File tree Expand file tree Collapse file tree 3 files changed +1
-10
lines changed Expand file tree Collapse file tree 3 files changed +1
-10
lines changed Original file line number Diff line number Diff line change 1
- /*
2
- * Copyright (c) Meta Platforms, Inc. and affiliates.
3
- * All rights reserved.
4
- *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree.
7
- */
1
+ // (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
8
2
9
3
// This file contains the implementation of create_regex with lookahead support
10
4
11
5
#include < pytorch/tokenizers/pcre2_regex.h>
12
6
#include < pytorch/tokenizers/regex.h>
13
7
#include < pytorch/tokenizers/std_regex.h>
14
8
15
- #include < iostream>
16
9
#include < memory>
17
10
18
11
namespace tokenizers {
Original file line number Diff line number Diff line change 28
28
#include < pytorch/tokenizers/base64.h>
29
29
#include < pytorch/tokenizers/tiktoken.h>
30
30
#include < algorithm>
31
- #include < cinttypes>
32
31
#include < fstream>
33
32
#include < limits>
34
33
#include < unordered_set>
Original file line number Diff line number Diff line change 9
9
#include < gtest/gtest.h>
10
10
11
11
#include " pytorch/tokenizers/pcre2_regex.h"
12
- #include " pytorch/tokenizers/re2_regex.h"
13
12
#include " pytorch/tokenizers/regex.h"
14
13
15
14
using namespace tokenizers ;
You can’t perform that action at this time.
0 commit comments