Home
last modified time | relevance | path

Searched refs:internal_tokenizer_codepoint_ranges (Results 1 – 11 of 11) sorted by relevance

/aosp_15_r20/external/libtextclassifier/native/annotator/grammar/
H A Dutils.cc39 if (options->internal_tokenizer_codepoint_ranges() != nullptr) { in BuildTokenizer()
42 options->internal_tokenizer_codepoint_ranges()->begin(), in BuildTokenizer()
43 options->internal_tokenizer_codepoint_ranges()->end()); in BuildTokenizer()
/aosp_15_r20/external/libtextclassifier/native/utils/
H A Dtokenizer.h57 internal_tokenizer_codepoint_ranges,
66 internal_tokenizer_codepoint_ranges, in Tokenizer() argument
70 internal_tokenizer_codepoint_ranges, split_on_script_change, in Tokenizer()
H A Dtokenizer.cc33 internal_tokenizer_codepoint_ranges, in Tokenizer() argument
53 SortCodepointRanges(internal_tokenizer_codepoint_ranges, in Tokenizer()
H A Dtokenizer_test.cc35 internal_tokenizer_codepoint_ranges, in TestingTokenizer() argument
40 internal_tokenizer_codepoint_ranges, split_on_script_change, in TestingTokenizer()
H A Dtokenizer.fbs29 // consisting only of codepoints in internal_tokenizer_codepoint_ranges
/aosp_15_r20/external/libtextclassifier/native/actions/
H A Dfeature-processor.cc54 if (options->internal_tokenizer_codepoint_ranges() != nullptr) { in CreateTokenizer()
57 options->internal_tokenizer_codepoint_ranges()->begin(), in CreateTokenizer()
58 options->internal_tokenizer_codepoint_ranges()->end()); in CreateTokenizer()
H A Dactions_model.fbs150 internal_tokenizer_codepoint_ranges:[CodepointRange];
/aosp_15_r20/out/soong/.intermediates/external/libtextclassifier/native/libtextclassifier_fbgen_annotator_model/gen/annotator/
Dmodel_generated.h1926 …vector<std::unique_ptr<libtextclassifier3::CodepointRangeT>> internal_tokenizer_codepoint_ranges{};
1953 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>> *internal_tokenizer_codepoint_ranges() c… in internal_tokenizer_codepoint_ranges() function
1967 verifier.VerifyVector(internal_tokenizer_codepoint_ranges()) && in Verify()
1968 verifier.VerifyVectorOfTables(internal_tokenizer_codepoint_ranges()) && in Verify()
1990 …or<flatbuffers::Offset<libtextclassifier3::CodepointRange>>> internal_tokenizer_codepoint_ranges) { in add_internal_tokenizer_codepoint_ranges()
1991 …mmarTokenizerOptions::VT_INTERNAL_TOKENIZER_CODEPOINT_RANGES, internal_tokenizer_codepoint_ranges); in add_internal_tokenizer_codepoint_ranges()
2012 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>>> internal_tokenizer_codepoint_ranges = 0,
2015 builder_.add_internal_tokenizer_codepoint_ranges(internal_tokenizer_codepoint_ranges);
2028 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>> *internal_tokenizer_codepoint_ranges = n…
2031 …= internal_tokenizer_codepoint_ranges ? _fbb.CreateVector<flatbuffers::Offset<libtextclassifier3::…
[all …]
/aosp_15_r20/out/soong/.intermediates/external/libtextclassifier/native/libtextclassifier_fbgen_actions_actions_model/gen/actions/
Dactions_model_generated.h687 …vector<std::unique_ptr<libtextclassifier3::CodepointRangeT>> internal_tokenizer_codepoint_ranges{};
714 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>> *internal_tokenizer_codepoint_ranges() c… in internal_tokenizer_codepoint_ranges() function
728 verifier.VerifyVector(internal_tokenizer_codepoint_ranges()) && in Verify()
729 verifier.VerifyVectorOfTables(internal_tokenizer_codepoint_ranges()) && in Verify()
751 …or<flatbuffers::Offset<libtextclassifier3::CodepointRange>>> internal_tokenizer_codepoint_ranges) { in add_internal_tokenizer_codepoint_ranges()
752 …ionsTokenizerOptions::VT_INTERNAL_TOKENIZER_CODEPOINT_RANGES, internal_tokenizer_codepoint_ranges); in add_internal_tokenizer_codepoint_ranges()
773 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>>> internal_tokenizer_codepoint_ranges = 0,
776 builder_.add_internal_tokenizer_codepoint_ranges(internal_tokenizer_codepoint_ranges);
789 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>> *internal_tokenizer_codepoint_ranges = n…
792 …= internal_tokenizer_codepoint_ranges ? _fbb.CreateVector<flatbuffers::Offset<libtextclassifier3::…
[all …]
/aosp_15_r20/external/libtextclassifier/native/annotator/
H A Dfeature-processor.cc40 if (options->internal_tokenizer_codepoint_ranges() != nullptr) { in BuildTokenizer()
43 options->internal_tokenizer_codepoint_ranges()->begin(), in BuildTokenizer()
44 options->internal_tokenizer_codepoint_ranges()->end()); in BuildTokenizer()
H A Dmodel.fbs369 internal_tokenizer_codepoint_ranges:[CodepointRange];
756 internal_tokenizer_codepoint_ranges:[CodepointRange];