Searched refs:internal_tokenizer_codepoint_ranges (Results 1 – 11 of 11) sorted by relevance
/aosp_15_r20/external/libtextclassifier/native/annotator/grammar/ |
H A D | utils.cc | 39 if (options->internal_tokenizer_codepoint_ranges() != nullptr) { in BuildTokenizer() 42 options->internal_tokenizer_codepoint_ranges()->begin(), in BuildTokenizer() 43 options->internal_tokenizer_codepoint_ranges()->end()); in BuildTokenizer()
|
/aosp_15_r20/external/libtextclassifier/native/utils/ |
H A D | tokenizer.h | 57 internal_tokenizer_codepoint_ranges, 66 internal_tokenizer_codepoint_ranges, in Tokenizer() argument 70 internal_tokenizer_codepoint_ranges, split_on_script_change, in Tokenizer()
|
H A D | tokenizer.cc | 33 internal_tokenizer_codepoint_ranges, in Tokenizer() argument 53 SortCodepointRanges(internal_tokenizer_codepoint_ranges, in Tokenizer()
|
H A D | tokenizer_test.cc | 35 internal_tokenizer_codepoint_ranges, in TestingTokenizer() argument 40 internal_tokenizer_codepoint_ranges, split_on_script_change, in TestingTokenizer()
|
H A D | tokenizer.fbs | 29 // consisting only of codepoints in internal_tokenizer_codepoint_ranges
|
/aosp_15_r20/external/libtextclassifier/native/actions/ |
H A D | feature-processor.cc | 54 if (options->internal_tokenizer_codepoint_ranges() != nullptr) { in CreateTokenizer() 57 options->internal_tokenizer_codepoint_ranges()->begin(), in CreateTokenizer() 58 options->internal_tokenizer_codepoint_ranges()->end()); in CreateTokenizer()
|
H A D | actions_model.fbs | 150 internal_tokenizer_codepoint_ranges:[CodepointRange];
|
/aosp_15_r20/out/soong/.intermediates/external/libtextclassifier/native/libtextclassifier_fbgen_annotator_model/gen/annotator/ |
D | model_generated.h | 1926 …vector<std::unique_ptr<libtextclassifier3::CodepointRangeT>> internal_tokenizer_codepoint_ranges{}; 1953 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>> *internal_tokenizer_codepoint_ranges() c… in internal_tokenizer_codepoint_ranges() function 1967 verifier.VerifyVector(internal_tokenizer_codepoint_ranges()) && in Verify() 1968 verifier.VerifyVectorOfTables(internal_tokenizer_codepoint_ranges()) && in Verify() 1990 …or<flatbuffers::Offset<libtextclassifier3::CodepointRange>>> internal_tokenizer_codepoint_ranges) { in add_internal_tokenizer_codepoint_ranges() 1991 …mmarTokenizerOptions::VT_INTERNAL_TOKENIZER_CODEPOINT_RANGES, internal_tokenizer_codepoint_ranges); in add_internal_tokenizer_codepoint_ranges() 2012 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>>> internal_tokenizer_codepoint_ranges = 0, 2015 builder_.add_internal_tokenizer_codepoint_ranges(internal_tokenizer_codepoint_ranges); 2028 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>> *internal_tokenizer_codepoint_ranges = n… 2031 …= internal_tokenizer_codepoint_ranges ? _fbb.CreateVector<flatbuffers::Offset<libtextclassifier3::… [all …]
|
/aosp_15_r20/out/soong/.intermediates/external/libtextclassifier/native/libtextclassifier_fbgen_actions_actions_model/gen/actions/ |
D | actions_model_generated.h | 687 …vector<std::unique_ptr<libtextclassifier3::CodepointRangeT>> internal_tokenizer_codepoint_ranges{}; 714 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>> *internal_tokenizer_codepoint_ranges() c… in internal_tokenizer_codepoint_ranges() function 728 verifier.VerifyVector(internal_tokenizer_codepoint_ranges()) && in Verify() 729 verifier.VerifyVectorOfTables(internal_tokenizer_codepoint_ranges()) && in Verify() 751 …or<flatbuffers::Offset<libtextclassifier3::CodepointRange>>> internal_tokenizer_codepoint_ranges) { in add_internal_tokenizer_codepoint_ranges() 752 …ionsTokenizerOptions::VT_INTERNAL_TOKENIZER_CODEPOINT_RANGES, internal_tokenizer_codepoint_ranges); in add_internal_tokenizer_codepoint_ranges() 773 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>>> internal_tokenizer_codepoint_ranges = 0, 776 builder_.add_internal_tokenizer_codepoint_ranges(internal_tokenizer_codepoint_ranges); 789 …<flatbuffers::Offset<libtextclassifier3::CodepointRange>> *internal_tokenizer_codepoint_ranges = n… 792 …= internal_tokenizer_codepoint_ranges ? _fbb.CreateVector<flatbuffers::Offset<libtextclassifier3::… [all …]
|
/aosp_15_r20/external/libtextclassifier/native/annotator/ |
H A D | feature-processor.cc | 40 if (options->internal_tokenizer_codepoint_ranges() != nullptr) { in BuildTokenizer() 43 options->internal_tokenizer_codepoint_ranges()->begin(), in BuildTokenizer() 44 options->internal_tokenizer_codepoint_ranges()->end()); in BuildTokenizer()
|
H A D | model.fbs | 369 internal_tokenizer_codepoint_ranges:[CodepointRange]; 756 internal_tokenizer_codepoint_ranges:[CodepointRange];
|