Searched refs:TokenizationCodepointRangeT (Results 1 – 8 of 8) sorted by relevance
20 struct TokenizationCodepointRangeT;109 struct TokenizationCodepointRangeT : public flatbuffers::NativeTable { struct118 typedef TokenizationCodepointRangeT NativeTableType; argument146 …TokenizationCodepointRangeT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) c…147 …void UnPackTo(TokenizationCodepointRangeT *_o, const flatbuffers::resolver_function_t *_resolver =…148 …depointRange> Pack(flatbuffers::FlatBufferBuilder &_fbb, const TokenizationCodepointRangeT* _o, co…192 …tionCodepointRange(flatbuffers::FlatBufferBuilder &_fbb, const TokenizationCodepointRangeT *_o, co…194 inline TokenizationCodepointRangeT *TokenizationCodepointRange::UnPack(const flatbuffers::resolver_… in UnPack()195 auto _o = std::unique_ptr<TokenizationCodepointRangeT>(new TokenizationCodepointRangeT()); in UnPack()200 inline void TokenizationCodepointRange::UnPackTo(TokenizationCodepointRangeT *_o, const flatbuffers… in UnPackTo()[all …]
50 const std::vector<TokenizationCodepointRangeT>& codepoint_range_configs, in TestingTokenizerProxy()85 const TokenizationCodepointRangeT* range = in TestFindTokenizationRole()105 std::vector<TokenizationCodepointRangeT> configs; in TEST()106 TokenizationCodepointRangeT* config; in TEST()163 std::vector<TokenizationCodepointRangeT> configs; in TEST()164 TokenizationCodepointRangeT* config; in TEST()185 std::vector<TokenizationCodepointRangeT> configs; in TEST()186 TokenizationCodepointRangeT* config; in TEST()221 std::vector<TokenizationCodepointRangeT> configs; in TEST()222 TokenizationCodepointRangeT* config; in TEST()[all …]
48 [](const std::unique_ptr<const TokenizationCodepointRangeT>& a, in Tokenizer()49 const std::unique_ptr<const TokenizationCodepointRangeT>& b) { in Tokenizer()61 const TokenizationCodepointRangeT* Tokenizer::FindTokenizationRange( in FindTokenizationRange()65 [](const std::unique_ptr<const TokenizationCodepointRangeT>& range, in FindTokenizationRange()89 const TokenizationCodepointRangeT* range = FindTokenizationRange(codepoint); in GetScriptAndRole()
102 const TokenizationCodepointRangeT* FindTokenizationRange(int codepoint) const;138 std::vector<std::unique_ptr<const TokenizationCodepointRangeT>>
373 new TokenizationCodepointRangeT()); in TEST_F()447 new TokenizationCodepointRangeT()); in TEST_F()597 new TokenizationCodepointRangeT()); in TEST_F()
107 new TokenizationCodepointRangeT()); in BuildFeatureProcessor()
1925 …std::vector<std::unique_ptr<libtextclassifier3::TokenizationCodepointRangeT>> tokenization_codepoi…3887 …std::vector<std::unique_ptr<libtextclassifier3::TokenizationCodepointRangeT>> tokenization_codepoi…6252 …ack((tokenization_codepoint_config_) ? new libtextclassifier3::TokenizationCodepointRangeT(*tokeni… in GrammarTokenizerOptionsT()6277 …ion_codepoint_config[_i] = std::unique_ptr<libtextclassifier3::TokenizationCodepointRangeT>(_e->Ge… in UnPackTo()7094 …ack((tokenization_codepoint_config_) ? new libtextclassifier3::TokenizationCodepointRangeT(*tokeni… in FeatureProcessorOptionsT()7164 …ion_codepoint_config[_i] = std::unique_ptr<libtextclassifier3::TokenizationCodepointRangeT>(_e->Ge… in UnPackTo()
686 …std::vector<std::unique_ptr<libtextclassifier3::TokenizationCodepointRangeT>> tokenization_codepoi…3513 …ack((tokenization_codepoint_config_) ? new libtextclassifier3::TokenizationCodepointRangeT(*tokeni… in ActionsTokenizerOptionsT()3538 …ion_codepoint_config[_i] = std::unique_ptr<libtextclassifier3::TokenizationCodepointRangeT>(_e->Ge… in UnPackTo()