Searched refs:internal_tokenizer_codepoint_ranges_ (Results 1 – 4 of 4) sorted by relevance
54 &internal_tokenizer_codepoint_ranges_); in Tokenizer()214 internal_tokenizer_codepoint_ranges_)) { in InternalRetokenize()
145 std::vector<CodepointRangeStruct> internal_tokenizer_codepoint_ranges_; variable
6254 …internal_tokenizer_codepoint_ranges_ : o.internal_tokenizer_codepoint_ranges) { internal_tokenizer… in GrammarTokenizerOptionsT() local7098 …internal_tokenizer_codepoint_ranges_ : o.internal_tokenizer_codepoint_ranges) { internal_tokenizer… in FeatureProcessorOptionsT() local
3515 …internal_tokenizer_codepoint_ranges_ : o.internal_tokenizer_codepoint_ranges) { internal_tokenizer… in ActionsTokenizerOptionsT() local