Lines Matching refs:UnicodeText

100   UnicodeText text_unicode = UTF8ToUnicodeText(text, /*do_copy=*/false);  in Tokenize()
104 std::vector<Token> Tokenizer::Tokenize(const UnicodeText& text_unicode) const {  in Tokenize()
133 void AppendCodepointToToken(UnicodeText::const_iterator it, Token* token) {  in AppendCodepointToToken()
139     const UnicodeText& text_unicode) const {  in InternalTokenize()
179 void Tokenizer::TokenizeSubstring(const UnicodeText& unicode_text,  in TokenizeSubstring()
188   UnicodeText text = UnicodeText::Substring(unicode_text, span.first,  in TokenizeSubstring()
204 void Tokenizer::InternalRetokenize(const UnicodeText& unicode_text,  in InternalRetokenize()
209     const UnicodeText unicode_token_value =  in InternalRetokenize()
236 bool Tokenizer::ICUTokenize(const UnicodeText& context_unicode,  in ICUTokenize()
282 bool Tokenizer::NumberTokenize(const UnicodeText& text_unicode,  in NumberTokenize()
297           UnicodeText::const_iterator it, bool is_whitespace = false) {  in NumberTokenize()
310           int codepoint_index, UnicodeText::const_iterator it) {  in NumberTokenize()