Home
last modified time | relevance | path

Searched full:tokenizer (Results 1 – 25 of 2751) sorted by relevance

12345678910>>...111

/aosp_15_r20/external/cronet/third_party/protobuf/src/google/protobuf/io/
H A Dtokenizer_unittest.cc35 #include <google/protobuf/io/tokenizer.h>
183 EXPECT_TRUE(Tokenizer::ParseInteger(text, kuint64max, &result)) in ParseInteger()
199 Tokenizer::TokenType type;
209 {"hello", Tokenizer::TYPE_IDENTIFIER},
212 {"123", Tokenizer::TYPE_INTEGER},
213 {"0xab6", Tokenizer::TYPE_INTEGER},
214 {"0XAB6", Tokenizer::TYPE_INTEGER},
215 {"0X1234567", Tokenizer::TYPE_INTEGER},
216 {"0x89abcdef", Tokenizer::TYPE_INTEGER},
217 {"0x89ABCDEF", Tokenizer::TYPE_INTEGER},
[all …]
/aosp_15_r20/external/protobuf/src/google/protobuf/io/
H A Dtokenizer_unittest.cc35 #include <google/protobuf/io/tokenizer.h>
183 EXPECT_TRUE(Tokenizer::ParseInteger(text, kuint64max, &result)) in ParseInteger()
199 Tokenizer::TokenType type;
209 {"hello", Tokenizer::TYPE_IDENTIFIER},
212 {"123", Tokenizer::TYPE_INTEGER},
213 {"0xab6", Tokenizer::TYPE_INTEGER},
214 {"0XAB6", Tokenizer::TYPE_INTEGER},
215 {"0X1234567", Tokenizer::TYPE_INTEGER},
216 {"0x89abcdef", Tokenizer::TYPE_INTEGER},
217 {"0x89ABCDEF", Tokenizer::TYPE_INTEGER},
[all …]
/aosp_15_r20/tools/metalava/metalava-model-text/src/main/java/com/android/tools/metalava/model/text/
H A DApiFile.kt451 val tokenizer = Tokenizer(path, apiText.toCharArray()) in parseApiSingleFile() constant
453 val token = tokenizer.getToken() ?: break in parseApiSingleFile()
456 parsePackage(tokenizer) in parseApiSingleFile()
458 throw ApiParseException("expected package got $token", tokenizer) in parseApiSingleFile()
463 private fun parsePackage(tokenizer: Tokenizer) { in parsePackage()
464 var token: String = tokenizer.requireToken() in parsePackage()
467 val annotations = getAnnotations(tokenizer, token) in parsePackage()
469 token = tokenizer.current in parsePackage()
470 tokenizer.assertIdent(token) in parsePackage()
479 fileLocation = tokenizer.fileLocation(), in parsePackage()
[all …]
/aosp_15_r20/external/doclava/src/com/google/doclava/apicheck/
H A DApiFile.java70 final Tokenizer tokenizer = new Tokenizer(filename, (new String(buf, 0, size)).toCharArray()); in parseApi() local
74 String token = tokenizer.getToken(); in parseApi()
79 parsePackage(api, tokenizer); in parseApi()
81 throw new ApiParseException("expected package got " + token, tokenizer.getLine()); in parseApi()
91 private static void parsePackage(ApiInfo api, Tokenizer tokenizer) in parsePackage() argument
97 token = tokenizer.requireToken(); in parsePackage()
98 assertIdent(tokenizer, token); in parsePackage()
100 pkg = new PackageInfo(name, tokenizer.pos()); in parsePackage()
101 token = tokenizer.requireToken(); in parsePackage()
103 throw new ApiParseException("expected '{' got " + token, tokenizer.getLine()); in parsePackage()
[all …]
/aosp_15_r20/system/core/init/parser/
H A Dtokenizer_test.cpp15 #include "tokenizer.h"
26 Tokenizer tokenizer(data); \
27 ASSERT_EQ(Tokenizer::TOK_START, tokenizer.current().type)
30 ASSERT_TRUE(tokenizer.Next()); \
31 ASSERT_EQ(test_text, tokenizer.current().text); \
32 ASSERT_EQ(Tokenizer::TOK_TEXT, tokenizer.current().type)
35 ASSERT_TRUE(tokenizer.Next()); \
36 ASSERT_EQ(Tokenizer::TOK_NEWLINE, tokenizer.current().type)
38 TEST(Tokenizer, Empty) { in TEST() argument
40 ASSERT_FALSE(tokenizer.Next()); in TEST()
[all …]
/aosp_15_r20/external/protobuf/python/google/protobuf/
H A Dtext_format.py862 tokenizer = Tokenizer(str_lines)
863 while not tokenizer.AtEnd():
864 self._MergeField(tokenizer, message)
866 def _MergeField(self, tokenizer, message): argument
870 tokenizer: A tokenizer to parse the field name and values.
878 tokenizer.TryConsume('[')):
879 type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
880 tokenizer.Consume(']')
881 tokenizer.TryConsume(':')
882 if tokenizer.TryConsume('<'):
[all …]
/aosp_15_r20/external/cronet/third_party/protobuf/python/google/protobuf/
H A Dtext_format.py862 tokenizer = Tokenizer(str_lines)
863 while not tokenizer.AtEnd():
864 self._MergeField(tokenizer, message)
866 def _MergeField(self, tokenizer, message): argument
870 tokenizer: A tokenizer to parse the field name and values.
878 tokenizer.TryConsume('[')):
879 type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
880 tokenizer.Consume(']')
881 tokenizer.TryConsume(':')
882 if tokenizer.TryConsume('<'):
[all …]
/aosp_15_r20/external/rust/android-crates-io/crates/protobuf-parse/src/pure/
Dparser.rs9 use protobuf_support::lexer::tokenizer::Tokenizer;
10 use protobuf_support::lexer::tokenizer::TokenizerError;
183 pub tokenizer: Tokenizer<'a>, field
296 tokenizer: Tokenizer::new(input, ParserLanguage::Proto), in new()
307 if self.tokenizer.next_symbol_if_eq('.')? { in next_full_ident()
310 full_ident.push_str(&self.tokenizer.next_ident()?); in next_full_ident()
311 while self.tokenizer.next_symbol_if_eq('.')? { in next_full_ident()
313 full_ident.push_str(&self.tokenizer.next_ident()?); in next_full_ident()
321 full_ident.push_str(&self.tokenizer.next_ident()?); in next_full_ident_rel()
322 while self.tokenizer.next_symbol_if_eq('.')? { in next_full_ident_rel()
[all …]
/aosp_15_r20/external/icing/icing/tokenization/
H A Dtrigram-tokenizer_test.cc15 #include "icing/tokenization/trigram-tokenizer.h"
24 #include "icing/tokenization/tokenizer.h"
39 TrigramTokenizer tokenizer; in TEST() local
43 tokenizer.TokenizeAll(s), in TEST()
52 TrigramTokenizer tokenizer; in TEST() local
56 tokenizer.TokenizeAll(s), in TEST()
66 TrigramTokenizer tokenizer; in TEST() local
70 tokenizer.TokenizeAll(s), in TEST()
81 TrigramTokenizer tokenizer; in TEST() local
85 tokenizer.TokenizeAll(s), in TEST()
[all …]
/aosp_15_r20/external/cronet/third_party/protobuf/python/google/protobuf/internal/
H A Dtext_format_test.py1983 tokenizer = text_format.Tokenizer(text.splitlines())
1984 methods = [(tokenizer.ConsumeIdentifier, 'identifier1'), ':',
1985 (tokenizer.ConsumeString, 'string1'),
1986 (tokenizer.ConsumeIdentifier, 'identifier2'), ':',
1987 (tokenizer.ConsumeInteger, 123),
1988 (tokenizer.ConsumeIdentifier, 'identifier3'), ':',
1989 (tokenizer.ConsumeString, 'string'),
1990 (tokenizer.ConsumeIdentifier, 'identifiER_4'), ':',
1991 (tokenizer.ConsumeFloat, 1.1e+2),
1992 (tokenizer.ConsumeIdentifier, 'ID5'), ':',
[all …]
/aosp_15_r20/external/protobuf/python/google/protobuf/internal/
H A Dtext_format_test.py1983 tokenizer = text_format.Tokenizer(text.splitlines())
1984 methods = [(tokenizer.ConsumeIdentifier, 'identifier1'), ':',
1985 (tokenizer.ConsumeString, 'string1'),
1986 (tokenizer.ConsumeIdentifier, 'identifier2'), ':',
1987 (tokenizer.ConsumeInteger, 123),
1988 (tokenizer.ConsumeIdentifier, 'identifier3'), ':',
1989 (tokenizer.ConsumeString, 'string'),
1990 (tokenizer.ConsumeIdentifier, 'identifiER_4'), ':',
1991 (tokenizer.ConsumeFloat, 1.1e+2),
1992 (tokenizer.ConsumeIdentifier, 'ID5'), ':',
[all …]
/aosp_15_r20/external/protobuf/csharp/src/Google.Protobuf.Test/
H A DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/aosp_15_r20/external/cronet/third_party/protobuf/csharp/src/Google.Protobuf.Test/
H A DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/aosp_15_r20/external/protobuf/csharp/compatibility_tests/v3.0.0/src/Google.Protobuf.Test/
H A DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/aosp_15_r20/external/cronet/third_party/protobuf/csharp/compatibility_tests/v3.0.0/src/Google.Protobuf.Test/
H A DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/aosp_15_r20/external/deqp-deps/amber/src/vkscript/
Dparser.cc43 std::string Parser::make_error(const Tokenizer& tokenizer, in make_error() argument
45 return std::to_string(tokenizer.GetCurrentLine()) + ": " + err; in make_error()
152 Tokenizer tokenizer(section.contents); in ProcessRequireBlock() local
153 tokenizer.SetCurrentLine(section.starting_line_number + 1); in ProcessRequireBlock()
155 for (auto token = tokenizer.NextToken(); !token->IsEOS(); in ProcessRequireBlock()
156 token = tokenizer.NextToken()) { in ProcessRequireBlock()
161 tokenizer, in ProcessRequireBlock()
169 token = tokenizer.NextToken(); in ProcessRequireBlock()
171 return Result(make_error(tokenizer, "Missing framebuffer format")); in ProcessRequireBlock()
177 make_error(tokenizer, "Failed to parse framebuffer format: " + in ProcessRequireBlock()
[all …]
/aosp_15_r20/external/cronet/third_party/boringssl/src/pki/
H A Dpem_unittest.cc20 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
21 EXPECT_TRUE(tokenizer.GetNext()); in TEST()
23 EXPECT_EQ("EXPECTED-BLOCK", tokenizer.block_type()); in TEST()
24 EXPECT_EQ("MatchesAcceptedBlockType", tokenizer.data()); in TEST()
26 EXPECT_FALSE(tokenizer.GetNext()); in TEST()
38 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
39 EXPECT_TRUE(tokenizer.GetNext()); in TEST()
41 EXPECT_EQ("EXPECTED-BLOCK", tokenizer.block_type()); in TEST()
42 EXPECT_EQ("MatchesAcceptedBlockType", tokenizer.data()); in TEST()
44 EXPECT_FALSE(tokenizer.GetNext()); in TEST()
[all …]
/aosp_15_r20/external/protobuf/java/core/src/main/java/com/google/protobuf/
H A DTextFormat.java916 private static final class Tokenizer { class in TextFormat
958 /** Construct a tokenizer that parses tokens from the given text. */
959 private Tokenizer(final CharSequence text) { in Tokenizer() method in TextFormat.Tokenizer
1554 private void detectSilentMarker(Tokenizer tokenizer) { in detectSilentMarker() argument
1787 final Tokenizer tokenizer = new Tokenizer(input); in merge() local
1792 while (!tokenizer.atEnd()) { in merge()
1793 mergeField(tokenizer, extensionRegistry, target, unknownFields); in merge()
1799 /** Parse a single field from {@code tokenizer} and merge it into {@code builder}. */
1801 final Tokenizer tokenizer, in mergeField() argument
1807 tokenizer, in mergeField()
[all …]
/aosp_15_r20/external/cronet/third_party/protobuf/java/core/src/main/java/com/google/protobuf/
H A DTextFormat.java916 private static final class Tokenizer { class in TextFormat
958 /** Construct a tokenizer that parses tokens from the given text. */
959 private Tokenizer(final CharSequence text) { in Tokenizer() method in TextFormat.Tokenizer
1554 private void detectSilentMarker(Tokenizer tokenizer) { in detectSilentMarker() argument
1787 final Tokenizer tokenizer = new Tokenizer(input); in merge() local
1792 while (!tokenizer.atEnd()) { in merge()
1793 mergeField(tokenizer, extensionRegistry, target, unknownFields); in merge()
1799 /** Parse a single field from {@code tokenizer} and merge it into {@code builder}. */
1801 final Tokenizer tokenizer, in mergeField() argument
1807 tokenizer, in mergeField()
[all …]
/aosp_15_r20/external/libtextclassifier/native/utils/
H A Dbert_tokenizer_test.cc32 void AssertTokenizerResults(std::unique_ptr<BertTokenizer> tokenizer) { in AssertTokenizerResults() argument
33 auto results = tokenizer->Tokenize("i'm question"); in AssertTokenizerResults()
41 auto tokenizer = in TEST() local
44 AssertTokenizerResults(std::move(tokenizer)); in TEST()
48 auto tokenizer = in TEST() local
51 AssertTokenizerResults(std::move(tokenizer)); in TEST()
60 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
62 AssertTokenizerResults(std::move(tokenizer)); in TEST()
66 auto tokenizer = in TEST() local
69 auto results = tokenizer->Tokenize("i'm questionansweraskask"); in TEST()
[all …]
H A Dtokenizer_test.cc17 #include "utils/tokenizer.h"
29 class TestingTokenizer : public Tokenizer {
39 : Tokenizer(type, unilib, codepoint_ranges, in TestingTokenizer()
43 using Tokenizer::FindTokenizationRange;
126 TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER, configs, in TEST() local
132 EXPECT_EQ(tokenizer.TestFindTokenizationRole(0), in TEST()
134 EXPECT_EQ(tokenizer.TestFindTokenizationRole(5), in TEST()
136 EXPECT_EQ(tokenizer.TestFindTokenizationRole(10), in TEST()
140 EXPECT_EQ(tokenizer.TestFindTokenizationRole(31), in TEST()
142 EXPECT_EQ(tokenizer.TestFindTokenizationRole(32), in TEST()
[all …]
/aosp_15_r20/external/sdv/vsomeip/third_party/boost/tokenizer/
DREADME.md4 # [Boost.Tokenizer](http://boost.org/libs/tokenizer)
8 Boost.Tokenizer is a part of [Boost C++ Libraries](http://github.com/boostorg). The Boost.Tokenize…
23 …tokenizer/tree/master) | [![Build Status](https://travis-ci.org/boostorg/tokenizer.svg?branch=mast…
24 …tokenizer/tree/develop) | [![Build Status](https://travis-ci.org/boostorg/tokenizer.svg?branch=dev…
36 #include <boost/tokenizer.hpp>
41 typedef boost::tokenizer<> Tok;
55 #include <boost/tokenizer.hpp>
60 boost::tokenizer<> tok(s);
69 Documentation can be found at [Boost.Tokenizer](http://boost.org/libs/tokenizer)
72 [Boost.Tokenizer](http://theboostcpplibraries.com/boost.tokenizer) Chapter 10 at theboostcpplibrari…
[all …]
/aosp_15_r20/external/protobuf/csharp/src/Google.Protobuf/
H A DJsonParser.cs71 …// TODO: Consider introducing a class containing parse state of the parser, tokenizer and depth. T…
76 …{ Timestamp.Descriptor.FullName, (parser, message, tokenizer) => MergeTimestamp(message, tokenizer…
77 …{ Duration.Descriptor.FullName, (parser, message, tokenizer) => MergeDuration(message, tokenizer.N…
78 …{ Value.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStructValue(message, toke…
79 { ListValue.Descriptor.FullName, (parser, message, tokenizer) =>
80 …r.MergeRepeatedField(message, message.Descriptor.Fields[ListValue.ValuesFieldNumber], tokenizer) },
81 …{ Struct.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStruct(message, tokenize…
82 … { Any.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeAny(message, tokenizer) },
83 …{ FieldMask.Descriptor.FullName, (parser, message, tokenizer) => MergeFieldMask(message, tokenizer…
97 …private static void MergeWrapperField(JsonParser parser, IMessage message, JsonTokenizer tokenizer) in MergeWrapperField() argument
[all …]
/aosp_15_r20/external/cronet/third_party/protobuf/csharp/src/Google.Protobuf/
H A DJsonParser.cs71 …// TODO: Consider introducing a class containing parse state of the parser, tokenizer and depth. T…
76 …{ Timestamp.Descriptor.FullName, (parser, message, tokenizer) => MergeTimestamp(message, tokenizer…
77 …{ Duration.Descriptor.FullName, (parser, message, tokenizer) => MergeDuration(message, tokenizer.N…
78 …{ Value.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStructValue(message, toke…
79 { ListValue.Descriptor.FullName, (parser, message, tokenizer) =>
80 …r.MergeRepeatedField(message, message.Descriptor.Fields[ListValue.ValuesFieldNumber], tokenizer) },
81 …{ Struct.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStruct(message, tokenize…
82 … { Any.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeAny(message, tokenizer) },
83 …{ FieldMask.Descriptor.FullName, (parser, message, tokenizer) => MergeFieldMask(message, tokenizer…
97 …private static void MergeWrapperField(JsonParser parser, IMessage message, JsonTokenizer tokenizer) in MergeWrapperField() argument
[all …]
/aosp_15_r20/external/parameter-framework/upstream/test/tokenizer/
H A DTest.cpp31 #include "Tokenizer.h"
44 SCENARIO("Tokenizer tests")
46 GIVEN ("A default tokenizer") {
49 Tokenizer tokenizer("a bcd ef"); variable
53 CHECK(tokenizer.split() == expected);
58 Tokenizer tokenizer(""); variable
62 CHECK(tokenizer.split() == expected);
67 Tokenizer tokenizer(" a \n\t bc "); variable
71 CHECK(tokenizer.split() == expected);
76 GIVEN ("A slash-separated string and tokenizer") {
[all …]

12345678910>>...111