1 // Copyright 2018 The Amber Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //     http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "src/tokenizer.h"
16 
17 #include <cmath>
18 #include <limits>
19 
20 #include "gtest/gtest.h"
21 
22 namespace amber {
23 
24 using TokenizerTest = testing::Test;
25 
TEST_F(TokenizerTest,ProcessEmpty)26 TEST_F(TokenizerTest, ProcessEmpty) {
27   Tokenizer t("");
28   auto next = t.NextToken();
29   ASSERT_TRUE(next != nullptr);
30   EXPECT_TRUE(next->IsEOS());
31 }
32 
TEST_F(TokenizerTest,ProcessIdentifier)33 TEST_F(TokenizerTest, ProcessIdentifier) {
34   Tokenizer t("TestIdentifier");
35   auto next = t.NextToken();
36   ASSERT_TRUE(next != nullptr);
37   EXPECT_TRUE(next->IsIdentifier());
38   EXPECT_EQ("TestIdentifier", next->AsString());
39 
40   next = t.NextToken();
41   ASSERT_TRUE(next != nullptr);
42   EXPECT_TRUE(next->IsEOS());
43 }
44 
TEST_F(TokenizerTest,ProcessInt)45 TEST_F(TokenizerTest, ProcessInt) {
46   Tokenizer t("123");
47   auto next = t.NextToken();
48   ASSERT_TRUE(next != nullptr);
49   EXPECT_TRUE(next->IsInteger());
50   EXPECT_EQ(123U, next->AsUint32());
51 
52   next = t.NextToken();
53   ASSERT_TRUE(next != nullptr);
54   EXPECT_TRUE(next->IsEOS());
55 }
56 
TEST_F(TokenizerTest,ProcessNegative)57 TEST_F(TokenizerTest, ProcessNegative) {
58   Tokenizer t("-123");
59   auto next = t.NextToken();
60   ASSERT_TRUE(next != nullptr);
61   EXPECT_TRUE(next->IsInteger());
62   EXPECT_EQ(-123, next->AsInt32());
63 
64   next = t.NextToken();
65   ASSERT_TRUE(next != nullptr);
66   EXPECT_TRUE(next->IsEOS());
67 }
68 
TEST_F(TokenizerTest,ProcessDouble)69 TEST_F(TokenizerTest, ProcessDouble) {
70   Tokenizer t("123.456");
71   auto next = t.NextToken();
72   ASSERT_TRUE(next != nullptr);
73   EXPECT_TRUE(next->IsDouble());
74   EXPECT_EQ(123.456f, next->AsFloat());
75 
76   next = t.NextToken();
77   ASSERT_TRUE(next != nullptr);
78   EXPECT_TRUE(next->IsEOS());
79 }
80 
81 namespace {
82 
TestNaN(const std::string & nan_str)83 void TestNaN(const std::string& nan_str) {
84   Tokenizer t(nan_str);
85   auto next = t.NextToken();
86   ASSERT_TRUE(next != nullptr);
87   EXPECT_TRUE(next->IsDouble());
88   EXPECT_TRUE(std::isnan(next->AsDouble()));
89 
90   next = t.NextToken();
91   ASSERT_TRUE(next != nullptr);
92   EXPECT_TRUE(next->IsEOS());
93 }
94 
95 }  // namespace
96 
TEST_F(TokenizerTest,ProcessNaN)97 TEST_F(TokenizerTest, ProcessNaN) {
98   TestNaN("nan");
99   TestNaN("naN");
100   TestNaN("nAn");
101   TestNaN("nAN");
102   TestNaN("Nan");
103   TestNaN("NaN");
104   TestNaN("NAn");
105   TestNaN("NAN");
106 }
107 
TEST_F(TokenizerTest,ProcessNegativeDouble)108 TEST_F(TokenizerTest, ProcessNegativeDouble) {
109   Tokenizer t("-123.456");
110   auto next = t.NextToken();
111   ASSERT_TRUE(next != nullptr);
112   EXPECT_TRUE(next->IsDouble());
113   EXPECT_EQ(-123.456f, next->AsFloat());
114 
115   next = t.NextToken();
116   ASSERT_TRUE(next != nullptr);
117   EXPECT_TRUE(next->IsEOS());
118 }
119 
TEST_F(TokenizerTest,ProcessDoubleStartWithDot)120 TEST_F(TokenizerTest, ProcessDoubleStartWithDot) {
121   Tokenizer t(".123456");
122   auto next = t.NextToken();
123   ASSERT_TRUE(next != nullptr);
124   EXPECT_TRUE(next->IsDouble());
125   EXPECT_EQ(.123456f, next->AsFloat());
126 
127   next = t.NextToken();
128   ASSERT_TRUE(next != nullptr);
129   EXPECT_TRUE(next->IsEOS());
130 }
131 
TEST_F(TokenizerTest,ProcessStringWithNumberInName)132 TEST_F(TokenizerTest, ProcessStringWithNumberInName) {
133   Tokenizer t("BufferAccess32");
134   auto next = t.NextToken();
135   ASSERT_TRUE(next != nullptr);
136   EXPECT_TRUE(next->IsIdentifier());
137   EXPECT_EQ("BufferAccess32", next->AsString());
138 
139   next = t.NextToken();
140   ASSERT_TRUE(next != nullptr);
141   EXPECT_TRUE(next->IsEOS());
142 }
143 
TEST_F(TokenizerTest,ProcessMultiStatement)144 TEST_F(TokenizerTest, ProcessMultiStatement) {
145   Tokenizer t("TestValue 123.456");
146   auto next = t.NextToken();
147   ASSERT_TRUE(next != nullptr);
148   EXPECT_TRUE(next->IsIdentifier());
149   EXPECT_EQ("TestValue", next->AsString());
150 
151   next = t.NextToken();
152   ASSERT_TRUE(next != nullptr);
153   EXPECT_TRUE(next->IsDouble());
154   EXPECT_EQ(123.456f, next->AsFloat());
155 
156   next = t.NextToken();
157   ASSERT_TRUE(next != nullptr);
158   EXPECT_TRUE(next->IsEOS());
159 }
160 
TEST_F(TokenizerTest,ProcessMultiLineStatement)161 TEST_F(TokenizerTest, ProcessMultiLineStatement) {
162   Tokenizer t("TestValue 123.456\nAnotherValue\n\nThirdValue 456");
163   auto next = t.NextToken();
164   ASSERT_TRUE(next != nullptr);
165   EXPECT_TRUE(next->IsIdentifier());
166   EXPECT_EQ("TestValue", next->AsString());
167   EXPECT_EQ(1U, t.GetCurrentLine());
168 
169   next = t.NextToken();
170   ASSERT_TRUE(next != nullptr);
171   EXPECT_TRUE(next->IsDouble());
172   EXPECT_EQ(123.456f, next->AsFloat());
173   EXPECT_EQ(1U, t.GetCurrentLine());
174 
175   next = t.NextToken();
176   ASSERT_TRUE(next != nullptr);
177   EXPECT_TRUE(next->IsEOL());
178 
179   next = t.NextToken();
180   ASSERT_TRUE(next != nullptr);
181   EXPECT_TRUE(next->IsIdentifier());
182   EXPECT_EQ("AnotherValue", next->AsString());
183   EXPECT_EQ(2U, t.GetCurrentLine());
184 
185   next = t.NextToken();
186   ASSERT_TRUE(next != nullptr);
187   EXPECT_TRUE(next->IsEOL());
188 
189   next = t.NextToken();
190   ASSERT_TRUE(next != nullptr);
191   EXPECT_TRUE(next->IsEOL());
192 
193   next = t.NextToken();
194   ASSERT_TRUE(next != nullptr);
195   EXPECT_TRUE(next->IsIdentifier());
196   EXPECT_EQ("ThirdValue", next->AsString());
197   EXPECT_EQ(4U, t.GetCurrentLine());
198 
199   next = t.NextToken();
200   ASSERT_TRUE(next != nullptr);
201   EXPECT_TRUE(next->IsInteger());
202   EXPECT_EQ(456U, next->AsUint16());
203   EXPECT_EQ(4U, t.GetCurrentLine());
204 
205   next = t.NextToken();
206   ASSERT_TRUE(next != nullptr);
207   EXPECT_TRUE(next->IsEOS());
208 }
209 
TEST_F(TokenizerTest,ProcessComments)210 TEST_F(TokenizerTest, ProcessComments) {
211   Tokenizer t(R"(# Initial comment string
212 TestValue 123.456
213     AnotherValue   # Space before, comment after
214 
215 ThirdValue 456)");
216   auto next = t.NextToken();
217   // The comment injects a blank line into the output
218   // so we can handle full line comment and end of line comment the same.
219   ASSERT_TRUE(next != nullptr);
220   EXPECT_TRUE(next->IsEOL());
221 
222   next = t.NextToken();
223   ASSERT_TRUE(next != nullptr);
224   EXPECT_TRUE(next->IsIdentifier());
225   EXPECT_EQ("TestValue", next->AsString());
226 
227   next = t.NextToken();
228   ASSERT_TRUE(next != nullptr);
229   EXPECT_TRUE(next->IsDouble());
230   EXPECT_EQ(123.456f, next->AsFloat());
231 
232   next = t.NextToken();
233   ASSERT_TRUE(next != nullptr);
234   EXPECT_TRUE(next->IsEOL());
235 
236   next = t.NextToken();
237   ASSERT_TRUE(next != nullptr);
238   EXPECT_TRUE(next->IsIdentifier());
239   EXPECT_EQ("AnotherValue", next->AsString());
240 
241   next = t.NextToken();
242   ASSERT_TRUE(next != nullptr);
243   EXPECT_TRUE(next->IsEOL());
244 
245   next = t.NextToken();
246   ASSERT_TRUE(next != nullptr);
247   EXPECT_TRUE(next->IsEOL());
248 
249   next = t.NextToken();
250   ASSERT_TRUE(next != nullptr);
251   EXPECT_TRUE(next->IsIdentifier());
252   EXPECT_EQ("ThirdValue", next->AsString());
253 
254   next = t.NextToken();
255   ASSERT_TRUE(next != nullptr);
256   EXPECT_TRUE(next->IsInteger());
257   EXPECT_EQ(456U, next->AsUint16());
258 
259   next = t.NextToken();
260   ASSERT_TRUE(next != nullptr);
261   EXPECT_TRUE(next->IsEOS());
262 }
263 
TEST_F(TokenizerTest,HexValue)264 TEST_F(TokenizerTest, HexValue) {
265   Tokenizer t("0xff00f0ff");
266   auto next = t.NextToken();
267   ASSERT_TRUE(next != nullptr);
268   EXPECT_TRUE(next->IsHex());
269   EXPECT_EQ(0xff00f0ff, next->AsHex());
270 
271   next = t.NextToken();
272   ASSERT_TRUE(next != nullptr);
273   EXPECT_TRUE(next->IsEOS());
274 }
275 
TEST_F(TokenizerTest,HexValueAfterWhiteSpace)276 TEST_F(TokenizerTest, HexValueAfterWhiteSpace) {
277   Tokenizer t("     \t  \t   0xff00f0ff");
278   auto next = t.NextToken();
279   ASSERT_TRUE(next != nullptr);
280   EXPECT_TRUE(next->IsHex());
281   EXPECT_EQ(0xff00f0ff, next->AsHex());
282 
283   next = t.NextToken();
284   ASSERT_TRUE(next != nullptr);
285   EXPECT_TRUE(next->IsEOS());
286 }
287 
TEST_F(TokenizerTest,StringStartingWithNum)288 TEST_F(TokenizerTest, StringStartingWithNum) {
289   Tokenizer t("1/ABC");
290   auto next = t.NextToken();
291   ASSERT_TRUE(next != nullptr);
292   EXPECT_TRUE(next->IsInteger());
293   EXPECT_EQ(1U, next->AsUint32());
294 
295   next = t.NextToken();
296   ASSERT_TRUE(next != nullptr);
297   EXPECT_TRUE(next->IsIdentifier());
298   EXPECT_EQ("/ABC", next->AsString());
299 }
300 
TEST_F(TokenizerTest,StringQuotedSingleLine)301 TEST_F(TokenizerTest, StringQuotedSingleLine) {
302   Tokenizer t("\"Hello world\"");
303   auto next = t.NextToken();
304   ASSERT_TRUE(next != nullptr);
305   EXPECT_TRUE(next->IsString());
306   EXPECT_EQ("Hello world", next->AsString());
307 
308   next = t.NextToken();
309   ASSERT_TRUE(next != nullptr);
310   EXPECT_TRUE(next->IsEOS());
311 }
312 
TEST_F(TokenizerTest,StringQuotedMultiLine)313 TEST_F(TokenizerTest, StringQuotedMultiLine) {
314   Tokenizer t("\"Hello\n\nworld\"");
315   auto next = t.NextToken();
316   ASSERT_TRUE(next != nullptr);
317   EXPECT_TRUE(next->IsString());
318   EXPECT_EQ("Hello\n\nworld", next->AsString());
319 
320   next = t.NextToken();
321   ASSERT_TRUE(next != nullptr);
322   EXPECT_TRUE(next->IsEOS());
323 }
324 
TEST_F(TokenizerTest,StringQuotedUnterminated)325 TEST_F(TokenizerTest, StringQuotedUnterminated) {
326   Tokenizer t("\"Hello\n\nworld");
327   auto next = t.NextToken();
328   ASSERT_TRUE(next != nullptr);
329   EXPECT_TRUE(next->IsString());
330   EXPECT_EQ("Hello\n\nworld", next->AsString());
331 
332   next = t.NextToken();
333   ASSERT_TRUE(next != nullptr);
334   EXPECT_TRUE(next->IsEOS());
335 }
336 
TEST_F(TokenizerTest,StringQuotedEscapeSequences)337 TEST_F(TokenizerTest, StringQuotedEscapeSequences) {
338   Tokenizer t(R"("_\"\\_a\aa_b\bb_t\tt_n\nn_v\vv_f\ff_r\rr_")");
339   auto next = t.NextToken();
340   ASSERT_TRUE(next != nullptr);
341   EXPECT_TRUE(next->IsString());
342 
343   std::string expect = "_\"\\_a\aa_b\bb_t\tt_n\nn_v\vv_f\ff_r\rr_";
344   EXPECT_EQ(expect, next->AsString());
345 
346   next = t.NextToken();
347   ASSERT_TRUE(next != nullptr);
348   EXPECT_TRUE(next->IsEOS());
349 }
350 
TEST_F(TokenizerTest,BracketsAndCommas)351 TEST_F(TokenizerTest, BracketsAndCommas) {
352   Tokenizer t("(1.0, 2, abc)");
353   auto next = t.NextToken();
354   ASSERT_TRUE(next != nullptr);
355   EXPECT_TRUE(next->IsOpenBracket());
356 
357   next = t.NextToken();
358   ASSERT_TRUE(next != nullptr);
359   EXPECT_TRUE(next->IsDouble());
360   EXPECT_FLOAT_EQ(1.0, next->AsFloat());
361 
362   next = t.NextToken();
363   ASSERT_TRUE(next != nullptr);
364   EXPECT_TRUE(next->IsComma());
365 
366   next = t.NextToken();
367   ASSERT_TRUE(next != nullptr);
368   EXPECT_TRUE(next->IsInteger());
369   EXPECT_EQ(2U, next->AsUint32());
370 
371   next = t.NextToken();
372   ASSERT_TRUE(next != nullptr);
373   EXPECT_TRUE(next->IsComma());
374 
375   next = t.NextToken();
376   ASSERT_TRUE(next != nullptr);
377   EXPECT_TRUE(next->IsIdentifier());
378   EXPECT_EQ("abc", next->AsString());
379 
380   next = t.NextToken();
381   ASSERT_TRUE(next != nullptr);
382   EXPECT_TRUE(next->IsCloseBracket());
383 
384   next = t.NextToken();
385   ASSERT_TRUE(next != nullptr);
386   EXPECT_TRUE(next->IsEOS());
387 }
388 
TEST_F(TokenizerTest,TokenToDoubleFromDouble)389 TEST_F(TokenizerTest, TokenToDoubleFromDouble) {
390   Tokenizer t("-1.234");
391   auto next = t.NextToken();
392   ASSERT_TRUE(next != nullptr);
393   ASSERT_TRUE(next->IsDouble());
394 
395   Result r = next->ConvertToDouble();
396   ASSERT_TRUE(r.IsSuccess());
397   EXPECT_FLOAT_EQ(-1.234f, next->AsFloat());
398 }
399 
TEST_F(TokenizerTest,TokenToDoubleFromInt)400 TEST_F(TokenizerTest, TokenToDoubleFromInt) {
401   Tokenizer t("-1");
402   auto next = t.NextToken();
403   ASSERT_TRUE(next != nullptr);
404   ASSERT_TRUE(next->IsInteger());
405 
406   Result r = next->ConvertToDouble();
407   ASSERT_TRUE(r.IsSuccess());
408   EXPECT_FLOAT_EQ(-1.0f, next->AsFloat());
409 }
410 
TEST_F(TokenizerTest,DashToken)411 TEST_F(TokenizerTest, DashToken) {
412   Tokenizer t("-");
413   auto next = t.NextToken();
414   ASSERT_TRUE(next != nullptr);
415   ASSERT_TRUE(next->IsIdentifier());
416   EXPECT_EQ("-", next->AsString());
417 }
418 
TEST_F(TokenizerTest,ParseUint64Max)419 TEST_F(TokenizerTest, ParseUint64Max) {
420   Tokenizer t(std::to_string(std::numeric_limits<uint64_t>::max()));
421   auto next = t.NextToken();
422   ASSERT_TRUE(next != nullptr);
423   ASSERT_TRUE(next->IsInteger());
424   EXPECT_EQ(std::numeric_limits<uint64_t>::max(), next->AsUint64());
425 }
426 
TEST_F(TokenizerTest,ParseInt64Min)427 TEST_F(TokenizerTest, ParseInt64Min) {
428   Tokenizer t(std::to_string(std::numeric_limits<int64_t>::min()));
429   auto next = t.NextToken();
430   ASSERT_TRUE(next != nullptr);
431   ASSERT_TRUE(next->IsInteger());
432   EXPECT_EQ(std::numeric_limits<int64_t>::min(), next->AsInt64());
433 }
434 
TEST_F(TokenizerTest,TokenToDoubleFromUint64Max)435 TEST_F(TokenizerTest, TokenToDoubleFromUint64Max) {
436   Tokenizer t(std::to_string(std::numeric_limits<uint64_t>::max()));
437   auto next = t.NextToken();
438   ASSERT_TRUE(next != nullptr);
439   ASSERT_TRUE(next->IsInteger());
440 
441   Result r = next->ConvertToDouble();
442   ASSERT_FALSE(r.IsSuccess());
443   EXPECT_EQ("uint64_t value too big to fit in double", r.Error());
444 }
445 
TEST_F(TokenizerTest,TokenToDoubleFromInt64Min)446 TEST_F(TokenizerTest, TokenToDoubleFromInt64Min) {
447   Tokenizer t(std::to_string(std::numeric_limits<int64_t>::min()));
448   auto next = t.NextToken();
449   ASSERT_TRUE(next != nullptr);
450   ASSERT_TRUE(next->IsInteger());
451 
452   Result r = next->ConvertToDouble();
453   ASSERT_TRUE(r.IsSuccess());
454   EXPECT_DOUBLE_EQ(static_cast<double>(std::numeric_limits<int64_t>::min()),
455                    next->AsDouble());
456 }
457 
TEST_F(TokenizerTest,TokenToDoubleFromInt64Max)458 TEST_F(TokenizerTest, TokenToDoubleFromInt64Max) {
459   Tokenizer t(std::to_string(std::numeric_limits<int64_t>::max()));
460   auto next = t.NextToken();
461   ASSERT_TRUE(next != nullptr);
462   ASSERT_TRUE(next->IsInteger());
463 
464   Result r = next->ConvertToDouble();
465   ASSERT_TRUE(r.IsSuccess());
466   EXPECT_DOUBLE_EQ(static_cast<double>(std::numeric_limits<int64_t>::max()),
467                    next->AsDouble());
468 }
469 
TEST_F(TokenizerTest,TokenToDoubleFromString)470 TEST_F(TokenizerTest, TokenToDoubleFromString) {
471   Tokenizer t("INVALID");
472   auto next = t.NextToken();
473   ASSERT_TRUE(next != nullptr);
474   ASSERT_TRUE(next->IsIdentifier());
475 
476   Result r = next->ConvertToDouble();
477   ASSERT_FALSE(r.IsSuccess());
478   EXPECT_EQ("Invalid conversion to double", r.Error());
479 }
480 
TEST_F(TokenizerTest,TokenToDoubleFromHex)481 TEST_F(TokenizerTest, TokenToDoubleFromHex) {
482   Tokenizer t("0xff00f0ff");
483   auto next = t.NextToken();
484   ASSERT_TRUE(next != nullptr);
485   ASSERT_TRUE(next->IsHex());
486 
487   Result r = next->ConvertToDouble();
488   ASSERT_TRUE(r.IsSuccess());
489   EXPECT_FLOAT_EQ(static_cast<float>(0xff00f0ff), next->AsFloat());
490 }
491 
TEST_F(TokenizerTest,TokenToDoubleFromEOS)492 TEST_F(TokenizerTest, TokenToDoubleFromEOS) {
493   Tokenizer t("");
494   auto next = t.NextToken();
495   ASSERT_TRUE(next != nullptr);
496   ASSERT_TRUE(next->IsEOS());
497 
498   Result r = next->ConvertToDouble();
499   ASSERT_FALSE(r.IsSuccess());
500   EXPECT_EQ("Invalid conversion to double", r.Error());
501 }
502 
TEST_F(TokenizerTest,TokenToDoubleFromEOL)503 TEST_F(TokenizerTest, TokenToDoubleFromEOL) {
504   Tokenizer t("-1\n-2");
505   auto next = t.NextToken();
506   next = t.NextToken();
507   ASSERT_TRUE(next != nullptr);
508   ASSERT_TRUE(next->IsEOL());
509 
510   Result r = next->ConvertToDouble();
511   ASSERT_FALSE(r.IsSuccess());
512   EXPECT_EQ("Invalid conversion to double", r.Error());
513 }
514 
TEST_F(TokenizerTest,Continuations)515 TEST_F(TokenizerTest, Continuations) {
516   Tokenizer t("1 \\\n2");
517   auto next = t.NextToken();
518   ASSERT_TRUE(next != nullptr);
519   ASSERT_TRUE(next->IsInteger());
520   EXPECT_EQ(1, next->AsInt32());
521   EXPECT_EQ(1u, t.GetCurrentLine());
522 
523   next = t.NextToken();
524   ASSERT_TRUE(next != nullptr);
525   ASSERT_TRUE(next->IsInteger());
526   EXPECT_EQ(2, next->AsInt32());
527   EXPECT_EQ(2u, t.GetCurrentLine());
528 
529   next = t.NextToken();
530   ASSERT_TRUE(next != nullptr);
531   EXPECT_TRUE(next->IsEOS());
532 }
533 
TEST_F(TokenizerTest,ContinuationAtEndOfString)534 TEST_F(TokenizerTest, ContinuationAtEndOfString) {
535   Tokenizer t("1 \\");
536   auto next = t.NextToken();
537   ASSERT_TRUE(next != nullptr);
538   ASSERT_TRUE(next->IsInteger());
539   EXPECT_EQ(1, next->AsInt32());
540 
541   next = t.NextToken();
542   ASSERT_TRUE(next != nullptr);
543   ASSERT_TRUE(next->IsIdentifier());
544   EXPECT_EQ("\\", next->AsString());
545 
546   next = t.NextToken();
547   ASSERT_TRUE(next != nullptr);
548   EXPECT_TRUE(next->IsEOS());
549 }
550 
TEST_F(TokenizerTest,ContinuationTokenAtOfLine)551 TEST_F(TokenizerTest, ContinuationTokenAtOfLine) {
552   Tokenizer t("1 \\2");
553   auto next = t.NextToken();
554   ASSERT_TRUE(next != nullptr);
555   ASSERT_TRUE(next->IsInteger());
556   EXPECT_EQ(1, next->AsInt32());
557 
558   next = t.NextToken();
559   ASSERT_TRUE(next != nullptr);
560   ASSERT_TRUE(next->IsIdentifier());
561   EXPECT_EQ("\\2", next->AsString());
562 
563   next = t.NextToken();
564   ASSERT_TRUE(next != nullptr);
565   EXPECT_TRUE(next->IsEOS());
566 }
567 
TEST_F(TokenizerTest,ContinuationTokenInMiddleOfLine)568 TEST_F(TokenizerTest, ContinuationTokenInMiddleOfLine) {
569   Tokenizer t("1 \\ 2");
570   auto next = t.NextToken();
571   ASSERT_TRUE(next != nullptr);
572   ASSERT_TRUE(next->IsInteger());
573   EXPECT_EQ(1, next->AsInt32());
574 
575   next = t.NextToken();
576   ASSERT_TRUE(next != nullptr);
577   ASSERT_TRUE(next->IsIdentifier());
578   EXPECT_EQ("\\", next->AsString());
579 
580   next = t.NextToken();
581   ASSERT_TRUE(next != nullptr);
582   ASSERT_TRUE(next->IsInteger());
583   EXPECT_EQ(2u, next->AsInt32());
584 
585   next = t.NextToken();
586   ASSERT_TRUE(next != nullptr);
587   EXPECT_TRUE(next->IsEOS());
588 }
589 
TEST_F(TokenizerTest,ExtractToNext)590 TEST_F(TokenizerTest, ExtractToNext) {
591   Tokenizer t("this\nis\na\ntest\nEND");
592 
593   auto next = t.NextToken();
594   EXPECT_TRUE(next->IsIdentifier());
595   EXPECT_EQ("this", next->AsString());
596 
597   std::string s = t.ExtractToNext("END");
598   ASSERT_EQ("\nis\na\ntest\n", s);
599 
600   next = t.NextToken();
601   EXPECT_TRUE(next->IsIdentifier());
602   EXPECT_EQ("END", next->AsString());
603   EXPECT_EQ(5U, t.GetCurrentLine());
604 
605   next = t.NextToken();
606   EXPECT_TRUE(next->IsEOS());
607 }
608 
TEST_F(TokenizerTest,ExtractToNextMissingNext)609 TEST_F(TokenizerTest, ExtractToNextMissingNext) {
610   Tokenizer t("this\nis\na\ntest\n");
611 
612   auto next = t.NextToken();
613   EXPECT_TRUE(next->IsIdentifier());
614   EXPECT_EQ("this", next->AsString());
615 
616   std::string s = t.ExtractToNext("END");
617   ASSERT_EQ("\nis\na\ntest\n", s);
618 
619   next = t.NextToken();
620   EXPECT_TRUE(next->IsEOS());
621   EXPECT_EQ(5U, t.GetCurrentLine());
622 }
623 
TEST_F(TokenizerTest,ExtractToNextCurrentIsNext)624 TEST_F(TokenizerTest, ExtractToNextCurrentIsNext) {
625   Tokenizer t("END");
626   std::string s = t.ExtractToNext("END");
627   ASSERT_EQ("", s);
628 
629   auto next = t.NextToken();
630   EXPECT_TRUE(next->IsIdentifier());
631   EXPECT_EQ("END", next->AsString());
632 
633   next = t.NextToken();
634   EXPECT_TRUE(next->IsEOS());
635 }
636 
637 }  // namespace amber
638