1 // Copyright 2016 The PDFium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "xfa/fxfa/formcalc/cxfa_fmlexer.h"
6
7 #include <vector>
8
9 #include "testing/gtest/include/gtest/gtest.h"
10
TEST(CXFA_FMLexerTest,NullString)11 TEST(CXFA_FMLexerTest, NullString) {
12 WideStringView null_string;
13 CXFA_FMLexer lexer(null_string);
14 CXFA_FMLexer::Token token = lexer.NextToken();
15 EXPECT_EQ(TOKeof, token.GetType());
16 EXPECT_TRUE(lexer.IsComplete());
17 }
18
TEST(CXFA_FMLexerTest,EmptyString)19 TEST(CXFA_FMLexerTest, EmptyString) {
20 CXFA_FMLexer lexer(L"");
21 CXFA_FMLexer::Token token = lexer.NextToken();
22 EXPECT_EQ(TOKeof, token.GetType());
23 EXPECT_TRUE(lexer.IsComplete());
24 }
25
TEST(CXFA_FMLexerTest,Numbers)26 TEST(CXFA_FMLexerTest, Numbers) {
27 {
28 CXFA_FMLexer lexer(L"-12");
29 CXFA_FMLexer::Token token = lexer.NextToken();
30 // TODO(dsinclair): Should this return -12 instead of two tokens?
31 EXPECT_EQ(TOKminus, token.GetType());
32 token = lexer.NextToken();
33 EXPECT_EQ(L"12", token.GetString());
34 token = lexer.NextToken();
35 EXPECT_EQ(TOKeof, token.GetType());
36 }
37 {
38 CXFA_FMLexer lexer(L"1.5362");
39 CXFA_FMLexer::Token token = lexer.NextToken();
40 EXPECT_EQ(TOKnumber, token.GetType());
41 EXPECT_EQ(L"1.5362", token.GetString());
42 }
43 {
44 CXFA_FMLexer lexer(L"0.875");
45 CXFA_FMLexer::Token token = lexer.NextToken();
46 EXPECT_EQ(TOKnumber, token.GetType());
47 EXPECT_EQ(L"0.875", token.GetString());
48 }
49 {
50 CXFA_FMLexer lexer(L"5.56e-2");
51 CXFA_FMLexer::Token token = lexer.NextToken();
52 EXPECT_EQ(TOKnumber, token.GetType());
53 EXPECT_EQ(L"5.56e-2", token.GetString());
54 }
55 {
56 CXFA_FMLexer lexer(L"1.234E10");
57 CXFA_FMLexer::Token token = lexer.NextToken();
58 EXPECT_EQ(TOKnumber, token.GetType());
59 EXPECT_EQ(L"1.234E10", token.GetString());
60 }
61 {
62 CXFA_FMLexer lexer(L"123456789.012345678");
63 CXFA_FMLexer::Token token = lexer.NextToken();
64 EXPECT_EQ(TOKnumber, token.GetType());
65 // TODO(dsinclair): This should round as per IEEE 64-bit values.
66 // EXPECT_EQ(L"123456789.01234567", token.GetString());
67 EXPECT_EQ(L"123456789.012345678", token.GetString());
68 }
69 {
70 CXFA_FMLexer lexer(L"99999999999999999");
71 CXFA_FMLexer::Token token = lexer.NextToken();
72 EXPECT_EQ(TOKnumber, token.GetType());
73 // TODO(dsinclair): This is spec'd as rounding when > 16 significant digits
74 // prior to the exponent.
75 // EXPECT_EQ(L"100000000000000000", token.GetString());
76 EXPECT_EQ(L"99999999999999999", token.GetString());
77 EXPECT_TRUE(lexer.IsComplete());
78 }
79 }
80
81 // The quotes are stripped in CXFA_FMStringExpression::ToJavaScript.
TEST(CXFA_FMLexerTest,Strings)82 TEST(CXFA_FMLexerTest, Strings) {
83 {
84 CXFA_FMLexer lexer(L"\"The cat jumped over the fence.\"");
85 CXFA_FMLexer::Token token = lexer.NextToken();
86 EXPECT_EQ(TOKstring, token.GetType());
87 EXPECT_EQ(L"\"The cat jumped over the fence.\"", token.GetString());
88
89 token = lexer.NextToken();
90 EXPECT_EQ(TOKeof, token.GetType());
91 }
92 {
93 CXFA_FMLexer lexer(L"\"\"");
94 CXFA_FMLexer::Token token = lexer.NextToken();
95 EXPECT_EQ(TOKstring, token.GetType());
96 EXPECT_EQ(L"\"\"", token.GetString());
97 }
98 {
99 CXFA_FMLexer lexer(
100 L"\"The message reads: \"\"Warning: Insufficient Memory\"\"\"");
101 CXFA_FMLexer::Token token = lexer.NextToken();
102 EXPECT_EQ(TOKstring, token.GetType());
103 EXPECT_EQ(L"\"The message reads: \"\"Warning: Insufficient Memory\"\"\"",
104 token.GetString());
105 }
106 {
107 CXFA_FMLexer lexer(
108 L"\"\\u0047\\u006f\\u0066\\u0069\\u0073\\u0068\\u0021\\u000d\\u000a\"");
109 CXFA_FMLexer::Token token = lexer.NextToken();
110 EXPECT_EQ(TOKstring, token.GetType());
111 EXPECT_EQ(
112 L"\"\\u0047\\u006f\\u0066\\u0069\\u0073\\u0068\\u0021\\u000d\\u000a\"",
113 token.GetString());
114 EXPECT_TRUE(lexer.IsComplete());
115 }
116 }
117
118 // Note, 'this' is a keyword but is not matched by the lexer.
TEST(CXFA_FMLexerTest,OperatorsAndKeywords)119 TEST(CXFA_FMLexerTest, OperatorsAndKeywords) {
120 struct {
121 const wchar_t* op;
122 XFA_FM_TOKEN token;
123 } op[] = {{L"+", TOKplus},
124 {L"/", TOKdiv},
125 {L"-", TOKminus},
126 {L"&", TOKand},
127 {L"|", TOKor},
128 {L"*", TOKmul},
129 {L"<", TOKlt},
130 {L">", TOKgt},
131 {L"==", TOKeq},
132 {L"<>", TOKne},
133 {L"<=", TOKle},
134 {L">=", TOKge},
135 {L"and", TOKksand},
136 {L"break", TOKbreak},
137 {L"continue", TOKcontinue},
138 {L"do", TOKdo},
139 {L"downto", TOKdownto},
140 {L"else", TOKelse},
141 {L"elseif", TOKelseif},
142 {L"end", TOKend},
143 {L"endfor", TOKendfor},
144 {L"endfunc", TOKendfunc},
145 {L"endif", TOKendif},
146 {L"endwhile", TOKendwhile},
147 {L"eq", TOKkseq},
148 {L"exit", TOKexit},
149 {L"for", TOKfor},
150 {L"foreach", TOKforeach},
151 {L"func", TOKfunc},
152 {L"ge", TOKksge},
153 {L"gt", TOKksgt},
154 {L"if", TOKif},
155 {L"in", TOKin},
156 {L"infinity", TOKinfinity},
157 {L"le", TOKksle},
158 {L"lt", TOKkslt},
159 {L"nan", TOKnan},
160 {L"ne", TOKksne},
161 {L"not", TOKksnot},
162 {L"null", TOKnull},
163 {L"or", TOKksor},
164 {L"return", TOKreturn},
165 {L"step", TOKstep},
166 {L"then", TOKthen},
167 {L"throw", TOKthrow},
168 {L"upto", TOKupto},
169 {L"var", TOKvar},
170 {L"while", TOKwhile},
171
172 // The following are defined but aren't in the spec.
173 {L"(", TOKlparen},
174 {L")", TOKrparen},
175 {L",", TOKcomma},
176 {L".", TOKdot},
177 {L"[", TOKlbracket},
178 {L"]", TOKrbracket},
179 {L"..", TOKdotdot},
180 {L".#", TOKdotscream},
181 {L".*", TOKdotstar}};
182
183 for (size_t i = 0; i < std::size(op); ++i) {
184 CXFA_FMLexer lexer(op[i].op);
185 CXFA_FMLexer::Token token = lexer.NextToken();
186 EXPECT_EQ(op[i].token, token.GetType());
187 EXPECT_TRUE(lexer.IsComplete());
188 }
189 }
190
TEST(CXFA_FMLexerTest,Comments)191 TEST(CXFA_FMLexerTest, Comments) {
192 {
193 CXFA_FMLexer lexer(L"// Empty.");
194 CXFA_FMLexer::Token token = lexer.NextToken();
195 EXPECT_EQ(TOKeof, token.GetType());
196 }
197 {
198 CXFA_FMLexer lexer(L"//");
199 CXFA_FMLexer::Token token = lexer.NextToken();
200 EXPECT_EQ(TOKeof, token.GetType());
201 }
202 {
203 CXFA_FMLexer lexer(L"123 // Empty.\n\"str\"");
204 CXFA_FMLexer::Token token = lexer.NextToken();
205 EXPECT_EQ(TOKnumber, token.GetType());
206 EXPECT_EQ(L"123", token.GetString());
207
208 token = lexer.NextToken();
209 EXPECT_EQ(TOKstring, token.GetType());
210 EXPECT_EQ(L"\"str\"", token.GetString());
211
212 token = lexer.NextToken();
213 EXPECT_EQ(TOKeof, token.GetType());
214 }
215 {
216 CXFA_FMLexer lexer(L";");
217 CXFA_FMLexer::Token token = lexer.NextToken();
218 EXPECT_EQ(TOKeof, token.GetType());
219 }
220 {
221 CXFA_FMLexer lexer(L"; Empty.");
222 CXFA_FMLexer::Token token = lexer.NextToken();
223 EXPECT_EQ(TOKeof, token.GetType());
224 }
225 {
226 CXFA_FMLexer lexer(L"123 ;Empty.\n\"str\"");
227 CXFA_FMLexer::Token token = lexer.NextToken();
228 EXPECT_EQ(TOKnumber, token.GetType());
229 EXPECT_EQ(L"123", token.GetString());
230
231 token = lexer.NextToken();
232 EXPECT_EQ(TOKstring, token.GetType());
233 EXPECT_EQ(L"\"str\"", token.GetString());
234
235 token = lexer.NextToken();
236 EXPECT_EQ(TOKeof, token.GetType());
237 EXPECT_TRUE(lexer.IsComplete());
238 }
239 }
240
TEST(CXFA_FMLexerTest,ValidIdentifiers)241 TEST(CXFA_FMLexerTest, ValidIdentifiers) {
242 std::vector<const wchar_t*> identifiers = {
243 L"a", L"an_identifier", L"_ident", L"$ident", L"!ident", L"GetAddr"};
244 for (const auto* ident : identifiers) {
245 CXFA_FMLexer lexer(ident);
246 CXFA_FMLexer::Token token = lexer.NextToken();
247 EXPECT_EQ(TOKidentifier, token.GetType());
248 EXPECT_EQ(ident, token.GetString());
249 EXPECT_TRUE(lexer.IsComplete());
250 }
251 }
252
TEST(CXFA_FMLexerTest,InvalidIdentifiers)253 TEST(CXFA_FMLexerTest, InvalidIdentifiers) {
254 {
255 CXFA_FMLexer lexer(L"#a");
256 CXFA_FMLexer::Token token = lexer.NextToken();
257 EXPECT_EQ(TOKreserver, token.GetType());
258 }
259 {
260 CXFA_FMLexer lexer(L"1a");
261 CXFA_FMLexer::Token token = lexer.NextToken();
262 EXPECT_EQ(TOKreserver, token.GetType());
263 }
264 {
265 CXFA_FMLexer lexer(L"an@identifier");
266 CXFA_FMLexer::Token token = lexer.NextToken();
267 EXPECT_NE(TOKreserver, token.GetType());
268 token = lexer.NextToken();
269 EXPECT_EQ(TOKreserver, token.GetType());
270 token = lexer.NextToken();
271 EXPECT_EQ(TOKreserver, token.GetType());
272 }
273 {
274 CXFA_FMLexer lexer(L"_ident@");
275 CXFA_FMLexer::Token token = lexer.NextToken();
276 EXPECT_NE(TOKreserver, token.GetType());
277 token = lexer.NextToken();
278 EXPECT_EQ(TOKreserver, token.GetType());
279 EXPECT_FALSE(lexer.IsComplete());
280 }
281 }
282
TEST(CXFA_FMLexerTest,Whitespace)283 TEST(CXFA_FMLexerTest, Whitespace) {
284 {
285 CXFA_FMLexer lexer(L" \t\xc\x9\xb");
286 CXFA_FMLexer::Token token = lexer.NextToken();
287 EXPECT_EQ(TOKeof, token.GetType());
288 }
289 {
290 CXFA_FMLexer lexer(L"123 \t\xc\x9\xb 456");
291 CXFA_FMLexer::Token token = lexer.NextToken();
292 EXPECT_EQ(TOKnumber, token.GetType());
293 EXPECT_EQ(L"123", token.GetString());
294
295 token = lexer.NextToken();
296 EXPECT_EQ(TOKnumber, token.GetType());
297 EXPECT_EQ(L"456", token.GetString());
298
299 token = lexer.NextToken();
300 EXPECT_EQ(TOKeof, token.GetType());
301 EXPECT_TRUE(lexer.IsComplete());
302 }
303 }
304
TEST(CXFA_FMLexerTest,NullData)305 TEST(CXFA_FMLexerTest, NullData) {
306 CXFA_FMLexer lexer(WideStringView(L"\x2d\x32\x00\x2d\x32", 5));
307 CXFA_FMLexer::Token token = lexer.NextToken();
308 EXPECT_EQ(TOKminus, token.GetType());
309
310 token = lexer.NextToken();
311 EXPECT_EQ(TOKnumber, token.GetType());
312 EXPECT_EQ(L"2", token.GetString());
313
314 token = lexer.NextToken();
315 EXPECT_EQ(TOKeof, token.GetType());
316 EXPECT_FALSE(lexer.IsComplete());
317 }
318