Lines Matching refs:Tokenizer
183 EXPECT_TRUE(Tokenizer::ParseInteger(text, kuint64max, &result));
198 Tokenizer::TokenType type;
208 {"hello", Tokenizer::TYPE_IDENTIFIER},
211 {"123", Tokenizer::TYPE_INTEGER},
212 {"0xab6", Tokenizer::TYPE_INTEGER},
213 {"0XAB6", Tokenizer::TYPE_INTEGER},
214 {"0X1234567", Tokenizer::TYPE_INTEGER},
215 {"0x89abcdef", Tokenizer::TYPE_INTEGER},
216 {"0x89ABCDEF", Tokenizer::TYPE_INTEGER},
217 {"01234567", Tokenizer::TYPE_INTEGER},
220 {"123.45", Tokenizer::TYPE_FLOAT},
221 {"1.", Tokenizer::TYPE_FLOAT},
222 {"1e3", Tokenizer::TYPE_FLOAT},
223 {"1E3", Tokenizer::TYPE_FLOAT},
224 {"1e-3", Tokenizer::TYPE_FLOAT},
225 {"1e+3", Tokenizer::TYPE_FLOAT},
226 {"1.e3", Tokenizer::TYPE_FLOAT},
227 {"1.2e3", Tokenizer::TYPE_FLOAT},
228 {".1", Tokenizer::TYPE_FLOAT},
229 {".1e3", Tokenizer::TYPE_FLOAT},
230 {".1e-3", Tokenizer::TYPE_FLOAT},
231 {".1e+3", Tokenizer::TYPE_FLOAT},
234 {"'hello'", Tokenizer::TYPE_STRING},
235 {"\"foo\"", Tokenizer::TYPE_STRING},
236 {"'a\"b'", Tokenizer::TYPE_STRING},
237 {"\"a'b\"", Tokenizer::TYPE_STRING},
238 {"'a\\'b'", Tokenizer::TYPE_STRING},
239 {"\"a\\\"b\"", Tokenizer::TYPE_STRING},
240 {"'\\xf'", Tokenizer::TYPE_STRING},
241 {"'\\0'", Tokenizer::TYPE_STRING},
244 {"+", Tokenizer::TYPE_SYMBOL},
245 {".", Tokenizer::TYPE_SYMBOL},
253 Tokenizer tokenizer(&input, &error_collector);
256 EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type);
279 EXPECT_EQ(Tokenizer::TYPE_END, tokenizer.current().type);
297 Tokenizer tokenizer(&input, &error_collector);
303 EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
306 EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
309 EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
312 EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
328 Tokenizer::Token output[10]; // The compiler wants a constant array
343 {Tokenizer::TYPE_END, "", 0, 0, 0},
349 {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
350 {Tokenizer::TYPE_INTEGER, "1", 0, 4, 5},
351 {Tokenizer::TYPE_FLOAT, "1.2", 0, 6, 9},
352 {Tokenizer::TYPE_SYMBOL, "+", 0, 10, 11},
353 {Tokenizer::TYPE_STRING, "'bar'", 0, 12, 17},
354 {Tokenizer::TYPE_END, "", 0, 17, 17},
360 {Tokenizer::TYPE_SYMBOL, "!", 0, 0, 1},
361 {Tokenizer::TYPE_SYMBOL, "@", 0, 1, 2},
362 {Tokenizer::TYPE_SYMBOL, "+", 0, 2, 3},
363 {Tokenizer::TYPE_SYMBOL, "%", 0, 3, 4},
364 {Tokenizer::TYPE_END, "", 0, 4, 4},
370 {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
371 {Tokenizer::TYPE_IDENTIFIER, "bar", 0, 4, 7},
372 {Tokenizer::TYPE_IDENTIFIER, "rab", 1, 0, 3},
373 {Tokenizer::TYPE_IDENTIFIER, "oof", 1, 4, 7},
374 {Tokenizer::TYPE_END, "", 1, 7, 7},
380 {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
381 {Tokenizer::TYPE_IDENTIFIER, "bar", 0, 8, 11},
382 {Tokenizer::TYPE_IDENTIFIER, "baz", 0, 16, 19},
383 {Tokenizer::TYPE_END, "", 0, 19, 19},
389 {Tokenizer::TYPE_STRING, "\"foo\tbar\"", 0, 0, 12},
390 {Tokenizer::TYPE_IDENTIFIER, "baz", 0, 13, 16},
391 {Tokenizer::TYPE_END, "", 0, 16, 16},
398 {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
399 {Tokenizer::TYPE_IDENTIFIER, "bar", 1, 0, 3},
400 {Tokenizer::TYPE_END, "", 1, 30, 30},
406 {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
407 {Tokenizer::TYPE_IDENTIFIER, "bar", 0, 34, 37},
408 {Tokenizer::TYPE_END, "", 0, 37, 37},
415 {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
416 {Tokenizer::TYPE_SYMBOL, "#", 0, 4, 5},
417 {Tokenizer::TYPE_IDENTIFIER, "bar", 0, 6, 9},
418 {Tokenizer::TYPE_IDENTIFIER, "baz", 1, 0, 3},
419 {Tokenizer::TYPE_END, "", 1, 3, 3},
425 {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
426 {Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11, 14},
427 {Tokenizer::TYPE_END, "", 1, 14, 14},
436 Tokenizer tokenizer(&input, &error_collector);
439 EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type);
447 Tokenizer::Token token;
453 Tokenizer::Token previous = tokenizer.current();
456 if (token.type != Tokenizer::TYPE_END) {
476 } while (token.type != Tokenizer::TYPE_END);
501 Tokenizer tokenizer(&input, &error_collector);
502 tokenizer.set_comment_style(Tokenizer::SH_COMMENT_STYLE);
661 Tokenizer tokenizer(&input, &error_collector);
666 Tokenizer tokenizer2(&input2, &error_collector);
718 EXPECT_FALSE(Tokenizer::ParseInteger("zxy", kuint64max, &i));
719 EXPECT_FALSE(Tokenizer::ParseInteger("1.2", kuint64max, &i));
720 EXPECT_FALSE(Tokenizer::ParseInteger("08", kuint64max, &i));
721 EXPECT_FALSE(Tokenizer::ParseInteger("0xg", kuint64max, &i));
722 EXPECT_FALSE(Tokenizer::ParseInteger("-1", kuint64max, &i));
725 EXPECT_TRUE(Tokenizer::ParseInteger("0", 0, &i));
726 EXPECT_FALSE(Tokenizer::ParseInteger("1", 0, &i));
727 EXPECT_TRUE(Tokenizer::ParseInteger("1", 1, &i));
728 EXPECT_TRUE(Tokenizer::ParseInteger("12345", 12345, &i));
729 EXPECT_FALSE(Tokenizer::ParseInteger("12346", 12345, &i));
730 EXPECT_TRUE(Tokenizer::ParseInteger("0xFFFFFFFFFFFFFFFF", kuint64max, &i));
731 EXPECT_FALSE(Tokenizer::ParseInteger("0x10000000000000000", kuint64max, &i));
735 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1."));
736 EXPECT_DOUBLE_EQ(1e3, Tokenizer::ParseFloat("1e3"));
737 EXPECT_DOUBLE_EQ(1e3, Tokenizer::ParseFloat("1E3"));
738 EXPECT_DOUBLE_EQ(1.5e3, Tokenizer::ParseFloat("1.5e3"));
739 EXPECT_DOUBLE_EQ(.1, Tokenizer::ParseFloat(".1"));
740 EXPECT_DOUBLE_EQ(.25, Tokenizer::ParseFloat(".25"));
741 EXPECT_DOUBLE_EQ(.1e3, Tokenizer::ParseFloat(".1e3"));
742 EXPECT_DOUBLE_EQ(.25e3, Tokenizer::ParseFloat(".25e3"));
743 EXPECT_DOUBLE_EQ(.1e+3, Tokenizer::ParseFloat(".1e+3"));
744 EXPECT_DOUBLE_EQ(.1e-3, Tokenizer::ParseFloat(".1e-3"));
745 EXPECT_DOUBLE_EQ(5, Tokenizer::ParseFloat("5"));
746 EXPECT_DOUBLE_EQ(6e-12, Tokenizer::ParseFloat("6e-12"));
747 EXPECT_DOUBLE_EQ(1.2, Tokenizer::ParseFloat("1.2"));
748 EXPECT_DOUBLE_EQ(1.e2, Tokenizer::ParseFloat("1.e2"));
751 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e"));
752 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e-"));
753 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.e"));
756 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1f"));
757 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.0f"));
758 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1F"));
762 EXPECT_EQ(0.0, Tokenizer::ParseFloat("1e-9999999999999999999999999999"));
763 EXPECT_EQ(HUGE_VAL, Tokenizer::ParseFloat("1e+9999999999999999999999999999"));
768 Tokenizer::ParseFloat("zxy"),
771 Tokenizer::ParseFloat("1-e0"),
774 Tokenizer::ParseFloat("-1.0"),
781 Tokenizer::ParseString("'hello'", &output);
783 Tokenizer::ParseString("\"blah\\nblah2\"", &output);
785 Tokenizer::ParseString("'\\1x\\1\\123\\739\\52\\334n\\3'", &output);
787 Tokenizer::ParseString("'\\x20\\x4'", &output);
791 Tokenizer::ParseString("\"\\a\\l\\v\\t", &output); // \l is invalid
793 Tokenizer::ParseString("'", &output);
795 Tokenizer::ParseString("'\\", &output);
800 Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\U00024b62XX'", &output);
803 Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\ud852\\udf62XX'", &output);
808 Tokenizer::ParseString("'\\ud852XX'", &output);
811 Tokenizer::ParseString("\\u0", &output);
817 Tokenizer::ParseString("", &output),
825 Tokenizer::ParseStringAppend("'hello'", &output);
827 Tokenizer::ParseString("'hello'", &output);
920 Tokenizer tokenizer(&input, &error_collector);
946 Tokenizer tokenizer(&input, &error_collector);