Lines Matching defs:token
194 // In each test case, the entire input text should parse as a single token
255 // Before Next() is called, the initial token should always be TYPE_START.
262 // Parse the token.
278 // After Next() returns false, the token should have type TYPE_END.
325 // last token in "output" must have type TYPE_END.
346 // Test all token types at the same time.
438 // Before Next() is called, the initial token should always be TYPE_START.
447 Tokenizer::Token token;
449 token = kMultiTokenCases_case.output[i++];
451 SCOPED_TRACE(testing::Message() << "Token #" << i << ": " << token.text);
455 // Next() should only return false when it hits the end token.
456 if (token.type != Tokenizer::TYPE_END) {
462 // Check that the previous token is set correctly.
469 // Check that the token matches the expected one.
470 EXPECT_EQ(token.type, tokenizer.current().type);
471 EXPECT_EQ(token.text, tokenizer.current().text);
472 EXPECT_EQ(token.line, tokenizer.current().line);
473 EXPECT_EQ(token.column, tokenizer.current().column);
474 EXPECT_EQ(token.end_column, tokenizer.current().end_column);
476 } while (token.type != Tokenizer::TYPE_END);
840 // the last token, which the test will check for.
922 // Ignore all input, except remember if the last token was "foo".
943 // Create a tokenizer, read one token, then destroy it.