Lines Matching refs:token
18 * file is a text file with one token definition per line. Each line is of the form:
34 static void writeH(const DFA& dfa, const char* lexer, const char* token,
46 out << "struct " << token << " {\n";
55 )" << token << "() {}";
57 out << token << R"((Kind kind, int32_t offset, int32_t length, int32_t line)
77 )" << token << R"( next();
104 static void writeCPP(const DFA& dfa, const char* lexer, const char* token, const char* include,
143 out << token << " " << lexer << "::next() {";
146 // where a token has a prefix which is not itself a valid token - for instance,
147 // maybe we have a valid token 'while', but 'w', 'wh', etc. are not valid
152 return )" << token << "(" << token << R"(::Kind::TK_END_OF_FILE, startOffset, 0, fLine);
176 Token::Kind kind = ()" << token << R"(::Kind) kAccepts[state];
177 return )" << token << R"((kind, startOffset, fOffset - startOffset, fLine);
184 static void process(const char* inPath, const char* lexer, const char* token, const char* hPath,
222 writeH(dfa, lexer, token, tokens, hPath);
223 writeCPP(dfa, lexer, token, (std::string("src/sksl/SkSL") + lexer + ".h").c_str(), cppPath);