Lines Matching defs:tokens
46 const LexerResult& tokens) const {
51 child->left()->RunAction(child, tokens);
54 MatchedInput matched_input = completed_item->GetMatchedInput(tokens);
82 std::string Item::SplitByChildren(const LexerResult& tokens) const {
85 return child->SplitByChildren(tokens);
92 s << item->GetMatchedInput(tokens).ToString();
98 void Item::CheckAmbiguity(const Item& other, const LexerResult& tokens) const {
103 << child_->GetMatchedInput(tokens).ToString() << "\":\n "
104 << child_->SplitByChildren(tokens) << "\nvs\n "
105 << other.child_->SplitByChildren(tokens);
110 s << "Ambiguous grammer rules for \"" << GetMatchedInput(tokens).ToString()
111 << "\":\n " << SplitByChildren(tokens) << " ...\nvs\n "
112 << other.SplitByChildren(tokens) << " ...";
188 Symbol* start, const LexerResult& tokens,
211 size_t input_length = tokens.token_symbols.size();
218 MatchedInput last_token = tokens.token_contents[pos];
221 if (!is_new) item.CheckAmbiguity(worklist.back(), tokens);
236 if (pos < tokens.token_symbols.size() &&
237 tokens.token_symbols[pos] == next) {
254 // (that is, it matched zero tokens), we might not yet have
276 if (last_item.pos() < tokens.token_symbols.size()) {
277 std::string next_token = tokens.token_contents[last_item.pos()].ToString();