Lines Matching refs:tokens
214 tokens = []
221 tokens.append((None, data[:m.start()], line, col))
222 tokens.append((m.lastgroup, m.group(), line, col))
225 tokens.append((None, data, line, col))
227 self.tokens = tokens
231 if self.off < len(self.tokens):
232 token = self.tokens[self.off]
248 raise ParseFailure(patterns, self.tokens[self.off:])
422 for i in range(p.off, len(p.tokens)):
423 f.write(p.tokens[i][1])