Lines Matching refs:tokens
209 tokens = []
214 tokens.append((None, data[:m.start()]))
215 tokens.append((m.lastgroup, m.group()))
218 tokens.append((None, data))
220 self.tokens = tokens
224 if self.off < len(self.tokens):
225 token = self.tokens[self.off]
241 raise ParseFailure(patterns, self.tokens[self.off:])
368 for i in range(p.off, len(p.tokens)):
369 outf.write(p.tokens[i][1])