Lines Matching refs:token
11 the token type (see token.py)
12 the token (a string)
13 the starting (row, column) indices of the token (a 2-tuple of ints)
14 the ending (row, column) indices of the token (a 2-tuple of ints)
26 each time a new token is found."""
34 from lib2to3.pgen2.token import *
36 from . import token
37 __all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize",
39 del token
142 def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing
146 (srow, scol, erow, ecol, tok_name[type], repr(token)))
158 called once for each token, with five arguments, corresponding to the
190 tok_type, token, start, end, line = t
192 self.tokens.append(token)
199 def compat(self, token, iterable):
203 toknum, tokval = token
317 Each element returned by the iterable must be a token sequence
318 with at least two elements, a token number and token value. If
344 The generator produces 5-tuples with these members: the token type; the
345 token string; a 2-tuple (srow, scol) of ints specifying the row and
346 column where the token begins in the source; a 2-tuple (erow, ecol) of
347 ints specifying the row and column where the token ends in the source;
348 and the line on which the token was found. The line passed is the
451 token, initial = line[start:end], line[start]
454 (initial == '.' and token != '.'): # ordinary number
455 yield (NUMBER, token, spos, epos, line)
465 yield (newline, token, spos, epos, line)
468 assert not token.endswith("\n")
472 yield (COMMENT, token, spos, epos, line)
473 elif token in triple_quoted:
474 endprog = endprogs[token]
478 token = line[start:pos]
482 yield (STRING, token, spos, (lnum, pos), line)
489 token[:2] in single_quoted or \
490 token[:3] in single_quoted:
491 if token[-1] == '\n': # continued string
493 endprog = (endprogs[initial] or endprogs[token[1]] or
494 endprogs[token[2]])
502 yield (STRING, token, spos, epos, line)
504 if token in ('async', 'await'):
506 yield (ASYNC if token == 'async' else AWAIT,
507 token, spos, epos, line)
510 tok = (NAME, token, spos, epos, line)
511 if token == 'async' and not stashed:
515 if token in ('def', 'for'):
520 if token == 'def':
539 yield (NL, token, spos, (lnum, pos), line)
547 yield (OP, token, spos, epos, line)