Lines Matching refs:TokenInfo
43 "untokenize", "TokenInfo"]
46 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
49 return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
444 yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
469 yield TokenInfo(STRING, contstr + line[:end],
474 yield TokenInfo(ERRORTOKEN, contstr + line,
503 yield TokenInfo(COMMENT, comment_token,
507 yield TokenInfo(NL, line[pos:],
513 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
521 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
539 yield TokenInfo(NUMBER, token, spos, epos, line)
542 yield TokenInfo(NL, token, spos, epos, line)
544 yield TokenInfo(NEWLINE, token, spos, epos, line)
548 yield TokenInfo(COMMENT, token, spos, epos, line)
556 yield TokenInfo(STRING, token, spos, (lnum, pos), line)
591 yield TokenInfo(STRING, token, spos, epos, line)
594 yield TokenInfo(NAME, token, spos, epos, line)
602 yield TokenInfo(OP, token, spos, epos, line)
604 yield TokenInfo(ERRORTOKEN, line[pos],
610 yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
612 yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
613 yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
690 yield TokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line)