Lines Matching refs:tokenize
147 import tokenize
1192 elif type == tokenize.NEWLINE:
1203 elif type == tokenize.INDENT:
1208 elif type == tokenize.DEDENT:
1215 elif type == tokenize.COMMENT:
1219 elif self.indent == 0 and type not in (tokenize.COMMENT, tokenize.NL):
1228 tokens = tokenize.generate_tokens(iter(lines).__next__)
2123 token_stream = tokenize.tokenize(generator)
2136 assert t.type == tokenize.ENCODING