Lines Matching refs:tokenize
46 import tokenize
122 encoding, _ = tokenize.detect_encoding(f.readline)
184 # that we can use tokenize's 1-based line numbering easily.
193 # signal that tokenize doesn't know what to do about them;
202 tokens = tokenize.generate_tokens(self.getline)
275 # Line-getter for tokenize.
284 # Line-eater for tokenize.
286 INDENT=tokenize.INDENT,
287 DEDENT=tokenize.DEDENT,
288 NEWLINE=tokenize.NEWLINE,
289 COMMENT=tokenize.COMMENT,
290 NL=tokenize.NL):