Lines Matching refs:tokenize
42 import tokenize
144 # Line-getter for tokenize.
156 STRING = tokenize.STRING
157 NL = tokenize.NL
158 NEWLINE = tokenize.NEWLINE
159 COMMENT = tokenize.COMMENT
160 NAME = tokenize.NAME
161 OP = tokenize.OP
164 get = tokenize.generate_tokens(self.getline).__next__
183 startline = srow - 1 # tokenize is one-based