Lines Matching defs:token
4 import token
184 return f"{tok.start[0]}.{tok.start[1]}: {token.tok_name[tok.type]}:{tok.string!r}"
189 if tok.type == token.NAME and tok.string not in self.KEYWORDS:
196 if tok.type == token.NUMBER:
203 if tok.type == token.STRING:
210 if tok.type == token.OP:
217 if tok.type == token.TYPE_COMMENT:
224 if tok.type == token.NAME and tok.string in self.SOFT_KEYWORDS:
236 if type in token.__dict__:
237 if tok.type == token.__dict__[type]:
239 if tok.type == token.OP and tok.string == type:
322 if diag.type == token.ENDMARKER:
332 print(f" token array : {len(tokenizer._tokens):10}")