Lines Matching refs:tokenize
35 Enter pygettext, which uses Python's standard tokenize module to scan
167 import tokenize
340 if ttype == tokenize.STRING and is_literal_string(tstring):
344 if ttype in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING):
348 if ttype == tokenize.NAME and tstring in ('class', 'def'):
351 if ttype == tokenize.NAME and tstring in opts.keywords:
354 if ttype == tokenize.STRING:
408 if ttype == tokenize.OP:
419 if ttype == tokenize.STRING and is_literal_string(tstring):
422 elif ttype not in (tokenize.NEWLINE, tokenize.INDENT,
423 tokenize.COMMENT):
428 if ttype == tokenize.OP and tstring == '(':
436 if ttype == tokenize.OP and tstring == ')':
444 elif ttype == tokenize.STRING and is_literal_string(tstring):
446 elif ttype not in [tokenize.COMMENT, token.INDENT, token.DEDENT,
447 token.NEWLINE, tokenize.NL]:
658 tokens = tokenize.tokenize(fp.readline)
661 except tokenize.TokenError as e: