Lines Matching refs:tokenize
5 import tokenize
187 def name(self) -> Optional[tokenize.TokenInfo]:
194 def number(self) -> Optional[tokenize.TokenInfo]:
201 def string(self) -> Optional[tokenize.TokenInfo]:
208 def op(self) -> Optional[tokenize.TokenInfo]:
215 def type_comment(self) -> Optional[tokenize.TokenInfo]:
222 def soft_keyword(self) -> Optional[tokenize.TokenInfo]:
229 def expect(self, type: str) -> Optional[tokenize.TokenInfo]:
243 def expect_forced(self, res: Any, expectation: str) -> Optional[tokenize.TokenInfo]:
293 tokengen = tokenize.generate_tokens(file.readline)