Lines Matching refs:token

109 # bind operators to token types
180 def describe_token(token):
181     """Returns a description of the token."""
182     if token.type == TOKEN_NAME:
183         return token.value
184     return _describe_token_type(token.type)
188     """Like `describe_token` but for token expressions."""
276         """Test a token against a token expression.  This can either be a
277         token type or ``'token_type:token_value'``.  This can only test
289         """Test against multiple token expressions."""
302     until the eof token is reached.
312         token = self.stream.current
313         if token.type is TOKEN_EOF:
317         return token
322     """A token stream is an iterable that yields :class:`Token`\\s.  The
324     one token ahead.  The current active token is stored as :attr:`current`.
349     def push(self, token):
350         """Push a token back to the stream."""
351         self._pushed.append(token)
354         """Look at the next token."""
367         """Perform the token test and return the token if it matched.
378         """Go one token ahead and return the old one.
399         """Expect a given token type and return it.  This accepts the same
412                 "expected token %r, got %r" % (expr, describe_token(self.current)),
614         """Calls tokeniter + tokenize and wraps it in a token stream."""
620         every token in a :class:`Token` and converts the value.
622         for lineno, token, value in stream:
623             if token in ignored_tokens:
625             elif token == TOKEN_LINESTATEMENT_BEGIN:
626                 token = TOKEN_BLOCK_BEGIN
627             elif token == TOKEN_LINESTATEMENT_END:
628                 token = TOKEN_BLOCK_END
630             elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END):
632             elif token == TOKEN_DATA:
634             elif token == "keyword":
635                 token = value
636             elif token == TOKEN_NAME:
642             elif token == TOKEN_STRING:
653             elif token == TOKEN_INTEGER:
655             elif token == TOKEN_FLOAT:
658             elif token == TOKEN_OPERATOR:
659                 token = operators[value]
660             yield Token(lineno, token, value)
741                     for idx, token in enumerate(tokens):
743                         if token.__class__ is Failure:
744                             raise token(lineno, filename)
746                         # yield for the current token the first named
748                         elif token == "#bygroup":
757                                     "the token dynamically"
763                             if data or token not in ignore_if_empty:
764                                 yield lineno, token, data
768                 # strings as token just are yielded as it.