Lines Matching refs:tokens
58 # internal the tokens and keep references to them
362 """Got n tokens ahead."""
629 # we are not interested in those tokens in the parser
663 """This method tokenizes the text and returns the tokens in a
689 for regex, tokens, new_state in statetokens:
699 if balancing_stack and tokens in (
707 if isinstance(tokens, tuple):
710 if isinstance(tokens, OptionalLStrip):
741 for idx, token in enumerate(tokens):
772 if tokens == TOKEN_OPERATOR:
794 if data or tokens not in ignore_if_empty:
795 yield lineno, tokens, data