Lines Matching refs:tokens
2 # Scanner produces tokens of the following types:
30 from .tokens import *
66 # List of processed tokens that are not yet emitted.
67 self.tokens = []
72 # Number of tokens that were emitted through the `get_token` method.
108 # '[', or '{' tokens.
117 if self.tokens:
121 if isinstance(self.tokens[0], choice):
127 # Return None if no more tokens.
130 if self.tokens:
131 return self.tokens[0]
139 if self.tokens:
141 return self.tokens.pop(0)
148 if not self.tokens:
164 # Compare the current indentation and column. It may add some tokens
307 token_number = self.tokens_taken+len(self.tokens)
327 ## In flow context, tokens should respect indentation.
343 # In block context, we may need to issue the BLOCK-END tokens.
347 self.tokens.append(BlockEndToken(mark, mark))
367 self.tokens.append(StreamStartToken(mark, mark,
385 self.tokens.append(StreamEndToken(mark, mark))
400 self.tokens.append(self.scan_directive())
422 self.tokens.append(TokenClass(start_mark, end_mark))
445 self.tokens.append(TokenClass(start_mark, end_mark))
468 self.tokens.append(TokenClass(start_mark, end_mark))
482 self.tokens.append(FlowEntryToken(start_mark, end_mark))
498 self.tokens.append(BlockSequenceStartToken(mark, mark))
515 self.tokens.append(BlockEntryToken(start_mark, end_mark))
531 self.tokens.append(BlockMappingStartToken(mark, mark))
543 self.tokens.append(KeyToken(start_mark, end_mark))
553 self.tokens.insert(key.token_number-self.tokens_taken,
560 self.tokens.insert(key.token_number-self.tokens_taken,
587 self.tokens.append(BlockMappingStartToken(mark, mark))
599 self.tokens.append(ValueToken(start_mark, end_mark))
610 self.tokens.append(self.scan_anchor(AliasToken))
621 self.tokens.append(self.scan_anchor(AnchorToken))
632 self.tokens.append(self.scan_tag())
649 self.tokens.append(self.scan_block_scalar(style))
666 self.tokens.append(self.scan_flow_scalar(style))
679 self.tokens.append(self.scan_plain())
762 # Tabs cannot precede tokens