Lines Matching refs:tokens
45 /// This tokenizer is stateful, and only returns "useful" tokens - names, values etc.
46 /// It does not create tokens for the separator between names and values, or for the comma
48 /// tokens it produces are appropriate. For example, it would never produce "start object, end array."
66 /// Creates a tokenizer that first replays the given list of tokens, then continues reading
71 internal static JsonTokenizer FromReplayedTokens(IList<JsonToken> tokens, JsonTokenizer continuation)
73 return new JsonReplayTokenizer(tokens, continuation);
169 /// Tokenizer which first exhausts a list of tokens, then consults another tokenizer.
173 private readonly IList<JsonToken> tokens;
177 internal JsonReplayTokenizer(IList<JsonToken> tokens, JsonTokenizer nextTokenizer)
179 this.tokens = tokens;
186 if (nextTokenIndex >= tokens.Count)
190 return tokens[nextTokenIndex++];
237 // Skip whitespace between tokens