Lines Matching refs:token
37 #include "token.h"
53 struct token **tokenlist;
54 struct token *token;
190 const char *show_token(const struct token *token)
194 if (!token)
195 return "<no token>";
196 switch (token_type(token)) {
204 return show_ident(token->ident);
207 return token->number;
210 return show_special(token->special);
213 return show_char(token->string->data,
214 token->string->length - 1, 0, '\'');
216 return show_char(token->embedded,
217 token_type(token) - TOKEN_CHAR, 0, '\'');
219 return show_char(token->string->data,
220 token->string->length - 1, 'L', '\'');
222 return show_char(token->embedded,
223 token_type(token) - TOKEN_WIDE_CHAR, 'L', '\'');
225 return show_char(token->string->data,
226 token->string->length - 1, 0, '"');
228 return show_char(token->string->data,
229 token->string->length - 1, 'L', '"');
232 sprintf(buffer, "<beginning of '%s'>", stream_name(token->pos.stream));
236 sprintf(buffer, "<end of '%s'>", stream_name(token->pos.stream));
248 sprintf(buffer, "unhandled token type '%d' ", token_type(token));
253 const char *quote_token(const struct token *token)
257 switch (token_type(token)) {
262 return show_ident(token->ident);
265 return token->number;
268 return show_special(token->special);
271 return quote_char(token->string->data,
272 token->string->length - 1, 0, '\'');
274 return quote_char(token->embedded,
275 token_type(token) - TOKEN_CHAR, 0, '\'');
277 return quote_char(token->string->data,
278 token->string->length - 1, 'L', '\'');
280 return quote_char(token->embedded,
281 token_type(token) - TOKEN_WIDE_CHAR, 'L', '\'');
283 return quote_char(token->string->data,
284 token->string->length - 1, 0, '"');
286 return quote_char(token->string->data,
287 token->string->length - 1, 'L', '"');
289 sprintf(buffer, "unhandled token type '%d' ", token_type(token));
343 static struct token * alloc_token(stream_t *stream)
345 struct token *token = __alloc_token(0);
346 token->pos = stream_pos(stream);
347 return token;
458 struct token eof_token_entry;
460 static struct token *mark_eof(stream_t *stream)
462 struct token *end;
480 struct token *token = stream->token;
482 stream->token = NULL;
483 token->next = NULL;
484 *stream->tokenlist = token;
485 stream->tokenlist = &token->next;
490 stream->newline |= stream->token->pos.newline;
491 stream->whitespace |= stream->token->pos.whitespace;
492 stream->token = NULL;
547 struct token *token;
569 sparse_error(stream_pos(stream), "number token exceeds %td characters",
577 token = stream->token;
578 token_type(token) = TOKEN_NUMBER;
579 token->number = xmemdup(buffer, p - buffer);
589 struct token *token = stream->token;
629 token_type(token) = type + len;
631 memcpy(token->embedded, buffer, 4);
633 token_type(token) = type;
638 token->string = string;
642 token = stream->token;
746 struct token *token;
788 token = stream->token;
789 token_type(token) = TOKEN_SPECIAL;
790 token->special = value;
904 struct token *built_in_token(int stream, struct ident *ident)
906 struct token *token;
908 token = __alloc_token(0);
909 token->pos.stream = stream;
910 token_type(token) = TOKEN_IDENT;
911 token->ident = ident;
912 return token;
917 struct token *token;
950 token = stream->token;
951 token_type(token) = TOKEN_IDENT;
952 token->ident = ident;
967 static struct token *setup_stream(stream_t *stream, int idx, int fd,
970 struct token *begin;
978 stream->token = NULL;
990 static struct token *tokenize_stream(stream_t *stream)
995 struct token *token = alloc_token(stream);
996 stream->token = token;
1008 struct token * tokenize_buffer(void *buffer, unsigned long size, struct token **endtoken)
1011 struct token *begin;
1018 struct token * tokenize(const struct position *pos, const char *name, int fd, struct token *endtoken, const char **next_path)
1020 struct token *begin, *end;