Lines Matching refs:token
2 * Do C preprocessing, based on a token list gathered by
42 #include "token.h"
94 static struct token *alloc_token(struct position *pos)
96 struct token *token = __alloc_token(0);
98 token->pos.stream = pos->stream;
99 token->pos.line = pos->line;
100 token->pos.pos = pos->pos;
101 token->pos.whitespace = 1;
102 return token;
106 static int expand(struct token **, struct symbol *);
108 static void replace_with_string(struct token *token, const char *str)
115 token_type(token) = TOKEN_STRING;
116 token->string = s;
119 static void replace_with_integer(struct token *token, unsigned int val)
123 token_type(token) = TOKEN_NUMBER;
124 token->number = buf;
135 static int token_defined(struct token *token)
137 if (token_type(token) == TOKEN_IDENT) {
138 struct symbol *sym = lookup_macro(token->ident);
146 sparse_error(token->pos, "expected preprocessor identifier");
150 static void replace_with_bool(struct token *token, bool val)
154 token_type(token) = TOKEN_NUMBER;
155 token->number = string[val];
158 static void replace_with_defined(struct token *token)
160 replace_with_bool(token, token_defined(token));
163 static void expand_line(struct token *token)
165 replace_with_integer(token, token->pos.line);
168 static void expand_file(struct token *token)
170 replace_with_string(token, stream_name(token->pos.stream));
173 static void expand_basefile(struct token *token)
175 replace_with_string(token, base_filename);
179 static void expand_date(struct token *token)
186 replace_with_string(token, buffer);
189 static void expand_time(struct token *token)
196 replace_with_string(token, buffer);
199 static void expand_counter(struct token *token)
201 replace_with_integer(token, counter_macro++);
204 static void expand_include_level(struct token *token)
206 replace_with_integer(token, include_level - 1);
209 static int expand_one_symbol(struct token **list)
211 struct token *token = *list;
214 if (token->pos.noexpand)
217 sym = lookup_macro(token->ident);
221 sym->expand_simple(token);
234 static inline struct token *scan_next(struct token **where)
236 struct token *token = *where;
237 if (token_type(token) != TOKEN_UNTAINT)
238 return token;
240 token->ident->tainted = 0;
241 token = token->next;
242 } while (token_type(token) == TOKEN_UNTAINT);
243 *where = token;
244 return token;
247 static void expand_list(struct token **list)
249 struct token *next;
256 static void preprocessor_line(struct stream *stream, struct token **line);
258 static struct token *collect_arg(struct token *prev, int vararg, struct position *pos, int count)
261 struct token **p = &prev->next;
262 struct token *next;
269 __free_token(next); /* Free the '#' token */
312 struct token *arg;
313 struct token *expanded;
314 struct token *str;
320 static int collect_arguments(struct token *start, struct token *arglist, struct arg *args, struct token *what)
323 struct token *next = NULL;
394 static struct token *dup_list(struct token *list)
396 struct token *res = NULL;
397 struct token **p = &res;
400 struct token *newtok = __alloc_token(0);
409 static const char *show_token_sequence(struct token *token, int quote)
415 if (!token && !quote)
417 while (!eof_token(token)) {
418 const char *val = quote ? quote_token(token) : show_token(token);
422 sparse_error(token->pos, "too long token expansion");
430 token = token->next;
431 whitespace = token->pos.whitespace;
437 static struct token *stringify(struct token *arg)
441 struct token *token = __alloc_token(0);
446 token->pos = arg->pos;
447 token_type(token) = TOKEN_STRING;
448 token->string = string;
449 token->next = &eof_token_entry;
450 return token;
457 struct token *arg = args[i].arg;
489 static enum token_type combine(struct token *left, struct token *right, char *p)
543 static int merge(struct token *left, struct token *right)
587 sparse_error(left->pos, "'##' failed: concatenation is not a valid token");
591 static struct token *dup_token(struct token *token, struct position *streampos)
593 struct token *alloc = alloc_token(streampos);
594 token_type(alloc) = token_type(token);
595 alloc->pos.newline = token->pos.newline;
596 alloc->pos.whitespace = token->pos.whitespace;
597 alloc->number = token->number;
598 alloc->pos.noexpand = token->pos.noexpand;
602 static struct token **copy(struct token **where, struct token *list, int *count)
606 struct token *token;
608 token = dup_token(list, &list->pos);
610 token = list;
611 if (token_type(token) == TOKEN_IDENT && token->ident->tainted)
612 token->pos.noexpand = 1;
613 *where = token;
614 where = &token->next;
621 static int handle_kludge(struct token **p, struct arg *args)
623 struct token *t = (*p)->next->next;
642 static struct token **substitute(struct token **list, struct token *body, struct arg *args)
649 struct token *added, *arg;
650 struct token **tail;
651 struct token *t;
745 static int expand(struct token **list, struct symbol *sym)
747 struct token *last;
748 struct token *token = *list;
749 struct ident *expanding = token->ident;
750 struct token **tail;
751 struct token *expansion = sym->expansion;
756 token->pos.noexpand = 1;
761 if (!match_op(scan_next(&token->next), '('))
763 if (!collect_arguments(token->next, sym->arglist, args, token))
769 return sym->expand(token, args) ? 0 : 1;
773 last = token->next;
781 (*list)->pos.newline = token->pos.newline;
782 (*list)->pos.whitespace = token->pos.whitespace;
788 static const char *token_name_sequence(struct token *token, int endop, struct token *start)
793 while (!eof_token(token) && !match_op(token, endop)) {
795 const char *val = token->string->data;
796 if (token_type(token) != TOKEN_STRING)
797 val = show_token(token);
801 token = token->next;
804 if (endop && !match_op(token, endop))
899 static int try_include(struct position pos, const char *path, const char *filename, int flen, struct token **where, const char **next_path)
923 static int do_include_path(const char **pptr, struct token **list, struct token *token, const char *filename, int flen)
928 if (!try_include(token->pos, path, filename, flen, list, pptr))
935 static int free_preprocessor_line(struct token *token)
937 while (token_type(token) != TOKEN_EOF) {
938 struct token *free = token;
939 token = token->next;
945 static int handle_include_path(struct stream *stream, struct token **list, struct token *token, int how)
948 struct token *next;
953 next = token->next;
956 expand_list(&token->next);
958 next = token;
959 if (match_op(token->next, '<')) {
960 next = token->next;
965 token = next->next;
966 filename = token_name_sequence(token, expect, token);
971 if (try_include(token->pos, "", filename, flen, list, includepath))
991 if (do_include_path(path, list, token, filename, flen))
994 error_die(token->pos, "unable to open '%s'", filename);
997 static int handle_include(struct stream *stream, struct token **list, struct token *token)
999 return handle_include_path(stream, list, token, 0);
1002 static int handle_include_next(struct stream *stream, struct token **list, struct token *token)
1004 return handle_include_path(stream, list, token, 1);
1007 static int handle_argv_include(struct stream *stream, struct token **list, struct token *token)
1009 return handle_include_path(stream, list, token, 2);
1012 static int token_different(struct token *t1, struct token *t2)
1065 static int token_list_different(struct token *list1, struct token *list2)
1079 static inline void set_arg_count(struct token *token)
1081 token_type(token) = TOKEN_ARG_COUNT;
1082 token->count.normal = token->count.quoted =
1083 token->count.str = token->count.vararg = 0;
1086 static struct token *parse_arguments(struct token *list)
1088 struct token *arg = list->next, *next = list;
1185 static int try_arg(struct token *token, enum token_type type, struct token *arglist)
1187 struct ident *ident = token->ident;
1190 if (!arglist || token_type(token) != TOKEN_IDENT)
1200 token->argnum = nr;
1201 token_type(token) = type;
1218 token_type(token) = TOKEN_ERROR;
1225 static struct token *handle_hash(struct token **p, struct token *arglist)
1227 struct token *token = *p;
1229 struct token *next = token->next;
1232 next->pos.whitespace = token->pos.whitespace;
1233 __free_token(token);
1234 token = *p = next;
1236 token->pos.noexpand = 1;
1238 return token;
1241 sparse_error(token->pos, "'#' is not followed by a macro parameter");
1245 /* token->next is ## */
1246 static struct token *handle_hashhash(struct token *token, struct token *arglist)
1248 struct token *last = token;
1249 struct token *concat;
1250 int state = match_op(token, ',');
1252 try_arg(token, TOKEN_QUOTED_ARGUMENT, arglist);
1255 struct token *t;
1259 concat = token->next;
1261 token->next = t;
1285 token = t;
1286 if (!match_op(token->next, SPECIAL_HASHHASH))
1292 return token;
1299 static struct token *parse_expansion(struct token *expansion, struct token *arglist, struct ident *name)
1301 struct token *token = expansion;
1302 struct token **p;
1304 if (match_op(token, SPECIAL_HASHHASH))
1307 for (p = &expansion; !eof_token(token); p = &token->next, token = *p) {
1308 if (match_op(token, '#')) {
1309 token = handle_hash(p, arglist);
1310 if (!token)
1313 if (match_op(token->next, SPECIAL_HASHHASH)) {
1314 token = handle_hashhash(token, arglist);
1315 if (!token)
1318 try_arg(token, TOKEN_MACRO_ARGUMENT, arglist);
1320 switch (token_type(token)) {
1326 token->string->immutable = 1;
1330 token = alloc_token(&expansion->pos);
1331 token_type(token) = TOKEN_UNTAINT;
1332 token->ident = name;
1333 token->next = *p;
1334 *p = token;
1338 sparse_error(token->pos, "'##' cannot appear at the ends of macro expansion");
1341 sparse_error(token->pos, "too many instances of argument in body");
1345 static int do_define(struct position pos, struct token *token, struct ident *name,
1346 struct token *arglist, struct token *expansion, int attr)
1369 warning(pos, "preprocessor token %.*s redefined",
1387 if (token) /* Free the "define" token, but not the rest of the line */
1388 __free_token(token);
1410 struct token *value = &eof_token_entry;
1470 static int do_handle_define(struct stream *stream, struct token **line, struct token *token, int attr)
1472 struct token *arglist, *expansion;
1473 struct token *left = token->next;
1477 sparse_error(token->pos, "expected identifier to 'define'");
1497 return do_define(left->pos, token, name, arglist, expansion, attr);
1500 static int handle_define(struct stream *stream, struct token **line, struct token *token)
1502 return do_handle_define(stream, line, token, SYM_ATTR_NORMAL);
1505 static int handle_weak_define(struct stream *stream, struct token **line, struct token *token)
1507 return do_handle_define(stream, line, token, SYM_ATTR_WEAK);
1510 static int handle_strong_define(struct stream *stream, struct token **line, struct token *token)
1512 return do_handle_define(stream, line, token, SYM_ATTR_STRONG);
1515 static int do_handle_undef(struct stream *stream, struct token **line, struct token *token, int attr)
1517 struct token *left = token->next;
1521 sparse_error(token->pos, "expected identifier to 'undef'");
1546 static int handle_undef(struct stream *stream, struct token **line, struct token *token)
1548 return do_handle_undef(stream, line, token, SYM_ATTR_NORMAL);
1551 static int handle_strong_undef(struct stream *stream, struct token **line, struct token *token)
1553 return do_handle_undef(stream, line, token, SYM_ATTR_STRONG);
1556 static int preprocessor_if(struct stream *stream, struct token *token, int cond)
1558 token_type(token) = false_nesting ? TOKEN_SKIP_GROUPS : TOKEN_IF;
1559 free_preprocessor_line(token->next);
1560 token->next = stream->top_if;
1561 stream->top_if = token;
1567 static int handle_ifdef(struct stream *stream, struct token **line, struct token *token)
1569 struct token *next = token->next;
1576 sparse_error(token->pos, "expected preprocessor identifier");
1579 return preprocessor_if(stream, token, arg);
1582 static int handle_ifndef(struct stream *stream, struct token **line, struct token *token)
1584 struct token *next = token->next;
1589 stream->ifndef = token;
1592 stream->ifndef = token;
1600 sparse_error(token->pos, "expected preprocessor identifier");
1604 return preprocessor_if(stream, token, arg);
1611 static int expression_value(struct token **where)
1614 struct token *p;
1615 struct token **list = where, **beginning = NULL;
1669 static int handle_if(struct stream *stream, struct token **line, struct token *token)
1673 value = expression_value(&token->next);
1676 return preprocessor_if(stream, token, value);
1679 static int handle_elif(struct stream * stream, struct token **line, struct token *token)
1681 struct token *top_if = stream->top_if;
1686 sparse_error(token->pos, "unmatched #elif within stream");
1692 sparse_error(token->pos, "#elif after #else");
1703 if (!expression_value(&token->next))
1712 static int handle_else(struct stream *stream, struct token **line, struct token *token)
1714 struct token *top_if = stream->top_if;
1719 sparse_error(token->pos, "unmatched #else within stream");
1725 sparse_error(token->pos, "#else after #else");
1737 static int handle_endif(struct stream *stream, struct token **line, struct token *token)
1739 struct token *top_if = stream->top_if;
1743 sparse_error(token->pos, "unmatched #endif in stream");
1753 static int handle_warning(struct stream *stream, struct token **line, struct token *token)
1755 warning(token->pos, "%s", show_token_sequence(token->next, 0));
1759 static int handle_error(struct stream *stream, struct token **line, struct token *token)
1761 sparse_error(token->pos, "%s", show_token_sequence(token->next, 0));
1765 static int handle_nostdinc(struct stream *stream, struct token **line, struct token *token)
1808 static void add_path_entry(struct token *token, const char *path,
1816 error_die(token->pos, "too many include path entries");
1842 static int handle_add_include(struct stream *stream, struct token **line, struct token *token)
1845 token = token->next;
1846 if (eof_token(token))
1848 if (token_type(token) != TOKEN_STRING) {
1849 warning(token->pos, "expected path string");
1852 add_path_entry(token, token->string->data, &isys_includepath);
1856 static int handle_add_isystem(struct stream *stream, struct token **line, struct token *token)
1859 token = token->next;
1860 if (eof_token(token))
1862 if (token_type(token) != TOKEN_STRING) {
1863 sparse_error(token->pos, "expected path string");
1866 add_path_entry(token, token->string->data, &sys_includepath);
1870 static int handle_add_system(struct stream *stream, struct token **line, struct token *token)
1873 token = token->next;
1874 if (eof_token(token))
1876 if (token_type(token) != TOKEN_STRING) {
1877 sparse_error(token->pos, "expected path string");
1880 add_path_entry(token, token->string->data, &dirafter_includepath);
1885 static void add_dirafter_entry(struct token *token, const char *path)
1891 error_die(token->pos, "too many include path entries");
1901 static int handle_add_dirafter(struct stream *stream, struct token **line, struct token *token)
1904 token = token->next;
1905 if (eof_token(token))
1907 if (token_type(token) != TOKEN_STRING) {
1908 sparse_error(token->pos, "expected path string");
1911 add_dirafter_entry(token, token->string->data);
1915 static int handle_split_include(struct stream *stream, struct token **line, struct token *token)
1935 * We replace "#pragma xxx" with "__pragma__" in the token
1939 * is that we can use this to insert arbitrary token sequences
1946 static int handle_pragma(struct stream *stream, struct token **line, struct token *token)
1948 struct token *next = *line;
1950 if (match_ident(token->next, &once_ident) && eof_token(token->next->next)) {
1954 token->ident = &pragma_ident;
1955 token->pos.newline = 1;
1956 token->pos.whitespace = 1;
1957 token->pos.pos = 1;
1958 *line = token;
1959 token->next = next;
1966 static int handle_line(struct stream *stream, struct token **line, struct token *token)
1971 static int handle_ident(struct stream *stream, struct token **line, struct token *token)
1976 static int handle_nondirective(struct stream *stream, struct token **line, struct token *token)
1978 sparse_error(token->pos, "unrecognized preprocessor line '%s'", show_token_sequence(token, 0));
1982 static bool expand_has_attribute(struct token *token, struct arg *args)
1984 struct token *arg = args[0].expanded;
1993 replace_with_bool(token, sym && sym->op && sym->op->attribute);
1997 static bool expand_has_builtin(struct token *token, struct arg *args)
1999 struct token *arg = args[0].expanded;
2008 replace_with_bool(token, sym && sym->builtin);
2012 static bool expand_has_extension(struct token *token, struct arg *args)
2014 struct token *arg = args[0].expanded;
2033 replace_with_bool(token, val);
2037 static bool expand_has_feature(struct token *token, struct arg *args)
2039 struct token *arg = args[0].expanded;
2060 replace_with_bool(token, val);
2066 struct token *token;
2067 struct token **next;
2072 token = __alloc_token(0);
2073 token_type(token) = TOKEN_ARG_COUNT;
2074 token->count.normal = count;
2075 sym->arglist = token;
2076 next = &token->next;
2079 struct token *id, *uses;
2099 int (*handler)(struct stream *, struct token **, struct token *);
2132 void (*expand_simple)(struct token *);
2133 bool (*expand)(struct token *, struct arg *args);
2171 static void handle_preprocessor_line(struct stream *stream, struct token **line, struct token *start)
2173 int (*handler)(struct stream *, struct token **, struct token *);
2174 struct token *token = start->next;
2178 if (eof_token(token))
2181 if (token_type(token) == TOKEN_IDENT) {
2182 struct symbol *sym = lookup_symbol(token->ident, NS_PREPROCESSOR);
2190 } else if (token_type(token) == TOKEN_NUMBER) {
2206 if (!handler(stream, line, token)) /* all set */
2210 free_preprocessor_line(token);
2213 static void preprocessor_line(struct stream *stream, struct token **line)
2215 struct token *start = *line, *next;
2216 struct token **tp = &start->next;
2229 static void do_preprocess(struct token **list)
2231 struct token *next;
2239 __free_token(next); /* Free the '#' token */
2277 struct token * preprocess(struct token *token)
2281 do_preprocess(&token);
2288 return token;
2291 static int is_VA_ARGS_token(struct token *token)
2293 return (token_type(token) == TOKEN_IDENT) &&
2294 (token->ident == &__VA_ARGS___ident);
2300 struct token *args[nargs];
2301 struct token *token;
2304 token = sym->arglist;
2305 if (token) {
2309 for (; !eof_token(token); token = token->next) {
2310 if (token_type(token) == TOKEN_ARG_COUNT)
2312 if (is_VA_ARGS_token(token))
2315 printf("%s%s", sep, show_token(token));
2316 args[narg++] = token;
2322 token = sym->expansion;
2323 while (token_type(token) != TOKEN_UNTAINT) {
2324 struct token *next = token->next;
2325 if (token->pos.whitespace)
2327 switch (token_type(token)) {
2336 token = args[token->argnum];
2339 printf("%s", show_token(token));
2341 token = next;