Lines Matching refs:tokens
10 //! structures into tokens of source code.
14 //! Procedural macros in Rust receive a stream of tokens as input, execute
15 //! arbitrary Rust code to determine how to manipulate those tokens, and produce
16 //! a stream of tokens to hand back to the compiler to compile into the caller's
18 //! tokens to return to the compiler.
25 //! it around, mutate it, and eventually hand it back to the compiler as tokens
43 //! syntax performs interpolation of runtime variables into the quoted tokens.
61 //! let tokens = quote! {
88 //! file, consider having the code generator pass the tokens through
133 /// Note: for returning tokens to the compiler in a procedural macro, use
144 /// scope and inserts it in that location in the output tokens. Any type
160 /// - `#( struct #var; )*` — the repetition can contain other tokens
167 /// Any interpolated tokens preserve the `Span` information provided by their
194 /// `quote!` from a procedural macro usually looks like `tokens.into()` or
195 /// `proc_macro::TokenStream::from(tokens)`.
243 /// // Hand the output tokens back to the compiler.
253 /// piece. Different parts may come from different helper functions. The tokens
263 /// let tokens = quote! {
279 /// continue to be two separate tokens as if you had written `_ x`.
460 /// # fn to_tokens(&self, tokens: &mut TokenStream) {
461 /// # tokens.append(Literal::usize_unsuffixed(self.0));
516 // Rule for any other number of tokens.
524 /// Same as `quote!`, but applies a given span to all tokens originating within
531 /// A span expression of type [`Span`], followed by `=>`, followed by the tokens
549 /// let tokens = quote_spanned!(span=> Box::into_raw(Box::new(#init)));
552 /// let tokens = quote_spanned! {span=>
561 /// tokens being evaluated in the generated code.
567 /// Any interpolated tokens preserve the `Span` information provided by their
659 // Rule for any other number of tokens.
806 // correspond to one of the original input tokens, so turns into nothing.
810 ($tokens:ident $($tts:tt)*) => {
811 $crate::quote_tokens_with_context!{$tokens
827 ($tokens:ident $span:ident $($tts:tt)*) => {
828 $crate::quote_tokens_with_context_spanned!{$tokens $span
844 ($tokens:ident
850 $crate::quote_token_with_context!{$tokens $b3 $b2 $b1 $curr $a1 $a2 $a3}
859 ($tokens:ident $span:ident
865 $crate::quote_token_with_context_spanned!{$tokens $span $b3 $b2 $b1 $curr $a1 $a2 $a3}
875 // original input tokens. Ignore it.
876 ($tokens:ident $b3:tt $b2:tt $b1:tt @ $a1:tt $a2:tt $a3:tt) => {};
879 ($tokens:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) * $a3:tt) => {{
892 $crate::quote_each_token!{$tokens $($inner)*}
896 ($tokens:ident $b3:tt $b2:tt # (( $($inner:tt)* )) * $a2:tt $a3:tt) => {};
898 ($tokens:ident $b3:tt # ( $($inner:tt)* ) (*) $a1:tt $a2:tt $a3:tt) => {};
901 ($tokens:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) $sep:tt *) => {{
910 $crate::quote_token!{$sep $tokens}
913 $crate::quote_each_token!{$tokens $($inner)*}
917 ($tokens:ident $b3:tt $b2:tt # (( $($inner:tt)* )) $sep:tt * $a3:tt) => {};
919 ($tokens:ident $b3:tt # ( $($inner:tt)* ) ($sep:tt) * $a2:tt $a3:tt) => {};
922 ($tokens:ident # ( $($inner:tt)* ) * (*) $a1:tt $a2:tt $a3:tt) => {
924 $crate::quote_token!{* $tokens}
927 ($tokens:ident # ( $($inner:tt)* ) $sep:tt (*) $a1:tt $a2:tt $a3:tt) => {};
930 ($tokens:ident $b3:tt $b2:tt $b1:tt (#) $var:ident $a2:tt $a3:tt) => {
931 $crate::ToTokens::to_tokens(&$var, &mut $tokens);
934 ($tokens:ident $b3:tt $b2:tt # ($var:ident) $a1:tt $a2:tt $a3:tt) => {};
937 ($tokens:ident $b3:tt $b2:tt $b1:tt ($curr:tt) $a1:tt $a2:tt $a3:tt) => {
938 $crate::quote_token!{$curr $tokens}
947 ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt @ $a1:tt $a2:tt $a3:tt) => {};
949 ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) * $a3:tt) => {{
956 $crate::quote_each_token_spanned!{$tokens $span $($inner)*}
959 ($tokens:ident $span:ident $b3:tt $b2:tt # (( $($inner:tt)* )) * $a2:tt $a3:tt) => {};
960 ($tokens:ident $span:ident $b3:tt # ( $($inner:tt)* ) (*) $a1:tt $a2:tt $a3:tt) => {};
962 ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) $sep:tt *) => {{
971 $crate::quote_token_spanned!{$sep $tokens $span}
974 $crate::quote_each_token_spanned!{$tokens $span $($inner)*}
977 ($tokens:ident $span:ident $b3:tt $b2:tt # (( $($inner:tt)* )) $sep:tt * $a3:tt) => {};
978 ($tokens:ident $span:ident $b3:tt # ( $($inner:tt)* ) ($sep:tt) * $a2:tt $a3:tt) => {};
979 ($tokens:ident $span:ident # ( $($inner:tt)* ) * (*) $a1:tt $a2:tt $a3:tt) => {
981 $crate::quote_token_spanned!{* $tokens $span}
983 ($tokens:ident $span:ident # ( $($inner:tt)* ) $sep:tt (*) $a1:tt $a2:tt $a3:tt) => {};
985 ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) $var:ident $a2:tt $a3:tt) => {
986 $crate::ToTokens::to_tokens(&$var, &mut $tokens);
988 ($tokens:ident $span:ident $b3:tt $b2:tt # ($var:ident) $a1:tt $a2:tt $a3:tt) => {};
990 ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt ($curr:tt) $a1:tt $a2:tt $a3:tt) => {
991 $crate::quote_token_spanned!{$curr $tokens $span}
1005 ($ident:ident $tokens:ident) => {
1006 $crate::__private::push_ident(&mut $tokens, stringify!($ident));
1009 (:: $tokens:ident) => {
1010 $crate::__private::push_colon2(&mut $tokens);
1013 (( $($inner:tt)* ) $tokens:ident) => {
1015 &mut $tokens,
1021 ([ $($inner:tt)* ] $tokens:ident) => {
1023 &mut $tokens,
1029 ({ $($inner:tt)* } $tokens:ident) => {
1031 &mut $tokens,
1037 (# $tokens:ident) => {
1038 $crate::__private::push_pound(&mut $tokens);
1041 (, $tokens:ident) => {
1042 $crate::__private::push_comma(&mut $tokens);
1045 (. $tokens:ident) => {
1046 $crate::__private::push_dot(&mut $tokens);
1049 (; $tokens:ident) => {
1050 $crate::__private::push_semi(&mut $tokens);
1053 (: $tokens:ident) => {
1054 $crate::__private::push_colon(&mut $tokens);
1057 (+ $tokens:ident) => {
1058 $crate::__private::push_add(&mut $tokens);
1061 (+= $tokens:ident) => {
1062 $crate::__private::push_add_eq(&mut $tokens);
1065 (& $tokens:ident) => {
1066 $crate::__private::push_and(&mut $tokens);
1069 (&& $tokens:ident) => {
1070 $crate::__private::push_and_and(&mut $tokens);
1073 (&= $tokens:ident) => {
1074 $crate::__private::push_and_eq(&mut $tokens);
1077 (@ $tokens:ident) => {
1078 $crate::__private::push_at(&mut $tokens);
1081 (! $tokens:ident) => {
1082 $crate::__private::push_bang(&mut $tokens);
1085 (^ $tokens:ident) => {
1086 $crate::__private::push_caret(&mut $tokens);
1089 (^= $tokens:ident) => {
1090 $crate::__private::push_caret_eq(&mut $tokens);
1093 (/ $tokens:ident) => {
1094 $crate::__private::push_div(&mut $tokens);
1097 (/= $tokens:ident) => {
1098 $crate::__private::push_div_eq(&mut $tokens);
1101 (.. $tokens:ident) => {
1102 $crate::__private::push_dot2(&mut $tokens);
1105 (... $tokens:ident) => {
1106 $crate::__private::push_dot3(&mut $tokens);
1109 (..= $tokens:ident) => {
1110 $crate::__private::push_dot_dot_eq(&mut $tokens);
1113 (= $tokens:ident) => {
1114 $crate::__private::push_eq(&mut $tokens);
1117 (== $tokens:ident) => {
1118 $crate::__private::push_eq_eq(&mut $tokens);
1121 (>= $tokens:ident) => {
1122 $crate::__private::push_ge(&mut $tokens);
1125 (> $tokens:ident) => {
1126 $crate::__private::push_gt(&mut $tokens);
1129 (<= $tokens:ident) => {
1130 $crate::__private::push_le(&mut $tokens);
1133 (< $tokens:ident) => {
1134 $crate::__private::push_lt(&mut $tokens);
1137 (*= $tokens:ident) => {
1138 $crate::__private::push_mul_eq(&mut $tokens);
1141 (!= $tokens:ident) => {
1142 $crate::__private::push_ne(&mut $tokens);
1145 (| $tokens:ident) => {
1146 $crate::__private::push_or(&mut $tokens);
1149 (|= $tokens:ident) => {
1150 $crate::__private::push_or_eq(&mut $tokens);
1153 (|| $tokens:ident) => {
1154 $crate::__private::push_or_or(&mut $tokens);
1157 (? $tokens:ident) => {
1158 $crate::__private::push_question(&mut $tokens);
1161 (-> $tokens:ident) => {
1162 $crate::__private::push_rarrow(&mut $tokens);
1165 (<- $tokens:ident) => {
1166 $crate::__private::push_larrow(&mut $tokens);
1169 (% $tokens:ident) => {
1170 $crate::__private::push_rem(&mut $tokens);
1173 (%= $tokens:ident) => {
1174 $crate::__private::push_rem_eq(&mut $tokens);
1177 (=> $tokens:ident) => {
1178 $crate::__private::push_fat_arrow(&mut $tokens);
1181 (<< $tokens:ident) => {
1182 $crate::__private::push_shl(&mut $tokens);
1185 (<<= $tokens:ident) => {
1186 $crate::__private::push_shl_eq(&mut $tokens);
1189 (>> $tokens:ident) => {
1190 $crate::__private::push_shr(&mut $tokens);
1193 (>>= $tokens:ident) => {
1194 $crate::__private::push_shr_eq(&mut $tokens);
1197 (* $tokens:ident) => {
1198 $crate::__private::push_star(&mut $tokens);
1201 (- $tokens:ident) => {
1202 $crate::__private::push_sub(&mut $tokens);
1205 (-= $tokens:ident) => {
1206 $crate::__private::push_sub_eq(&mut $tokens);
1209 ($lifetime:lifetime $tokens:ident) => {
1210 $crate::__private::push_lifetime(&mut $tokens, stringify!($lifetime));
1213 (_ $tokens:ident) => {
1214 $crate::__private::push_underscore(&mut $tokens);
1217 ($other:tt $tokens:ident) => {
1218 $crate::__private::parse(&mut $tokens, stringify!($other));
1226 ($ident:ident $tokens:ident $span:ident) => {
1227 $crate::__private::push_ident_spanned(&mut $tokens, $span, stringify!($ident));
1230 (:: $tokens:ident $span:ident) => {
1231 $crate::__private::push_colon2_spanned(&mut $tokens, $span);
1234 (( $($inner:tt)* ) $tokens:ident $span:ident) => {
1236 &mut $tokens,
1243 ([ $($inner:tt)* ] $tokens:ident $span:ident) => {
1245 &mut $tokens,
1252 ({ $($inner:tt)* } $tokens:ident $span:ident) => {
1254 &mut $tokens,
1261 (# $tokens:ident $span:ident) => {
1262 $crate::__private::push_pound_spanned(&mut $tokens, $span);
1265 (, $tokens:ident $span:ident) => {
1266 $crate::__private::push_comma_spanned(&mut $tokens, $span);
1269 (. $tokens:ident $span:ident) => {
1270 $crate::__private::push_dot_spanned(&mut $tokens, $span);
1273 (; $tokens:ident $span:ident) => {
1274 $crate::__private::push_semi_spanned(&mut $tokens, $span);
1277 (: $tokens:ident $span:ident) => {
1278 $crate::__private::push_colon_spanned(&mut $tokens, $span);
1281 (+ $tokens:ident $span:ident) => {
1282 $crate::__private::push_add_spanned(&mut $tokens, $span);
1285 (+= $tokens:ident $span:ident) => {
1286 $crate::__private::push_add_eq_spanned(&mut $tokens, $span);
1289 (& $tokens:ident $span:ident) => {
1290 $crate::__private::push_and_spanned(&mut $tokens, $span);
1293 (&& $tokens:ident $span:ident) => {
1294 $crate::__private::push_and_and_spanned(&mut $tokens, $span);
1297 (&= $tokens:ident $span:ident) => {
1298 $crate::__private::push_and_eq_spanned(&mut $tokens, $span);
1301 (@ $tokens:ident $span:ident) => {
1302 $crate::__private::push_at_spanned(&mut $tokens, $span);
1305 (! $tokens:ident $span:ident) => {
1306 $crate::__private::push_bang_spanned(&mut $tokens, $span);
1309 (^ $tokens:ident $span:ident) => {
1310 $crate::__private::push_caret_spanned(&mut $tokens, $span);
1313 (^= $tokens:ident $span:ident) => {
1314 $crate::__private::push_caret_eq_spanned(&mut $tokens, $span);
1317 (/ $tokens:ident $span:ident) => {
1318 $crate::__private::push_div_spanned(&mut $tokens, $span);
1321 (/= $tokens:ident $span:ident) => {
1322 $crate::__private::push_div_eq_spanned(&mut $tokens, $span);
1325 (.. $tokens:ident $span:ident) => {
1326 $crate::__private::push_dot2_spanned(&mut $tokens, $span);
1329 (... $tokens:ident $span:ident) => {
1330 $crate::__private::push_dot3_spanned(&mut $tokens, $span);
1333 (..= $tokens:ident $span:ident) => {
1334 $crate::__private::push_dot_dot_eq_spanned(&mut $tokens, $span);
1337 (= $tokens:ident $span:ident) => {
1338 $crate::__private::push_eq_spanned(&mut $tokens, $span);
1341 (== $tokens:ident $span:ident) => {
1342 $crate::__private::push_eq_eq_spanned(&mut $tokens, $span);
1345 (>= $tokens:ident $span:ident) => {
1346 $crate::__private::push_ge_spanned(&mut $tokens, $span);
1349 (> $tokens:ident $span:ident) => {
1350 $crate::__private::push_gt_spanned(&mut $tokens, $span);
1353 (<= $tokens:ident $span:ident) => {
1354 $crate::__private::push_le_spanned(&mut $tokens, $span);
1357 (< $tokens:ident $span:ident) => {
1358 $crate::__private::push_lt_spanned(&mut $tokens, $span);
1361 (*= $tokens:ident $span:ident) => {
1362 $crate::__private::push_mul_eq_spanned(&mut $tokens, $span);
1365 (!= $tokens:ident $span:ident) => {
1366 $crate::__private::push_ne_spanned(&mut $tokens, $span);
1369 (| $tokens:ident $span:ident) => {
1370 $crate::__private::push_or_spanned(&mut $tokens, $span);
1373 (|= $tokens:ident $span:ident) => {
1374 $crate::__private::push_or_eq_spanned(&mut $tokens, $span);
1377 (|| $tokens:ident $span:ident) => {
1378 $crate::__private::push_or_or_spanned(&mut $tokens, $span);
1381 (? $tokens:ident $span:ident) => {
1382 $crate::__private::push_question_spanned(&mut $tokens, $span);
1385 (-> $tokens:ident $span:ident) => {
1386 $crate::__private::push_rarrow_spanned(&mut $tokens, $span);
1389 (<- $tokens:ident $span:ident) => {
1390 $crate::__private::push_larrow_spanned(&mut $tokens, $span);
1393 (% $tokens:ident $span:ident) => {
1394 $crate::__private::push_rem_spanned(&mut $tokens, $span);
1397 (%= $tokens:ident $span:ident) => {
1398 $crate::__private::push_rem_eq_spanned(&mut $tokens, $span);
1401 (=> $tokens:ident $span:ident) => {
1402 $crate::__private::push_fat_arrow_spanned(&mut $tokens, $span);
1405 (<< $tokens:ident $span:ident) => {
1406 $crate::__private::push_shl_spanned(&mut $tokens, $span);
1409 (<<= $tokens:ident $span:ident) => {
1410 $crate::__private::push_shl_eq_spanned(&mut $tokens, $span);
1413 (>> $tokens:ident $span:ident) => {
1414 $crate::__private::push_shr_spanned(&mut $tokens, $span);
1417 (>>= $tokens:ident $span:ident) => {
1418 $crate::__private::push_shr_eq_spanned(&mut $tokens, $span);
1421 (* $tokens:ident $span:ident) => {
1422 $crate::__private::push_star_spanned(&mut $tokens, $span);
1425 (- $tokens:ident $span:ident) => {
1426 $crate::__private::push_sub_spanned(&mut $tokens, $span);
1429 (-= $tokens:ident $span:ident) => {
1430 $crate::__private::push_sub_eq_spanned(&mut $tokens, $span);
1433 ($lifetime:lifetime $tokens:ident $span:ident) => {
1434 $crate::__private::push_lifetime_spanned(&mut $tokens, $span, stringify!($lifetime));
1437 (_ $tokens:ident $span:ident) => {
1438 $crate::__private::push_underscore_spanned(&mut $tokens, $span);
1441 ($other:tt $tokens:ident $span:ident) => {
1442 $crate::__private::parse_spanned(&mut $tokens, $span, stringify!($other));