Lines Matching refs:token
55 possibly empty list all of the defects found while creating the token. Defects
56 may appear on any token in the tree, and a composite list of all defects in the
60 Each object in a parse tree is called a 'token', and each has a 'token_type'
67 XXX: provide complete list of token types.
151 for token in self:
152 comments.extend(token.comments)
169 for token in self:
170 if not hasattr(token, '_pp'):
171 yield (indent + ' !! invalid element in token '
172 'list: {!r}'.format(token))
174 yield from token._pp(indent+' ')
213 token_type = 'token'
246 for token in self:
247 if token.token_type == 'bare-quoted-string':
248 return token.value
666 # Because the first token, the attribute (name) eats CFWS, the second
667 # token is always the section if there is one.
673 for token in self:
674 if token.token_type == 'value':
675 return token.stripped_value
676 if token.token_type == 'quoted-string':
677 for token in token:
678 if token.token_type == 'bare-quoted-string':
679 for token in token:
680 if token.token_type == 'value':
681 return token.stripped_value
696 for token in self:
697 if token.token_type.endswith('attrtext'):
698 return token.value
712 token = self[0]
713 if token.token_type == 'cfws':
714 token = self[1]
715 if token.token_type.endswith(
717 return token.stripped_value
734 for token in self:
735 if not token.token_type.endswith('parameter'):
737 if token[0].token_type != 'attribute':
739 name = token[0].value.strip()
742 params[name].append((token.section_number, token))
815 for token in reversed(self):
816 if token.token_type == 'mime-parameters':
817 return token.params
964 # of a given type and returns a token representing the construct plus the
987 """If input token contains ASCII non-printables, register a defect."""
994 "Non-ASCII characters found in header token"))
1073 token, text = get_fws(text)
1074 ew.append(token)
1113 token, value = get_fws(value)
1114 unstructured.append(token)
1119 token, value = get_encoded_word(value)
1137 unstructured.append(token)
1160 ASCII, a NonPrintableDefect is added to the token's defects list. Since
1162 a 'ptext' token. In this case it is a WhiteSpaceTerminal, so it's value
1176 added to the token's defects list. Any quoted pairs are converted to their
1177 unquoted values, so what is returned is a 'ptext' token. In this case it
1190 the token's defects list if we find non-atext characters.
1215 token, value = get_qcontent(value)
1216 bare_quoted_string.append(token)
1219 token, value = get_fws(value)
1223 token, value = get_encoded_word(value)
1228 token, value = get_qcontent(value)
1237 token, value = get_qcontent(value)
1238 bare_quoted_string.append(token)
1258 token, value = get_fws(value)
1260 token, value = get_comment(value)
1262 token, value = get_qp_ctext(value)
1263 comment.append(token)
1277 token, value = get_fws(value)
1279 token, value = get_comment(value)
1280 cfws.append(token)
1292 token, value = get_cfws(value)
1293 quoted_string.append(token)
1294 token, value = get_bare_quoted_string(value)
1295 quoted_string.append(token)
1297 token, value = get_cfws(value)
1298 quoted_string.append(token)
1308 token, value = get_cfws(value)
1309 atom.append(token)
1315 token, value = get_encoded_word(value)
1319 token, value = get_atext(value)
1321 token, value = get_atext(value)
1322 atom.append(token)
1324 token, value = get_cfws(value)
1325 atom.append(token)
1337 token, value = get_atext(value)
1338 dot_atom_text.append(token)
1355 token, value = get_cfws(value)
1356 dot_atom.append(token)
1359 token, value = get_encoded_word(value)
1363 token, value = get_dot_atom_text(value)
1365 token, value = get_dot_atom_text(value)
1366 dot_atom.append(token)
1368 token, value = get_cfws(value)
1369 dot_atom.append(token)
1377 the leading CFWS, if any, into the parsed sub-token.
1382 The token returned is either an Atom or a QuotedString, as appropriate.
1396 token, value = get_quoted_string(value)
1401 token, value = get_atom(value)
1403 token[:0] = [leader]
1404 return token, value
1412 words is detected, an ObsoleteHeaderDefect is added to the token's defect
1420 token, value = get_word(value)
1421 phrase.append(token)
1433 token, value = get_word(value)
1436 token, value = get_cfws(value)
1441 phrase.append(token)
1456 token, value = get_dot_atom(value)
1459 token, value = get_word(value)
1463 token = TokenList()
1465 token[:0] = [leader]
1466 local_part.append(token)
1509 token, value = get_word(value)
1514 token, value = get_cfws(value)
1515 obs_local_part.append(token)
1536 added to the token's defects list. Quoted pairs are converted to their
1537 unquoted values, so what is returned is a ptext token, in this case a
1539 added to the returned token's defect list.
1564 token, value = get_cfws(value)
1565 domain_literal.append(token)
1576 token, value = get_fws(value)
1577 domain_literal.append(token)
1578 token, value = get_dtext(value)
1579 domain_literal.append(token)
1583 token, value = get_fws(value)
1584 domain_literal.append(token)
1593 token, value = get_cfws(value)
1594 domain_literal.append(token)
1610 token, value = get_domain_literal(value)
1612 token[:0] = [leader]
1613 domain.append(token)
1616 token, value = get_dot_atom(value)
1618 token, value = get_atom(value)
1622 token[:0] = [leader]
1623 domain.append(token)
1631 token, value = get_atom(value[1:])
1632 domain.append(token)
1640 token, value = get_local_part(value)
1641 addr_spec.append(token)
1647 token, value = get_domain(value[1:])
1648 addr_spec.append(token)
1655 Returns an obs-route token with the appropriate sub-tokens (that is,
1661 token, value = get_cfws(value)
1662 obs_route.append(token)
1670 token, value = get_domain(value[1:])
1671 obs_route.append(token)
1678 token, value = get_cfws(value)
1679 obs_route.append(token)
1682 token, value = get_domain(value[1:])
1683 obs_route.append(token)
1699 token, value = get_cfws(value)
1700 angle_addr.append(token)
1715 token, value = get_addr_spec(value)
1718 token, value = get_obs_route(value)
1724 angle_addr.append(token)
1725 token, value = get_addr_spec(value)
1726 angle_addr.append(token)
1734 token, value = get_cfws(value)
1735 angle_addr.append(token)
1742 token containing a phrase, but rather a display-name token with
1747 token, value = get_phrase(value)
1748 display_name.extend(token[:])
1749 display_name.defects = token.defects[:]
1769 token, value = get_display_name(value)
1772 "expected name-addr but found '{}'".format(token))
1774 token[0][:0] = [leader]
1776 name_addr.append(token)
1777 token, value = get_angle_addr(value)
1779 token[:0] = [leader]
1780 name_addr.append(token)
1791 token, value = get_name_addr(value)
1794 token, value = get_addr_spec(value)
1799 for x in token.all_defects):
1801 mailbox.append(token)
1818 token, value = get_phrase(value)
1819 invalid_mailbox.append(token)
1837 token, value = get_mailbox(value)
1838 mailbox_list.append(token)
1848 token, value = get_invalid_mailbox(value, ',;')
1850 token[:0] = [leader]
1851 mailbox_list.append(token)
1858 token, value = get_invalid_mailbox(value, ',;')
1860 token[:0] = [leader]
1861 mailbox_list.append(token)
1869 token, value = get_invalid_mailbox(value, ',;')
1870 mailbox.extend(token)
1903 token, value = get_mailbox_list(value)
1904 if len(token.all_mailboxes)==0:
1907 group_list.extend(token)
1912 token[:0] = [leader]
1913 group_list.append(token)
1921 token, value = get_display_name(value)
1925 group.append(token)
1931 token, value = get_group_list(value)
1932 group.append(token)
1942 token, value = get_cfws(value)
1943 group.append(token)
1953 element, which is either a mailbox or a group token.
1965 token, value = get_group(value)
1968 token, value = get_mailbox(value)
1972 address.append(token)
1988 token, value = get_address(value)
1989 address_list.append(token)
1999 token, value = get_invalid_mailbox(value, ',')
2001 token[:0] = [leader]
2002 address_list.append(Address([token]))
2009 token, value = get_invalid_mailbox(value, ',')
2011 token[:0] = [leader]
2012 address_list.append(Address([token]))
2020 token, value = get_invalid_mailbox(value, ',')
2021 mailbox.extend(token)
2043 token, value = get_dtext(value)
2044 no_fold_literal.append(token)
2060 token, value = get_cfws(value)
2061 msg_id.append(token)
2069 token, value = get_dot_atom_text(value)
2073 token, value = get_obs_local_part(value)
2080 msg_id.append(token)
2095 token, value = get_dot_atom_text(value)
2098 token, value = get_no_fold_literal(value)
2101 token, value = get_domain(value)
2108 msg_id.append(token)
2116 token, value = get_cfws(value)
2117 msg_id.append(token)
2126 token, value = get_msg_id(value)
2127 message_id.append(token)
2129 token = get_unstructured(value)
2130 message_id = InvalidMessageID(token)
2143 # have two level of parser routines: the get_XXX methods that get a token in
2161 token, value = get_cfws(value)
2162 mime_version.append(token)
2178 token, value = get_cfws(value)
2179 mime_version.append(token)
2190 token, value = get_cfws(value)
2191 mime_version.append(token)
2209 token, value = get_cfws(value)
2210 mime_version.append(token)
2231 token, value = get_phrase(value)
2232 invalid_parameter.append(token)
2238 We allow any non-TOKEN_ENDS in ttext, but add defects to the token's
2255 """token = [CFWS] 1*ttext [CFWS]
2265 token, value = get_cfws(value)
2266 mtoken.append(token)
2269 "expected token but found '{}'".format(value))
2270 token, value = get_ttext(value)
2271 mtoken.append(token)
2273 token, value = get_cfws(value)
2274 mtoken.append(token)
2281 token's defects list if we find non-attrtext characters. We also register
2301 attrtext is the token characters, with the subtraction of '*', "'", and '%'.
2302 We include tab in the excluded set just as we do for token.
2307 token, value = get_cfws(value)
2308 attribute.append(token)
2311 "expected token but found '{}'".format(value))
2312 token, value = get_attrtext(value)
2313 attribute.append(token)
2315 token, value = get_cfws(value)
2316 attribute.append(token)
2347 token, value = get_cfws(value)
2348 attribute.append(token)
2351 "expected token but found '{}'".format(value))
2352 token, value = get_extended_attrtext(value)
2353 attribute.append(token)
2355 token, value = get_cfws(value)
2356 attribute.append(token)
2403 token, value = get_quoted_string(value)
2405 token, value = get_extended_attribute(value)
2407 token[:0] = [leader]
2408 v.append(token)
2423 token, value = get_attribute(value)
2424 param.append(token)
2427 "name ({}) but no value".format(token)))
2431 token, value = get_section(value)
2433 param.append(token)
2448 token, value = get_cfws(value)
2449 param.append(token)
2463 token, rest = get_attrtext(inner_value)
2468 token, rest = get_extended_attrtext(inner_value)
2490 token = None
2492 token, value = get_value(value)
2495 appendto.append(token)
2504 # Assume the charset/lang is missing and the token is the value.
2507 appendto.append(token)
2511 if token is not None:
2512 for t in token:
2524 token, value = get_attrtext(value)
2525 appendto.append(token)
2526 param.lang = token.value
2537 token, value = get_fws(value)
2539 token = ValueTerminal('"', 'DQUOTE')
2542 token, value = get_qcontent(value)
2543 v.append(token)
2544 token = v
2546 token, value = get_value(value)
2547 appendto.append(token)
2569 token, value = get_parameter(value)
2570 mime_parameters.append(token)
2584 token, value = get_invalid_parameter(value)
2586 token[:0] = [leader]
2587 mime_parameters.append(token)
2589 "invalid parameter {!r}".format(token)))
2595 token, value = get_invalid_parameter(value)
2596 param.extend(token)
2598 "parameter with invalid trailing text {!r}".format(token)))
2614 token, value = get_phrase(value)
2615 tokenlist.append(token)
2625 be checked against the official IANA list + x-token, but we
2635 token, value = get_token(value)
2641 ctype.append(token)
2650 ctype.maintype = token.value.strip().lower()
2654 token, value = get_token(value)
2660 ctype.append(token)
2661 ctype.subtype = token.value.strip().lower()
2688 token, value = get_token(value)
2694 disp_header.append(token)
2695 disp_header.content_disposition = token.value.strip().lower()
2719 token, value = get_token(value)
2724 cte_header.append(token)
2725 cte_header.cte = token.value.strip().lower()
2735 token, value = get_phrase(value)
2736 cte_header.append(token)
2748 # version of each token. The tokens contain information that supports the