Lines Matching defs:token
3139 // CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII
3140 // string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes
3141 // Unicode (UTF-8) strings from byte strings. MUST means that the the token
3142 // producer (e.g. parser) must verify this. SHOULD means that the token
3146 // code points (with multi-byte UTF-8 encodings) do not straddle token
3147 // boundaries. Checking UTF-8 validity can inspect each token separately.
3150 // strings, in a single- or multi-token chain, to have none of these bits set.
3209 // token length (after adjusting for FORMAT_IGNORE_ETC) discriminates
3219 // wuffs_base__token__value returns the token's high 46 bits, sign-extended. A
3220 // negative value means an extended token, non-negative means a simple token.
3226 // wuffs_base__token__value_extension returns a negative value if the token was
3227 // not an extended token.
3233 // wuffs_base__token__value_major returns a negative value if the token was not
3234 // a simple token.
3240 // wuffs_base__token__value_base_category returns a negative value if the token
3241 // was not a simple token.
24586 const char wuffs_cbor__error__internal_error_inconsistent_token_length[] = "#cbor: internal error: inconsistent token length";
44342 // Valid token's VBCs range in 0 ..= 15. Values over that are for tokens
44347 // 1. Get the next token.
44348 // 2. Process that token.
44350 // 1. Get the next token.
44401 wuffs_base__token token = tok_buf.data.ptr[tok_buf.meta.ri++];
44402 uint64_t token_len = token.length();
44406 "wuffs_aux::DecodeCbor: internal error: bad token indexes";
44412 // 2. Process that token.
44414 uint64_t vbd = token.value_base_detail();
44417 int64_t ext = token.value_extension();
44418 if ((ext >= 0) && !token.continued()) {
44442 "wuffs_aux::DecodeCbor: internal error: bad extended token";
44446 switch (token.value_base_category()) {
44485 if (token.continued()) {
44505 if (token.continued()) {
44553 if (token.continued()) {
44556 static_cast<uint64_t>(token.value_base_detail__sign_extended());
44560 callbacks.AppendI64(token.value_base_detail__sign_extended());
44565 if (token.continued()) {
44576 if (token.value_major() == WUFFS_CBOR__TOKEN_VALUE_MAJOR) {
44577 uint64_t value_minor = token.value_minor();
44590 if (token.continued()) {
44607 "wuffs_aux::DecodeCbor: internal error: unexpected token";
45280 wuffs_base__token token = tok_buf.data.ptr[tok_buf.meta.ri++]; \
45281 uint64_t token_len = token.length(); \
45285 "wuffs_aux::DecodeJson: internal error: bad token indexes"; \
45367 int64_t vbc = token.value_base_category();
45368 uint64_t vbd = token.value_base_detail();
45390 int64_t vbc = token.value_base_category();
45391 uint64_t vbd = token.value_base_detail();
45432 if (token.continued()) {
45445 int64_t vbc = token.value_base_category();
45446 uint64_t vbd = token.value_base_detail();
45447 if (token.continued() || (vbc == WUFFS_BASE__TOKEN__VBC__FILLER)) {
45481 int64_t vbc = token.value_base_category();
45482 uint64_t vbd = token.value_base_detail();
45483 if (token.continued() || (vbc == WUFFS_BASE__TOKEN__VBC__FILLER)) {
45510 int64_t vbc = token.value_base_category();
45511 uint64_t vbd = token.value_base_detail();
45517 // that we're only peeking at the next token.
45529 return "wuffs_aux::DecodeJson: internal error: unexpected token";
45612 // 1. Get the next token.
45613 // 2. Process that token.
45617 int64_t vbc = token.value_base_category();
45618 uint64_t vbd = token.value_base_detail();
45658 if (token.continued()) {
45675 if (token.continued()) {
45739 "wuffs_aux::DecodeJson: internal error: unexpected token";