Lines Matching refs:token
312 Token token;
313 skipCommentTokens(token);
318 // Set error location to start of doc, ideally should be first token found
320 token.type_ = tokenError;
321 token.start_ = beginDoc;
322 token.end_ = endDoc;
325 token);
340 Token token;
341 skipCommentTokens(token);
349 switch (token.type_) {
351 successful = readObject(token);
355 successful = readArray(token);
359 successful = decodeNumber(token);
362 successful = decodeString(token);
368 currentValue().setOffsetStart(token.start_ - begin_);
369 currentValue().setOffsetLimit(token.end_ - begin_);
376 currentValue().setOffsetStart(token.start_ - begin_);
377 currentValue().setOffsetLimit(token.end_ - begin_);
384 currentValue().setOffsetStart(token.start_ - begin_);
385 currentValue().setOffsetLimit(token.end_ - begin_);
392 // "Un-read" the current token and mark the current value as a null
393 // token.
402 currentValue().setOffsetStart(token.start_ - begin_);
403 currentValue().setOffsetLimit(token.end_ - begin_);
404 return addError("Syntax error: value, object or array expected.", token);
416 void Reader::skipCommentTokens(Token& token) {
419 readToken(token);
420 } while (token.type_ == tokenComment);
422 readToken(token);
426 bool Reader::readToken(Token& token) {
428 token.start_ = current_;
433 token.type_ = tokenObjectBegin;
436 token.type_ = tokenObjectEnd;
439 token.type_ = tokenArrayBegin;
442 token.type_ = tokenArrayEnd;
445 token.type_ = tokenString;
449 token.type_ = tokenComment;
463 token.type_ = tokenNumber;
467 token.type_ = tokenTrue;
471 token.type_ = tokenFalse;
475 token.type_ = tokenNull;
479 token.type_ = tokenArraySeparator;
482 token.type_ = tokenMemberSeparator;
485 token.type_ = tokenEndOfStream;
492 token.type_ = tokenError;
493 token.end_ = current_;
707 Token token;
709 ok = readToken(token);
710 while (token.type_ == tokenComment && ok) {
711 ok = readToken(token);
714 (token.type_ != tokenArraySeparator && token.type_ != tokenArrayEnd);
717 "Missing ',' or ']' in array declaration", token, tokenArrayEnd);
719 if (token.type_ == tokenArrayEnd)
725 bool Reader::decodeNumber(Token& token) {
727 if (!decodeNumber(token, decoded))
730 currentValue().setOffsetStart(token.start_ - begin_);
731 currentValue().setOffsetLimit(token.end_ - begin_);
735 bool Reader::decodeNumber(Token& token, Value& decoded) {
739 Location current = token.start_;
749 while (current < token.end_) {
752 return decodeDouble(token, decoded);
759 if (value > threshold || current != token.end_ ||
761 return decodeDouble(token, decoded);
775 bool Reader::decodeDouble(Token& token) {
777 if (!decodeDouble(token, decoded))
780 currentValue().setOffsetStart(token.start_ - begin_);
781 currentValue().setOffsetLimit(token.end_ - begin_);
785 bool Reader::decodeDouble(Token& token, Value& decoded) {
789 int length = int(token.end_ - token.start_);
793 return addError("Unable to parse token length", token);
805 memcpy(buffer, token.start_, length);
809 std::string buffer(token.start_, token.end_);
814 return addError("'" + std::string(token.start_, token.end_) +
816 token);
821 bool Reader::decodeString(Token& token) {
823 if (!decodeString(token, decoded_string))
827 currentValue().setOffsetStart(token.start_ - begin_);
828 currentValue().setOffsetLimit(token.end_ - begin_);
832 bool Reader::decodeString(Token& token, std::string& decoded) {
833 decoded.reserve(token.end_ - token.start_ - 2);
834 Location current = token.start_ + 1; // skip '"'
835 Location end = token.end_ - 1; // do not include '"'
842 return addError("Empty escape sequence in string", token, current);
871 if (!decodeUnicodeCodePoint(token, current, end, unicode))
876 return addError("Bad escape sequence in string", token, current);
885 bool Reader::decodeUnicodeCodePoint(Token& token,
890 if (!decodeUnicodeEscapeSequence(token, current, end, unicode))
897 token,
901 if (decodeUnicodeEscapeSequence(token, current, end, surrogatePair)) {
906 return addError("expecting another \\u token to begin the second half of "
908 token,
914 bool Reader::decodeUnicodeEscapeSequence(Token& token,
921 token,
936 token,
943 Reader::addError(const std::string& message, Token& token, Location extra) {
945 info.token_ = token;
966 Token& token,
968 addError(message, token);
1060 Token token;
1061 token.type_ = tokenError;
1062 token.start_ = begin_ + value.getOffsetStart();
1063 token.end_ = end_ + value.getOffsetLimit();
1065 info.token_ = token;
1078 Token token;
1079 token.type_ = tokenError;
1080 token.start_ = begin_ + value.getOffsetStart();
1081 token.end_ = begin_ + value.getOffsetLimit();
1083 info.token_ = token;
1184 bool readToken(Token& token);
1194 bool readObject(Token& token);
1195 bool readArray(Token& token);
1196 bool decodeNumber(Token& token);
1197 bool decodeNumber(Token& token, Value& decoded);
1198 bool decodeString(Token& token);
1199 bool decodeString(Token& token, std::string& decoded);
1200 bool decodeDouble(Token& token);
1201 bool decodeDouble(Token& token, Value& decoded);
1202 bool decodeUnicodeCodePoint(Token& token,
1206 bool decodeUnicodeEscapeSequence(Token& token,
1210 bool addError(const std::string& message, Token& token, Location extra = 0);
1213 Token& token,
1222 void skipCommentTokens(Token& token);
1269 Token token;
1270 skipCommentTokens(token);
1272 if (token.type_ != tokenError && token.type_ != tokenEndOfStream) {
1273 addError("Extra non-whitespace after JSON value.", token);
1281 // Set error location to start of doc, ideally should be first token found
1283 token.type_ = tokenError;
1284 token.start_ = beginDoc;
1285 token.end_ = endDoc;
1288 token);
1298 Token token;
1299 skipCommentTokens(token);
1307 switch (token.type_) {
1309 successful = readObject(token);
1313 successful = readArray(token);
1317 successful = decodeNumber(token);
1320 successful = decodeString(token);
1326 currentValue().setOffsetStart(token.start_ - begin_);
1327 currentValue().setOffsetLimit(token.end_ - begin_);
1334 currentValue().setOffsetStart(token.start_ - begin_);
1335 currentValue().setOffsetLimit(token.end_ - begin_);
1342 currentValue().setOffsetStart(token.start_ - begin_);
1343 currentValue().setOffsetLimit(token.end_ - begin_);
1350 // "Un-read" the current token and mark the current value as a null
1351 // token.
1360 currentValue().setOffsetStart(token.start_ - begin_);
1361 currentValue().setOffsetLimit(token.end_ - begin_);
1362 return addError("Syntax error: value, object or array expected.", token);
1374 void OurReader::skipCommentTokens(Token& token) {
1377 readToken(token);
1378 } while (token.type_ == tokenComment);
1380 readToken(token);
1384 bool OurReader::readToken(Token& token) {
1386 token.start_ = current_;
1391 token.type_ = tokenObjectBegin;
1394 token.type_ = tokenObjectEnd;
1397 token.type_ = tokenArrayBegin;
1400 token.type_ = tokenArrayEnd;
1403 token.type_ = tokenString;
1408 token.type_ = tokenString;
1413 token.type_ = tokenComment;
1427 token.type_ = tokenNumber;
1431 token.type_ = tokenTrue;
1435 token.type_ = tokenFalse;
1439 token.type_ = tokenNull;
1443 token.type_ = tokenArraySeparator;
1446 token.type_ = tokenMemberSeparator;
1449 token.type_ = tokenEndOfStream;
1456 token.type_ = tokenError;
1457 token.end_ = current_;
1670 Token token;
1672 ok = readToken(token);
1673 while (token.type_ == tokenComment && ok) {
1674 ok = readToken(token);
1677 (token.type_ != tokenArraySeparator && token.type_ != tokenArrayEnd);
1680 "Missing ',' or ']' in array declaration", token, tokenArrayEnd);
1682 if (token.type_ == tokenArrayEnd)
1688 bool OurReader::decodeNumber(Token& token) {
1690 if (!decodeNumber(token, decoded))
1693 currentValue().setOffsetStart(token.start_ - begin_);
1694 currentValue().setOffsetLimit(token.end_ - begin_);
1698 bool OurReader::decodeNumber(Token& token, Value& decoded) {
1702 Location current = token.start_;
1712 while (current < token.end_) {
1715 return decodeDouble(token, decoded);
1722 if (value > threshold || current != token.end_ ||
1724 return decodeDouble(token, decoded);
1738 bool OurReader::decodeDouble(Token& token) {
1740 if (!decodeDouble(token, decoded))
1743 currentValue().setOffsetStart(token.start_ - begin_);
1744 currentValue().setOffsetLimit(token.end_ - begin_);
1748 bool OurReader::decodeDouble(Token& token, Value& decoded) {
1752 int length = int(token.end_ - token.start_);
1756 return addError("Unable to parse token length", token);
1768 memcpy(buffer, token.start_, length);
1772 std::string buffer(token.start_, token.end_);
1777 return addError("'" + std::string(token.start_, token.end_) +
1779 token);
1784 bool OurReader::decodeString(Token& token) {
1786 if (!decodeString(token, decoded_string))
1790 currentValue().setOffsetStart(token.start_ - begin_);
1791 currentValue().setOffsetLimit(token.end_ - begin_);
1795 bool OurReader::decodeString(Token& token, std::string& decoded) {
1796 decoded.reserve(token.end_ - token.start_ - 2);
1797 Location current = token.start_ + 1; // skip '"'
1798 Location end = token.end_ - 1; // do not include '"'
1805 return addError("Empty escape sequence in string", token, current);
1834 if (!decodeUnicodeCodePoint(token, current, end, unicode))
1839 return addError("Bad escape sequence in string", token, current);
1848 bool OurReader::decodeUnicodeCodePoint(Token& token,
1853 if (!decodeUnicodeEscapeSequence(token, current, end, unicode))
1860 token,
1864 if (decodeUnicodeEscapeSequence(token, current, end, surrogatePair)) {
1869 return addError("expecting another \\u token to begin the second half of "
1871 token,
1877 bool OurReader::decodeUnicodeEscapeSequence(Token& token,
1884 token,
1899 token,
1906 OurReader::addError(const std::string& message, Token& token, Location extra) {
1908 info.token_ = token;
1929 Token& token,
1931 addError(message, token);
2018 Token token;
2019 token.type_ = tokenError;
2020 token.start_ = begin_ + value.getOffsetStart();
2021 token.end_ = end_ + value.getOffsetLimit();
2023 info.token_ = token;
2036 Token token;
2037 token.type_ = tokenError;
2038 token.start_ = begin_ + value.getOffsetStart();
2039 token.end_ = begin_ + value.getOffsetLimit();
2041 info.token_ = token;