Lines Matching refs:token

177     Token token = PeekToken();

178     while (token.kind != tokenType) {

180         token = PeekToken();

190 bool Lexer::ReadCacheableTime(Token &token)

215     token.value = sb.ToString();

216     if (token.value.empty()) {

223 void Lexer::ReadToken(Token &token, bool skipComment)

226         InitCurToken(token);

234         token.location.row = file_->GetCharLineNumber();

235         token.location.col = file_->GetCharColumnNumber();

237             ReadId(token);

240             ReadNum(token);

243             ReadShiftLeftOp(token);

246             ReadShiftRightOp(token);

249             ReadPPlusOp(token);

252             ReadMMinusOp(token);

255             ReadComment(token);

256             if ((token.kind == TokenType::COMMENT_BLOCK || token.kind == TokenType::COMMENT_LINE) && skipComment) {

257                 InitCurToken(token);

262         ReadSymbolToken(token);

265     token.kind = TokenType::END_OF_FILE;

266     token.value = "";

269 void Lexer::InitCurToken(Token &token)

271     token.kind = TokenType::UNKNOWN;

272     token.location.filePath = file_->GetPath();

273     token.location.row = 1;

274     token.location.col = 1;

275     token.value = "";

278 void Lexer::ReadId(Token &token)

298     token.kind = (it != keyWords_.end()) ? it->second : TokenType::ID;

299     token.value = sb.ToString();

302 void Lexer::ReadNum(Token &token)

310             ReadBinaryNum(token);

313             return ReadOctNum(token);

316             return ReadHexNum(token);

319             token.kind = TokenType::NUM;

320             token.value = "0";

323         ReadDecNum(token);

325     ReadNumSuffix(token);

328 void Lexer::ReadBinaryNum(Token &token)

346     token.kind = err ? TokenType::UNKNOWN : TokenType::NUM;

347     token.value = sb.ToString();

350 void Lexer::ReadOctNum(Token &token)

369     token.kind = err ? TokenType::UNKNOWN : TokenType::NUM;

370     token.value = sb.ToString();

373 void Lexer::ReadHexNum(Token &token)

391     token.kind = err ? TokenType::UNKNOWN : TokenType::NUM;

392     token.value = sb.ToString();

395 void Lexer::ReadDecNum(Token &token)

411     token.kind = TokenType::NUM;

412     token.value = sb.ToString();

415 void Lexer::ReadNumSuffix(Token &token)

420             token.value += c;

428 void Lexer::ReadShiftLeftOp(Token &token)

434         token.kind = TokenType::LEFT_SHIFT;

435         token.value = "<<";

441     token.kind = (iter != symbols_.end()) ? iter->second : TokenType::UNKNOWN;

442     token.value = symbol;

445 void Lexer::ReadShiftRightOp(Token &token)

451         token.kind = TokenType::RIGHT_SHIFT;

452         token.value = ">>";

458     token.kind = (iter != symbols_.end()) ? iter->second : TokenType::UNKNOWN;

459     token.value = symbol;

462 void Lexer::ReadPPlusOp(Token &token)

468         token.kind = TokenType::PPLUS;

469         token.value = "++";

475     token.kind = (iter != symbols_.end()) ? iter->second : TokenType::UNKNOWN;

476     token.value = symbol;

479 void Lexer::ReadMMinusOp(Token &token)

485         token.kind = TokenType::MMINUS;

486         token.value = "--";

492     token.kind = (iter != symbols_.end()) ? iter->second : TokenType::UNKNOWN;

493     token.value = symbol;

496 void Lexer::ReadComment(Token &token)

501         ReadLineComment(token);

504         ReadBlockComment(token);

510     token.kind = (iter != symbols_.end()) ? iter->second : TokenType::UNKNOWN;

511     token.value = symbol;

514 void Lexer::ReadLineComment(Token &token)

528     token.kind = TokenType::COMMENT_LINE;

529     token.value = sb.ToString();

532 void Lexer::ReadBlockComment(Token &token)

549     token.kind = TokenType::COMMENT_BLOCK;

550     token.value = sb.ToString();

553 void Lexer::ReadSymbolToken(Token &token)

558     token.kind = (iter != symbols_.end()) ? iter->second : TokenType::UNKNOWN;

559     token.value = symbol;