Lines Matching defs:token
149 initialize_token(Parser *p, Token *token, const char *start, const char *end, int token_type) {
150 assert(token != NULL);
152 token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, start, (int)(end - start)) : token_type;
153 token->bytes = PyBytes_FromStringAndSize(start, end - start);
154 if (token->bytes == NULL) {
158 if (_PyArena_AddPyObject(p->arena, token->bytes) < 0) {
159 Py_DECREF(token->bytes);
163 token->level = p->tok->level;
172 token->lineno = lineno;
173 token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + col_offset : col_offset;
174 token->end_lineno = end_lineno;
175 token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + end_col_offset : end_col_offset;
247 // Check if we are at the limit of the token array capacity and resize if needed
443 Token *token = NULL;
445 token = p->tokens[m];
446 if (token->type != ENDMARKER && (token->type < NEWLINE || token->type > DEDENT)) {
450 return token;