Lines Matching refs:tok
163 token->level = p->tok->level;
165 const char *line_start = token_type == STRING ? p->tok->multi_line_start : p->tok->line_start;
166 int lineno = token_type == STRING ? p->tok->first_lineno : p->tok->lineno;
167 int end_lineno = p->tok->lineno;
170 int end_col_offset = (end != NULL && end >= p->tok->line_start) ? (int)(end - p->tok->line_start) : -1;
173 token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + col_offset : col_offset;
175 token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + end_col_offset : end_col_offset;
179 if (token_type == ERRORTOKEN && p->tok->done == E_DECODE) {
213 int type = _PyTokenizer_Get(p->tok, &start, &end);
226 if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
230 type = _PyTokenizer_Get(p->tok, &start, &end);
238 if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
239 p->tok->pendin = -p->tok->indent;
240 p->tok->indent = 0;
696 char *cur = p->tok->cur;
750 _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
757 assert(tok != NULL);
758 tok->type_comments = (flags & PyPARSE_TYPE_COMMENTS) > 0;
759 tok->async_hacks = (flags & PyPARSE_ASYNC_HACKS) > 0;
760 p->tok = tok;
825 p->tok->interactive_underflow = IUNDERFLOW_STOP;
830 int err = p->tok->done;
861 p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
884 struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2);
885 if (tok == NULL) {
892 if (!tok->fp || ps1 != NULL || ps2 != NULL ||
894 tok->fp_interactive = 1;
897 tok->filename = filename_ob;
904 Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
914 _PyTokenizer_Free(tok);
924 struct tok_state *tok;
926 tok = _PyTokenizer_FromUTF8(str, exec_input);
928 tok = _PyTokenizer_FromString(str, exec_input);
930 if (tok == NULL) {
937 tok->filename = filename_ob;
946 Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
956 _PyTokenizer_Free(tok);