Lines Matching defs:tok
28 PyObject_HEAD struct tok_state *tok;
50 self->tok = _PyTokenizer_FromUTF8(source, 1);
51 if (self->tok == NULL) {
55 self->tok->filename = filename;
64 int type = _PyTokenizer_Get(it->tok, &start, &end);
83 Py_ssize_t size = it->tok->inp - it->tok->buf;
84 PyObject *line = PyUnicode_DecodeUTF8(it->tok->buf, size, "replace");
89 const char *line_start = type == STRING ? it->tok->multi_line_start : it->tok->line_start;
90 int lineno = type == STRING ? it->tok->first_lineno : it->tok->lineno;
91 int end_lineno = it->tok->lineno;
97 if (end != NULL && end >= it->tok->line_start) {
98 end_col_offset = (int)(end - it->tok->line_start);
108 _PyTokenizer_Free(it->tok);