Lines Matching defs:str
90 tok->str = NULL;
380 contains_null_bytes(const char* str, size_t size) {
381 return memchr(str, 0, size) != NULL;
587 return Py_CHARMASK(*tok->str++);
594 tok->str--;
595 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
611 translate_into_utf8(const char* str, const char* enc) {
613 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
676 char *str;
680 tok->input = str = translate_newlines(input, single, tok);
681 if (str == NULL)
684 tok->str = str;
687 str = tok->str; /* string after BOM if any */
688 assert(str);
690 utf8 = translate_into_utf8(str, tok->enc);
693 str = PyBytes_AsString(utf8);
695 for (s = str;; s++) {
708 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) {
719 utf8 = translate_into_utf8(str, tok->enc);
722 str = PyBytes_AS_STRING(utf8);
726 return str;
732 _PyTokenizer_FromString(const char *str, int exec_input)
739 decoded = decode_str(str, exec_input, tok);
753 _PyTokenizer_FromUTF8(const char *str, int exec_input)
759 tok->input = translated = translate_newlines(str, exec_input, tok);
766 tok->str = translated;