1/*
2 * C Extension module to test Python internal C APIs (Include/internal).
3 */
4
5#ifndef Py_BUILD_CORE_BUILTIN
6#  define Py_BUILD_CORE_MODULE 1
7#endif
8
9/* Always enable assertions */
10#undef NDEBUG
11
12#define PY_SSIZE_T_CLEAN
13
14#include "Python.h"
15#include "pycore_atomic_funcs.h" // _Py_atomic_int_get()
16#include "pycore_bitutils.h"     // _Py_bswap32()
17#include "pycore_fileutils.h"    // _Py_normpath
18#include "pycore_frame.h"        // _PyInterpreterFrame
19#include "pycore_gc.h"           // PyGC_Head
20#include "pycore_hashtable.h"    // _Py_hashtable_new()
21#include "pycore_initconfig.h"   // _Py_GetConfigsAsDict()
22#include "pycore_pathconfig.h"   // _PyPathConfig_ClearGlobal()
23#include "pycore_interp.h"       // _PyInterpreterState_GetConfigCopy()
24#include "pycore_pyerrors.h"     // _Py_UTF8_Edit_Cost()
25#include "pycore_pystate.h"      // _PyThreadState_GET()
26#include "osdefs.h"              // MAXPATHLEN
27
28
29static PyObject *
30get_configs(PyObject *self, PyObject *Py_UNUSED(args))
31{
32    return _Py_GetConfigsAsDict();
33}
34
35
36static PyObject*
37get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args))
38{
39    PyThreadState *tstate = _PyThreadState_GET();
40
41    /* subtract one to ignore the frame of the get_recursion_depth() call */
42
43    return PyLong_FromLong(tstate->recursion_limit - tstate->recursion_remaining - 1);
44}
45
46
47static PyObject*
48test_bswap(PyObject *self, PyObject *Py_UNUSED(args))
49{
50    uint16_t u16 = _Py_bswap16(UINT16_C(0x3412));
51    if (u16 != UINT16_C(0x1234)) {
52        PyErr_Format(PyExc_AssertionError,
53                     "_Py_bswap16(0x3412) returns %u", u16);
54        return NULL;
55    }
56
57    uint32_t u32 = _Py_bswap32(UINT32_C(0x78563412));
58    if (u32 != UINT32_C(0x12345678)) {
59        PyErr_Format(PyExc_AssertionError,
60                     "_Py_bswap32(0x78563412) returns %lu", u32);
61        return NULL;
62    }
63
64    uint64_t u64 = _Py_bswap64(UINT64_C(0xEFCDAB9078563412));
65    if (u64 != UINT64_C(0x1234567890ABCDEF)) {
66        PyErr_Format(PyExc_AssertionError,
67                     "_Py_bswap64(0xEFCDAB9078563412) returns %llu", u64);
68        return NULL;
69    }
70
71    Py_RETURN_NONE;
72}
73
74
75static int
76check_popcount(uint32_t x, int expected)
77{
78    // Use volatile to prevent the compiler to optimize out the whole test
79    volatile uint32_t u = x;
80    int bits = _Py_popcount32(u);
81    if (bits != expected) {
82        PyErr_Format(PyExc_AssertionError,
83                     "_Py_popcount32(%lu) returns %i, expected %i",
84                     (unsigned long)x, bits, expected);
85        return -1;
86    }
87    return 0;
88}
89
90
91static PyObject*
92test_popcount(PyObject *self, PyObject *Py_UNUSED(args))
93{
94#define CHECK(X, RESULT) \
95    do { \
96        if (check_popcount(X, RESULT) < 0) { \
97            return NULL; \
98        } \
99    } while (0)
100
101    CHECK(0, 0);
102    CHECK(1, 1);
103    CHECK(0x08080808, 4);
104    CHECK(0x10000001, 2);
105    CHECK(0x10101010, 4);
106    CHECK(0x10204080, 4);
107    CHECK(0xDEADCAFE, 22);
108    CHECK(0xFFFFFFFF, 32);
109    Py_RETURN_NONE;
110
111#undef CHECK
112}
113
114
115static int
116check_bit_length(unsigned long x, int expected)
117{
118    // Use volatile to prevent the compiler to optimize out the whole test
119    volatile unsigned long u = x;
120    int len = _Py_bit_length(u);
121    if (len != expected) {
122        PyErr_Format(PyExc_AssertionError,
123                     "_Py_bit_length(%lu) returns %i, expected %i",
124                     x, len, expected);
125        return -1;
126    }
127    return 0;
128}
129
130
131static PyObject*
132test_bit_length(PyObject *self, PyObject *Py_UNUSED(args))
133{
134#define CHECK(X, RESULT) \
135    do { \
136        if (check_bit_length(X, RESULT) < 0) { \
137            return NULL; \
138        } \
139    } while (0)
140
141    CHECK(0, 0);
142    CHECK(1, 1);
143    CHECK(0x1000, 13);
144    CHECK(0x1234, 13);
145    CHECK(0x54321, 19);
146    CHECK(0x7FFFFFFF, 31);
147    CHECK(0xFFFFFFFF, 32);
148    Py_RETURN_NONE;
149
150#undef CHECK
151}
152
153
154#define TO_PTR(ch) ((void*)(uintptr_t)ch)
155#define FROM_PTR(ptr) ((uintptr_t)ptr)
156#define VALUE(key) (1 + ((int)(key) - 'a'))
157
158static Py_uhash_t
159hash_char(const void *key)
160{
161    char ch = (char)FROM_PTR(key);
162    return ch;
163}
164
165
166static int
167hashtable_cb(_Py_hashtable_t *table,
168             const void *key_ptr, const void *value_ptr,
169             void *user_data)
170{
171    int *count = (int *)user_data;
172    char key = (char)FROM_PTR(key_ptr);
173    int value = (int)FROM_PTR(value_ptr);
174    assert(value == VALUE(key));
175    *count += 1;
176    return 0;
177}
178
179
180static PyObject*
181test_hashtable(PyObject *self, PyObject *Py_UNUSED(args))
182{
183    _Py_hashtable_t *table = _Py_hashtable_new(hash_char,
184                                               _Py_hashtable_compare_direct);
185    if (table == NULL) {
186        return PyErr_NoMemory();
187    }
188
189    // Using an newly allocated table must not crash
190    assert(table->nentries == 0);
191    assert(table->nbuckets > 0);
192    assert(_Py_hashtable_get(table, TO_PTR('x')) == NULL);
193
194    // Test _Py_hashtable_set()
195    char key;
196    for (key='a'; key <= 'z'; key++) {
197        int value = VALUE(key);
198        if (_Py_hashtable_set(table, TO_PTR(key), TO_PTR(value)) < 0) {
199            _Py_hashtable_destroy(table);
200            return PyErr_NoMemory();
201        }
202    }
203    assert(table->nentries == 26);
204    assert(table->nbuckets > table->nentries);
205
206    // Test _Py_hashtable_get_entry()
207    for (key='a'; key <= 'z'; key++) {
208        _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(table, TO_PTR(key));
209        assert(entry != NULL);
210        assert(entry->key == TO_PTR(key));
211        assert(entry->value == TO_PTR(VALUE(key)));
212    }
213
214    // Test _Py_hashtable_get()
215    for (key='a'; key <= 'z'; key++) {
216        void *value_ptr = _Py_hashtable_get(table, TO_PTR(key));
217        assert((int)FROM_PTR(value_ptr) == VALUE(key));
218    }
219
220    // Test _Py_hashtable_steal()
221    key = 'p';
222    void *value_ptr = _Py_hashtable_steal(table, TO_PTR(key));
223    assert((int)FROM_PTR(value_ptr) == VALUE(key));
224    assert(table->nentries == 25);
225    assert(_Py_hashtable_get_entry(table, TO_PTR(key)) == NULL);
226
227    // Test _Py_hashtable_foreach()
228    int count = 0;
229    int res = _Py_hashtable_foreach(table, hashtable_cb, &count);
230    assert(res == 0);
231    assert(count == 25);
232
233    // Test _Py_hashtable_clear()
234    _Py_hashtable_clear(table);
235    assert(table->nentries == 0);
236    assert(table->nbuckets > 0);
237    assert(_Py_hashtable_get(table, TO_PTR('x')) == NULL);
238
239    _Py_hashtable_destroy(table);
240    Py_RETURN_NONE;
241}
242
243
244static PyObject *
245test_get_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args))
246{
247    PyConfig config;
248    PyConfig_InitIsolatedConfig(&config);
249    if (_PyInterpreterState_GetConfigCopy(&config) < 0) {
250        PyConfig_Clear(&config);
251        return NULL;
252    }
253    PyObject *dict = _PyConfig_AsDict(&config);
254    PyConfig_Clear(&config);
255    return dict;
256}
257
258
259static PyObject *
260test_set_config(PyObject *Py_UNUSED(self), PyObject *dict)
261{
262    PyConfig config;
263    PyConfig_InitIsolatedConfig(&config);
264    if (_PyConfig_FromDict(&config, dict) < 0) {
265        goto error;
266    }
267    if (_PyInterpreterState_SetConfig(&config) < 0) {
268        goto error;
269    }
270    PyConfig_Clear(&config);
271    Py_RETURN_NONE;
272
273error:
274    PyConfig_Clear(&config);
275    return NULL;
276}
277
278
279static PyObject *
280test_reset_path_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(arg))
281{
282    _PyPathConfig_ClearGlobal();
283    Py_RETURN_NONE;
284}
285
286
287static PyObject*
288test_atomic_funcs(PyObject *self, PyObject *Py_UNUSED(args))
289{
290    // Test _Py_atomic_size_get() and _Py_atomic_size_set()
291    Py_ssize_t var = 1;
292    _Py_atomic_size_set(&var, 2);
293    assert(_Py_atomic_size_get(&var) == 2);
294    Py_RETURN_NONE;
295}
296
297
298static int
299check_edit_cost(const char *a, const char *b, Py_ssize_t expected)
300{
301    int ret = -1;
302    PyObject *a_obj = NULL;
303    PyObject *b_obj = NULL;
304
305    a_obj = PyUnicode_FromString(a);
306    if (a_obj == NULL) {
307        goto exit;
308    }
309    b_obj = PyUnicode_FromString(b);
310    if (b_obj == NULL) {
311        goto exit;
312    }
313    Py_ssize_t result = _Py_UTF8_Edit_Cost(a_obj, b_obj, -1);
314    if (result != expected) {
315        PyErr_Format(PyExc_AssertionError,
316                     "Edit cost from '%s' to '%s' returns %zd, expected %zd",
317                     a, b, result, expected);
318        goto exit;
319    }
320    // Check that smaller max_edits thresholds are exceeded.
321    Py_ssize_t max_edits = result;
322    while (max_edits > 0) {
323        max_edits /= 2;
324        Py_ssize_t result2 = _Py_UTF8_Edit_Cost(a_obj, b_obj, max_edits);
325        if (result2 <= max_edits) {
326            PyErr_Format(PyExc_AssertionError,
327                         "Edit cost from '%s' to '%s' (threshold %zd) "
328                         "returns %zd, expected greater than %zd",
329                         a, b, max_edits, result2, max_edits);
330            goto exit;
331        }
332    }
333    // Check that bigger max_edits thresholds don't change anything
334    Py_ssize_t result3 = _Py_UTF8_Edit_Cost(a_obj, b_obj, result * 2 + 1);
335    if (result3 != result) {
336        PyErr_Format(PyExc_AssertionError,
337                     "Edit cost from '%s' to '%s' (threshold %zd) "
338                     "returns %zd, expected %zd",
339                     a, b, result * 2, result3, result);
340        goto exit;
341    }
342    ret = 0;
343exit:
344    Py_XDECREF(a_obj);
345    Py_XDECREF(b_obj);
346    return ret;
347}
348
349static PyObject *
350test_edit_cost(PyObject *self, PyObject *Py_UNUSED(args))
351{
352    #define CHECK(a, b, n) do {              \
353        if (check_edit_cost(a, b, n) < 0) {  \
354            return NULL;                     \
355        }                                    \
356    } while (0)                              \
357
358    CHECK("", "", 0);
359    CHECK("", "a", 2);
360    CHECK("a", "A", 1);
361    CHECK("Apple", "Aple", 2);
362    CHECK("Banana", "B@n@n@", 6);
363    CHECK("Cherry", "Cherry!", 2);
364    CHECK("---0---", "------", 2);
365    CHECK("abc", "y", 6);
366    CHECK("aa", "bb", 4);
367    CHECK("aaaaa", "AAAAA", 5);
368    CHECK("wxyz", "wXyZ", 2);
369    CHECK("wxyz", "wXyZ123", 8);
370    CHECK("Python", "Java", 12);
371    CHECK("Java", "C#", 8);
372    CHECK("AbstractFoobarManager", "abstract_foobar_manager", 3+2*2);
373    CHECK("CPython", "PyPy", 10);
374    CHECK("CPython", "pypy", 11);
375    CHECK("AttributeError", "AttributeErrop", 2);
376    CHECK("AttributeError", "AttributeErrorTests", 10);
377
378    #undef CHECK
379    Py_RETURN_NONE;
380}
381
382
383static PyObject *
384normalize_path(PyObject *self, PyObject *filename)
385{
386    Py_ssize_t size = -1;
387    wchar_t *encoded = PyUnicode_AsWideCharString(filename, &size);
388    if (encoded == NULL) {
389        return NULL;
390    }
391
392    PyObject *result = PyUnicode_FromWideChar(_Py_normpath(encoded, size), -1);
393    PyMem_Free(encoded);
394
395    return result;
396}
397
398static PyObject *
399get_getpath_codeobject(PyObject *self, PyObject *Py_UNUSED(args)) {
400    return _Py_Get_Getpath_CodeObject();
401}
402
403
404static PyObject *
405encode_locale_ex(PyObject *self, PyObject *args)
406{
407    PyObject *unicode;
408    int current_locale = 0;
409    wchar_t *wstr;
410    PyObject *res = NULL;
411    const char *errors = NULL;
412
413    if (!PyArg_ParseTuple(args, "U|is", &unicode, &current_locale, &errors)) {
414        return NULL;
415    }
416    wstr = PyUnicode_AsWideCharString(unicode, NULL);
417    if (wstr == NULL) {
418        return NULL;
419    }
420    _Py_error_handler error_handler = _Py_GetErrorHandler(errors);
421
422    char *str = NULL;
423    size_t error_pos;
424    const char *reason = NULL;
425    int ret = _Py_EncodeLocaleEx(wstr,
426                                 &str, &error_pos, &reason,
427                                 current_locale, error_handler);
428    PyMem_Free(wstr);
429
430    switch(ret) {
431    case 0:
432        res = PyBytes_FromString(str);
433        PyMem_RawFree(str);
434        break;
435    case -1:
436        PyErr_NoMemory();
437        break;
438    case -2:
439        PyErr_Format(PyExc_RuntimeError, "encode error: pos=%zu, reason=%s",
440                     error_pos, reason);
441        break;
442    case -3:
443        PyErr_SetString(PyExc_ValueError, "unsupported error handler");
444        break;
445    default:
446        PyErr_SetString(PyExc_ValueError, "unknown error code");
447        break;
448    }
449    return res;
450}
451
452
453static PyObject *
454decode_locale_ex(PyObject *self, PyObject *args)
455{
456    char *str;
457    int current_locale = 0;
458    PyObject *res = NULL;
459    const char *errors = NULL;
460
461    if (!PyArg_ParseTuple(args, "y|is", &str, &current_locale, &errors)) {
462        return NULL;
463    }
464    _Py_error_handler error_handler = _Py_GetErrorHandler(errors);
465
466    wchar_t *wstr = NULL;
467    size_t wlen = 0;
468    const char *reason = NULL;
469    int ret = _Py_DecodeLocaleEx(str,
470                                 &wstr, &wlen, &reason,
471                                 current_locale, error_handler);
472
473    switch(ret) {
474    case 0:
475        res = PyUnicode_FromWideChar(wstr, wlen);
476        PyMem_RawFree(wstr);
477        break;
478    case -1:
479        PyErr_NoMemory();
480        break;
481    case -2:
482        PyErr_Format(PyExc_RuntimeError, "decode error: pos=%zu, reason=%s",
483                     wlen, reason);
484        break;
485    case -3:
486        PyErr_SetString(PyExc_ValueError, "unsupported error handler");
487        break;
488    default:
489        PyErr_SetString(PyExc_ValueError, "unknown error code");
490        break;
491    }
492    return res;
493}
494
495static PyObject *record_list = NULL;
496
497static PyObject *
498set_eval_frame_default(PyObject *self, PyObject *Py_UNUSED(args))
499{
500    _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState_Get(), _PyEval_EvalFrameDefault);
501    Py_CLEAR(record_list);
502    Py_RETURN_NONE;
503}
504
505static PyObject *
506record_eval(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc)
507{
508    PyList_Append(record_list, f->f_func->func_name);
509    return _PyEval_EvalFrameDefault(tstate, f, exc);
510}
511
512
513static PyObject *
514set_eval_frame_record(PyObject *self, PyObject *list)
515{
516    if (!PyList_Check(list)) {
517        PyErr_SetString(PyExc_TypeError, "argument must be a list");
518        return NULL;
519    }
520    Py_CLEAR(record_list);
521    Py_INCREF(list);
522    record_list = list;
523    _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState_Get(), record_eval);
524    Py_RETURN_NONE;
525}
526
527
528static PyMethodDef TestMethods[] = {
529    {"get_configs", get_configs, METH_NOARGS},
530    {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
531    {"test_bswap", test_bswap, METH_NOARGS},
532    {"test_popcount", test_popcount, METH_NOARGS},
533    {"test_bit_length", test_bit_length, METH_NOARGS},
534    {"test_hashtable", test_hashtable, METH_NOARGS},
535    {"get_config", test_get_config, METH_NOARGS},
536    {"set_config", test_set_config, METH_O},
537    {"reset_path_config", test_reset_path_config, METH_NOARGS},
538    {"test_atomic_funcs", test_atomic_funcs, METH_NOARGS},
539    {"test_edit_cost", test_edit_cost, METH_NOARGS},
540    {"normalize_path", normalize_path, METH_O, NULL},
541    {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL},
542    {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS},
543    {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS},
544    {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL},
545    {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL},
546    {NULL, NULL} /* sentinel */
547};
548
549
550static struct PyModuleDef _testcapimodule = {
551    PyModuleDef_HEAD_INIT,
552    "_testinternalcapi",
553    NULL,
554    -1,
555    TestMethods,
556    NULL,
557    NULL,
558    NULL,
559    NULL
560};
561
562
563PyMODINIT_FUNC
564PyInit__testinternalcapi(void)
565{
566    PyObject *module = PyModule_Create(&_testcapimodule);
567    if (module == NULL) {
568        return NULL;
569    }
570
571    if (PyModule_AddObject(module, "SIZEOF_PYGC_HEAD",
572                           PyLong_FromSsize_t(sizeof(PyGC_Head))) < 0) {
573        goto error;
574    }
575
576    return module;
577
578error:
579    Py_DECREF(module);
580    return NULL;
581}
582