1 /*
2 * C Extension module to test Python internal C APIs (Include/internal).
3 */
4
5 #ifndef Py_BUILD_CORE_BUILTIN
6 # define Py_BUILD_CORE_MODULE 1
7 #endif
8
9 /* Always enable assertions */
10 #undef NDEBUG
11
12 #define PY_SSIZE_T_CLEAN
13
14 #include "Python.h"
15 #include "pycore_atomic_funcs.h" // _Py_atomic_int_get()
16 #include "pycore_bitutils.h" // _Py_bswap32()
17 #include "pycore_fileutils.h" // _Py_normpath
18 #include "pycore_frame.h" // _PyInterpreterFrame
19 #include "pycore_gc.h" // PyGC_Head
20 #include "pycore_hashtable.h" // _Py_hashtable_new()
21 #include "pycore_initconfig.h" // _Py_GetConfigsAsDict()
22 #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal()
23 #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy()
24 #include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost()
25 #include "pycore_pystate.h" // _PyThreadState_GET()
26 #include "osdefs.h" // MAXPATHLEN
27
28
29 static PyObject *
get_configs(PyObject * self,PyObject * Py_UNUSED (args))30 get_configs(PyObject *self, PyObject *Py_UNUSED(args))
31 {
32 return _Py_GetConfigsAsDict();
33 }
34
35
36 static PyObject*
get_recursion_depth(PyObject * self,PyObject * Py_UNUSED (args))37 get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args))
38 {
39 PyThreadState *tstate = _PyThreadState_GET();
40
41 /* subtract one to ignore the frame of the get_recursion_depth() call */
42
43 return PyLong_FromLong(tstate->recursion_limit - tstate->recursion_remaining - 1);
44 }
45
46
47 static PyObject*
test_bswap(PyObject * self,PyObject * Py_UNUSED (args))48 test_bswap(PyObject *self, PyObject *Py_UNUSED(args))
49 {
50 uint16_t u16 = _Py_bswap16(UINT16_C(0x3412));
51 if (u16 != UINT16_C(0x1234)) {
52 PyErr_Format(PyExc_AssertionError,
53 "_Py_bswap16(0x3412) returns %u", u16);
54 return NULL;
55 }
56
57 uint32_t u32 = _Py_bswap32(UINT32_C(0x78563412));
58 if (u32 != UINT32_C(0x12345678)) {
59 PyErr_Format(PyExc_AssertionError,
60 "_Py_bswap32(0x78563412) returns %lu", u32);
61 return NULL;
62 }
63
64 uint64_t u64 = _Py_bswap64(UINT64_C(0xEFCDAB9078563412));
65 if (u64 != UINT64_C(0x1234567890ABCDEF)) {
66 PyErr_Format(PyExc_AssertionError,
67 "_Py_bswap64(0xEFCDAB9078563412) returns %llu", u64);
68 return NULL;
69 }
70
71 Py_RETURN_NONE;
72 }
73
74
75 static int
check_popcount(uint32_t x,int expected)76 check_popcount(uint32_t x, int expected)
77 {
78 // Use volatile to prevent the compiler to optimize out the whole test
79 volatile uint32_t u = x;
80 int bits = _Py_popcount32(u);
81 if (bits != expected) {
82 PyErr_Format(PyExc_AssertionError,
83 "_Py_popcount32(%lu) returns %i, expected %i",
84 (unsigned long)x, bits, expected);
85 return -1;
86 }
87 return 0;
88 }
89
90
91 static PyObject*
test_popcount(PyObject * self,PyObject * Py_UNUSED (args))92 test_popcount(PyObject *self, PyObject *Py_UNUSED(args))
93 {
94 #define CHECK(X, RESULT) \
95 do { \
96 if (check_popcount(X, RESULT) < 0) { \
97 return NULL; \
98 } \
99 } while (0)
100
101 CHECK(0, 0);
102 CHECK(1, 1);
103 CHECK(0x08080808, 4);
104 CHECK(0x10000001, 2);
105 CHECK(0x10101010, 4);
106 CHECK(0x10204080, 4);
107 CHECK(0xDEADCAFE, 22);
108 CHECK(0xFFFFFFFF, 32);
109 Py_RETURN_NONE;
110
111 #undef CHECK
112 }
113
114
115 static int
check_bit_length(unsigned long x,int expected)116 check_bit_length(unsigned long x, int expected)
117 {
118 // Use volatile to prevent the compiler to optimize out the whole test
119 volatile unsigned long u = x;
120 int len = _Py_bit_length(u);
121 if (len != expected) {
122 PyErr_Format(PyExc_AssertionError,
123 "_Py_bit_length(%lu) returns %i, expected %i",
124 x, len, expected);
125 return -1;
126 }
127 return 0;
128 }
129
130
131 static PyObject*
test_bit_length(PyObject * self,PyObject * Py_UNUSED (args))132 test_bit_length(PyObject *self, PyObject *Py_UNUSED(args))
133 {
134 #define CHECK(X, RESULT) \
135 do { \
136 if (check_bit_length(X, RESULT) < 0) { \
137 return NULL; \
138 } \
139 } while (0)
140
141 CHECK(0, 0);
142 CHECK(1, 1);
143 CHECK(0x1000, 13);
144 CHECK(0x1234, 13);
145 CHECK(0x54321, 19);
146 CHECK(0x7FFFFFFF, 31);
147 CHECK(0xFFFFFFFF, 32);
148 Py_RETURN_NONE;
149
150 #undef CHECK
151 }
152
153
154 #define TO_PTR(ch) ((void*)(uintptr_t)ch)
155 #define FROM_PTR(ptr) ((uintptr_t)ptr)
156 #define VALUE(key) (1 + ((int)(key) - 'a'))
157
158 static Py_uhash_t
hash_char(const void * key)159 hash_char(const void *key)
160 {
161 char ch = (char)FROM_PTR(key);
162 return ch;
163 }
164
165
166 static int
hashtable_cb(_Py_hashtable_t * table,const void * key_ptr,const void * value_ptr,void * user_data)167 hashtable_cb(_Py_hashtable_t *table,
168 const void *key_ptr, const void *value_ptr,
169 void *user_data)
170 {
171 int *count = (int *)user_data;
172 char key = (char)FROM_PTR(key_ptr);
173 int value = (int)FROM_PTR(value_ptr);
174 assert(value == VALUE(key));
175 *count += 1;
176 return 0;
177 }
178
179
180 static PyObject*
test_hashtable(PyObject * self,PyObject * Py_UNUSED (args))181 test_hashtable(PyObject *self, PyObject *Py_UNUSED(args))
182 {
183 _Py_hashtable_t *table = _Py_hashtable_new(hash_char,
184 _Py_hashtable_compare_direct);
185 if (table == NULL) {
186 return PyErr_NoMemory();
187 }
188
189 // Using an newly allocated table must not crash
190 assert(table->nentries == 0);
191 assert(table->nbuckets > 0);
192 assert(_Py_hashtable_get(table, TO_PTR('x')) == NULL);
193
194 // Test _Py_hashtable_set()
195 char key;
196 for (key='a'; key <= 'z'; key++) {
197 int value = VALUE(key);
198 if (_Py_hashtable_set(table, TO_PTR(key), TO_PTR(value)) < 0) {
199 _Py_hashtable_destroy(table);
200 return PyErr_NoMemory();
201 }
202 }
203 assert(table->nentries == 26);
204 assert(table->nbuckets > table->nentries);
205
206 // Test _Py_hashtable_get_entry()
207 for (key='a'; key <= 'z'; key++) {
208 _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(table, TO_PTR(key));
209 assert(entry != NULL);
210 assert(entry->key == TO_PTR(key));
211 assert(entry->value == TO_PTR(VALUE(key)));
212 }
213
214 // Test _Py_hashtable_get()
215 for (key='a'; key <= 'z'; key++) {
216 void *value_ptr = _Py_hashtable_get(table, TO_PTR(key));
217 assert((int)FROM_PTR(value_ptr) == VALUE(key));
218 }
219
220 // Test _Py_hashtable_steal()
221 key = 'p';
222 void *value_ptr = _Py_hashtable_steal(table, TO_PTR(key));
223 assert((int)FROM_PTR(value_ptr) == VALUE(key));
224 assert(table->nentries == 25);
225 assert(_Py_hashtable_get_entry(table, TO_PTR(key)) == NULL);
226
227 // Test _Py_hashtable_foreach()
228 int count = 0;
229 int res = _Py_hashtable_foreach(table, hashtable_cb, &count);
230 assert(res == 0);
231 assert(count == 25);
232
233 // Test _Py_hashtable_clear()
234 _Py_hashtable_clear(table);
235 assert(table->nentries == 0);
236 assert(table->nbuckets > 0);
237 assert(_Py_hashtable_get(table, TO_PTR('x')) == NULL);
238
239 _Py_hashtable_destroy(table);
240 Py_RETURN_NONE;
241 }
242
243
244 static PyObject *
test_get_config(PyObject * Py_UNUSED (self),PyObject * Py_UNUSED (args))245 test_get_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args))
246 {
247 PyConfig config;
248 PyConfig_InitIsolatedConfig(&config);
249 if (_PyInterpreterState_GetConfigCopy(&config) < 0) {
250 PyConfig_Clear(&config);
251 return NULL;
252 }
253 PyObject *dict = _PyConfig_AsDict(&config);
254 PyConfig_Clear(&config);
255 return dict;
256 }
257
258
259 static PyObject *
test_set_config(PyObject * Py_UNUSED (self),PyObject * dict)260 test_set_config(PyObject *Py_UNUSED(self), PyObject *dict)
261 {
262 PyConfig config;
263 PyConfig_InitIsolatedConfig(&config);
264 if (_PyConfig_FromDict(&config, dict) < 0) {
265 goto error;
266 }
267 if (_PyInterpreterState_SetConfig(&config) < 0) {
268 goto error;
269 }
270 PyConfig_Clear(&config);
271 Py_RETURN_NONE;
272
273 error:
274 PyConfig_Clear(&config);
275 return NULL;
276 }
277
278
279 static PyObject *
test_reset_path_config(PyObject * Py_UNUSED (self),PyObject * Py_UNUSED (arg))280 test_reset_path_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(arg))
281 {
282 _PyPathConfig_ClearGlobal();
283 Py_RETURN_NONE;
284 }
285
286
287 static PyObject*
test_atomic_funcs(PyObject * self,PyObject * Py_UNUSED (args))288 test_atomic_funcs(PyObject *self, PyObject *Py_UNUSED(args))
289 {
290 // Test _Py_atomic_size_get() and _Py_atomic_size_set()
291 Py_ssize_t var = 1;
292 _Py_atomic_size_set(&var, 2);
293 assert(_Py_atomic_size_get(&var) == 2);
294 Py_RETURN_NONE;
295 }
296
297
298 static int
check_edit_cost(const char * a,const char * b,Py_ssize_t expected)299 check_edit_cost(const char *a, const char *b, Py_ssize_t expected)
300 {
301 int ret = -1;
302 PyObject *a_obj = NULL;
303 PyObject *b_obj = NULL;
304
305 a_obj = PyUnicode_FromString(a);
306 if (a_obj == NULL) {
307 goto exit;
308 }
309 b_obj = PyUnicode_FromString(b);
310 if (b_obj == NULL) {
311 goto exit;
312 }
313 Py_ssize_t result = _Py_UTF8_Edit_Cost(a_obj, b_obj, -1);
314 if (result != expected) {
315 PyErr_Format(PyExc_AssertionError,
316 "Edit cost from '%s' to '%s' returns %zd, expected %zd",
317 a, b, result, expected);
318 goto exit;
319 }
320 // Check that smaller max_edits thresholds are exceeded.
321 Py_ssize_t max_edits = result;
322 while (max_edits > 0) {
323 max_edits /= 2;
324 Py_ssize_t result2 = _Py_UTF8_Edit_Cost(a_obj, b_obj, max_edits);
325 if (result2 <= max_edits) {
326 PyErr_Format(PyExc_AssertionError,
327 "Edit cost from '%s' to '%s' (threshold %zd) "
328 "returns %zd, expected greater than %zd",
329 a, b, max_edits, result2, max_edits);
330 goto exit;
331 }
332 }
333 // Check that bigger max_edits thresholds don't change anything
334 Py_ssize_t result3 = _Py_UTF8_Edit_Cost(a_obj, b_obj, result * 2 + 1);
335 if (result3 != result) {
336 PyErr_Format(PyExc_AssertionError,
337 "Edit cost from '%s' to '%s' (threshold %zd) "
338 "returns %zd, expected %zd",
339 a, b, result * 2, result3, result);
340 goto exit;
341 }
342 ret = 0;
343 exit:
344 Py_XDECREF(a_obj);
345 Py_XDECREF(b_obj);
346 return ret;
347 }
348
349 static PyObject *
test_edit_cost(PyObject * self,PyObject * Py_UNUSED (args))350 test_edit_cost(PyObject *self, PyObject *Py_UNUSED(args))
351 {
352 #define CHECK(a, b, n) do { \
353 if (check_edit_cost(a, b, n) < 0) { \
354 return NULL; \
355 } \
356 } while (0) \
357
358 CHECK("", "", 0);
359 CHECK("", "a", 2);
360 CHECK("a", "A", 1);
361 CHECK("Apple", "Aple", 2);
362 CHECK("Banana", "B@n@n@", 6);
363 CHECK("Cherry", "Cherry!", 2);
364 CHECK("---0---", "------", 2);
365 CHECK("abc", "y", 6);
366 CHECK("aa", "bb", 4);
367 CHECK("aaaaa", "AAAAA", 5);
368 CHECK("wxyz", "wXyZ", 2);
369 CHECK("wxyz", "wXyZ123", 8);
370 CHECK("Python", "Java", 12);
371 CHECK("Java", "C#", 8);
372 CHECK("AbstractFoobarManager", "abstract_foobar_manager", 3+2*2);
373 CHECK("CPython", "PyPy", 10);
374 CHECK("CPython", "pypy", 11);
375 CHECK("AttributeError", "AttributeErrop", 2);
376 CHECK("AttributeError", "AttributeErrorTests", 10);
377
378 #undef CHECK
379 Py_RETURN_NONE;
380 }
381
382
383 static PyObject *
normalize_path(PyObject * self,PyObject * filename)384 normalize_path(PyObject *self, PyObject *filename)
385 {
386 Py_ssize_t size = -1;
387 wchar_t *encoded = PyUnicode_AsWideCharString(filename, &size);
388 if (encoded == NULL) {
389 return NULL;
390 }
391
392 PyObject *result = PyUnicode_FromWideChar(_Py_normpath(encoded, size), -1);
393 PyMem_Free(encoded);
394
395 return result;
396 }
397
398 static PyObject *
get_getpath_codeobject(PyObject * self,PyObject * Py_UNUSED (args))399 get_getpath_codeobject(PyObject *self, PyObject *Py_UNUSED(args)) {
400 return _Py_Get_Getpath_CodeObject();
401 }
402
403
404 static PyObject *
encode_locale_ex(PyObject * self,PyObject * args)405 encode_locale_ex(PyObject *self, PyObject *args)
406 {
407 PyObject *unicode;
408 int current_locale = 0;
409 wchar_t *wstr;
410 PyObject *res = NULL;
411 const char *errors = NULL;
412
413 if (!PyArg_ParseTuple(args, "U|is", &unicode, ¤t_locale, &errors)) {
414 return NULL;
415 }
416 wstr = PyUnicode_AsWideCharString(unicode, NULL);
417 if (wstr == NULL) {
418 return NULL;
419 }
420 _Py_error_handler error_handler = _Py_GetErrorHandler(errors);
421
422 char *str = NULL;
423 size_t error_pos;
424 const char *reason = NULL;
425 int ret = _Py_EncodeLocaleEx(wstr,
426 &str, &error_pos, &reason,
427 current_locale, error_handler);
428 PyMem_Free(wstr);
429
430 switch(ret) {
431 case 0:
432 res = PyBytes_FromString(str);
433 PyMem_RawFree(str);
434 break;
435 case -1:
436 PyErr_NoMemory();
437 break;
438 case -2:
439 PyErr_Format(PyExc_RuntimeError, "encode error: pos=%zu, reason=%s",
440 error_pos, reason);
441 break;
442 case -3:
443 PyErr_SetString(PyExc_ValueError, "unsupported error handler");
444 break;
445 default:
446 PyErr_SetString(PyExc_ValueError, "unknown error code");
447 break;
448 }
449 return res;
450 }
451
452
453 static PyObject *
decode_locale_ex(PyObject * self,PyObject * args)454 decode_locale_ex(PyObject *self, PyObject *args)
455 {
456 char *str;
457 int current_locale = 0;
458 PyObject *res = NULL;
459 const char *errors = NULL;
460
461 if (!PyArg_ParseTuple(args, "y|is", &str, ¤t_locale, &errors)) {
462 return NULL;
463 }
464 _Py_error_handler error_handler = _Py_GetErrorHandler(errors);
465
466 wchar_t *wstr = NULL;
467 size_t wlen = 0;
468 const char *reason = NULL;
469 int ret = _Py_DecodeLocaleEx(str,
470 &wstr, &wlen, &reason,
471 current_locale, error_handler);
472
473 switch(ret) {
474 case 0:
475 res = PyUnicode_FromWideChar(wstr, wlen);
476 PyMem_RawFree(wstr);
477 break;
478 case -1:
479 PyErr_NoMemory();
480 break;
481 case -2:
482 PyErr_Format(PyExc_RuntimeError, "decode error: pos=%zu, reason=%s",
483 wlen, reason);
484 break;
485 case -3:
486 PyErr_SetString(PyExc_ValueError, "unsupported error handler");
487 break;
488 default:
489 PyErr_SetString(PyExc_ValueError, "unknown error code");
490 break;
491 }
492 return res;
493 }
494
495 static PyObject *record_list = NULL;
496
497 static PyObject *
set_eval_frame_default(PyObject * self,PyObject * Py_UNUSED (args))498 set_eval_frame_default(PyObject *self, PyObject *Py_UNUSED(args))
499 {
500 _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState_Get(), _PyEval_EvalFrameDefault);
501 Py_CLEAR(record_list);
502 Py_RETURN_NONE;
503 }
504
505 static PyObject *
record_eval(PyThreadState * tstate,struct _PyInterpreterFrame * f,int exc)506 record_eval(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc)
507 {
508 PyList_Append(record_list, f->f_func->func_name);
509 return _PyEval_EvalFrameDefault(tstate, f, exc);
510 }
511
512
513 static PyObject *
set_eval_frame_record(PyObject * self,PyObject * list)514 set_eval_frame_record(PyObject *self, PyObject *list)
515 {
516 if (!PyList_Check(list)) {
517 PyErr_SetString(PyExc_TypeError, "argument must be a list");
518 return NULL;
519 }
520 Py_CLEAR(record_list);
521 Py_INCREF(list);
522 record_list = list;
523 _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState_Get(), record_eval);
524 Py_RETURN_NONE;
525 }
526
527
528 static PyMethodDef TestMethods[] = {
529 {"get_configs", get_configs, METH_NOARGS},
530 {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
531 {"test_bswap", test_bswap, METH_NOARGS},
532 {"test_popcount", test_popcount, METH_NOARGS},
533 {"test_bit_length", test_bit_length, METH_NOARGS},
534 {"test_hashtable", test_hashtable, METH_NOARGS},
535 {"get_config", test_get_config, METH_NOARGS},
536 {"set_config", test_set_config, METH_O},
537 {"reset_path_config", test_reset_path_config, METH_NOARGS},
538 {"test_atomic_funcs", test_atomic_funcs, METH_NOARGS},
539 {"test_edit_cost", test_edit_cost, METH_NOARGS},
540 {"normalize_path", normalize_path, METH_O, NULL},
541 {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL},
542 {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS},
543 {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS},
544 {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL},
545 {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL},
546 {NULL, NULL} /* sentinel */
547 };
548
549
550 static struct PyModuleDef _testcapimodule = {
551 PyModuleDef_HEAD_INIT,
552 "_testinternalcapi",
553 NULL,
554 -1,
555 TestMethods,
556 NULL,
557 NULL,
558 NULL,
559 NULL
560 };
561
562
563 PyMODINIT_FUNC
PyInit__testinternalcapi(void)564 PyInit__testinternalcapi(void)
565 {
566 PyObject *module = PyModule_Create(&_testcapimodule);
567 if (module == NULL) {
568 return NULL;
569 }
570
571 if (PyModule_AddObject(module, "SIZEOF_PYGC_HEAD",
572 PyLong_FromSsize_t(sizeof(PyGC_Head))) < 0) {
573 goto error;
574 }
575
576 return module;
577
578 error:
579 Py_DECREF(module);
580 return NULL;
581 }
582