1# Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) 2from test import support 3from test.support import import_helper 4from test.support import os_helper 5import unittest 6 7from collections import namedtuple 8import contextlib 9import json 10import os 11import os.path 12import re 13import shutil 14import subprocess 15import sys 16import sysconfig 17import tempfile 18import textwrap 19 20if not support.has_subprocess_support: 21 raise unittest.SkipTest("test module requires subprocess") 22 23MS_WINDOWS = (os.name == 'nt') 24MACOS = (sys.platform == 'darwin') 25Py_DEBUG = hasattr(sys, 'gettotalrefcount') 26PYMEM_ALLOCATOR_NOT_SET = 0 27PYMEM_ALLOCATOR_DEBUG = 2 28PYMEM_ALLOCATOR_MALLOC = 3 29 30# _PyCoreConfig_InitCompatConfig() 31API_COMPAT = 1 32# _PyCoreConfig_InitPythonConfig() 33API_PYTHON = 2 34# _PyCoreConfig_InitIsolatedConfig() 35API_ISOLATED = 3 36 37INIT_LOOPS = 4 38MAX_HASH_SEED = 4294967295 39 40 41# If we are running from a build dir, but the stdlib has been installed, 42# some tests need to expect different results. 43STDLIB_INSTALL = os.path.join(sys.prefix, sys.platlibdir, 44 f'python{sys.version_info.major}.{sys.version_info.minor}') 45if not os.path.isfile(os.path.join(STDLIB_INSTALL, 'os.py')): 46 STDLIB_INSTALL = None 47 48def debug_build(program): 49 program = os.path.basename(program) 50 name = os.path.splitext(program)[0] 51 return name.casefold().endswith("_d".casefold()) 52 53 54def remove_python_envvars(): 55 env = dict(os.environ) 56 # Remove PYTHON* environment variables to get deterministic environment 57 for key in list(env): 58 if key.startswith('PYTHON'): 59 del env[key] 60 return env 61 62 63class EmbeddingTestsMixin: 64 def setUp(self): 65 exename = "_testembed" 66 builddir = os.path.dirname(sys.executable) 67 if MS_WINDOWS: 68 ext = ("_d" if debug_build(sys.executable) else "") + ".exe" 69 exename += ext 70 exepath = builddir 71 else: 72 exepath = os.path.join(builddir, 'Programs') 73 self.test_exe = exe = os.path.join(exepath, exename) 74 if not os.path.exists(exe): 75 self.skipTest("%r doesn't exist" % exe) 76 # This is needed otherwise we get a fatal error: 77 # "Py_Initialize: Unable to get the locale encoding 78 # LookupError: no codec search functions registered: can't find encoding" 79 self.oldcwd = os.getcwd() 80 os.chdir(builddir) 81 82 def tearDown(self): 83 os.chdir(self.oldcwd) 84 85 def run_embedded_interpreter(self, *args, env=None, 86 timeout=None, returncode=0, input=None, 87 cwd=None): 88 """Runs a test in the embedded interpreter""" 89 cmd = [self.test_exe] 90 cmd.extend(args) 91 if env is not None and MS_WINDOWS: 92 # Windows requires at least the SYSTEMROOT environment variable to 93 # start Python. 94 env = env.copy() 95 env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] 96 97 p = subprocess.Popen(cmd, 98 stdout=subprocess.PIPE, 99 stderr=subprocess.PIPE, 100 universal_newlines=True, 101 env=env, 102 cwd=cwd) 103 try: 104 (out, err) = p.communicate(input=input, timeout=timeout) 105 except: 106 p.terminate() 107 p.wait() 108 raise 109 if p.returncode != returncode and support.verbose: 110 print(f"--- {cmd} failed ---") 111 print(f"stdout:\n{out}") 112 print(f"stderr:\n{err}") 113 print(f"------") 114 115 self.assertEqual(p.returncode, returncode, 116 "bad returncode %d, stderr is %r" % 117 (p.returncode, err)) 118 return out, err 119 120 def run_repeated_init_and_subinterpreters(self): 121 out, err = self.run_embedded_interpreter("test_repeated_init_and_subinterpreters") 122 self.assertEqual(err, "") 123 124 # The output from _testembed looks like this: 125 # --- Pass 1 --- 126 # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 127 # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784 128 # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368 129 # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200 130 # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 131 # --- Pass 2 --- 132 # ... 133 134 interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, " 135 r"thread state <(0x[\dA-F]+)>: " 136 r"id\(modules\) = ([\d]+)$") 137 Interp = namedtuple("Interp", "id interp tstate modules") 138 139 numloops = 1 140 current_run = [] 141 for line in out.splitlines(): 142 if line == "--- Pass {} ---".format(numloops): 143 self.assertEqual(len(current_run), 0) 144 if support.verbose > 1: 145 print(line) 146 numloops += 1 147 continue 148 149 self.assertLess(len(current_run), 5) 150 match = re.match(interp_pat, line) 151 if match is None: 152 self.assertRegex(line, interp_pat) 153 154 # Parse the line from the loop. The first line is the main 155 # interpreter and the 3 afterward are subinterpreters. 156 interp = Interp(*match.groups()) 157 if support.verbose > 1: 158 print(interp) 159 self.assertTrue(interp.interp) 160 self.assertTrue(interp.tstate) 161 self.assertTrue(interp.modules) 162 current_run.append(interp) 163 164 # The last line in the loop should be the same as the first. 165 if len(current_run) == 5: 166 main = current_run[0] 167 self.assertEqual(interp, main) 168 yield current_run 169 current_run = [] 170 171 172class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase): 173 maxDiff = 100 * 50 174 175 def test_subinterps_main(self): 176 for run in self.run_repeated_init_and_subinterpreters(): 177 main = run[0] 178 179 self.assertEqual(main.id, '0') 180 181 def test_subinterps_different_ids(self): 182 for run in self.run_repeated_init_and_subinterpreters(): 183 main, *subs, _ = run 184 185 mainid = int(main.id) 186 for i, sub in enumerate(subs): 187 self.assertEqual(sub.id, str(mainid + i + 1)) 188 189 def test_subinterps_distinct_state(self): 190 for run in self.run_repeated_init_and_subinterpreters(): 191 main, *subs, _ = run 192 193 if '0x0' in main: 194 # XXX Fix on Windows (and other platforms): something 195 # is going on with the pointers in Programs/_testembed.c. 196 # interp.interp is 0x0 and interp.modules is the same 197 # between interpreters. 198 raise unittest.SkipTest('platform prints pointers as 0x0') 199 200 for sub in subs: 201 # A new subinterpreter may have the same 202 # PyInterpreterState pointer as a previous one if 203 # the earlier one has already been destroyed. So 204 # we compare with the main interpreter. The same 205 # applies to tstate. 206 self.assertNotEqual(sub.interp, main.interp) 207 self.assertNotEqual(sub.tstate, main.tstate) 208 self.assertNotEqual(sub.modules, main.modules) 209 210 def test_repeated_init_and_inittab(self): 211 out, err = self.run_embedded_interpreter("test_repeated_init_and_inittab") 212 self.assertEqual(err, "") 213 214 lines = [f"--- Pass {i} ---" for i in range(1, INIT_LOOPS+1)] 215 lines = "\n".join(lines) + "\n" 216 self.assertEqual(out, lines) 217 218 def test_forced_io_encoding(self): 219 # Checks forced configuration of embedded interpreter IO streams 220 env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") 221 out, err = self.run_embedded_interpreter("test_forced_io_encoding", env=env) 222 if support.verbose > 1: 223 print() 224 print(out) 225 print(err) 226 expected_stream_encoding = "utf-8" 227 expected_errors = "surrogateescape" 228 expected_output = '\n'.join([ 229 "--- Use defaults ---", 230 "Expected encoding: default", 231 "Expected errors: default", 232 "stdin: {in_encoding}:{errors}", 233 "stdout: {out_encoding}:{errors}", 234 "stderr: {out_encoding}:backslashreplace", 235 "--- Set errors only ---", 236 "Expected encoding: default", 237 "Expected errors: ignore", 238 "stdin: {in_encoding}:ignore", 239 "stdout: {out_encoding}:ignore", 240 "stderr: {out_encoding}:backslashreplace", 241 "--- Set encoding only ---", 242 "Expected encoding: iso8859-1", 243 "Expected errors: default", 244 "stdin: iso8859-1:{errors}", 245 "stdout: iso8859-1:{errors}", 246 "stderr: iso8859-1:backslashreplace", 247 "--- Set encoding and errors ---", 248 "Expected encoding: iso8859-1", 249 "Expected errors: replace", 250 "stdin: iso8859-1:replace", 251 "stdout: iso8859-1:replace", 252 "stderr: iso8859-1:backslashreplace"]) 253 expected_output = expected_output.format( 254 in_encoding=expected_stream_encoding, 255 out_encoding=expected_stream_encoding, 256 errors=expected_errors) 257 # This is useful if we ever trip over odd platform behaviour 258 self.maxDiff = None 259 self.assertEqual(out.strip(), expected_output) 260 261 def test_pre_initialization_api(self): 262 """ 263 Checks some key parts of the C-API that need to work before the runtime 264 is initialized (via Py_Initialize()). 265 """ 266 env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) 267 out, err = self.run_embedded_interpreter("test_pre_initialization_api", env=env) 268 if MS_WINDOWS: 269 expected_path = self.test_exe 270 else: 271 expected_path = os.path.join(os.getcwd(), "spam") 272 expected_output = f"sys.executable: {expected_path}\n" 273 self.assertIn(expected_output, out) 274 self.assertEqual(err, '') 275 276 def test_pre_initialization_sys_options(self): 277 """ 278 Checks that sys.warnoptions and sys._xoptions can be set before the 279 runtime is initialized (otherwise they won't be effective). 280 """ 281 env = remove_python_envvars() 282 env['PYTHONPATH'] = os.pathsep.join(sys.path) 283 out, err = self.run_embedded_interpreter( 284 "test_pre_initialization_sys_options", env=env) 285 expected_output = ( 286 "sys.warnoptions: ['once', 'module', 'default']\n" 287 "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n" 288 "warnings.filters[:3]: ['default', 'module', 'once']\n" 289 ) 290 self.assertIn(expected_output, out) 291 self.assertEqual(err, '') 292 293 def test_bpo20891(self): 294 """ 295 bpo-20891: Calling PyGILState_Ensure in a non-Python thread must not 296 crash. 297 """ 298 out, err = self.run_embedded_interpreter("test_bpo20891") 299 self.assertEqual(out, '') 300 self.assertEqual(err, '') 301 302 def test_initialize_twice(self): 303 """ 304 bpo-33932: Calling Py_Initialize() twice should do nothing (and not 305 crash!). 306 """ 307 out, err = self.run_embedded_interpreter("test_initialize_twice") 308 self.assertEqual(out, '') 309 self.assertEqual(err, '') 310 311 def test_initialize_pymain(self): 312 """ 313 bpo-34008: Calling Py_Main() after Py_Initialize() must not fail. 314 """ 315 out, err = self.run_embedded_interpreter("test_initialize_pymain") 316 self.assertEqual(out.rstrip(), "Py_Main() after Py_Initialize: sys.argv=['-c', 'arg2']") 317 self.assertEqual(err, '') 318 319 def test_run_main(self): 320 out, err = self.run_embedded_interpreter("test_run_main") 321 self.assertEqual(out.rstrip(), "Py_RunMain(): sys.argv=['-c', 'arg2']") 322 self.assertEqual(err, '') 323 324 def test_run_main_loop(self): 325 # bpo-40413: Calling Py_InitializeFromConfig()+Py_RunMain() multiple 326 # times must not crash. 327 nloop = 5 328 out, err = self.run_embedded_interpreter("test_run_main_loop") 329 self.assertEqual(out, "Py_RunMain(): sys.argv=['-c', 'arg2']\n" * nloop) 330 self.assertEqual(err, '') 331 332 def test_finalize_structseq(self): 333 # bpo-46417: Py_Finalize() clears structseq static types. Check that 334 # sys attributes using struct types still work when 335 # Py_Finalize()/Py_Initialize() is called multiple times. 336 # print() calls type->tp_repr(instance) and so checks that the types 337 # are still working properly. 338 script = support.findfile('_test_embed_structseq.py') 339 with open(script, encoding="utf-8") as fp: 340 code = fp.read() 341 out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) 342 self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS) 343 344 def test_simple_initialization_api(self): 345 # _testembed now uses Py_InitializeFromConfig by default 346 # This case specifically checks Py_Initialize(Ex) still works 347 out, err = self.run_embedded_interpreter("test_repeated_simple_init") 348 self.assertEqual(out, 'Finalized\n' * INIT_LOOPS) 349 350 def test_quickened_static_code_gets_unquickened_at_Py_FINALIZE(self): 351 # https://github.com/python/cpython/issues/92031 352 353 # Do these imports outside of the code string to avoid using 354 # importlib too much from within the code string, so that 355 # _handle_fromlist doesn't get quickened until we intend it to. 356 from dis import _all_opmap 357 resume = _all_opmap["RESUME"] 358 resume_quick = _all_opmap["RESUME_QUICK"] 359 from test.test_dis import QUICKENING_WARMUP_DELAY 360 361 code = textwrap.dedent(f"""\ 362 import importlib._bootstrap 363 func = importlib._bootstrap._handle_fromlist 364 code = func.__code__ 365 366 # Assert initially unquickened. 367 # Use sets to account for byte order. 368 if set(code._co_code_adaptive[:2]) != set([{resume}, 0]): 369 raise AssertionError() 370 371 for i in range({QUICKENING_WARMUP_DELAY}): 372 func(importlib._bootstrap, ["x"], lambda *args: None) 373 374 # Assert quickening worked 375 if set(code._co_code_adaptive[:2]) != set([{resume_quick}, 0]): 376 raise AssertionError() 377 378 print("Tests passed") 379 """) 380 run = self.run_embedded_interpreter 381 out, err = run("test_repeated_init_exec", code) 382 self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS) 383 384 def test_ucnhash_capi_reset(self): 385 # bpo-47182: unicodeobject.c:ucnhash_capi was not reset on shutdown. 386 code = "print('\\N{digit nine}')" 387 out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) 388 self.assertEqual(out, '9\n' * INIT_LOOPS) 389 390class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 391 maxDiff = 4096 392 UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape') 393 394 # Marker to read the default configuration: get_default_config() 395 GET_DEFAULT_CONFIG = object() 396 397 # Marker to ignore a configuration parameter 398 IGNORE_CONFIG = object() 399 400 PRE_CONFIG_COMPAT = { 401 '_config_init': API_COMPAT, 402 'allocator': PYMEM_ALLOCATOR_NOT_SET, 403 'parse_argv': 0, 404 'configure_locale': 1, 405 'coerce_c_locale': 0, 406 'coerce_c_locale_warn': 0, 407 'utf8_mode': 0, 408 } 409 if MS_WINDOWS: 410 PRE_CONFIG_COMPAT.update({ 411 'legacy_windows_fs_encoding': 0, 412 }) 413 PRE_CONFIG_PYTHON = dict(PRE_CONFIG_COMPAT, 414 _config_init=API_PYTHON, 415 parse_argv=1, 416 coerce_c_locale=GET_DEFAULT_CONFIG, 417 utf8_mode=GET_DEFAULT_CONFIG, 418 ) 419 PRE_CONFIG_ISOLATED = dict(PRE_CONFIG_COMPAT, 420 _config_init=API_ISOLATED, 421 configure_locale=0, 422 isolated=1, 423 use_environment=0, 424 utf8_mode=0, 425 dev_mode=0, 426 coerce_c_locale=0, 427 ) 428 429 COPY_PRE_CONFIG = [ 430 'dev_mode', 431 'isolated', 432 'use_environment', 433 ] 434 435 CONFIG_COMPAT = { 436 '_config_init': API_COMPAT, 437 'isolated': 0, 438 'use_environment': 1, 439 'dev_mode': 0, 440 441 'install_signal_handlers': 1, 442 'use_hash_seed': 0, 443 'hash_seed': 0, 444 'faulthandler': 0, 445 'tracemalloc': 0, 446 'import_time': 0, 447 'code_debug_ranges': 1, 448 'show_ref_count': 0, 449 'dump_refs': 0, 450 'malloc_stats': 0, 451 452 'filesystem_encoding': GET_DEFAULT_CONFIG, 453 'filesystem_errors': GET_DEFAULT_CONFIG, 454 455 'pycache_prefix': None, 456 'program_name': GET_DEFAULT_CONFIG, 457 'parse_argv': 0, 458 'argv': [""], 459 'orig_argv': [], 460 461 'xoptions': [], 462 'warnoptions': [], 463 464 'pythonpath_env': None, 465 'home': None, 466 'executable': GET_DEFAULT_CONFIG, 467 'base_executable': GET_DEFAULT_CONFIG, 468 469 'prefix': GET_DEFAULT_CONFIG, 470 'base_prefix': GET_DEFAULT_CONFIG, 471 'exec_prefix': GET_DEFAULT_CONFIG, 472 'base_exec_prefix': GET_DEFAULT_CONFIG, 473 'module_search_paths': GET_DEFAULT_CONFIG, 474 'module_search_paths_set': 1, 475 'platlibdir': sys.platlibdir, 476 'stdlib_dir': GET_DEFAULT_CONFIG, 477 478 'site_import': 1, 479 'bytes_warning': 0, 480 'warn_default_encoding': 0, 481 'inspect': 0, 482 'interactive': 0, 483 'optimization_level': 0, 484 'parser_debug': 0, 485 'write_bytecode': 1, 486 'verbose': 0, 487 'quiet': 0, 488 'user_site_directory': 1, 489 'configure_c_stdio': 0, 490 'buffered_stdio': 1, 491 492 'stdio_encoding': GET_DEFAULT_CONFIG, 493 'stdio_errors': GET_DEFAULT_CONFIG, 494 495 'skip_source_first_line': 0, 496 'run_command': None, 497 'run_module': None, 498 'run_filename': None, 499 500 '_install_importlib': 1, 501 'check_hash_pycs_mode': 'default', 502 'pathconfig_warnings': 1, 503 '_init_main': 1, 504 '_isolated_interpreter': 0, 505 'use_frozen_modules': not Py_DEBUG, 506 'safe_path': 0, 507 '_is_python_build': IGNORE_CONFIG, 508 } 509 if MS_WINDOWS: 510 CONFIG_COMPAT.update({ 511 'legacy_windows_stdio': 0, 512 }) 513 514 CONFIG_PYTHON = dict(CONFIG_COMPAT, 515 _config_init=API_PYTHON, 516 configure_c_stdio=1, 517 parse_argv=2, 518 ) 519 CONFIG_ISOLATED = dict(CONFIG_COMPAT, 520 _config_init=API_ISOLATED, 521 isolated=1, 522 use_environment=0, 523 user_site_directory=0, 524 safe_path=1, 525 dev_mode=0, 526 install_signal_handlers=0, 527 use_hash_seed=0, 528 faulthandler=0, 529 tracemalloc=0, 530 pathconfig_warnings=0, 531 ) 532 if MS_WINDOWS: 533 CONFIG_ISOLATED['legacy_windows_stdio'] = 0 534 535 # global config 536 DEFAULT_GLOBAL_CONFIG = { 537 'Py_HasFileSystemDefaultEncoding': 0, 538 'Py_HashRandomizationFlag': 1, 539 '_Py_HasFileSystemDefaultEncodeErrors': 0, 540 } 541 COPY_GLOBAL_PRE_CONFIG = [ 542 ('Py_UTF8Mode', 'utf8_mode'), 543 ] 544 COPY_GLOBAL_CONFIG = [ 545 # Copy core config to global config for expected values 546 # True means that the core config value is inverted (0 => 1 and 1 => 0) 547 ('Py_BytesWarningFlag', 'bytes_warning'), 548 ('Py_DebugFlag', 'parser_debug'), 549 ('Py_DontWriteBytecodeFlag', 'write_bytecode', True), 550 ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'), 551 ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'), 552 ('Py_FrozenFlag', 'pathconfig_warnings', True), 553 ('Py_IgnoreEnvironmentFlag', 'use_environment', True), 554 ('Py_InspectFlag', 'inspect'), 555 ('Py_InteractiveFlag', 'interactive'), 556 ('Py_IsolatedFlag', 'isolated'), 557 ('Py_NoSiteFlag', 'site_import', True), 558 ('Py_NoUserSiteDirectory', 'user_site_directory', True), 559 ('Py_OptimizeFlag', 'optimization_level'), 560 ('Py_QuietFlag', 'quiet'), 561 ('Py_UnbufferedStdioFlag', 'buffered_stdio', True), 562 ('Py_VerboseFlag', 'verbose'), 563 ] 564 if MS_WINDOWS: 565 COPY_GLOBAL_PRE_CONFIG.extend(( 566 ('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'), 567 )) 568 COPY_GLOBAL_CONFIG.extend(( 569 ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'), 570 )) 571 572 EXPECTED_CONFIG = None 573 574 @classmethod 575 def tearDownClass(cls): 576 # clear cache 577 cls.EXPECTED_CONFIG = None 578 579 def main_xoptions(self, xoptions_list): 580 xoptions = {} 581 for opt in xoptions_list: 582 if '=' in opt: 583 key, value = opt.split('=', 1) 584 xoptions[key] = value 585 else: 586 xoptions[opt] = True 587 return xoptions 588 589 def _get_expected_config_impl(self): 590 env = remove_python_envvars() 591 code = textwrap.dedent(''' 592 import json 593 import sys 594 import _testinternalcapi 595 596 configs = _testinternalcapi.get_configs() 597 598 data = json.dumps(configs) 599 data = data.encode('utf-8') 600 sys.stdout.buffer.write(data) 601 sys.stdout.buffer.flush() 602 ''') 603 604 # Use -S to not import the site module: get the proper configuration 605 # when test_embed is run from a venv (bpo-35313) 606 args = [sys.executable, '-S', '-c', code] 607 proc = subprocess.run(args, env=env, 608 stdout=subprocess.PIPE, 609 stderr=subprocess.PIPE) 610 if proc.returncode: 611 raise Exception(f"failed to get the default config: " 612 f"stdout={proc.stdout!r} stderr={proc.stderr!r}") 613 stdout = proc.stdout.decode('utf-8') 614 # ignore stderr 615 try: 616 return json.loads(stdout) 617 except json.JSONDecodeError: 618 self.fail(f"fail to decode stdout: {stdout!r}") 619 620 def _get_expected_config(self): 621 cls = InitConfigTests 622 if cls.EXPECTED_CONFIG is None: 623 cls.EXPECTED_CONFIG = self._get_expected_config_impl() 624 625 # get a copy 626 configs = {} 627 for config_key, config_value in cls.EXPECTED_CONFIG.items(): 628 config = {} 629 for key, value in config_value.items(): 630 if isinstance(value, list): 631 value = value.copy() 632 config[key] = value 633 configs[config_key] = config 634 return configs 635 636 def get_expected_config(self, expected_preconfig, expected, 637 env, api, modify_path_cb=None): 638 configs = self._get_expected_config() 639 640 pre_config = configs['pre_config'] 641 for key, value in expected_preconfig.items(): 642 if value is self.GET_DEFAULT_CONFIG: 643 expected_preconfig[key] = pre_config[key] 644 645 if not expected_preconfig['configure_locale'] or api == API_COMPAT: 646 # there is no easy way to get the locale encoding before 647 # setlocale(LC_CTYPE, "") is called: don't test encodings 648 for key in ('filesystem_encoding', 'filesystem_errors', 649 'stdio_encoding', 'stdio_errors'): 650 expected[key] = self.IGNORE_CONFIG 651 652 if not expected_preconfig['configure_locale']: 653 # UTF-8 Mode depends on the locale. There is no easy way 654 # to guess if UTF-8 Mode will be enabled or not if the locale 655 # is not configured. 656 expected_preconfig['utf8_mode'] = self.IGNORE_CONFIG 657 658 if expected_preconfig['utf8_mode'] == 1: 659 if expected['filesystem_encoding'] is self.GET_DEFAULT_CONFIG: 660 expected['filesystem_encoding'] = 'utf-8' 661 if expected['filesystem_errors'] is self.GET_DEFAULT_CONFIG: 662 expected['filesystem_errors'] = self.UTF8_MODE_ERRORS 663 if expected['stdio_encoding'] is self.GET_DEFAULT_CONFIG: 664 expected['stdio_encoding'] = 'utf-8' 665 if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG: 666 expected['stdio_errors'] = 'surrogateescape' 667 668 if MS_WINDOWS: 669 default_executable = self.test_exe 670 elif expected['program_name'] is not self.GET_DEFAULT_CONFIG: 671 default_executable = os.path.abspath(expected['program_name']) 672 else: 673 default_executable = os.path.join(os.getcwd(), '_testembed') 674 if expected['executable'] is self.GET_DEFAULT_CONFIG: 675 expected['executable'] = default_executable 676 if expected['base_executable'] is self.GET_DEFAULT_CONFIG: 677 expected['base_executable'] = default_executable 678 if expected['program_name'] is self.GET_DEFAULT_CONFIG: 679 expected['program_name'] = './_testembed' 680 681 config = configs['config'] 682 for key, value in expected.items(): 683 if value is self.GET_DEFAULT_CONFIG: 684 expected[key] = config[key] 685 686 if expected['module_search_paths'] is not self.IGNORE_CONFIG: 687 pythonpath_env = expected['pythonpath_env'] 688 if pythonpath_env is not None: 689 paths = pythonpath_env.split(os.path.pathsep) 690 expected['module_search_paths'] = [*paths, *expected['module_search_paths']] 691 if modify_path_cb is not None: 692 expected['module_search_paths'] = expected['module_search_paths'].copy() 693 modify_path_cb(expected['module_search_paths']) 694 695 for key in self.COPY_PRE_CONFIG: 696 if key not in expected_preconfig: 697 expected_preconfig[key] = expected[key] 698 699 def check_pre_config(self, configs, expected): 700 pre_config = dict(configs['pre_config']) 701 for key, value in list(expected.items()): 702 if value is self.IGNORE_CONFIG: 703 pre_config.pop(key, None) 704 del expected[key] 705 self.assertEqual(pre_config, expected) 706 707 def check_config(self, configs, expected): 708 config = dict(configs['config']) 709 if MS_WINDOWS: 710 value = config.get(key := 'program_name') 711 if value and isinstance(value, str): 712 value = value[:len(value.lower().removesuffix('.exe'))] 713 if debug_build(sys.executable): 714 value = value[:len(value.lower().removesuffix('_d'))] 715 config[key] = value 716 for key, value in list(expected.items()): 717 if value is self.IGNORE_CONFIG: 718 config.pop(key, None) 719 del expected[key] 720 self.assertEqual(config, expected) 721 722 def check_global_config(self, configs): 723 pre_config = configs['pre_config'] 724 config = configs['config'] 725 726 expected = dict(self.DEFAULT_GLOBAL_CONFIG) 727 for item in self.COPY_GLOBAL_CONFIG: 728 if len(item) == 3: 729 global_key, core_key, opposite = item 730 expected[global_key] = 0 if config[core_key] else 1 731 else: 732 global_key, core_key = item 733 expected[global_key] = config[core_key] 734 for item in self.COPY_GLOBAL_PRE_CONFIG: 735 if len(item) == 3: 736 global_key, core_key, opposite = item 737 expected[global_key] = 0 if pre_config[core_key] else 1 738 else: 739 global_key, core_key = item 740 expected[global_key] = pre_config[core_key] 741 742 self.assertEqual(configs['global_config'], expected) 743 744 def check_all_configs(self, testname, expected_config=None, 745 expected_preconfig=None, 746 modify_path_cb=None, 747 stderr=None, *, api, preconfig_api=None, 748 env=None, ignore_stderr=False, cwd=None): 749 new_env = remove_python_envvars() 750 if env is not None: 751 new_env.update(env) 752 env = new_env 753 754 if preconfig_api is None: 755 preconfig_api = api 756 if preconfig_api == API_ISOLATED: 757 default_preconfig = self.PRE_CONFIG_ISOLATED 758 elif preconfig_api == API_PYTHON: 759 default_preconfig = self.PRE_CONFIG_PYTHON 760 else: 761 default_preconfig = self.PRE_CONFIG_COMPAT 762 if expected_preconfig is None: 763 expected_preconfig = {} 764 expected_preconfig = dict(default_preconfig, **expected_preconfig) 765 766 if expected_config is None: 767 expected_config = {} 768 769 if api == API_PYTHON: 770 default_config = self.CONFIG_PYTHON 771 elif api == API_ISOLATED: 772 default_config = self.CONFIG_ISOLATED 773 else: 774 default_config = self.CONFIG_COMPAT 775 expected_config = dict(default_config, **expected_config) 776 777 self.get_expected_config(expected_preconfig, 778 expected_config, 779 env, 780 api, modify_path_cb) 781 782 out, err = self.run_embedded_interpreter(testname, 783 env=env, cwd=cwd) 784 if stderr is None and not expected_config['verbose']: 785 stderr = "" 786 if stderr is not None and not ignore_stderr: 787 self.assertEqual(err.rstrip(), stderr) 788 try: 789 configs = json.loads(out) 790 except json.JSONDecodeError: 791 self.fail(f"fail to decode stdout: {out!r}") 792 793 self.check_pre_config(configs, expected_preconfig) 794 self.check_config(configs, expected_config) 795 self.check_global_config(configs) 796 return configs 797 798 def test_init_default_config(self): 799 self.check_all_configs("test_init_initialize_config", api=API_COMPAT) 800 801 def test_preinit_compat_config(self): 802 self.check_all_configs("test_preinit_compat_config", api=API_COMPAT) 803 804 def test_init_compat_config(self): 805 self.check_all_configs("test_init_compat_config", api=API_COMPAT) 806 807 def test_init_global_config(self): 808 preconfig = { 809 'utf8_mode': 1, 810 } 811 config = { 812 'program_name': './globalvar', 813 'site_import': 0, 814 'bytes_warning': 1, 815 'warnoptions': ['default::BytesWarning'], 816 'inspect': 1, 817 'interactive': 1, 818 'optimization_level': 2, 819 'write_bytecode': 0, 820 'verbose': 1, 821 'quiet': 1, 822 'buffered_stdio': 0, 823 824 'user_site_directory': 0, 825 'pathconfig_warnings': 0, 826 } 827 self.check_all_configs("test_init_global_config", config, preconfig, 828 api=API_COMPAT) 829 830 def test_init_from_config(self): 831 preconfig = { 832 'allocator': PYMEM_ALLOCATOR_MALLOC, 833 'utf8_mode': 1, 834 } 835 config = { 836 'install_signal_handlers': 0, 837 'use_hash_seed': 1, 838 'hash_seed': 123, 839 'tracemalloc': 2, 840 'import_time': 1, 841 'code_debug_ranges': 0, 842 'show_ref_count': 1, 843 'malloc_stats': 1, 844 845 'stdio_encoding': 'iso8859-1', 846 'stdio_errors': 'replace', 847 848 'pycache_prefix': 'conf_pycache_prefix', 849 'program_name': './conf_program_name', 850 'argv': ['-c', 'arg2'], 851 'orig_argv': ['python3', 852 '-W', 'cmdline_warnoption', 853 '-X', 'cmdline_xoption', 854 '-c', 'pass', 855 'arg2'], 856 'parse_argv': 2, 857 'xoptions': [ 858 'config_xoption1=3', 859 'config_xoption2=', 860 'config_xoption3', 861 'cmdline_xoption', 862 ], 863 'warnoptions': [ 864 'cmdline_warnoption', 865 'default::BytesWarning', 866 'config_warnoption', 867 ], 868 'run_command': 'pass\n', 869 870 'site_import': 0, 871 'bytes_warning': 1, 872 'inspect': 1, 873 'interactive': 1, 874 'optimization_level': 2, 875 'write_bytecode': 0, 876 'verbose': 1, 877 'quiet': 1, 878 'configure_c_stdio': 1, 879 'buffered_stdio': 0, 880 'user_site_directory': 0, 881 'faulthandler': 1, 882 'platlibdir': 'my_platlibdir', 883 'module_search_paths': self.IGNORE_CONFIG, 884 'safe_path': 1, 885 886 'check_hash_pycs_mode': 'always', 887 'pathconfig_warnings': 0, 888 889 '_isolated_interpreter': 1, 890 } 891 self.check_all_configs("test_init_from_config", config, preconfig, 892 api=API_COMPAT) 893 894 def test_init_compat_env(self): 895 preconfig = { 896 'allocator': PYMEM_ALLOCATOR_MALLOC, 897 } 898 config = { 899 'use_hash_seed': 1, 900 'hash_seed': 42, 901 'tracemalloc': 2, 902 'import_time': 1, 903 'code_debug_ranges': 0, 904 'malloc_stats': 1, 905 'inspect': 1, 906 'optimization_level': 2, 907 'pythonpath_env': '/my/path', 908 'pycache_prefix': 'env_pycache_prefix', 909 'write_bytecode': 0, 910 'verbose': 1, 911 'buffered_stdio': 0, 912 'stdio_encoding': 'iso8859-1', 913 'stdio_errors': 'replace', 914 'user_site_directory': 0, 915 'faulthandler': 1, 916 'warnoptions': ['EnvVar'], 917 'platlibdir': 'env_platlibdir', 918 'module_search_paths': self.IGNORE_CONFIG, 919 'safe_path': 1, 920 } 921 self.check_all_configs("test_init_compat_env", config, preconfig, 922 api=API_COMPAT) 923 924 def test_init_python_env(self): 925 preconfig = { 926 'allocator': PYMEM_ALLOCATOR_MALLOC, 927 'utf8_mode': 1, 928 } 929 config = { 930 'use_hash_seed': 1, 931 'hash_seed': 42, 932 'tracemalloc': 2, 933 'import_time': 1, 934 'code_debug_ranges': 0, 935 'malloc_stats': 1, 936 'inspect': 1, 937 'optimization_level': 2, 938 'pythonpath_env': '/my/path', 939 'pycache_prefix': 'env_pycache_prefix', 940 'write_bytecode': 0, 941 'verbose': 1, 942 'buffered_stdio': 0, 943 'stdio_encoding': 'iso8859-1', 944 'stdio_errors': 'replace', 945 'user_site_directory': 0, 946 'faulthandler': 1, 947 'warnoptions': ['EnvVar'], 948 'platlibdir': 'env_platlibdir', 949 'module_search_paths': self.IGNORE_CONFIG, 950 'safe_path': 1, 951 } 952 self.check_all_configs("test_init_python_env", config, preconfig, 953 api=API_PYTHON) 954 955 def test_init_env_dev_mode(self): 956 preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG) 957 config = dict(dev_mode=1, 958 faulthandler=1, 959 warnoptions=['default']) 960 self.check_all_configs("test_init_env_dev_mode", config, preconfig, 961 api=API_COMPAT) 962 963 def test_init_env_dev_mode_alloc(self): 964 preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC) 965 config = dict(dev_mode=1, 966 faulthandler=1, 967 warnoptions=['default']) 968 self.check_all_configs("test_init_env_dev_mode_alloc", config, preconfig, 969 api=API_COMPAT) 970 971 def test_init_dev_mode(self): 972 preconfig = { 973 'allocator': PYMEM_ALLOCATOR_DEBUG, 974 } 975 config = { 976 'faulthandler': 1, 977 'dev_mode': 1, 978 'warnoptions': ['default'], 979 } 980 self.check_all_configs("test_init_dev_mode", config, preconfig, 981 api=API_PYTHON) 982 983 def test_preinit_parse_argv(self): 984 # Pre-initialize implicitly using argv: make sure that -X dev 985 # is used to configure the allocation in preinitialization 986 preconfig = { 987 'allocator': PYMEM_ALLOCATOR_DEBUG, 988 } 989 config = { 990 'argv': ['script.py'], 991 'orig_argv': ['python3', '-X', 'dev', '-P', 'script.py'], 992 'run_filename': os.path.abspath('script.py'), 993 'dev_mode': 1, 994 'faulthandler': 1, 995 'warnoptions': ['default'], 996 'xoptions': ['dev'], 997 'safe_path': 1, 998 } 999 self.check_all_configs("test_preinit_parse_argv", config, preconfig, 1000 api=API_PYTHON) 1001 1002 def test_preinit_dont_parse_argv(self): 1003 # -X dev must be ignored by isolated preconfiguration 1004 preconfig = { 1005 'isolated': 0, 1006 } 1007 argv = ["python3", 1008 "-E", "-I", "-P", 1009 "-X", "dev", 1010 "-X", "utf8", 1011 "script.py"] 1012 config = { 1013 'argv': argv, 1014 'orig_argv': argv, 1015 'isolated': 0, 1016 } 1017 self.check_all_configs("test_preinit_dont_parse_argv", config, preconfig, 1018 api=API_ISOLATED) 1019 1020 def test_init_isolated_flag(self): 1021 config = { 1022 'isolated': 1, 1023 'safe_path': 1, 1024 'use_environment': 0, 1025 'user_site_directory': 0, 1026 } 1027 self.check_all_configs("test_init_isolated_flag", config, api=API_PYTHON) 1028 1029 def test_preinit_isolated1(self): 1030 # _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set 1031 config = { 1032 'isolated': 1, 1033 'safe_path': 1, 1034 'use_environment': 0, 1035 'user_site_directory': 0, 1036 } 1037 self.check_all_configs("test_preinit_isolated1", config, api=API_COMPAT) 1038 1039 def test_preinit_isolated2(self): 1040 # _PyPreConfig.isolated=0, _PyCoreConfig.isolated=1 1041 config = { 1042 'isolated': 1, 1043 'safe_path': 1, 1044 'use_environment': 0, 1045 'user_site_directory': 0, 1046 } 1047 self.check_all_configs("test_preinit_isolated2", config, api=API_COMPAT) 1048 1049 def test_preinit_isolated_config(self): 1050 self.check_all_configs("test_preinit_isolated_config", api=API_ISOLATED) 1051 1052 def test_init_isolated_config(self): 1053 self.check_all_configs("test_init_isolated_config", api=API_ISOLATED) 1054 1055 def test_preinit_python_config(self): 1056 self.check_all_configs("test_preinit_python_config", api=API_PYTHON) 1057 1058 def test_init_python_config(self): 1059 self.check_all_configs("test_init_python_config", api=API_PYTHON) 1060 1061 def test_init_dont_configure_locale(self): 1062 # _PyPreConfig.configure_locale=0 1063 preconfig = { 1064 'configure_locale': 0, 1065 'coerce_c_locale': 0, 1066 } 1067 self.check_all_configs("test_init_dont_configure_locale", {}, preconfig, 1068 api=API_PYTHON) 1069 1070 @unittest.skip('as of 3.11 this test no longer works because ' 1071 'path calculations do not occur on read') 1072 def test_init_read_set(self): 1073 config = { 1074 'program_name': './init_read_set', 1075 'executable': 'my_executable', 1076 'base_executable': 'my_executable', 1077 } 1078 def modify_path(path): 1079 path.insert(1, "test_path_insert1") 1080 path.append("test_path_append") 1081 self.check_all_configs("test_init_read_set", config, 1082 api=API_PYTHON, 1083 modify_path_cb=modify_path) 1084 1085 def test_init_sys_add(self): 1086 config = { 1087 'faulthandler': 1, 1088 'xoptions': [ 1089 'config_xoption', 1090 'cmdline_xoption', 1091 'sysadd_xoption', 1092 'faulthandler', 1093 ], 1094 'warnoptions': [ 1095 'ignore:::cmdline_warnoption', 1096 'ignore:::sysadd_warnoption', 1097 'ignore:::config_warnoption', 1098 ], 1099 'orig_argv': ['python3', 1100 '-W', 'ignore:::cmdline_warnoption', 1101 '-X', 'cmdline_xoption'], 1102 } 1103 self.check_all_configs("test_init_sys_add", config, api=API_PYTHON) 1104 1105 def test_init_run_main(self): 1106 code = ('import _testinternalcapi, json; ' 1107 'print(json.dumps(_testinternalcapi.get_configs()))') 1108 config = { 1109 'argv': ['-c', 'arg2'], 1110 'orig_argv': ['python3', '-c', code, 'arg2'], 1111 'program_name': './python3', 1112 'run_command': code + '\n', 1113 'parse_argv': 2, 1114 } 1115 self.check_all_configs("test_init_run_main", config, api=API_PYTHON) 1116 1117 def test_init_main(self): 1118 code = ('import _testinternalcapi, json; ' 1119 'print(json.dumps(_testinternalcapi.get_configs()))') 1120 config = { 1121 'argv': ['-c', 'arg2'], 1122 'orig_argv': ['python3', 1123 '-c', code, 1124 'arg2'], 1125 'program_name': './python3', 1126 'run_command': code + '\n', 1127 'parse_argv': 2, 1128 '_init_main': 0, 1129 } 1130 self.check_all_configs("test_init_main", config, 1131 api=API_PYTHON, 1132 stderr="Run Python code before _Py_InitializeMain") 1133 1134 def test_init_parse_argv(self): 1135 config = { 1136 'parse_argv': 2, 1137 'argv': ['-c', 'arg1', '-v', 'arg3'], 1138 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 1139 'program_name': './argv0', 1140 'run_command': 'pass\n', 1141 'use_environment': 0, 1142 } 1143 self.check_all_configs("test_init_parse_argv", config, api=API_PYTHON) 1144 1145 def test_init_dont_parse_argv(self): 1146 pre_config = { 1147 'parse_argv': 0, 1148 } 1149 config = { 1150 'parse_argv': 0, 1151 'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 1152 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 1153 'program_name': './argv0', 1154 } 1155 self.check_all_configs("test_init_dont_parse_argv", config, pre_config, 1156 api=API_PYTHON) 1157 1158 def default_program_name(self, config): 1159 if MS_WINDOWS: 1160 program_name = 'python' 1161 executable = self.test_exe 1162 else: 1163 program_name = 'python3' 1164 if MACOS: 1165 executable = self.test_exe 1166 else: 1167 executable = shutil.which(program_name) or '' 1168 config.update({ 1169 'program_name': program_name, 1170 'base_executable': executable, 1171 'executable': executable, 1172 }) 1173 1174 def test_init_setpath(self): 1175 # Test Py_SetPath() 1176 config = self._get_expected_config() 1177 paths = config['config']['module_search_paths'] 1178 1179 config = { 1180 'module_search_paths': paths, 1181 'prefix': '', 1182 'base_prefix': '', 1183 'exec_prefix': '', 1184 'base_exec_prefix': '', 1185 # The current getpath.c doesn't determine the stdlib dir 1186 # in this case. 1187 'stdlib_dir': '', 1188 } 1189 self.default_program_name(config) 1190 env = {'TESTPATH': os.path.pathsep.join(paths)} 1191 1192 self.check_all_configs("test_init_setpath", config, 1193 api=API_COMPAT, env=env, 1194 ignore_stderr=True) 1195 1196 def test_init_setpath_config(self): 1197 # Test Py_SetPath() with PyConfig 1198 config = self._get_expected_config() 1199 paths = config['config']['module_search_paths'] 1200 1201 config = { 1202 # set by Py_SetPath() 1203 'module_search_paths': paths, 1204 'prefix': '', 1205 'base_prefix': '', 1206 'exec_prefix': '', 1207 'base_exec_prefix': '', 1208 # The current getpath.c doesn't determine the stdlib dir 1209 # in this case. 1210 'stdlib_dir': '', 1211 'use_frozen_modules': not Py_DEBUG, 1212 # overridden by PyConfig 1213 'program_name': 'conf_program_name', 1214 'base_executable': 'conf_executable', 1215 'executable': 'conf_executable', 1216 } 1217 env = {'TESTPATH': os.path.pathsep.join(paths)} 1218 self.check_all_configs("test_init_setpath_config", config, 1219 api=API_PYTHON, env=env, ignore_stderr=True) 1220 1221 def module_search_paths(self, prefix=None, exec_prefix=None): 1222 config = self._get_expected_config() 1223 if prefix is None: 1224 prefix = config['config']['prefix'] 1225 if exec_prefix is None: 1226 exec_prefix = config['config']['prefix'] 1227 if MS_WINDOWS: 1228 return config['config']['module_search_paths'] 1229 else: 1230 ver = sys.version_info 1231 return [ 1232 os.path.join(prefix, sys.platlibdir, 1233 f'python{ver.major}{ver.minor}.zip'), 1234 os.path.join(prefix, sys.platlibdir, 1235 f'python{ver.major}.{ver.minor}'), 1236 os.path.join(exec_prefix, sys.platlibdir, 1237 f'python{ver.major}.{ver.minor}', 'lib-dynload'), 1238 ] 1239 1240 @contextlib.contextmanager 1241 def tmpdir_with_python(self, subdir=None): 1242 # Temporary directory with a copy of the Python program 1243 with tempfile.TemporaryDirectory() as tmpdir: 1244 # bpo-38234: On macOS and FreeBSD, the temporary directory 1245 # can be symbolic link. For example, /tmp can be a symbolic link 1246 # to /var/tmp. Call realpath() to resolve all symbolic links. 1247 tmpdir = os.path.realpath(tmpdir) 1248 if subdir: 1249 tmpdir = os.path.normpath(os.path.join(tmpdir, subdir)) 1250 os.makedirs(tmpdir) 1251 1252 if MS_WINDOWS: 1253 # Copy pythonXY.dll (or pythonXY_d.dll) 1254 import fnmatch 1255 exedir = os.path.dirname(self.test_exe) 1256 for f in os.listdir(exedir): 1257 if fnmatch.fnmatch(f, '*.dll'): 1258 shutil.copyfile(os.path.join(exedir, f), os.path.join(tmpdir, f)) 1259 1260 # Copy Python program 1261 exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe)) 1262 shutil.copyfile(self.test_exe, exec_copy) 1263 shutil.copystat(self.test_exe, exec_copy) 1264 self.test_exe = exec_copy 1265 1266 yield tmpdir 1267 1268 def test_init_setpythonhome(self): 1269 # Test Py_SetPythonHome(home) with PYTHONPATH env var 1270 config = self._get_expected_config() 1271 paths = config['config']['module_search_paths'] 1272 paths_str = os.path.pathsep.join(paths) 1273 1274 for path in paths: 1275 if not os.path.isdir(path): 1276 continue 1277 if os.path.exists(os.path.join(path, 'os.py')): 1278 home = os.path.dirname(path) 1279 break 1280 else: 1281 self.fail(f"Unable to find home in {paths!r}") 1282 1283 prefix = exec_prefix = home 1284 if MS_WINDOWS: 1285 stdlib = os.path.join(home, "Lib") 1286 # Because we are specifying 'home', module search paths 1287 # are fairly static 1288 expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] 1289 else: 1290 version = f'{sys.version_info.major}.{sys.version_info.minor}' 1291 stdlib = os.path.join(home, sys.platlibdir, f'python{version}') 1292 expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) 1293 1294 config = { 1295 'home': home, 1296 'module_search_paths': expected_paths, 1297 'prefix': prefix, 1298 'base_prefix': prefix, 1299 'exec_prefix': exec_prefix, 1300 'base_exec_prefix': exec_prefix, 1301 'pythonpath_env': paths_str, 1302 'stdlib_dir': stdlib, 1303 } 1304 self.default_program_name(config) 1305 env = {'TESTHOME': home, 'PYTHONPATH': paths_str} 1306 self.check_all_configs("test_init_setpythonhome", config, 1307 api=API_COMPAT, env=env) 1308 1309 def test_init_is_python_build_with_home(self): 1310 # Test _Py_path_config._is_python_build configuration (gh-91985) 1311 config = self._get_expected_config() 1312 paths = config['config']['module_search_paths'] 1313 paths_str = os.path.pathsep.join(paths) 1314 1315 for path in paths: 1316 if not os.path.isdir(path): 1317 continue 1318 if os.path.exists(os.path.join(path, 'os.py')): 1319 home = os.path.dirname(path) 1320 break 1321 else: 1322 self.fail(f"Unable to find home in {paths!r}") 1323 1324 prefix = exec_prefix = home 1325 if MS_WINDOWS: 1326 stdlib = os.path.join(home, "Lib") 1327 # Because we are specifying 'home', module search paths 1328 # are fairly static 1329 expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] 1330 else: 1331 version = f'{sys.version_info.major}.{sys.version_info.minor}' 1332 stdlib = os.path.join(home, sys.platlibdir, f'python{version}') 1333 expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) 1334 1335 config = { 1336 'home': home, 1337 'module_search_paths': expected_paths, 1338 'prefix': prefix, 1339 'base_prefix': prefix, 1340 'exec_prefix': exec_prefix, 1341 'base_exec_prefix': exec_prefix, 1342 'pythonpath_env': paths_str, 1343 'stdlib_dir': stdlib, 1344 } 1345 # The code above is taken from test_init_setpythonhome() 1346 env = {'TESTHOME': home, 'PYTHONPATH': paths_str} 1347 1348 env['NEGATIVE_ISPYTHONBUILD'] = '1' 1349 config['_is_python_build'] = 0 1350 self.check_all_configs("test_init_is_python_build", config, 1351 api=API_COMPAT, env=env) 1352 1353 env['NEGATIVE_ISPYTHONBUILD'] = '0' 1354 config['_is_python_build'] = 1 1355 exedir = os.path.dirname(sys.executable) 1356 with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f: 1357 expected_paths[1 if MS_WINDOWS else 2] = os.path.normpath( 1358 os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0])) 1359 if not MS_WINDOWS: 1360 # PREFIX (default) is set when running in build directory 1361 prefix = exec_prefix = sys.prefix 1362 # stdlib calculation (/Lib) is not yet supported 1363 expected_paths[0] = self.module_search_paths(prefix=prefix)[0] 1364 config.update(prefix=prefix, base_prefix=prefix, 1365 exec_prefix=exec_prefix, base_exec_prefix=exec_prefix) 1366 self.check_all_configs("test_init_is_python_build", config, 1367 api=API_COMPAT, env=env) 1368 1369 def copy_paths_by_env(self, config): 1370 all_configs = self._get_expected_config() 1371 paths = all_configs['config']['module_search_paths'] 1372 paths_str = os.path.pathsep.join(paths) 1373 config['pythonpath_env'] = paths_str 1374 env = {'PYTHONPATH': paths_str} 1375 return env 1376 1377 @unittest.skipIf(MS_WINDOWS, 'See test_init_pybuilddir_win32') 1378 def test_init_pybuilddir(self): 1379 # Test path configuration with pybuilddir.txt configuration file 1380 1381 with self.tmpdir_with_python() as tmpdir: 1382 # pybuilddir.txt is a sub-directory relative to the current 1383 # directory (tmpdir) 1384 vpath = sysconfig.get_config_var("VPATH") or '' 1385 subdir = 'libdir' 1386 libdir = os.path.join(tmpdir, subdir) 1387 # The stdlib dir is dirname(executable) + VPATH + 'Lib' 1388 stdlibdir = os.path.normpath(os.path.join(tmpdir, vpath, 'Lib')) 1389 os.mkdir(libdir) 1390 1391 filename = os.path.join(tmpdir, 'pybuilddir.txt') 1392 with open(filename, "w", encoding="utf8") as fp: 1393 fp.write(subdir) 1394 1395 module_search_paths = self.module_search_paths() 1396 module_search_paths[-2] = stdlibdir 1397 module_search_paths[-1] = libdir 1398 1399 executable = self.test_exe 1400 config = { 1401 'base_exec_prefix': sysconfig.get_config_var("exec_prefix"), 1402 'base_prefix': sysconfig.get_config_var("prefix"), 1403 'base_executable': executable, 1404 'executable': executable, 1405 'module_search_paths': module_search_paths, 1406 'stdlib_dir': stdlibdir, 1407 } 1408 env = self.copy_paths_by_env(config) 1409 self.check_all_configs("test_init_compat_config", config, 1410 api=API_COMPAT, env=env, 1411 ignore_stderr=True, cwd=tmpdir) 1412 1413 @unittest.skipUnless(MS_WINDOWS, 'See test_init_pybuilddir') 1414 def test_init_pybuilddir_win32(self): 1415 # Test path configuration with pybuilddir.txt configuration file 1416 1417 vpath = sysconfig.get_config_var("VPATH") 1418 subdir = r'PCbuild\arch' 1419 if os.path.normpath(vpath).count(os.sep) == 2: 1420 subdir = os.path.join(subdir, 'instrumented') 1421 1422 with self.tmpdir_with_python(subdir) as tmpdir: 1423 # The prefix is dirname(executable) + VPATH 1424 prefix = os.path.normpath(os.path.join(tmpdir, vpath)) 1425 # The stdlib dir is dirname(executable) + VPATH + 'Lib' 1426 stdlibdir = os.path.normpath(os.path.join(tmpdir, vpath, 'Lib')) 1427 1428 filename = os.path.join(tmpdir, 'pybuilddir.txt') 1429 with open(filename, "w", encoding="utf8") as fp: 1430 fp.write(tmpdir) 1431 1432 module_search_paths = self.module_search_paths() 1433 module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3])) 1434 module_search_paths[-2] = tmpdir 1435 module_search_paths[-1] = stdlibdir 1436 1437 executable = self.test_exe 1438 config = { 1439 'base_exec_prefix': prefix, 1440 'base_prefix': prefix, 1441 'base_executable': executable, 1442 'executable': executable, 1443 'prefix': prefix, 1444 'exec_prefix': prefix, 1445 'module_search_paths': module_search_paths, 1446 'stdlib_dir': stdlibdir, 1447 } 1448 env = self.copy_paths_by_env(config) 1449 self.check_all_configs("test_init_compat_config", config, 1450 api=API_COMPAT, env=env, 1451 ignore_stderr=False, cwd=tmpdir) 1452 1453 def test_init_pyvenv_cfg(self): 1454 # Test path configuration with pyvenv.cfg configuration file 1455 1456 with self.tmpdir_with_python() as tmpdir, \ 1457 tempfile.TemporaryDirectory() as pyvenv_home: 1458 ver = sys.version_info 1459 1460 if not MS_WINDOWS: 1461 lib_dynload = os.path.join(pyvenv_home, 1462 sys.platlibdir, 1463 f'python{ver.major}.{ver.minor}', 1464 'lib-dynload') 1465 os.makedirs(lib_dynload) 1466 else: 1467 lib_folder = os.path.join(pyvenv_home, 'Lib') 1468 os.makedirs(lib_folder) 1469 # getpath.py uses Lib\os.py as the LANDMARK 1470 shutil.copyfile( 1471 os.path.join(support.STDLIB_DIR, 'os.py'), 1472 os.path.join(lib_folder, 'os.py'), 1473 ) 1474 1475 filename = os.path.join(tmpdir, 'pyvenv.cfg') 1476 with open(filename, "w", encoding="utf8") as fp: 1477 print("home = %s" % pyvenv_home, file=fp) 1478 print("include-system-site-packages = false", file=fp) 1479 1480 paths = self.module_search_paths() 1481 if not MS_WINDOWS: 1482 paths[-1] = lib_dynload 1483 else: 1484 paths = [ 1485 os.path.join(tmpdir, os.path.basename(paths[0])), 1486 pyvenv_home, 1487 os.path.join(pyvenv_home, "Lib"), 1488 ] 1489 1490 executable = self.test_exe 1491 base_executable = os.path.join(pyvenv_home, os.path.basename(executable)) 1492 exec_prefix = pyvenv_home 1493 config = { 1494 'base_prefix': sysconfig.get_config_var("prefix"), 1495 'base_exec_prefix': exec_prefix, 1496 'exec_prefix': exec_prefix, 1497 'base_executable': base_executable, 1498 'executable': executable, 1499 'module_search_paths': paths, 1500 } 1501 if MS_WINDOWS: 1502 config['base_prefix'] = pyvenv_home 1503 config['prefix'] = pyvenv_home 1504 config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib') 1505 config['use_frozen_modules'] = int(not Py_DEBUG) 1506 else: 1507 # cannot reliably assume stdlib_dir here because it 1508 # depends too much on our build. But it ought to be found 1509 config['stdlib_dir'] = self.IGNORE_CONFIG 1510 config['use_frozen_modules'] = int(not Py_DEBUG) 1511 1512 env = self.copy_paths_by_env(config) 1513 self.check_all_configs("test_init_compat_config", config, 1514 api=API_COMPAT, env=env, 1515 ignore_stderr=True, cwd=tmpdir) 1516 1517 @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') 1518 def test_getpath_abspath_win32(self): 1519 # Check _Py_abspath() is passed a backslashed path not to fall back to 1520 # GetFullPathNameW() on startup, which (re-)normalizes the path overly. 1521 # Currently, _Py_normpath() doesn't trim trailing dots and spaces. 1522 CASES = [ 1523 ("C:/a. . .", "C:\\a. . ."), 1524 ("C:\\a. . .", "C:\\a. . ."), 1525 ("\\\\?\\C:////a////b. . .", "\\\\?\\C:\\a\\b. . ."), 1526 ("//a/b/c. . .", "\\\\a\\b\\c. . ."), 1527 ("\\\\a\\b\\c. . .", "\\\\a\\b\\c. . ."), 1528 ("a. . .", f"{os.getcwd()}\\a"), # relpath gets fully normalized 1529 ] 1530 out, err = self.run_embedded_interpreter( 1531 "test_init_initialize_config", 1532 env={**remove_python_envvars(), 1533 "PYTHONPATH": os.path.pathsep.join(c[0] for c in CASES)} 1534 ) 1535 self.assertEqual(err, "") 1536 try: 1537 out = json.loads(out) 1538 except json.JSONDecodeError: 1539 self.fail(f"fail to decode stdout: {out!r}") 1540 1541 results = out['config']["module_search_paths"] 1542 for (_, expected), result in zip(CASES, results): 1543 self.assertEqual(result, expected) 1544 1545 def test_global_pathconfig(self): 1546 # Test C API functions getting the path configuration: 1547 # 1548 # - Py_GetExecPrefix() 1549 # - Py_GetPath() 1550 # - Py_GetPrefix() 1551 # - Py_GetProgramFullPath() 1552 # - Py_GetProgramName() 1553 # - Py_GetPythonHome() 1554 # 1555 # The global path configuration (_Py_path_config) must be a copy 1556 # of the path configuration of PyInterpreter.config (PyConfig). 1557 ctypes = import_helper.import_module('ctypes') 1558 _testinternalcapi = import_helper.import_module('_testinternalcapi') 1559 1560 def get_func(name): 1561 func = getattr(ctypes.pythonapi, name) 1562 func.argtypes = () 1563 func.restype = ctypes.c_wchar_p 1564 return func 1565 1566 Py_GetPath = get_func('Py_GetPath') 1567 Py_GetPrefix = get_func('Py_GetPrefix') 1568 Py_GetExecPrefix = get_func('Py_GetExecPrefix') 1569 Py_GetProgramName = get_func('Py_GetProgramName') 1570 Py_GetProgramFullPath = get_func('Py_GetProgramFullPath') 1571 Py_GetPythonHome = get_func('Py_GetPythonHome') 1572 1573 config = _testinternalcapi.get_configs()['config'] 1574 1575 self.assertEqual(Py_GetPath().split(os.path.pathsep), 1576 config['module_search_paths']) 1577 self.assertEqual(Py_GetPrefix(), config['prefix']) 1578 self.assertEqual(Py_GetExecPrefix(), config['exec_prefix']) 1579 self.assertEqual(Py_GetProgramName(), config['program_name']) 1580 self.assertEqual(Py_GetProgramFullPath(), config['executable']) 1581 self.assertEqual(Py_GetPythonHome(), config['home']) 1582 1583 def test_init_warnoptions(self): 1584 # lowest to highest priority 1585 warnoptions = [ 1586 'ignore:::PyConfig_Insert0', # PyWideStringList_Insert(0) 1587 'default', # PyConfig.dev_mode=1 1588 'ignore:::env1', # PYTHONWARNINGS env var 1589 'ignore:::env2', # PYTHONWARNINGS env var 1590 'ignore:::cmdline1', # -W opt command line option 1591 'ignore:::cmdline2', # -W opt command line option 1592 'default::BytesWarning', # PyConfig.bytes_warnings=1 1593 'ignore:::PySys_AddWarnOption1', # PySys_AddWarnOption() 1594 'ignore:::PySys_AddWarnOption2', # PySys_AddWarnOption() 1595 'ignore:::PyConfig_BeforeRead', # PyConfig.warnoptions 1596 'ignore:::PyConfig_AfterRead'] # PyWideStringList_Append() 1597 preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG) 1598 config = { 1599 'dev_mode': 1, 1600 'faulthandler': 1, 1601 'bytes_warning': 1, 1602 'warnoptions': warnoptions, 1603 'orig_argv': ['python3', 1604 '-Wignore:::cmdline1', 1605 '-Wignore:::cmdline2'], 1606 } 1607 self.check_all_configs("test_init_warnoptions", config, preconfig, 1608 api=API_PYTHON) 1609 1610 def test_init_set_config(self): 1611 config = { 1612 '_init_main': 0, 1613 'bytes_warning': 2, 1614 'warnoptions': ['error::BytesWarning'], 1615 } 1616 self.check_all_configs("test_init_set_config", config, 1617 api=API_ISOLATED) 1618 1619 def test_get_argc_argv(self): 1620 self.run_embedded_interpreter("test_get_argc_argv") 1621 # ignore output 1622 1623 def test_init_use_frozen_modules(self): 1624 tests = { 1625 ('=on', 1), 1626 ('=off', 0), 1627 ('=', 1), 1628 ('', 1), 1629 } 1630 for raw, expected in tests: 1631 optval = f'frozen_modules{raw}' 1632 config = { 1633 'parse_argv': 2, 1634 'argv': ['-c'], 1635 'orig_argv': ['./argv0', '-X', optval, '-c', 'pass'], 1636 'program_name': './argv0', 1637 'run_command': 'pass\n', 1638 'use_environment': 1, 1639 'xoptions': [optval], 1640 'use_frozen_modules': expected, 1641 } 1642 env = {'TESTFROZEN': raw[1:]} if raw else None 1643 with self.subTest(repr(raw)): 1644 self.check_all_configs("test_init_use_frozen_modules", config, 1645 api=API_PYTHON, env=env) 1646 1647 1648class SetConfigTests(unittest.TestCase): 1649 def test_set_config(self): 1650 # bpo-42260: Test _PyInterpreterState_SetConfig() 1651 import_helper.import_module('_testcapi') 1652 cmd = [sys.executable, '-X', 'utf8', '-I', '-m', 'test._test_embed_set_config'] 1653 proc = subprocess.run(cmd, 1654 stdout=subprocess.PIPE, 1655 stderr=subprocess.PIPE, 1656 encoding='utf-8', errors='backslashreplace') 1657 if proc.returncode and support.verbose: 1658 print(proc.stdout) 1659 print(proc.stderr) 1660 self.assertEqual(proc.returncode, 0, 1661 (proc.returncode, proc.stdout, proc.stderr)) 1662 1663 1664class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): 1665 def test_open_code_hook(self): 1666 self.run_embedded_interpreter("test_open_code_hook") 1667 1668 def test_audit(self): 1669 self.run_embedded_interpreter("test_audit") 1670 1671 def test_audit_subinterpreter(self): 1672 self.run_embedded_interpreter("test_audit_subinterpreter") 1673 1674 def test_audit_run_command(self): 1675 self.run_embedded_interpreter("test_audit_run_command", 1676 timeout=support.SHORT_TIMEOUT, 1677 returncode=1) 1678 1679 def test_audit_run_file(self): 1680 self.run_embedded_interpreter("test_audit_run_file", 1681 timeout=support.SHORT_TIMEOUT, 1682 returncode=1) 1683 1684 def test_audit_run_interactivehook(self): 1685 startup = os.path.join(self.oldcwd, os_helper.TESTFN) + ".py" 1686 with open(startup, "w", encoding="utf-8") as f: 1687 print("import sys", file=f) 1688 print("sys.__interactivehook__ = lambda: None", file=f) 1689 try: 1690 env = {**remove_python_envvars(), "PYTHONSTARTUP": startup} 1691 self.run_embedded_interpreter("test_audit_run_interactivehook", 1692 timeout=support.SHORT_TIMEOUT, 1693 returncode=10, env=env) 1694 finally: 1695 os.unlink(startup) 1696 1697 def test_audit_run_startup(self): 1698 startup = os.path.join(self.oldcwd, os_helper.TESTFN) + ".py" 1699 with open(startup, "w", encoding="utf-8") as f: 1700 print("pass", file=f) 1701 try: 1702 env = {**remove_python_envvars(), "PYTHONSTARTUP": startup} 1703 self.run_embedded_interpreter("test_audit_run_startup", 1704 timeout=support.SHORT_TIMEOUT, 1705 returncode=10, env=env) 1706 finally: 1707 os.unlink(startup) 1708 1709 def test_audit_run_stdin(self): 1710 self.run_embedded_interpreter("test_audit_run_stdin", 1711 timeout=support.SHORT_TIMEOUT, 1712 returncode=1) 1713 1714 def test_get_incomplete_frame(self): 1715 self.run_embedded_interpreter("test_get_incomplete_frame") 1716 1717 1718class MiscTests(EmbeddingTestsMixin, unittest.TestCase): 1719 def test_unicode_id_init(self): 1720 # bpo-42882: Test that _PyUnicode_FromId() works 1721 # when Python is initialized multiples times. 1722 self.run_embedded_interpreter("test_unicode_id_init") 1723 1724 # See bpo-44133 1725 @unittest.skipIf(os.name == 'nt', 1726 'Py_FrozenMain is not exported on Windows') 1727 def test_frozenmain(self): 1728 env = dict(os.environ) 1729 env['PYTHONUNBUFFERED'] = '1' 1730 out, err = self.run_embedded_interpreter("test_frozenmain", env=env) 1731 executable = os.path.realpath('./argv0') 1732 expected = textwrap.dedent(f""" 1733 Frozen Hello World 1734 sys.argv ['./argv0', '-E', 'arg1', 'arg2'] 1735 config program_name: ./argv0 1736 config executable: {executable} 1737 config use_environment: 1 1738 config configure_c_stdio: 1 1739 config buffered_stdio: 0 1740 """).lstrip() 1741 self.assertEqual(out, expected) 1742 1743 @unittest.skipUnless(hasattr(sys, 'gettotalrefcount'), 1744 '-X showrefcount requires a Python debug build') 1745 def test_no_memleak(self): 1746 # bpo-1635741: Python must release all memory at exit 1747 tests = ( 1748 ('off', 'pass'), 1749 ('on', 'pass'), 1750 ('off', 'import __hello__'), 1751 ('on', 'import __hello__'), 1752 ) 1753 for flag, stmt in tests: 1754 xopt = f"frozen_modules={flag}" 1755 cmd = [sys.executable, "-I", "-X", "showrefcount", "-X", xopt, "-c", stmt] 1756 proc = subprocess.run(cmd, 1757 stdout=subprocess.PIPE, 1758 stderr=subprocess.STDOUT, 1759 text=True) 1760 self.assertEqual(proc.returncode, 0) 1761 out = proc.stdout.rstrip() 1762 match = re.match(r'^\[(-?\d+) refs, (-?\d+) blocks\]', out) 1763 if not match: 1764 self.fail(f"unexpected output: {out!a}") 1765 refs = int(match.group(1)) 1766 blocks = int(match.group(2)) 1767 with self.subTest(frozen_modules=flag, stmt=stmt): 1768 self.assertEqual(refs, 0, out) 1769 self.assertEqual(blocks, 0, out) 1770 1771 1772class StdPrinterTests(EmbeddingTestsMixin, unittest.TestCase): 1773 # Test PyStdPrinter_Type which is used by _PySys_SetPreliminaryStderr(): 1774 # "Set up a preliminary stderr printer until we have enough 1775 # infrastructure for the io module in place." 1776 1777 STDOUT_FD = 1 1778 1779 def create_printer(self, fd): 1780 ctypes = import_helper.import_module('ctypes') 1781 PyFile_NewStdPrinter = ctypes.pythonapi.PyFile_NewStdPrinter 1782 PyFile_NewStdPrinter.argtypes = (ctypes.c_int,) 1783 PyFile_NewStdPrinter.restype = ctypes.py_object 1784 return PyFile_NewStdPrinter(fd) 1785 1786 def test_write(self): 1787 message = "unicode:\xe9-\u20ac-\udc80!\n" 1788 1789 stdout_fd = self.STDOUT_FD 1790 stdout_fd_copy = os.dup(stdout_fd) 1791 self.addCleanup(os.close, stdout_fd_copy) 1792 1793 rfd, wfd = os.pipe() 1794 self.addCleanup(os.close, rfd) 1795 self.addCleanup(os.close, wfd) 1796 try: 1797 # PyFile_NewStdPrinter() only accepts fileno(stdout) 1798 # or fileno(stderr) file descriptor. 1799 os.dup2(wfd, stdout_fd) 1800 1801 printer = self.create_printer(stdout_fd) 1802 printer.write(message) 1803 finally: 1804 os.dup2(stdout_fd_copy, stdout_fd) 1805 1806 data = os.read(rfd, 100) 1807 self.assertEqual(data, message.encode('utf8', 'backslashreplace')) 1808 1809 def test_methods(self): 1810 fd = self.STDOUT_FD 1811 printer = self.create_printer(fd) 1812 self.assertEqual(printer.fileno(), fd) 1813 self.assertEqual(printer.isatty(), os.isatty(fd)) 1814 printer.flush() # noop 1815 printer.close() # noop 1816 1817 def test_disallow_instantiation(self): 1818 fd = self.STDOUT_FD 1819 printer = self.create_printer(fd) 1820 support.check_disallow_instantiation(self, type(printer)) 1821 1822 1823if __name__ == "__main__": 1824 unittest.main() 1825