1import gc
2import importlib
3import importlib.util
4import os
5import os.path
6import py_compile
7import sys
8from test import support
9from test.support import import_helper
10from test.support import os_helper
11from test.support import script_helper
12from test.support import warnings_helper
13import unittest
14import warnings
15imp = warnings_helper.import_deprecated('imp')
16import _imp
17
18
19OS_PATH_NAME = os.path.__name__
20
21
22def requires_load_dynamic(meth):
23    """Decorator to skip a test if not running under CPython or lacking
24    imp.load_dynamic()."""
25    meth = support.cpython_only(meth)
26    return unittest.skipIf(getattr(imp, 'load_dynamic', None) is None,
27                           'imp.load_dynamic() required')(meth)
28
29
30class LockTests(unittest.TestCase):
31
32    """Very basic test of import lock functions."""
33
34    def verify_lock_state(self, expected):
35        self.assertEqual(imp.lock_held(), expected,
36                             "expected imp.lock_held() to be %r" % expected)
37    def testLock(self):
38        LOOPS = 50
39
40        # The import lock may already be held, e.g. if the test suite is run
41        # via "import test.autotest".
42        lock_held_at_start = imp.lock_held()
43        self.verify_lock_state(lock_held_at_start)
44
45        for i in range(LOOPS):
46            imp.acquire_lock()
47            self.verify_lock_state(True)
48
49        for i in range(LOOPS):
50            imp.release_lock()
51
52        # The original state should be restored now.
53        self.verify_lock_state(lock_held_at_start)
54
55        if not lock_held_at_start:
56            try:
57                imp.release_lock()
58            except RuntimeError:
59                pass
60            else:
61                self.fail("release_lock() without lock should raise "
62                            "RuntimeError")
63
64class ImportTests(unittest.TestCase):
65    def setUp(self):
66        mod = importlib.import_module('test.encoded_modules')
67        self.test_strings = mod.test_strings
68        self.test_path = mod.__path__
69
70    def test_import_encoded_module(self):
71        for modname, encoding, teststr in self.test_strings:
72            mod = importlib.import_module('test.encoded_modules.'
73                                          'module_' + modname)
74            self.assertEqual(teststr, mod.test)
75
76    def test_find_module_encoding(self):
77        for mod, encoding, _ in self.test_strings:
78            with imp.find_module('module_' + mod, self.test_path)[0] as fd:
79                self.assertEqual(fd.encoding, encoding)
80
81        path = [os.path.dirname(__file__)]
82        with self.assertRaises(SyntaxError):
83            imp.find_module('badsyntax_pep3120', path)
84
85    def test_issue1267(self):
86        for mod, encoding, _ in self.test_strings:
87            fp, filename, info  = imp.find_module('module_' + mod,
88                                                  self.test_path)
89            with fp:
90                self.assertNotEqual(fp, None)
91                self.assertEqual(fp.encoding, encoding)
92                self.assertEqual(fp.tell(), 0)
93                self.assertEqual(fp.readline(), '# test %s encoding\n'
94                                 % encoding)
95
96        fp, filename, info = imp.find_module("tokenize")
97        with fp:
98            self.assertNotEqual(fp, None)
99            self.assertEqual(fp.encoding, "utf-8")
100            self.assertEqual(fp.tell(), 0)
101            self.assertEqual(fp.readline(),
102                             '"""Tokenization help for Python programs.\n')
103
104    def test_issue3594(self):
105        temp_mod_name = 'test_imp_helper'
106        sys.path.insert(0, '.')
107        try:
108            with open(temp_mod_name + '.py', 'w', encoding="latin-1") as file:
109                file.write("# coding: cp1252\nu = 'test.test_imp'\n")
110            file, filename, info = imp.find_module(temp_mod_name)
111            file.close()
112            self.assertEqual(file.encoding, 'cp1252')
113        finally:
114            del sys.path[0]
115            os_helper.unlink(temp_mod_name + '.py')
116            os_helper.unlink(temp_mod_name + '.pyc')
117
118    def test_issue5604(self):
119        # Test cannot cover imp.load_compiled function.
120        # Martin von Loewis note what shared library cannot have non-ascii
121        # character because init_xxx function cannot be compiled
122        # and issue never happens for dynamic modules.
123        # But sources modified to follow generic way for processing paths.
124
125        # the return encoding could be uppercase or None
126        fs_encoding = sys.getfilesystemencoding()
127
128        # covers utf-8 and Windows ANSI code pages
129        # one non-space symbol from every page
130        # (http://en.wikipedia.org/wiki/Code_page)
131        known_locales = {
132            'utf-8' : b'\xc3\xa4',
133            'cp1250' : b'\x8C',
134            'cp1251' : b'\xc0',
135            'cp1252' : b'\xc0',
136            'cp1253' : b'\xc1',
137            'cp1254' : b'\xc0',
138            'cp1255' : b'\xe0',
139            'cp1256' : b'\xe0',
140            'cp1257' : b'\xc0',
141            'cp1258' : b'\xc0',
142            }
143
144        if sys.platform == 'darwin':
145            self.assertEqual(fs_encoding, 'utf-8')
146            # Mac OS X uses the Normal Form D decomposition
147            # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
148            special_char = b'a\xcc\x88'
149        else:
150            special_char = known_locales.get(fs_encoding)
151
152        if not special_char:
153            self.skipTest("can't run this test with %s as filesystem encoding"
154                          % fs_encoding)
155        decoded_char = special_char.decode(fs_encoding)
156        temp_mod_name = 'test_imp_helper_' + decoded_char
157        test_package_name = 'test_imp_helper_package_' + decoded_char
158        init_file_name = os.path.join(test_package_name, '__init__.py')
159        try:
160            # if the curdir is not in sys.path the test fails when run with
161            # ./python ./Lib/test/regrtest.py test_imp
162            sys.path.insert(0, os.curdir)
163            with open(temp_mod_name + '.py', 'w', encoding="utf-8") as file:
164                file.write('a = 1\n')
165            file, filename, info = imp.find_module(temp_mod_name)
166            with file:
167                self.assertIsNotNone(file)
168                self.assertTrue(filename[:-3].endswith(temp_mod_name))
169                self.assertEqual(info[0], '.py')
170                self.assertEqual(info[1], 'r')
171                self.assertEqual(info[2], imp.PY_SOURCE)
172
173                mod = imp.load_module(temp_mod_name, file, filename, info)
174                self.assertEqual(mod.a, 1)
175
176            with warnings.catch_warnings():
177                warnings.simplefilter('ignore')
178                mod = imp.load_source(temp_mod_name, temp_mod_name + '.py')
179            self.assertEqual(mod.a, 1)
180
181            with warnings.catch_warnings():
182                warnings.simplefilter('ignore')
183                if not sys.dont_write_bytecode:
184                    mod = imp.load_compiled(
185                        temp_mod_name,
186                        imp.cache_from_source(temp_mod_name + '.py'))
187            self.assertEqual(mod.a, 1)
188
189            if not os.path.exists(test_package_name):
190                os.mkdir(test_package_name)
191            with open(init_file_name, 'w', encoding="utf-8") as file:
192                file.write('b = 2\n')
193            with warnings.catch_warnings():
194                warnings.simplefilter('ignore')
195                package = imp.load_package(test_package_name, test_package_name)
196            self.assertEqual(package.b, 2)
197        finally:
198            del sys.path[0]
199            for ext in ('.py', '.pyc'):
200                os_helper.unlink(temp_mod_name + ext)
201                os_helper.unlink(init_file_name + ext)
202            os_helper.rmtree(test_package_name)
203            os_helper.rmtree('__pycache__')
204
205    def test_issue9319(self):
206        path = os.path.dirname(__file__)
207        self.assertRaises(SyntaxError,
208                          imp.find_module, "badsyntax_pep3120", [path])
209
210    def test_load_from_source(self):
211        # Verify that the imp module can correctly load and find .py files
212        # XXX (ncoghlan): It would be nice to use import_helper.CleanImport
213        # here, but that breaks because the os module registers some
214        # handlers in copy_reg on import. Since CleanImport doesn't
215        # revert that registration, the module is left in a broken
216        # state after reversion. Reinitialising the module contents
217        # and just reverting os.environ to its previous state is an OK
218        # workaround
219        with import_helper.CleanImport('os', 'os.path', OS_PATH_NAME):
220            import os
221            orig_path = os.path
222            orig_getenv = os.getenv
223            with os_helper.EnvironmentVarGuard():
224                x = imp.find_module("os")
225                self.addCleanup(x[0].close)
226                new_os = imp.load_module("os", *x)
227                self.assertIs(os, new_os)
228                self.assertIs(orig_path, new_os.path)
229                self.assertIsNot(orig_getenv, new_os.getenv)
230
231    @requires_load_dynamic
232    def test_issue15828_load_extensions(self):
233        # Issue 15828 picked up that the adapter between the old imp API
234        # and importlib couldn't handle C extensions
235        example = "_heapq"
236        x = imp.find_module(example)
237        file_ = x[0]
238        if file_ is not None:
239            self.addCleanup(file_.close)
240        mod = imp.load_module(example, *x)
241        self.assertEqual(mod.__name__, example)
242
243    @requires_load_dynamic
244    def test_issue16421_multiple_modules_in_one_dll(self):
245        # Issue 16421: loading several modules from the same compiled file fails
246        m = '_testimportmultiple'
247        fileobj, pathname, description = imp.find_module(m)
248        fileobj.close()
249        mod0 = imp.load_dynamic(m, pathname)
250        mod1 = imp.load_dynamic('_testimportmultiple_foo', pathname)
251        mod2 = imp.load_dynamic('_testimportmultiple_bar', pathname)
252        self.assertEqual(mod0.__name__, m)
253        self.assertEqual(mod1.__name__, '_testimportmultiple_foo')
254        self.assertEqual(mod2.__name__, '_testimportmultiple_bar')
255        with self.assertRaises(ImportError):
256            imp.load_dynamic('nonexistent', pathname)
257
258    @requires_load_dynamic
259    def test_load_dynamic_ImportError_path(self):
260        # Issue #1559549 added `name` and `path` attributes to ImportError
261        # in order to provide better detail. Issue #10854 implemented those
262        # attributes on import failures of extensions on Windows.
263        path = 'bogus file path'
264        name = 'extension'
265        with self.assertRaises(ImportError) as err:
266            imp.load_dynamic(name, path)
267        self.assertIn(path, err.exception.path)
268        self.assertEqual(name, err.exception.name)
269
270    @requires_load_dynamic
271    def test_load_module_extension_file_is_None(self):
272        # When loading an extension module and the file is None, open one
273        # on the behalf of imp.load_dynamic().
274        # Issue #15902
275        name = '_testimportmultiple'
276        found = imp.find_module(name)
277        if found[0] is not None:
278            found[0].close()
279        if found[2][2] != imp.C_EXTENSION:
280            self.skipTest("found module doesn't appear to be a C extension")
281        imp.load_module(name, None, *found[1:])
282
283    @requires_load_dynamic
284    def test_issue24748_load_module_skips_sys_modules_check(self):
285        name = 'test.imp_dummy'
286        try:
287            del sys.modules[name]
288        except KeyError:
289            pass
290        try:
291            module = importlib.import_module(name)
292            spec = importlib.util.find_spec('_testmultiphase')
293            module = imp.load_dynamic(name, spec.origin)
294            self.assertEqual(module.__name__, name)
295            self.assertEqual(module.__spec__.name, name)
296            self.assertEqual(module.__spec__.origin, spec.origin)
297            self.assertRaises(AttributeError, getattr, module, 'dummy_name')
298            self.assertEqual(module.int_const, 1969)
299            self.assertIs(sys.modules[name], module)
300        finally:
301            try:
302                del sys.modules[name]
303            except KeyError:
304                pass
305
306    @unittest.skipIf(sys.dont_write_bytecode,
307        "test meaningful only when writing bytecode")
308    def test_bug7732(self):
309        with os_helper.temp_cwd():
310            source = os_helper.TESTFN + '.py'
311            os.mkdir(source)
312            self.assertRaisesRegex(ImportError, '^No module',
313                imp.find_module, os_helper.TESTFN, ["."])
314
315    def test_multiple_calls_to_get_data(self):
316        # Issue #18755: make sure multiple calls to get_data() can succeed.
317        loader = imp._LoadSourceCompatibility('imp', imp.__file__,
318                                              open(imp.__file__, encoding="utf-8"))
319        loader.get_data(imp.__file__)  # File should be closed
320        loader.get_data(imp.__file__)  # Will need to create a newly opened file
321
322    def test_load_source(self):
323        # Create a temporary module since load_source(name) modifies
324        # sys.modules[name] attributes like __loader___
325        modname = f"tmp{__name__}"
326        mod = type(sys.modules[__name__])(modname)
327        with support.swap_item(sys.modules, modname, mod):
328            with self.assertRaisesRegex(ValueError, 'embedded null'):
329                imp.load_source(modname, __file__ + "\0")
330
331    @support.cpython_only
332    def test_issue31315(self):
333        # There shouldn't be an assertion failure in imp.create_dynamic(),
334        # when spec.name is not a string.
335        create_dynamic = support.get_attribute(imp, 'create_dynamic')
336        class BadSpec:
337            name = None
338            origin = 'foo'
339        with self.assertRaises(TypeError):
340            create_dynamic(BadSpec())
341
342    def test_issue_35321(self):
343        # Both _frozen_importlib and _frozen_importlib_external
344        # should have a spec origin of "frozen" and
345        # no need to clean up imports in this case.
346
347        import _frozen_importlib_external
348        self.assertEqual(_frozen_importlib_external.__spec__.origin, "frozen")
349
350        import _frozen_importlib
351        self.assertEqual(_frozen_importlib.__spec__.origin, "frozen")
352
353    def test_source_hash(self):
354        self.assertEqual(_imp.source_hash(42, b'hi'), b'\xfb\xd9G\x05\xaf$\x9b~')
355        self.assertEqual(_imp.source_hash(43, b'hi'), b'\xd0/\x87C\xccC\xff\xe2')
356
357    def test_pyc_invalidation_mode_from_cmdline(self):
358        cases = [
359            ([], "default"),
360            (["--check-hash-based-pycs", "default"], "default"),
361            (["--check-hash-based-pycs", "always"], "always"),
362            (["--check-hash-based-pycs", "never"], "never"),
363        ]
364        for interp_args, expected in cases:
365            args = interp_args + [
366                "-c",
367                "import _imp; print(_imp.check_hash_based_pycs)",
368            ]
369            res = script_helper.assert_python_ok(*args)
370            self.assertEqual(res.out.strip().decode('utf-8'), expected)
371
372    def test_find_and_load_checked_pyc(self):
373        # issue 34056
374        with os_helper.temp_cwd():
375            with open('mymod.py', 'wb') as fp:
376                fp.write(b'x = 42\n')
377            py_compile.compile(
378                'mymod.py',
379                doraise=True,
380                invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
381            )
382            file, path, description = imp.find_module('mymod', path=['.'])
383            mod = imp.load_module('mymod', file, path, description)
384        self.assertEqual(mod.x, 42)
385
386
387    @support.cpython_only
388    def test_create_builtin_subinterp(self):
389        # gh-99578: create_builtin() behavior changes after the creation of the
390        # first sub-interpreter. Test both code paths, before and after the
391        # creation of a sub-interpreter. Previously, create_builtin() had
392        # a reference leak after the creation of the first sub-interpreter.
393
394        import builtins
395        create_builtin = support.get_attribute(_imp, "create_builtin")
396        class Spec:
397            name = "builtins"
398        spec = Spec()
399
400        def check_get_builtins():
401            refcnt = sys.getrefcount(builtins)
402            mod = _imp.create_builtin(spec)
403            self.assertIs(mod, builtins)
404            self.assertEqual(sys.getrefcount(builtins), refcnt + 1)
405            # Check that a GC collection doesn't crash
406            gc.collect()
407
408        check_get_builtins()
409
410        ret = support.run_in_subinterp("import builtins")
411        self.assertEqual(ret, 0)
412
413        check_get_builtins()
414
415
416class ReloadTests(unittest.TestCase):
417
418    """Very basic tests to make sure that imp.reload() operates just like
419    reload()."""
420
421    def test_source(self):
422        # XXX (ncoghlan): It would be nice to use test.import_helper.CleanImport
423        # here, but that breaks because the os module registers some
424        # handlers in copy_reg on import. Since CleanImport doesn't
425        # revert that registration, the module is left in a broken
426        # state after reversion. Reinitialising the module contents
427        # and just reverting os.environ to its previous state is an OK
428        # workaround
429        with os_helper.EnvironmentVarGuard():
430            import os
431            imp.reload(os)
432
433    def test_extension(self):
434        with import_helper.CleanImport('time'):
435            import time
436            imp.reload(time)
437
438    def test_builtin(self):
439        with import_helper.CleanImport('marshal'):
440            import marshal
441            imp.reload(marshal)
442
443    def test_with_deleted_parent(self):
444        # see #18681
445        from html import parser
446        html = sys.modules.pop('html')
447        def cleanup():
448            sys.modules['html'] = html
449        self.addCleanup(cleanup)
450        with self.assertRaisesRegex(ImportError, 'html'):
451            imp.reload(parser)
452
453
454class PEP3147Tests(unittest.TestCase):
455    """Tests of PEP 3147."""
456
457    tag = imp.get_tag()
458
459    @unittest.skipUnless(sys.implementation.cache_tag is not None,
460                         'requires sys.implementation.cache_tag not be None')
461    def test_cache_from_source(self):
462        # Given the path to a .py file, return the path to its PEP 3147
463        # defined .pyc file (i.e. under __pycache__).
464        path = os.path.join('foo', 'bar', 'baz', 'qux.py')
465        expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
466                              'qux.{}.pyc'.format(self.tag))
467        self.assertEqual(imp.cache_from_source(path, True), expect)
468
469    @unittest.skipUnless(sys.implementation.cache_tag is not None,
470                         'requires sys.implementation.cache_tag to not be '
471                         'None')
472    def test_source_from_cache(self):
473        # Given the path to a PEP 3147 defined .pyc file, return the path to
474        # its source.  This tests the good path.
475        path = os.path.join('foo', 'bar', 'baz', '__pycache__',
476                            'qux.{}.pyc'.format(self.tag))
477        expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
478        self.assertEqual(imp.source_from_cache(path), expect)
479
480
481class NullImporterTests(unittest.TestCase):
482    @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None,
483                     "Need an undecodeable filename")
484    def test_unencodeable(self):
485        name = os_helper.TESTFN_UNENCODABLE
486        os.mkdir(name)
487        try:
488            self.assertRaises(ImportError, imp.NullImporter, name)
489        finally:
490            os.rmdir(name)
491
492
493if __name__ == "__main__":
494    unittest.main()
495