1# This script lists the names of standard library modules
2# to update Python/stdlib_mod_names.h
3import _imp
4import os.path
5import re
6import subprocess
7import sys
8import sysconfig
9
10
11SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
12STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
13MODULES_SETUP = os.path.join(SRC_DIR, 'Modules', 'Setup')
14SETUP_PY = os.path.join(SRC_DIR, 'setup.py')
15
16IGNORE = {
17    '__init__',
18    '__pycache__',
19    'site-packages',
20
21    # Test modules and packages
22    '__hello__',
23    '__phello__',
24    '__hello_alias__',
25    '__phello_alias__',
26    '__hello_only__',
27    '_ctypes_test',
28    '_testbuffer',
29    '_testcapi',
30    '_testclinic',
31    '_testconsole',
32    '_testimportmultiple',
33    '_testinternalcapi',
34    '_testmultiphase',
35    '_xxsubinterpreters',
36    '_xxtestfuzz',
37    'distutils.tests',
38    'idlelib.idle_test',
39    'lib2to3.tests',
40    'test',
41    'xxlimited',
42    'xxlimited_35',
43    'xxsubtype',
44}
45
46# Windows extension modules
47WINDOWS_MODULES = (
48    '_msi',
49    '_overlapped',
50    '_testconsole',
51    '_winapi',
52    'msvcrt',
53    'nt',
54    'winreg',
55    'winsound'
56)
57
58# macOS extension modules
59MACOS_MODULES = (
60    '_scproxy',
61)
62
63# Pure Python modules (Lib/*.py)
64def list_python_modules(names):
65    for filename in os.listdir(STDLIB_PATH):
66        if not filename.endswith(".py"):
67            continue
68        name = filename.removesuffix(".py")
69        names.add(name)
70
71
72# Packages in Lib/
73def list_packages(names):
74    for name in os.listdir(STDLIB_PATH):
75        if name in IGNORE:
76            continue
77        package_path = os.path.join(STDLIB_PATH, name)
78        if not os.path.isdir(package_path):
79            continue
80        if any(package_file.endswith(".py")
81               for package_file in os.listdir(package_path)):
82            names.add(name)
83
84
85# Extension modules built by setup.py
86def list_setup_extensions(names):
87    cmd = [sys.executable, SETUP_PY, "-q", "build", "--list-module-names"]
88    output = subprocess.check_output(cmd)
89    output = output.decode("utf8")
90    extensions = output.splitlines()
91    names |= set(extensions)
92
93
94# Built-in and extension modules built by Modules/Setup
95def list_modules_setup_extensions(names):
96    assign_var = re.compile("^[A-Z]+=")
97
98    with open(MODULES_SETUP, encoding="utf-8") as modules_fp:
99        for line in modules_fp:
100            # Strip comment
101            line = line.partition("#")[0]
102            line = line.rstrip()
103            if not line:
104                continue
105            if assign_var.match(line):
106                # Ignore "VAR=VALUE"
107                continue
108            if line in ("*disabled*", "*shared*"):
109                continue
110            parts = line.split()
111            if len(parts) < 2:
112                continue
113            # "errno errnomodule.c" => write "errno"
114            name = parts[0]
115            names.add(name)
116
117
118# List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
119# Use the "./Programs/_testembed list_frozen" command.
120def list_frozen(names):
121    submodules = set()
122    for name in _imp._frozen_module_names():
123        # To skip __hello__, __hello_alias__ and etc.
124        if name.startswith('__'):
125            continue
126        if '.' in name:
127            submodules.add(name)
128        else:
129            names.add(name)
130    # Make sure all frozen submodules have a known parent.
131    for name in list(submodules):
132        if name.partition('.')[0] in names:
133            submodules.remove(name)
134    if submodules:
135        raise Exception(f'unexpected frozen submodules: {sorted(submodules)}')
136
137
138def list_modules():
139    names = set(sys.builtin_module_names) | set(WINDOWS_MODULES) | set(MACOS_MODULES)
140    list_modules_setup_extensions(names)
141    list_setup_extensions(names)
142    list_packages(names)
143    list_python_modules(names)
144    list_frozen(names)
145
146    # Remove ignored packages and modules
147    for name in list(names):
148        package_name = name.split('.')[0]
149        # package_name can be equal to name
150        if package_name in IGNORE:
151            names.discard(name)
152
153    for name in names:
154        if "." in name:
155            raise Exception("sub-modules must not be listed")
156
157    return names
158
159
160def write_modules(fp, names):
161    print("// Auto-generated by Tools/scripts/generate_stdlib_module_names.py.",
162          file=fp)
163    print("// List used to create sys.stdlib_module_names.", file=fp)
164    print(file=fp)
165    print("static const char* _Py_stdlib_module_names[] = {", file=fp)
166    for name in sorted(names):
167        print(f'"{name}",', file=fp)
168    print("};", file=fp)
169
170
171def main():
172    if not sysconfig.is_python_build():
173        print(f"ERROR: {sys.executable} is not a Python build",
174              file=sys.stderr)
175        sys.exit(1)
176
177    fp = sys.stdout
178    names = list_modules()
179    write_modules(fp, names)
180
181
182if __name__ == "__main__":
183    main()
184