1"""
2Collect various information about Python to help debugging test failures.
3"""
4from __future__ import print_function
5import errno
6import re
7import sys
8import traceback
9
10
11def normalize_text(text):
12    if text is None:
13        return None
14    text = str(text)
15    text = re.sub(r'\s+', ' ', text)
16    return text.strip()
17
18
19class PythonInfo:
20    def __init__(self):
21        self.info = {}
22
23    def add(self, key, value):
24        if key in self.info:
25            raise ValueError("duplicate key: %r" % key)
26
27        if value is None:
28            return
29
30        if not isinstance(value, int):
31            if not isinstance(value, str):
32                # convert other objects like sys.flags to string
33                value = str(value)
34
35            value = value.strip()
36            if not value:
37                return
38
39        self.info[key] = value
40
41    def get_infos(self):
42        """
43        Get information as a key:value dictionary where values are strings.
44        """
45        return {key: str(value) for key, value in self.info.items()}
46
47
48def copy_attributes(info_add, obj, name_fmt, attributes, formatter=None):
49    for attr in attributes:
50        value = getattr(obj, attr, None)
51        if value is None:
52            continue
53        name = name_fmt % attr
54        if formatter is not None:
55            value = formatter(attr, value)
56        info_add(name, value)
57
58
59def copy_attr(info_add, name, mod, attr_name):
60    try:
61        value = getattr(mod, attr_name)
62    except AttributeError:
63        return
64    info_add(name, value)
65
66
67def call_func(info_add, name, mod, func_name, formatter=None):
68    try:
69        func = getattr(mod, func_name)
70    except AttributeError:
71        return
72    value = func()
73    if formatter is not None:
74        value = formatter(value)
75    info_add(name, value)
76
77
78def collect_sys(info_add):
79    attributes = (
80        '_framework',
81        'abiflags',
82        'api_version',
83        'builtin_module_names',
84        'byteorder',
85        'dont_write_bytecode',
86        'executable',
87        'flags',
88        'float_info',
89        'float_repr_style',
90        'hash_info',
91        'hexversion',
92        'implementation',
93        'int_info',
94        'maxsize',
95        'maxunicode',
96        'path',
97        'platform',
98        'prefix',
99        'thread_info',
100        'version',
101        'version_info',
102        'winver',
103    )
104    copy_attributes(info_add, sys, 'sys.%s', attributes)
105
106    call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel')
107    call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion')
108
109    encoding = sys.getfilesystemencoding()
110    if hasattr(sys, 'getfilesystemencodeerrors'):
111        encoding = '%s/%s' % (encoding, sys.getfilesystemencodeerrors())
112    info_add('sys.filesystem_encoding', encoding)
113
114    for name in ('stdin', 'stdout', 'stderr'):
115        stream = getattr(sys, name)
116        if stream is None:
117            continue
118        encoding = getattr(stream, 'encoding', None)
119        if not encoding:
120            continue
121        errors = getattr(stream, 'errors', None)
122        if errors:
123            encoding = '%s/%s' % (encoding, errors)
124        info_add('sys.%s.encoding' % name, encoding)
125
126    # Were we compiled --with-pydebug or with #define Py_DEBUG?
127    Py_DEBUG = hasattr(sys, 'gettotalrefcount')
128    if Py_DEBUG:
129        text = 'Yes (sys.gettotalrefcount() present)'
130    else:
131        text = 'No (sys.gettotalrefcount() missing)'
132    info_add('Py_DEBUG', text)
133
134
135def collect_platform(info_add):
136    import platform
137
138    arch = platform.architecture()
139    arch = ' '.join(filter(bool, arch))
140    info_add('platform.architecture', arch)
141
142    info_add('platform.python_implementation',
143             platform.python_implementation())
144    info_add('platform.platform',
145             platform.platform(aliased=True, terse=True))
146
147
148def collect_locale(info_add):
149    import locale
150
151    info_add('locale.encoding', locale.getpreferredencoding(False))
152
153
154def collect_builtins(info_add):
155    info_add('builtins.float.float_format', float.__getformat__("float"))
156    info_add('builtins.float.double_format', float.__getformat__("double"))
157
158
159def collect_os(info_add):
160    import os
161
162    def format_attr(attr, value):
163        if attr in ('supports_follow_symlinks', 'supports_fd',
164                    'supports_effective_ids'):
165            return str(sorted(func.__name__ for func in value))
166        else:
167            return value
168
169    attributes = (
170        'name',
171        'supports_bytes_environ',
172        'supports_effective_ids',
173        'supports_fd',
174        'supports_follow_symlinks',
175    )
176    copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr)
177
178    call_func(info_add, 'os.cwd', os, 'getcwd')
179
180    call_func(info_add, 'os.uid', os, 'getuid')
181    call_func(info_add, 'os.gid', os, 'getgid')
182    call_func(info_add, 'os.uname', os, 'uname')
183
184    def format_groups(groups):
185        return ', '.join(map(str, groups))
186
187    call_func(info_add, 'os.groups', os, 'getgroups', formatter=format_groups)
188
189    if hasattr(os, 'getlogin'):
190        try:
191            login = os.getlogin()
192        except OSError:
193            # getlogin() fails with "OSError: [Errno 25] Inappropriate ioctl
194            # for device" on Travis CI
195            pass
196        else:
197            info_add("os.login", login)
198
199    call_func(info_add, 'os.cpu_count', os, 'cpu_count')
200    call_func(info_add, 'os.loadavg', os, 'getloadavg')
201
202    # Get environment variables: filter to list
203    # to not leak sensitive information
204    ENV_VARS = (
205        "CC",
206        "COMSPEC",
207        "DISPLAY",
208        "DISTUTILS_USE_SDK",
209        "DYLD_LIBRARY_PATH",
210        "HOME",
211        "HOMEDRIVE",
212        "HOMEPATH",
213        "LANG",
214        "LD_LIBRARY_PATH",
215        "MACOSX_DEPLOYMENT_TARGET",
216        "MAKEFLAGS",
217        "MSSDK",
218        "PATH",
219        "SDK_TOOLS_BIN",
220        "SHELL",
221        "TEMP",
222        "TERM",
223        "TMP",
224        "TMPDIR",
225        "USERPROFILE",
226        "WAYLAND_DISPLAY",
227    )
228    for name, value in os.environ.items():
229        uname = name.upper()
230        if (uname in ENV_VARS
231           # Copy PYTHON* and LC_* variables
232           or uname.startswith(("PYTHON", "LC_"))
233           # Visual Studio: VS140COMNTOOLS
234           or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))):
235            info_add('os.environ[%s]' % name, value)
236
237    if hasattr(os, 'umask'):
238        mask = os.umask(0)
239        os.umask(mask)
240        info_add("os.umask", '%03o' % mask)
241
242    if hasattr(os, 'getrandom'):
243        # PEP 524: Check if system urandom is initialized
244        try:
245            try:
246                os.getrandom(1, os.GRND_NONBLOCK)
247                state = 'ready (initialized)'
248            except BlockingIOError as exc:
249                state = 'not seeded yet (%s)' % exc
250            info_add('os.getrandom', state)
251        except OSError as exc:
252            # Python was compiled on a more recent Linux version
253            # than the current Linux kernel: ignore OSError(ENOSYS)
254            if exc.errno != errno.ENOSYS:
255                raise
256
257
258def collect_readline(info_add):
259    try:
260        import readline
261    except ImportError:
262        return
263
264    def format_attr(attr, value):
265        if isinstance(value, int):
266            return "%#x" % value
267        else:
268            return value
269
270    attributes = (
271        "_READLINE_VERSION",
272        "_READLINE_RUNTIME_VERSION",
273        "_READLINE_LIBRARY_VERSION",
274    )
275    copy_attributes(info_add, readline, 'readline.%s', attributes,
276                    formatter=format_attr)
277
278    if not hasattr(readline, "_READLINE_LIBRARY_VERSION"):
279        # _READLINE_LIBRARY_VERSION has been added to CPython 3.7
280        doc = getattr(readline, '__doc__', '')
281        if 'libedit readline' in doc:
282            info_add('readline.library', 'libedit readline')
283        elif 'GNU readline' in doc:
284            info_add('readline.library', 'GNU readline')
285
286
287def collect_gdb(info_add):
288    import subprocess
289
290    try:
291        proc = subprocess.Popen(["gdb", "-nx", "--version"],
292                                stdout=subprocess.PIPE,
293                                stderr=subprocess.PIPE,
294                                universal_newlines=True)
295        version = proc.communicate()[0]
296    except OSError:
297        return
298
299    # Only keep the first line
300    version = version.splitlines()[0]
301    info_add('gdb_version', version)
302
303
304def collect_tkinter(info_add):
305    try:
306        import _tkinter
307    except ImportError:
308        pass
309    else:
310        attributes = ('TK_VERSION', 'TCL_VERSION')
311        copy_attributes(info_add, _tkinter, 'tkinter.%s', attributes)
312
313    try:
314        import tkinter
315    except ImportError:
316        pass
317    else:
318        tcl = tkinter.Tcl()
319        patchlevel = tcl.call('info', 'patchlevel')
320        info_add('tkinter.info_patchlevel', patchlevel)
321
322
323def collect_time(info_add):
324    import time
325
326    info_add('time.time', time.time())
327
328    attributes = (
329        'altzone',
330        'daylight',
331        'timezone',
332        'tzname',
333    )
334    copy_attributes(info_add, time, 'time.%s', attributes)
335
336    if hasattr(time, 'get_clock_info'):
337        for clock in ('time', 'perf_counter'):
338            tinfo = time.get_clock_info(clock)
339            info_add('time.get_clock_info(%s)' % clock, tinfo)
340
341
342def collect_datetime(info_add):
343    try:
344        import datetime
345    except ImportError:
346        return
347
348    info_add('datetime.datetime.now', datetime.datetime.now())
349
350
351def collect_sysconfig(info_add):
352    import sysconfig
353
354    for name in (
355        'ABIFLAGS',
356        'ANDROID_API_LEVEL',
357        'CC',
358        'CCSHARED',
359        'CFLAGS',
360        'CFLAGSFORSHARED',
361        'CONFIG_ARGS',
362        'HOST_GNU_TYPE',
363        'MACHDEP',
364        'MULTIARCH',
365        'OPT',
366        'PY_CFLAGS',
367        'PY_CFLAGS_NODIST',
368        'PY_LDFLAGS',
369        'Py_DEBUG',
370        'Py_ENABLE_SHARED',
371        'SHELL',
372        'SOABI',
373        'prefix',
374    ):
375        value = sysconfig.get_config_var(name)
376        if name == 'ANDROID_API_LEVEL' and not value:
377            # skip ANDROID_API_LEVEL=0
378            continue
379        value = normalize_text(value)
380        info_add('sysconfig[%s]' % name, value)
381
382
383def collect_ssl(info_add):
384    try:
385        import ssl
386    except ImportError:
387        return
388
389    def format_attr(attr, value):
390        if attr.startswith('OP_'):
391            return '%#8x' % value
392        else:
393            return value
394
395    attributes = (
396        'OPENSSL_VERSION',
397        'OPENSSL_VERSION_INFO',
398        'HAS_SNI',
399        'OP_ALL',
400        'OP_NO_TLSv1_1',
401    )
402    copy_attributes(info_add, ssl, 'ssl.%s', attributes, formatter=format_attr)
403
404
405def collect_socket(info_add):
406    import socket
407
408    hostname = socket.gethostname()
409    info_add('socket.hostname', hostname)
410
411
412def collect_sqlite(info_add):
413    try:
414        import sqlite3
415    except ImportError:
416        return
417
418    attributes = ('version', 'sqlite_version')
419    copy_attributes(info_add, sqlite3, 'sqlite3.%s', attributes)
420
421
422def collect_zlib(info_add):
423    try:
424        import zlib
425    except ImportError:
426        return
427
428    attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION')
429    copy_attributes(info_add, zlib, 'zlib.%s', attributes)
430
431
432def collect_expat(info_add):
433    try:
434        from xml.parsers import expat
435    except ImportError:
436        return
437
438    attributes = ('EXPAT_VERSION',)
439    copy_attributes(info_add, expat, 'expat.%s', attributes)
440
441
442def collect_decimal(info_add):
443    try:
444        import _decimal
445    except ImportError:
446        return
447
448    attributes = ('__libmpdec_version__',)
449    copy_attributes(info_add, _decimal, '_decimal.%s', attributes)
450
451
452def collect_testcapi(info_add):
453    try:
454        import _testcapi
455    except ImportError:
456        return
457
458    call_func(info_add, 'pymem.allocator', _testcapi, 'pymem_getallocatorsname')
459    copy_attr(info_add, 'pymem.with_pymalloc', _testcapi, 'WITH_PYMALLOC')
460
461
462def collect_resource(info_add):
463    try:
464        import resource
465    except ImportError:
466        return
467
468    limits = [attr for attr in dir(resource) if attr.startswith('RLIMIT_')]
469    for name in limits:
470        key = getattr(resource, name)
471        value = resource.getrlimit(key)
472        info_add('resource.%s' % name, value)
473
474
475def collect_test_socket(info_add):
476    try:
477        from test import test_socket
478    except ImportError:
479        return
480
481    # all check attributes like HAVE_SOCKET_CAN
482    attributes = [name for name in dir(test_socket)
483                  if name.startswith('HAVE_')]
484    copy_attributes(info_add, test_socket, 'test_socket.%s', attributes)
485
486
487def collect_test_support(info_add):
488    try:
489        from test import support
490    except ImportError:
491        return
492
493    attributes = ('IPV6_ENABLED',)
494    copy_attributes(info_add, support, 'test_support.%s', attributes)
495
496    call_func(info_add, 'test_support._is_gui_available', support, '_is_gui_available')
497    call_func(info_add, 'test_support.python_is_optimized', support, 'python_is_optimized')
498
499
500def collect_cc(info_add):
501    import subprocess
502    import sysconfig
503
504    CC = sysconfig.get_config_var('CC')
505    if not CC:
506        return
507
508    try:
509        import shlex
510        args = shlex.split(CC)
511    except ImportError:
512        args = CC.split()
513    args.append('--version')
514    proc = subprocess.Popen(args,
515                            stdout=subprocess.PIPE,
516                            stderr=subprocess.STDOUT,
517                            universal_newlines=True)
518    stdout = proc.communicate()[0]
519    if proc.returncode:
520        # CC --version failed: ignore error
521        return
522
523    text = stdout.splitlines()[0]
524    text = normalize_text(text)
525    info_add('CC.version', text)
526
527
528def collect_info(info):
529    error = False
530    info_add = info.add
531
532    for collect_func in (
533        # collect_os() should be the first, to check the getrandom() status
534        collect_os,
535
536        collect_builtins,
537        collect_gdb,
538        collect_locale,
539        collect_platform,
540        collect_readline,
541        collect_socket,
542        collect_sqlite,
543        collect_ssl,
544        collect_sys,
545        collect_sysconfig,
546        collect_time,
547        collect_datetime,
548        collect_tkinter,
549        collect_zlib,
550        collect_expat,
551        collect_decimal,
552        collect_testcapi,
553        collect_resource,
554        collect_cc,
555
556        # Collecting from tests should be last as they have side effects.
557        collect_test_socket,
558        collect_test_support,
559    ):
560        try:
561            collect_func(info_add)
562        except Exception as exc:
563            error = True
564            print("ERROR: %s() failed" % (collect_func.__name__),
565                  file=sys.stderr)
566            traceback.print_exc(file=sys.stderr)
567            print(file=sys.stderr)
568            sys.stderr.flush()
569
570    return error
571
572
573def dump_info(info, file=None):
574    title = "Python debug information"
575    print(title)
576    print("=" * len(title))
577    print()
578
579    infos = info.get_infos()
580    infos = sorted(infos.items())
581    for key, value in infos:
582        value = value.replace("\n", " ")
583        print("%s: %s" % (key, value))
584    print()
585
586
587def main():
588    info = PythonInfo()
589    error = collect_info(info)
590    dump_info(info)
591
592    if error:
593        print("Collection failed: exit with error", file=sys.stderr)
594        sys.exit(1)
595
596
597if __name__ == "__main__":
598    main()
599