1import argparse
2import os
3import shlex
4import sys
5from test.support import os_helper
6
7
8USAGE = """\
9python -m test [options] [test_name1 [test_name2 ...]]
10python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]]
11"""
12
13DESCRIPTION = """\
14Run Python regression tests.
15
16If no arguments or options are provided, finds all files matching
17the pattern "test_*" in the Lib/test subdirectory and runs
18them in alphabetical order (but see -M and -u, below, for exceptions).
19
20For more rigorous testing, it is useful to use the following
21command line:
22
23python -E -Wd -m test [options] [test_name1 ...]
24"""
25
26EPILOG = """\
27Additional option details:
28
29-r randomizes test execution order. You can use --randseed=int to provide an
30int seed value for the randomizer; this is useful for reproducing troublesome
31test orders.
32
33-s On the first invocation of regrtest using -s, the first test file found
34or the first test file given on the command line is run, and the name of
35the next test is recorded in a file named pynexttest.  If run from the
36Python build directory, pynexttest is located in the 'build' subdirectory,
37otherwise it is located in tempfile.gettempdir().  On subsequent runs,
38the test in pynexttest is run, and the next test is written to pynexttest.
39When the last test has been run, pynexttest is deleted.  In this way it
40is possible to single step through the test files.  This is useful when
41doing memory analysis on the Python interpreter, which process tends to
42consume too many resources to run the full regression test non-stop.
43
44-S is used to continue running tests after an aborted run.  It will
45maintain the order a standard run (ie, this assumes -r is not used).
46This is useful after the tests have prematurely stopped for some external
47reason and you want to start running from where you left off rather
48than starting from the beginning.
49
50-f reads the names of tests from the file given as f's argument, one
51or more test names per line.  Whitespace is ignored.  Blank lines and
52lines beginning with '#' are ignored.  This is especially useful for
53whittling down failures involving interactions among tests.
54
55-L causes the leaks(1) command to be run just before exit if it exists.
56leaks(1) is available on Mac OS X and presumably on some other
57FreeBSD-derived systems.
58
59-R runs each test several times and examines sys.gettotalrefcount() to
60see if the test appears to be leaking references.  The argument should
61be of the form stab:run:fname where 'stab' is the number of times the
62test is run to let gettotalrefcount settle down, 'run' is the number
63of times further it is run and 'fname' is the name of the file the
64reports are written to.  These parameters all have defaults (5, 4 and
65"reflog.txt" respectively), and the minimal invocation is '-R :'.
66
67-M runs tests that require an exorbitant amount of memory. These tests
68typically try to ascertain containers keep working when containing more than
692 billion objects, which only works on 64-bit systems. There are also some
70tests that try to exhaust the address space of the process, which only makes
71sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit,
72which is a string in the form of '2.5Gb', determines how much memory the
73tests will limit themselves to (but they may go slightly over.) The number
74shouldn't be more memory than the machine has (including swap memory). You
75should also keep in mind that swap memory is generally much, much slower
76than RAM, and setting memlimit to all available RAM or higher will heavily
77tax the machine. On the other hand, it is no use running these tests with a
78limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect
79to use more than memlimit memory will be skipped. The big-memory tests
80generally run very, very long.
81
82-u is used to specify which special resource intensive tests to run,
83such as those requiring large file support or network connectivity.
84The argument is a comma-separated list of words indicating the
85resources to test.  Currently only the following are defined:
86
87    all -       Enable all special resources.
88
89    none -      Disable all special resources (this is the default).
90
91    audio -     Tests that use the audio device.  (There are known
92                cases of broken audio drivers that can crash Python or
93                even the Linux kernel.)
94
95    curses -    Tests that use curses and will modify the terminal's
96                state and output modes.
97
98    largefile - It is okay to run some test that may create huge
99                files.  These tests can take a long time and may
100                consume >2 GiB of disk space temporarily.
101
102    network -   It is okay to run tests that use external network
103                resource, e.g. testing SSL support for sockets.
104
105    decimal -   Test the decimal module against a large suite that
106                verifies compliance with standards.
107
108    cpu -       Used for certain CPU-heavy tests.
109
110    subprocess  Run all tests for the subprocess module.
111
112    urlfetch -  It is okay to download files required on testing.
113
114    gui -       Run tests that require a running GUI.
115
116    tzdata -    Run tests that require timezone data.
117
118To enable all resources except one, use '-uall,-<resource>'.  For
119example, to run all the tests except for the gui tests, give the
120option '-uall,-gui'.
121
122--matchfile filters tests using a text file, one pattern per line.
123Pattern examples:
124
125- test method: test_stat_attributes
126- test class: FileTests
127- test identifier: test_os.FileTests.test_stat_attributes
128"""
129
130
131ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network',
132                 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui')
133
134# Other resources excluded from --use=all:
135#
136# - extralagefile (ex: test_zipfile64): really too slow to be enabled
137#   "by default"
138# - tzdata: while needed to validate fully test_datetime, it makes
139#   test_datetime too slow (15-20 min on some buildbots) and so is disabled by
140#   default (see bpo-30822).
141RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata')
142
143
144class Namespace(argparse.Namespace):
145    def __init__(self, **kwargs) -> None:
146        self.testdir = None
147        self.verbose = 0
148        self.quiet = False
149        self.exclude = False
150        self.single = False
151        self.randomize = False
152        self.fromfile = None
153        self.fail_env_changed = False
154        self.use_resources = None
155        self.trace = False
156        self.coverdir = 'coverage'
157        self.runleaks = False
158        self.huntrleaks = False
159        self.verbose2 = False
160        self.verbose3 = False
161        self.print_slow = False
162        self.random_seed = None
163        self.use_mp = None
164        self.forever = False
165        self.header = False
166        self.failfast = False
167        self.match_tests = None
168        self.ignore_tests = None
169        self.pgo = False
170        self.pgo_extended = False
171
172        super().__init__(**kwargs)
173
174
175class _ArgParser(argparse.ArgumentParser):
176
177    def error(self, message):
178        super().error(message + "\nPass -h or --help for complete help.")
179
180
181def _create_parser():
182    # Set prog to prevent the uninformative "__main__.py" from displaying in
183    # error messages when using "python -m test ...".
184    parser = _ArgParser(prog='regrtest.py',
185                        usage=USAGE,
186                        description=DESCRIPTION,
187                        epilog=EPILOG,
188                        add_help=False,
189                        formatter_class=argparse.RawDescriptionHelpFormatter)
190
191    # Arguments with this clause added to its help are described further in
192    # the epilog's "Additional option details" section.
193    more_details = '  See the section at bottom for more details.'
194
195    group = parser.add_argument_group('General options')
196    # We add help explicitly to control what argument group it renders under.
197    group.add_argument('-h', '--help', action='help',
198                       help='show this help message and exit')
199    group.add_argument('--timeout', metavar='TIMEOUT', type=float,
200                        help='dump the traceback and exit if a test takes '
201                             'more than TIMEOUT seconds; disabled if TIMEOUT '
202                             'is negative or equals to zero')
203    group.add_argument('--wait', action='store_true',
204                       help='wait for user input, e.g., allow a debugger '
205                            'to be attached')
206    group.add_argument('--worker-args', metavar='ARGS')
207    group.add_argument('-S', '--start', metavar='START',
208                       help='the name of the test at which to start.' +
209                            more_details)
210    group.add_argument('-p', '--python', metavar='PYTHON',
211                       help='Command to run Python test subprocesses with.')
212
213    group = parser.add_argument_group('Verbosity')
214    group.add_argument('-v', '--verbose', action='count',
215                       help='run tests in verbose mode with output to stdout')
216    group.add_argument('-w', '--verbose2', action='store_true',
217                       help='re-run failed tests in verbose mode')
218    group.add_argument('-W', '--verbose3', action='store_true',
219                       help='display test output on failure')
220    group.add_argument('-q', '--quiet', action='store_true',
221                       help='no output unless one or more tests fail')
222    group.add_argument('-o', '--slowest', action='store_true', dest='print_slow',
223                       help='print the slowest 10 tests')
224    group.add_argument('--header', action='store_true',
225                       help='print header with interpreter info')
226
227    group = parser.add_argument_group('Selecting tests')
228    group.add_argument('-r', '--randomize', action='store_true',
229                       help='randomize test execution order.' + more_details)
230    group.add_argument('--randseed', metavar='SEED',
231                       dest='random_seed', type=int,
232                       help='pass a random seed to reproduce a previous '
233                            'random run')
234    group.add_argument('-f', '--fromfile', metavar='FILE',
235                       help='read names of tests to run from a file.' +
236                            more_details)
237    group.add_argument('-x', '--exclude', action='store_true',
238                       help='arguments are tests to *exclude*')
239    group.add_argument('-s', '--single', action='store_true',
240                       help='single step through a set of tests.' +
241                            more_details)
242    group.add_argument('-m', '--match', metavar='PAT',
243                       dest='match_tests', action='append',
244                       help='match test cases and methods with glob pattern PAT')
245    group.add_argument('-i', '--ignore', metavar='PAT',
246                       dest='ignore_tests', action='append',
247                       help='ignore test cases and methods with glob pattern PAT')
248    group.add_argument('--matchfile', metavar='FILENAME',
249                       dest='match_filename',
250                       help='similar to --match but get patterns from a '
251                            'text file, one pattern per line')
252    group.add_argument('--ignorefile', metavar='FILENAME',
253                       dest='ignore_filename',
254                       help='similar to --matchfile but it receives patterns '
255                            'from text file to ignore')
256    group.add_argument('-G', '--failfast', action='store_true',
257                       help='fail as soon as a test fails (only with -v or -W)')
258    group.add_argument('-u', '--use', metavar='RES1,RES2,...',
259                       action='append', type=resources_list,
260                       help='specify which special resource intensive tests '
261                            'to run.' + more_details)
262    group.add_argument('-M', '--memlimit', metavar='LIMIT',
263                       help='run very large memory-consuming tests.' +
264                            more_details)
265    group.add_argument('--testdir', metavar='DIR',
266                       type=relative_filename,
267                       help='execute test files in the specified directory '
268                            '(instead of the Python stdlib test suite)')
269
270    group = parser.add_argument_group('Special runs')
271    group.add_argument('-L', '--runleaks', action='store_true',
272                       help='run the leaks(1) command just before exit.' +
273                            more_details)
274    group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS',
275                       type=huntrleaks,
276                       help='search for reference leaks (needs debug build, '
277                            'very slow).' + more_details)
278    group.add_argument('-j', '--multiprocess', metavar='PROCESSES',
279                       dest='use_mp', type=int,
280                       help='run PROCESSES processes at once')
281    group.add_argument('-T', '--coverage', action='store_true',
282                       dest='trace',
283                       help='turn on code coverage tracing using the trace '
284                            'module')
285    group.add_argument('-D', '--coverdir', metavar='DIR',
286                       type=relative_filename,
287                       help='directory where coverage files are put')
288    group.add_argument('-N', '--nocoverdir',
289                       action='store_const', const=None, dest='coverdir',
290                       help='put coverage files alongside modules')
291    group.add_argument('-t', '--threshold', metavar='THRESHOLD',
292                       type=int,
293                       help='call gc.set_threshold(THRESHOLD)')
294    group.add_argument('-n', '--nowindows', action='store_true',
295                       help='suppress error message boxes on Windows')
296    group.add_argument('-F', '--forever', action='store_true',
297                       help='run the specified tests in a loop, until an '
298                            'error happens; imply --failfast')
299    group.add_argument('--list-tests', action='store_true',
300                       help="only write the name of tests that will be run, "
301                            "don't execute them")
302    group.add_argument('--list-cases', action='store_true',
303                       help='only write the name of test cases that will be run'
304                            ' , don\'t execute them')
305    group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
306                       help='enable Profile Guided Optimization (PGO) training')
307    group.add_argument('--pgo-extended', action='store_true',
308                       help='enable extended PGO training (slower training)')
309    group.add_argument('--fail-env-changed', action='store_true',
310                       help='if a test file alters the environment, mark '
311                            'the test as failed')
312
313    group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
314                       help='writes JUnit-style XML results to the specified '
315                            'file')
316    group.add_argument('--tempdir', metavar='PATH',
317                       help='override the working directory for the test run')
318    group.add_argument('--cleanup', action='store_true',
319                       help='remove old test_python_* directories')
320    return parser
321
322
323def relative_filename(string):
324    # CWD is replaced with a temporary dir before calling main(), so we
325    # join it with the saved CWD so it ends up where the user expects.
326    return os.path.join(os_helper.SAVEDCWD, string)
327
328
329def huntrleaks(string):
330    args = string.split(':')
331    if len(args) not in (2, 3):
332        raise argparse.ArgumentTypeError(
333            'needs 2 or 3 colon-separated arguments')
334    nwarmup = int(args[0]) if args[0] else 5
335    ntracked = int(args[1]) if args[1] else 4
336    fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt'
337    return nwarmup, ntracked, fname
338
339
340def resources_list(string):
341    u = [x.lower() for x in string.split(',')]
342    for r in u:
343        if r == 'all' or r == 'none':
344            continue
345        if r[0] == '-':
346            r = r[1:]
347        if r not in RESOURCE_NAMES:
348            raise argparse.ArgumentTypeError('invalid resource: ' + r)
349    return u
350
351
352def _parse_args(args, **kwargs):
353    # Defaults
354    ns = Namespace()
355    for k, v in kwargs.items():
356        if not hasattr(ns, k):
357            raise TypeError('%r is an invalid keyword argument '
358                            'for this function' % k)
359        setattr(ns, k, v)
360    if ns.use_resources is None:
361        ns.use_resources = []
362
363    parser = _create_parser()
364    # Issue #14191: argparse doesn't support "intermixed" positional and
365    # optional arguments. Use parse_known_args() as workaround.
366    ns.args = parser.parse_known_args(args=args, namespace=ns)[1]
367    for arg in ns.args:
368        if arg.startswith('-'):
369            parser.error("unrecognized arguments: %s" % arg)
370            sys.exit(1)
371
372    if ns.single and ns.fromfile:
373        parser.error("-s and -f don't go together!")
374    if ns.use_mp is not None and ns.trace:
375        parser.error("-T and -j don't go together!")
376    if ns.python is not None:
377        if ns.use_mp is None:
378            parser.error("-p requires -j!")
379        # The "executable" may be two or more parts, e.g. "node python.js"
380        ns.python = shlex.split(ns.python)
381    if ns.failfast and not (ns.verbose or ns.verbose3):
382        parser.error("-G/--failfast needs either -v or -W")
383    if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3):
384        parser.error("--pgo/-v don't go together!")
385    if ns.pgo_extended:
386        ns.pgo = True  # pgo_extended implies pgo
387
388    if ns.nowindows:
389        print("Warning: the --nowindows (-n) option is deprecated. "
390              "Use -vv to display assertions in stderr.", file=sys.stderr)
391
392    if ns.quiet:
393        ns.verbose = 0
394    if ns.timeout is not None:
395        if ns.timeout <= 0:
396            ns.timeout = None
397    if ns.use_mp is not None:
398        if ns.use_mp <= 0:
399            # Use all cores + extras for tests that like to sleep
400            ns.use_mp = 2 + (os.cpu_count() or 1)
401    if ns.use:
402        for a in ns.use:
403            for r in a:
404                if r == 'all':
405                    ns.use_resources[:] = ALL_RESOURCES
406                    continue
407                if r == 'none':
408                    del ns.use_resources[:]
409                    continue
410                remove = False
411                if r[0] == '-':
412                    remove = True
413                    r = r[1:]
414                if remove:
415                    if r in ns.use_resources:
416                        ns.use_resources.remove(r)
417                elif r not in ns.use_resources:
418                    ns.use_resources.append(r)
419    if ns.random_seed is not None:
420        ns.randomize = True
421    if ns.verbose:
422        ns.header = True
423    if ns.huntrleaks and ns.verbose3:
424        ns.verbose3 = False
425        print("WARNING: Disable --verbose3 because it's incompatible with "
426              "--huntrleaks: see http://bugs.python.org/issue27103",
427              file=sys.stderr)
428    if ns.match_filename:
429        if ns.match_tests is None:
430            ns.match_tests = []
431        with open(ns.match_filename) as fp:
432            for line in fp:
433                ns.match_tests.append(line.strip())
434    if ns.ignore_filename:
435        if ns.ignore_tests is None:
436            ns.ignore_tests = []
437        with open(ns.ignore_filename) as fp:
438            for line in fp:
439                ns.ignore_tests.append(line.strip())
440    if ns.forever:
441        # --forever implies --failfast
442        ns.failfast = True
443
444    return ns
445