xref: /aosp_15_r20/external/pytorch/torch/testing/_internal/dynamo_test_failures.py (revision da0073e96a02ea20f0ac840b70461e3646d07c45) 
1# mypy: allow-untyped-defs
2import logging
3import os
4import sys
5
6
7# NOTE: [dynamo_test_failures.py]
8#
9# We generate xFailIfTorchDynamo* for all tests in `dynamo_expected_failures`
10# We generate skipIfTorchDynamo* for all tests in `dynamo_skips`
11#
12# For an easier-than-manual way of generating and updating these lists,
13# see scripts/compile_tests/update_failures.py
14#
15# If you're adding a new test, and it's failing PYTORCH_TEST_WITH_DYNAMO=1,
16# either add the appropriate decorators to your test or add skips for them
17# via test/dynamo_skips and test/dynamo_expected_failures.
18#
19# *These are not exactly unittest.expectedFailure and unittest.skip. We'll
20# always execute the test and then suppress the signal, if necessary.
21# If your tests crashes, or is slow, please use @skipIfTorchDynamo instead.
22#
23# The expected failure and skip files are located in test/dynamo_skips and
24# test/dynamo_expected_failures. They're individual files rather than a list so
25# git will merge changes easier.
26
27
28def find_test_dir():
29    # Find the path to the dynamo expected failure and skip files.
30    from os.path import abspath, basename, dirname, exists, join, normpath
31
32    if sys.platform == "win32":
33        return None
34
35    # Check relative to this file (local build):
36    test_dir = normpath(join(dirname(abspath(__file__)), "../../../test"))
37    if exists(join(test_dir, "dynamo_expected_failures")):
38        return test_dir
39
40    # Check relative to __main__ (installed builds relative to test file):
41    main = sys.modules["__main__"]
42    file = getattr(main, "__file__", None)
43    if file is None:
44        # Generated files do not have a module.__file__
45        return None
46    test_dir = dirname(abspath(file))
47    while dirname(test_dir) != test_dir:
48        if basename(test_dir) == "test" and exists(
49            join(test_dir, "dynamo_expected_failures")
50        ):
51            return test_dir
52        test_dir = dirname(test_dir)
53
54    # Not found
55    return None
56
57
58test_dir = find_test_dir()
59if not test_dir:
60    logger = logging.getLogger(__name__)
61    logger.warning(
62        "test/dynamo_expected_failures directory not found - known dynamo errors won't be skipped."
63    )
64
65# Tests that run without strict mode in PYTORCH_TEST_WITH_INDUCTOR=1.
66# Please don't add anything to this list.
67FIXME_inductor_non_strict = {
68    "test_modules",
69    "test_ops",
70    "test_ops_gradients",
71    "test_torch",
72}
73
74# Tests that run without resetting dynamo in PYTORCH_TEST_WITH_INDUCTOR=1.
75# Please don't add anything to this list.
76#
77# Instead we will gradually remove items from this list. Once the list is empty,
78# we will remove the list.
79FIXME_inductor_dont_reset_dynamo = {
80    "test_modules",
81    "test_ops",
82    "test_ops_gradients",
83}
84
85# We generate unittest.expectedFailure for all of the following tests
86# when run under PYTORCH_TEST_WITH_DYNAMO=1.
87# see NOTE [dynamo_test_failures.py] for more details
88#
89# This lists exists so we can more easily add large numbers of failing tests,
90if test_dir is None:
91    dynamo_expected_failures = set()
92    dynamo_skips = set()
93else:
94    failures_directory = os.path.join(test_dir, "dynamo_expected_failures")
95    skips_directory = os.path.join(test_dir, "dynamo_skips")
96
97    dynamo_expected_failures = set(os.listdir(failures_directory))
98    dynamo_skips = set(os.listdir(skips_directory))
99
100# TODO: due to case sensitivity problems, for now list these files by hand
101extra_dynamo_skips = {
102    "TestProxyTensorOpInfoCPU.test_make_fx_exhaustive_T_cpu_float32",
103    "TestProxyTensorOpInfoCPU.test_make_fx_exhaustive_t_cpu_float32",
104    "TestProxyTensorOpInfoCPU.test_make_fx_fake_exhaustive_T_cpu_float32",
105    "TestProxyTensorOpInfoCPU.test_make_fx_fake_exhaustive_t_cpu_float32",
106    "TestProxyTensorOpInfoCPU.test_make_fx_symbolic_exhaustive_T_cpu_float32",
107    "TestProxyTensorOpInfoCPU.test_make_fx_symbolic_exhaustive_t_cpu_float32",
108    "TestProxyTensorOpInfoCPU.test_make_fx_symbolic_exhaustive_inplace_T_cpu_float32",
109    "TestProxyTensorOpInfoCPU.test_make_fx_symbolic_exhaustive_inplace_t_cpu_float32",
110    "TestProxyTensorOpInfoCPU.test_make_fx_symbolic_exhaustive_out_T_cpu_float32",
111    "TestProxyTensorOpInfoCPU.test_make_fx_symbolic_exhaustive_out_t_cpu_float32",
112}
113dynamo_skips = dynamo_skips.union(extra_dynamo_skips)
114
115
116# verify some invariants
117for test in dynamo_expected_failures.union(dynamo_skips):
118    if len(test.split(".")) != 2:
119        raise AssertionError(f'Invalid test name: "{test}"')
120
121intersection = dynamo_expected_failures.intersection(dynamo_skips)
122if len(intersection) > 0:
123    raise AssertionError(
124        "there should be no overlap between dynamo_expected_failures "
125        "and dynamo_skips, got " + str(intersection)
126    )
127