1*795d594fSAndroid Build Coastguard Worker#! /usr/bin/env python3 2*795d594fSAndroid Build Coastguard Worker# 3*795d594fSAndroid Build Coastguard Worker# Copyright 2020 The Android Open Source Project 4*795d594fSAndroid Build Coastguard Worker# 5*795d594fSAndroid Build Coastguard Worker# Licensed under the Apache License, Version 2.0 (the "License"); 6*795d594fSAndroid Build Coastguard Worker# you may not use this file except in compliance with the License. 7*795d594fSAndroid Build Coastguard Worker# You may obtain a copy of the License at 8*795d594fSAndroid Build Coastguard Worker# 9*795d594fSAndroid Build Coastguard Worker# http://www.apache.org/licenses/LICENSE-2.0 10*795d594fSAndroid Build Coastguard Worker# 11*795d594fSAndroid Build Coastguard Worker# Unless required by applicable law or agreed to in writing, software 12*795d594fSAndroid Build Coastguard Worker# distributed under the License is distributed on an "AS IS" BASIS, 13*795d594fSAndroid Build Coastguard Worker# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14*795d594fSAndroid Build Coastguard Worker# See the License for the specific language governing permissions and 15*795d594fSAndroid Build Coastguard Worker# limitations under the License. 16*795d594fSAndroid Build Coastguard Worker 17*795d594fSAndroid Build Coastguard Worker# Regenerate some ART test related files. 18*795d594fSAndroid Build Coastguard Worker 19*795d594fSAndroid Build Coastguard Worker# This script handles only a subset of ART run-tests at the moment; additional 20*795d594fSAndroid Build Coastguard Worker# cases will be added later. 21*795d594fSAndroid Build Coastguard Worker 22*795d594fSAndroid Build Coastguard Workerimport argparse 23*795d594fSAndroid Build Coastguard Workerimport copy 24*795d594fSAndroid Build Coastguard Workerimport collections 25*795d594fSAndroid Build Coastguard Workerimport itertools 26*795d594fSAndroid Build Coastguard Workerimport json 27*795d594fSAndroid Build Coastguard Workerimport logging 28*795d594fSAndroid Build Coastguard Workerimport os 29*795d594fSAndroid Build Coastguard Workerimport re 30*795d594fSAndroid Build Coastguard Workerimport sys 31*795d594fSAndroid Build Coastguard Workerimport textwrap 32*795d594fSAndroid Build Coastguard Workerimport xml.dom.minidom 33*795d594fSAndroid Build Coastguard Worker 34*795d594fSAndroid Build Coastguard Workerlogging.basicConfig(format='%(levelname)s: %(message)s') 35*795d594fSAndroid Build Coastguard Worker 36*795d594fSAndroid Build Coastguard WorkerME = os.path.basename(sys.argv[0]) 37*795d594fSAndroid Build Coastguard Worker 38*795d594fSAndroid Build Coastguard Worker# Common advisory placed at the top of all generated files. 39*795d594fSAndroid Build Coastguard WorkerADVISORY = f"Generated by `{ME}`. Do not edit manually." 40*795d594fSAndroid Build Coastguard Worker 41*795d594fSAndroid Build Coastguard Worker# Default indentation unit. 42*795d594fSAndroid Build Coastguard WorkerINDENT = " " 43*795d594fSAndroid Build Coastguard Worker 44*795d594fSAndroid Build Coastguard Worker# Indentation unit for XML files. 45*795d594fSAndroid Build Coastguard WorkerXML_INDENT = " " 46*795d594fSAndroid Build Coastguard Worker 47*795d594fSAndroid Build Coastguard Workerdef reindent(str, indent = ""): 48*795d594fSAndroid Build Coastguard Worker """Reindent literal string while removing common leading spaces.""" 49*795d594fSAndroid Build Coastguard Worker return textwrap.indent(textwrap.dedent(str), indent) 50*795d594fSAndroid Build Coastguard Worker 51*795d594fSAndroid Build Coastguard Workerdef copyright_header_text(year): 52*795d594fSAndroid Build Coastguard Worker """Return the copyright header text used in XML files.""" 53*795d594fSAndroid Build Coastguard Worker return reindent(f"""\ 54*795d594fSAndroid Build Coastguard Worker Copyright (C) {year} The Android Open Source Project 55*795d594fSAndroid Build Coastguard Worker 56*795d594fSAndroid Build Coastguard Worker Licensed under the Apache License, Version 2.0 (the "License"); 57*795d594fSAndroid Build Coastguard Worker you may not use this file except in compliance with the License. 58*795d594fSAndroid Build Coastguard Worker You may obtain a copy of the License at 59*795d594fSAndroid Build Coastguard Worker 60*795d594fSAndroid Build Coastguard Worker http://www.apache.org/licenses/LICENSE-2.0 61*795d594fSAndroid Build Coastguard Worker 62*795d594fSAndroid Build Coastguard Worker Unless required by applicable law or agreed to in writing, software 63*795d594fSAndroid Build Coastguard Worker distributed under the License is distributed on an "AS IS" BASIS, 64*795d594fSAndroid Build Coastguard Worker WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 65*795d594fSAndroid Build Coastguard Worker See the License for the specific language governing permissions and 66*795d594fSAndroid Build Coastguard Worker limitations under the License. 67*795d594fSAndroid Build Coastguard Worker """, " ") 68*795d594fSAndroid Build Coastguard Worker 69*795d594fSAndroid Build Coastguard Workerdef split_list(l, n): 70*795d594fSAndroid Build Coastguard Worker """Return a list of `n` sublists of (contiguous) elements of list `l`.""" 71*795d594fSAndroid Build Coastguard Worker assert n > 0 72*795d594fSAndroid Build Coastguard Worker (d, m) = divmod(len(l), n) 73*795d594fSAndroid Build Coastguard Worker # If the length of `l` is divisible by `n`, use that that divisor (`d`) as size of each sublist; 74*795d594fSAndroid Build Coastguard Worker # otherwise, the next integer value (`d + 1`). 75*795d594fSAndroid Build Coastguard Worker s = d if m == 0 else d + 1 76*795d594fSAndroid Build Coastguard Worker result = [l[i:i + s] for i in range(0, len(l), s)] 77*795d594fSAndroid Build Coastguard Worker assert len(result) == n 78*795d594fSAndroid Build Coastguard Worker return result 79*795d594fSAndroid Build Coastguard Worker 80*795d594fSAndroid Build Coastguard Worker# The prefix used in the Soong module name of all ART run-tests. 81*795d594fSAndroid Build Coastguard WorkerART_RUN_TEST_MODULE_NAME_PREFIX = "art-run-test-" 82*795d594fSAndroid Build Coastguard Worker 83*795d594fSAndroid Build Coastguard Worker# Number of shards used to declare ART run-tests in the sharded ART MTS test plan. 84*795d594fSAndroid Build Coastguard WorkerNUM_MTS_ART_RUN_TEST_SHARDS = 1 85*795d594fSAndroid Build Coastguard Worker 86*795d594fSAndroid Build Coastguard Worker# Name of the ART MTS test list containing "eng-only" test modules, 87*795d594fSAndroid Build Coastguard Worker# which require a device-under-test running a `userdebug` or `eng` 88*795d594fSAndroid Build Coastguard Worker# build. 89*795d594fSAndroid Build Coastguard WorkerENG_ONLY_TEST_LIST_NAME = "mts-art-tests-list-eng-only" 90*795d594fSAndroid Build Coastguard Worker 91*795d594fSAndroid Build Coastguard Worker# Name of Lint baseline filename used in certain ART run-tests, 92*795d594fSAndroid Build Coastguard Worker# e.g. for the `NewApi` check (see e.g. b/268261262). 93*795d594fSAndroid Build Coastguard WorkerLINT_BASELINE_FILENAME = "lint-baseline.xml" 94*795d594fSAndroid Build Coastguard Worker 95*795d594fSAndroid Build Coastguard Worker# Curated list of tests that have a custom `run` script, but that are 96*795d594fSAndroid Build Coastguard Worker# known to work fine with the default test execution strategy (i.e. 97*795d594fSAndroid Build Coastguard Worker# when ignoring their `run` script), even if not exactly as they would 98*795d594fSAndroid Build Coastguard Worker# with the original ART run-test harness. 99*795d594fSAndroid Build Coastguard Workerrunnable_test_exceptions = frozenset([ 100*795d594fSAndroid Build Coastguard Worker "055-enum-performance", 101*795d594fSAndroid Build Coastguard Worker "059-finalizer-throw", 102*795d594fSAndroid Build Coastguard Worker "080-oom-throw", 103*795d594fSAndroid Build Coastguard Worker "133-static-invoke-super", 104*795d594fSAndroid Build Coastguard Worker "159-app-image-fields", 105*795d594fSAndroid Build Coastguard Worker "160-read-barrier-stress", 106*795d594fSAndroid Build Coastguard Worker "163-app-image-methods", 107*795d594fSAndroid Build Coastguard Worker "165-lock-owner-proxy", 108*795d594fSAndroid Build Coastguard Worker "168-vmstack-annotated", 109*795d594fSAndroid Build Coastguard Worker "176-app-image-string", 110*795d594fSAndroid Build Coastguard Worker "304-method-tracing", 111*795d594fSAndroid Build Coastguard Worker "628-vdex", 112*795d594fSAndroid Build Coastguard Worker "643-checker-bogus-ic", 113*795d594fSAndroid Build Coastguard Worker "676-proxy-jit-at-first-use", 114*795d594fSAndroid Build Coastguard Worker "677-fsi2", 115*795d594fSAndroid Build Coastguard Worker "678-quickening", 116*795d594fSAndroid Build Coastguard Worker "818-clinit-nterp", 117*795d594fSAndroid Build Coastguard Worker "821-madvise-willneed", 118*795d594fSAndroid Build Coastguard Worker "1004-checker-volatile-ref-load", 119*795d594fSAndroid Build Coastguard Worker "1338-gc-no-los", 120*795d594fSAndroid Build Coastguard Worker]) 121*795d594fSAndroid Build Coastguard Worker 122*795d594fSAndroid Build Coastguard Worker# Known slow tests, for which the timeout value is raised. 123*795d594fSAndroid Build Coastguard Workerknown_slow_tests = frozenset([ 124*795d594fSAndroid Build Coastguard Worker "080-oom-throw", 125*795d594fSAndroid Build Coastguard Worker "099-vmdebug", 126*795d594fSAndroid Build Coastguard Worker "109-suspend-check", 127*795d594fSAndroid Build Coastguard Worker "175-alloc-big-bignums", 128*795d594fSAndroid Build Coastguard Worker]) 129*795d594fSAndroid Build Coastguard Worker 130*795d594fSAndroid Build Coastguard Worker# Known failing ART run-tests. 131*795d594fSAndroid Build Coastguard Worker# TODO(rpl): Investigate and address the causes of failures. 132*795d594fSAndroid Build Coastguard Workerknown_failing_tests = frozenset([ 133*795d594fSAndroid Build Coastguard Worker "004-SignalTest", 134*795d594fSAndroid Build Coastguard Worker "004-UnsafeTest", 135*795d594fSAndroid Build Coastguard Worker "051-thread", 136*795d594fSAndroid Build Coastguard Worker "086-null-super", 137*795d594fSAndroid Build Coastguard Worker "087-gc-after-link", 138*795d594fSAndroid Build Coastguard Worker "136-daemon-jni-shutdown", 139*795d594fSAndroid Build Coastguard Worker "139-register-natives", 140*795d594fSAndroid Build Coastguard Worker "148-multithread-gc-annotations", 141*795d594fSAndroid Build Coastguard Worker "149-suspend-all-stress", 142*795d594fSAndroid Build Coastguard Worker "150-loadlibrary", 143*795d594fSAndroid Build Coastguard Worker "154-gc-loop", 144*795d594fSAndroid Build Coastguard Worker "169-threadgroup-jni", 145*795d594fSAndroid Build Coastguard Worker "177-visibly-initialized-deadlock", 146*795d594fSAndroid Build Coastguard Worker "179-nonvirtual-jni", 147*795d594fSAndroid Build Coastguard Worker "203-multi-checkpoint", 148*795d594fSAndroid Build Coastguard Worker "305-other-fault-handler", 149*795d594fSAndroid Build Coastguard Worker # 449-checker-bce: Dependency on `libarttest`. 150*795d594fSAndroid Build Coastguard Worker "449-checker-bce", 151*795d594fSAndroid Build Coastguard Worker "454-get-vreg", 152*795d594fSAndroid Build Coastguard Worker "461-get-reference-vreg", 153*795d594fSAndroid Build Coastguard Worker "466-get-live-vreg", 154*795d594fSAndroid Build Coastguard Worker "497-inlining-and-class-loader", 155*795d594fSAndroid Build Coastguard Worker "530-regression-lse", 156*795d594fSAndroid Build Coastguard Worker "555-UnsafeGetLong-regression", 157*795d594fSAndroid Build Coastguard Worker # 596-monitor-inflation: Dependency on `libarttest`. 158*795d594fSAndroid Build Coastguard Worker "596-monitor-inflation", 159*795d594fSAndroid Build Coastguard Worker "602-deoptimizeable", 160*795d594fSAndroid Build Coastguard Worker "604-hot-static-interface", 161*795d594fSAndroid Build Coastguard Worker "616-cha-native", 162*795d594fSAndroid Build Coastguard Worker "616-cha-regression-proxy-method", 163*795d594fSAndroid Build Coastguard Worker # 623-checker-loop-regressions: Dependency on `libarttest`. 164*795d594fSAndroid Build Coastguard Worker "623-checker-loop-regressions", 165*795d594fSAndroid Build Coastguard Worker "626-set-resolved-string", 166*795d594fSAndroid Build Coastguard Worker "642-fp-callees", 167*795d594fSAndroid Build Coastguard Worker "647-jni-get-field-id", 168*795d594fSAndroid Build Coastguard Worker "655-jit-clinit", 169*795d594fSAndroid Build Coastguard Worker "656-loop-deopt", 170*795d594fSAndroid Build Coastguard Worker "664-aget-verifier", 171*795d594fSAndroid Build Coastguard Worker # 680-checker-deopt-dex-pc-0: Dependency on `libarttest`. 172*795d594fSAndroid Build Coastguard Worker "680-checker-deopt-dex-pc-0", 173*795d594fSAndroid Build Coastguard Worker "685-deoptimizeable", 174*795d594fSAndroid Build Coastguard Worker "687-deopt", 175*795d594fSAndroid Build Coastguard Worker "693-vdex-inmem-loader-evict", 176*795d594fSAndroid Build Coastguard Worker "708-jit-cache-churn", 177*795d594fSAndroid Build Coastguard Worker # 716-jli-jit-samples: Dependency on `libarttest`. 178*795d594fSAndroid Build Coastguard Worker "716-jli-jit-samples", 179*795d594fSAndroid Build Coastguard Worker "717-integer-value-of", 180*795d594fSAndroid Build Coastguard Worker "720-thread-priority", 181*795d594fSAndroid Build Coastguard Worker # 730-cha-deopt: Fails with: 182*795d594fSAndroid Build Coastguard Worker # 183*795d594fSAndroid Build Coastguard Worker # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 184*795d594fSAndroid Build Coastguard Worker # at Main.main(Main.java:24) 185*795d594fSAndroid Build Coastguard Worker # 186*795d594fSAndroid Build Coastguard Worker "730-cha-deopt", 187*795d594fSAndroid Build Coastguard Worker # 813-fp-args: Dependency on `libarttest`. 188*795d594fSAndroid Build Coastguard Worker "813-fp-args", 189*795d594fSAndroid Build Coastguard Worker # 821-many-args: Fails with: 190*795d594fSAndroid Build Coastguard Worker # 191*795d594fSAndroid Build Coastguard Worker # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 192*795d594fSAndroid Build Coastguard Worker # at Main.main(Main.java:20) 193*795d594fSAndroid Build Coastguard Worker # 194*795d594fSAndroid Build Coastguard Worker "821-many-args", 195*795d594fSAndroid Build Coastguard Worker # 823-cha-inlining: Dependency on `libarttest`. 196*795d594fSAndroid Build Coastguard Worker "823-cha-inlining", 197*795d594fSAndroid Build Coastguard Worker # 826-infinite-loop: The test expects an argument passed to `Main.main` (the test library, 198*795d594fSAndroid Build Coastguard Worker # usually `arttestd` or `arttest)`, but the ART run-test TradeFed test runner 199*795d594fSAndroid Build Coastguard Worker # (`com.android.tradefed.testtype.ArtRunTest`) does not implement this yet. 200*795d594fSAndroid Build Coastguard Worker "826-infinite-loop", 201*795d594fSAndroid Build Coastguard Worker # 832-cha-recursive: Dependency on `libarttest`. 202*795d594fSAndroid Build Coastguard Worker "832-cha-recursive", 203*795d594fSAndroid Build Coastguard Worker # 837-deopt: Dependency on `libarttest`. 204*795d594fSAndroid Build Coastguard Worker "837-deopt", 205*795d594fSAndroid Build Coastguard Worker # 844-exception: Dependency on `libarttest`. 206*795d594fSAndroid Build Coastguard Worker "844-exception", 207*795d594fSAndroid Build Coastguard Worker # 844-exception2: Dependency on `libarttest`. 208*795d594fSAndroid Build Coastguard Worker "844-exception2", 209*795d594fSAndroid Build Coastguard Worker # 966-default-conflict: Dependency on `libarttest`. 210*795d594fSAndroid Build Coastguard Worker "966-default-conflict", 211*795d594fSAndroid Build Coastguard Worker # These tests need native code. 212*795d594fSAndroid Build Coastguard Worker "993-breakpoints-non-debuggable", 213*795d594fSAndroid Build Coastguard Worker # 1002-notify-startup: Dependency on `libarttest` + custom `check` script. 214*795d594fSAndroid Build Coastguard Worker "1002-notify-startup", 215*795d594fSAndroid Build Coastguard Worker "1337-gc-coverage", 216*795d594fSAndroid Build Coastguard Worker "1339-dead-reference-safe", 217*795d594fSAndroid Build Coastguard Worker "1945-proxy-method-arguments", 218*795d594fSAndroid Build Coastguard Worker "2011-stack-walk-concurrent-instrument", 219*795d594fSAndroid Build Coastguard Worker "2033-shutdown-mechanics", 220*795d594fSAndroid Build Coastguard Worker "2036-jni-filechannel", 221*795d594fSAndroid Build Coastguard Worker "2037-thread-name-inherit", 222*795d594fSAndroid Build Coastguard Worker # 2040-huge-native-alloc: Fails with: 223*795d594fSAndroid Build Coastguard Worker # 224*795d594fSAndroid Build Coastguard Worker # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 225*795d594fSAndroid Build Coastguard Worker # at Main.main(Main.java:56) 226*795d594fSAndroid Build Coastguard Worker # 227*795d594fSAndroid Build Coastguard Worker "2040-huge-native-alloc", 228*795d594fSAndroid Build Coastguard Worker "2235-JdkUnsafeTest", 229*795d594fSAndroid Build Coastguard Worker "2243-single-step-default", 230*795d594fSAndroid Build Coastguard Worker "2262-miranda-methods", 231*795d594fSAndroid Build Coastguard Worker "2262-default-conflict-methods", 232*795d594fSAndroid Build Coastguard Worker # 2275-pthread-name: Dependency on `libarttest`. 233*795d594fSAndroid Build Coastguard Worker "2275-pthread-name", 234*795d594fSAndroid Build Coastguard Worker]) 235*795d594fSAndroid Build Coastguard Worker 236*795d594fSAndroid Build Coastguard Worker# These ART run-tests are new and have not had enough post-submit runs 237*795d594fSAndroid Build Coastguard Worker# to meet pre-submit SLOs. Monitor their post-submit runs before 238*795d594fSAndroid Build Coastguard Worker# removing them from this set (in order to promote them to 239*795d594fSAndroid Build Coastguard Worker# presubmits). 240*795d594fSAndroid Build Coastguard Workerpostsubmit_only_tests = frozenset([ 241*795d594fSAndroid Build Coastguard Worker]) 242*795d594fSAndroid Build Coastguard Worker 243*795d594fSAndroid Build Coastguard Workerknown_failing_on_hwasan_tests = frozenset([ 244*795d594fSAndroid Build Coastguard Worker "CtsJdwpTestCases", # times out 245*795d594fSAndroid Build Coastguard Worker # apexd fails to unmount com.android.runtime on ASan builds. 246*795d594fSAndroid Build Coastguard Worker "art_standalone_dexopt_chroot_setup_tests", 247*795d594fSAndroid Build Coastguard Worker]) 248*795d594fSAndroid Build Coastguard Worker 249*795d594fSAndroid Build Coastguard Worker# ART gtests that do not need root access to the device. 250*795d594fSAndroid Build Coastguard Workerart_gtest_user_module_names = [ 251*795d594fSAndroid Build Coastguard Worker "art_libnativebridge_cts_tests", 252*795d594fSAndroid Build Coastguard Worker "art_standalone_artd_tests", 253*795d594fSAndroid Build Coastguard Worker "art_standalone_cmdline_tests", 254*795d594fSAndroid Build Coastguard Worker "art_standalone_compiler_tests", 255*795d594fSAndroid Build Coastguard Worker "art_standalone_dex2oat_cts_tests", 256*795d594fSAndroid Build Coastguard Worker "art_standalone_dex2oat_tests", 257*795d594fSAndroid Build Coastguard Worker "art_standalone_dexdump_tests", 258*795d594fSAndroid Build Coastguard Worker "art_standalone_dexlist_tests", 259*795d594fSAndroid Build Coastguard Worker "art_standalone_libartbase_tests", 260*795d594fSAndroid Build Coastguard Worker "art_standalone_libartpalette_tests", 261*795d594fSAndroid Build Coastguard Worker "art_standalone_libartservice_tests", 262*795d594fSAndroid Build Coastguard Worker "art_standalone_libarttools_tests", 263*795d594fSAndroid Build Coastguard Worker "art_standalone_libdexfile_support_tests", 264*795d594fSAndroid Build Coastguard Worker "art_standalone_libdexfile_tests", 265*795d594fSAndroid Build Coastguard Worker "art_standalone_libprofile_tests", 266*795d594fSAndroid Build Coastguard Worker "art_standalone_oatdump_tests", 267*795d594fSAndroid Build Coastguard Worker "art_standalone_odrefresh_tests", 268*795d594fSAndroid Build Coastguard Worker "art_standalone_runtime_tests", 269*795d594fSAndroid Build Coastguard Worker "art_standalone_sigchain_tests", 270*795d594fSAndroid Build Coastguard Worker "libnativebridge-lazy-tests", 271*795d594fSAndroid Build Coastguard Worker "libnativebridge-tests", 272*795d594fSAndroid Build Coastguard Worker "libnativeloader_test", 273*795d594fSAndroid Build Coastguard Worker] 274*795d594fSAndroid Build Coastguard Worker 275*795d594fSAndroid Build Coastguard Worker# ART gtests that need root access to the device. 276*795d594fSAndroid Build Coastguard Workerart_gtest_eng_only_module_names = [ 277*795d594fSAndroid Build Coastguard Worker "art_standalone_dexopt_chroot_setup_tests", 278*795d594fSAndroid Build Coastguard Worker "art_standalone_dexoptanalyzer_tests", 279*795d594fSAndroid Build Coastguard Worker "art_standalone_profman_tests", 280*795d594fSAndroid Build Coastguard Worker "libnativeloader_e2e_tests", 281*795d594fSAndroid Build Coastguard Worker] 282*795d594fSAndroid Build Coastguard Worker 283*795d594fSAndroid Build Coastguard Worker# All supported ART gtests. 284*795d594fSAndroid Build Coastguard Workerart_gtest_module_names = sorted(art_gtest_user_module_names + art_gtest_eng_only_module_names) 285*795d594fSAndroid Build Coastguard Worker 286*795d594fSAndroid Build Coastguard Worker# These ART gtests are new and have not had enough post-submit runs 287*795d594fSAndroid Build Coastguard Worker# to meet pre-submit SLOs. Monitor their post-submit runs before 288*795d594fSAndroid Build Coastguard Worker# removing them from this set (in order to promote them to 289*795d594fSAndroid Build Coastguard Worker# presubmits). 290*795d594fSAndroid Build Coastguard Workerart_gtest_postsubmit_only_module_names = [ 291*795d594fSAndroid Build Coastguard Worker] 292*795d594fSAndroid Build Coastguard Worker 293*795d594fSAndroid Build Coastguard Worker# ART gtests not supported in MTS. 294*795d594fSAndroid Build Coastguard Workerart_gtest_modules_excluded_from_mts = [ 295*795d594fSAndroid Build Coastguard Worker # TODO(b/347717488): Consider adding this test to ART MTS. 296*795d594fSAndroid Build Coastguard Worker "libnativebridge-tests", 297*795d594fSAndroid Build Coastguard Worker] 298*795d594fSAndroid Build Coastguard Worker 299*795d594fSAndroid Build Coastguard Worker# ART gtests supported in MTS that do not need root access to the device. 300*795d594fSAndroid Build Coastguard Workerart_gtest_mts_user_module_names = [t for t in art_gtest_user_module_names 301*795d594fSAndroid Build Coastguard Worker if t not in art_gtest_modules_excluded_from_mts] 302*795d594fSAndroid Build Coastguard Worker 303*795d594fSAndroid Build Coastguard Worker# ART gtests supported in presubmits. 304*795d594fSAndroid Build Coastguard Workerart_gtest_presubmit_module_names = [t for t in art_gtest_module_names 305*795d594fSAndroid Build Coastguard Worker if t not in art_gtest_postsubmit_only_module_names] 306*795d594fSAndroid Build Coastguard Worker 307*795d594fSAndroid Build Coastguard Worker# ART gtests supported in Mainline presubmits. 308*795d594fSAndroid Build Coastguard Workerart_gtest_mainline_presubmit_module_names = copy.copy(art_gtest_presubmit_module_names) 309*795d594fSAndroid Build Coastguard Worker 310*795d594fSAndroid Build Coastguard Worker# ART gtests supported in postsubmits. 311*795d594fSAndroid Build Coastguard Workerunknown_art_gtest_postsubmit_only_module_names = [t for t in art_gtest_postsubmit_only_module_names 312*795d594fSAndroid Build Coastguard Worker if t not in art_gtest_module_names] 313*795d594fSAndroid Build Coastguard Workerif unknown_art_gtest_postsubmit_only_module_names: 314*795d594fSAndroid Build Coastguard Worker logging.error(textwrap.dedent("""\ 315*795d594fSAndroid Build Coastguard Worker The following `art_gtest_postsubmit_only_module_names` elements are not part of 316*795d594fSAndroid Build Coastguard Worker `art_gtest_module_names`: """) + str(unknown_art_gtest_postsubmit_only_module_names)) 317*795d594fSAndroid Build Coastguard Worker sys.exit(1) 318*795d594fSAndroid Build Coastguard Workerart_gtest_postsubmit_module_names = copy.copy(art_gtest_postsubmit_only_module_names) 319*795d594fSAndroid Build Coastguard Worker 320*795d594fSAndroid Build Coastguard Worker# Tests exhibiting a flaky behavior, currently exluded from MTS for 321*795d594fSAndroid Build Coastguard Worker# the stake of stability / confidence (b/209958457). 322*795d594fSAndroid Build Coastguard Workerflaky_tests_excluded_from_mts = { 323*795d594fSAndroid Build Coastguard Worker "CtsLibcoreFileIOTestCases": [ 324*795d594fSAndroid Build Coastguard Worker ("android.cts.FileChannelInterProcessLockTest#" + m) for m in [ 325*795d594fSAndroid Build Coastguard Worker "test_lockJJZ_Exclusive_asyncChannel", 326*795d594fSAndroid Build Coastguard Worker "test_lockJJZ_Exclusive_syncChannel", 327*795d594fSAndroid Build Coastguard Worker "test_lock_differentChannelTypes", 328*795d594fSAndroid Build Coastguard Worker "test_lockJJZ_Shared_asyncChannel", 329*795d594fSAndroid Build Coastguard Worker "test_lockJJZ_Shared_syncChannel", 330*795d594fSAndroid Build Coastguard Worker ] 331*795d594fSAndroid Build Coastguard Worker ], 332*795d594fSAndroid Build Coastguard Worker "CtsLibcoreTestCases": [ 333*795d594fSAndroid Build Coastguard Worker ("com.android.org.conscrypt.javax.net.ssl.SSLSocketVersionCompatibilityTest#" + m + c) 334*795d594fSAndroid Build Coastguard Worker for (m, c) in itertools.product( 335*795d594fSAndroid Build Coastguard Worker [ 336*795d594fSAndroid Build Coastguard Worker "test_SSLSocket_interrupt_read_withoutAutoClose", 337*795d594fSAndroid Build Coastguard Worker "test_SSLSocket_setSoWriteTimeout", 338*795d594fSAndroid Build Coastguard Worker ], 339*795d594fSAndroid Build Coastguard Worker [ 340*795d594fSAndroid Build Coastguard Worker "[0: TLSv1.2 client, TLSv1.2 server]", 341*795d594fSAndroid Build Coastguard Worker "[1: TLSv1.2 client, TLSv1.3 server]", 342*795d594fSAndroid Build Coastguard Worker "[2: TLSv1.3 client, TLSv1.2 server]", 343*795d594fSAndroid Build Coastguard Worker "[3: TLSv1.3 client, TLSv1.3 server]", 344*795d594fSAndroid Build Coastguard Worker ] 345*795d594fSAndroid Build Coastguard Worker ) 346*795d594fSAndroid Build Coastguard Worker ] + [ 347*795d594fSAndroid Build Coastguard Worker ("libcore.dalvik.system.DelegateLastClassLoaderTest#" + m) for m in [ 348*795d594fSAndroid Build Coastguard Worker "testLookupOrderNodelegate_getResource", 349*795d594fSAndroid Build Coastguard Worker "testLookupOrder_getResource", 350*795d594fSAndroid Build Coastguard Worker ] 351*795d594fSAndroid Build Coastguard Worker ] 352*795d594fSAndroid Build Coastguard Worker} 353*795d594fSAndroid Build Coastguard Worker 354*795d594fSAndroid Build Coastguard Worker# Tests excluded from all test mapping test groups. 355*795d594fSAndroid Build Coastguard Worker# 356*795d594fSAndroid Build Coastguard Worker# Example of admissible values in this dictionary: 357*795d594fSAndroid Build Coastguard Worker# 358*795d594fSAndroid Build Coastguard Worker# "art_standalone_cmdline_tests": ["CmdlineParserTest#TestCompilerOption"], 359*795d594fSAndroid Build Coastguard Worker# "art_standalone_dexopt_chroot_setup_tests": ["DexoptChrootSetupTest#HelloWorld"], 360*795d594fSAndroid Build Coastguard Worker# 361*795d594fSAndroid Build Coastguard Workerfailing_tests_excluded_from_test_mapping = { 362*795d594fSAndroid Build Coastguard Worker # Empty. 363*795d594fSAndroid Build Coastguard Worker} 364*795d594fSAndroid Build Coastguard Worker 365*795d594fSAndroid Build Coastguard Worker# Tests failing because of linking issues, currently exluded from MTS 366*795d594fSAndroid Build Coastguard Worker# and Mainline Presubmits to minimize noise in continuous runs while 367*795d594fSAndroid Build Coastguard Worker# we investigate. 368*795d594fSAndroid Build Coastguard Worker# 369*795d594fSAndroid Build Coastguard Worker# Example of admissible values in this dictionary: same as for 370*795d594fSAndroid Build Coastguard Worker# `failing_tests_excluded_from_test_mapping` (see above). 371*795d594fSAndroid Build Coastguard Worker# 372*795d594fSAndroid Build Coastguard Worker# TODO(b/247108425): Address the linking issues and re-enable these 373*795d594fSAndroid Build Coastguard Worker# tests. 374*795d594fSAndroid Build Coastguard Workerfailing_tests_excluded_from_mts_and_mainline_presubmits = { 375*795d594fSAndroid Build Coastguard Worker "art_standalone_compiler_tests": ["JniCompilerTest*"], 376*795d594fSAndroid Build Coastguard Worker "art_standalone_libartpalette_tests": ["PaletteClientJniTest*"], 377*795d594fSAndroid Build Coastguard Worker} 378*795d594fSAndroid Build Coastguard Worker 379*795d594fSAndroid Build Coastguard Workerfailing_tests_excluded_from_mainline_presubmits = ( 380*795d594fSAndroid Build Coastguard Worker failing_tests_excluded_from_test_mapping | 381*795d594fSAndroid Build Coastguard Worker failing_tests_excluded_from_mts_and_mainline_presubmits 382*795d594fSAndroid Build Coastguard Worker) 383*795d594fSAndroid Build Coastguard Worker 384*795d594fSAndroid Build Coastguard Worker# Is `run_test` a Checker test (i.e. a test containing Checker 385*795d594fSAndroid Build Coastguard Worker# assertions)? 386*795d594fSAndroid Build Coastguard Workerdef is_checker_test(run_test): 387*795d594fSAndroid Build Coastguard Worker return re.match("^[0-9]+-checker-", run_test) 388*795d594fSAndroid Build Coastguard Worker 389*795d594fSAndroid Build Coastguard Workerdef gen_mts_test_list_file(tests, test_list_file, copyright_year, configuration_description, 390*795d594fSAndroid Build Coastguard Worker tests_description, comments = []): 391*795d594fSAndroid Build Coastguard Worker """Generate an ART MTS test list file.""" 392*795d594fSAndroid Build Coastguard Worker root = xml.dom.minidom.Document() 393*795d594fSAndroid Build Coastguard Worker 394*795d594fSAndroid Build Coastguard Worker advisory_header = root.createComment(f" {ADVISORY} ") 395*795d594fSAndroid Build Coastguard Worker root.appendChild(advisory_header) 396*795d594fSAndroid Build Coastguard Worker copyright_header = root.createComment(copyright_header_text(copyright_year)) 397*795d594fSAndroid Build Coastguard Worker root.appendChild(copyright_header) 398*795d594fSAndroid Build Coastguard Worker 399*795d594fSAndroid Build Coastguard Worker configuration = root.createElement("configuration") 400*795d594fSAndroid Build Coastguard Worker root.appendChild(configuration) 401*795d594fSAndroid Build Coastguard Worker configuration.setAttribute("description", configuration_description) 402*795d594fSAndroid Build Coastguard Worker 403*795d594fSAndroid Build Coastguard Worker def append_option(name, value): 404*795d594fSAndroid Build Coastguard Worker option = root.createElement("option") 405*795d594fSAndroid Build Coastguard Worker option.setAttribute("name", name) 406*795d594fSAndroid Build Coastguard Worker option.setAttribute("value", value) 407*795d594fSAndroid Build Coastguard Worker configuration.appendChild(option) 408*795d594fSAndroid Build Coastguard Worker 409*795d594fSAndroid Build Coastguard Worker def append_comment(comment): 410*795d594fSAndroid Build Coastguard Worker xml_comment = root.createComment(f" {comment} ") 411*795d594fSAndroid Build Coastguard Worker configuration.appendChild(xml_comment) 412*795d594fSAndroid Build Coastguard Worker 413*795d594fSAndroid Build Coastguard Worker # Test declarations. 414*795d594fSAndroid Build Coastguard Worker # ------------------ 415*795d594fSAndroid Build Coastguard Worker 416*795d594fSAndroid Build Coastguard Worker test_declarations_comments = [tests_description + "."] 417*795d594fSAndroid Build Coastguard Worker test_declarations_comments.extend(comments) 418*795d594fSAndroid Build Coastguard Worker for c in test_declarations_comments: 419*795d594fSAndroid Build Coastguard Worker append_comment(c) 420*795d594fSAndroid Build Coastguard Worker for t in tests: 421*795d594fSAndroid Build Coastguard Worker append_option("compatibility:include-filter", t) 422*795d594fSAndroid Build Coastguard Worker 423*795d594fSAndroid Build Coastguard Worker # `MainlineTestModuleController` configurations. 424*795d594fSAndroid Build Coastguard Worker # ---------------------------------------------- 425*795d594fSAndroid Build Coastguard Worker 426*795d594fSAndroid Build Coastguard Worker module_controller_configuration_comments = [ 427*795d594fSAndroid Build Coastguard Worker f"Enable MainlineTestModuleController for {tests_description}."] 428*795d594fSAndroid Build Coastguard Worker module_controller_configuration_comments.extend(comments) 429*795d594fSAndroid Build Coastguard Worker for c in module_controller_configuration_comments: 430*795d594fSAndroid Build Coastguard Worker append_comment(c) 431*795d594fSAndroid Build Coastguard Worker for t in tests: 432*795d594fSAndroid Build Coastguard Worker append_option("compatibility:module-arg", f"{t}:enable:true") 433*795d594fSAndroid Build Coastguard Worker for t in tests: 434*795d594fSAndroid Build Coastguard Worker if t in ["CtsLibcoreTestCases", "CtsLibcoreOjTestCases"]: 435*795d594fSAndroid Build Coastguard Worker append_comment("core-test-mode=mts tells ExpectationBasedFilter to exclude @NonMts Tests") 436*795d594fSAndroid Build Coastguard Worker append_option("compatibility:module-arg", f"{t}:instrumentation-arg:core-test-mode:=mts") 437*795d594fSAndroid Build Coastguard Worker 438*795d594fSAndroid Build Coastguard Worker xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 439*795d594fSAndroid Build Coastguard Worker 440*795d594fSAndroid Build Coastguard Worker with open(test_list_file, "wb") as f: 441*795d594fSAndroid Build Coastguard Worker logging.debug(f"Writing `{test_list_file}`.") 442*795d594fSAndroid Build Coastguard Worker f.write(xml_str) 443*795d594fSAndroid Build Coastguard Worker 444*795d594fSAndroid Build Coastguard Workerclass Generator: 445*795d594fSAndroid Build Coastguard Worker def __init__(self, top_dir): 446*795d594fSAndroid Build Coastguard Worker """Generator of ART test files for an Android source tree anchored at `top_dir`.""" 447*795d594fSAndroid Build Coastguard Worker # Path to the Android top source tree. 448*795d594fSAndroid Build Coastguard Worker self.top_dir = top_dir 449*795d594fSAndroid Build Coastguard Worker # Path to the ART directory 450*795d594fSAndroid Build Coastguard Worker self.art_dir = os.path.join(top_dir, "art") 451*795d594fSAndroid Build Coastguard Worker # Path to the ART tests top-level directory. 452*795d594fSAndroid Build Coastguard Worker self.art_test_dir = os.path.join(self.art_dir, "test") 453*795d594fSAndroid Build Coastguard Worker # Path to the MTS configuration directory. 454*795d594fSAndroid Build Coastguard Worker self.mts_config_dir = os.path.join( 455*795d594fSAndroid Build Coastguard Worker top_dir, "test", "mts", "tools", "mts-tradefed", "res", "config") 456*795d594fSAndroid Build Coastguard Worker # Path to the ART JVM TI CTS tests top-level directory. 457*795d594fSAndroid Build Coastguard Worker self.jvmti_cts_test_dir = os.path.join(top_dir, "cts/hostsidetests/jvmti/run-tests") 458*795d594fSAndroid Build Coastguard Worker 459*795d594fSAndroid Build Coastguard Worker # Return the list of ART run-tests (in short form, i.e. `001-HelloWorld`, 460*795d594fSAndroid Build Coastguard Worker # not `art-run-test-001-HelloWorld`). 461*795d594fSAndroid Build Coastguard Worker def enumerate_run_tests(self): 462*795d594fSAndroid Build Coastguard Worker return sorted([run_test 463*795d594fSAndroid Build Coastguard Worker for run_test in os.listdir(self.art_test_dir) 464*795d594fSAndroid Build Coastguard Worker if re.match("^[0-9]{3,}-", run_test)]) 465*795d594fSAndroid Build Coastguard Worker 466*795d594fSAndroid Build Coastguard Worker # Return the list of ART JVM TI CTS tests. 467*795d594fSAndroid Build Coastguard Worker def enumerate_jvmti_cts_tests(self): 468*795d594fSAndroid Build Coastguard Worker return sorted([re.sub(r"test-(\d+)", r"CtsJvmtiRunTest\1HostTestCases", cts_jvmti_test_dir) 469*795d594fSAndroid Build Coastguard Worker for cts_jvmti_test_dir in os.listdir(self.jvmti_cts_test_dir) 470*795d594fSAndroid Build Coastguard Worker if re.match(r"^test-\d+$", cts_jvmti_test_dir)]) 471*795d594fSAndroid Build Coastguard Worker 472*795d594fSAndroid Build Coastguard Worker # Return the metadata of a test, if any. 473*795d594fSAndroid Build Coastguard Worker def get_test_metadata(self, run_test): 474*795d594fSAndroid Build Coastguard Worker run_test_path = os.path.join(self.art_test_dir, run_test) 475*795d594fSAndroid Build Coastguard Worker metadata_file = os.path.join(run_test_path, "test-metadata.json") 476*795d594fSAndroid Build Coastguard Worker metadata = {} 477*795d594fSAndroid Build Coastguard Worker if os.path.exists(metadata_file): 478*795d594fSAndroid Build Coastguard Worker with open(metadata_file, "r") as f: 479*795d594fSAndroid Build Coastguard Worker try: 480*795d594fSAndroid Build Coastguard Worker metadata = json.load(f) 481*795d594fSAndroid Build Coastguard Worker except json.decoder.JSONDecodeError: 482*795d594fSAndroid Build Coastguard Worker logging.error(f"Unable to parse test metadata file `{metadata_file}`") 483*795d594fSAndroid Build Coastguard Worker raise 484*795d594fSAndroid Build Coastguard Worker return metadata 485*795d594fSAndroid Build Coastguard Worker 486*795d594fSAndroid Build Coastguard Worker # Can the build script of `run_test` be safely ignored? 487*795d594fSAndroid Build Coastguard Worker def can_ignore_build_script(self, run_test): 488*795d594fSAndroid Build Coastguard Worker # Check whether there are test metadata with build parameters 489*795d594fSAndroid Build Coastguard Worker # enabling us to safely ignore the build script. 490*795d594fSAndroid Build Coastguard Worker metadata = self.get_test_metadata(run_test) 491*795d594fSAndroid Build Coastguard Worker build_param = metadata.get("build-param", {}) 492*795d594fSAndroid Build Coastguard Worker # Ignore build scripts that are just about preventing building for 493*795d594fSAndroid Build Coastguard Worker # the JVM and/or using VarHandles (Soong builds JARs with 494*795d594fSAndroid Build Coastguard Worker # VarHandle support by default (i.e. by using an API level greater 495*795d594fSAndroid Build Coastguard Worker # or equal to 28), so we can ignore build scripts that just 496*795d594fSAndroid Build Coastguard Worker # request support for this feature.) 497*795d594fSAndroid Build Coastguard Worker experimental_var_handles = {"experimental": "var-handles"} 498*795d594fSAndroid Build Coastguard Worker jvm_supported_false = {"jvm-supported": "false"} 499*795d594fSAndroid Build Coastguard Worker if (build_param == experimental_var_handles or 500*795d594fSAndroid Build Coastguard Worker build_param == jvm_supported_false or 501*795d594fSAndroid Build Coastguard Worker build_param == experimental_var_handles | jvm_supported_false): 502*795d594fSAndroid Build Coastguard Worker return True 503*795d594fSAndroid Build Coastguard Worker return False 504*795d594fSAndroid Build Coastguard Worker 505*795d594fSAndroid Build Coastguard Worker # Can `run_test` be built with Soong? 506*795d594fSAndroid Build Coastguard Worker # TODO(b/147814778): Add build support for more tests. 507*795d594fSAndroid Build Coastguard Worker def is_soong_buildable(self, run_test): 508*795d594fSAndroid Build Coastguard Worker run_test_path = os.path.join(self.art_test_dir, run_test) 509*795d594fSAndroid Build Coastguard Worker 510*795d594fSAndroid Build Coastguard Worker # Skip tests with non-default build rules, unless these build 511*795d594fSAndroid Build Coastguard Worker # rules can be safely ignored. 512*795d594fSAndroid Build Coastguard Worker if (os.path.isfile(os.path.join(run_test_path, "generate-sources")) or 513*795d594fSAndroid Build Coastguard Worker os.path.isfile(os.path.join(run_test_path, "javac_post.sh"))): 514*795d594fSAndroid Build Coastguard Worker return False 515*795d594fSAndroid Build Coastguard Worker if os.path.isfile(os.path.join(run_test_path, "build.py")): 516*795d594fSAndroid Build Coastguard Worker if not self.can_ignore_build_script(run_test): 517*795d594fSAndroid Build Coastguard Worker return False 518*795d594fSAndroid Build Coastguard Worker # Skip tests with sources outside the `src` directory. 519*795d594fSAndroid Build Coastguard Worker for subdir in ["jasmin", 520*795d594fSAndroid Build Coastguard Worker "jasmin-multidex", 521*795d594fSAndroid Build Coastguard Worker "smali", 522*795d594fSAndroid Build Coastguard Worker "smali-ex", 523*795d594fSAndroid Build Coastguard Worker "smali-multidex", 524*795d594fSAndroid Build Coastguard Worker "src-aotex", 525*795d594fSAndroid Build Coastguard Worker "src-bcpex", 526*795d594fSAndroid Build Coastguard Worker "src-ex", 527*795d594fSAndroid Build Coastguard Worker "src-ex2", 528*795d594fSAndroid Build Coastguard Worker "src-multidex"]: 529*795d594fSAndroid Build Coastguard Worker if os.path.isdir(os.path.join(run_test_path, subdir)): 530*795d594fSAndroid Build Coastguard Worker return False 531*795d594fSAndroid Build Coastguard Worker # Skip tests that have both an `src` directory and an `src-art` directory. 532*795d594fSAndroid Build Coastguard Worker if os.path.isdir(os.path.join(run_test_path, "src")) and \ 533*795d594fSAndroid Build Coastguard Worker os.path.isdir(os.path.join(run_test_path, "src-art")): 534*795d594fSAndroid Build Coastguard Worker return False 535*795d594fSAndroid Build Coastguard Worker # Skip tests that have neither an `src` directory nor an `src-art` directory. 536*795d594fSAndroid Build Coastguard Worker if not os.path.isdir(os.path.join(run_test_path, "src")) and \ 537*795d594fSAndroid Build Coastguard Worker not os.path.isdir(os.path.join(run_test_path, "src-art")): 538*795d594fSAndroid Build Coastguard Worker return False 539*795d594fSAndroid Build Coastguard Worker # Skip test with a copy of `sun.misc.Unsafe`. 540*795d594fSAndroid Build Coastguard Worker if os.path.isfile(os.path.join(run_test_path, "src", "sun", "misc", "Unsafe.java")): 541*795d594fSAndroid Build Coastguard Worker return False 542*795d594fSAndroid Build Coastguard Worker # Skip tests with Hidden API specs. 543*795d594fSAndroid Build Coastguard Worker if os.path.isfile(os.path.join(run_test_path, "hiddenapi-flags.csv")): 544*795d594fSAndroid Build Coastguard Worker return False 545*795d594fSAndroid Build Coastguard Worker # All other tests are considered buildable. 546*795d594fSAndroid Build Coastguard Worker return True 547*795d594fSAndroid Build Coastguard Worker 548*795d594fSAndroid Build Coastguard Worker # Can the run script of `run_test` be safely ignored? 549*795d594fSAndroid Build Coastguard Worker def can_ignore_run_script(self, run_test): 550*795d594fSAndroid Build Coastguard Worker # Unconditionally consider some identified tests that have a 551*795d594fSAndroid Build Coastguard Worker # (not-yet-handled) custom `run` script as runnable. 552*795d594fSAndroid Build Coastguard Worker # 553*795d594fSAndroid Build Coastguard Worker # TODO(rpl): Get rid of this exception mechanism by supporting 554*795d594fSAndroid Build Coastguard Worker # these tests' `run` scripts properly. 555*795d594fSAndroid Build Coastguard Worker if run_test in runnable_test_exceptions: 556*795d594fSAndroid Build Coastguard Worker return True 557*795d594fSAndroid Build Coastguard Worker # Check whether there are test metadata with run parameters 558*795d594fSAndroid Build Coastguard Worker # enabling us to safely ignore the run script. 559*795d594fSAndroid Build Coastguard Worker metadata = self.get_test_metadata(run_test) 560*795d594fSAndroid Build Coastguard Worker run_param = metadata.get("run-param", {}) 561*795d594fSAndroid Build Coastguard Worker if run_param.get("default-run", ""): 562*795d594fSAndroid Build Coastguard Worker return True 563*795d594fSAndroid Build Coastguard Worker return False 564*795d594fSAndroid Build Coastguard Worker 565*795d594fSAndroid Build Coastguard Worker # Generate a Blueprint property group as a string, i.e. something looking like 566*795d594fSAndroid Build Coastguard Worker # this: 567*795d594fSAndroid Build Coastguard Worker # 568*795d594fSAndroid Build Coastguard Worker # ``` 569*795d594fSAndroid Build Coastguard Worker # <group_name>: { 570*795d594fSAndroid Build Coastguard Worker # <key0>: "<value0>", 571*795d594fSAndroid Build Coastguard Worker # ... 572*795d594fSAndroid Build Coastguard Worker # <keyN>: "<valueN>", 573*795d594fSAndroid Build Coastguard Worker # } 574*795d594fSAndroid Build Coastguard Worker # ``` 575*795d594fSAndroid Build Coastguard Worker # 576*795d594fSAndroid Build Coastguard Worker # where `(key0, value0), ..., (keyN, valueN)` are key-value pairs in `props`. 577*795d594fSAndroid Build Coastguard Worker def gen_prop_group(self, group_name, props): 578*795d594fSAndroid Build Coastguard Worker props_joined = """, 579*795d594fSAndroid Build Coastguard Worker """.join([f"{k}: \"{v}\"" for (k, v) in props.items()]) 580*795d594fSAndroid Build Coastguard Worker return f""" 581*795d594fSAndroid Build Coastguard Worker {group_name}: {{ 582*795d594fSAndroid Build Coastguard Worker {props_joined}, 583*795d594fSAndroid Build Coastguard Worker }},""" 584*795d594fSAndroid Build Coastguard Worker 585*795d594fSAndroid Build Coastguard Worker def gen_libs_list_impl(self, library_type, libraries): 586*795d594fSAndroid Build Coastguard Worker if len(libraries) == 0: 587*795d594fSAndroid Build Coastguard Worker return "" 588*795d594fSAndroid Build Coastguard Worker libraries_joined = """, 589*795d594fSAndroid Build Coastguard Worker """.join(libraries) 590*795d594fSAndroid Build Coastguard Worker return f""" 591*795d594fSAndroid Build Coastguard Worker {library_type}: [ 592*795d594fSAndroid Build Coastguard Worker {libraries_joined}, 593*795d594fSAndroid Build Coastguard Worker ],""" 594*795d594fSAndroid Build Coastguard Worker 595*795d594fSAndroid Build Coastguard Worker def gen_libs_list(self, libraries): 596*795d594fSAndroid Build Coastguard Worker return self.gen_libs_list_impl("libs", libraries); 597*795d594fSAndroid Build Coastguard Worker 598*795d594fSAndroid Build Coastguard Worker def gen_static_libs_list(self, libraries): 599*795d594fSAndroid Build Coastguard Worker return self.gen_libs_list_impl("static_libs", libraries); 600*795d594fSAndroid Build Coastguard Worker 601*795d594fSAndroid Build Coastguard Worker def gen_java_library_rule(self, name, src_dir, libraries, extra_props): 602*795d594fSAndroid Build Coastguard Worker return f"""\ 603*795d594fSAndroid Build Coastguard Worker 604*795d594fSAndroid Build Coastguard Worker 605*795d594fSAndroid Build Coastguard Worker // Library with {src_dir}/ sources for the test. 606*795d594fSAndroid Build Coastguard Worker java_library {{ 607*795d594fSAndroid Build Coastguard Worker name: "{name}", 608*795d594fSAndroid Build Coastguard Worker defaults: ["art-run-test-defaults"],{self.gen_libs_list(libraries)} 609*795d594fSAndroid Build Coastguard Worker srcs: ["{src_dir}/**/*.java"],{extra_props} 610*795d594fSAndroid Build Coastguard Worker }}""" 611*795d594fSAndroid Build Coastguard Worker 612*795d594fSAndroid Build Coastguard Worker # Can `run_test` be succesfully run with TradeFed? 613*795d594fSAndroid Build Coastguard Worker # TODO(b/147812905): Add run-time support for more tests. 614*795d594fSAndroid Build Coastguard Worker def is_tradefed_runnable(self, run_test): 615*795d594fSAndroid Build Coastguard Worker run_test_path = os.path.join(self.art_test_dir, run_test) 616*795d594fSAndroid Build Coastguard Worker 617*795d594fSAndroid Build Coastguard Worker # Skip tests with non-default run rules, unless these run rules 618*795d594fSAndroid Build Coastguard Worker # can be safely ignored. 619*795d594fSAndroid Build Coastguard Worker if os.path.isfile(os.path.join(run_test_path, "run.py")): 620*795d594fSAndroid Build Coastguard Worker if not self.can_ignore_run_script(run_test): 621*795d594fSAndroid Build Coastguard Worker return False 622*795d594fSAndroid Build Coastguard Worker # Skip tests known to fail. 623*795d594fSAndroid Build Coastguard Worker if run_test in known_failing_tests: 624*795d594fSAndroid Build Coastguard Worker return False 625*795d594fSAndroid Build Coastguard Worker # All other tests are considered runnable. 626*795d594fSAndroid Build Coastguard Worker return True 627*795d594fSAndroid Build Coastguard Worker 628*795d594fSAndroid Build Coastguard Worker def is_slow(self, run_test): 629*795d594fSAndroid Build Coastguard Worker return run_test in known_slow_tests 630*795d594fSAndroid Build Coastguard Worker 631*795d594fSAndroid Build Coastguard Worker def regen_bp_files(self, run_tests, buildable_tests): 632*795d594fSAndroid Build Coastguard Worker for run_test in run_tests: 633*795d594fSAndroid Build Coastguard Worker # Remove any previously generated file. 634*795d594fSAndroid Build Coastguard Worker bp_file = os.path.join(self.art_test_dir, run_test, "Android.bp") 635*795d594fSAndroid Build Coastguard Worker if os.path.exists(bp_file): 636*795d594fSAndroid Build Coastguard Worker logging.debug(f"Removing `{bp_file}`.") 637*795d594fSAndroid Build Coastguard Worker os.remove(bp_file) 638*795d594fSAndroid Build Coastguard Worker 639*795d594fSAndroid Build Coastguard Worker for run_test in buildable_tests: 640*795d594fSAndroid Build Coastguard Worker self.regen_bp_file(run_test) 641*795d594fSAndroid Build Coastguard Worker 642*795d594fSAndroid Build Coastguard Worker def regen_bp_file(self, run_test): 643*795d594fSAndroid Build Coastguard Worker """Regenerate Blueprint file for an ART run-test.""" 644*795d594fSAndroid Build Coastguard Worker 645*795d594fSAndroid Build Coastguard Worker run_test_path = os.path.join(self.art_test_dir, run_test) 646*795d594fSAndroid Build Coastguard Worker bp_file = os.path.join(run_test_path, "Android.bp") 647*795d594fSAndroid Build Coastguard Worker 648*795d594fSAndroid Build Coastguard Worker # Optional test metadata (JSON file). 649*795d594fSAndroid Build Coastguard Worker metadata = self.get_test_metadata(run_test) 650*795d594fSAndroid Build Coastguard Worker test_suites = metadata.get("test_suites", []) 651*795d594fSAndroid Build Coastguard Worker is_cts_test = "cts" in test_suites 652*795d594fSAndroid Build Coastguard Worker is_mcts_test = "mcts-art" in test_suites 653*795d594fSAndroid Build Coastguard Worker 654*795d594fSAndroid Build Coastguard Worker # For now we make it mandatory for an ART CTS test to be an ART 655*795d594fSAndroid Build Coastguard Worker # MCTS test and vice versa. 656*795d594fSAndroid Build Coastguard Worker if is_cts_test != is_mcts_test: 657*795d594fSAndroid Build Coastguard Worker (present, absent) = ("mts", "mcts-art") if is_cts_test else ("mcts-art", "mts") 658*795d594fSAndroid Build Coastguard Worker logging.error(f"Inconsistent test suites state in metadata for ART run-test `{run_test}`: " + 659*795d594fSAndroid Build Coastguard Worker f"`test_suites` contains `{present}` but not `{absent}`") 660*795d594fSAndroid Build Coastguard Worker sys.exit(1) 661*795d594fSAndroid Build Coastguard Worker 662*795d594fSAndroid Build Coastguard Worker # Do not package non-runnable ART run-tests in ART MTS (see b/363075236). 663*795d594fSAndroid Build Coastguard Worker if self.is_tradefed_runnable(run_test): 664*795d594fSAndroid Build Coastguard Worker test_suites.append("mts-art") 665*795d594fSAndroid Build Coastguard Worker 666*795d594fSAndroid Build Coastguard Worker run_test_module_name = ART_RUN_TEST_MODULE_NAME_PREFIX + run_test 667*795d594fSAndroid Build Coastguard Worker 668*795d594fSAndroid Build Coastguard Worker # Set the test configuration template. 669*795d594fSAndroid Build Coastguard Worker if self.is_tradefed_runnable(run_test): 670*795d594fSAndroid Build Coastguard Worker if is_cts_test: 671*795d594fSAndroid Build Coastguard Worker test_config_template = "art-run-test-target-cts-template" 672*795d594fSAndroid Build Coastguard Worker elif self.is_slow(run_test): 673*795d594fSAndroid Build Coastguard Worker test_config_template = "art-run-test-target-slow-template" 674*795d594fSAndroid Build Coastguard Worker else: 675*795d594fSAndroid Build Coastguard Worker test_config_template = "art-run-test-target-template" 676*795d594fSAndroid Build Coastguard Worker else: 677*795d594fSAndroid Build Coastguard Worker test_config_template = "art-run-test-target-no-test-suite-tag-template" 678*795d594fSAndroid Build Coastguard Worker 679*795d594fSAndroid Build Coastguard Worker # Define the `test_suites` property, if test suites are present in 680*795d594fSAndroid Build Coastguard Worker # the test's metadata. 681*795d594fSAndroid Build Coastguard Worker test_suites_prop = "" 682*795d594fSAndroid Build Coastguard Worker if test_suites: 683*795d594fSAndroid Build Coastguard Worker test_suites_joined = """, 684*795d594fSAndroid Build Coastguard Worker """.join([f"\"{s}\"" for s in test_suites]) 685*795d594fSAndroid Build Coastguard Worker test_suites_prop = f"""\ 686*795d594fSAndroid Build Coastguard Worker 687*795d594fSAndroid Build Coastguard Worker test_suites: [ 688*795d594fSAndroid Build Coastguard Worker {test_suites_joined}, 689*795d594fSAndroid Build Coastguard Worker ],""" 690*795d594fSAndroid Build Coastguard Worker 691*795d594fSAndroid Build Coastguard Worker include_srcs_prop = "" 692*795d594fSAndroid Build Coastguard Worker if is_checker_test(run_test): 693*795d594fSAndroid Build Coastguard Worker include_srcs_prop = """\ 694*795d594fSAndroid Build Coastguard Worker 695*795d594fSAndroid Build Coastguard Worker // Include the Java source files in the test's artifacts, to make Checker assertions 696*795d594fSAndroid Build Coastguard Worker // available to the TradeFed test runner. 697*795d594fSAndroid Build Coastguard Worker include_srcs: true,""" 698*795d594fSAndroid Build Coastguard Worker 699*795d594fSAndroid Build Coastguard Worker # Set the version of the SDK to compile the Java test module 700*795d594fSAndroid Build Coastguard Worker # against, if needed. 701*795d594fSAndroid Build Coastguard Worker sdk_version_prop = "" 702*795d594fSAndroid Build Coastguard Worker if is_cts_test: 703*795d594fSAndroid Build Coastguard Worker # Have CTS and MCTS test modules use the test API 704*795d594fSAndroid Build Coastguard Worker # (`test_current`) so that they do not depend on the framework 705*795d594fSAndroid Build Coastguard Worker # private platform API (`private`), which is the default. 706*795d594fSAndroid Build Coastguard Worker sdk_version_prop = """ 707*795d594fSAndroid Build Coastguard Worker sdk_version: "test_current",""" 708*795d594fSAndroid Build Coastguard Worker 709*795d594fSAndroid Build Coastguard Worker # The default source directory is `src`, except if `src-art` exists. 710*795d594fSAndroid Build Coastguard Worker if os.path.isdir(os.path.join(run_test_path, "src-art")): 711*795d594fSAndroid Build Coastguard Worker source_dir = "src-art" 712*795d594fSAndroid Build Coastguard Worker else: 713*795d594fSAndroid Build Coastguard Worker source_dir = "src" 714*795d594fSAndroid Build Coastguard Worker 715*795d594fSAndroid Build Coastguard Worker src_library_rules = [] 716*795d594fSAndroid Build Coastguard Worker test_libraries = [] 717*795d594fSAndroid Build Coastguard Worker extra_props = "" 718*795d594fSAndroid Build Coastguard Worker # Honor the Lint baseline file, if present. 719*795d594fSAndroid Build Coastguard Worker if os.path.isfile(os.path.join(run_test_path, LINT_BASELINE_FILENAME)): 720*795d594fSAndroid Build Coastguard Worker extra_props += self.gen_prop_group("lint", {"baseline_filename": LINT_BASELINE_FILENAME}) 721*795d594fSAndroid Build Coastguard Worker if os.path.isdir(os.path.join(run_test_path, "src2")): 722*795d594fSAndroid Build Coastguard Worker test_library = f"{run_test_module_name}-{source_dir}" 723*795d594fSAndroid Build Coastguard Worker src_library_rules.append( 724*795d594fSAndroid Build Coastguard Worker self.gen_java_library_rule(test_library, source_dir, test_libraries, extra_props)) 725*795d594fSAndroid Build Coastguard Worker test_libraries.append(f"\"{test_library}\"") 726*795d594fSAndroid Build Coastguard Worker source_dir = "src2" 727*795d594fSAndroid Build Coastguard Worker 728*795d594fSAndroid Build Coastguard Worker with open(bp_file, "w") as f: 729*795d594fSAndroid Build Coastguard Worker logging.debug(f"Writing `{bp_file}`.") 730*795d594fSAndroid Build Coastguard Worker f.write(textwrap.dedent(f"""\ 731*795d594fSAndroid Build Coastguard Worker // {ADVISORY} 732*795d594fSAndroid Build Coastguard Worker 733*795d594fSAndroid Build Coastguard Worker // Build rules for ART run-test `{run_test}`. 734*795d594fSAndroid Build Coastguard Worker 735*795d594fSAndroid Build Coastguard Worker package {{ 736*795d594fSAndroid Build Coastguard Worker // See: http://go/android-license-faq 737*795d594fSAndroid Build Coastguard Worker // A large-scale-change added 'default_applicable_licenses' to import 738*795d594fSAndroid Build Coastguard Worker // all of the 'license_kinds' from "art_license" 739*795d594fSAndroid Build Coastguard Worker // to get the below license kinds: 740*795d594fSAndroid Build Coastguard Worker // SPDX-license-identifier-Apache-2.0 741*795d594fSAndroid Build Coastguard Worker default_applicable_licenses: ["art_license"], 742*795d594fSAndroid Build Coastguard Worker }}{''.join(src_library_rules)} 743*795d594fSAndroid Build Coastguard Worker 744*795d594fSAndroid Build Coastguard Worker // Test's Dex code. 745*795d594fSAndroid Build Coastguard Worker java_test {{ 746*795d594fSAndroid Build Coastguard Worker name: "{run_test_module_name}", 747*795d594fSAndroid Build Coastguard Worker defaults: ["art-run-test-defaults"], 748*795d594fSAndroid Build Coastguard Worker test_config_template: ":{test_config_template}", 749*795d594fSAndroid Build Coastguard Worker srcs: ["{source_dir}/**/*.java"],{self.gen_static_libs_list(test_libraries)} 750*795d594fSAndroid Build Coastguard Worker data: [ 751*795d594fSAndroid Build Coastguard Worker ":{run_test_module_name}-expected-stdout", 752*795d594fSAndroid Build Coastguard Worker ":{run_test_module_name}-expected-stderr", 753*795d594fSAndroid Build Coastguard Worker ],{test_suites_prop}{include_srcs_prop}{sdk_version_prop} 754*795d594fSAndroid Build Coastguard Worker }} 755*795d594fSAndroid Build Coastguard Worker """)) 756*795d594fSAndroid Build Coastguard Worker 757*795d594fSAndroid Build Coastguard Worker def add_expected_output_genrule(type_str): 758*795d594fSAndroid Build Coastguard Worker type_str_long = "standard output" if type_str == "stdout" else "standard error" 759*795d594fSAndroid Build Coastguard Worker in_file = os.path.join(run_test_path, f"expected-{type_str}.txt") 760*795d594fSAndroid Build Coastguard Worker if os.path.islink(in_file): 761*795d594fSAndroid Build Coastguard Worker # Genrules are sandboxed, so if we just added the symlink to the srcs list, it would 762*795d594fSAndroid Build Coastguard Worker # be a dangling symlink in the sandbox. Instead, if we see a symlink, depend on the 763*795d594fSAndroid Build Coastguard Worker # genrule from the test that the symlink is pointing to instead of the symlink itself. 764*795d594fSAndroid Build Coastguard Worker link_target = os.readlink(in_file) 765*795d594fSAndroid Build Coastguard Worker basename = os.path.basename(in_file) 766*795d594fSAndroid Build Coastguard Worker match = re.fullmatch('\.\./([a-zA-Z0-9_-]+)/' + re.escape(basename), link_target) 767*795d594fSAndroid Build Coastguard Worker if not match: 768*795d594fSAndroid Build Coastguard Worker sys.exit(f"Error: expected symlink to be '../something/{basename}', got {link_target}") 769*795d594fSAndroid Build Coastguard Worker f.write(textwrap.dedent(f"""\ 770*795d594fSAndroid Build Coastguard Worker 771*795d594fSAndroid Build Coastguard Worker // Test's expected {type_str_long}. 772*795d594fSAndroid Build Coastguard Worker genrule {{ 773*795d594fSAndroid Build Coastguard Worker name: "{run_test_module_name}-expected-{type_str}", 774*795d594fSAndroid Build Coastguard Worker out: ["{run_test_module_name}-expected-{type_str}.txt"], 775*795d594fSAndroid Build Coastguard Worker srcs: [":{ART_RUN_TEST_MODULE_NAME_PREFIX}{match.group(1)}-expected-{type_str}"], 776*795d594fSAndroid Build Coastguard Worker cmd: "cp -f $(in) $(out)", 777*795d594fSAndroid Build Coastguard Worker }} 778*795d594fSAndroid Build Coastguard Worker """)) 779*795d594fSAndroid Build Coastguard Worker else: 780*795d594fSAndroid Build Coastguard Worker f.write(textwrap.dedent(f"""\ 781*795d594fSAndroid Build Coastguard Worker 782*795d594fSAndroid Build Coastguard Worker // Test's expected {type_str_long}. 783*795d594fSAndroid Build Coastguard Worker genrule {{ 784*795d594fSAndroid Build Coastguard Worker name: "{run_test_module_name}-expected-{type_str}", 785*795d594fSAndroid Build Coastguard Worker out: ["{run_test_module_name}-expected-{type_str}.txt"], 786*795d594fSAndroid Build Coastguard Worker srcs: ["expected-{type_str}.txt"], 787*795d594fSAndroid Build Coastguard Worker cmd: "cp -f $(in) $(out)", 788*795d594fSAndroid Build Coastguard Worker }} 789*795d594fSAndroid Build Coastguard Worker """)) 790*795d594fSAndroid Build Coastguard Worker 791*795d594fSAndroid Build Coastguard Worker add_expected_output_genrule("stdout") 792*795d594fSAndroid Build Coastguard Worker add_expected_output_genrule("stderr") 793*795d594fSAndroid Build Coastguard Worker 794*795d594fSAndroid Build Coastguard Worker 795*795d594fSAndroid Build Coastguard Worker def regen_test_mapping_file(self, art_run_tests): 796*795d594fSAndroid Build Coastguard Worker """Regenerate ART's `TEST_MAPPING`.""" 797*795d594fSAndroid Build Coastguard Worker 798*795d594fSAndroid Build Coastguard Worker # See go/test-mapping#attributes and 799*795d594fSAndroid Build Coastguard Worker # https://source.android.com/docs/core/tests/development/test-mapping 800*795d594fSAndroid Build Coastguard Worker # for more information about Test Mapping test groups. 801*795d594fSAndroid Build Coastguard Worker 802*795d594fSAndroid Build Coastguard Worker # ART run-tests used in `*presubmit` test groups, used both in pre- and post-submit runs. 803*795d594fSAndroid Build Coastguard Worker presubmit_run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t 804*795d594fSAndroid Build Coastguard Worker for t in art_run_tests 805*795d594fSAndroid Build Coastguard Worker if t not in postsubmit_only_tests] 806*795d594fSAndroid Build Coastguard Worker # ART run-tests used in the `postsubmit` test group, used in post-submit runs only. 807*795d594fSAndroid Build Coastguard Worker postsubmit_run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t 808*795d594fSAndroid Build Coastguard Worker for t in art_run_tests 809*795d594fSAndroid Build Coastguard Worker if t in postsubmit_only_tests] 810*795d594fSAndroid Build Coastguard Worker 811*795d594fSAndroid Build Coastguard Worker def gen_tests_dict(tests, excluded_test_cases = {}, excluded_test_modules = [], suffix = ""): 812*795d594fSAndroid Build Coastguard Worker return [ 813*795d594fSAndroid Build Coastguard Worker ({"name": t + suffix, 814*795d594fSAndroid Build Coastguard Worker "options": [ 815*795d594fSAndroid Build Coastguard Worker {"exclude-filter": e} 816*795d594fSAndroid Build Coastguard Worker for e in excluded_test_cases[t] 817*795d594fSAndroid Build Coastguard Worker ]} 818*795d594fSAndroid Build Coastguard Worker if t in excluded_test_cases 819*795d594fSAndroid Build Coastguard Worker else {"name": t + suffix}) 820*795d594fSAndroid Build Coastguard Worker for t in tests 821*795d594fSAndroid Build Coastguard Worker if t not in excluded_test_modules 822*795d594fSAndroid Build Coastguard Worker ] 823*795d594fSAndroid Build Coastguard Worker 824*795d594fSAndroid Build Coastguard Worker # Mainline presubmits. 825*795d594fSAndroid Build Coastguard Worker mainline_presubmit_apex_suffix = "[com.google.android.art.apex]" 826*795d594fSAndroid Build Coastguard Worker mainline_other_presubmit_tests = [] 827*795d594fSAndroid Build Coastguard Worker mainline_presubmit_tests = (mainline_other_presubmit_tests + presubmit_run_test_module_names + 828*795d594fSAndroid Build Coastguard Worker art_gtest_mainline_presubmit_module_names) 829*795d594fSAndroid Build Coastguard Worker mainline_presubmit_tests_dict = \ 830*795d594fSAndroid Build Coastguard Worker gen_tests_dict(mainline_presubmit_tests, 831*795d594fSAndroid Build Coastguard Worker failing_tests_excluded_from_mainline_presubmits, 832*795d594fSAndroid Build Coastguard Worker [], 833*795d594fSAndroid Build Coastguard Worker mainline_presubmit_apex_suffix) 834*795d594fSAndroid Build Coastguard Worker 835*795d594fSAndroid Build Coastguard Worker # ART mainline presubmits tests without APEX suffix 836*795d594fSAndroid Build Coastguard Worker art_mainline_presubmit_tests_dict = \ 837*795d594fSAndroid Build Coastguard Worker gen_tests_dict(mainline_presubmit_tests, 838*795d594fSAndroid Build Coastguard Worker failing_tests_excluded_from_mainline_presubmits, 839*795d594fSAndroid Build Coastguard Worker [], 840*795d594fSAndroid Build Coastguard Worker "") 841*795d594fSAndroid Build Coastguard Worker 842*795d594fSAndroid Build Coastguard Worker # Android Virtualization Framework presubmits 843*795d594fSAndroid Build Coastguard Worker avf_presubmit_tests = ["ComposHostTestCases"] 844*795d594fSAndroid Build Coastguard Worker avf_presubmit_tests_dict = gen_tests_dict(avf_presubmit_tests, 845*795d594fSAndroid Build Coastguard Worker failing_tests_excluded_from_test_mapping) 846*795d594fSAndroid Build Coastguard Worker 847*795d594fSAndroid Build Coastguard Worker # Presubmits. 848*795d594fSAndroid Build Coastguard Worker other_presubmit_tests = [ 849*795d594fSAndroid Build Coastguard Worker "ArtServiceTests", 850*795d594fSAndroid Build Coastguard Worker "BootImageProfileTest", 851*795d594fSAndroid Build Coastguard Worker "CtsJdwpTestCases", 852*795d594fSAndroid Build Coastguard Worker "art-apex-update-rollback", 853*795d594fSAndroid Build Coastguard Worker "art_standalone_dexpreopt_tests", 854*795d594fSAndroid Build Coastguard Worker ] 855*795d594fSAndroid Build Coastguard Worker presubmit_tests = (other_presubmit_tests + presubmit_run_test_module_names + 856*795d594fSAndroid Build Coastguard Worker art_gtest_presubmit_module_names) 857*795d594fSAndroid Build Coastguard Worker presubmit_tests_dict = gen_tests_dict(presubmit_tests, 858*795d594fSAndroid Build Coastguard Worker failing_tests_excluded_from_test_mapping) 859*795d594fSAndroid Build Coastguard Worker hwasan_presubmit_tests_dict = gen_tests_dict(presubmit_tests, 860*795d594fSAndroid Build Coastguard Worker failing_tests_excluded_from_test_mapping, 861*795d594fSAndroid Build Coastguard Worker known_failing_on_hwasan_tests) 862*795d594fSAndroid Build Coastguard Worker 863*795d594fSAndroid Build Coastguard Worker # Postsubmits. 864*795d594fSAndroid Build Coastguard Worker postsubmit_tests = postsubmit_run_test_module_names + art_gtest_postsubmit_module_names 865*795d594fSAndroid Build Coastguard Worker postsubmit_tests_dict = [{"name": t} for t in postsubmit_tests] 866*795d594fSAndroid Build Coastguard Worker postsubmit_tests_dict = gen_tests_dict(postsubmit_tests, 867*795d594fSAndroid Build Coastguard Worker failing_tests_excluded_from_test_mapping) 868*795d594fSAndroid Build Coastguard Worker 869*795d594fSAndroid Build Coastguard Worker # Use an `OrderedDict` container to preserve the order in which items are inserted. 870*795d594fSAndroid Build Coastguard Worker # Do not produce an entry for a test group if it is empty. 871*795d594fSAndroid Build Coastguard Worker test_mapping_dict = collections.OrderedDict([ 872*795d594fSAndroid Build Coastguard Worker (test_group_name, test_group_dict) 873*795d594fSAndroid Build Coastguard Worker for (test_group_name, test_group_dict) 874*795d594fSAndroid Build Coastguard Worker in [ 875*795d594fSAndroid Build Coastguard Worker ("art-mainline-presubmit", art_mainline_presubmit_tests_dict), 876*795d594fSAndroid Build Coastguard Worker ("mainline-presubmit", mainline_presubmit_tests_dict), 877*795d594fSAndroid Build Coastguard Worker ("presubmit", presubmit_tests_dict), 878*795d594fSAndroid Build Coastguard Worker ("hwasan-presubmit", hwasan_presubmit_tests_dict), 879*795d594fSAndroid Build Coastguard Worker ("avf-presubmit", avf_presubmit_tests_dict), 880*795d594fSAndroid Build Coastguard Worker ("postsubmit", postsubmit_tests_dict), 881*795d594fSAndroid Build Coastguard Worker ] 882*795d594fSAndroid Build Coastguard Worker if test_group_dict 883*795d594fSAndroid Build Coastguard Worker ]) 884*795d594fSAndroid Build Coastguard Worker test_mapping_contents = json.dumps(test_mapping_dict, indent = INDENT) 885*795d594fSAndroid Build Coastguard Worker 886*795d594fSAndroid Build Coastguard Worker test_mapping_file = os.path.join(self.art_dir, "TEST_MAPPING") 887*795d594fSAndroid Build Coastguard Worker with open(test_mapping_file, "w") as f: 888*795d594fSAndroid Build Coastguard Worker logging.debug(f"Writing `{test_mapping_file}`.") 889*795d594fSAndroid Build Coastguard Worker f.write(f"// {ADVISORY}\n") 890*795d594fSAndroid Build Coastguard Worker f.write(test_mapping_contents) 891*795d594fSAndroid Build Coastguard Worker f.write("\n") 892*795d594fSAndroid Build Coastguard Worker 893*795d594fSAndroid Build Coastguard Worker def create_mts_test_shard(self, tests_description, tests, shard_num, copyright_year, 894*795d594fSAndroid Build Coastguard Worker comments = []): 895*795d594fSAndroid Build Coastguard Worker """Factory method instantiating an `MtsTestShard`.""" 896*795d594fSAndroid Build Coastguard Worker return self.MtsTestShard(self.mts_config_dir, tests_description, tests, shard_num, 897*795d594fSAndroid Build Coastguard Worker copyright_year, comments) 898*795d594fSAndroid Build Coastguard Worker 899*795d594fSAndroid Build Coastguard Worker class MtsTestShard: 900*795d594fSAndroid Build Coastguard Worker """Class encapsulating data and generation logic for an ART MTS test shard.""" 901*795d594fSAndroid Build Coastguard Worker 902*795d594fSAndroid Build Coastguard Worker def __init__(self, mts_config_dir, tests_description, tests, shard_num, copyright_year, 903*795d594fSAndroid Build Coastguard Worker comments): 904*795d594fSAndroid Build Coastguard Worker self.mts_config_dir = mts_config_dir 905*795d594fSAndroid Build Coastguard Worker self.tests_description = tests_description 906*795d594fSAndroid Build Coastguard Worker self.tests = tests 907*795d594fSAndroid Build Coastguard Worker self.shard_num = shard_num 908*795d594fSAndroid Build Coastguard Worker self.copyright_year = copyright_year 909*795d594fSAndroid Build Coastguard Worker self.comments = comments 910*795d594fSAndroid Build Coastguard Worker 911*795d594fSAndroid Build Coastguard Worker def shard_id(self): 912*795d594fSAndroid Build Coastguard Worker return f"{self.shard_num:02}" 913*795d594fSAndroid Build Coastguard Worker 914*795d594fSAndroid Build Coastguard Worker def test_plan_name(self): 915*795d594fSAndroid Build Coastguard Worker return "mts-art-shard-" + self.shard_id() 916*795d594fSAndroid Build Coastguard Worker 917*795d594fSAndroid Build Coastguard Worker def test_list_name(self): 918*795d594fSAndroid Build Coastguard Worker return "mts-art-tests-list-user-shard-" + self.shard_id() 919*795d594fSAndroid Build Coastguard Worker 920*795d594fSAndroid Build Coastguard Worker def regen_test_plan_file(self): 921*795d594fSAndroid Build Coastguard Worker """Regenerate ART MTS test plan file shard (`mts-art-shard-<shard_num>.xml`).""" 922*795d594fSAndroid Build Coastguard Worker root = xml.dom.minidom.Document() 923*795d594fSAndroid Build Coastguard Worker 924*795d594fSAndroid Build Coastguard Worker advisory_header = root.createComment(f" {ADVISORY} ") 925*795d594fSAndroid Build Coastguard Worker root.appendChild(advisory_header) 926*795d594fSAndroid Build Coastguard Worker copyright_header = root.createComment(copyright_header_text(self.copyright_year)) 927*795d594fSAndroid Build Coastguard Worker root.appendChild(copyright_header) 928*795d594fSAndroid Build Coastguard Worker 929*795d594fSAndroid Build Coastguard Worker configuration = root.createElement("configuration") 930*795d594fSAndroid Build Coastguard Worker root.appendChild(configuration) 931*795d594fSAndroid Build Coastguard Worker configuration.setAttribute( 932*795d594fSAndroid Build Coastguard Worker "description", 933*795d594fSAndroid Build Coastguard Worker f"Run {self.test_plan_name()} from a preexisting MTS installation.") 934*795d594fSAndroid Build Coastguard Worker 935*795d594fSAndroid Build Coastguard Worker # Included XML files. 936*795d594fSAndroid Build Coastguard Worker included_xml_files = ["mts", self.test_list_name()] 937*795d594fSAndroid Build Coastguard Worker # Special case for the test plan of shard 03 (ART gtests), where we also 938*795d594fSAndroid Build Coastguard Worker # include ART MTS eng-only tests. 939*795d594fSAndroid Build Coastguard Worker # 940*795d594fSAndroid Build Coastguard Worker # TODO(rpl): Restucture the MTS generation logic to avoid special-casing 941*795d594fSAndroid Build Coastguard Worker # at that level of the generator. 942*795d594fSAndroid Build Coastguard Worker if self.shard_num == 3: 943*795d594fSAndroid Build Coastguard Worker included_xml_files.append(ENG_ONLY_TEST_LIST_NAME) 944*795d594fSAndroid Build Coastguard Worker for xml_file in included_xml_files: 945*795d594fSAndroid Build Coastguard Worker include = root.createElement("include") 946*795d594fSAndroid Build Coastguard Worker include.setAttribute("name", xml_file) 947*795d594fSAndroid Build Coastguard Worker configuration.appendChild(include) 948*795d594fSAndroid Build Coastguard Worker 949*795d594fSAndroid Build Coastguard Worker # Test plan name. 950*795d594fSAndroid Build Coastguard Worker option = root.createElement("option") 951*795d594fSAndroid Build Coastguard Worker option.setAttribute("name", "plan") 952*795d594fSAndroid Build Coastguard Worker option.setAttribute("value", self.test_plan_name()) 953*795d594fSAndroid Build Coastguard Worker configuration.appendChild(option) 954*795d594fSAndroid Build Coastguard Worker 955*795d594fSAndroid Build Coastguard Worker xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 956*795d594fSAndroid Build Coastguard Worker 957*795d594fSAndroid Build Coastguard Worker test_plan_file = os.path.join(self.mts_config_dir, self.test_plan_name() + ".xml") 958*795d594fSAndroid Build Coastguard Worker with open(test_plan_file, "wb") as f: 959*795d594fSAndroid Build Coastguard Worker logging.debug(f"Writing `{test_plan_file}`.") 960*795d594fSAndroid Build Coastguard Worker f.write(xml_str) 961*795d594fSAndroid Build Coastguard Worker 962*795d594fSAndroid Build Coastguard Worker def regen_test_list_file(self): 963*795d594fSAndroid Build Coastguard Worker """Regenerate ART MTS test list file (`mts-art-tests-list-user-shard-<shard_num>.xml`).""" 964*795d594fSAndroid Build Coastguard Worker configuration_description = \ 965*795d594fSAndroid Build Coastguard Worker f"List of ART MTS tests that do not need root access (shard {self.shard_id()})" 966*795d594fSAndroid Build Coastguard Worker test_list_file = os.path.join(self.mts_config_dir, self.test_list_name() + ".xml") 967*795d594fSAndroid Build Coastguard Worker gen_mts_test_list_file(self.tests, test_list_file, self.copyright_year, 968*795d594fSAndroid Build Coastguard Worker configuration_description, self.tests_description, self.comments) 969*795d594fSAndroid Build Coastguard Worker 970*795d594fSAndroid Build Coastguard Worker def regen_mts_art_tests_list_user_file(self, num_mts_art_run_test_shards): 971*795d594fSAndroid Build Coastguard Worker """Regenerate ART MTS test list file (`mts-art-tests-list-user.xml`).""" 972*795d594fSAndroid Build Coastguard Worker root = xml.dom.minidom.Document() 973*795d594fSAndroid Build Coastguard Worker 974*795d594fSAndroid Build Coastguard Worker advisory_header = root.createComment(f" {ADVISORY} ") 975*795d594fSAndroid Build Coastguard Worker root.appendChild(advisory_header) 976*795d594fSAndroid Build Coastguard Worker copyright_header = root.createComment(copyright_header_text(2020)) 977*795d594fSAndroid Build Coastguard Worker root.appendChild(copyright_header) 978*795d594fSAndroid Build Coastguard Worker 979*795d594fSAndroid Build Coastguard Worker configuration = root.createElement("configuration") 980*795d594fSAndroid Build Coastguard Worker root.appendChild(configuration) 981*795d594fSAndroid Build Coastguard Worker configuration.setAttribute("description", "List of ART MTS tests that do not need root access.") 982*795d594fSAndroid Build Coastguard Worker 983*795d594fSAndroid Build Coastguard Worker # Included XML files. 984*795d594fSAndroid Build Coastguard Worker for s in range(num_mts_art_run_test_shards): 985*795d594fSAndroid Build Coastguard Worker include = root.createElement("include") 986*795d594fSAndroid Build Coastguard Worker include.setAttribute("name", f"mts-art-tests-list-user-shard-{s:02}") 987*795d594fSAndroid Build Coastguard Worker configuration.appendChild(include) 988*795d594fSAndroid Build Coastguard Worker 989*795d594fSAndroid Build Coastguard Worker def append_test_exclusion(test): 990*795d594fSAndroid Build Coastguard Worker option = root.createElement("option") 991*795d594fSAndroid Build Coastguard Worker option.setAttribute("name", "compatibility:exclude-filter") 992*795d594fSAndroid Build Coastguard Worker option.setAttribute("value", test) 993*795d594fSAndroid Build Coastguard Worker configuration.appendChild(option) 994*795d594fSAndroid Build Coastguard Worker 995*795d594fSAndroid Build Coastguard Worker # Excluded flaky tests. 996*795d594fSAndroid Build Coastguard Worker xml_comment = root.createComment(" Excluded flaky tests (b/209958457). ") 997*795d594fSAndroid Build Coastguard Worker configuration.appendChild(xml_comment) 998*795d594fSAndroid Build Coastguard Worker for module in flaky_tests_excluded_from_mts: 999*795d594fSAndroid Build Coastguard Worker for testcase in flaky_tests_excluded_from_mts[module]: 1000*795d594fSAndroid Build Coastguard Worker append_test_exclusion(f"{module} {testcase}") 1001*795d594fSAndroid Build Coastguard Worker 1002*795d594fSAndroid Build Coastguard Worker # Excluded failing tests. 1003*795d594fSAndroid Build Coastguard Worker xml_comment = root.createComment(" Excluded failing tests (b/247108425). ") 1004*795d594fSAndroid Build Coastguard Worker configuration.appendChild(xml_comment) 1005*795d594fSAndroid Build Coastguard Worker for module in failing_tests_excluded_from_mts_and_mainline_presubmits: 1006*795d594fSAndroid Build Coastguard Worker for testcase in failing_tests_excluded_from_mts_and_mainline_presubmits[module]: 1007*795d594fSAndroid Build Coastguard Worker append_test_exclusion(f"{module} {testcase}") 1008*795d594fSAndroid Build Coastguard Worker 1009*795d594fSAndroid Build Coastguard Worker xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 1010*795d594fSAndroid Build Coastguard Worker 1011*795d594fSAndroid Build Coastguard Worker mts_art_tests_list_user_file = os.path.join(self.mts_config_dir, "mts-art-tests-list-user.xml") 1012*795d594fSAndroid Build Coastguard Worker with open(mts_art_tests_list_user_file, "wb") as f: 1013*795d594fSAndroid Build Coastguard Worker logging.debug(f"Writing `{mts_art_tests_list_user_file}`.") 1014*795d594fSAndroid Build Coastguard Worker f.write(xml_str) 1015*795d594fSAndroid Build Coastguard Worker 1016*795d594fSAndroid Build Coastguard Worker def regen_art_mts_files(self, art_run_tests, art_jvmti_cts_tests): 1017*795d594fSAndroid Build Coastguard Worker """Regenerate ART MTS definition files.""" 1018*795d594fSAndroid Build Coastguard Worker 1019*795d594fSAndroid Build Coastguard Worker # Remove any previously MTS ART test plan shard (`mts-art-shard-[0-9]+.xml`) 1020*795d594fSAndroid Build Coastguard Worker # and any test list shard (`mts-art-tests-list-user-shard-[0-9]+.xml`). 1021*795d594fSAndroid Build Coastguard Worker old_test_plan_shards = sorted([ 1022*795d594fSAndroid Build Coastguard Worker test_plan_shard 1023*795d594fSAndroid Build Coastguard Worker for test_plan_shard in os.listdir(self.mts_config_dir) 1024*795d594fSAndroid Build Coastguard Worker if re.match("^mts-art-(tests-list-user-)?shard-[0-9]+.xml$", test_plan_shard)]) 1025*795d594fSAndroid Build Coastguard Worker for shard in old_test_plan_shards: 1026*795d594fSAndroid Build Coastguard Worker shard_path = os.path.join(self.mts_config_dir, shard) 1027*795d594fSAndroid Build Coastguard Worker if os.path.exists(shard_path): 1028*795d594fSAndroid Build Coastguard Worker logging.debug(f"Removing `{shard_path}`.") 1029*795d594fSAndroid Build Coastguard Worker os.remove(shard_path) 1030*795d594fSAndroid Build Coastguard Worker 1031*795d594fSAndroid Build Coastguard Worker mts_test_shards = [] 1032*795d594fSAndroid Build Coastguard Worker 1033*795d594fSAndroid Build Coastguard Worker # ART run-tests shard(s). 1034*795d594fSAndroid Build Coastguard Worker art_run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t for t in art_run_tests] 1035*795d594fSAndroid Build Coastguard Worker art_run_test_shards = split_list(art_run_test_module_names, NUM_MTS_ART_RUN_TEST_SHARDS) 1036*795d594fSAndroid Build Coastguard Worker for i in range(len(art_run_test_shards)): 1037*795d594fSAndroid Build Coastguard Worker art_tests_shard_i_tests = art_run_test_shards[i] 1038*795d594fSAndroid Build Coastguard Worker art_tests_shard_i = self.create_mts_test_shard( 1039*795d594fSAndroid Build Coastguard Worker "ART run-tests", art_tests_shard_i_tests, i, 2020, 1040*795d594fSAndroid Build Coastguard Worker ["TODO(rpl): Find a way to express this list in a more concise fashion."]) 1041*795d594fSAndroid Build Coastguard Worker mts_test_shards.append(art_tests_shard_i) 1042*795d594fSAndroid Build Coastguard Worker 1043*795d594fSAndroid Build Coastguard Worker # CTS Libcore non-OJ tests (`CtsLibcoreTestCases`) shard. 1044*795d594fSAndroid Build Coastguard Worker cts_libcore_tests_shard_num = len(mts_test_shards) 1045*795d594fSAndroid Build Coastguard Worker cts_libcore_tests_shard = self.create_mts_test_shard( 1046*795d594fSAndroid Build Coastguard Worker "CTS Libcore non-OJ tests", ["CtsLibcoreTestCases"], cts_libcore_tests_shard_num, 2020) 1047*795d594fSAndroid Build Coastguard Worker mts_test_shards.append(cts_libcore_tests_shard) 1048*795d594fSAndroid Build Coastguard Worker 1049*795d594fSAndroid Build Coastguard Worker # Other CTS tests shard. 1050*795d594fSAndroid Build Coastguard Worker other_cts_tests_shard_num = len(mts_test_shards) 1051*795d594fSAndroid Build Coastguard Worker other_cts_libcore_tests_shard_tests = [ 1052*795d594fSAndroid Build Coastguard Worker "CtsLibcoreApiEvolutionTestCases", 1053*795d594fSAndroid Build Coastguard Worker "CtsLibcoreFileIOTestCases", 1054*795d594fSAndroid Build Coastguard Worker "CtsLibcoreJsr166TestCases", 1055*795d594fSAndroid Build Coastguard Worker "CtsLibcoreLegacy22TestCases", 1056*795d594fSAndroid Build Coastguard Worker "CtsLibcoreOjTestCases", 1057*795d594fSAndroid Build Coastguard Worker "CtsLibcoreWycheproofBCTestCases", 1058*795d594fSAndroid Build Coastguard Worker "MtsLibcoreOkHttpTestCases", 1059*795d594fSAndroid Build Coastguard Worker "MtsLibcoreBouncyCastleTestCases", 1060*795d594fSAndroid Build Coastguard Worker ] 1061*795d594fSAndroid Build Coastguard Worker other_cts_tests_shard_tests = art_jvmti_cts_tests + other_cts_libcore_tests_shard_tests 1062*795d594fSAndroid Build Coastguard Worker other_cts_tests_shard = self.create_mts_test_shard( 1063*795d594fSAndroid Build Coastguard Worker "Other CTS tests", other_cts_tests_shard_tests, other_cts_tests_shard_num, 2021) 1064*795d594fSAndroid Build Coastguard Worker mts_test_shards.append(other_cts_tests_shard) 1065*795d594fSAndroid Build Coastguard Worker 1066*795d594fSAndroid Build Coastguard Worker # ART gtests shard. 1067*795d594fSAndroid Build Coastguard Worker art_gtests_shard_num = len(mts_test_shards) 1068*795d594fSAndroid Build Coastguard Worker art_gtests_shard_tests = art_gtest_mts_user_module_names 1069*795d594fSAndroid Build Coastguard Worker art_gtests_shard = self.create_mts_test_shard( 1070*795d594fSAndroid Build Coastguard Worker "ART gtests", art_gtests_shard_tests, art_gtests_shard_num, 2022) 1071*795d594fSAndroid Build Coastguard Worker mts_test_shards.append(art_gtests_shard) 1072*795d594fSAndroid Build Coastguard Worker 1073*795d594fSAndroid Build Coastguard Worker for s in mts_test_shards: 1074*795d594fSAndroid Build Coastguard Worker s.regen_test_plan_file() 1075*795d594fSAndroid Build Coastguard Worker s.regen_test_list_file() 1076*795d594fSAndroid Build Coastguard Worker 1077*795d594fSAndroid Build Coastguard Worker # Generate the MTS test list file of "eng-only" tests (tests that 1078*795d594fSAndroid Build Coastguard Worker # need root access to the device-under-test and are not part of 1079*795d594fSAndroid Build Coastguard Worker # "user" test plans). 1080*795d594fSAndroid Build Coastguard Worker # 1081*795d594fSAndroid Build Coastguard Worker # TODO(rpl): Refactor the MTS file generation logic to better 1082*795d594fSAndroid Build Coastguard Worker # handle the special case of "eng-only" tests, which do not play 1083*795d594fSAndroid Build Coastguard Worker # well with `MtsTestShard` at the moment). 1084*795d594fSAndroid Build Coastguard Worker eng_only_test_list_file = os.path.join(self.mts_config_dir, ENG_ONLY_TEST_LIST_NAME + ".xml") 1085*795d594fSAndroid Build Coastguard Worker gen_mts_test_list_file( 1086*795d594fSAndroid Build Coastguard Worker art_gtest_eng_only_module_names, eng_only_test_list_file, 1087*795d594fSAndroid Build Coastguard Worker copyright_year = 2020, 1088*795d594fSAndroid Build Coastguard Worker configuration_description = "List of ART MTS tests that need root access.", 1089*795d594fSAndroid Build Coastguard Worker tests_description = "ART gtests") 1090*795d594fSAndroid Build Coastguard Worker 1091*795d594fSAndroid Build Coastguard Worker self.regen_mts_art_tests_list_user_file(len(mts_test_shards)) 1092*795d594fSAndroid Build Coastguard Worker 1093*795d594fSAndroid Build Coastguard Worker def regen_test_files(self, regen_art_mts): 1094*795d594fSAndroid Build Coastguard Worker """Regenerate ART test files. 1095*795d594fSAndroid Build Coastguard Worker 1096*795d594fSAndroid Build Coastguard Worker Args: 1097*795d594fSAndroid Build Coastguard Worker regen_art_mts: If true, also regenerate the ART MTS definition. 1098*795d594fSAndroid Build Coastguard Worker """ 1099*795d594fSAndroid Build Coastguard Worker run_tests = self.enumerate_run_tests() 1100*795d594fSAndroid Build Coastguard Worker 1101*795d594fSAndroid Build Coastguard Worker # Create a list of the tests that can currently be built, and for 1102*795d594fSAndroid Build Coastguard Worker # which a Blueprint file is to be generated. 1103*795d594fSAndroid Build Coastguard Worker buildable_tests = list(filter(self.is_soong_buildable, run_tests)) 1104*795d594fSAndroid Build Coastguard Worker 1105*795d594fSAndroid Build Coastguard Worker # Create a list of the tests that can be built and run 1106*795d594fSAndroid Build Coastguard Worker # (successfully). These tests are to be added to ART's 1107*795d594fSAndroid Build Coastguard Worker # `TEST_MAPPING` file and also tagged as part of TradeFed's 1108*795d594fSAndroid Build Coastguard Worker # `art-target-run-test` test suite via the `test-suite-tag` option 1109*795d594fSAndroid Build Coastguard Worker # in their configuration file. 1110*795d594fSAndroid Build Coastguard Worker expected_succeeding_tests = list(filter(self.is_tradefed_runnable, 1111*795d594fSAndroid Build Coastguard Worker buildable_tests)) 1112*795d594fSAndroid Build Coastguard Worker 1113*795d594fSAndroid Build Coastguard Worker # Regenerate Blueprint files. 1114*795d594fSAndroid Build Coastguard Worker # --------------------------- 1115*795d594fSAndroid Build Coastguard Worker 1116*795d594fSAndroid Build Coastguard Worker self.regen_bp_files(run_tests, buildable_tests) 1117*795d594fSAndroid Build Coastguard Worker 1118*795d594fSAndroid Build Coastguard Worker buildable_tests_percentage = int(len(buildable_tests) * 100 / len(run_tests)) 1119*795d594fSAndroid Build Coastguard Worker 1120*795d594fSAndroid Build Coastguard Worker print(f"Generated Blueprint files for {len(buildable_tests)} ART run-tests out of" 1121*795d594fSAndroid Build Coastguard Worker f" {len(run_tests)} ({buildable_tests_percentage}%).") 1122*795d594fSAndroid Build Coastguard Worker 1123*795d594fSAndroid Build Coastguard Worker # Regenerate `TEST_MAPPING` file. 1124*795d594fSAndroid Build Coastguard Worker # ------------------------------- 1125*795d594fSAndroid Build Coastguard Worker 1126*795d594fSAndroid Build Coastguard Worker # Note: We only include ART run-tests expected to succeed for now. 1127*795d594fSAndroid Build Coastguard Worker num_expected_succeeding_tests = len(expected_succeeding_tests) 1128*795d594fSAndroid Build Coastguard Worker 1129*795d594fSAndroid Build Coastguard Worker presubmit_run_tests = set(expected_succeeding_tests).difference(postsubmit_only_tests) 1130*795d594fSAndroid Build Coastguard Worker num_presubmit_run_tests = len(presubmit_run_tests) 1131*795d594fSAndroid Build Coastguard Worker presubmit_run_tests_percentage = int( 1132*795d594fSAndroid Build Coastguard Worker num_presubmit_run_tests * 100 / num_expected_succeeding_tests) 1133*795d594fSAndroid Build Coastguard Worker 1134*795d594fSAndroid Build Coastguard Worker num_mainline_presubmit_run_tests = num_presubmit_run_tests 1135*795d594fSAndroid Build Coastguard Worker mainline_presubmit_run_tests_percentage = presubmit_run_tests_percentage 1136*795d594fSAndroid Build Coastguard Worker 1137*795d594fSAndroid Build Coastguard Worker postsubmit_run_tests = set(expected_succeeding_tests).intersection(postsubmit_only_tests) 1138*795d594fSAndroid Build Coastguard Worker num_postsubmit_run_tests = len(postsubmit_run_tests) 1139*795d594fSAndroid Build Coastguard Worker postsubmit_run_tests_percentage = int( 1140*795d594fSAndroid Build Coastguard Worker num_postsubmit_run_tests * 100 / num_expected_succeeding_tests) 1141*795d594fSAndroid Build Coastguard Worker 1142*795d594fSAndroid Build Coastguard Worker self.regen_test_mapping_file(expected_succeeding_tests) 1143*795d594fSAndroid Build Coastguard Worker 1144*795d594fSAndroid Build Coastguard Worker expected_succeeding_tests_percentage = int( 1145*795d594fSAndroid Build Coastguard Worker num_expected_succeeding_tests * 100 / len(run_tests)) 1146*795d594fSAndroid Build Coastguard Worker 1147*795d594fSAndroid Build Coastguard Worker num_gtests = len(art_gtest_module_names) 1148*795d594fSAndroid Build Coastguard Worker 1149*795d594fSAndroid Build Coastguard Worker num_presubmit_gtests = len(art_gtest_presubmit_module_names) 1150*795d594fSAndroid Build Coastguard Worker presubmit_gtests_percentage = int(num_presubmit_gtests * 100 / num_gtests) 1151*795d594fSAndroid Build Coastguard Worker 1152*795d594fSAndroid Build Coastguard Worker num_mainline_presubmit_gtests = len(art_gtest_mainline_presubmit_module_names) 1153*795d594fSAndroid Build Coastguard Worker mainline_presubmit_gtests_percentage = int(num_mainline_presubmit_gtests * 100 / num_gtests) 1154*795d594fSAndroid Build Coastguard Worker 1155*795d594fSAndroid Build Coastguard Worker num_postsubmit_gtests = len(art_gtest_postsubmit_module_names) 1156*795d594fSAndroid Build Coastguard Worker postsubmit_gtests_percentage = int(num_postsubmit_gtests * 100 / num_gtests) 1157*795d594fSAndroid Build Coastguard Worker 1158*795d594fSAndroid Build Coastguard Worker print(f"Generated TEST_MAPPING entries for {num_expected_succeeding_tests} ART run-tests out" 1159*795d594fSAndroid Build Coastguard Worker f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%):") 1160*795d594fSAndroid Build Coastguard Worker for (num_tests, test_kind, tests_percentage, test_group_name) in [ 1161*795d594fSAndroid Build Coastguard Worker (num_mainline_presubmit_run_tests, "ART run-tests", mainline_presubmit_run_tests_percentage, 1162*795d594fSAndroid Build Coastguard Worker "art-mainline-presubmit"), 1163*795d594fSAndroid Build Coastguard Worker (num_mainline_presubmit_run_tests, "ART run-tests", mainline_presubmit_run_tests_percentage, 1164*795d594fSAndroid Build Coastguard Worker "mainline-presubmit"), 1165*795d594fSAndroid Build Coastguard Worker (num_presubmit_run_tests, "ART run-tests", presubmit_run_tests_percentage, "presubmit"), 1166*795d594fSAndroid Build Coastguard Worker (num_postsubmit_run_tests, "ART run-tests", postsubmit_run_tests_percentage, "postsubmit"), 1167*795d594fSAndroid Build Coastguard Worker (num_mainline_presubmit_gtests, "ART gtests", mainline_presubmit_gtests_percentage, 1168*795d594fSAndroid Build Coastguard Worker "mainline-presubmit"), 1169*795d594fSAndroid Build Coastguard Worker (num_presubmit_gtests, "ART gtests", presubmit_gtests_percentage, "presubmit"), 1170*795d594fSAndroid Build Coastguard Worker (num_postsubmit_gtests, "ART gtests", postsubmit_gtests_percentage, "postsubmit"), 1171*795d594fSAndroid Build Coastguard Worker ]: 1172*795d594fSAndroid Build Coastguard Worker print( 1173*795d594fSAndroid Build Coastguard Worker f" {num_tests:3d} {test_kind} ({tests_percentage}%) in `{test_group_name}` test group.") 1174*795d594fSAndroid Build Coastguard Worker print(""" Note: Tests in `*presubmit` test groups are executed in pre- and 1175*795d594fSAndroid Build Coastguard Worker post-submit test runs. Tests in the `postsubmit` test group 1176*795d594fSAndroid Build Coastguard Worker are only executed in post-submit test runs.""") 1177*795d594fSAndroid Build Coastguard Worker 1178*795d594fSAndroid Build Coastguard Worker # Regenerate ART MTS definition (optional). 1179*795d594fSAndroid Build Coastguard Worker # ----------------------------------------- 1180*795d594fSAndroid Build Coastguard Worker 1181*795d594fSAndroid Build Coastguard Worker if regen_art_mts: 1182*795d594fSAndroid Build Coastguard Worker self.regen_art_mts_files(expected_succeeding_tests, self.enumerate_jvmti_cts_tests()) 1183*795d594fSAndroid Build Coastguard Worker print(f"Generated ART MTS entries for {num_expected_succeeding_tests} ART run-tests out" 1184*795d594fSAndroid Build Coastguard Worker f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%).") 1185*795d594fSAndroid Build Coastguard Worker 1186*795d594fSAndroid Build Coastguard Workerdef main(): 1187*795d594fSAndroid Build Coastguard Worker if "ANDROID_BUILD_TOP" not in os.environ: 1188*795d594fSAndroid Build Coastguard Worker logging.error("ANDROID_BUILD_TOP environment variable is empty; did you forget to run `lunch`?") 1189*795d594fSAndroid Build Coastguard Worker sys.exit(1) 1190*795d594fSAndroid Build Coastguard Worker 1191*795d594fSAndroid Build Coastguard Worker parser = argparse.ArgumentParser( 1192*795d594fSAndroid Build Coastguard Worker formatter_class=argparse.RawDescriptionHelpFormatter, 1193*795d594fSAndroid Build Coastguard Worker description=textwrap.dedent("Regenerate some ART test related files."), 1194*795d594fSAndroid Build Coastguard Worker epilog=textwrap.dedent("""\ 1195*795d594fSAndroid Build Coastguard Worker Regenerate ART run-tests Blueprint files, ART's `TEST_MAPPING` file, and 1196*795d594fSAndroid Build Coastguard Worker optionally the ART MTS (Mainline Test Suite) definition. 1197*795d594fSAndroid Build Coastguard Worker """)) 1198*795d594fSAndroid Build Coastguard Worker parser.add_argument("-m", "--regen-art-mts", help="regenerate the ART MTS definition as well", 1199*795d594fSAndroid Build Coastguard Worker action="store_true") 1200*795d594fSAndroid Build Coastguard Worker parser.add_argument("-v", "--verbose", help="enable verbose output", action="store_true") 1201*795d594fSAndroid Build Coastguard Worker args = parser.parse_args() 1202*795d594fSAndroid Build Coastguard Worker 1203*795d594fSAndroid Build Coastguard Worker if args.verbose: 1204*795d594fSAndroid Build Coastguard Worker logging.getLogger().setLevel(logging.DEBUG) 1205*795d594fSAndroid Build Coastguard Worker 1206*795d594fSAndroid Build Coastguard Worker generator = Generator(os.path.join(os.environ["ANDROID_BUILD_TOP"])) 1207*795d594fSAndroid Build Coastguard Worker generator.regen_test_files(args.regen_art_mts) 1208*795d594fSAndroid Build Coastguard Worker 1209*795d594fSAndroid Build Coastguard Worker 1210*795d594fSAndroid Build Coastguard Workerif __name__ == "__main__": 1211*795d594fSAndroid Build Coastguard Worker main() 1212