1# ===----------------------------------------------------------------------===## 2# 3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4# See https://llvm.org/LICENSE.txt for license information. 5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6# 7# ===----------------------------------------------------------------------===## 8 9import lit 10import libcxx.test.config as config 11import lit.formats 12import os 13import re 14 15 16def _getTempPaths(test): 17 """ 18 Return the values to use for the %T and %t substitutions, respectively. 19 20 The difference between this and Lit's default behavior is that we guarantee 21 that %T is a path unique to the test being run. 22 """ 23 tmpDir, _ = lit.TestRunner.getTempPaths(test) 24 _, testName = os.path.split(test.getExecPath()) 25 tmpDir = os.path.join(tmpDir, testName + ".dir") 26 tmpBase = os.path.join(tmpDir, "t") 27 return tmpDir, tmpBase 28 29 30def _checkBaseSubstitutions(substitutions): 31 substitutions = [s for (s, _) in substitutions] 32 for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]: 33 assert s in substitutions, "Required substitution {} was not provided".format(s) 34 35def _executeScriptInternal(test, litConfig, commands): 36 """ 37 Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands) 38 39 TODO: This really should be easier to access from Lit itself 40 """ 41 parsedCommands = parseScript(test, preamble=commands) 42 43 _, tmpBase = _getTempPaths(test) 44 execDir = os.path.dirname(test.getExecPath()) 45 try: 46 res = lit.TestRunner.executeScriptInternal( 47 test, litConfig, tmpBase, parsedCommands, execDir, debug=False 48 ) 49 except lit.TestRunner.ScriptFatal as e: 50 res = ("", str(e), 127, None) 51 (out, err, exitCode, timeoutInfo) = res 52 53 return (out, err, exitCode, timeoutInfo, parsedCommands) 54 55 56def _validateModuleDependencies(modules): 57 for m in modules: 58 if m not in ("std", "std.compat"): 59 raise RuntimeError( 60 f"Invalid module dependency '{m}', only 'std' and 'std.compat' are valid" 61 ) 62 63 64def parseScript(test, preamble): 65 """ 66 Extract the script from a test, with substitutions applied. 67 68 Returns a list of commands ready to be executed. 69 70 - test 71 The lit.Test to parse. 72 73 - preamble 74 A list of commands to perform before any command in the test. 75 These commands can contain unexpanded substitutions, but they 76 must not be of the form 'RUN:' -- they must be proper commands 77 once substituted. 78 """ 79 # Get the default substitutions 80 tmpDir, tmpBase = _getTempPaths(test) 81 substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase) 82 83 # Check base substitutions and add the %{build}, %{verify} and %{run} convenience substitutions 84 # 85 # Note: We use -Wno-error with %{verify} to make sure that we don't treat all diagnostics as 86 # errors, which doesn't make sense for clang-verify tests because we may want to check 87 # for specific warning diagnostics. 88 _checkBaseSubstitutions(substitutions) 89 substitutions.append( 90 ("%{build}", "%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe") 91 ) 92 substitutions.append( 93 ( 94 "%{verify}", 95 "%{cxx} %s %{flags} %{compile_flags} -fsyntax-only -Wno-error -Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0", 96 ) 97 ) 98 substitutions.append(("%{run}", "%{exec} %t.exe")) 99 100 # Parse the test file, including custom directives 101 additionalCompileFlags = [] 102 fileDependencies = [] 103 modules = [] # The enabled modules 104 moduleCompileFlags = [] # The compilation flags to use modules 105 parsers = [ 106 lit.TestRunner.IntegratedTestKeywordParser( 107 "FILE_DEPENDENCIES:", 108 lit.TestRunner.ParserKind.LIST, 109 initial_value=fileDependencies, 110 ), 111 lit.TestRunner.IntegratedTestKeywordParser( 112 "ADDITIONAL_COMPILE_FLAGS:", 113 lit.TestRunner.ParserKind.SPACE_LIST, 114 initial_value=additionalCompileFlags, 115 ), 116 lit.TestRunner.IntegratedTestKeywordParser( 117 "MODULE_DEPENDENCIES:", 118 lit.TestRunner.ParserKind.SPACE_LIST, 119 initial_value=modules, 120 ), 121 ] 122 123 # Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first 124 # class support for conditional keywords in Lit, which would allow evaluating arbitrary 125 # Lit boolean expressions instead. 126 for feature in test.config.available_features: 127 parser = lit.TestRunner.IntegratedTestKeywordParser( 128 "ADDITIONAL_COMPILE_FLAGS({}):".format(feature), 129 lit.TestRunner.ParserKind.SPACE_LIST, 130 initial_value=additionalCompileFlags, 131 ) 132 parsers.append(parser) 133 134 scriptInTest = lit.TestRunner.parseIntegratedTestScript( 135 test, additional_parsers=parsers, require_script=not preamble 136 ) 137 if isinstance(scriptInTest, lit.Test.Result): 138 return scriptInTest 139 140 script = [] 141 142 # For each file dependency in FILE_DEPENDENCIES, inject a command to copy 143 # that file to the execution directory. Execute the copy from %S to allow 144 # relative paths from the test directory. 145 for dep in fileDependencies: 146 script += ["%dbg(SETUP) cd %S && cp {} %T".format(dep)] 147 script += preamble 148 script += scriptInTest 149 150 # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS. 151 # Modules need to be built with the same compilation flags as the 152 # test. So add these flags before adding the modules. 153 substitutions = config._appendToSubstitution( 154 substitutions, "%{compile_flags}", " ".join(additionalCompileFlags) 155 ) 156 157 if modules: 158 _validateModuleDependencies(modules) 159 160 # The moduleCompileFlags are added to the %{compile_flags}, but 161 # the modules need to be built without these flags. So expand the 162 # %{compile_flags} eagerly and hardcode them in the build script. 163 compileFlags = config._getSubstitution("%{compile_flags}", test.config) 164 165 # Building the modules needs to happen before the other script 166 # commands are executed. Therefore the commands are added to the 167 # front of the list. 168 if "std.compat" in modules: 169 script.insert( 170 0, 171 "%dbg(MODULE std.compat) %{cxx} %{flags} " 172 f"{compileFlags} " 173 "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal " 174 "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std. 175 "--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm", 176 ) 177 moduleCompileFlags.extend( 178 ["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"] 179 ) 180 181 # Make sure the std module is built before std.compat. Libc++'s 182 # std.compat module depends on the std module. It is not 183 # known whether the compiler expects the modules in the order of 184 # their dependencies. However it's trivial to provide them in 185 # that order. 186 script.insert( 187 0, 188 "%dbg(MODULE std) %{cxx} %{flags} " 189 f"{compileFlags} " 190 "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal " 191 "--precompile -o %T/std.pcm -c %{module-dir}/std.cppm", 192 ) 193 moduleCompileFlags.extend(["-fmodule-file=std=%T/std.pcm", "%T/std.pcm"]) 194 195 # Add compile flags required for the modules. 196 substitutions = config._appendToSubstitution( 197 substitutions, "%{compile_flags}", " ".join(moduleCompileFlags) 198 ) 199 200 # Perform substitutions in the script itself. 201 script = lit.TestRunner.applySubstitutions( 202 script, substitutions, recursion_limit=test.config.recursiveExpansionLimit 203 ) 204 205 return script 206 207 208class CxxStandardLibraryTest(lit.formats.FileBasedTest): 209 """ 210 Lit test format for the C++ Standard Library conformance test suite. 211 212 Lit tests are contained in files that follow a certain pattern, which determines the semantics of the test. 213 Under the hood, we basically generate a builtin Lit shell test that follows the ShTest format, and perform 214 the appropriate operations (compile/link/run). See 215 https://libcxx.llvm.org/TestingLibcxx.html#test-names 216 for a complete description of those semantics. 217 218 Substitution requirements 219 =============================== 220 The test format operates by assuming that each test's configuration provides 221 the following substitutions, which it will reuse in the shell scripts it 222 constructs: 223 %{cxx} - A command that can be used to invoke the compiler 224 %{compile_flags} - Flags to use when compiling a test case 225 %{link_flags} - Flags to use when linking a test case 226 %{flags} - Flags to use either when compiling or linking a test case 227 %{exec} - A command to prefix the execution of executables 228 229 Note that when building an executable (as opposed to only compiling a source 230 file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used 231 in the same command line. In other words, the test format doesn't perform 232 separate compilation and linking steps in this case. 233 234 Additional provided substitutions and features 235 ============================================== 236 The test format will define the following substitutions for use inside tests: 237 238 %{build} 239 Expands to a command-line that builds the current source 240 file with the %{flags}, %{compile_flags} and %{link_flags} 241 substitutions, and that produces an executable named %t.exe. 242 243 %{verify} 244 Expands to a command-line that builds the current source 245 file with the %{flags} and %{compile_flags} substitutions 246 and enables clang-verify. This can be used to write .sh.cpp 247 tests that use clang-verify. Note that this substitution can 248 only be used when the 'verify-support' feature is available. 249 250 %{run} 251 Equivalent to `%{exec} %t.exe`. This is intended to be used 252 in conjunction with the %{build} substitution. 253 """ 254 255 def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig): 256 SUPPORTED_SUFFIXES = [ 257 "[.]pass[.]cpp$", 258 "[.]pass[.]mm$", 259 "[.]compile[.]pass[.]cpp$", 260 "[.]compile[.]pass[.]mm$", 261 "[.]compile[.]fail[.]cpp$", 262 "[.]link[.]pass[.]cpp$", 263 "[.]link[.]pass[.]mm$", 264 "[.]link[.]fail[.]cpp$", 265 "[.]sh[.][^.]+$", 266 "[.]gen[.][^.]+$", 267 "[.]verify[.]cpp$", 268 ] 269 270 sourcePath = testSuite.getSourcePath(pathInSuite) 271 filename = os.path.basename(sourcePath) 272 273 # Ignore dot files, excluded tests and tests with an unsupported suffix 274 hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES]) 275 if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename): 276 return 277 278 # If this is a generated test, run the generation step and add 279 # as many Lit tests as necessary. 280 if re.search('[.]gen[.][^.]+$', filename): 281 for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig): 282 yield test 283 else: 284 yield lit.Test.Test(testSuite, pathInSuite, localConfig) 285 286 def execute(self, test, litConfig): 287 supportsVerify = "verify-support" in test.config.available_features 288 filename = test.path_in_suite[-1] 289 290 if re.search("[.]sh[.][^.]+$", filename): 291 steps = [] # The steps are already in the script 292 return self._executeShTest(test, litConfig, steps) 293 elif filename.endswith(".compile.pass.cpp") or filename.endswith( 294 ".compile.pass.mm" 295 ): 296 steps = [ 297 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 298 ] 299 return self._executeShTest(test, litConfig, steps) 300 elif filename.endswith(".compile.fail.cpp"): 301 steps = [ 302 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 303 ] 304 return self._executeShTest(test, litConfig, steps) 305 elif filename.endswith(".link.pass.cpp") or filename.endswith(".link.pass.mm"): 306 steps = [ 307 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe" 308 ] 309 return self._executeShTest(test, litConfig, steps) 310 elif filename.endswith(".link.fail.cpp"): 311 steps = [ 312 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o", 313 "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe", 314 ] 315 return self._executeShTest(test, litConfig, steps) 316 elif filename.endswith(".verify.cpp"): 317 if not supportsVerify: 318 return lit.Test.Result( 319 lit.Test.UNSUPPORTED, 320 "Test {} requires support for Clang-verify, which isn't supported by the compiler".format( 321 test.getFullName() 322 ), 323 ) 324 steps = ["%dbg(COMPILED WITH) %{verify}"] 325 return self._executeShTest(test, litConfig, steps) 326 # Make sure to check these ones last, since they will match other 327 # suffixes above too. 328 elif filename.endswith(".pass.cpp") or filename.endswith(".pass.mm"): 329 steps = [ 330 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", 331 "%dbg(EXECUTED AS) %{exec} %t.exe", 332 ] 333 return self._executeShTest(test, litConfig, steps) 334 else: 335 return lit.Test.Result( 336 lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename) 337 ) 338 339 def _executeShTest(self, test, litConfig, steps): 340 if test.config.unsupported: 341 return lit.Test.Result(lit.Test.UNSUPPORTED, "Test is unsupported") 342 343 script = parseScript(test, steps) 344 if isinstance(script, lit.Test.Result): 345 return script 346 347 if litConfig.noExecute: 348 return lit.Test.Result( 349 lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS 350 ) 351 else: 352 _, tmpBase = _getTempPaths(test) 353 useExternalSh = False 354 return lit.TestRunner._runShTest( 355 test, litConfig, useExternalSh, script, tmpBase 356 ) 357 358 def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig): 359 generator = lit.Test.Test(testSuite, pathInSuite, localConfig) 360 361 # Make sure we have a directory to execute the generator test in 362 generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite)) 363 os.makedirs(generatorExecDir, exist_ok=True) 364 365 # Run the generator test 366 steps = [] # Steps must already be in the script 367 (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps) 368 if exitCode != 0: 369 raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}") 370 371 # Split the generated output into multiple files and generate one test for each file 372 for subfile, content in self._splitFile(out): 373 generatedFile = testSuite.getExecPath(pathInSuite + (subfile,)) 374 os.makedirs(os.path.dirname(generatedFile), exist_ok=True) 375 with open(generatedFile, 'w') as f: 376 f.write(content) 377 yield lit.Test.Test(testSuite, (generatedFile,), localConfig) 378 379 def _splitFile(self, input): 380 DELIM = r'^(//|#)---(.+)' 381 lines = input.splitlines() 382 currentFile = None 383 thisFileContent = [] 384 for line in lines: 385 match = re.match(DELIM, line) 386 if match: 387 if currentFile is not None: 388 yield (currentFile, '\n'.join(thisFileContent)) 389 currentFile = match.group(2).strip() 390 thisFileContent = [] 391 assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}" 392 thisFileContent.append(line) 393 if currentFile is not None: 394 yield (currentFile, '\n'.join(thisFileContent)) 395