1# ===----------------------------------------------------------------------===## 2# 3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4# See https://llvm.org/LICENSE.txt for license information. 5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6# 7# ===----------------------------------------------------------------------===## 8 9import lit 10import libcxx.test.config as config 11import lit.formats 12import os 13import re 14 15 16def _getTempPaths(test): 17 """ 18 Return the values to use for the %T and %t substitutions, respectively. 19 20 The difference between this and Lit's default behavior is that we guarantee 21 that %T is a path unique to the test being run. 22 """ 23 tmpDir, _ = lit.TestRunner.getTempPaths(test) 24 _, testName = os.path.split(test.getExecPath()) 25 tmpDir = os.path.join(tmpDir, testName + ".dir") 26 tmpBase = os.path.join(tmpDir, "t") 27 return tmpDir, tmpBase 28 29 30def _checkBaseSubstitutions(substitutions): 31 substitutions = [s for (s, _) in substitutions] 32 for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{benchmark_flags}", "%{flags}", "%{exec}"]: 33 assert s in substitutions, "Required substitution {} was not provided".format(s) 34 35def _executeScriptInternal(test, litConfig, commands): 36 """ 37 Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands) 38 39 TODO: This really should be easier to access from Lit itself 40 """ 41 parsedCommands = parseScript(test, preamble=commands) 42 43 _, tmpBase = _getTempPaths(test) 44 execDir = os.path.dirname(test.getExecPath()) 45 try: 46 res = lit.TestRunner.executeScriptInternal( 47 test, litConfig, tmpBase, parsedCommands, execDir, debug=False 48 ) 49 except lit.TestRunner.ScriptFatal as e: 50 res = ("", str(e), 127, None) 51 (out, err, exitCode, timeoutInfo) = res 52 53 return (out, err, exitCode, timeoutInfo, parsedCommands) 54 55 56def _validateModuleDependencies(modules): 57 for m in modules: 58 if m not in ("std", "std.compat"): 59 raise RuntimeError( 60 f"Invalid module dependency '{m}', only 'std' and 'std.compat' are valid" 61 ) 62 63 64def parseScript(test, preamble): 65 """ 66 Extract the script from a test, with substitutions applied. 67 68 Returns a list of commands ready to be executed. 69 70 - test 71 The lit.Test to parse. 72 73 - preamble 74 A list of commands to perform before any command in the test. 75 These commands can contain unexpanded substitutions, but they 76 must not be of the form 'RUN:' -- they must be proper commands 77 once substituted. 78 """ 79 # Get the default substitutions 80 tmpDir, tmpBase = _getTempPaths(test) 81 substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase) 82 83 # Check base substitutions and add the %{build}, %{verify} and %{run} convenience substitutions 84 # 85 # Note: We use -Wno-error with %{verify} to make sure that we don't treat all diagnostics as 86 # errors, which doesn't make sense for clang-verify tests because we may want to check 87 # for specific warning diagnostics. 88 _checkBaseSubstitutions(substitutions) 89 substitutions.append( 90 ("%{build}", "%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe") 91 ) 92 substitutions.append( 93 ( 94 "%{verify}", 95 "%{cxx} %s %{flags} %{compile_flags} -fsyntax-only -Wno-error -Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0", 96 ) 97 ) 98 substitutions.append(("%{run}", "%{exec} %t.exe")) 99 100 # Parse the test file, including custom directives 101 additionalCompileFlags = [] 102 fileDependencies = [] 103 modules = [] # The enabled modules 104 moduleCompileFlags = [] # The compilation flags to use modules 105 parsers = [ 106 lit.TestRunner.IntegratedTestKeywordParser( 107 "FILE_DEPENDENCIES:", 108 lit.TestRunner.ParserKind.LIST, 109 initial_value=fileDependencies, 110 ), 111 lit.TestRunner.IntegratedTestKeywordParser( 112 "ADDITIONAL_COMPILE_FLAGS:", 113 lit.TestRunner.ParserKind.SPACE_LIST, 114 initial_value=additionalCompileFlags, 115 ), 116 lit.TestRunner.IntegratedTestKeywordParser( 117 "MODULE_DEPENDENCIES:", 118 lit.TestRunner.ParserKind.SPACE_LIST, 119 initial_value=modules, 120 ), 121 ] 122 123 # Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first 124 # class support for conditional keywords in Lit, which would allow evaluating arbitrary 125 # Lit boolean expressions instead. 126 for feature in test.config.available_features: 127 parser = lit.TestRunner.IntegratedTestKeywordParser( 128 "ADDITIONAL_COMPILE_FLAGS({}):".format(feature), 129 lit.TestRunner.ParserKind.SPACE_LIST, 130 initial_value=additionalCompileFlags, 131 ) 132 parsers.append(parser) 133 134 scriptInTest = lit.TestRunner.parseIntegratedTestScript( 135 test, additional_parsers=parsers, require_script=not preamble 136 ) 137 if isinstance(scriptInTest, lit.Test.Result): 138 return scriptInTest 139 140 script = [] 141 142 # For each file dependency in FILE_DEPENDENCIES, inject a command to copy 143 # that file to the execution directory. Execute the copy from %S to allow 144 # relative paths from the test directory. 145 for dep in fileDependencies: 146 script += ["%dbg(SETUP) cd %S && cp {} %T".format(dep)] 147 script += preamble 148 script += scriptInTest 149 150 # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS. 151 # Modules need to be built with the same compilation flags as the 152 # test. So add these flags before adding the modules. 153 substitutions = config._appendToSubstitution( 154 substitutions, "%{compile_flags}", " ".join(additionalCompileFlags) 155 ) 156 157 if modules: 158 _validateModuleDependencies(modules) 159 160 # The moduleCompileFlags are added to the %{compile_flags}, but 161 # the modules need to be built without these flags. So expand the 162 # %{compile_flags} eagerly and hardcode them in the build script. 163 compileFlags = config._getSubstitution("%{compile_flags}", test.config) 164 165 # Building the modules needs to happen before the other script 166 # commands are executed. Therefore the commands are added to the 167 # front of the list. 168 if "std.compat" in modules: 169 script.insert( 170 0, 171 "%dbg(MODULE std.compat) %{cxx} %{flags} " 172 f"{compileFlags} " 173 "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal " 174 "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std. 175 "--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm", 176 ) 177 moduleCompileFlags.extend( 178 ["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"] 179 ) 180 181 # Make sure the std module is built before std.compat. Libc++'s 182 # std.compat module depends on the std module. It is not 183 # known whether the compiler expects the modules in the order of 184 # their dependencies. However it's trivial to provide them in 185 # that order. 186 script.insert( 187 0, 188 "%dbg(MODULE std) %{cxx} %{flags} " 189 f"{compileFlags} " 190 "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal " 191 "--precompile -o %T/std.pcm -c %{module-dir}/std.cppm", 192 ) 193 moduleCompileFlags.extend(["-fmodule-file=std=%T/std.pcm", "%T/std.pcm"]) 194 195 # Add compile flags required for the modules. 196 substitutions = config._appendToSubstitution( 197 substitutions, "%{compile_flags}", " ".join(moduleCompileFlags) 198 ) 199 200 # Perform substitutions in the script itself. 201 script = lit.TestRunner.applySubstitutions( 202 script, substitutions, recursion_limit=test.config.recursiveExpansionLimit 203 ) 204 205 return script 206 207 208class CxxStandardLibraryTest(lit.formats.FileBasedTest): 209 """ 210 Lit test format for the C++ Standard Library conformance test suite. 211 212 Lit tests are contained in files that follow a certain pattern, which determines the semantics of the test. 213 Under the hood, we basically generate a builtin Lit shell test that follows the ShTest format, and perform 214 the appropriate operations (compile/link/run). See 215 https://libcxx.llvm.org/TestingLibcxx.html#test-names 216 for a complete description of those semantics. 217 218 Substitution requirements 219 =============================== 220 The test format operates by assuming that each test's configuration provides 221 the following substitutions, which it will reuse in the shell scripts it 222 constructs: 223 %{cxx} - A command that can be used to invoke the compiler 224 %{compile_flags} - Flags to use when compiling a test case 225 %{link_flags} - Flags to use when linking a test case 226 %{flags} - Flags to use either when compiling or linking a test case 227 %{benchmark_flags} - Flags to use when compiling benchmarks. These flags should provide access to 228 GoogleBenchmark but shouldn't hardcode any optimization level or other settings, 229 since the benchmarks should be run under the same configuration as the rest of 230 the test suite. 231 %{exec} - A command to prefix the execution of executables 232 233 Note that when building an executable (as opposed to only compiling a source 234 file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used 235 in the same command line. In other words, the test format doesn't perform 236 separate compilation and linking steps in this case. 237 238 Additional provided substitutions and features 239 ============================================== 240 The test format will define the following substitutions for use inside tests: 241 242 %{build} 243 Expands to a command-line that builds the current source 244 file with the %{flags}, %{compile_flags} and %{link_flags} 245 substitutions, and that produces an executable named %t.exe. 246 247 %{verify} 248 Expands to a command-line that builds the current source 249 file with the %{flags} and %{compile_flags} substitutions 250 and enables clang-verify. This can be used to write .sh.cpp 251 tests that use clang-verify. Note that this substitution can 252 only be used when the 'verify-support' feature is available. 253 254 %{run} 255 Equivalent to `%{exec} %t.exe`. This is intended to be used 256 in conjunction with the %{build} substitution. 257 """ 258 259 def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig): 260 SUPPORTED_SUFFIXES = [ 261 "[.]bench[.]cpp$", 262 "[.]pass[.]cpp$", 263 "[.]pass[.]mm$", 264 "[.]compile[.]pass[.]cpp$", 265 "[.]compile[.]pass[.]mm$", 266 "[.]compile[.]fail[.]cpp$", 267 "[.]link[.]pass[.]cpp$", 268 "[.]link[.]pass[.]mm$", 269 "[.]link[.]fail[.]cpp$", 270 "[.]sh[.][^.]+$", 271 "[.]gen[.][^.]+$", 272 "[.]verify[.]cpp$", 273 ] 274 275 sourcePath = testSuite.getSourcePath(pathInSuite) 276 filename = os.path.basename(sourcePath) 277 278 # Ignore dot files, excluded tests and tests with an unsupported suffix 279 hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES]) 280 if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename): 281 return 282 283 # If this is a generated test, run the generation step and add 284 # as many Lit tests as necessary. 285 if re.search('[.]gen[.][^.]+$', filename): 286 for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig): 287 yield test 288 else: 289 yield lit.Test.Test(testSuite, pathInSuite, localConfig) 290 291 def execute(self, test, litConfig): 292 supportsVerify = "verify-support" in test.config.available_features 293 filename = test.path_in_suite[-1] 294 295 if re.search("[.]sh[.][^.]+$", filename): 296 steps = [] # The steps are already in the script 297 return self._executeShTest(test, litConfig, steps) 298 elif filename.endswith(".compile.pass.cpp") or filename.endswith( 299 ".compile.pass.mm" 300 ): 301 steps = [ 302 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 303 ] 304 return self._executeShTest(test, litConfig, steps) 305 elif filename.endswith(".compile.fail.cpp"): 306 steps = [ 307 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 308 ] 309 return self._executeShTest(test, litConfig, steps) 310 elif filename.endswith(".link.pass.cpp") or filename.endswith(".link.pass.mm"): 311 steps = [ 312 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe" 313 ] 314 return self._executeShTest(test, litConfig, steps) 315 elif filename.endswith(".link.fail.cpp"): 316 steps = [ 317 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o", 318 "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe", 319 ] 320 return self._executeShTest(test, litConfig, steps) 321 elif filename.endswith(".verify.cpp"): 322 if not supportsVerify: 323 return lit.Test.Result( 324 lit.Test.UNSUPPORTED, 325 "Test {} requires support for Clang-verify, which isn't supported by the compiler".format( 326 test.getFullName() 327 ), 328 ) 329 steps = ["%dbg(COMPILED WITH) %{verify}"] 330 return self._executeShTest(test, litConfig, steps) 331 # Make sure to check these ones last, since they will match other 332 # suffixes above too. 333 elif filename.endswith(".pass.cpp") or filename.endswith(".pass.mm"): 334 steps = [ 335 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", 336 "%dbg(EXECUTED AS) %{exec} %t.exe", 337 ] 338 return self._executeShTest(test, litConfig, steps) 339 elif filename.endswith(".bench.cpp"): 340 if "enable-benchmarks=no" in test.config.available_features: 341 return lit.Test.Result( 342 lit.Test.UNSUPPORTED, 343 "Test {} requires support for benchmarks, which isn't supported by this configuration".format( 344 test.getFullName() 345 ), 346 ) 347 steps = [ 348 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{benchmark_flags} %{link_flags} -o %t.exe", 349 ] 350 if "enable-benchmarks=run" in test.config.available_features: 351 steps += ["%dbg(EXECUTED AS) %{exec} %t.exe --benchmark_out=%T/benchmark-result.json --benchmark_out_format=json"] 352 return self._executeShTest(test, litConfig, steps) 353 else: 354 return lit.Test.Result( 355 lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename) 356 ) 357 358 def _executeShTest(self, test, litConfig, steps): 359 if test.config.unsupported: 360 return lit.Test.Result(lit.Test.UNSUPPORTED, "Test is unsupported") 361 362 script = parseScript(test, steps) 363 if isinstance(script, lit.Test.Result): 364 return script 365 366 if litConfig.noExecute: 367 return lit.Test.Result( 368 lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS 369 ) 370 else: 371 _, tmpBase = _getTempPaths(test) 372 useExternalSh = False 373 return lit.TestRunner._runShTest( 374 test, litConfig, useExternalSh, script, tmpBase 375 ) 376 377 def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig): 378 generator = lit.Test.Test(testSuite, pathInSuite, localConfig) 379 380 # Make sure we have a directory to execute the generator test in 381 generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite)) 382 os.makedirs(generatorExecDir, exist_ok=True) 383 384 # Run the generator test 385 steps = [] # Steps must already be in the script 386 (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps) 387 if exitCode != 0: 388 raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}") 389 390 # Split the generated output into multiple files and generate one test for each file 391 for subfile, content in self._splitFile(out): 392 generatedFile = testSuite.getExecPath(pathInSuite + (subfile,)) 393 os.makedirs(os.path.dirname(generatedFile), exist_ok=True) 394 with open(generatedFile, 'w') as f: 395 f.write(content) 396 yield lit.Test.Test(testSuite, (generatedFile,), localConfig) 397 398 def _splitFile(self, input): 399 DELIM = r'^(//|#)---(.+)' 400 lines = input.splitlines() 401 currentFile = None 402 thisFileContent = [] 403 for line in lines: 404 match = re.match(DELIM, line) 405 if match: 406 if currentFile is not None: 407 yield (currentFile, '\n'.join(thisFileContent)) 408 currentFile = match.group(2).strip() 409 thisFileContent = [] 410 assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}" 411 thisFileContent.append(line) 412 if currentFile is not None: 413 yield (currentFile, '\n'.join(thisFileContent)) 414