1#===----------------------------------------------------------------------===## 2# 3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4# See https://llvm.org/LICENSE.txt for license information. 5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6# 7#===----------------------------------------------------------------------===## 8 9import lit 10import lit.formats 11import os 12import pipes 13import re 14import shutil 15import subprocess 16 17def _supportsVerify(config): 18 """ 19 Determine whether clang-verify is supported by the given configuration. 20 21 This is done by checking whether the %{cxx} substitution in that 22 configuration supports certain compiler flags. 23 """ 24 command = "%{{cxx}} -xc++ {} -Werror -fsyntax-only -Xclang -verify-ignore-unexpected".format(os.devnull) 25 command = lit.TestRunner.applySubstitutions([command], config.substitutions, 26 recursion_limit=config.recursiveExpansionLimit)[0] 27 devNull = open(os.devnull, 'w') 28 result = subprocess.call(command, shell=True, stdout=devNull, stderr=devNull) 29 return result == 0 30 31def _getTempPaths(test): 32 """ 33 Return the values to use for the %T and %t substitutions, respectively. 34 35 The difference between this and Lit's default behavior is that we guarantee 36 that %T is a path unique to the test being run. 37 """ 38 tmpDir, _ = lit.TestRunner.getTempPaths(test) 39 _, testName = os.path.split(test.getExecPath()) 40 tmpDir = os.path.join(tmpDir, testName + '.dir') 41 tmpBase = os.path.join(tmpDir, 't') 42 return tmpDir, tmpBase 43 44def _checkBaseSubstitutions(substitutions): 45 substitutions = [s for (s, _) in substitutions] 46 for s in ['%{cxx}', '%{compile_flags}', '%{link_flags}', '%{flags}', '%{exec}']: 47 assert s in substitutions, "Required substitution {} was not provided".format(s) 48 49def parseScript(test, preamble): 50 """ 51 Extract the script from a test, with substitutions applied. 52 53 Returns a list of commands ready to be executed. 54 55 - test 56 The lit.Test to parse. 57 58 - preamble 59 A list of commands to perform before any command in the test. 60 These commands can contain unexpanded substitutions, but they 61 must not be of the form 'RUN:' -- they must be proper commands 62 once substituted. 63 """ 64 65 # Get the default substitutions 66 tmpDir, tmpBase = _getTempPaths(test) 67 substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase) 68 69 # Check base substitutions and add the %{build} and %{run} convenience substitutions 70 _checkBaseSubstitutions(substitutions) 71 substitutions.append(('%{build}', '%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe')) 72 substitutions.append(('%{run}', '%{exec} %t.exe')) 73 74 # Parse the test file, including custom directives 75 additionalCompileFlags = [] 76 fileDependencies = [] 77 parsers = [ 78 lit.TestRunner.IntegratedTestKeywordParser('FILE_DEPENDENCIES:', 79 lit.TestRunner.ParserKind.LIST, 80 initial_value=fileDependencies), 81 lit.TestRunner.IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS:', 82 lit.TestRunner.ParserKind.LIST, 83 initial_value=additionalCompileFlags) 84 ] 85 86 scriptInTest = lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers, 87 require_script=not preamble) 88 if isinstance(scriptInTest, lit.Test.Result): 89 return scriptInTest 90 91 script = [] 92 93 # For each file dependency in FILE_DEPENDENCIES, inject a command to copy 94 # that file to the execution directory. Execute the copy from %S to allow 95 # relative paths from the test directory. 96 for dep in fileDependencies: 97 script += ['%dbg(SETUP) cd %S && cp {} %T'.format(dep)] 98 script += preamble 99 script += scriptInTest 100 101 # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS. 102 substitutions = [(s, x + ' ' + ' '.join(additionalCompileFlags)) if s == '%{compile_flags}' 103 else (s, x) for (s, x) in substitutions] 104 105 # Perform substitutions in the script itself. 106 script = lit.TestRunner.applySubstitutions(script, substitutions, 107 recursion_limit=test.config.recursiveExpansionLimit) 108 109 return script 110 111 112class CxxStandardLibraryTest(lit.formats.TestFormat): 113 """ 114 Lit test format for the C++ Standard Library conformance test suite. 115 116 This test format is based on top of the ShTest format -- it basically 117 creates a shell script performing the right operations (compile/link/run) 118 based on the extension of the test file it encounters. It supports files 119 with the following extensions: 120 121 FOO.pass.cpp - Compiles, links and runs successfully 122 FOO.pass.mm - Same as .pass.cpp, but for Objective-C++ 123 124 FOO.compile.pass.cpp - Compiles successfully, link and run not attempted 125 FOO.compile.fail.cpp - Does not compile successfully 126 127 FOO.link.pass.cpp - Compiles and links successfully, run not attempted 128 FOO.link.fail.cpp - Compiles successfully, but fails to link 129 130 FOO.sh.<anything> - A builtin Lit Shell test 131 132 FOO.verify.cpp - Compiles with clang-verify. This type of test is 133 automatically marked as UNSUPPORTED if the compiler 134 does not support Clang-verify. 135 136 FOO.fail.cpp - Compiled with clang-verify if clang-verify is 137 supported, and equivalent to a .compile.fail.cpp 138 test otherwise. This is supported only for backwards 139 compatibility with the test suite. 140 141 142 Substitution requirements 143 =============================== 144 The test format operates by assuming that each test's configuration provides 145 the following substitutions, which it will reuse in the shell scripts it 146 constructs: 147 %{cxx} - A command that can be used to invoke the compiler 148 %{compile_flags} - Flags to use when compiling a test case 149 %{link_flags} - Flags to use when linking a test case 150 %{flags} - Flags to use either when compiling or linking a test case 151 %{exec} - A command to prefix the execution of executables 152 153 Note that when building an executable (as opposed to only compiling a source 154 file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used 155 in the same command line. In other words, the test format doesn't perform 156 separate compilation and linking steps in this case. 157 158 159 Additional supported directives 160 =============================== 161 In addition to everything that's supported in Lit ShTests, this test format 162 also understands the following directives inside test files: 163 164 // FILE_DEPENDENCIES: file, directory, /path/to/file 165 166 This directive expresses that the test requires the provided files 167 or directories in order to run. An example is a test that requires 168 some test input stored in a data file. When a test file contains 169 such a directive, this test format will collect them and copy them 170 to the directory represented by %T. The intent is that %T contains 171 all the inputs necessary to run the test, such that e.g. execution 172 on a remote host can be done by simply copying %T to the host. 173 174 // ADDITIONAL_COMPILE_FLAGS: flag1, flag2, flag3 175 176 This directive will cause the provided flags to be added to the 177 %{compile_flags} substitution for the test that contains it. This 178 allows adding special compilation flags without having to use a 179 .sh.cpp test, which would be more powerful but perhaps overkill. 180 181 182 Additional provided substitutions and features 183 ============================================== 184 The test format will define the following substitutions for use inside tests: 185 186 %{build} 187 Expands to a command-line that builds the current source 188 file with the %{flags}, %{compile_flags} and %{link_flags} 189 substitutions, and that produces an executable named %t.exe. 190 191 %{run} 192 Equivalent to `%{exec} %t.exe`. This is intended to be used 193 in conjunction with the %{build} substitution. 194 """ 195 def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig): 196 SUPPORTED_SUFFIXES = ['[.]pass[.]cpp$', '[.]pass[.]mm$', 197 '[.]compile[.]pass[.]cpp$', '[.]compile[.]fail[.]cpp$', 198 '[.]link[.]pass[.]cpp$', '[.]link[.]fail[.]cpp$', 199 '[.]sh[.][^.]+$', 200 '[.]verify[.]cpp$', 201 '[.]fail[.]cpp$'] 202 sourcePath = testSuite.getSourcePath(pathInSuite) 203 for filename in os.listdir(sourcePath): 204 # Ignore dot files and excluded tests. 205 if filename.startswith('.') or filename in localConfig.excludes: 206 continue 207 208 filepath = os.path.join(sourcePath, filename) 209 if not os.path.isdir(filepath): 210 if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]): 211 yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig) 212 213 def _disableWithModules(self, test): 214 with open(test.getSourcePath(), 'rb') as f: 215 contents = f.read() 216 return b'#define _LIBCPP_ASSERT' in contents 217 218 def execute(self, test, litConfig): 219 VERIFY_FLAGS = '-Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0' 220 supportsVerify = _supportsVerify(test.config) 221 filename = test.path_in_suite[-1] 222 223 # TODO(ldionne): We currently disable tests that re-define _LIBCPP_ASSERT 224 # when we run with modules enabled. Instead, we should 225 # split the part that does a death test outside of the 226 # test, and only disable that part when modules are 227 # enabled. 228 if '-fmodules' in test.config.available_features and self._disableWithModules(test): 229 return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test {} is unsupported when modules are enabled') 230 231 if re.search('[.]sh[.][^.]+$', filename): 232 steps = [ ] # The steps are already in the script 233 return self._executeShTest(test, litConfig, steps) 234 elif filename.endswith('.compile.pass.cpp'): 235 steps = [ 236 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 237 ] 238 return self._executeShTest(test, litConfig, steps) 239 elif filename.endswith('.compile.fail.cpp'): 240 steps = [ 241 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 242 ] 243 return self._executeShTest(test, litConfig, steps) 244 elif filename.endswith('.link.pass.cpp'): 245 steps = [ 246 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe" 247 ] 248 return self._executeShTest(test, litConfig, steps) 249 elif filename.endswith('.link.fail.cpp'): 250 steps = [ 251 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o", 252 "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe" 253 ] 254 return self._executeShTest(test, litConfig, steps) 255 elif filename.endswith('.verify.cpp'): 256 if not supportsVerify: 257 return lit.Test.Result(lit.Test.UNSUPPORTED, 258 "Test {} requires support for Clang-verify, which isn't supported by the compiler".format(test.getFullName())) 259 steps = [ 260 # Note: Use -Wno-error to make sure all diagnostics are not treated as errors, 261 # which doesn't make sense for clang-verify tests. 262 "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS) 263 ] 264 return self._executeShTest(test, litConfig, steps) 265 # Make sure to check these ones last, since they will match other 266 # suffixes above too. 267 elif filename.endswith('.pass.cpp') or filename.endswith('.pass.mm'): 268 steps = [ 269 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", 270 "%dbg(EXECUTED AS) %{exec} %t.exe" 271 ] 272 return self._executeShTest(test, litConfig, steps) 273 # This is like a .verify.cpp test when clang-verify is supported, 274 # otherwise it's like a .compile.fail.cpp test. This is only provided 275 # for backwards compatibility with the test suite. 276 elif filename.endswith('.fail.cpp'): 277 if supportsVerify: 278 steps = [ 279 "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS) 280 ] 281 else: 282 steps = [ 283 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 284 ] 285 return self._executeShTest(test, litConfig, steps) 286 else: 287 return lit.Test.Result(lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)) 288 289 # Utility function to add compile flags in lit.local.cfg files. 290 def addCompileFlags(self, config, *flags): 291 string = ' '.join(flags) 292 config.substitutions = [(s, x + ' ' + string) if s == '%{compile_flags}' else (s, x) for (s, x) in config.substitutions] 293 294 def _executeShTest(self, test, litConfig, steps): 295 if test.config.unsupported: 296 return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test is unsupported') 297 298 script = parseScript(test, steps) 299 if isinstance(script, lit.Test.Result): 300 return script 301 302 if litConfig.noExecute: 303 return lit.Test.Result(lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS) 304 else: 305 _, tmpBase = _getTempPaths(test) 306 useExternalSh = False 307 return lit.TestRunner._runShTest(test, litConfig, useExternalSh, script, tmpBase) 308