1#===----------------------------------------------------------------------===## 2# 3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4# See https://llvm.org/LICENSE.txt for license information. 5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6# 7#===----------------------------------------------------------------------===## 8 9import copy 10import os 11import pkgutil 12import pipes 13import re 14import shlex 15import shutil 16import sys 17 18from libcxx.compiler import CXXCompiler 19from libcxx.test.target_info import make_target_info 20import libcxx.util 21import libcxx.test.features 22import libcxx.test.newconfig 23import libcxx.test.params 24 25def loadSiteConfig(lit_config, config, param_name, env_name): 26 # We haven't loaded the site specific configuration (the user is 27 # probably trying to run on a test file directly, and either the site 28 # configuration hasn't been created by the build system, or we are in an 29 # out-of-tree build situation). 30 site_cfg = lit_config.params.get(param_name, 31 os.environ.get(env_name)) 32 if not site_cfg: 33 lit_config.warning('No site specific configuration file found!' 34 ' Running the tests in the default configuration.') 35 elif not os.path.isfile(site_cfg): 36 lit_config.fatal( 37 "Specified site configuration file does not exist: '%s'" % 38 site_cfg) 39 else: 40 lit_config.note('using site specific configuration at %s' % site_cfg) 41 ld_fn = lit_config.load_config 42 43 # Null out the load_config function so that lit.site.cfg doesn't 44 # recursively load a config even if it tries. 45 # TODO: This is one hell of a hack. Fix it. 46 def prevent_reload_fn(*args, **kwargs): 47 pass 48 lit_config.load_config = prevent_reload_fn 49 ld_fn(config, site_cfg) 50 lit_config.load_config = ld_fn 51 52# Extract the value of a numeric macro such as __cplusplus or a feature-test 53# macro. 54def intMacroValue(token): 55 return int(token.rstrip('LlUu')) 56 57class Configuration(object): 58 # pylint: disable=redefined-outer-name 59 def __init__(self, lit_config, config): 60 self.lit_config = lit_config 61 self.config = config 62 self.cxx = None 63 self.cxx_is_clang_cl = None 64 self.cxx_stdlib_under_test = None 65 self.project_obj_root = None 66 self.libcxx_src_root = None 67 self.libcxx_obj_root = None 68 self.cxx_library_root = None 69 self.cxx_runtime_root = None 70 self.abi_library_root = None 71 self.link_shared = self.get_lit_bool('enable_shared', default=True) 72 self.debug_build = self.get_lit_bool('debug_build', default=False) 73 self.exec_env = dict() 74 self.use_clang_verify = False 75 76 def get_lit_conf(self, name, default=None): 77 val = self.lit_config.params.get(name, None) 78 if val is None: 79 val = getattr(self.config, name, None) 80 if val is None: 81 val = default 82 return val 83 84 def get_lit_bool(self, name, default=None, env_var=None): 85 def check_value(value, var_name): 86 if value is None: 87 return default 88 if isinstance(value, bool): 89 return value 90 if not isinstance(value, str): 91 raise TypeError('expected bool or string') 92 if value.lower() in ('1', 'true'): 93 return True 94 if value.lower() in ('', '0', 'false'): 95 return False 96 self.lit_config.fatal( 97 "parameter '{}' should be true or false".format(var_name)) 98 99 conf_val = self.get_lit_conf(name) 100 if env_var is not None and env_var in os.environ and \ 101 os.environ[env_var] is not None: 102 val = os.environ[env_var] 103 if conf_val is not None: 104 self.lit_config.warning( 105 'Environment variable %s=%s is overriding explicit ' 106 '--param=%s=%s' % (env_var, val, name, conf_val)) 107 return check_value(val, env_var) 108 return check_value(conf_val, name) 109 110 def make_static_lib_name(self, name): 111 """Return the full filename for the specified library name""" 112 if self.target_info.is_windows() and not self.target_info.is_mingw(): 113 assert name == 'c++' # Only allow libc++ to use this function for now. 114 return 'lib' + name + '.lib' 115 else: 116 return 'lib' + name + '.a' 117 118 def configure(self): 119 self.target_info = make_target_info(self) 120 self.executor = self.get_lit_conf('executor') 121 self.configure_cxx() 122 self.configure_src_root() 123 self.configure_obj_root() 124 self.cxx_stdlib_under_test = self.get_lit_conf('cxx_stdlib_under_test', 'libc++') 125 self.cxx_library_root = self.get_lit_conf('cxx_library_root', self.libcxx_obj_root) 126 self.abi_library_root = self.get_lit_conf('abi_library_root') or self.cxx_library_root 127 self.cxx_runtime_root = self.get_lit_conf('cxx_runtime_root', self.cxx_library_root) 128 self.abi_runtime_root = self.get_lit_conf('abi_runtime_root', self.abi_library_root) 129 self.configure_compile_flags() 130 self.configure_link_flags() 131 self.configure_env() 132 self.configure_coverage() 133 self.configure_modules() 134 self.configure_substitutions() 135 self.configure_features() 136 137 libcxx.test.newconfig.configure( 138 libcxx.test.params.DEFAULT_PARAMETERS, 139 libcxx.test.features.DEFAULT_FEATURES, 140 self.config, 141 self.lit_config 142 ) 143 144 self.lit_config.note("All available features: {}".format(self.config.available_features)) 145 146 def print_config_info(self): 147 if self.cxx.use_modules: 148 self.lit_config.note('Using modules flags: %s' % 149 self.cxx.modules_flags) 150 if len(self.cxx.warning_flags): 151 self.lit_config.note('Using warnings: %s' % self.cxx.warning_flags) 152 show_env_vars = {} 153 for k,v in self.exec_env.items(): 154 if k not in os.environ or os.environ[k] != v: 155 show_env_vars[k] = v 156 self.lit_config.note('Adding environment variables: %r' % show_env_vars) 157 self.lit_config.note("Linking against the C++ Library at {}".format(self.cxx_library_root)) 158 self.lit_config.note("Running against the C++ Library at {}".format(self.cxx_runtime_root)) 159 self.lit_config.note("Linking against the ABI Library at {}".format(self.abi_library_root)) 160 self.lit_config.note("Running against the ABI Library at {}".format(self.abi_runtime_root)) 161 sys.stderr.flush() # Force flushing to avoid broken output on Windows 162 163 def get_test_format(self): 164 from libcxx.test.format import LibcxxTestFormat 165 return LibcxxTestFormat( 166 self.cxx, 167 self.use_clang_verify, 168 self.executor, 169 exec_env=self.exec_env) 170 171 def configure_cxx(self): 172 # Gather various compiler parameters. 173 cxx = self.get_lit_conf('cxx_under_test') 174 self.cxx_is_clang_cl = cxx is not None and \ 175 os.path.basename(cxx).startswith('clang-cl') 176 # If no specific cxx_under_test was given, attempt to infer it as 177 # clang++. 178 if cxx is None or self.cxx_is_clang_cl: 179 search_paths = self.config.environment['PATH'] 180 if cxx is not None and os.path.isabs(cxx): 181 search_paths = os.path.dirname(cxx) 182 clangxx = libcxx.util.which('clang++', search_paths) 183 if clangxx: 184 cxx = clangxx 185 self.lit_config.note( 186 "inferred cxx_under_test as: %r" % cxx) 187 elif self.cxx_is_clang_cl: 188 self.lit_config.fatal('Failed to find clang++ substitution for' 189 ' clang-cl') 190 if not cxx: 191 self.lit_config.fatal('must specify user parameter cxx_under_test ' 192 '(e.g., --param=cxx_under_test=clang++)') 193 self.cxx = CXXCompiler(self, cxx) if not self.cxx_is_clang_cl else \ 194 self._configure_clang_cl(cxx) 195 self.cxx.compile_env = dict(os.environ) 196 197 def _configure_clang_cl(self, clang_path): 198 def _split_env_var(var): 199 return [p.strip() for p in os.environ.get(var, '').split(';') if p.strip()] 200 201 def _prefixed_env_list(var, prefix): 202 from itertools import chain 203 return list(chain.from_iterable((prefix, path) for path in _split_env_var(var))) 204 205 assert self.cxx_is_clang_cl 206 flags = [] 207 compile_flags = [] 208 link_flags = _prefixed_env_list('LIB', '-L') 209 return CXXCompiler(self, clang_path, flags=flags, 210 compile_flags=compile_flags, 211 link_flags=link_flags) 212 213 def configure_src_root(self): 214 self.libcxx_src_root = self.get_lit_conf( 215 'libcxx_src_root', os.path.dirname(self.config.test_source_root)) 216 217 def configure_obj_root(self): 218 self.project_obj_root = self.get_lit_conf('project_obj_root') 219 self.libcxx_obj_root = self.get_lit_conf('libcxx_obj_root') 220 if not self.libcxx_obj_root and self.project_obj_root is not None: 221 possible_roots = [ 222 os.path.join(self.project_obj_root, 'libcxx'), 223 os.path.join(self.project_obj_root, 'projects', 'libcxx'), 224 os.path.join(self.project_obj_root, 'runtimes', 'libcxx'), 225 ] 226 for possible_root in possible_roots: 227 if os.path.isdir(possible_root): 228 self.libcxx_obj_root = possible_root 229 break 230 else: 231 self.libcxx_obj_root = self.project_obj_root 232 233 def configure_features(self): 234 additional_features = self.get_lit_conf('additional_features') 235 if additional_features: 236 for f in additional_features.split(','): 237 self.config.available_features.add(f.strip()) 238 239 if self.target_info.is_windows(): 240 if self.cxx_stdlib_under_test == 'libc++': 241 # LIBCXX-WINDOWS-FIXME is the feature name used to XFAIL the 242 # initial Windows failures until they can be properly diagnosed 243 # and fixed. This allows easier detection of new test failures 244 # and regressions. Note: New failures should not be suppressed 245 # using this feature. (Also see llvm.org/PR32730) 246 self.config.available_features.add('LIBCXX-WINDOWS-FIXME') 247 248 def configure_compile_flags(self): 249 self.configure_default_compile_flags() 250 # Configure extra flags 251 compile_flags_str = self.get_lit_conf('compile_flags', '') 252 self.cxx.compile_flags += shlex.split(compile_flags_str) 253 if self.target_info.is_windows(): 254 self.cxx.compile_flags += ['-D_CRT_SECURE_NO_WARNINGS'] 255 # Don't warn about using common but nonstandard unprefixed functions 256 # like chdir, fileno. 257 self.cxx.compile_flags += ['-D_CRT_NONSTDC_NO_WARNINGS'] 258 # Build the tests in the same configuration as libcxx itself, 259 # to avoid mismatches if linked statically. 260 self.cxx.compile_flags += ['-D_CRT_STDIO_ISO_WIDE_SPECIFIERS'] 261 # Required so that tests using min/max don't fail on Windows, 262 # and so that those tests don't have to be changed to tolerate 263 # this insanity. 264 self.cxx.compile_flags += ['-DNOMINMAX'] 265 additional_flags = self.get_lit_conf('test_compiler_flags') 266 if additional_flags: 267 self.cxx.compile_flags += shlex.split(additional_flags) 268 269 def configure_default_compile_flags(self): 270 # Configure include paths 271 self.configure_compile_flags_header_includes() 272 self.target_info.add_cxx_compile_flags(self.cxx.compile_flags) 273 self.target_info.add_cxx_flags(self.cxx.flags) 274 # Configure feature flags. 275 enable_32bit = self.get_lit_bool('enable_32bit', False) 276 if enable_32bit: 277 self.cxx.flags += ['-m32'] 278 # Use verbose output for better errors 279 self.cxx.flags += ['-v'] 280 sysroot = self.get_lit_conf('sysroot') 281 if sysroot: 282 self.cxx.flags += ['--sysroot=' + sysroot] 283 gcc_toolchain = self.get_lit_conf('gcc_toolchain') 284 if gcc_toolchain: 285 self.cxx.flags += ['--gcc-toolchain=' + gcc_toolchain] 286 # NOTE: the _DEBUG definition must preceed the triple check because for 287 # the Windows build of libc++, the forced inclusion of a header requires 288 # that _DEBUG is defined. Incorrect ordering will result in -target 289 # being elided. 290 if self.target_info.is_windows() and self.debug_build: 291 self.cxx.compile_flags += ['-D_DEBUG'] 292 293 # Add includes for support headers used in the tests. 294 support_path = os.path.join(self.libcxx_src_root, 'test/support') 295 self.cxx.compile_flags += ['-I' + support_path] 296 297 # On GCC, the libc++ headers cause errors due to throw() decorators 298 # on operator new clashing with those from the test suite, so we 299 # don't enable warnings in system headers on GCC. 300 if self.cxx.type != 'gcc': 301 self.cxx.compile_flags += ['-D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER'] 302 303 # Add includes for the PSTL headers 304 pstl_src_root = self.get_lit_conf('pstl_src_root') 305 pstl_obj_root = self.get_lit_conf('pstl_obj_root') 306 if pstl_src_root is not None and pstl_obj_root is not None: 307 self.cxx.compile_flags += ['-I' + os.path.join(pstl_src_root, 'include')] 308 self.cxx.compile_flags += ['-I' + os.path.join(pstl_obj_root, 'generated_headers')] 309 self.cxx.compile_flags += ['-I' + os.path.join(pstl_src_root, 'test')] 310 self.config.available_features.add('parallel-algorithms') 311 312 def configure_compile_flags_header_includes(self): 313 support_path = os.path.join(self.libcxx_src_root, 'test', 'support') 314 if self.cxx_stdlib_under_test != 'libstdc++' and \ 315 not self.target_info.is_windows() and \ 316 not self.target_info.is_zos(): 317 self.cxx.compile_flags += [ 318 '-include', os.path.join(support_path, 'nasty_macros.h')] 319 if self.cxx_stdlib_under_test == 'msvc': 320 self.cxx.compile_flags += [ 321 '-include', os.path.join(support_path, 322 'msvc_stdlib_force_include.h')] 323 pass 324 if self.target_info.is_windows() and self.debug_build and \ 325 self.cxx_stdlib_under_test != 'msvc': 326 self.cxx.compile_flags += [ 327 '-include', os.path.join(support_path, 328 'set_windows_crt_report_mode.h') 329 ] 330 cxx_headers = self.get_lit_conf('cxx_headers') 331 if cxx_headers is None and self.cxx_stdlib_under_test != 'libc++': 332 self.lit_config.note('using the system cxx headers') 333 return 334 self.cxx.compile_flags += ['-nostdinc++'] 335 if not os.path.isdir(cxx_headers): 336 self.lit_config.fatal("cxx_headers='{}' is not a directory.".format(cxx_headers)) 337 (path, version) = os.path.split(cxx_headers) 338 (path, cxx) = os.path.split(path) 339 triple = self.get_lit_conf('target_triple', None) 340 if triple is not None: 341 cxx_target_headers = os.path.join(path, triple, cxx, version) 342 if os.path.isdir(cxx_target_headers): 343 self.cxx.compile_flags += ['-I' + cxx_target_headers] 344 self.cxx.compile_flags += ['-I' + cxx_headers] 345 if self.libcxx_obj_root is not None: 346 cxxabi_headers = os.path.join(self.libcxx_obj_root, 'include', 347 'c++build') 348 if os.path.isdir(cxxabi_headers): 349 self.cxx.compile_flags += ['-I' + cxxabi_headers] 350 351 def configure_link_flags(self): 352 # Configure library path 353 self.configure_link_flags_cxx_library_path() 354 self.configure_link_flags_abi_library_path() 355 356 # Configure libraries 357 if self.cxx_stdlib_under_test == 'libc++': 358 if self.target_info.is_mingw(): 359 self.cxx.link_flags += ['-nostdlib++'] 360 else: 361 self.cxx.link_flags += ['-nodefaultlibs'] 362 # FIXME: Handle MSVCRT as part of the ABI library handling. 363 if self.target_info.is_windows() and not self.target_info.is_mingw(): 364 self.cxx.link_flags += ['-nostdlib'] 365 self.configure_link_flags_cxx_library() 366 self.configure_link_flags_abi_library() 367 self.configure_extra_library_flags() 368 elif self.cxx_stdlib_under_test == 'libstdc++': 369 self.cxx.link_flags += ['-lstdc++fs', '-lm', '-pthread'] 370 elif self.cxx_stdlib_under_test == 'msvc': 371 # FIXME: Correctly setup debug/release flags here. 372 pass 373 elif self.cxx_stdlib_under_test == 'cxx_default': 374 self.cxx.link_flags += ['-pthread'] 375 else: 376 self.lit_config.fatal('invalid stdlib under test') 377 378 link_flags_str = self.get_lit_conf('link_flags', '') 379 self.cxx.link_flags += shlex.split(link_flags_str) 380 381 def configure_link_flags_cxx_library_path(self): 382 if self.cxx_library_root: 383 self.cxx.link_flags += ['-L' + self.cxx_library_root] 384 if self.target_info.is_windows() and self.link_shared: 385 self.add_path(self.cxx.compile_env, self.cxx_library_root) 386 if self.cxx_runtime_root: 387 if not self.target_info.is_windows(): 388 self.cxx.link_flags += ['-Wl,-rpath,' + 389 self.cxx_runtime_root] 390 elif self.target_info.is_windows() and self.link_shared: 391 self.add_path(self.exec_env, self.cxx_runtime_root) 392 additional_flags = self.get_lit_conf('test_linker_flags') 393 if additional_flags: 394 self.cxx.link_flags += shlex.split(additional_flags) 395 396 def configure_link_flags_abi_library_path(self): 397 # Configure ABI library paths. 398 if self.abi_library_root: 399 self.cxx.link_flags += ['-L' + self.abi_library_root] 400 if self.abi_runtime_root: 401 if not self.target_info.is_windows(): 402 self.cxx.link_flags += ['-Wl,-rpath,' + self.abi_runtime_root] 403 else: 404 self.add_path(self.exec_env, self.abi_runtime_root) 405 406 def configure_link_flags_cxx_library(self): 407 if self.link_shared: 408 self.cxx.link_flags += ['-lc++'] 409 else: 410 if self.cxx_library_root: 411 libname = self.make_static_lib_name('c++') 412 abs_path = os.path.join(self.cxx_library_root, libname) 413 assert os.path.exists(abs_path) and \ 414 "static libc++ library does not exist" 415 self.cxx.link_flags += [abs_path] 416 else: 417 self.cxx.link_flags += ['-lc++'] 418 419 def configure_link_flags_abi_library(self): 420 cxx_abi = self.get_lit_conf('cxx_abi', 'libcxxabi') 421 if cxx_abi == 'libstdc++': 422 self.cxx.link_flags += ['-lstdc++'] 423 elif cxx_abi == 'libsupc++': 424 self.cxx.link_flags += ['-lsupc++'] 425 elif cxx_abi == 'libcxxabi': 426 # If the C++ library requires explicitly linking to libc++abi, or 427 # if we're testing libc++abi itself (the test configs are shared), 428 # then link it. 429 testing_libcxxabi = self.get_lit_conf('name', '') == 'libc++abi' 430 if self.target_info.allow_cxxabi_link() or testing_libcxxabi: 431 libcxxabi_shared = self.get_lit_bool('libcxxabi_shared', default=True) 432 if libcxxabi_shared: 433 self.cxx.link_flags += ['-lc++abi'] 434 else: 435 if self.abi_library_root: 436 libname = self.make_static_lib_name('c++abi') 437 abs_path = os.path.join(self.abi_library_root, libname) 438 self.cxx.link_flags += [abs_path] 439 else: 440 self.cxx.link_flags += ['-lc++abi'] 441 elif cxx_abi == 'libcxxrt': 442 self.cxx.link_flags += ['-lcxxrt'] 443 elif cxx_abi == 'vcruntime': 444 debug_suffix = 'd' if self.debug_build else '' 445 # This matches the set of libraries linked in the toplevel 446 # libcxx CMakeLists.txt if building targeting msvc. 447 self.cxx.link_flags += ['-l%s%s' % (lib, debug_suffix) for lib in 448 ['vcruntime', 'ucrt', 'msvcrt', 'msvcprt']] 449 # The compiler normally links in oldnames.lib too, but we've 450 # specified -nostdlib above, so we need to specify it manually. 451 self.cxx.link_flags += ['-loldnames'] 452 elif cxx_abi == 'none' or cxx_abi == 'default': 453 if self.target_info.is_windows(): 454 debug_suffix = 'd' if self.debug_build else '' 455 self.cxx.link_flags += ['-lmsvcrt%s' % debug_suffix] 456 else: 457 self.lit_config.fatal( 458 'C++ ABI setting %s unsupported for tests' % cxx_abi) 459 460 def configure_extra_library_flags(self): 461 if self.get_lit_bool('cxx_ext_threads', default=False): 462 self.cxx.link_flags += ['-lc++external_threads'] 463 self.target_info.add_cxx_link_flags(self.cxx.link_flags) 464 465 def configure_coverage(self): 466 self.generate_coverage = self.get_lit_bool('generate_coverage', False) 467 if self.generate_coverage: 468 self.cxx.flags += ['-g', '--coverage'] 469 self.cxx.compile_flags += ['-O0'] 470 471 def configure_modules(self): 472 modules_flags = ['-fmodules', '-Xclang', '-fmodules-local-submodule-visibility'] 473 supports_modules = self.cxx.hasCompileFlag(modules_flags) 474 enable_modules = self.get_lit_bool('enable_modules', default=False, 475 env_var='LIBCXX_ENABLE_MODULES') 476 if enable_modules and not supports_modules: 477 self.lit_config.fatal( 478 '-fmodules is enabled but not supported by the compiler') 479 if not supports_modules: 480 return 481 module_cache = os.path.join(self.config.test_exec_root, 482 'modules.cache') 483 module_cache = os.path.realpath(module_cache) 484 if os.path.isdir(module_cache): 485 shutil.rmtree(module_cache) 486 os.makedirs(module_cache) 487 self.cxx.modules_flags += modules_flags + \ 488 ['-fmodules-cache-path=' + module_cache] 489 if enable_modules: 490 self.config.available_features.add('-fmodules') 491 self.cxx.useModules() 492 493 def configure_substitutions(self): 494 sub = self.config.substitutions 495 sub.append(('%{cxx}', pipes.quote(self.cxx.path))) 496 flags = self.cxx.flags + (self.cxx.modules_flags if self.cxx.use_modules else []) 497 compile_flags = self.cxx.compile_flags + (self.cxx.warning_flags if self.cxx.use_warnings else []) 498 sub.append(('%{flags}', ' '.join(map(pipes.quote, flags)))) 499 sub.append(('%{compile_flags}', ' '.join(map(pipes.quote, compile_flags)))) 500 sub.append(('%{link_flags}', ' '.join(map(pipes.quote, self.cxx.link_flags)))) 501 502 codesign_ident = self.get_lit_conf('llvm_codesign_identity', '') 503 env_vars = ' '.join('%s=%s' % (k, pipes.quote(v)) for (k, v) in self.exec_env.items()) 504 exec_args = [ 505 '--execdir %T', 506 '--codesign_identity "{}"'.format(codesign_ident), 507 '--env {}'.format(env_vars) 508 ] 509 sub.append(('%{exec}', '{} {} -- '.format(self.executor, ' '.join(exec_args)))) 510 511 def configure_env(self): 512 self.config.environment = dict(os.environ) 513 514 def add_path(self, dest_env, new_path): 515 self.target_info.add_path(dest_env, new_path) 516