1# ===----------------------------------------------------------------------===## 2# 3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4# See https://llvm.org/LICENSE.txt for license information. 5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6# 7# ===----------------------------------------------------------------------===## 8import sys 9import re 10import shlex 11from pathlib import Path 12 13from libcxx.test.dsl import * 14from libcxx.test.features import _isClang, _isAppleClang, _isGCC, _isMSVC 15 16 17_warningFlags = [ 18 "-Werror", 19 "-Wall", 20 "-Wctad-maybe-unsupported", 21 "-Wextra", 22 "-Wshadow", 23 "-Wundef", 24 "-Wunused-template", 25 "-Wno-unused-command-line-argument", 26 "-Wno-attributes", 27 "-Wno-pessimizing-move", 28 "-Wno-noexcept-type", 29 "-Wno-aligned-allocation-unavailable", 30 "-Wno-atomic-alignment", 31 "-Wno-reserved-module-identifier", 32 '-Wdeprecated-copy', 33 '-Wdeprecated-copy-dtor', 34 # GCC warns about places where we might want to add sized allocation/deallocation 35 # functions, but we know better what we're doing/testing in the test suite. 36 "-Wno-sized-deallocation", 37 # Turn off warnings about user-defined literals with reserved suffixes. Those are 38 # just noise since we are testing the Standard Library itself. 39 "-Wno-literal-suffix", # GCC 40 "-Wno-user-defined-literals", # Clang 41 # GCC warns about this when TEST_IS_CONSTANT_EVALUATED is used on a non-constexpr 42 # function. (This mostly happens in C++11 mode.) 43 # TODO(mordante) investigate a solution for this issue. 44 "-Wno-tautological-compare", 45 # -Wstringop-overread and -Wstringop-overflow seem to be a bit buggy currently 46 "-Wno-stringop-overread", 47 "-Wno-stringop-overflow", 48 # These warnings should be enabled in order to support the MSVC 49 # team using the test suite; They enable the warnings below and 50 # expect the test suite to be clean. 51 "-Wsign-compare", 52 "-Wunused-variable", 53 "-Wunused-parameter", 54 "-Wunreachable-code", 55 "-Wno-unused-local-typedef", 56 57 # Disable warnings for extensions used in C++03 58 "-Wno-local-type-template-args", 59 "-Wno-c++11-extensions", 60 61 # TODO(philnik) This fails with the PSTL. 62 "-Wno-unknown-pragmas", 63 # Don't fail compilation in case the compiler fails to perform the requested 64 # loop vectorization. 65 "-Wno-pass-failed", 66 67 # TODO: Find out why GCC warns in lots of places (is this a problem with always_inline?) 68 "-Wno-dangling-reference", 69 "-Wno-mismatched-new-delete", 70 "-Wno-redundant-move", 71 72 # This doesn't make sense in real code, but we have to test it because the standard requires us to not break 73 "-Wno-self-move", 74] 75 76_allStandards = ["c++03", "c++11", "c++14", "c++17", "c++20", "c++23", "c++26"] 77 78 79def getStdFlag(cfg, std): 80 if hasCompileFlag(cfg, "-std=" + std): 81 return "-std=" + std 82 # TODO(LLVM-19) Remove the fallbacks needed for Clang 16. 83 fallbacks = { 84 "c++23": "c++2b", 85 } 86 if std in fallbacks and hasCompileFlag(cfg, "-std=" + fallbacks[std]): 87 return "-std=" + fallbacks[std] 88 return None 89 90 91def getDefaultStdValue(cfg): 92 viable = [s for s in reversed(_allStandards) if getStdFlag(cfg, s)] 93 94 if not viable: 95 raise RuntimeError( 96 "Unable to successfully detect the presence of any -std=c++NN flag. This likely indicates an issue with your compiler." 97 ) 98 99 return viable[0] 100 101 102def getSpeedOptimizationFlag(cfg): 103 if _isClang(cfg) or _isAppleClang(cfg) or _isGCC(cfg): 104 return "-O3" 105 elif _isMSVC(cfg): 106 return "/O2" 107 else: 108 raise RuntimeError( 109 "Can't figure out what compiler is used in the configuration" 110 ) 111 112 113def getSizeOptimizationFlag(cfg): 114 if _isClang(cfg) or _isAppleClang(cfg) or _isGCC(cfg): 115 return "-Os" 116 elif _isMSVC(cfg): 117 return "/O1" 118 else: 119 raise RuntimeError( 120 "Can't figure out what compiler is used in the configuration" 121 ) 122 123 124def testClangTidy(cfg, version, executable): 125 try: 126 if version in commandOutput(cfg, [f"{executable} --version"]): 127 return executable 128 except ConfigurationRuntimeError: 129 return None 130 131 132def getSuitableClangTidy(cfg): 133 # If we didn't build the libcxx-tidy plugin via CMake, we can't run the clang-tidy tests. 134 if ( 135 runScriptExitCode( 136 cfg, ["stat %{test-tools-dir}/clang_tidy_checks/libcxx-tidy.plugin"] 137 ) 138 != 0 139 ): 140 return None 141 142 version = "{__clang_major__}.{__clang_minor__}.{__clang_patchlevel__}".format( 143 **compilerMacros(cfg) 144 ) 145 exe = testClangTidy( 146 cfg, version, "clang-tidy-{__clang_major__}".format(**compilerMacros(cfg)) 147 ) 148 149 if not exe: 150 exe = testClangTidy(cfg, version, "clang-tidy") 151 152 return exe 153 154 155# fmt: off 156DEFAULT_PARAMETERS = [ 157 Parameter( 158 name="compiler", 159 type=str, 160 help="The path of the compiler to use for testing.", 161 actions=lambda cxx: [ 162 AddSubstitution("%{cxx}", shlex.quote(cxx)), 163 ], 164 ), 165 Parameter( 166 name="target_triple", 167 type=str, 168 help="The target triple to compile the test suite for. This must be " 169 "compatible with the target that the tests will be run on.", 170 actions=lambda triple: filter( 171 None, 172 [ 173 AddFeature("target={}".format(triple)), 174 AddFlagIfSupported("--target={}".format(triple)), 175 AddSubstitution("%{triple}", triple), 176 ], 177 ), 178 ), 179 Parameter( 180 name="std", 181 choices=_allStandards, 182 type=str, 183 help="The version of the standard to compile the test suite with.", 184 default=lambda cfg: getDefaultStdValue(cfg), 185 actions=lambda std: [ 186 AddFeature(std), 187 AddSubstitution("%{cxx_std}", re.sub(r"\+", "x", std)), 188 AddCompileFlag(lambda cfg: getStdFlag(cfg, std)), 189 ] 190 + [AddFeature(f"std-at-least-{s}") for s in _allStandards if s <= std], 191 ), 192 Parameter( 193 name="optimization", 194 choices=["none", "speed", "size"], 195 type=str, 196 help="The optimization level to use when compiling the test suite.", 197 default="none", 198 actions=lambda opt: filter(None, [ 199 AddCompileFlag(lambda cfg: getSpeedOptimizationFlag(cfg)) if opt == "speed" else None, 200 AddCompileFlag(lambda cfg: getSizeOptimizationFlag(cfg)) if opt == "size" else None, 201 AddFeature(f'optimization={opt}'), 202 ]), 203 ), 204 Parameter( 205 name="enable_modules", 206 choices=["none", "clang", "clang-lsv"], 207 type=str, 208 help="Whether to build the test suite with modules enabled. " 209 "Select `clang` for Clang modules, and 'clang-lsv' for Clang modules with Local Submodule Visibility.", 210 default="none", 211 actions=lambda modules: filter(None, [ 212 AddFeature("clang-modules-build") if modules in ("clang", "clang-lsv") else None, 213 214 # Note: AppleClang disregards -fmodules entirely when compiling C++, so we also pass -fcxx-modules 215 # to enable modules for C++. 216 AddCompileFlag("-fmodules -fcxx-modules") if modules in ("clang", "clang-lsv") else None, 217 218 # Note: We use a custom modules cache path to make sure that we don't reuse 219 # the default one, which can be shared across CI builds with different 220 # configurations. 221 AddCompileFlag(lambda cfg: f"-fmodules-cache-path={cfg.test_exec_root}/ModuleCache") if modules in ("clang", "clang-lsv") else None, 222 223 AddCompileFlag("-Xclang -fmodules-local-submodule-visibility") if modules == "clang-lsv" else None, 224 ]) 225 ), 226 Parameter( 227 name="enable_exceptions", 228 choices=[True, False], 229 type=bool, 230 default=True, 231 help="Whether to enable exceptions when compiling the test suite.", 232 actions=lambda exceptions: [] if exceptions else [ 233 AddFeature("no-exceptions"), 234 AddCompileFlag("-fno-exceptions") 235 ], 236 ), 237 Parameter( 238 name="enable_rtti", 239 choices=[True, False], 240 type=bool, 241 default=True, 242 help="Whether to enable RTTI when compiling the test suite.", 243 actions=lambda rtti: [] if rtti else [ 244 AddFeature("no-rtti"), 245 AddCompileFlag("-fno-rtti") 246 ], 247 ), 248 Parameter( 249 name="stdlib", 250 choices=["llvm-libc++", "apple-libc++", "libstdc++", "msvc"], 251 type=str, 252 default="llvm-libc++", 253 help="""The C++ Standard Library implementation being tested. 254 255 Note that this parameter can also be used to encode different 'flavors' of the same 256 standard library, such as libc++ as shipped by a different vendor, if it has different 257 properties worth testing. 258 259 The Standard libraries currently supported are: 260 - llvm-libc++: The 'upstream' libc++ as shipped with LLVM. 261 - apple-libc++: libc++ as shipped by Apple. This is basically like the LLVM one, but 262 there are a few differences like installation paths, the use of 263 universal dylibs and the existence of availability markup. 264 - libstdc++: The GNU C++ library typically shipped with GCC. 265 - msvc: The Microsoft implementation of the C++ Standard Library. 266 """, 267 actions=lambda stdlib: filter( 268 None, 269 [ 270 AddFeature("stdlib={}".format(stdlib)), 271 # Also add an umbrella feature 'stdlib=libc++' for all flavors of libc++, to simplify 272 # the test suite. 273 AddFeature("stdlib=libc++") if re.match(r".+-libc\+\+", stdlib) else None, 274 ], 275 ), 276 ), 277 Parameter( 278 name="using_system_stdlib", 279 choices=[True, False], 280 type=bool, 281 default=False, 282 help="""Whether the Standard Library being tested is the one that shipped with the system by default. 283 284 This is different from the 'stdlib' parameter, which describes the flavor of libc++ being 285 tested. 'using_system_stdlib' describes whether the target system passed with 'target_triple' 286 also corresponds to the version of the library being tested. 287 """, 288 actions=lambda is_system: [AddFeature("stdlib=system")] if is_system else [], 289 ), 290 Parameter( 291 name="enable_warnings", 292 choices=[True, False], 293 type=bool, 294 default=True, 295 help="Whether to enable warnings when compiling the test suite.", 296 actions=lambda warnings: [] if not warnings else 297 [AddOptionalWarningFlag(w) for w in _warningFlags] + 298 [AddCompileFlag("-D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER")], 299 ), 300 Parameter( 301 name="use_sanitizer", 302 choices=[ 303 "", 304 "Address", 305 "HWAddress", 306 "Undefined", 307 "Memory", 308 "MemoryWithOrigins", 309 "Thread", 310 "DataFlow", 311 "Leaks", 312 ], 313 type=str, 314 default="", 315 help="An optional sanitizer to enable when building and running the test suite.", 316 actions=lambda sanitizer: filter( 317 None, 318 [ 319 AddFlag("-g -fno-omit-frame-pointer") if sanitizer else None, 320 321 AddFlag("-fsanitize=undefined -fno-sanitize=float-divide-by-zero -fno-sanitize-recover=all") if sanitizer == "Undefined" else None, 322 AddFeature("ubsan") if sanitizer == "Undefined" else None, 323 324 AddFlag("-fsanitize=address") if sanitizer == "Address" else None, 325 AddFeature("asan") if sanitizer == "Address" else None, 326 327 AddFlag("-fsanitize=hwaddress") if sanitizer == "HWAddress" else None, 328 AddFeature("hwasan") if sanitizer == "HWAddress" else None, 329 330 AddFlag("-fsanitize=memory") if sanitizer in ["Memory", "MemoryWithOrigins"] else None, 331 AddFeature("msan") if sanitizer in ["Memory", "MemoryWithOrigins"] else None, 332 AddFlag("-fsanitize-memory-track-origins") if sanitizer == "MemoryWithOrigins" else None, 333 334 AddFlag("-fsanitize=thread") if sanitizer == "Thread" else None, 335 AddFeature("tsan") if sanitizer == "Thread" else None, 336 337 AddFlag("-fsanitize=dataflow") if sanitizer == "DataFlow" else None, 338 AddFlag("-fsanitize=leaks") if sanitizer == "Leaks" else None, 339 340 AddFeature("sanitizer-new-delete") if sanitizer in ["Address", "HWAddress", "Memory", "MemoryWithOrigins", "Thread"] else None, 341 AddFeature("lsan") if sanitizer in ["Address", "HWAddress", "Leaks"] else None, 342 ] 343 ) 344 ), 345 Parameter( 346 name="enable_experimental", 347 choices=[True, False], 348 type=bool, 349 default=True, 350 help="Whether to enable tests for experimental C++ Library features.", 351 actions=lambda experimental: [ 352 # When linking in MSVC mode via the Clang driver, a -l<foo> 353 # maps to <foo>.lib, so we need to use -llibc++experimental here 354 # to make it link against the static libc++experimental.lib. 355 # We can't check for the feature 'msvc' in available_features 356 # as those features are added after processing parameters. 357 AddFeature("c++experimental"), 358 PrependLinkFlag(lambda cfg: "-llibc++experimental" if _isMSVC(cfg) else "-lc++experimental"), 359 AddCompileFlag("-D_LIBCPP_ENABLE_EXPERIMENTAL"), 360 ] 361 if experimental 362 else [ 363 AddFeature("libcpp-has-no-incomplete-pstl"), 364 AddFeature("libcpp-has-no-experimental-tzdb"), 365 AddFeature("libcpp-has-no-experimental-syncstream"), 366 ], 367 ), 368 # TODO: This can be improved once we use a version of GoogleBenchmark that supports the dry-run mode. 369 # See https://github.com/google/benchmark/issues/1827. 370 Parameter( 371 name="enable_benchmarks", 372 choices=["no", "run", "dry-run"], 373 type=str, 374 default="run", 375 help="Whether to run the benchmarks in the test suite, to only dry-run them or to disable them entirely.", 376 actions=lambda mode: [AddFeature(f"enable-benchmarks={mode}")], 377 ), 378 Parameter( 379 name="long_tests", 380 choices=[True, False], 381 type=bool, 382 default=True, 383 help="Whether to enable tests that take longer to run. This can be useful when running on a very slow device.", 384 actions=lambda enabled: [] if not enabled else [AddFeature("long_tests")], 385 ), 386 Parameter( 387 name="large_tests", 388 choices=[True, False], 389 type=bool, 390 default=True, 391 help="Whether to enable tests that use a lot of memory. This can be useful when running on a device with limited amounts of memory.", 392 actions=lambda enabled: [] if not enabled else [AddFeature("large_tests")], 393 ), 394 Parameter( 395 name="hardening_mode", 396 choices=["none", "fast", "extensive", "debug", "undefined"], 397 type=str, 398 default="undefined", 399 help="Whether to enable one of the hardening modes when compiling the test suite. This is only " 400 "meaningful when running the tests against libc++. By default, no hardening mode is specified " 401 "so the default hardening mode of the standard library will be used (if any).", 402 actions=lambda hardening_mode: filter( 403 None, 404 [ 405 AddCompileFlag("-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_NONE") if hardening_mode == "none" else None, 406 AddCompileFlag("-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST") if hardening_mode == "fast" else None, 407 AddCompileFlag("-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE") if hardening_mode == "extensive" else None, 408 AddCompileFlag("-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG") if hardening_mode == "debug" else None, 409 AddFeature("libcpp-hardening-mode={}".format(hardening_mode)) if hardening_mode != "undefined" else None, 410 ], 411 ), 412 ), 413 Parameter( 414 name="additional_features", 415 type=list, 416 default=[], 417 help="A comma-delimited list of additional features that will be enabled when running the tests. " 418 "This should be used sparingly since specifying ad-hoc features manually is error-prone and " 419 "brittle in the long run as changes are made to the test suite.", 420 actions=lambda features: [AddFeature(f) for f in features], 421 ), 422 Parameter( 423 name="enable_transitive_includes", 424 choices=[True, False], 425 type=bool, 426 default=True, 427 help="Whether to enable backwards-compatibility transitive includes when running the tests. This " 428 "is provided to ensure that the trimmed-down version of libc++ does not bit-rot in between " 429 "points at which we bulk-remove transitive includes.", 430 actions=lambda enabled: [] if enabled else [ 431 AddFeature("transitive-includes-disabled"), 432 AddCompileFlag("-D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES"), 433 ], 434 ), 435 Parameter( 436 name="executor", 437 type=str, 438 default=f"{shlex.quote(sys.executable)} {shlex.quote(str(Path(__file__).resolve().parent.parent.parent / 'run.py'))}", 439 help="Custom executor to use instead of the configured default.", 440 actions=lambda executor: [AddSubstitution("%{executor}", executor)], 441 ), 442 Parameter( 443 name='clang-tidy-executable', 444 type=str, 445 default=lambda cfg: getSuitableClangTidy(cfg), 446 help="Selects the clang-tidy executable to use.", 447 actions=lambda exe: [] if exe is None else [ 448 AddFeature('has-clang-tidy'), 449 AddSubstitution('%{clang-tidy}', exe), 450 ] 451 ), 452 Parameter( 453 name='test_frozen_cxx03_headers', 454 type=bool, 455 default=False, 456 help="Whether to test the main or C++03-specific headers. Only changes behaviour when std=c++03.", 457 actions=lambda enabled: [] if not enabled else [AddFlag("-D_LIBCPP_USE_FROZEN_CXX03_HEADERS"), AddFeature("FROZEN-CXX03-HEADERS-FIXME")], 458 ), 459] 460# fmt: on 461