1""" 2A simple testing framework for lldb using python's unit testing framework. 3 4Tests for lldb are written as python scripts which take advantage of the script 5bridging provided by LLDB.framework to interact with lldb core. 6 7A specific naming pattern is followed by the .py script to be recognized as 8a module which implements a test scenario, namely, Test*.py. 9 10To specify the directories where "Test*.py" python test scripts are located, 11you need to pass in a list of directory names. By default, the current 12working directory is searched if nothing is specified on the command line. 13 14Type: 15 16./dotest.py -h 17 18for available options. 19""" 20 21from __future__ import absolute_import 22from __future__ import print_function 23 24# System modules 25import atexit 26import datetime 27import errno 28import logging 29import os 30import platform 31import re 32import signal 33import subprocess 34import sys 35import tempfile 36 37# Third-party modules 38import six 39import unittest2 40 41# LLDB Modules 42import lldbsuite 43from . import configuration 44from . import dotest_args 45from . import lldbtest_config 46from . import test_categories 47from lldbsuite.test_event import formatter 48from . import test_result 49from lldbsuite.test_event.event_builder import EventBuilder 50from ..support import seven 51 52 53def is_exe(fpath): 54 """Returns true if fpath is an executable.""" 55 if fpath == None: 56 return False 57 return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 58 59 60def which(program): 61 """Returns the full path to a program; None otherwise.""" 62 fpath, _ = os.path.split(program) 63 if fpath: 64 if is_exe(program): 65 return program 66 else: 67 for path in os.environ["PATH"].split(os.pathsep): 68 exe_file = os.path.join(path, program) 69 if is_exe(exe_file): 70 return exe_file 71 return None 72 73 74def usage(parser): 75 parser.print_help() 76 if configuration.verbose > 0: 77 print(""" 78Examples: 79 80This is an example of using the -f option to pinpoint to a specific test class 81and test method to be run: 82 83$ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command 84---------------------------------------------------------------------- 85Collected 1 test 86 87test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase) 88Test 'frame variable this' when stopped on a class constructor. ... ok 89 90---------------------------------------------------------------------- 91Ran 1 test in 1.396s 92 93OK 94 95And this is an example of using the -p option to run a single file (the filename 96matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'): 97 98$ ./dotest.py -v -p ObjC 99---------------------------------------------------------------------- 100Collected 4 tests 101 102test_break_with_dsym (TestObjCMethods.FoundationTestCase) 103Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok 104test_break_with_dwarf (TestObjCMethods.FoundationTestCase) 105Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok 106test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase) 107Lookup objective-c data types and evaluate expressions. ... ok 108test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase) 109Lookup objective-c data types and evaluate expressions. ... ok 110 111---------------------------------------------------------------------- 112Ran 4 tests in 16.661s 113 114OK 115 116Running of this script also sets up the LLDB_TEST environment variable so that 117individual test cases can locate their supporting files correctly. The script 118tries to set up Python's search paths for modules by looking at the build tree 119relative to this script. See also the '-i' option in the following example. 120 121Finally, this is an example of using the lldb.py module distributed/installed by 122Xcode4 to run against the tests under the 'forward' directory, and with the '-w' 123option to add some delay between two tests. It uses ARCH=x86_64 to specify that 124as the architecture and CC=clang to specify the compiler used for the test run: 125 126$ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward 127 128Session logs for test failures/errors will go into directory '2010-11-11-13_56_16' 129---------------------------------------------------------------------- 130Collected 2 tests 131 132test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase) 133Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok 134test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase) 135Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok 136 137---------------------------------------------------------------------- 138Ran 2 tests in 5.659s 139 140OK 141 142The 'Session ...' verbiage is recently introduced (see also the '-s' option) to 143notify the directory containing the session logs for test failures or errors. 144In case there is any test failure/error, a similar message is appended at the 145end of the stderr output for your convenience. 146 147ENABLING LOGS FROM TESTS 148 149Option 1: 150 151Writing logs into different files per test case:: 152 153$ ./dotest.py --channel "lldb all" 154 155$ ./dotest.py --channel "lldb all" --channel "gdb-remote packets" 156 157These log files are written to: 158 159<session-dir>/<test-id>-host.log (logs from lldb host process) 160<session-dir>/<test-id>-server.log (logs from debugserver/lldb-server) 161<session-dir>/<test-id>-<test-result>.log (console logs) 162 163By default, logs from successful runs are deleted. Use the --log-success flag 164to create reference logs for debugging. 165 166$ ./dotest.py --log-success 167 168""") 169 sys.exit(0) 170 171 172def parseExclusion(exclusion_file): 173 """Parse an exclusion file, of the following format, where 174 'skip files', 'skip methods', 'xfail files', and 'xfail methods' 175 are the possible list heading values: 176 177 skip files 178 <file name> 179 <file name> 180 181 xfail methods 182 <method name> 183 """ 184 excl_type = None 185 186 with open(exclusion_file) as f: 187 for line in f: 188 line = line.strip() 189 if not excl_type: 190 excl_type = line 191 continue 192 193 if not line: 194 excl_type = None 195 elif excl_type == 'skip': 196 if not configuration.skip_tests: 197 configuration.skip_tests = [] 198 configuration.skip_tests.append(line) 199 elif excl_type == 'xfail': 200 if not configuration.xfail_tests: 201 configuration.xfail_tests = [] 202 configuration.xfail_tests.append(line) 203 204 205def parseOptionsAndInitTestdirs(): 206 """Initialize the list of directories containing our unittest scripts. 207 208 '-h/--help as the first option prints out usage info and exit the program. 209 """ 210 211 do_help = False 212 213 platform_system = platform.system() 214 platform_machine = platform.machine() 215 216 try: 217 parser = dotest_args.create_parser() 218 args = parser.parse_args() 219 except: 220 raise 221 222 if args.unset_env_varnames: 223 for env_var in args.unset_env_varnames: 224 if env_var in os.environ: 225 # From Python Doc: When unsetenv() is supported, deletion of items in os.environ 226 # is automatically translated into a corresponding call to 227 # unsetenv(). 228 del os.environ[env_var] 229 # os.unsetenv(env_var) 230 231 if args.set_env_vars: 232 for env_var in args.set_env_vars: 233 parts = env_var.split('=', 1) 234 if len(parts) == 1: 235 os.environ[parts[0]] = "" 236 else: 237 os.environ[parts[0]] = parts[1] 238 239 if args.set_inferior_env_vars: 240 lldbtest_config.inferior_env = ' '.join(args.set_inferior_env_vars) 241 242 if args.h: 243 do_help = True 244 245 if args.compiler: 246 configuration.compiler = os.path.realpath(args.compiler) 247 if not is_exe(configuration.compiler): 248 configuration.compiler = which(args.compiler) 249 if not is_exe(configuration.compiler): 250 logging.error( 251 '%s is not a valid compiler executable; aborting...', 252 args.compiler) 253 sys.exit(-1) 254 else: 255 # Use a compiler appropriate appropriate for the Apple SDK if one was 256 # specified 257 if platform_system == 'Darwin' and args.apple_sdk: 258 configuration.compiler = seven.get_command_output( 259 'xcrun -sdk "%s" -find clang 2> /dev/null' % 260 (args.apple_sdk)) 261 else: 262 # 'clang' on ubuntu 14.04 is 3.4 so we try clang-3.5 first 263 candidateCompilers = ['clang-3.5', 'clang', 'gcc'] 264 for candidate in candidateCompilers: 265 if which(candidate): 266 configuration.compiler = candidate 267 break 268 269 if args.dsymutil: 270 configuration.dsymutil = args.dsymutil 271 elif platform_system == 'Darwin': 272 configuration.dsymutil = seven.get_command_output( 273 'xcrun -find -toolchain default dsymutil') 274 275 276 # The lldb-dotest script produced by the CMake build passes in a path to a 277 # working FileCheck and yaml2obj binary. So does one specific Xcode 278 # project target. However, when invoking dotest.py directly, a valid 279 # --filecheck and --yaml2obj option needs to be given. 280 if args.filecheck: 281 configuration.filecheck = os.path.abspath(args.filecheck) 282 283 if args.yaml2obj: 284 configuration.yaml2obj = os.path.abspath(args.yaml2obj) 285 286 if not configuration.get_filecheck_path(): 287 logging.warning('No valid FileCheck executable; some tests may fail...') 288 logging.warning('(Double-check the --filecheck argument to dotest.py)') 289 290 if args.channels: 291 lldbtest_config.channels = args.channels 292 293 if args.log_success: 294 lldbtest_config.log_success = args.log_success 295 296 if args.out_of_tree_debugserver: 297 lldbtest_config.out_of_tree_debugserver = args.out_of_tree_debugserver 298 299 # Set SDKROOT if we are using an Apple SDK 300 if platform_system == 'Darwin' and args.apple_sdk: 301 configuration.sdkroot = seven.get_command_output( 302 'xcrun --sdk "%s" --show-sdk-path 2> /dev/null' % 303 (args.apple_sdk)) 304 305 if args.arch: 306 configuration.arch = args.arch 307 else: 308 configuration.arch = platform_machine 309 310 if args.categories_list: 311 configuration.categories_list = set( 312 test_categories.validate( 313 args.categories_list, False)) 314 configuration.use_categories = True 315 else: 316 configuration.categories_list = [] 317 318 if args.skip_categories: 319 configuration.skip_categories += test_categories.validate( 320 args.skip_categories, False) 321 322 if args.xfail_categories: 323 configuration.xfail_categories += test_categories.validate( 324 args.xfail_categories, False) 325 326 if args.E: 327 os.environ['CFLAGS_EXTRAS'] = args.E 328 329 if args.dwarf_version: 330 configuration.dwarf_version = args.dwarf_version 331 # We cannot modify CFLAGS_EXTRAS because they're used in test cases 332 # that explicitly require no debug info. 333 os.environ['CFLAGS'] = '-gdwarf-{}'.format(configuration.dwarf_version) 334 335 if args.settings: 336 for setting in args.settings: 337 if not len(setting) == 1 or not setting[0].count('='): 338 logging.error('"%s" is not a setting in the form "key=value"', 339 setting[0]) 340 sys.exit(-1) 341 setting_list = setting[0].split('=', 1) 342 configuration.settings.append((setting_list[0], setting_list[1])) 343 344 if args.d: 345 sys.stdout.write( 346 "Suspending the process %d to wait for debugger to attach...\n" % 347 os.getpid()) 348 sys.stdout.flush() 349 os.kill(os.getpid(), signal.SIGSTOP) 350 351 if args.f: 352 if any([x.startswith('-') for x in args.f]): 353 usage(parser) 354 configuration.filters.extend(args.f) 355 356 if args.framework: 357 configuration.lldb_framework_path = args.framework 358 359 if args.executable: 360 # lldb executable is passed explicitly 361 lldbtest_config.lldbExec = os.path.realpath(args.executable) 362 if not is_exe(lldbtest_config.lldbExec): 363 lldbtest_config.lldbExec = which(args.executable) 364 if not is_exe(lldbtest_config.lldbExec): 365 logging.error( 366 '%s is not a valid executable to test; aborting...', 367 args.executable) 368 sys.exit(-1) 369 370 if args.server and args.out_of_tree_debugserver: 371 logging.warning('Both --server and --out-of-tree-debugserver are set') 372 373 if args.server and not args.out_of_tree_debugserver: 374 os.environ['LLDB_DEBUGSERVER_PATH'] = args.server 375 376 if args.excluded: 377 for excl_file in args.excluded: 378 parseExclusion(excl_file) 379 380 if args.p: 381 if args.p.startswith('-'): 382 usage(parser) 383 configuration.regexp = args.p 384 385 if args.s: 386 configuration.sdir_name = args.s 387 else: 388 timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S") 389 configuration.sdir_name = os.path.join(os.getcwd(), timestamp_started) 390 391 configuration.session_file_format = args.session_file_format 392 393 if args.t: 394 os.environ['LLDB_COMMAND_TRACE'] = 'YES' 395 396 if args.v: 397 configuration.verbose = 2 398 399 # argparse makes sure we have a number 400 if args.sharp: 401 configuration.count = args.sharp 402 403 if sys.platform.startswith('win32'): 404 os.environ['LLDB_DISABLE_CRASH_DIALOG'] = str( 405 args.disable_crash_dialog) 406 os.environ['LLDB_LAUNCH_INFERIORS_WITHOUT_CONSOLE'] = str(True) 407 408 if do_help: 409 usage(parser) 410 411 # Reproducer arguments 412 if args.capture_path and args.replay_path: 413 logging.error('Cannot specify both a capture and a replay path.') 414 sys.exit(-1) 415 416 if args.capture_path: 417 configuration.capture_path = args.capture_path 418 419 if args.replay_path: 420 configuration.replay_path = args.replay_path 421 422 # rerun-related arguments 423 configuration.rerun_all_issues = args.rerun_all_issues 424 425 if args.lldb_platform_name: 426 configuration.lldb_platform_name = args.lldb_platform_name 427 if args.lldb_platform_url: 428 configuration.lldb_platform_url = args.lldb_platform_url 429 if args.lldb_platform_working_dir: 430 configuration.lldb_platform_working_dir = args.lldb_platform_working_dir 431 if args.test_build_dir: 432 configuration.test_build_dir = args.test_build_dir 433 if args.lldb_module_cache_dir: 434 configuration.lldb_module_cache_dir = args.lldb_module_cache_dir 435 else: 436 configuration.lldb_module_cache_dir = os.path.join( 437 configuration.test_build_dir, 'module-cache-lldb') 438 if args.clang_module_cache_dir: 439 configuration.clang_module_cache_dir = args.clang_module_cache_dir 440 else: 441 configuration.clang_module_cache_dir = os.path.join( 442 configuration.test_build_dir, 'module-cache-clang') 443 444 if args.lldb_libs_dir: 445 configuration.lldb_libs_dir = args.lldb_libs_dir 446 447 if args.enabled_plugins: 448 configuration.enabled_plugins = args.enabled_plugins 449 450 # Gather all the dirs passed on the command line. 451 if len(args.args) > 0: 452 configuration.testdirs = [os.path.realpath(os.path.abspath(x)) for x in args.args] 453 454 lldbtest_config.codesign_identity = args.codesign_identity 455 456 457def setupTestResults(): 458 """Sets up test results-related objects based on arg settings.""" 459 460 # Create the results formatter. 461 formatter_spec = formatter.create_results_formatter( 462 "lldbsuite.test_event.formatter.results_formatter.ResultsFormatter") 463 if formatter_spec is not None and formatter_spec.formatter is not None: 464 configuration.results_formatter_object = formatter_spec.formatter 465 466 # Send an initialize message to the formatter. 467 initialize_event = EventBuilder.bare_event("initialize") 468 initialize_event["worker_count"] = 1 469 470 formatter_spec.formatter.handle_event(initialize_event) 471 472 # Make sure we clean up the formatter on shutdown. 473 if formatter_spec.cleanup_func is not None: 474 atexit.register(formatter_spec.cleanup_func) 475 476 477def setupSysPath(): 478 """ 479 Add LLDB.framework/Resources/Python to the search paths for modules. 480 As a side effect, we also discover the 'lldb' executable and export it here. 481 """ 482 483 # Get the directory containing the current script. 484 if "DOTEST_PROFILE" in os.environ and "DOTEST_SCRIPT_DIR" in os.environ: 485 scriptPath = os.environ["DOTEST_SCRIPT_DIR"] 486 else: 487 scriptPath = os.path.dirname(os.path.realpath(__file__)) 488 if not scriptPath.endswith('test'): 489 print("This script expects to reside in lldb's test directory.") 490 sys.exit(-1) 491 492 os.environ["LLDB_TEST"] = scriptPath 493 os.environ["LLDB_TEST_SRC"] = lldbsuite.lldb_test_root 494 495 # Set up the root build directory. 496 if not configuration.test_build_dir: 497 raise Exception("test_build_dir is not set") 498 configuration.test_build_dir = os.path.abspath(configuration.test_build_dir) 499 500 # Set up the LLDB_SRC environment variable, so that the tests can locate 501 # the LLDB source code. 502 os.environ["LLDB_SRC"] = lldbsuite.lldb_root 503 504 pluginPath = os.path.join(scriptPath, 'plugins') 505 toolsLLDBVSCode = os.path.join(scriptPath, 'tools', 'lldb-vscode') 506 toolsLLDBServerPath = os.path.join(scriptPath, 'tools', 'lldb-server') 507 508 # Insert script dir, plugin dir and lldb-server dir to the sys.path. 509 sys.path.insert(0, pluginPath) 510 # Adding test/tools/lldb-vscode to the path makes it easy to 511 # "import lldb_vscode_testcase" from the VSCode tests 512 sys.path.insert(0, toolsLLDBVSCode) 513 # Adding test/tools/lldb-server to the path makes it easy 514 sys.path.insert(0, toolsLLDBServerPath) 515 # to "import lldbgdbserverutils" from the lldb-server tests 516 517 # This is the root of the lldb git/svn checkout 518 # When this changes over to a package instead of a standalone script, this 519 # will be `lldbsuite.lldb_root` 520 lldbRootDirectory = lldbsuite.lldb_root 521 522 # Some of the tests can invoke the 'lldb' command directly. 523 # We'll try to locate the appropriate executable right here. 524 525 # The lldb executable can be set from the command line 526 # if it's not set, we try to find it now 527 # first, we try the environment 528 if not lldbtest_config.lldbExec: 529 # First, you can define an environment variable LLDB_EXEC specifying the 530 # full pathname of the lldb executable. 531 if "LLDB_EXEC" in os.environ: 532 lldbtest_config.lldbExec = os.environ["LLDB_EXEC"] 533 534 if not lldbtest_config.lldbExec: 535 # Last, check the path 536 lldbtest_config.lldbExec = which('lldb') 537 538 if lldbtest_config.lldbExec and not is_exe(lldbtest_config.lldbExec): 539 print( 540 "'{}' is not a path to a valid executable".format( 541 lldbtest_config.lldbExec)) 542 lldbtest_config.lldbExec = None 543 544 if not lldbtest_config.lldbExec: 545 print("The 'lldb' executable cannot be located. Some of the tests may not be run as a result.") 546 sys.exit(-1) 547 548 # confusingly, this is the "bin" directory 549 lldbLibDir = os.path.dirname(lldbtest_config.lldbExec) 550 os.environ["LLDB_LIB_DIR"] = lldbLibDir 551 lldbImpLibDir = configuration.lldb_libs_dir 552 os.environ["LLDB_IMPLIB_DIR"] = lldbImpLibDir 553 print("LLDB library dir:", os.environ["LLDB_LIB_DIR"]) 554 print("LLDB import library dir:", os.environ["LLDB_IMPLIB_DIR"]) 555 os.system('%s -v' % lldbtest_config.lldbExec) 556 557 lldbDir = os.path.dirname(lldbtest_config.lldbExec) 558 559 lldbVSCodeExec = os.path.join(lldbDir, "lldb-vscode") 560 if is_exe(lldbVSCodeExec): 561 os.environ["LLDBVSCODE_EXEC"] = lldbVSCodeExec 562 else: 563 if not configuration.shouldSkipBecauseOfCategories(["lldb-vscode"]): 564 print( 565 "The 'lldb-vscode' executable cannot be located. The lldb-vscode tests can not be run as a result.") 566 configuration.skip_categories.append("lldb-vscode") 567 568 lldbPythonDir = None # The directory that contains 'lldb/__init__.py' 569 if not configuration.lldb_framework_path and os.path.exists(os.path.join(lldbLibDir, "LLDB.framework")): 570 configuration.lldb_framework_path = os.path.join(lldbLibDir, "LLDB.framework") 571 if configuration.lldb_framework_path: 572 lldbtest_config.lldb_framework_path = configuration.lldb_framework_path 573 candidatePath = os.path.join( 574 configuration.lldb_framework_path, 'Resources', 'Python') 575 if os.path.isfile(os.path.join(candidatePath, 'lldb/__init__.py')): 576 lldbPythonDir = candidatePath 577 if not lldbPythonDir: 578 print( 579 'Resources/Python/lldb/__init__.py was not found in ' + 580 configuration.lldb_framework_path) 581 sys.exit(-1) 582 else: 583 # If our lldb supports the -P option, use it to find the python path: 584 init_in_python_dir = os.path.join('lldb', '__init__.py') 585 586 lldb_dash_p_result = subprocess.check_output( 587 [lldbtest_config.lldbExec, "-P"], stderr=subprocess.STDOUT, universal_newlines=True) 588 589 if lldb_dash_p_result and not lldb_dash_p_result.startswith( 590 ("<", "lldb: invalid option:")) and not lldb_dash_p_result.startswith("Traceback"): 591 lines = lldb_dash_p_result.splitlines() 592 593 # Workaround for readline vs libedit issue on FreeBSD. If stdout 594 # is not a terminal Python executes 595 # rl_variable_bind ("enable-meta-key", "off"); 596 # This produces a warning with FreeBSD's libedit because the 597 # enable-meta-key variable is unknown. Not an issue on Apple 598 # because cpython commit f0ab6f9f0603 added a #ifndef __APPLE__ 599 # around the call. See http://bugs.python.org/issue19884 for more 600 # information. For now we just discard the warning output. 601 if len(lines) >= 1 and lines[0].startswith( 602 "bind: Invalid command"): 603 lines.pop(0) 604 605 # Taking the last line because lldb outputs 606 # 'Cannot read termcap database;\nusing dumb terminal settings.\n' 607 # before the path 608 if len(lines) >= 1 and os.path.isfile( 609 os.path.join(lines[-1], init_in_python_dir)): 610 lldbPythonDir = lines[-1] 611 if "freebsd" in sys.platform or "linux" in sys.platform: 612 os.environ['LLDB_LIB_DIR'] = os.path.join( 613 lldbPythonDir, '..', '..') 614 615 if not lldbPythonDir: 616 print( 617 "Unable to load lldb extension module. Possible reasons for this include:") 618 print(" 1) LLDB was built with LLDB_ENABLE_PYTHON=0") 619 print( 620 " 2) PYTHONPATH and PYTHONHOME are not set correctly. PYTHONHOME should refer to") 621 print( 622 " the version of Python that LLDB built and linked against, and PYTHONPATH") 623 print( 624 " should contain the Lib directory for the same python distro, as well as the") 625 print(" location of LLDB\'s site-packages folder.") 626 print( 627 " 3) A different version of Python than that which was built against is exported in") 628 print(" the system\'s PATH environment variable, causing conflicts.") 629 print( 630 " 4) The executable '%s' could not be found. Please check " % 631 lldbtest_config.lldbExec) 632 print(" that it exists and is executable.") 633 634 if lldbPythonDir: 635 lldbPythonDir = os.path.normpath(lldbPythonDir) 636 # Some of the code that uses this path assumes it hasn't resolved the Versions... link. 637 # If the path we've constructed looks like that, then we'll strip out 638 # the Versions/A part. 639 (before, frameWithVersion, after) = lldbPythonDir.rpartition( 640 "LLDB.framework/Versions/A") 641 if frameWithVersion != "": 642 lldbPythonDir = before + "LLDB.framework" + after 643 644 lldbPythonDir = os.path.abspath(lldbPythonDir) 645 646 # If tests need to find LLDB_FRAMEWORK, now they can do it 647 os.environ["LLDB_FRAMEWORK"] = os.path.dirname( 648 os.path.dirname(lldbPythonDir)) 649 650 # This is to locate the lldb.py module. Insert it right after 651 # sys.path[0]. 652 sys.path[1:1] = [lldbPythonDir] 653 654 655def visit_file(dir, name): 656 # Try to match the regexp pattern, if specified. 657 if configuration.regexp: 658 if not re.search(configuration.regexp, name): 659 # We didn't match the regex, we're done. 660 return 661 662 if configuration.skip_tests: 663 for file_regexp in configuration.skip_tests: 664 if re.search(file_regexp, name): 665 return 666 667 # We found a match for our test. Add it to the suite. 668 669 # Update the sys.path first. 670 if not sys.path.count(dir): 671 sys.path.insert(0, dir) 672 base = os.path.splitext(name)[0] 673 674 # Thoroughly check the filterspec against the base module and admit 675 # the (base, filterspec) combination only when it makes sense. 676 677 def check(obj, parts): 678 for part in parts: 679 try: 680 parent, obj = obj, getattr(obj, part) 681 except AttributeError: 682 # The filterspec has failed. 683 return False 684 return True 685 686 module = __import__(base) 687 688 def iter_filters(): 689 for filterspec in configuration.filters: 690 parts = filterspec.split('.') 691 if check(module, parts): 692 yield filterspec 693 elif parts[0] == base and len(parts) > 1 and check(module, parts[1:]): 694 yield '.'.join(parts[1:]) 695 else: 696 for key,value in module.__dict__.items(): 697 if check(value, parts): 698 yield key + '.' + filterspec 699 700 filtered = False 701 for filterspec in iter_filters(): 702 filtered = True 703 print("adding filter spec %s to module %s" % (filterspec, repr(module))) 704 tests = unittest2.defaultTestLoader.loadTestsFromName(filterspec, module) 705 configuration.suite.addTests(tests) 706 707 # Forgo this module if the (base, filterspec) combo is invalid 708 if configuration.filters and not filtered: 709 return 710 711 if not filtered: 712 # Add the entire file's worth of tests since we're not filtered. 713 # Also the fail-over case when the filterspec branch 714 # (base, filterspec) combo doesn't make sense. 715 configuration.suite.addTests( 716 unittest2.defaultTestLoader.loadTestsFromName(base)) 717 718 719def visit(prefix, dir, names): 720 """Visitor function for os.path.walk(path, visit, arg).""" 721 722 dir_components = set(dir.split(os.sep)) 723 excluded_components = set(['.svn', '.git']) 724 if dir_components.intersection(excluded_components): 725 return 726 727 # Gather all the Python test file names that follow the Test*.py pattern. 728 python_test_files = [ 729 name 730 for name in names 731 if name.endswith('.py') and name.startswith(prefix)] 732 733 # Visit all the python test files. 734 for name in python_test_files: 735 try: 736 # Ensure we error out if we have multiple tests with the same 737 # base name. 738 # Future improvement: find all the places where we work with base 739 # names and convert to full paths. We have directory structure 740 # to disambiguate these, so we shouldn't need this constraint. 741 if name in configuration.all_tests: 742 raise Exception("Found multiple tests with the name %s" % name) 743 configuration.all_tests.add(name) 744 745 # Run the relevant tests in the python file. 746 visit_file(dir, name) 747 except Exception as ex: 748 # Convert this exception to a test event error for the file. 749 test_filename = os.path.abspath(os.path.join(dir, name)) 750 if configuration.results_formatter_object is not None: 751 # Grab the backtrace for the exception. 752 import traceback 753 backtrace = traceback.format_exc() 754 755 # Generate the test event. 756 configuration.results_formatter_object.handle_event( 757 EventBuilder.event_for_job_test_add_error( 758 test_filename, ex, backtrace)) 759 raise 760 761 762# ======================================== # 763# # 764# Execution of the test driver starts here # 765# # 766# ======================================== # 767 768 769def checkDsymForUUIDIsNotOn(): 770 cmd = ["defaults", "read", "com.apple.DebugSymbols"] 771 process = subprocess.Popen( 772 cmd, 773 stdout=subprocess.PIPE, 774 stderr=subprocess.STDOUT) 775 cmd_output = process.stdout.read() 776 output_str = cmd_output.decode("utf-8") 777 if "DBGFileMappedPaths = " in output_str: 778 print("%s =>" % ' '.join(cmd)) 779 print(output_str) 780 print( 781 "Disable automatic lookup and caching of dSYMs before running the test suite!") 782 print("Exiting...") 783 sys.exit(0) 784 785 786def exitTestSuite(exitCode=None): 787 # lldb.py does SBDebugger.Initialize(). 788 # Call SBDebugger.Terminate() on exit. 789 import lldb 790 lldb.SBDebugger.Terminate() 791 if exitCode: 792 sys.exit(exitCode) 793 794 795def getVersionForSDK(sdk): 796 sdk = str.lower(sdk) 797 full_path = seven.get_command_output('xcrun -sdk %s --show-sdk-path' % sdk) 798 basename = os.path.basename(full_path) 799 basename = os.path.splitext(basename)[0] 800 basename = str.lower(basename) 801 ver = basename.replace(sdk, '') 802 return ver 803 804 805def setDefaultTripleForPlatform(): 806 if configuration.lldb_platform_name == 'ios-simulator': 807 triple_str = 'x86_64-apple-ios%s' % ( 808 getVersionForSDK('iphonesimulator')) 809 os.environ['TRIPLE'] = triple_str 810 return {'TRIPLE': triple_str} 811 return {} 812 813 814def checkCompiler(): 815 # Add some intervention here to sanity check that the compiler requested is sane. 816 # If found not to be an executable program, we abort. 817 c = configuration.compiler 818 if which(c): 819 return 820 821 if not sys.platform.startswith("darwin"): 822 raise Exception(c + " is not a valid compiler") 823 824 pipe = subprocess.Popen( 825 ['xcrun', '-find', c], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 826 cmd_output = pipe.stdout.read() 827 if not cmd_output or "not found" in cmd_output: 828 raise Exception(c + " is not a valid compiler") 829 830 configuration.compiler = cmd_output.split('\n')[0] 831 print("'xcrun -find %s' returning %s" % (c, configuration.compiler)) 832 833def canRunLibcxxTests(): 834 from lldbsuite.test import lldbplatformutil 835 836 platform = lldbplatformutil.getPlatform() 837 838 if lldbplatformutil.target_is_android() or lldbplatformutil.platformIsDarwin(): 839 return True, "libc++ always present" 840 841 if platform == "linux": 842 if os.path.isdir("/usr/include/c++/v1"): 843 return True, "Headers found, let's hope they work" 844 with tempfile.NamedTemporaryFile() as f: 845 cmd = [configuration.compiler, "-xc++", "-stdlib=libc++", "-o", f.name, "-"] 846 p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) 847 _, stderr = p.communicate("#include <algorithm>\nint main() {}") 848 if not p.returncode: 849 return True, "Compiling with -stdlib=libc++ works" 850 return False, "Compiling with -stdlib=libc++ fails with the error: %s" % stderr 851 852 return False, "Don't know how to build with libc++ on %s" % platform 853 854def checkLibcxxSupport(): 855 result, reason = canRunLibcxxTests() 856 if result: 857 return # libc++ supported 858 if "libc++" in configuration.categories_list: 859 return # libc++ category explicitly requested, let it run. 860 print("Libc++ tests will not be run because: " + reason) 861 configuration.skip_categories.append("libc++") 862 863def canRunLibstdcxxTests(): 864 from lldbsuite.test import lldbplatformutil 865 866 platform = lldbplatformutil.getPlatform() 867 if lldbplatformutil.target_is_android(): 868 platform = "android" 869 if platform == "linux": 870 return True, "libstdcxx always present" 871 return False, "Don't know how to build with libstdcxx on %s" % platform 872 873def checkLibstdcxxSupport(): 874 result, reason = canRunLibstdcxxTests() 875 if result: 876 return # libstdcxx supported 877 if "libstdcxx" in configuration.categories_list: 878 return # libstdcxx category explicitly requested, let it run. 879 print("libstdcxx tests will not be run because: " + reason) 880 configuration.skip_categories.append("libstdcxx") 881 882def canRunWatchpointTests(): 883 from lldbsuite.test import lldbplatformutil 884 885 platform = lldbplatformutil.getPlatform() 886 if platform == "netbsd": 887 if os.geteuid() == 0: 888 return True, "root can always write dbregs" 889 try: 890 output = subprocess.check_output(["/sbin/sysctl", "-n", 891 "security.models.extensions.user_set_dbregs"]).decode().strip() 892 if output == "1": 893 return True, "security.models.extensions.user_set_dbregs enabled" 894 except subprocess.CalledProcessError: 895 pass 896 return False, "security.models.extensions.user_set_dbregs disabled" 897 return True, "watchpoint support available" 898 899def checkWatchpointSupport(): 900 result, reason = canRunWatchpointTests() 901 if result: 902 return # watchpoints supported 903 if "watchpoint" in configuration.categories_list: 904 return # watchpoint category explicitly requested, let it run. 905 print("watchpoint tests will not be run because: " + reason) 906 configuration.skip_categories.append("watchpoint") 907 908def checkDebugInfoSupport(): 909 import lldb 910 911 platform = lldb.selected_platform.GetTriple().split('-')[2] 912 compiler = configuration.compiler 913 skipped = [] 914 for cat in test_categories.debug_info_categories: 915 if cat in configuration.categories_list: 916 continue # Category explicitly requested, let it run. 917 if test_categories.is_supported_on_platform(cat, platform, compiler): 918 continue 919 configuration.skip_categories.append(cat) 920 skipped.append(cat) 921 if skipped: 922 print("Skipping following debug info categories:", skipped) 923 924def run_suite(): 925 # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults 926 # does not exist before proceeding to running the test suite. 927 if sys.platform.startswith("darwin"): 928 checkDsymForUUIDIsNotOn() 929 930 # Start the actions by first parsing the options while setting up the test 931 # directories, followed by setting up the search paths for lldb utilities; 932 # then, we walk the directory trees and collect the tests into our test suite. 933 # 934 parseOptionsAndInitTestdirs() 935 936 # Setup test results (test results formatter and output handling). 937 setupTestResults() 938 939 setupSysPath() 940 941 import lldbconfig 942 if configuration.capture_path or configuration.replay_path: 943 lldbconfig.INITIALIZE = False 944 import lldb 945 946 if configuration.capture_path: 947 lldb.SBReproducer.Capture(configuration.capture_path) 948 lldb.SBReproducer.SetAutoGenerate(True) 949 elif configuration.replay_path: 950 lldb.SBReproducer.PassiveReplay(configuration.replay_path) 951 952 if not lldbconfig.INITIALIZE: 953 lldb.SBDebugger.Initialize() 954 955 # Use host platform by default. 956 lldb.selected_platform = lldb.SBPlatform.GetHostPlatform() 957 958 # Now we can also import lldbutil 959 from lldbsuite.test import lldbutil 960 961 if configuration.lldb_platform_name: 962 print("Setting up remote platform '%s'" % 963 (configuration.lldb_platform_name)) 964 lldb.remote_platform = lldb.SBPlatform( 965 configuration.lldb_platform_name) 966 if not lldb.remote_platform.IsValid(): 967 print( 968 "error: unable to create the LLDB platform named '%s'." % 969 (configuration.lldb_platform_name)) 970 exitTestSuite(1) 971 if configuration.lldb_platform_url: 972 # We must connect to a remote platform if a LLDB platform URL was 973 # specified 974 print( 975 "Connecting to remote platform '%s' at '%s'..." % 976 (configuration.lldb_platform_name, configuration.lldb_platform_url)) 977 platform_connect_options = lldb.SBPlatformConnectOptions( 978 configuration.lldb_platform_url) 979 err = lldb.remote_platform.ConnectRemote(platform_connect_options) 980 if err.Success(): 981 print("Connected.") 982 else: 983 print("error: failed to connect to remote platform using URL '%s': %s" % ( 984 configuration.lldb_platform_url, err)) 985 exitTestSuite(1) 986 else: 987 configuration.lldb_platform_url = None 988 989 platform_changes = setDefaultTripleForPlatform() 990 first = True 991 for key in platform_changes: 992 if first: 993 print("Environment variables setup for platform support:") 994 first = False 995 print("%s = %s" % (key, platform_changes[key])) 996 997 if configuration.lldb_platform_working_dir: 998 print("Setting remote platform working directory to '%s'..." % 999 (configuration.lldb_platform_working_dir)) 1000 error = lldb.remote_platform.MakeDirectory( 1001 configuration.lldb_platform_working_dir, 448) # 448 = 0o700 1002 if error.Fail(): 1003 raise Exception("making remote directory '%s': %s" % ( 1004 configuration.lldb_platform_working_dir, error)) 1005 1006 if not lldb.remote_platform.SetWorkingDirectory( 1007 configuration.lldb_platform_working_dir): 1008 raise Exception("failed to set working directory '%s'" % configuration.lldb_platform_working_dir) 1009 lldb.selected_platform = lldb.remote_platform 1010 else: 1011 lldb.remote_platform = None 1012 configuration.lldb_platform_working_dir = None 1013 configuration.lldb_platform_url = None 1014 1015 # Set up the working directory. 1016 # Note that it's not dotest's job to clean this directory. 1017 lldbutil.mkdir_p(configuration.test_build_dir) 1018 1019 target_platform = lldb.selected_platform.GetTriple().split('-')[2] 1020 1021 checkLibcxxSupport() 1022 checkLibstdcxxSupport() 1023 checkWatchpointSupport() 1024 checkDebugInfoSupport() 1025 1026 # Don't do debugserver tests on anything except OS X. 1027 configuration.dont_do_debugserver_test = ( 1028 "linux" in target_platform or 1029 "freebsd" in target_platform or 1030 "netbsd" in target_platform or 1031 "windows" in target_platform) 1032 1033 # Don't do lldb-server (llgs) tests on anything except Linux and Windows. 1034 configuration.dont_do_llgs_test = not ( 1035 "linux" in target_platform or 1036 "netbsd" in target_platform or 1037 "windows" in target_platform) 1038 1039 for testdir in configuration.testdirs: 1040 for (dirpath, dirnames, filenames) in os.walk(testdir): 1041 visit('Test', dirpath, filenames) 1042 1043 # 1044 # Now that we have loaded all the test cases, run the whole test suite. 1045 # 1046 1047 # Install the control-c handler. 1048 unittest2.signals.installHandler() 1049 1050 lldbutil.mkdir_p(configuration.sdir_name) 1051 os.environ["LLDB_SESSION_DIRNAME"] = configuration.sdir_name 1052 1053 sys.stderr.write( 1054 "\nSession logs for test failures/errors/unexpected successes" 1055 " will go into directory '%s'\n" % 1056 configuration.sdir_name) 1057 1058 # 1059 # Invoke the default TextTestRunner to run the test suite 1060 # 1061 checkCompiler() 1062 1063 if configuration.verbose: 1064 print("compiler=%s" % configuration.compiler) 1065 1066 # Iterating over all possible architecture and compiler combinations. 1067 configString = "arch=%s compiler=%s" % (configuration.arch, 1068 configuration.compiler) 1069 1070 # Output the configuration. 1071 if configuration.verbose: 1072 sys.stderr.write("\nConfiguration: " + configString + "\n") 1073 1074 # First, write out the number of collected test cases. 1075 if configuration.verbose: 1076 sys.stderr.write(configuration.separator + "\n") 1077 sys.stderr.write( 1078 "Collected %d test%s\n\n" % 1079 (configuration.suite.countTestCases(), 1080 configuration.suite.countTestCases() != 1 and "s" or "")) 1081 1082 # Invoke the test runner. 1083 if configuration.count == 1: 1084 result = unittest2.TextTestRunner( 1085 stream=sys.stderr, 1086 verbosity=configuration.verbose, 1087 resultclass=test_result.LLDBTestResult).run( 1088 configuration.suite) 1089 else: 1090 # We are invoking the same test suite more than once. In this case, 1091 # mark __ignore_singleton__ flag as True so the signleton pattern is 1092 # not enforced. 1093 test_result.LLDBTestResult.__ignore_singleton__ = True 1094 for i in range(configuration.count): 1095 1096 result = unittest2.TextTestRunner( 1097 stream=sys.stderr, 1098 verbosity=configuration.verbose, 1099 resultclass=test_result.LLDBTestResult).run( 1100 configuration.suite) 1101 1102 configuration.failed = not result.wasSuccessful() 1103 1104 if configuration.sdir_has_content and configuration.verbose: 1105 sys.stderr.write( 1106 "Session logs for test failures/errors/unexpected successes" 1107 " can be found in directory '%s'\n" % 1108 configuration.sdir_name) 1109 1110 if configuration.use_categories and len( 1111 configuration.failures_per_category) > 0: 1112 sys.stderr.write("Failures per category:\n") 1113 for category in configuration.failures_per_category: 1114 sys.stderr.write( 1115 "%s - %d\n" % 1116 (category, configuration.failures_per_category[category])) 1117 1118 # Exiting. 1119 exitTestSuite(configuration.failed) 1120 1121if __name__ == "__main__": 1122 print( 1123 __file__ + 1124 " is for use as a module only. It should not be run as a standalone script.") 1125 sys.exit(-1) 1126