1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright(c) 2010-2019 Intel Corporation 3# Copyright(c) 2022-2023 PANTHEON.tech s.r.o. 4# Copyright(c) 2022-2023 University of New Hampshire 5 6"""Test suite runner module. 7 8The module is responsible for running DTS in a series of stages: 9 10 #. Test run stage, 11 #. Build target stage, 12 #. Test suite stage, 13 #. Test case stage. 14 15The test run and build target stages set up the environment before running test suites. 16The test suite stage sets up steps common to all test cases 17and the test case stage runs test cases individually. 18""" 19 20import importlib 21import inspect 22import os 23import random 24import sys 25from pathlib import Path 26from types import MethodType 27from typing import Iterable 28 29from framework.testbed_model.capability import Capability, get_supported_capabilities 30from framework.testbed_model.sut_node import SutNode 31from framework.testbed_model.tg_node import TGNode 32 33from .config import ( 34 BuildTargetConfiguration, 35 Configuration, 36 TestRunConfiguration, 37 TestSuiteConfig, 38 load_config, 39) 40from .exception import ( 41 BlockingTestSuiteError, 42 ConfigurationError, 43 SSHTimeoutError, 44 TestCaseVerifyError, 45) 46from .logger import DTSLogger, DtsStage, get_dts_logger 47from .settings import SETTINGS 48from .test_result import ( 49 BuildTargetResult, 50 DTSResult, 51 Result, 52 TestCaseResult, 53 TestRunResult, 54 TestSuiteResult, 55 TestSuiteWithCases, 56) 57from .test_suite import TestCase, TestSuite 58from .testbed_model.topology import Topology 59 60 61class DTSRunner: 62 r"""Test suite runner class. 63 64 The class is responsible for running tests on testbeds defined in the test run configuration. 65 Each setup or teardown of each stage is recorded in a :class:`~framework.test_result.DTSResult` 66 or one of its subclasses. The test case results are also recorded. 67 68 If an error occurs, the current stage is aborted, the error is recorded, everything in 69 the inner stages is marked as blocked and the run continues in the next iteration 70 of the same stage. The return code is the highest `severity` of all 71 :class:`~.framework.exception.DTSError`\s. 72 73 Example: 74 An error occurs in a build target setup. The current build target is aborted, 75 all test suites and their test cases are marked as blocked and the run continues 76 with the next build target. If the errored build target was the last one in the 77 given test run, the next test run begins. 78 """ 79 80 _configuration: Configuration 81 _logger: DTSLogger 82 _result: DTSResult 83 _test_suite_class_prefix: str 84 _test_suite_module_prefix: str 85 _func_test_case_regex: str 86 _perf_test_case_regex: str 87 88 def __init__(self): 89 """Initialize the instance with configuration, logger, result and string constants.""" 90 self._configuration = load_config(SETTINGS.config_file_path) 91 self._logger = get_dts_logger() 92 if not os.path.exists(SETTINGS.output_dir): 93 os.makedirs(SETTINGS.output_dir) 94 self._logger.add_dts_root_logger_handlers(SETTINGS.verbose, SETTINGS.output_dir) 95 self._result = DTSResult(self._logger) 96 self._test_suite_class_prefix = "Test" 97 self._test_suite_module_prefix = "tests.TestSuite_" 98 self._func_test_case_regex = r"test_(?!perf_)" 99 self._perf_test_case_regex = r"test_perf_" 100 101 def run(self) -> None: 102 """Run all build targets in all test runs from the test run configuration. 103 104 Before running test suites, test runs and build targets are first set up. 105 The test runs and build targets defined in the test run configuration are iterated over. 106 The test runs define which tests to run and where to run them and build targets define 107 the DPDK build setup. 108 109 The tests suites are set up for each test run/build target tuple and each discovered 110 test case within the test suite is set up, executed and torn down. After all test cases 111 have been executed, the test suite is torn down and the next build target will be tested. 112 113 In order to properly mark test suites and test cases as blocked in case of a failure, 114 we need to have discovered which test suites and test cases to run before any failures 115 happen. The discovery happens at the earliest point at the start of each test run. 116 117 All the nested steps look like this: 118 119 #. Test run setup 120 121 #. Build target setup 122 123 #. Test suite setup 124 125 #. Test case setup 126 #. Test case logic 127 #. Test case teardown 128 129 #. Test suite teardown 130 131 #. Build target teardown 132 133 #. Test run teardown 134 135 The test cases are filtered according to the specification in the test run configuration and 136 the :option:`--test-suite` command line argument or 137 the :envvar:`DTS_TESTCASES` environment variable. 138 """ 139 sut_nodes: dict[str, SutNode] = {} 140 tg_nodes: dict[str, TGNode] = {} 141 try: 142 # check the python version of the server that runs dts 143 self._check_dts_python_version() 144 self._result.update_setup(Result.PASS) 145 146 # for all test run sections 147 for test_run_config in self._configuration.test_runs: 148 self._logger.set_stage(DtsStage.test_run_setup) 149 self._logger.info( 150 f"Running test run with SUT '{test_run_config.system_under_test_node.name}'." 151 ) 152 self._init_random_seed(test_run_config) 153 test_run_result = self._result.add_test_run(test_run_config) 154 # we don't want to modify the original config, so create a copy 155 test_run_test_suites = list( 156 SETTINGS.test_suites if SETTINGS.test_suites else test_run_config.test_suites 157 ) 158 if not test_run_config.skip_smoke_tests: 159 test_run_test_suites[:0] = [TestSuiteConfig.from_dict("smoke_tests")] 160 try: 161 test_suites_with_cases = self._get_test_suites_with_cases( 162 test_run_test_suites, test_run_config.func, test_run_config.perf 163 ) 164 test_run_result.test_suites_with_cases = test_suites_with_cases 165 except Exception as e: 166 self._logger.exception( 167 f"Invalid test suite configuration found: " f"{test_run_test_suites}." 168 ) 169 test_run_result.update_setup(Result.FAIL, e) 170 171 else: 172 self._connect_nodes_and_run_test_run( 173 sut_nodes, 174 tg_nodes, 175 test_run_config, 176 test_run_result, 177 test_suites_with_cases, 178 ) 179 180 except Exception as e: 181 self._logger.exception("An unexpected error has occurred.") 182 self._result.add_error(e) 183 raise 184 185 finally: 186 try: 187 self._logger.set_stage(DtsStage.post_run) 188 for node in (sut_nodes | tg_nodes).values(): 189 node.close() 190 self._result.update_teardown(Result.PASS) 191 except Exception as e: 192 self._logger.exception("The final cleanup of nodes failed.") 193 self._result.update_teardown(Result.ERROR, e) 194 195 # we need to put the sys.exit call outside the finally clause to make sure 196 # that unexpected exceptions will propagate 197 # in that case, the error that should be reported is the uncaught exception as 198 # that is a severe error originating from the framework 199 # at that point, we'll only have partial results which could be impacted by the 200 # error causing the uncaught exception, making them uninterpretable 201 self._exit_dts() 202 203 def _check_dts_python_version(self) -> None: 204 """Check the required Python version - v3.10.""" 205 if sys.version_info.major < 3 or ( 206 sys.version_info.major == 3 and sys.version_info.minor < 10 207 ): 208 self._logger.warning( 209 "DTS execution node's python version is lower than Python 3.10, " 210 "is deprecated and will not work in future releases." 211 ) 212 self._logger.warning("Please use Python >= 3.10 instead.") 213 214 def _get_test_suites_with_cases( 215 self, 216 test_suite_configs: list[TestSuiteConfig], 217 func: bool, 218 perf: bool, 219 ) -> list[TestSuiteWithCases]: 220 """Test suites with test cases discovery. 221 222 The test suites with test cases defined in the user configuration are discovered 223 and stored for future use so that we don't import the modules twice and so that 224 the list of test suites with test cases is available for recording right away. 225 226 Args: 227 test_suite_configs: Test suite configurations. 228 func: Whether to include functional test cases in the final list. 229 perf: Whether to include performance test cases in the final list. 230 231 Returns: 232 The discovered test suites, each with test cases. 233 """ 234 test_suites_with_cases = [] 235 236 for test_suite_config in test_suite_configs: 237 test_suite_class = self._get_test_suite_class(test_suite_config.test_suite) 238 test_cases: list[type[TestCase]] = [] 239 func_test_cases, perf_test_cases = test_suite_class.get_test_cases( 240 test_suite_config.test_cases 241 ) 242 if func: 243 test_cases.extend(func_test_cases) 244 if perf: 245 test_cases.extend(perf_test_cases) 246 247 test_suites_with_cases.append( 248 TestSuiteWithCases(test_suite_class=test_suite_class, test_cases=test_cases) 249 ) 250 251 return test_suites_with_cases 252 253 def _get_test_suite_class(self, module_name: str) -> type[TestSuite]: 254 """Find the :class:`TestSuite` class in `module_name`. 255 256 The full module name is `module_name` prefixed with `self._test_suite_module_prefix`. 257 The module name is a standard filename with words separated with underscores. 258 Search the `module_name` for a :class:`TestSuite` class which starts 259 with `self._test_suite_class_prefix`, continuing with CamelCase `module_name`. 260 The first matching class is returned. 261 262 The CamelCase convention applies to abbreviations, acronyms, initialisms and so on:: 263 264 OS -> Os 265 TCP -> Tcp 266 267 Args: 268 module_name: The module name without prefix where to search for the test suite. 269 270 Returns: 271 The found test suite class. 272 273 Raises: 274 ConfigurationError: If the corresponding module is not found or 275 a valid :class:`TestSuite` is not found in the module. 276 """ 277 278 def is_test_suite(object) -> bool: 279 """Check whether `object` is a :class:`TestSuite`. 280 281 The `object` is a subclass of :class:`TestSuite`, but not :class:`TestSuite` itself. 282 283 Args: 284 object: The object to be checked. 285 286 Returns: 287 :data:`True` if `object` is a subclass of `TestSuite`. 288 """ 289 try: 290 if issubclass(object, TestSuite) and object is not TestSuite: 291 return True 292 except TypeError: 293 return False 294 return False 295 296 testsuite_module_path = f"{self._test_suite_module_prefix}{module_name}" 297 try: 298 test_suite_module = importlib.import_module(testsuite_module_path) 299 except ModuleNotFoundError as e: 300 raise ConfigurationError( 301 f"Test suite module '{testsuite_module_path}' not found." 302 ) from e 303 304 camel_case_suite_name = "".join( 305 [suite_word.capitalize() for suite_word in module_name.split("_")] 306 ) 307 full_suite_name_to_find = f"{self._test_suite_class_prefix}{camel_case_suite_name}" 308 for class_name, class_obj in inspect.getmembers(test_suite_module, is_test_suite): 309 if class_name == full_suite_name_to_find: 310 return class_obj 311 raise ConfigurationError( 312 f"Couldn't find any valid test suites in {test_suite_module.__name__}." 313 ) 314 315 def _connect_nodes_and_run_test_run( 316 self, 317 sut_nodes: dict[str, SutNode], 318 tg_nodes: dict[str, TGNode], 319 test_run_config: TestRunConfiguration, 320 test_run_result: TestRunResult, 321 test_suites_with_cases: Iterable[TestSuiteWithCases], 322 ) -> None: 323 """Connect nodes, then continue to run the given test run. 324 325 Connect the :class:`SutNode` and the :class:`TGNode` of this `test_run_config`. 326 If either has already been connected, it's going to be in either `sut_nodes` or `tg_nodes`, 327 respectively. 328 If not, connect and add the node to the respective `sut_nodes` or `tg_nodes` :class:`dict`. 329 330 Args: 331 sut_nodes: A dictionary storing connected/to be connected SUT nodes. 332 tg_nodes: A dictionary storing connected/to be connected TG nodes. 333 test_run_config: A test run configuration. 334 test_run_result: The test run's result. 335 test_suites_with_cases: The test suites with test cases to run. 336 """ 337 sut_node = sut_nodes.get(test_run_config.system_under_test_node.name) 338 tg_node = tg_nodes.get(test_run_config.traffic_generator_node.name) 339 340 try: 341 if not sut_node: 342 sut_node = SutNode(test_run_config.system_under_test_node) 343 sut_nodes[sut_node.name] = sut_node 344 if not tg_node: 345 tg_node = TGNode(test_run_config.traffic_generator_node) 346 tg_nodes[tg_node.name] = tg_node 347 except Exception as e: 348 failed_node = test_run_config.system_under_test_node.name 349 if sut_node: 350 failed_node = test_run_config.traffic_generator_node.name 351 self._logger.exception(f"The Creation of node {failed_node} failed.") 352 test_run_result.update_setup(Result.FAIL, e) 353 354 else: 355 self._run_test_run( 356 sut_node, tg_node, test_run_config, test_run_result, test_suites_with_cases 357 ) 358 359 def _run_test_run( 360 self, 361 sut_node: SutNode, 362 tg_node: TGNode, 363 test_run_config: TestRunConfiguration, 364 test_run_result: TestRunResult, 365 test_suites_with_cases: Iterable[TestSuiteWithCases], 366 ) -> None: 367 """Run the given test run. 368 369 This involves running the test run setup as well as running all build targets 370 in the given test run. After that, the test run teardown is run. 371 372 Args: 373 sut_node: The test run's SUT node. 374 tg_node: The test run's TG node. 375 test_run_config: A test run configuration. 376 test_run_result: The test run's result. 377 test_suites_with_cases: The test suites with test cases to run. 378 """ 379 self._logger.info( 380 f"Running test run with SUT '{test_run_config.system_under_test_node.name}'." 381 ) 382 test_run_result.add_sut_info(sut_node.node_info) 383 try: 384 sut_node.set_up_test_run(test_run_config) 385 tg_node.set_up_test_run(test_run_config) 386 test_run_result.update_setup(Result.PASS) 387 except Exception as e: 388 self._logger.exception("Test run setup failed.") 389 test_run_result.update_setup(Result.FAIL, e) 390 391 else: 392 for build_target_config in test_run_config.build_targets: 393 build_target_result = test_run_result.add_build_target(build_target_config) 394 self._run_build_target( 395 sut_node, 396 tg_node, 397 build_target_config, 398 build_target_result, 399 test_suites_with_cases, 400 ) 401 402 finally: 403 try: 404 self._logger.set_stage(DtsStage.test_run_teardown) 405 sut_node.tear_down_test_run() 406 tg_node.tear_down_test_run() 407 test_run_result.update_teardown(Result.PASS) 408 except Exception as e: 409 self._logger.exception("Test run teardown failed.") 410 test_run_result.update_teardown(Result.FAIL, e) 411 412 def _run_build_target( 413 self, 414 sut_node: SutNode, 415 tg_node: TGNode, 416 build_target_config: BuildTargetConfiguration, 417 build_target_result: BuildTargetResult, 418 test_suites_with_cases: Iterable[TestSuiteWithCases], 419 ) -> None: 420 """Run the given build target. 421 422 This involves running the build target setup as well as running all test suites 423 of the build target's test run. 424 After that, build target teardown is run. 425 426 Args: 427 sut_node: The test run's sut node. 428 tg_node: The test run's tg node. 429 build_target_config: A build target's test run configuration. 430 build_target_result: The build target level result object associated 431 with the current build target. 432 test_suites_with_cases: The test suites with test cases to run. 433 """ 434 self._logger.set_stage(DtsStage.build_target_setup) 435 self._logger.info(f"Running build target '{build_target_config.name}'.") 436 437 try: 438 sut_node.set_up_build_target(build_target_config) 439 self._result.dpdk_version = sut_node.dpdk_version 440 build_target_result.add_build_target_info(sut_node.get_build_target_info()) 441 build_target_result.update_setup(Result.PASS) 442 except Exception as e: 443 self._logger.exception("Build target setup failed.") 444 build_target_result.update_setup(Result.FAIL, e) 445 446 else: 447 self._run_test_suites(sut_node, tg_node, build_target_result, test_suites_with_cases) 448 449 finally: 450 try: 451 self._logger.set_stage(DtsStage.build_target_teardown) 452 sut_node.tear_down_build_target() 453 build_target_result.update_teardown(Result.PASS) 454 except Exception as e: 455 self._logger.exception("Build target teardown failed.") 456 build_target_result.update_teardown(Result.FAIL, e) 457 458 def _get_supported_capabilities( 459 self, 460 sut_node: SutNode, 461 topology_config: Topology, 462 test_suites_with_cases: Iterable[TestSuiteWithCases], 463 ) -> set[Capability]: 464 465 capabilities_to_check = set() 466 for test_suite_with_cases in test_suites_with_cases: 467 capabilities_to_check.update(test_suite_with_cases.required_capabilities) 468 469 self._logger.debug(f"Found capabilities to check: {capabilities_to_check}") 470 471 return get_supported_capabilities(sut_node, topology_config, capabilities_to_check) 472 473 def _run_test_suites( 474 self, 475 sut_node: SutNode, 476 tg_node: TGNode, 477 build_target_result: BuildTargetResult, 478 test_suites_with_cases: Iterable[TestSuiteWithCases], 479 ) -> None: 480 """Run `test_suites_with_cases` with the current build target. 481 482 The method assumes the build target we're testing has already been built on the SUT node. 483 The current build target thus corresponds to the current DPDK build present on the SUT node. 484 485 Before running any suites, the method determines whether they should be skipped 486 by inspecting any required capabilities the test suite needs and comparing those 487 to capabilities supported by the tested environment. If all capabilities are supported, 488 the suite is run. If all test cases in a test suite would be skipped, the whole test suite 489 is skipped (the setup and teardown is not run). 490 491 If a blocking test suite (such as the smoke test suite) fails, the rest of the test suites 492 in the current build target won't be executed. 493 494 Args: 495 sut_node: The test run's SUT node. 496 tg_node: The test run's TG node. 497 build_target_result: The build target level result object associated 498 with the current build target. 499 test_suites_with_cases: The test suites with test cases to run. 500 """ 501 end_build_target = False 502 topology = Topology(sut_node.ports, tg_node.ports) 503 supported_capabilities = self._get_supported_capabilities( 504 sut_node, topology, test_suites_with_cases 505 ) 506 for test_suite_with_cases in test_suites_with_cases: 507 test_suite_with_cases.mark_skip_unsupported(supported_capabilities) 508 test_suite_result = build_target_result.add_test_suite(test_suite_with_cases) 509 try: 510 if not test_suite_with_cases.skip: 511 self._run_test_suite( 512 sut_node, 513 tg_node, 514 topology, 515 test_suite_result, 516 test_suite_with_cases, 517 ) 518 else: 519 self._logger.info( 520 f"Test suite execution SKIPPED: " 521 f"'{test_suite_with_cases.test_suite_class.__name__}'. Reason: " 522 f"{test_suite_with_cases.test_suite_class.skip_reason}" 523 ) 524 test_suite_result.update_setup(Result.SKIP) 525 except BlockingTestSuiteError as e: 526 self._logger.exception( 527 f"An error occurred within {test_suite_with_cases.test_suite_class.__name__}. " 528 "Skipping build target..." 529 ) 530 self._result.add_error(e) 531 end_build_target = True 532 # if a blocking test failed and we need to bail out of suite executions 533 if end_build_target: 534 break 535 536 def _run_test_suite( 537 self, 538 sut_node: SutNode, 539 tg_node: TGNode, 540 topology: Topology, 541 test_suite_result: TestSuiteResult, 542 test_suite_with_cases: TestSuiteWithCases, 543 ) -> None: 544 """Set up, execute and tear down `test_suite_with_cases`. 545 546 The method assumes the build target we're testing has already been built on the SUT node. 547 The current build target thus corresponds to the current DPDK build present on the SUT node. 548 549 Test suite execution consists of running the discovered test cases. 550 A test case run consists of setup, execution and teardown of said test case. 551 552 Record the setup and the teardown and handle failures. 553 554 Args: 555 sut_node: The test run's SUT node. 556 tg_node: The test run's TG node. 557 test_suite_result: The test suite level result object associated 558 with the current test suite. 559 test_suite_with_cases: The test suite with test cases to run. 560 561 Raises: 562 BlockingTestSuiteError: If a blocking test suite fails. 563 """ 564 test_suite_name = test_suite_with_cases.test_suite_class.__name__ 565 self._logger.set_stage( 566 DtsStage.test_suite_setup, Path(SETTINGS.output_dir, test_suite_name) 567 ) 568 test_suite = test_suite_with_cases.test_suite_class(sut_node, tg_node, topology) 569 try: 570 self._logger.info(f"Starting test suite setup: {test_suite_name}") 571 test_suite.set_up_suite() 572 test_suite_result.update_setup(Result.PASS) 573 self._logger.info(f"Test suite setup successful: {test_suite_name}") 574 except Exception as e: 575 self._logger.exception(f"Test suite setup ERROR: {test_suite_name}") 576 test_suite_result.update_setup(Result.ERROR, e) 577 578 else: 579 self._execute_test_suite( 580 test_suite, 581 test_suite_with_cases.test_cases, 582 test_suite_result, 583 ) 584 finally: 585 try: 586 self._logger.set_stage(DtsStage.test_suite_teardown) 587 test_suite.tear_down_suite() 588 sut_node.kill_cleanup_dpdk_apps() 589 test_suite_result.update_teardown(Result.PASS) 590 except Exception as e: 591 self._logger.exception(f"Test suite teardown ERROR: {test_suite_name}") 592 self._logger.warning( 593 f"Test suite '{test_suite_name}' teardown failed, " 594 "the next test suite may be affected." 595 ) 596 test_suite_result.update_setup(Result.ERROR, e) 597 if len(test_suite_result.get_errors()) > 0 and test_suite.is_blocking: 598 raise BlockingTestSuiteError(test_suite_name) 599 600 def _execute_test_suite( 601 self, 602 test_suite: TestSuite, 603 test_cases: Iterable[type[TestCase]], 604 test_suite_result: TestSuiteResult, 605 ) -> None: 606 """Execute all `test_cases` in `test_suite`. 607 608 If the :option:`--re-run` command line argument or the :envvar:`DTS_RERUN` environment 609 variable is set, in case of a test case failure, the test case will be executed again 610 until it passes or it fails that many times in addition of the first failure. 611 612 Args: 613 test_suite: The test suite object. 614 test_cases: The list of test case functions. 615 test_suite_result: The test suite level result object associated 616 with the current test suite. 617 """ 618 self._logger.set_stage(DtsStage.test_suite) 619 for test_case in test_cases: 620 test_case_name = test_case.__name__ 621 test_case_result = test_suite_result.add_test_case(test_case_name) 622 all_attempts = SETTINGS.re_run + 1 623 attempt_nr = 1 624 if not test_case.skip: 625 self._run_test_case(test_suite, test_case, test_case_result) 626 while not test_case_result and attempt_nr < all_attempts: 627 attempt_nr += 1 628 self._logger.info( 629 f"Re-running FAILED test case '{test_case_name}'. " 630 f"Attempt number {attempt_nr} out of {all_attempts}." 631 ) 632 self._run_test_case(test_suite, test_case, test_case_result) 633 else: 634 self._logger.info( 635 f"Test case execution SKIPPED: {test_case_name}. Reason: " 636 f"{test_case.skip_reason}" 637 ) 638 test_case_result.update_setup(Result.SKIP) 639 640 def _run_test_case( 641 self, 642 test_suite: TestSuite, 643 test_case: type[TestCase], 644 test_case_result: TestCaseResult, 645 ) -> None: 646 """Setup, execute and teardown `test_case_method` from `test_suite`. 647 648 Record the result of the setup and the teardown and handle failures. 649 650 Args: 651 test_suite: The test suite object. 652 test_case: The test case function. 653 test_case_result: The test case level result object associated 654 with the current test case. 655 """ 656 test_case_name = test_case.__name__ 657 658 try: 659 # run set_up function for each case 660 test_suite.set_up_test_case() 661 test_case_result.update_setup(Result.PASS) 662 except SSHTimeoutError as e: 663 self._logger.exception(f"Test case setup FAILED: {test_case_name}") 664 test_case_result.update_setup(Result.FAIL, e) 665 except Exception as e: 666 self._logger.exception(f"Test case setup ERROR: {test_case_name}") 667 test_case_result.update_setup(Result.ERROR, e) 668 669 else: 670 # run test case if setup was successful 671 self._execute_test_case(test_suite, test_case, test_case_result) 672 673 finally: 674 try: 675 test_suite.tear_down_test_case() 676 test_case_result.update_teardown(Result.PASS) 677 except Exception as e: 678 self._logger.exception(f"Test case teardown ERROR: {test_case_name}") 679 self._logger.warning( 680 f"Test case '{test_case_name}' teardown failed, " 681 f"the next test case may be affected." 682 ) 683 test_case_result.update_teardown(Result.ERROR, e) 684 test_case_result.update(Result.ERROR) 685 686 def _execute_test_case( 687 self, 688 test_suite: TestSuite, 689 test_case: type[TestCase], 690 test_case_result: TestCaseResult, 691 ) -> None: 692 """Execute `test_case_method` from `test_suite`, record the result and handle failures. 693 694 Args: 695 test_suite: The test suite object. 696 test_case: The test case function. 697 test_case_result: The test case level result object associated 698 with the current test case. 699 """ 700 test_case_name = test_case.__name__ 701 try: 702 self._logger.info(f"Starting test case execution: {test_case_name}") 703 # Explicit method binding is required, otherwise mypy complains 704 MethodType(test_case, test_suite)() 705 test_case_result.update(Result.PASS) 706 self._logger.info(f"Test case execution PASSED: {test_case_name}") 707 708 except TestCaseVerifyError as e: 709 self._logger.exception(f"Test case execution FAILED: {test_case_name}") 710 test_case_result.update(Result.FAIL, e) 711 except Exception as e: 712 self._logger.exception(f"Test case execution ERROR: {test_case_name}") 713 test_case_result.update(Result.ERROR, e) 714 except KeyboardInterrupt: 715 self._logger.error(f"Test case execution INTERRUPTED by user: {test_case_name}") 716 test_case_result.update(Result.SKIP) 717 raise KeyboardInterrupt("Stop DTS") 718 719 def _exit_dts(self) -> None: 720 """Process all errors and exit with the proper exit code.""" 721 self._result.process() 722 723 if self._logger: 724 self._logger.info("DTS execution has ended.") 725 726 sys.exit(self._result.get_return_code()) 727 728 def _init_random_seed(self, conf: TestRunConfiguration) -> None: 729 """Initialize the random seed to use for the test run.""" 730 seed = SETTINGS.random_seed or conf.random_seed or random.randrange(0xFFFF_FFFF) 731 self._logger.info(f"Initializing test run with random seed {seed}.") 732 random.seed(seed) 733