1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright(c) 2010-2019 Intel Corporation 3# Copyright(c) 2022-2023 PANTHEON.tech s.r.o. 4# Copyright(c) 2022-2023 University of New Hampshire 5 6"""Test suite runner module. 7 8The module is responsible for running DTS in a series of stages: 9 10 #. Test run stage, 11 #. Build target stage, 12 #. Test suite stage, 13 #. Test case stage. 14 15The test run and build target stages set up the environment before running test suites. 16The test suite stage sets up steps common to all test cases 17and the test case stage runs test cases individually. 18""" 19 20import importlib 21import inspect 22import os 23import re 24import sys 25from pathlib import Path 26from types import FunctionType 27from typing import Iterable, Sequence 28 29from framework.testbed_model.sut_node import SutNode 30from framework.testbed_model.tg_node import TGNode 31 32from .config import ( 33 BuildTargetConfiguration, 34 Configuration, 35 TestRunConfiguration, 36 TestSuiteConfig, 37 load_config, 38) 39from .exception import ( 40 BlockingTestSuiteError, 41 ConfigurationError, 42 SSHTimeoutError, 43 TestCaseVerifyError, 44) 45from .logger import DTSLogger, DtsStage, get_dts_logger 46from .settings import SETTINGS 47from .test_result import ( 48 BuildTargetResult, 49 DTSResult, 50 Result, 51 TestCaseResult, 52 TestRunResult, 53 TestSuiteResult, 54 TestSuiteWithCases, 55) 56from .test_suite import TestSuite 57 58 59class DTSRunner: 60 r"""Test suite runner class. 61 62 The class is responsible for running tests on testbeds defined in the test run configuration. 63 Each setup or teardown of each stage is recorded in a :class:`~framework.test_result.DTSResult` 64 or one of its subclasses. The test case results are also recorded. 65 66 If an error occurs, the current stage is aborted, the error is recorded, everything in 67 the inner stages is marked as blocked and the run continues in the next iteration 68 of the same stage. The return code is the highest `severity` of all 69 :class:`~.framework.exception.DTSError`\s. 70 71 Example: 72 An error occurs in a build target setup. The current build target is aborted, 73 all test suites and their test cases are marked as blocked and the run continues 74 with the next build target. If the errored build target was the last one in the 75 given test run, the next test run begins. 76 """ 77 78 _configuration: Configuration 79 _logger: DTSLogger 80 _result: DTSResult 81 _test_suite_class_prefix: str 82 _test_suite_module_prefix: str 83 _func_test_case_regex: str 84 _perf_test_case_regex: str 85 86 def __init__(self): 87 """Initialize the instance with configuration, logger, result and string constants.""" 88 self._configuration = load_config(SETTINGS.config_file_path) 89 self._logger = get_dts_logger() 90 if not os.path.exists(SETTINGS.output_dir): 91 os.makedirs(SETTINGS.output_dir) 92 self._logger.add_dts_root_logger_handlers(SETTINGS.verbose, SETTINGS.output_dir) 93 self._result = DTSResult(self._logger) 94 self._test_suite_class_prefix = "Test" 95 self._test_suite_module_prefix = "tests.TestSuite_" 96 self._func_test_case_regex = r"test_(?!perf_)" 97 self._perf_test_case_regex = r"test_perf_" 98 99 def run(self) -> None: 100 """Run all build targets in all test runs from the test run configuration. 101 102 Before running test suites, test runs and build targets are first set up. 103 The test runs and build targets defined in the test run configuration are iterated over. 104 The test runs define which tests to run and where to run them and build targets define 105 the DPDK build setup. 106 107 The tests suites are set up for each test run/build target tuple and each discovered 108 test case within the test suite is set up, executed and torn down. After all test cases 109 have been executed, the test suite is torn down and the next build target will be tested. 110 111 In order to properly mark test suites and test cases as blocked in case of a failure, 112 we need to have discovered which test suites and test cases to run before any failures 113 happen. The discovery happens at the earliest point at the start of each test run. 114 115 All the nested steps look like this: 116 117 #. Test run setup 118 119 #. Build target setup 120 121 #. Test suite setup 122 123 #. Test case setup 124 #. Test case logic 125 #. Test case teardown 126 127 #. Test suite teardown 128 129 #. Build target teardown 130 131 #. Test run teardown 132 133 The test cases are filtered according to the specification in the test run configuration and 134 the :option:`--test-suite` command line argument or 135 the :envvar:`DTS_TESTCASES` environment variable. 136 """ 137 sut_nodes: dict[str, SutNode] = {} 138 tg_nodes: dict[str, TGNode] = {} 139 try: 140 # check the python version of the server that runs dts 141 self._check_dts_python_version() 142 self._result.update_setup(Result.PASS) 143 144 # for all test run sections 145 for test_run_config in self._configuration.test_runs: 146 self._logger.set_stage(DtsStage.test_run_setup) 147 self._logger.info( 148 f"Running test run with SUT '{test_run_config.system_under_test_node.name}'." 149 ) 150 test_run_result = self._result.add_test_run(test_run_config) 151 # we don't want to modify the original config, so create a copy 152 test_run_test_suites = list( 153 SETTINGS.test_suites if SETTINGS.test_suites else test_run_config.test_suites 154 ) 155 if not test_run_config.skip_smoke_tests: 156 test_run_test_suites[:0] = [TestSuiteConfig.from_dict("smoke_tests")] 157 try: 158 test_suites_with_cases = self._get_test_suites_with_cases( 159 test_run_test_suites, test_run_config.func, test_run_config.perf 160 ) 161 test_run_result.test_suites_with_cases = test_suites_with_cases 162 except Exception as e: 163 self._logger.exception( 164 f"Invalid test suite configuration found: " f"{test_run_test_suites}." 165 ) 166 test_run_result.update_setup(Result.FAIL, e) 167 168 else: 169 self._connect_nodes_and_run_test_run( 170 sut_nodes, 171 tg_nodes, 172 test_run_config, 173 test_run_result, 174 test_suites_with_cases, 175 ) 176 177 except Exception as e: 178 self._logger.exception("An unexpected error has occurred.") 179 self._result.add_error(e) 180 raise 181 182 finally: 183 try: 184 self._logger.set_stage(DtsStage.post_run) 185 for node in (sut_nodes | tg_nodes).values(): 186 node.close() 187 self._result.update_teardown(Result.PASS) 188 except Exception as e: 189 self._logger.exception("The final cleanup of nodes failed.") 190 self._result.update_teardown(Result.ERROR, e) 191 192 # we need to put the sys.exit call outside the finally clause to make sure 193 # that unexpected exceptions will propagate 194 # in that case, the error that should be reported is the uncaught exception as 195 # that is a severe error originating from the framework 196 # at that point, we'll only have partial results which could be impacted by the 197 # error causing the uncaught exception, making them uninterpretable 198 self._exit_dts() 199 200 def _check_dts_python_version(self) -> None: 201 """Check the required Python version - v3.10.""" 202 if sys.version_info.major < 3 or ( 203 sys.version_info.major == 3 and sys.version_info.minor < 10 204 ): 205 self._logger.warning( 206 "DTS execution node's python version is lower than Python 3.10, " 207 "is deprecated and will not work in future releases." 208 ) 209 self._logger.warning("Please use Python >= 3.10 instead.") 210 211 def _get_test_suites_with_cases( 212 self, 213 test_suite_configs: list[TestSuiteConfig], 214 func: bool, 215 perf: bool, 216 ) -> list[TestSuiteWithCases]: 217 """Test suites with test cases discovery. 218 219 The test suites with test cases defined in the user configuration are discovered 220 and stored for future use so that we don't import the modules twice and so that 221 the list of test suites with test cases is available for recording right away. 222 223 Args: 224 test_suite_configs: Test suite configurations. 225 func: Whether to include functional test cases in the final list. 226 perf: Whether to include performance test cases in the final list. 227 228 Returns: 229 The discovered test suites, each with test cases. 230 """ 231 test_suites_with_cases = [] 232 233 for test_suite_config in test_suite_configs: 234 test_suite_class = self._get_test_suite_class(test_suite_config.test_suite) 235 test_cases = [] 236 func_test_cases, perf_test_cases = self._filter_test_cases( 237 test_suite_class, test_suite_config.test_cases 238 ) 239 if func: 240 test_cases.extend(func_test_cases) 241 if perf: 242 test_cases.extend(perf_test_cases) 243 244 test_suites_with_cases.append( 245 TestSuiteWithCases(test_suite_class=test_suite_class, test_cases=test_cases) 246 ) 247 248 return test_suites_with_cases 249 250 def _get_test_suite_class(self, module_name: str) -> type[TestSuite]: 251 """Find the :class:`TestSuite` class in `module_name`. 252 253 The full module name is `module_name` prefixed with `self._test_suite_module_prefix`. 254 The module name is a standard filename with words separated with underscores. 255 Search the `module_name` for a :class:`TestSuite` class which starts 256 with `self._test_suite_class_prefix`, continuing with CamelCase `module_name`. 257 The first matching class is returned. 258 259 The CamelCase convention applies to abbreviations, acronyms, initialisms and so on:: 260 261 OS -> Os 262 TCP -> Tcp 263 264 Args: 265 module_name: The module name without prefix where to search for the test suite. 266 267 Returns: 268 The found test suite class. 269 270 Raises: 271 ConfigurationError: If the corresponding module is not found or 272 a valid :class:`TestSuite` is not found in the module. 273 """ 274 275 def is_test_suite(object) -> bool: 276 """Check whether `object` is a :class:`TestSuite`. 277 278 The `object` is a subclass of :class:`TestSuite`, but not :class:`TestSuite` itself. 279 280 Args: 281 object: The object to be checked. 282 283 Returns: 284 :data:`True` if `object` is a subclass of `TestSuite`. 285 """ 286 try: 287 if issubclass(object, TestSuite) and object is not TestSuite: 288 return True 289 except TypeError: 290 return False 291 return False 292 293 testsuite_module_path = f"{self._test_suite_module_prefix}{module_name}" 294 try: 295 test_suite_module = importlib.import_module(testsuite_module_path) 296 except ModuleNotFoundError as e: 297 raise ConfigurationError( 298 f"Test suite module '{testsuite_module_path}' not found." 299 ) from e 300 301 camel_case_suite_name = "".join( 302 [suite_word.capitalize() for suite_word in module_name.split("_")] 303 ) 304 full_suite_name_to_find = f"{self._test_suite_class_prefix}{camel_case_suite_name}" 305 for class_name, class_obj in inspect.getmembers(test_suite_module, is_test_suite): 306 if class_name == full_suite_name_to_find: 307 return class_obj 308 raise ConfigurationError( 309 f"Couldn't find any valid test suites in {test_suite_module.__name__}." 310 ) 311 312 def _filter_test_cases( 313 self, test_suite_class: type[TestSuite], test_cases_to_run: Sequence[str] 314 ) -> tuple[list[FunctionType], list[FunctionType]]: 315 """Filter `test_cases_to_run` from `test_suite_class`. 316 317 There are two rounds of filtering if `test_cases_to_run` is not empty. 318 The first filters `test_cases_to_run` from all methods of `test_suite_class`. 319 Then the methods are separated into functional and performance test cases. 320 If a method matches neither the functional nor performance name prefix, it's an error. 321 322 Args: 323 test_suite_class: The class of the test suite. 324 test_cases_to_run: Test case names to filter from `test_suite_class`. 325 If empty, return all matching test cases. 326 327 Returns: 328 A list of test case methods that should be executed. 329 330 Raises: 331 ConfigurationError: If a test case from `test_cases_to_run` is not found 332 or it doesn't match either the functional nor performance name prefix. 333 """ 334 func_test_cases = [] 335 perf_test_cases = [] 336 name_method_tuples = inspect.getmembers(test_suite_class, inspect.isfunction) 337 if test_cases_to_run: 338 name_method_tuples = [ 339 (name, method) for name, method in name_method_tuples if name in test_cases_to_run 340 ] 341 if len(name_method_tuples) < len(test_cases_to_run): 342 missing_test_cases = set(test_cases_to_run) - { 343 name for name, _ in name_method_tuples 344 } 345 raise ConfigurationError( 346 f"Test cases {missing_test_cases} not found among methods " 347 f"of {test_suite_class.__name__}." 348 ) 349 350 for test_case_name, test_case_method in name_method_tuples: 351 if re.match(self._func_test_case_regex, test_case_name): 352 func_test_cases.append(test_case_method) 353 elif re.match(self._perf_test_case_regex, test_case_name): 354 perf_test_cases.append(test_case_method) 355 elif test_cases_to_run: 356 raise ConfigurationError( 357 f"Method '{test_case_name}' matches neither " 358 f"a functional nor a performance test case name." 359 ) 360 361 return func_test_cases, perf_test_cases 362 363 def _connect_nodes_and_run_test_run( 364 self, 365 sut_nodes: dict[str, SutNode], 366 tg_nodes: dict[str, TGNode], 367 test_run_config: TestRunConfiguration, 368 test_run_result: TestRunResult, 369 test_suites_with_cases: Iterable[TestSuiteWithCases], 370 ) -> None: 371 """Connect nodes, then continue to run the given test run. 372 373 Connect the :class:`SutNode` and the :class:`TGNode` of this `test_run_config`. 374 If either has already been connected, it's going to be in either `sut_nodes` or `tg_nodes`, 375 respectively. 376 If not, connect and add the node to the respective `sut_nodes` or `tg_nodes` :class:`dict`. 377 378 Args: 379 sut_nodes: A dictionary storing connected/to be connected SUT nodes. 380 tg_nodes: A dictionary storing connected/to be connected TG nodes. 381 test_run_config: A test run configuration. 382 test_run_result: The test run's result. 383 test_suites_with_cases: The test suites with test cases to run. 384 """ 385 sut_node = sut_nodes.get(test_run_config.system_under_test_node.name) 386 tg_node = tg_nodes.get(test_run_config.traffic_generator_node.name) 387 388 try: 389 if not sut_node: 390 sut_node = SutNode(test_run_config.system_under_test_node) 391 sut_nodes[sut_node.name] = sut_node 392 if not tg_node: 393 tg_node = TGNode(test_run_config.traffic_generator_node) 394 tg_nodes[tg_node.name] = tg_node 395 except Exception as e: 396 failed_node = test_run_config.system_under_test_node.name 397 if sut_node: 398 failed_node = test_run_config.traffic_generator_node.name 399 self._logger.exception(f"The Creation of node {failed_node} failed.") 400 test_run_result.update_setup(Result.FAIL, e) 401 402 else: 403 self._run_test_run( 404 sut_node, tg_node, test_run_config, test_run_result, test_suites_with_cases 405 ) 406 407 def _run_test_run( 408 self, 409 sut_node: SutNode, 410 tg_node: TGNode, 411 test_run_config: TestRunConfiguration, 412 test_run_result: TestRunResult, 413 test_suites_with_cases: Iterable[TestSuiteWithCases], 414 ) -> None: 415 """Run the given test run. 416 417 This involves running the test run setup as well as running all build targets 418 in the given test run. After that, the test run teardown is run. 419 420 Args: 421 sut_node: The test run's SUT node. 422 tg_node: The test run's TG node. 423 test_run_config: A test run configuration. 424 test_run_result: The test run's result. 425 test_suites_with_cases: The test suites with test cases to run. 426 """ 427 self._logger.info( 428 f"Running test run with SUT '{test_run_config.system_under_test_node.name}'." 429 ) 430 test_run_result.add_sut_info(sut_node.node_info) 431 try: 432 sut_node.set_up_test_run(test_run_config) 433 tg_node.set_up_test_run(test_run_config) 434 test_run_result.update_setup(Result.PASS) 435 except Exception as e: 436 self._logger.exception("Test run setup failed.") 437 test_run_result.update_setup(Result.FAIL, e) 438 439 else: 440 for build_target_config in test_run_config.build_targets: 441 build_target_result = test_run_result.add_build_target(build_target_config) 442 self._run_build_target( 443 sut_node, 444 tg_node, 445 build_target_config, 446 build_target_result, 447 test_suites_with_cases, 448 ) 449 450 finally: 451 try: 452 self._logger.set_stage(DtsStage.test_run_teardown) 453 sut_node.tear_down_test_run() 454 tg_node.tear_down_test_run() 455 test_run_result.update_teardown(Result.PASS) 456 except Exception as e: 457 self._logger.exception("Test run teardown failed.") 458 test_run_result.update_teardown(Result.FAIL, e) 459 460 def _run_build_target( 461 self, 462 sut_node: SutNode, 463 tg_node: TGNode, 464 build_target_config: BuildTargetConfiguration, 465 build_target_result: BuildTargetResult, 466 test_suites_with_cases: Iterable[TestSuiteWithCases], 467 ) -> None: 468 """Run the given build target. 469 470 This involves running the build target setup as well as running all test suites 471 of the build target's test run. 472 After that, build target teardown is run. 473 474 Args: 475 sut_node: The test run's sut node. 476 tg_node: The test run's tg node. 477 build_target_config: A build target's test run configuration. 478 build_target_result: The build target level result object associated 479 with the current build target. 480 test_suites_with_cases: The test suites with test cases to run. 481 """ 482 self._logger.set_stage(DtsStage.build_target_setup) 483 self._logger.info(f"Running build target '{build_target_config.name}'.") 484 485 try: 486 sut_node.set_up_build_target(build_target_config) 487 self._result.dpdk_version = sut_node.dpdk_version 488 build_target_result.add_build_target_info(sut_node.get_build_target_info()) 489 build_target_result.update_setup(Result.PASS) 490 except Exception as e: 491 self._logger.exception("Build target setup failed.") 492 build_target_result.update_setup(Result.FAIL, e) 493 494 else: 495 self._run_test_suites(sut_node, tg_node, build_target_result, test_suites_with_cases) 496 497 finally: 498 try: 499 self._logger.set_stage(DtsStage.build_target_teardown) 500 sut_node.tear_down_build_target() 501 build_target_result.update_teardown(Result.PASS) 502 except Exception as e: 503 self._logger.exception("Build target teardown failed.") 504 build_target_result.update_teardown(Result.FAIL, e) 505 506 def _run_test_suites( 507 self, 508 sut_node: SutNode, 509 tg_node: TGNode, 510 build_target_result: BuildTargetResult, 511 test_suites_with_cases: Iterable[TestSuiteWithCases], 512 ) -> None: 513 """Run `test_suites_with_cases` with the current build target. 514 515 The method assumes the build target we're testing has already been built on the SUT node. 516 The current build target thus corresponds to the current DPDK build present on the SUT node. 517 518 If a blocking test suite (such as the smoke test suite) fails, the rest of the test suites 519 in the current build target won't be executed. 520 521 Args: 522 sut_node: The test run's SUT node. 523 tg_node: The test run's TG node. 524 build_target_result: The build target level result object associated 525 with the current build target. 526 test_suites_with_cases: The test suites with test cases to run. 527 """ 528 end_build_target = False 529 for test_suite_with_cases in test_suites_with_cases: 530 test_suite_result = build_target_result.add_test_suite(test_suite_with_cases) 531 try: 532 self._run_test_suite(sut_node, tg_node, test_suite_result, test_suite_with_cases) 533 except BlockingTestSuiteError as e: 534 self._logger.exception( 535 f"An error occurred within {test_suite_with_cases.test_suite_class.__name__}. " 536 "Skipping build target..." 537 ) 538 self._result.add_error(e) 539 end_build_target = True 540 # if a blocking test failed and we need to bail out of suite executions 541 if end_build_target: 542 break 543 544 def _run_test_suite( 545 self, 546 sut_node: SutNode, 547 tg_node: TGNode, 548 test_suite_result: TestSuiteResult, 549 test_suite_with_cases: TestSuiteWithCases, 550 ) -> None: 551 """Set up, execute and tear down `test_suite_with_cases`. 552 553 The method assumes the build target we're testing has already been built on the SUT node. 554 The current build target thus corresponds to the current DPDK build present on the SUT node. 555 556 Test suite execution consists of running the discovered test cases. 557 A test case run consists of setup, execution and teardown of said test case. 558 559 Record the setup and the teardown and handle failures. 560 561 Args: 562 sut_node: The test run's SUT node. 563 tg_node: The test run's TG node. 564 test_suite_result: The test suite level result object associated 565 with the current test suite. 566 test_suite_with_cases: The test suite with test cases to run. 567 568 Raises: 569 BlockingTestSuiteError: If a blocking test suite fails. 570 """ 571 test_suite_name = test_suite_with_cases.test_suite_class.__name__ 572 self._logger.set_stage( 573 DtsStage.test_suite_setup, Path(SETTINGS.output_dir, test_suite_name) 574 ) 575 test_suite = test_suite_with_cases.test_suite_class(sut_node, tg_node) 576 try: 577 self._logger.info(f"Starting test suite setup: {test_suite_name}") 578 test_suite.set_up_suite() 579 test_suite_result.update_setup(Result.PASS) 580 self._logger.info(f"Test suite setup successful: {test_suite_name}") 581 except Exception as e: 582 self._logger.exception(f"Test suite setup ERROR: {test_suite_name}") 583 test_suite_result.update_setup(Result.ERROR, e) 584 585 else: 586 self._execute_test_suite( 587 test_suite, 588 test_suite_with_cases.test_cases, 589 test_suite_result, 590 ) 591 finally: 592 try: 593 self._logger.set_stage(DtsStage.test_suite_teardown) 594 test_suite.tear_down_suite() 595 sut_node.kill_cleanup_dpdk_apps() 596 test_suite_result.update_teardown(Result.PASS) 597 except Exception as e: 598 self._logger.exception(f"Test suite teardown ERROR: {test_suite_name}") 599 self._logger.warning( 600 f"Test suite '{test_suite_name}' teardown failed, " 601 "the next test suite may be affected." 602 ) 603 test_suite_result.update_setup(Result.ERROR, e) 604 if len(test_suite_result.get_errors()) > 0 and test_suite.is_blocking: 605 raise BlockingTestSuiteError(test_suite_name) 606 607 def _execute_test_suite( 608 self, 609 test_suite: TestSuite, 610 test_cases: Iterable[FunctionType], 611 test_suite_result: TestSuiteResult, 612 ) -> None: 613 """Execute all `test_cases` in `test_suite`. 614 615 If the :option:`--re-run` command line argument or the :envvar:`DTS_RERUN` environment 616 variable is set, in case of a test case failure, the test case will be executed again 617 until it passes or it fails that many times in addition of the first failure. 618 619 Args: 620 test_suite: The test suite object. 621 test_cases: The list of test case methods. 622 test_suite_result: The test suite level result object associated 623 with the current test suite. 624 """ 625 self._logger.set_stage(DtsStage.test_suite) 626 for test_case_method in test_cases: 627 test_case_name = test_case_method.__name__ 628 test_case_result = test_suite_result.add_test_case(test_case_name) 629 all_attempts = SETTINGS.re_run + 1 630 attempt_nr = 1 631 self._run_test_case(test_suite, test_case_method, test_case_result) 632 while not test_case_result and attempt_nr < all_attempts: 633 attempt_nr += 1 634 self._logger.info( 635 f"Re-running FAILED test case '{test_case_name}'. " 636 f"Attempt number {attempt_nr} out of {all_attempts}." 637 ) 638 self._run_test_case(test_suite, test_case_method, test_case_result) 639 640 def _run_test_case( 641 self, 642 test_suite: TestSuite, 643 test_case_method: FunctionType, 644 test_case_result: TestCaseResult, 645 ) -> None: 646 """Setup, execute and teardown `test_case_method` from `test_suite`. 647 648 Record the result of the setup and the teardown and handle failures. 649 650 Args: 651 test_suite: The test suite object. 652 test_case_method: The test case method. 653 test_case_result: The test case level result object associated 654 with the current test case. 655 """ 656 test_case_name = test_case_method.__name__ 657 658 try: 659 # run set_up function for each case 660 test_suite.set_up_test_case() 661 test_case_result.update_setup(Result.PASS) 662 except SSHTimeoutError as e: 663 self._logger.exception(f"Test case setup FAILED: {test_case_name}") 664 test_case_result.update_setup(Result.FAIL, e) 665 except Exception as e: 666 self._logger.exception(f"Test case setup ERROR: {test_case_name}") 667 test_case_result.update_setup(Result.ERROR, e) 668 669 else: 670 # run test case if setup was successful 671 self._execute_test_case(test_suite, test_case_method, test_case_result) 672 673 finally: 674 try: 675 test_suite.tear_down_test_case() 676 test_case_result.update_teardown(Result.PASS) 677 except Exception as e: 678 self._logger.exception(f"Test case teardown ERROR: {test_case_name}") 679 self._logger.warning( 680 f"Test case '{test_case_name}' teardown failed, " 681 f"the next test case may be affected." 682 ) 683 test_case_result.update_teardown(Result.ERROR, e) 684 test_case_result.update(Result.ERROR) 685 686 def _execute_test_case( 687 self, 688 test_suite: TestSuite, 689 test_case_method: FunctionType, 690 test_case_result: TestCaseResult, 691 ) -> None: 692 """Execute `test_case_method` from `test_suite`, record the result and handle failures. 693 694 Args: 695 test_suite: The test suite object. 696 test_case_method: The test case method. 697 test_case_result: The test case level result object associated 698 with the current test case. 699 """ 700 test_case_name = test_case_method.__name__ 701 try: 702 self._logger.info(f"Starting test case execution: {test_case_name}") 703 test_case_method(test_suite) 704 test_case_result.update(Result.PASS) 705 self._logger.info(f"Test case execution PASSED: {test_case_name}") 706 707 except TestCaseVerifyError as e: 708 self._logger.exception(f"Test case execution FAILED: {test_case_name}") 709 test_case_result.update(Result.FAIL, e) 710 except Exception as e: 711 self._logger.exception(f"Test case execution ERROR: {test_case_name}") 712 test_case_result.update(Result.ERROR, e) 713 except KeyboardInterrupt: 714 self._logger.error(f"Test case execution INTERRUPTED by user: {test_case_name}") 715 test_case_result.update(Result.SKIP) 716 raise KeyboardInterrupt("Stop DTS") 717 718 def _exit_dts(self) -> None: 719 """Process all errors and exit with the proper exit code.""" 720 self._result.process() 721 722 if self._logger: 723 self._logger.info("DTS execution has ended.") 724 725 sys.exit(self._result.get_return_code()) 726