1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright(c) 2010-2019 Intel Corporation 3# Copyright(c) 2022-2023 PANTHEON.tech s.r.o. 4# Copyright(c) 2022-2023 University of New Hampshire 5 6"""Test suite runner module. 7 8The module is responsible for running DTS in a series of stages: 9 10 #. Test run stage, 11 #. DPDK build stage, 12 #. Test suite stage, 13 #. Test case stage. 14 15The test run stage sets up the environment before running test suites. 16The test suite stage sets up steps common to all test cases 17and the test case stage runs test cases individually. 18""" 19 20import importlib 21import inspect 22import os 23import random 24import sys 25from pathlib import Path 26from types import MethodType 27from typing import Iterable 28 29from framework.testbed_model.capability import Capability, get_supported_capabilities 30from framework.testbed_model.sut_node import SutNode 31from framework.testbed_model.tg_node import TGNode 32 33from .config import ( 34 Configuration, 35 DPDKPrecompiledBuildConfiguration, 36 SutNodeConfiguration, 37 TestRunConfiguration, 38 TestSuiteConfig, 39 TGNodeConfiguration, 40 load_config, 41) 42from .exception import ( 43 BlockingTestSuiteError, 44 ConfigurationError, 45 SSHTimeoutError, 46 TestCaseVerifyError, 47) 48from .logger import DTSLogger, DtsStage, get_dts_logger 49from .settings import SETTINGS 50from .test_result import ( 51 DTSResult, 52 Result, 53 TestCaseResult, 54 TestRunResult, 55 TestSuiteResult, 56 TestSuiteWithCases, 57) 58from .test_suite import TestCase, TestSuite 59from .testbed_model.topology import Topology 60 61 62class DTSRunner: 63 r"""Test suite runner class. 64 65 The class is responsible for running tests on testbeds defined in the test run configuration. 66 Each setup or teardown of each stage is recorded in a :class:`~framework.test_result.DTSResult` 67 or one of its subclasses. The test case results are also recorded. 68 69 If an error occurs, the current stage is aborted, the error is recorded, everything in 70 the inner stages is marked as blocked and the run continues in the next iteration 71 of the same stage. The return code is the highest `severity` of all 72 :class:`~.framework.exception.DTSError`\s. 73 74 Example: 75 An error occurs in a test suite setup. The current test suite is aborted, 76 all its test cases are marked as blocked and the run continues 77 with the next test suite. If the errored test suite was the last one in the 78 given test run, the next test run begins. 79 """ 80 81 _configuration: Configuration 82 _logger: DTSLogger 83 _result: DTSResult 84 _test_suite_class_prefix: str 85 _test_suite_module_prefix: str 86 _func_test_case_regex: str 87 _perf_test_case_regex: str 88 89 def __init__(self): 90 """Initialize the instance with configuration, logger, result and string constants.""" 91 self._configuration = load_config(SETTINGS.config_file_path) 92 self._logger = get_dts_logger() 93 if not os.path.exists(SETTINGS.output_dir): 94 os.makedirs(SETTINGS.output_dir) 95 self._logger.add_dts_root_logger_handlers(SETTINGS.verbose, SETTINGS.output_dir) 96 self._result = DTSResult(self._logger) 97 self._test_suite_class_prefix = "Test" 98 self._test_suite_module_prefix = "tests.TestSuite_" 99 self._func_test_case_regex = r"test_(?!perf_)" 100 self._perf_test_case_regex = r"test_perf_" 101 102 def run(self) -> None: 103 """Run all test runs from the test run configuration. 104 105 Before running test suites, test runs are first set up. 106 The test runs defined in the test run configuration are iterated over. 107 The test runs define which tests to run and where to run them. 108 109 The test suites are set up for each test run and each discovered 110 test case within the test suite is set up, executed and torn down. After all test cases 111 have been executed, the test suite is torn down and the next test suite will be run. Once 112 all test suites have been run, the next test run will be tested. 113 114 In order to properly mark test suites and test cases as blocked in case of a failure, 115 we need to have discovered which test suites and test cases to run before any failures 116 happen. The discovery happens at the earliest point at the start of each test run. 117 118 All the nested steps look like this: 119 120 #. Test run setup 121 122 #. Test suite setup 123 124 #. Test case setup 125 #. Test case logic 126 #. Test case teardown 127 128 #. Test suite teardown 129 130 #. Test run teardown 131 132 The test cases are filtered according to the specification in the test run configuration and 133 the :option:`--test-suite` command line argument or 134 the :envvar:`DTS_TESTCASES` environment variable. 135 """ 136 sut_nodes: dict[str, SutNode] = {} 137 tg_nodes: dict[str, TGNode] = {} 138 try: 139 # check the python version of the server that runs dts 140 self._check_dts_python_version() 141 self._result.update_setup(Result.PASS) 142 143 # for all test run sections 144 for test_run_with_nodes_config in self._configuration.test_runs_with_nodes: 145 test_run_config, sut_node_config, tg_node_config = test_run_with_nodes_config 146 self._logger.set_stage(DtsStage.test_run_setup) 147 self._logger.info(f"Running test run with SUT '{sut_node_config.name}'.") 148 self._init_random_seed(test_run_config) 149 test_run_result = self._result.add_test_run(test_run_config) 150 # we don't want to modify the original config, so create a copy 151 test_run_test_suites = list( 152 SETTINGS.test_suites if SETTINGS.test_suites else test_run_config.test_suites 153 ) 154 if not test_run_config.skip_smoke_tests: 155 test_run_test_suites[:0] = [TestSuiteConfig(test_suite="smoke_tests")] 156 try: 157 test_suites_with_cases = self._get_test_suites_with_cases( 158 test_run_test_suites, test_run_config.func, test_run_config.perf 159 ) 160 test_run_result.test_suites_with_cases = test_suites_with_cases 161 except Exception as e: 162 self._logger.exception( 163 f"Invalid test suite configuration found: " f"{test_run_test_suites}." 164 ) 165 test_run_result.update_setup(Result.FAIL, e) 166 167 else: 168 self._connect_nodes_and_run_test_run( 169 sut_nodes, 170 tg_nodes, 171 sut_node_config, 172 tg_node_config, 173 test_run_config, 174 test_run_result, 175 test_suites_with_cases, 176 ) 177 178 except Exception as e: 179 self._logger.exception("An unexpected error has occurred.") 180 self._result.add_error(e) 181 raise 182 183 finally: 184 try: 185 self._logger.set_stage(DtsStage.post_run) 186 for node in (sut_nodes | tg_nodes).values(): 187 node.close() 188 self._result.update_teardown(Result.PASS) 189 except Exception as e: 190 self._logger.exception("The final cleanup of nodes failed.") 191 self._result.update_teardown(Result.ERROR, e) 192 193 # we need to put the sys.exit call outside the finally clause to make sure 194 # that unexpected exceptions will propagate 195 # in that case, the error that should be reported is the uncaught exception as 196 # that is a severe error originating from the framework 197 # at that point, we'll only have partial results which could be impacted by the 198 # error causing the uncaught exception, making them uninterpretable 199 self._exit_dts() 200 201 def _check_dts_python_version(self) -> None: 202 """Check the required Python version - v3.10.""" 203 if sys.version_info.major < 3 or ( 204 sys.version_info.major == 3 and sys.version_info.minor < 10 205 ): 206 self._logger.warning( 207 "DTS execution node's python version is lower than Python 3.10, " 208 "is deprecated and will not work in future releases." 209 ) 210 self._logger.warning("Please use Python >= 3.10 instead.") 211 212 def _get_test_suites_with_cases( 213 self, 214 test_suite_configs: list[TestSuiteConfig], 215 func: bool, 216 perf: bool, 217 ) -> list[TestSuiteWithCases]: 218 """Test suites with test cases discovery. 219 220 The test suites with test cases defined in the user configuration are discovered 221 and stored for future use so that we don't import the modules twice and so that 222 the list of test suites with test cases is available for recording right away. 223 224 Args: 225 test_suite_configs: Test suite configurations. 226 func: Whether to include functional test cases in the final list. 227 perf: Whether to include performance test cases in the final list. 228 229 Returns: 230 The discovered test suites, each with test cases. 231 """ 232 test_suites_with_cases = [] 233 234 for test_suite_config in test_suite_configs: 235 test_suite_class = self._get_test_suite_class(test_suite_config.test_suite_name) 236 test_cases: list[type[TestCase]] = [] 237 func_test_cases, perf_test_cases = test_suite_class.filter_test_cases( 238 test_suite_config.test_cases_names 239 ) 240 if func: 241 test_cases.extend(func_test_cases) 242 if perf: 243 test_cases.extend(perf_test_cases) 244 245 test_suites_with_cases.append( 246 TestSuiteWithCases(test_suite_class=test_suite_class, test_cases=test_cases) 247 ) 248 249 return test_suites_with_cases 250 251 def _get_test_suite_class(self, module_name: str) -> type[TestSuite]: 252 """Find the :class:`TestSuite` class in `module_name`. 253 254 The full module name is `module_name` prefixed with `self._test_suite_module_prefix`. 255 The module name is a standard filename with words separated with underscores. 256 Search the `module_name` for a :class:`TestSuite` class which starts 257 with `self._test_suite_class_prefix`, continuing with CamelCase `module_name`. 258 The first matching class is returned. 259 260 The CamelCase convention applies to abbreviations, acronyms, initialisms and so on:: 261 262 OS -> Os 263 TCP -> Tcp 264 265 Args: 266 module_name: The module name without prefix where to search for the test suite. 267 268 Returns: 269 The found test suite class. 270 271 Raises: 272 ConfigurationError: If the corresponding module is not found or 273 a valid :class:`TestSuite` is not found in the module. 274 """ 275 276 def is_test_suite(object) -> bool: 277 """Check whether `object` is a :class:`TestSuite`. 278 279 The `object` is a subclass of :class:`TestSuite`, but not :class:`TestSuite` itself. 280 281 Args: 282 object: The object to be checked. 283 284 Returns: 285 :data:`True` if `object` is a subclass of `TestSuite`. 286 """ 287 try: 288 if issubclass(object, TestSuite) and object is not TestSuite: 289 return True 290 except TypeError: 291 return False 292 return False 293 294 testsuite_module_path = f"{self._test_suite_module_prefix}{module_name}" 295 try: 296 test_suite_module = importlib.import_module(testsuite_module_path) 297 except ModuleNotFoundError as e: 298 raise ConfigurationError( 299 f"Test suite module '{testsuite_module_path}' not found." 300 ) from e 301 302 camel_case_suite_name = "".join( 303 [suite_word.capitalize() for suite_word in module_name.split("_")] 304 ) 305 full_suite_name_to_find = f"{self._test_suite_class_prefix}{camel_case_suite_name}" 306 for class_name, class_obj in inspect.getmembers(test_suite_module, is_test_suite): 307 if class_name == full_suite_name_to_find: 308 return class_obj 309 raise ConfigurationError( 310 f"Couldn't find any valid test suites in {test_suite_module.__name__}." 311 ) 312 313 def _connect_nodes_and_run_test_run( 314 self, 315 sut_nodes: dict[str, SutNode], 316 tg_nodes: dict[str, TGNode], 317 sut_node_config: SutNodeConfiguration, 318 tg_node_config: TGNodeConfiguration, 319 test_run_config: TestRunConfiguration, 320 test_run_result: TestRunResult, 321 test_suites_with_cases: Iterable[TestSuiteWithCases], 322 ) -> None: 323 """Connect nodes, then continue to run the given test run. 324 325 Connect the :class:`SutNode` and the :class:`TGNode` of this `test_run_config`. 326 If either has already been connected, it's going to be in either `sut_nodes` or `tg_nodes`, 327 respectively. 328 If not, connect and add the node to the respective `sut_nodes` or `tg_nodes` :class:`dict`. 329 330 Args: 331 sut_nodes: A dictionary storing connected/to be connected SUT nodes. 332 tg_nodes: A dictionary storing connected/to be connected TG nodes. 333 sut_node_config: The test run's SUT node configuration. 334 tg_node_config: The test run's TG node configuration. 335 test_run_config: A test run configuration. 336 test_run_result: The test run's result. 337 test_suites_with_cases: The test suites with test cases to run. 338 """ 339 sut_node = sut_nodes.get(sut_node_config.name) 340 tg_node = tg_nodes.get(tg_node_config.name) 341 342 try: 343 if not sut_node: 344 sut_node = SutNode(sut_node_config) 345 sut_nodes[sut_node.name] = sut_node 346 if not tg_node: 347 tg_node = TGNode(tg_node_config) 348 tg_nodes[tg_node.name] = tg_node 349 except Exception as e: 350 failed_node = test_run_config.system_under_test_node.node_name 351 if sut_node: 352 failed_node = test_run_config.traffic_generator_node 353 self._logger.exception(f"The Creation of node {failed_node} failed.") 354 test_run_result.update_setup(Result.FAIL, e) 355 356 else: 357 self._run_test_run( 358 sut_node, tg_node, test_run_config, test_run_result, test_suites_with_cases 359 ) 360 361 def _run_test_run( 362 self, 363 sut_node: SutNode, 364 tg_node: TGNode, 365 test_run_config: TestRunConfiguration, 366 test_run_result: TestRunResult, 367 test_suites_with_cases: Iterable[TestSuiteWithCases], 368 ) -> None: 369 """Run the given test run. 370 371 This involves running the test run setup as well as running all test suites 372 in the given test run. After that, the test run teardown is run. 373 374 Args: 375 sut_node: The test run's SUT node. 376 tg_node: The test run's TG node. 377 test_run_config: A test run configuration. 378 test_run_result: The test run's result. 379 test_suites_with_cases: The test suites with test cases to run. 380 381 Raises: 382 ConfigurationError: If the DPDK sources or build is not set up from config or settings. 383 """ 384 self._logger.info( 385 f"Running test run with SUT '{test_run_config.system_under_test_node.node_name}'." 386 ) 387 test_run_result.add_sut_info(sut_node.node_info) 388 try: 389 dpdk_build_config = test_run_config.dpdk_config 390 if new_location := SETTINGS.dpdk_location: 391 dpdk_build_config = dpdk_build_config.model_copy( 392 update={"dpdk_location": new_location} 393 ) 394 if dir := SETTINGS.precompiled_build_dir: 395 dpdk_build_config = DPDKPrecompiledBuildConfiguration( 396 dpdk_location=dpdk_build_config.dpdk_location, precompiled_build_dir=dir 397 ) 398 sut_node.set_up_test_run(test_run_config, dpdk_build_config) 399 test_run_result.add_dpdk_build_info(sut_node.get_dpdk_build_info()) 400 tg_node.set_up_test_run(test_run_config, dpdk_build_config) 401 test_run_result.update_setup(Result.PASS) 402 except Exception as e: 403 self._logger.exception("Test run setup failed.") 404 test_run_result.update_setup(Result.FAIL, e) 405 406 else: 407 self._run_test_suites(sut_node, tg_node, test_run_result, test_suites_with_cases) 408 409 finally: 410 try: 411 self._logger.set_stage(DtsStage.test_run_teardown) 412 sut_node.tear_down_test_run() 413 tg_node.tear_down_test_run() 414 test_run_result.update_teardown(Result.PASS) 415 except Exception as e: 416 self._logger.exception("Test run teardown failed.") 417 test_run_result.update_teardown(Result.FAIL, e) 418 419 def _get_supported_capabilities( 420 self, 421 sut_node: SutNode, 422 topology_config: Topology, 423 test_suites_with_cases: Iterable[TestSuiteWithCases], 424 ) -> set[Capability]: 425 426 capabilities_to_check = set() 427 for test_suite_with_cases in test_suites_with_cases: 428 capabilities_to_check.update(test_suite_with_cases.required_capabilities) 429 430 self._logger.debug(f"Found capabilities to check: {capabilities_to_check}") 431 432 return get_supported_capabilities(sut_node, topology_config, capabilities_to_check) 433 434 def _run_test_suites( 435 self, 436 sut_node: SutNode, 437 tg_node: TGNode, 438 test_run_result: TestRunResult, 439 test_suites_with_cases: Iterable[TestSuiteWithCases], 440 ) -> None: 441 """Run `test_suites_with_cases` with the current test run. 442 443 The method assumes the DPDK we're testing has already been built on the SUT node. 444 445 Before running any suites, the method determines whether they should be skipped 446 by inspecting any required capabilities the test suite needs and comparing those 447 to capabilities supported by the tested environment. If all capabilities are supported, 448 the suite is run. If all test cases in a test suite would be skipped, the whole test suite 449 is skipped (the setup and teardown is not run). 450 451 If a blocking test suite (such as the smoke test suite) fails, the rest of the test suites 452 in the current test run won't be executed. 453 454 Args: 455 sut_node: The test run's SUT node. 456 tg_node: The test run's TG node. 457 test_run_result: The test run's result. 458 test_suites_with_cases: The test suites with test cases to run. 459 """ 460 end_dpdk_build = False 461 topology = Topology(sut_node.ports, tg_node.ports) 462 supported_capabilities = self._get_supported_capabilities( 463 sut_node, topology, test_suites_with_cases 464 ) 465 for test_suite_with_cases in test_suites_with_cases: 466 test_suite_with_cases.mark_skip_unsupported(supported_capabilities) 467 test_suite_result = test_run_result.add_test_suite(test_suite_with_cases) 468 try: 469 if not test_suite_with_cases.skip: 470 self._run_test_suite( 471 sut_node, 472 tg_node, 473 topology, 474 test_suite_result, 475 test_suite_with_cases, 476 ) 477 else: 478 self._logger.info( 479 f"Test suite execution SKIPPED: " 480 f"'{test_suite_with_cases.test_suite_class.__name__}'. Reason: " 481 f"{test_suite_with_cases.test_suite_class.skip_reason}" 482 ) 483 test_suite_result.update_setup(Result.SKIP) 484 except BlockingTestSuiteError as e: 485 self._logger.exception( 486 f"An error occurred within {test_suite_with_cases.test_suite_class.__name__}. " 487 "Skipping the rest of the test suites in this test run." 488 ) 489 self._result.add_error(e) 490 end_dpdk_build = True 491 # if a blocking test failed and we need to bail out of suite executions 492 if end_dpdk_build: 493 break 494 495 def _run_test_suite( 496 self, 497 sut_node: SutNode, 498 tg_node: TGNode, 499 topology: Topology, 500 test_suite_result: TestSuiteResult, 501 test_suite_with_cases: TestSuiteWithCases, 502 ) -> None: 503 """Set up, execute and tear down `test_suite_with_cases`. 504 505 The method assumes the DPDK we're testing has already been built on the SUT node. 506 507 Test suite execution consists of running the discovered test cases. 508 A test case run consists of setup, execution and teardown of said test case. 509 510 Record the setup and the teardown and handle failures. 511 512 Args: 513 sut_node: The test run's SUT node. 514 tg_node: The test run's TG node. 515 test_suite_result: The test suite level result object associated 516 with the current test suite. 517 test_suite_with_cases: The test suite with test cases to run. 518 519 Raises: 520 BlockingTestSuiteError: If a blocking test suite fails. 521 """ 522 test_suite_name = test_suite_with_cases.test_suite_class.__name__ 523 self._logger.set_stage( 524 DtsStage.test_suite_setup, Path(SETTINGS.output_dir, test_suite_name) 525 ) 526 test_suite = test_suite_with_cases.test_suite_class(sut_node, tg_node, topology) 527 try: 528 self._logger.info(f"Starting test suite setup: {test_suite_name}") 529 test_suite.set_up_suite() 530 test_suite_result.update_setup(Result.PASS) 531 self._logger.info(f"Test suite setup successful: {test_suite_name}") 532 except Exception as e: 533 self._logger.exception(f"Test suite setup ERROR: {test_suite_name}") 534 test_suite_result.update_setup(Result.ERROR, e) 535 536 else: 537 self._execute_test_suite( 538 test_suite, 539 test_suite_with_cases.test_cases, 540 test_suite_result, 541 ) 542 finally: 543 try: 544 self._logger.set_stage(DtsStage.test_suite_teardown) 545 test_suite.tear_down_suite() 546 sut_node.kill_cleanup_dpdk_apps() 547 test_suite_result.update_teardown(Result.PASS) 548 except Exception as e: 549 self._logger.exception(f"Test suite teardown ERROR: {test_suite_name}") 550 self._logger.warning( 551 f"Test suite '{test_suite_name}' teardown failed, " 552 "the next test suite may be affected." 553 ) 554 test_suite_result.update_setup(Result.ERROR, e) 555 if len(test_suite_result.get_errors()) > 0 and test_suite.is_blocking: 556 raise BlockingTestSuiteError(test_suite_name) 557 558 def _execute_test_suite( 559 self, 560 test_suite: TestSuite, 561 test_cases: Iterable[type[TestCase]], 562 test_suite_result: TestSuiteResult, 563 ) -> None: 564 """Execute all `test_cases` in `test_suite`. 565 566 If the :option:`--re-run` command line argument or the :envvar:`DTS_RERUN` environment 567 variable is set, in case of a test case failure, the test case will be executed again 568 until it passes or it fails that many times in addition of the first failure. 569 570 Args: 571 test_suite: The test suite object. 572 test_cases: The list of test case functions. 573 test_suite_result: The test suite level result object associated 574 with the current test suite. 575 """ 576 self._logger.set_stage(DtsStage.test_suite) 577 for test_case in test_cases: 578 test_case_name = test_case.__name__ 579 test_case_result = test_suite_result.add_test_case(test_case_name) 580 all_attempts = SETTINGS.re_run + 1 581 attempt_nr = 1 582 if not test_case.skip: 583 self._run_test_case(test_suite, test_case, test_case_result) 584 while not test_case_result and attempt_nr < all_attempts: 585 attempt_nr += 1 586 self._logger.info( 587 f"Re-running FAILED test case '{test_case_name}'. " 588 f"Attempt number {attempt_nr} out of {all_attempts}." 589 ) 590 self._run_test_case(test_suite, test_case, test_case_result) 591 else: 592 self._logger.info( 593 f"Test case execution SKIPPED: {test_case_name}. Reason: " 594 f"{test_case.skip_reason}" 595 ) 596 test_case_result.update_setup(Result.SKIP) 597 598 def _run_test_case( 599 self, 600 test_suite: TestSuite, 601 test_case: type[TestCase], 602 test_case_result: TestCaseResult, 603 ) -> None: 604 """Setup, execute and teardown `test_case_method` from `test_suite`. 605 606 Record the result of the setup and the teardown and handle failures. 607 608 Args: 609 test_suite: The test suite object. 610 test_case: The test case function. 611 test_case_result: The test case level result object associated 612 with the current test case. 613 """ 614 test_case_name = test_case.__name__ 615 616 try: 617 # run set_up function for each case 618 test_suite.set_up_test_case() 619 test_case_result.update_setup(Result.PASS) 620 except SSHTimeoutError as e: 621 self._logger.exception(f"Test case setup FAILED: {test_case_name}") 622 test_case_result.update_setup(Result.FAIL, e) 623 except Exception as e: 624 self._logger.exception(f"Test case setup ERROR: {test_case_name}") 625 test_case_result.update_setup(Result.ERROR, e) 626 627 else: 628 # run test case if setup was successful 629 self._execute_test_case(test_suite, test_case, test_case_result) 630 631 finally: 632 try: 633 test_suite.tear_down_test_case() 634 test_case_result.update_teardown(Result.PASS) 635 except Exception as e: 636 self._logger.exception(f"Test case teardown ERROR: {test_case_name}") 637 self._logger.warning( 638 f"Test case '{test_case_name}' teardown failed, " 639 f"the next test case may be affected." 640 ) 641 test_case_result.update_teardown(Result.ERROR, e) 642 test_case_result.update(Result.ERROR) 643 644 def _execute_test_case( 645 self, 646 test_suite: TestSuite, 647 test_case: type[TestCase], 648 test_case_result: TestCaseResult, 649 ) -> None: 650 """Execute `test_case_method` from `test_suite`, record the result and handle failures. 651 652 Args: 653 test_suite: The test suite object. 654 test_case: The test case function. 655 test_case_result: The test case level result object associated 656 with the current test case. 657 """ 658 test_case_name = test_case.__name__ 659 try: 660 self._logger.info(f"Starting test case execution: {test_case_name}") 661 # Explicit method binding is required, otherwise mypy complains 662 MethodType(test_case, test_suite)() 663 test_case_result.update(Result.PASS) 664 self._logger.info(f"Test case execution PASSED: {test_case_name}") 665 666 except TestCaseVerifyError as e: 667 self._logger.exception(f"Test case execution FAILED: {test_case_name}") 668 test_case_result.update(Result.FAIL, e) 669 except Exception as e: 670 self._logger.exception(f"Test case execution ERROR: {test_case_name}") 671 test_case_result.update(Result.ERROR, e) 672 except KeyboardInterrupt: 673 self._logger.error(f"Test case execution INTERRUPTED by user: {test_case_name}") 674 test_case_result.update(Result.SKIP) 675 raise KeyboardInterrupt("Stop DTS") 676 677 def _exit_dts(self) -> None: 678 """Process all errors and exit with the proper exit code.""" 679 self._result.process() 680 681 if self._logger: 682 self._logger.info("DTS execution has ended.") 683 684 sys.exit(self._result.get_return_code()) 685 686 def _init_random_seed(self, conf: TestRunConfiguration) -> None: 687 """Initialize the random seed to use for the test run.""" 688 seed = SETTINGS.random_seed or conf.random_seed or random.randrange(0xFFFF_FFFF) 689 self._logger.info(f"Initializing test run with random seed {seed}.") 690 random.seed(seed) 691