1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright(c) 2010-2019 Intel Corporation 3# Copyright(c) 2022-2023 PANTHEON.tech s.r.o. 4# Copyright(c) 2022-2023 University of New Hampshire 5# Copyright(c) 2024 Arm Limited 6 7"""Test suite runner module. 8 9The module is responsible for running DTS in a series of stages: 10 11 #. Test run stage, 12 #. DPDK build stage, 13 #. Test suite stage, 14 #. Test case stage. 15 16The test run stage sets up the environment before running test suites. 17The test suite stage sets up steps common to all test cases 18and the test case stage runs test cases individually. 19""" 20 21import os 22import random 23import sys 24from pathlib import Path 25from types import MethodType 26from typing import Iterable 27 28from framework.testbed_model.capability import Capability, get_supported_capabilities 29from framework.testbed_model.sut_node import SutNode 30from framework.testbed_model.tg_node import TGNode 31 32from .config import ( 33 Configuration, 34 DPDKPrecompiledBuildConfiguration, 35 SutNodeConfiguration, 36 TestRunConfiguration, 37 TestSuiteConfig, 38 TGNodeConfiguration, 39 load_config, 40) 41from .exception import BlockingTestSuiteError, SSHTimeoutError, TestCaseVerifyError 42from .logger import DTSLogger, DtsStage, get_dts_logger 43from .settings import SETTINGS 44from .test_result import ( 45 DTSResult, 46 Result, 47 TestCaseResult, 48 TestRunResult, 49 TestSuiteResult, 50 TestSuiteWithCases, 51) 52from .test_suite import TestCase, TestSuite 53from .testbed_model.topology import Topology 54 55 56class DTSRunner: 57 r"""Test suite runner class. 58 59 The class is responsible for running tests on testbeds defined in the test run configuration. 60 Each setup or teardown of each stage is recorded in a :class:`~framework.test_result.DTSResult` 61 or one of its subclasses. The test case results are also recorded. 62 63 If an error occurs, the current stage is aborted, the error is recorded, everything in 64 the inner stages is marked as blocked and the run continues in the next iteration 65 of the same stage. The return code is the highest `severity` of all 66 :class:`~.framework.exception.DTSError`\s. 67 68 Example: 69 An error occurs in a test suite setup. The current test suite is aborted, 70 all its test cases are marked as blocked and the run continues 71 with the next test suite. If the errored test suite was the last one in the 72 given test run, the next test run begins. 73 """ 74 75 _configuration: Configuration 76 _logger: DTSLogger 77 _result: DTSResult 78 _test_suite_class_prefix: str 79 _test_suite_module_prefix: str 80 _func_test_case_regex: str 81 _perf_test_case_regex: str 82 83 def __init__(self): 84 """Initialize the instance with configuration, logger, result and string constants.""" 85 self._configuration = load_config(SETTINGS.config_file_path) 86 self._logger = get_dts_logger() 87 if not os.path.exists(SETTINGS.output_dir): 88 os.makedirs(SETTINGS.output_dir) 89 self._logger.add_dts_root_logger_handlers(SETTINGS.verbose, SETTINGS.output_dir) 90 self._result = DTSResult(SETTINGS.output_dir, self._logger) 91 self._test_suite_class_prefix = "Test" 92 self._test_suite_module_prefix = "tests.TestSuite_" 93 self._func_test_case_regex = r"test_(?!perf_)" 94 self._perf_test_case_regex = r"test_perf_" 95 96 def run(self) -> None: 97 """Run all test runs from the test run configuration. 98 99 Before running test suites, test runs are first set up. 100 The test runs defined in the test run configuration are iterated over. 101 The test runs define which tests to run and where to run them. 102 103 The test suites are set up for each test run and each discovered 104 test case within the test suite is set up, executed and torn down. After all test cases 105 have been executed, the test suite is torn down and the next test suite will be run. Once 106 all test suites have been run, the next test run will be tested. 107 108 In order to properly mark test suites and test cases as blocked in case of a failure, 109 we need to have discovered which test suites and test cases to run before any failures 110 happen. The discovery happens at the earliest point at the start of each test run. 111 112 All the nested steps look like this: 113 114 #. Test run setup 115 116 #. Test suite setup 117 118 #. Test case setup 119 #. Test case logic 120 #. Test case teardown 121 122 #. Test suite teardown 123 124 #. Test run teardown 125 126 The test cases are filtered according to the specification in the test run configuration and 127 the :option:`--test-suite` command line argument or 128 the :envvar:`DTS_TESTCASES` environment variable. 129 """ 130 sut_nodes: dict[str, SutNode] = {} 131 tg_nodes: dict[str, TGNode] = {} 132 try: 133 # check the python version of the server that runs dts 134 self._check_dts_python_version() 135 self._result.update_setup(Result.PASS) 136 137 # for all test run sections 138 for test_run_with_nodes_config in self._configuration.test_runs_with_nodes: 139 test_run_config, sut_node_config, tg_node_config = test_run_with_nodes_config 140 self._logger.set_stage(DtsStage.test_run_setup) 141 self._logger.info(f"Running test run with SUT '{sut_node_config.name}'.") 142 self._init_random_seed(test_run_config) 143 test_run_result = self._result.add_test_run(test_run_config) 144 # we don't want to modify the original config, so create a copy 145 test_run_test_suites = list( 146 SETTINGS.test_suites if SETTINGS.test_suites else test_run_config.test_suites 147 ) 148 if not test_run_config.skip_smoke_tests: 149 test_run_test_suites[:0] = [TestSuiteConfig(test_suite="smoke_tests")] 150 try: 151 test_suites_with_cases = self._get_test_suites_with_cases( 152 test_run_test_suites, test_run_config.func, test_run_config.perf 153 ) 154 test_run_result.test_suites_with_cases = test_suites_with_cases 155 except Exception as e: 156 self._logger.exception( 157 f"Invalid test suite configuration found: " f"{test_run_test_suites}." 158 ) 159 test_run_result.update_setup(Result.FAIL, e) 160 161 else: 162 self._connect_nodes_and_run_test_run( 163 sut_nodes, 164 tg_nodes, 165 sut_node_config, 166 tg_node_config, 167 test_run_config, 168 test_run_result, 169 test_suites_with_cases, 170 ) 171 172 except Exception as e: 173 self._logger.exception("An unexpected error has occurred.") 174 self._result.add_error(e) 175 raise 176 177 finally: 178 try: 179 self._logger.set_stage(DtsStage.post_run) 180 for node in (sut_nodes | tg_nodes).values(): 181 node.close() 182 self._result.update_teardown(Result.PASS) 183 except Exception as e: 184 self._logger.exception("The final cleanup of nodes failed.") 185 self._result.update_teardown(Result.ERROR, e) 186 187 # we need to put the sys.exit call outside the finally clause to make sure 188 # that unexpected exceptions will propagate 189 # in that case, the error that should be reported is the uncaught exception as 190 # that is a severe error originating from the framework 191 # at that point, we'll only have partial results which could be impacted by the 192 # error causing the uncaught exception, making them uninterpretable 193 self._exit_dts() 194 195 def _check_dts_python_version(self) -> None: 196 """Check the required Python version - v3.10.""" 197 if sys.version_info.major < 3 or ( 198 sys.version_info.major == 3 and sys.version_info.minor < 10 199 ): 200 self._logger.warning( 201 "DTS execution node's python version is lower than Python 3.10, " 202 "is deprecated and will not work in future releases." 203 ) 204 self._logger.warning("Please use Python >= 3.10 instead.") 205 206 def _get_test_suites_with_cases( 207 self, 208 test_suite_configs: list[TestSuiteConfig], 209 func: bool, 210 perf: bool, 211 ) -> list[TestSuiteWithCases]: 212 """Get test suites with selected cases. 213 214 The test suites with test cases defined in the user configuration are selected 215 and the corresponding functions and classes are gathered. 216 217 Args: 218 test_suite_configs: Test suite configurations. 219 func: Whether to include functional test cases in the final list. 220 perf: Whether to include performance test cases in the final list. 221 222 Returns: 223 The test suites, each with test cases. 224 """ 225 test_suites_with_cases = [] 226 227 for test_suite_config in test_suite_configs: 228 test_suite_class = test_suite_config.test_suite_spec.class_obj 229 test_cases: list[type[TestCase]] = [] 230 func_test_cases, perf_test_cases = test_suite_class.filter_test_cases( 231 test_suite_config.test_cases_names 232 ) 233 if func: 234 test_cases.extend(func_test_cases) 235 if perf: 236 test_cases.extend(perf_test_cases) 237 238 test_suites_with_cases.append( 239 TestSuiteWithCases(test_suite_class=test_suite_class, test_cases=test_cases) 240 ) 241 return test_suites_with_cases 242 243 def _connect_nodes_and_run_test_run( 244 self, 245 sut_nodes: dict[str, SutNode], 246 tg_nodes: dict[str, TGNode], 247 sut_node_config: SutNodeConfiguration, 248 tg_node_config: TGNodeConfiguration, 249 test_run_config: TestRunConfiguration, 250 test_run_result: TestRunResult, 251 test_suites_with_cases: Iterable[TestSuiteWithCases], 252 ) -> None: 253 """Connect nodes, then continue to run the given test run. 254 255 Connect the :class:`SutNode` and the :class:`TGNode` of this `test_run_config`. 256 If either has already been connected, it's going to be in either `sut_nodes` or `tg_nodes`, 257 respectively. 258 If not, connect and add the node to the respective `sut_nodes` or `tg_nodes` :class:`dict`. 259 260 Args: 261 sut_nodes: A dictionary storing connected/to be connected SUT nodes. 262 tg_nodes: A dictionary storing connected/to be connected TG nodes. 263 sut_node_config: The test run's SUT node configuration. 264 tg_node_config: The test run's TG node configuration. 265 test_run_config: A test run configuration. 266 test_run_result: The test run's result. 267 test_suites_with_cases: The test suites with test cases to run. 268 """ 269 sut_node = sut_nodes.get(sut_node_config.name) 270 tg_node = tg_nodes.get(tg_node_config.name) 271 272 try: 273 if not sut_node: 274 sut_node = SutNode(sut_node_config) 275 sut_nodes[sut_node.name] = sut_node 276 if not tg_node: 277 tg_node = TGNode(tg_node_config) 278 tg_nodes[tg_node.name] = tg_node 279 except Exception as e: 280 failed_node = test_run_config.system_under_test_node.node_name 281 if sut_node: 282 failed_node = test_run_config.traffic_generator_node 283 self._logger.exception(f"The Creation of node {failed_node} failed.") 284 test_run_result.update_setup(Result.FAIL, e) 285 286 else: 287 self._run_test_run( 288 sut_node, tg_node, test_run_config, test_run_result, test_suites_with_cases 289 ) 290 291 def _run_test_run( 292 self, 293 sut_node: SutNode, 294 tg_node: TGNode, 295 test_run_config: TestRunConfiguration, 296 test_run_result: TestRunResult, 297 test_suites_with_cases: Iterable[TestSuiteWithCases], 298 ) -> None: 299 """Run the given test run. 300 301 This involves running the test run setup as well as running all test suites 302 in the given test run. After that, the test run teardown is run. 303 304 Args: 305 sut_node: The test run's SUT node. 306 tg_node: The test run's TG node. 307 test_run_config: A test run configuration. 308 test_run_result: The test run's result. 309 test_suites_with_cases: The test suites with test cases to run. 310 311 Raises: 312 ConfigurationError: If the DPDK sources or build is not set up from config or settings. 313 """ 314 self._logger.info( 315 f"Running test run with SUT '{test_run_config.system_under_test_node.node_name}'." 316 ) 317 test_run_result.ports = sut_node.ports 318 test_run_result.sut_info = sut_node.node_info 319 try: 320 dpdk_build_config = test_run_config.dpdk_config 321 if new_location := SETTINGS.dpdk_location: 322 dpdk_build_config = dpdk_build_config.model_copy( 323 update={"dpdk_location": new_location} 324 ) 325 if dir := SETTINGS.precompiled_build_dir: 326 dpdk_build_config = DPDKPrecompiledBuildConfiguration( 327 dpdk_location=dpdk_build_config.dpdk_location, precompiled_build_dir=dir 328 ) 329 sut_node.set_up_test_run(test_run_config, dpdk_build_config) 330 test_run_result.dpdk_build_info = sut_node.get_dpdk_build_info() 331 tg_node.set_up_test_run(test_run_config, dpdk_build_config) 332 test_run_result.update_setup(Result.PASS) 333 except Exception as e: 334 self._logger.exception("Test run setup failed.") 335 test_run_result.update_setup(Result.FAIL, e) 336 337 else: 338 self._run_test_suites(sut_node, tg_node, test_run_result, test_suites_with_cases) 339 340 finally: 341 try: 342 self._logger.set_stage(DtsStage.test_run_teardown) 343 sut_node.tear_down_test_run() 344 tg_node.tear_down_test_run() 345 test_run_result.update_teardown(Result.PASS) 346 except Exception as e: 347 self._logger.exception("Test run teardown failed.") 348 test_run_result.update_teardown(Result.FAIL, e) 349 350 def _get_supported_capabilities( 351 self, 352 sut_node: SutNode, 353 topology_config: Topology, 354 test_suites_with_cases: Iterable[TestSuiteWithCases], 355 ) -> set[Capability]: 356 357 capabilities_to_check = set() 358 for test_suite_with_cases in test_suites_with_cases: 359 capabilities_to_check.update(test_suite_with_cases.required_capabilities) 360 361 self._logger.debug(f"Found capabilities to check: {capabilities_to_check}") 362 363 return get_supported_capabilities(sut_node, topology_config, capabilities_to_check) 364 365 def _run_test_suites( 366 self, 367 sut_node: SutNode, 368 tg_node: TGNode, 369 test_run_result: TestRunResult, 370 test_suites_with_cases: Iterable[TestSuiteWithCases], 371 ) -> None: 372 """Run `test_suites_with_cases` with the current test run. 373 374 The method assumes the DPDK we're testing has already been built on the SUT node. 375 376 Before running any suites, the method determines whether they should be skipped 377 by inspecting any required capabilities the test suite needs and comparing those 378 to capabilities supported by the tested environment. If all capabilities are supported, 379 the suite is run. If all test cases in a test suite would be skipped, the whole test suite 380 is skipped (the setup and teardown is not run). 381 382 If a blocking test suite (such as the smoke test suite) fails, the rest of the test suites 383 in the current test run won't be executed. 384 385 Args: 386 sut_node: The test run's SUT node. 387 tg_node: The test run's TG node. 388 test_run_result: The test run's result. 389 test_suites_with_cases: The test suites with test cases to run. 390 """ 391 end_dpdk_build = False 392 topology = Topology(sut_node.ports, tg_node.ports) 393 supported_capabilities = self._get_supported_capabilities( 394 sut_node, topology, test_suites_with_cases 395 ) 396 for test_suite_with_cases in test_suites_with_cases: 397 test_suite_with_cases.mark_skip_unsupported(supported_capabilities) 398 test_suite_result = test_run_result.add_test_suite(test_suite_with_cases) 399 try: 400 if not test_suite_with_cases.skip: 401 self._run_test_suite( 402 sut_node, 403 tg_node, 404 topology, 405 test_suite_result, 406 test_suite_with_cases, 407 ) 408 else: 409 self._logger.info( 410 f"Test suite execution SKIPPED: " 411 f"'{test_suite_with_cases.test_suite_class.__name__}'. Reason: " 412 f"{test_suite_with_cases.test_suite_class.skip_reason}" 413 ) 414 test_suite_result.update_setup(Result.SKIP) 415 except BlockingTestSuiteError as e: 416 self._logger.exception( 417 f"An error occurred within {test_suite_with_cases.test_suite_class.__name__}. " 418 "Skipping the rest of the test suites in this test run." 419 ) 420 self._result.add_error(e) 421 end_dpdk_build = True 422 # if a blocking test failed and we need to bail out of suite executions 423 if end_dpdk_build: 424 break 425 426 def _run_test_suite( 427 self, 428 sut_node: SutNode, 429 tg_node: TGNode, 430 topology: Topology, 431 test_suite_result: TestSuiteResult, 432 test_suite_with_cases: TestSuiteWithCases, 433 ) -> None: 434 """Set up, execute and tear down `test_suite_with_cases`. 435 436 The method assumes the DPDK we're testing has already been built on the SUT node. 437 438 Test suite execution consists of running the discovered test cases. 439 A test case run consists of setup, execution and teardown of said test case. 440 441 Record the setup and the teardown and handle failures. 442 443 Args: 444 sut_node: The test run's SUT node. 445 tg_node: The test run's TG node. 446 test_suite_result: The test suite level result object associated 447 with the current test suite. 448 test_suite_with_cases: The test suite with test cases to run. 449 450 Raises: 451 BlockingTestSuiteError: If a blocking test suite fails. 452 """ 453 test_suite_name = test_suite_with_cases.test_suite_class.__name__ 454 self._logger.set_stage( 455 DtsStage.test_suite_setup, Path(SETTINGS.output_dir, test_suite_name) 456 ) 457 test_suite = test_suite_with_cases.test_suite_class(sut_node, tg_node, topology) 458 try: 459 self._logger.info(f"Starting test suite setup: {test_suite_name}") 460 test_suite.set_up_suite() 461 test_suite_result.update_setup(Result.PASS) 462 self._logger.info(f"Test suite setup successful: {test_suite_name}") 463 except Exception as e: 464 self._logger.exception(f"Test suite setup ERROR: {test_suite_name}") 465 test_suite_result.update_setup(Result.ERROR, e) 466 467 else: 468 self._execute_test_suite( 469 test_suite, 470 test_suite_with_cases.test_cases, 471 test_suite_result, 472 ) 473 finally: 474 try: 475 self._logger.set_stage(DtsStage.test_suite_teardown) 476 test_suite.tear_down_suite() 477 sut_node.kill_cleanup_dpdk_apps() 478 test_suite_result.update_teardown(Result.PASS) 479 except Exception as e: 480 self._logger.exception(f"Test suite teardown ERROR: {test_suite_name}") 481 self._logger.warning( 482 f"Test suite '{test_suite_name}' teardown failed, " 483 "the next test suite may be affected." 484 ) 485 test_suite_result.update_setup(Result.ERROR, e) 486 if len(test_suite_result.get_errors()) > 0 and test_suite.is_blocking: 487 raise BlockingTestSuiteError(test_suite_name) 488 489 def _execute_test_suite( 490 self, 491 test_suite: TestSuite, 492 test_cases: Iterable[type[TestCase]], 493 test_suite_result: TestSuiteResult, 494 ) -> None: 495 """Execute all `test_cases` in `test_suite`. 496 497 If the :option:`--re-run` command line argument or the :envvar:`DTS_RERUN` environment 498 variable is set, in case of a test case failure, the test case will be executed again 499 until it passes or it fails that many times in addition of the first failure. 500 501 Args: 502 test_suite: The test suite object. 503 test_cases: The list of test case functions. 504 test_suite_result: The test suite level result object associated 505 with the current test suite. 506 """ 507 self._logger.set_stage(DtsStage.test_suite) 508 for test_case in test_cases: 509 test_case_name = test_case.__name__ 510 test_case_result = test_suite_result.add_test_case(test_case_name) 511 all_attempts = SETTINGS.re_run + 1 512 attempt_nr = 1 513 if not test_case.skip: 514 self._run_test_case(test_suite, test_case, test_case_result) 515 while not test_case_result and attempt_nr < all_attempts: 516 attempt_nr += 1 517 self._logger.info( 518 f"Re-running FAILED test case '{test_case_name}'. " 519 f"Attempt number {attempt_nr} out of {all_attempts}." 520 ) 521 self._run_test_case(test_suite, test_case, test_case_result) 522 else: 523 self._logger.info( 524 f"Test case execution SKIPPED: {test_case_name}. Reason: " 525 f"{test_case.skip_reason}" 526 ) 527 test_case_result.update_setup(Result.SKIP) 528 529 def _run_test_case( 530 self, 531 test_suite: TestSuite, 532 test_case: type[TestCase], 533 test_case_result: TestCaseResult, 534 ) -> None: 535 """Setup, execute and teardown `test_case_method` from `test_suite`. 536 537 Record the result of the setup and the teardown and handle failures. 538 539 Args: 540 test_suite: The test suite object. 541 test_case: The test case function. 542 test_case_result: The test case level result object associated 543 with the current test case. 544 """ 545 test_case_name = test_case.__name__ 546 547 try: 548 # run set_up function for each case 549 test_suite.set_up_test_case() 550 test_case_result.update_setup(Result.PASS) 551 except SSHTimeoutError as e: 552 self._logger.exception(f"Test case setup FAILED: {test_case_name}") 553 test_case_result.update_setup(Result.FAIL, e) 554 except Exception as e: 555 self._logger.exception(f"Test case setup ERROR: {test_case_name}") 556 test_case_result.update_setup(Result.ERROR, e) 557 558 else: 559 # run test case if setup was successful 560 self._execute_test_case(test_suite, test_case, test_case_result) 561 562 finally: 563 try: 564 test_suite.tear_down_test_case() 565 test_case_result.update_teardown(Result.PASS) 566 except Exception as e: 567 self._logger.exception(f"Test case teardown ERROR: {test_case_name}") 568 self._logger.warning( 569 f"Test case '{test_case_name}' teardown failed, " 570 f"the next test case may be affected." 571 ) 572 test_case_result.update_teardown(Result.ERROR, e) 573 test_case_result.update(Result.ERROR) 574 575 def _execute_test_case( 576 self, 577 test_suite: TestSuite, 578 test_case: type[TestCase], 579 test_case_result: TestCaseResult, 580 ) -> None: 581 """Execute `test_case_method` from `test_suite`, record the result and handle failures. 582 583 Args: 584 test_suite: The test suite object. 585 test_case: The test case function. 586 test_case_result: The test case level result object associated 587 with the current test case. 588 """ 589 test_case_name = test_case.__name__ 590 try: 591 self._logger.info(f"Starting test case execution: {test_case_name}") 592 # Explicit method binding is required, otherwise mypy complains 593 MethodType(test_case, test_suite)() 594 test_case_result.update(Result.PASS) 595 self._logger.info(f"Test case execution PASSED: {test_case_name}") 596 597 except TestCaseVerifyError as e: 598 self._logger.exception(f"Test case execution FAILED: {test_case_name}") 599 test_case_result.update(Result.FAIL, e) 600 except Exception as e: 601 self._logger.exception(f"Test case execution ERROR: {test_case_name}") 602 test_case_result.update(Result.ERROR, e) 603 except KeyboardInterrupt: 604 self._logger.error(f"Test case execution INTERRUPTED by user: {test_case_name}") 605 test_case_result.update(Result.SKIP) 606 raise KeyboardInterrupt("Stop DTS") 607 608 def _exit_dts(self) -> None: 609 """Process all errors and exit with the proper exit code.""" 610 self._result.process() 611 612 if self._logger: 613 self._logger.info("DTS execution has ended.") 614 615 sys.exit(self._result.get_return_code()) 616 617 def _init_random_seed(self, conf: TestRunConfiguration) -> None: 618 """Initialize the random seed to use for the test run.""" 619 seed = SETTINGS.random_seed or conf.random_seed or random.randrange(0xFFFF_FFFF) 620 self._logger.info(f"Initializing test run with random seed {seed}.") 621 random.seed(seed) 622