1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright(c) 2023 PANTHEON.tech s.r.o. 3# Copyright(c) 2023 University of New Hampshire 4# Copyright(c) 2024 Arm Limited 5 6r"""Record and process DTS results. 7 8The results are recorded in a hierarchical manner: 9 10 * :class:`DTSResult` contains 11 * :class:`TestRunResult` contains 12 * :class:`TestSuiteResult` contains 13 * :class:`TestCaseResult` 14 15Each result may contain multiple lower level results, e.g. there are multiple 16:class:`TestSuiteResult`\s in a :class:`TestRunResult`. 17The results have common parts, such as setup and teardown results, captured in :class:`BaseResult`, 18which also defines some common behaviors in its methods. 19 20Each result class has its own idiosyncrasies which they implement in overridden methods. 21 22The :option:`--output` command line argument and the :envvar:`DTS_OUTPUT_DIR` environment 23variable modify the directory where the files with results will be stored. 24""" 25 26import json 27from collections.abc import MutableSequence 28from dataclasses import asdict, dataclass, field 29from enum import Enum, auto 30from pathlib import Path 31from typing import Any, Callable, TypedDict 32 33from framework.testbed_model.capability import Capability 34 35from .config import TestRunConfiguration, TestSuiteConfig 36from .exception import DTSError, ErrorSeverity 37from .logger import DTSLogger 38from .test_suite import TestCase, TestSuite 39from .testbed_model.os_session import OSSessionInfo 40from .testbed_model.port import Port 41from .testbed_model.sut_node import DPDKBuildInfo 42 43 44@dataclass(slots=True, frozen=True) 45class TestSuiteWithCases: 46 """A test suite class with test case methods. 47 48 An auxiliary class holding a test case class with test case methods. The intended use of this 49 class is to hold a subset of test cases (which could be all test cases) because we don't have 50 all the data to instantiate the class at the point of inspection. The knowledge of this subset 51 is needed in case an error occurs before the class is instantiated and we need to record 52 which test cases were blocked by the error. 53 54 Attributes: 55 test_suite_class: The test suite class. 56 test_cases: The test case methods. 57 required_capabilities: The combined required capabilities of both the test suite 58 and the subset of test cases. 59 """ 60 61 test_suite_class: type[TestSuite] 62 test_cases: list[type[TestCase]] 63 required_capabilities: set[Capability] = field(default_factory=set, init=False) 64 65 def __post_init__(self): 66 """Gather the required capabilities of the test suite and all test cases.""" 67 for test_object in [self.test_suite_class] + self.test_cases: 68 self.required_capabilities.update(test_object.required_capabilities) 69 70 def create_config(self) -> TestSuiteConfig: 71 """Generate a :class:`TestSuiteConfig` from the stored test suite with test cases. 72 73 Returns: 74 The :class:`TestSuiteConfig` representation. 75 """ 76 return TestSuiteConfig( 77 test_suite=self.test_suite_class.__name__, 78 test_cases=[test_case.__name__ for test_case in self.test_cases], 79 ) 80 81 def mark_skip_unsupported(self, supported_capabilities: set[Capability]) -> None: 82 """Mark the test suite and test cases to be skipped. 83 84 The mark is applied if object to be skipped requires any capabilities and at least one of 85 them is not among `supported_capabilities`. 86 87 Args: 88 supported_capabilities: The supported capabilities. 89 """ 90 for test_object in [self.test_suite_class, *self.test_cases]: 91 capabilities_not_supported = test_object.required_capabilities - supported_capabilities 92 if capabilities_not_supported: 93 test_object.skip = True 94 capability_str = ( 95 "capability" if len(capabilities_not_supported) == 1 else "capabilities" 96 ) 97 test_object.skip_reason = ( 98 f"Required {capability_str} '{capabilities_not_supported}' not found." 99 ) 100 if not self.test_suite_class.skip: 101 if all(test_case.skip for test_case in self.test_cases): 102 self.test_suite_class.skip = True 103 104 self.test_suite_class.skip_reason = ( 105 "All test cases are marked to be skipped with reasons: " 106 f"{' '.join(test_case.skip_reason for test_case in self.test_cases)}" 107 ) 108 109 @property 110 def skip(self) -> bool: 111 """Skip the test suite if all test cases or the suite itself are to be skipped. 112 113 Returns: 114 :data:`True` if the test suite should be skipped, :data:`False` otherwise. 115 """ 116 return all(test_case.skip for test_case in self.test_cases) or self.test_suite_class.skip 117 118 119class Result(Enum): 120 """The possible states that a setup, a teardown or a test case may end up in.""" 121 122 #: 123 PASS = auto() 124 #: 125 FAIL = auto() 126 #: 127 ERROR = auto() 128 #: 129 BLOCK = auto() 130 #: 131 SKIP = auto() 132 133 def __bool__(self) -> bool: 134 """Only :attr:`PASS` is True.""" 135 return self is self.PASS 136 137 138class TestCaseResultDict(TypedDict): 139 """Represents the `TestCaseResult` results. 140 141 Attributes: 142 test_case_name: The name of the test case. 143 result: The result name of the test case. 144 """ 145 146 test_case_name: str 147 result: str 148 149 150class TestSuiteResultDict(TypedDict): 151 """Represents the `TestSuiteResult` results. 152 153 Attributes: 154 test_suite_name: The name of the test suite. 155 test_cases: A list of test case results contained in this test suite. 156 """ 157 158 test_suite_name: str 159 test_cases: list[TestCaseResultDict] 160 161 162class TestRunResultDict(TypedDict, total=False): 163 """Represents the `TestRunResult` results. 164 165 Attributes: 166 compiler_version: The version of the compiler used for the DPDK build. 167 dpdk_version: The version of DPDK being tested. 168 ports: A list of ports associated with the test run. 169 test_suites: A list of test suite results included in this test run. 170 summary: A dictionary containing overall results, such as pass/fail counts. 171 """ 172 173 compiler_version: str | None 174 dpdk_version: str | None 175 ports: list[dict[str, Any]] 176 test_suites: list[TestSuiteResultDict] 177 summary: dict[str, int | float] 178 179 180class DtsRunResultDict(TypedDict): 181 """Represents the `DtsRunResult` results. 182 183 Attributes: 184 test_runs: A list of test run results. 185 summary: A summary dictionary containing overall statistics for the test runs. 186 """ 187 188 test_runs: list[TestRunResultDict] 189 summary: dict[str, int | float] 190 191 192class FixtureResult: 193 """A record that stores the result of a setup or a teardown. 194 195 :attr:`~Result.FAIL` is a sensible default since it prevents false positives (which could happen 196 if the default was :attr:`~Result.PASS`). 197 198 Preventing false positives or other false results is preferable since a failure 199 is mostly likely to be investigated (the other false results may not be investigated at all). 200 201 Attributes: 202 result: The associated result. 203 error: The error in case of a failure. 204 """ 205 206 result: Result 207 error: Exception | None = None 208 209 def __init__( 210 self, 211 result: Result = Result.FAIL, 212 error: Exception | None = None, 213 ): 214 """Initialize the constructor with the fixture result and store a possible error. 215 216 Args: 217 result: The result to store. 218 error: The error which happened when a failure occurred. 219 """ 220 self.result = result 221 self.error = error 222 223 def __bool__(self) -> bool: 224 """A wrapper around the stored :class:`Result`.""" 225 return bool(self.result) 226 227 228class BaseResult: 229 """Common data and behavior of DTS results. 230 231 Stores the results of the setup and teardown portions of the corresponding stage. 232 The hierarchical nature of DTS results is captured recursively in an internal list. 233 A stage is each level in this particular hierarchy (pre-run or the top-most level, 234 test run, test suite and test case). 235 236 Attributes: 237 setup_result: The result of the setup of the particular stage. 238 teardown_result: The results of the teardown of the particular stage. 239 child_results: The results of the descendants in the results hierarchy. 240 """ 241 242 setup_result: FixtureResult 243 teardown_result: FixtureResult 244 child_results: MutableSequence["BaseResult"] 245 246 def __init__(self): 247 """Initialize the constructor.""" 248 self.setup_result = FixtureResult() 249 self.teardown_result = FixtureResult() 250 self.child_results = [] 251 252 def update_setup(self, result: Result, error: Exception | None = None) -> None: 253 """Store the setup result. 254 255 If the result is :attr:`~Result.BLOCK`, :attr:`~Result.ERROR` or :attr:`~Result.FAIL`, 256 then the corresponding child results in result hierarchy 257 are also marked with :attr:`~Result.BLOCK`. 258 259 Args: 260 result: The result of the setup. 261 error: The error that occurred in case of a failure. 262 """ 263 self.setup_result.result = result 264 self.setup_result.error = error 265 266 if result != Result.PASS: 267 result_to_mark = Result.BLOCK if result != Result.SKIP else Result.SKIP 268 self.update_teardown(result_to_mark) 269 self._mark_results(result_to_mark) 270 271 def _mark_results(self, result) -> None: 272 """Mark the child results or the result of the level itself as `result`. 273 274 The marking of results should be done in overloaded methods. 275 """ 276 277 def update_teardown(self, result: Result, error: Exception | None = None) -> None: 278 """Store the teardown result. 279 280 Args: 281 result: The result of the teardown. 282 error: The error that occurred in case of a failure. 283 """ 284 self.teardown_result.result = result 285 self.teardown_result.error = error 286 287 def _get_setup_teardown_errors(self) -> list[Exception]: 288 errors = [] 289 if self.setup_result.error: 290 errors.append(self.setup_result.error) 291 if self.teardown_result.error: 292 errors.append(self.teardown_result.error) 293 return errors 294 295 def _get_child_errors(self) -> list[Exception]: 296 return [error for child_result in self.child_results for error in child_result.get_errors()] 297 298 def get_errors(self) -> list[Exception]: 299 """Compile errors from the whole result hierarchy. 300 301 Returns: 302 The errors from setup, teardown and all errors found in the whole result hierarchy. 303 """ 304 return self._get_setup_teardown_errors() + self._get_child_errors() 305 306 def to_dict(self): 307 """Convert the results hierarchy into a dictionary representation.""" 308 309 def add_result(self, results: dict[str, int]): 310 """Collate the test case result to the given result hierarchy. 311 312 Args: 313 results: The dictionary in which results will be collated. 314 """ 315 for child_result in self.child_results: 316 child_result.add_result(results) 317 318 def generate_pass_rate_dict(self, test_run_summary) -> dict[str, float]: 319 """Generate a dictionary with the PASS/FAIL ratio of all test cases. 320 321 Args: 322 test_run_summary: The summary dictionary containing test result counts. 323 324 Returns: 325 A dictionary with the PASS/FAIL ratio of all test cases. 326 """ 327 return { 328 "PASS_RATE": ( 329 float(test_run_summary[Result.PASS.name]) 330 * 100 331 / sum(test_run_summary[result.name] for result in Result if result != Result.SKIP) 332 ) 333 } 334 335 336class DTSResult(BaseResult): 337 """Stores environment information and test results from a DTS run. 338 339 * Test run level information, such as testbed, the test suite list and 340 DPDK build configuration (compiler, target OS and cpu), 341 * Test suite and test case results, 342 * All errors that are caught and recorded during DTS execution. 343 344 The information is stored hierarchically. This is the first level of the hierarchy 345 and as such is where the data form the whole hierarchy is collated or processed. 346 347 The internal list stores the results of all test runs. 348 """ 349 350 _output_dir: str 351 _logger: DTSLogger 352 _errors: list[Exception] 353 _return_code: ErrorSeverity 354 355 def __init__(self, output_dir: str, logger: DTSLogger): 356 """Extend the constructor with top-level specifics. 357 358 Args: 359 output_dir: The directory where DTS logs and results are saved. 360 logger: The logger instance the whole result will use. 361 """ 362 super().__init__() 363 self._output_dir = output_dir 364 self._logger = logger 365 self._errors = [] 366 self._return_code = ErrorSeverity.NO_ERR 367 368 def add_test_run(self, test_run_config: TestRunConfiguration) -> "TestRunResult": 369 """Add and return the child result (test run). 370 371 Args: 372 test_run_config: A test run configuration. 373 374 Returns: 375 The test run's result. 376 """ 377 result = TestRunResult(test_run_config) 378 self.child_results.append(result) 379 return result 380 381 def add_error(self, error: Exception) -> None: 382 """Record an error that occurred outside any test run. 383 384 Args: 385 error: The exception to record. 386 """ 387 self._errors.append(error) 388 389 def process(self) -> None: 390 """Process the data after a whole DTS run. 391 392 The data is added to child objects during runtime and this object is not updated 393 at that time. This requires us to process the child data after it's all been gathered. 394 395 The processing gathers all errors and the statistics of test case results. 396 """ 397 self._errors += self.get_errors() 398 if self._errors and self._logger: 399 self._logger.debug("Summary of errors:") 400 for error in self._errors: 401 self._logger.debug(repr(error)) 402 403 TextSummary(self).save(Path(self._output_dir, "results_summary.txt")) 404 JsonResults(self).save(Path(self._output_dir, "results.json")) 405 406 def get_return_code(self) -> int: 407 """Go through all stored Exceptions and return the final DTS error code. 408 409 Returns: 410 The highest error code found. 411 """ 412 for error in self._errors: 413 error_return_code = ErrorSeverity.GENERIC_ERR 414 if isinstance(error, DTSError): 415 error_return_code = error.severity 416 417 if error_return_code > self._return_code: 418 self._return_code = error_return_code 419 420 return int(self._return_code) 421 422 def to_dict(self) -> DtsRunResultDict: 423 """Convert DTS result into a dictionary format. 424 425 The dictionary contains test runs and summary of test runs. 426 427 Returns: 428 A dictionary representation of the DTS result 429 """ 430 431 def merge_test_run_summaries(test_run_summaries: list[dict[str, int]]) -> dict[str, int]: 432 """Merge multiple test run summaries into one dictionary. 433 434 Args: 435 test_run_summaries: List of test run summary dictionaries. 436 437 Returns: 438 A merged dictionary containing the aggregated summary. 439 """ 440 return { 441 key.name: sum(test_run_summary[key.name] for test_run_summary in test_run_summaries) 442 for key in Result 443 } 444 445 test_runs = [child.to_dict() for child in self.child_results] 446 test_run_summary = merge_test_run_summaries([test_run["summary"] for test_run in test_runs]) 447 448 return { 449 "test_runs": test_runs, 450 "summary": test_run_summary | self.generate_pass_rate_dict(test_run_summary), 451 } 452 453 454class TestRunResult(BaseResult): 455 """The test run specific result. 456 457 The internal list stores the results of all test suites in a given test run. 458 459 Attributes: 460 compiler_version: The DPDK build compiler version. 461 dpdk_version: The built DPDK version. 462 sut_os_name: The operating system of the SUT node. 463 sut_os_version: The operating system version of the SUT node. 464 sut_kernel_version: The operating system kernel version of the SUT node. 465 """ 466 467 _config: TestRunConfiguration 468 _test_suites_with_cases: list[TestSuiteWithCases] 469 _ports: list[Port] 470 _sut_info: OSSessionInfo | None 471 _dpdk_build_info: DPDKBuildInfo | None 472 473 def __init__(self, test_run_config: TestRunConfiguration): 474 """Extend the constructor with the test run's config. 475 476 Args: 477 test_run_config: A test run configuration. 478 """ 479 super().__init__() 480 self._config = test_run_config 481 self._test_suites_with_cases = [] 482 self._ports = [] 483 self._sut_info = None 484 self._dpdk_build_info = None 485 486 def add_test_suite( 487 self, 488 test_suite_with_cases: TestSuiteWithCases, 489 ) -> "TestSuiteResult": 490 """Add and return the child result (test suite). 491 492 Args: 493 test_suite_with_cases: The test suite with test cases. 494 495 Returns: 496 The test suite's result. 497 """ 498 result = TestSuiteResult(test_suite_with_cases) 499 self.child_results.append(result) 500 return result 501 502 @property 503 def test_suites_with_cases(self) -> list[TestSuiteWithCases]: 504 """The test suites with test cases to be executed in this test run. 505 506 The test suites can only be assigned once. 507 508 Returns: 509 The list of test suites with test cases. If an error occurs between 510 the initialization of :class:`TestRunResult` and assigning test cases to the instance, 511 return an empty list, representing that we don't know what to execute. 512 """ 513 return self._test_suites_with_cases 514 515 @test_suites_with_cases.setter 516 def test_suites_with_cases(self, test_suites_with_cases: list[TestSuiteWithCases]) -> None: 517 if self._test_suites_with_cases: 518 raise ValueError( 519 "Attempted to assign test suites to a test run result " 520 "which already has test suites." 521 ) 522 self._test_suites_with_cases = test_suites_with_cases 523 524 @property 525 def ports(self) -> list[Port]: 526 """Get the list of ports associated with this test run.""" 527 return self._ports 528 529 @ports.setter 530 def ports(self, ports: list[Port]) -> None: 531 """Set the list of ports associated with this test run. 532 533 Args: 534 ports: The list of ports to associate with this test run. 535 536 Raises: 537 ValueError: If the ports have already been assigned to this test run. 538 """ 539 if self._ports: 540 raise ValueError( 541 "Attempted to assign `ports` to a test run result which already has `ports`." 542 ) 543 self._ports = ports 544 545 @property 546 def sut_info(self) -> OSSessionInfo | None: 547 """Get the SUT OS session information associated with this test run.""" 548 return self._sut_info 549 550 @sut_info.setter 551 def sut_info(self, sut_info: OSSessionInfo) -> None: 552 """Set the SUT node information associated with this test run. 553 554 Args: 555 sut_info: The SUT node information to associate with this test run. 556 557 Raises: 558 ValueError: If the SUT information has already been assigned to this test run. 559 """ 560 if self._sut_info: 561 raise ValueError( 562 "Attempted to assign `sut_info` to a test run result which already has `sut_info`." 563 ) 564 self._sut_info = sut_info 565 566 @property 567 def dpdk_build_info(self) -> DPDKBuildInfo | None: 568 """Get the DPDK build information associated with this test run.""" 569 return self._dpdk_build_info 570 571 @dpdk_build_info.setter 572 def dpdk_build_info(self, dpdk_build_info: DPDKBuildInfo) -> None: 573 """Set the DPDK build information associated with this test run. 574 575 Args: 576 dpdk_build_info: The DPDK build information to associate with this test run. 577 578 Raises: 579 ValueError: If the DPDK build information has already been assigned to this test run. 580 """ 581 if self._dpdk_build_info: 582 raise ValueError( 583 "Attempted to assign `dpdk_build_info` to a test run result which already " 584 "has `dpdk_build_info`." 585 ) 586 self._dpdk_build_info = dpdk_build_info 587 588 def to_dict(self) -> TestRunResultDict: 589 """Convert the test run result into a dictionary. 590 591 The dictionary contains test suites in this test run, and a summary of the test run and 592 information about the DPDK version, compiler version and associated ports. 593 594 Returns: 595 TestRunResultDict: A dictionary representation of the test run result. 596 """ 597 results = {result.name: 0 for result in Result} 598 self.add_result(results) 599 600 compiler_version = None 601 dpdk_version = None 602 603 if self.dpdk_build_info: 604 compiler_version = self.dpdk_build_info.compiler_version 605 dpdk_version = self.dpdk_build_info.dpdk_version 606 607 return { 608 "compiler_version": compiler_version, 609 "dpdk_version": dpdk_version, 610 "ports": [asdict(port) for port in self.ports], 611 "test_suites": [child.to_dict() for child in self.child_results], 612 "summary": results | self.generate_pass_rate_dict(results), 613 } 614 615 def _mark_results(self, result) -> None: 616 """Mark the test suite results as `result`.""" 617 for test_suite_with_cases in self._test_suites_with_cases: 618 child_result = self.add_test_suite(test_suite_with_cases) 619 child_result.update_setup(result) 620 621 622class TestSuiteResult(BaseResult): 623 """The test suite specific result. 624 625 The internal list stores the results of all test cases in a given test suite. 626 627 Attributes: 628 test_suite_name: The test suite name. 629 """ 630 631 test_suite_name: str 632 _test_suite_with_cases: TestSuiteWithCases 633 _child_configs: list[str] 634 635 def __init__(self, test_suite_with_cases: TestSuiteWithCases): 636 """Extend the constructor with test suite's config. 637 638 Args: 639 test_suite_with_cases: The test suite with test cases. 640 """ 641 super().__init__() 642 self.test_suite_name = test_suite_with_cases.test_suite_class.__name__ 643 self._test_suite_with_cases = test_suite_with_cases 644 645 def add_test_case(self, test_case_name: str) -> "TestCaseResult": 646 """Add and return the child result (test case). 647 648 Args: 649 test_case_name: The name of the test case. 650 651 Returns: 652 The test case's result. 653 """ 654 result = TestCaseResult(test_case_name) 655 self.child_results.append(result) 656 return result 657 658 def to_dict(self) -> TestSuiteResultDict: 659 """Convert the test suite result into a dictionary. 660 661 The dictionary contains a test suite name and test cases given in this test suite. 662 """ 663 return { 664 "test_suite_name": self.test_suite_name, 665 "test_cases": [child.to_dict() for child in self.child_results], 666 } 667 668 def _mark_results(self, result) -> None: 669 """Mark the test case results as `result`.""" 670 for test_case_method in self._test_suite_with_cases.test_cases: 671 child_result = self.add_test_case(test_case_method.__name__) 672 child_result.update_setup(result) 673 674 675class TestCaseResult(BaseResult, FixtureResult): 676 r"""The test case specific result. 677 678 Stores the result of the actual test case. This is done by adding an extra superclass 679 in :class:`FixtureResult`. The setup and teardown results are :class:`FixtureResult`\s and 680 the class is itself a record of the test case. 681 682 Attributes: 683 test_case_name: The test case name. 684 """ 685 686 test_case_name: str 687 688 def __init__(self, test_case_name: str): 689 """Extend the constructor with test case's name. 690 691 Args: 692 test_case_name: The test case's name. 693 """ 694 super().__init__() 695 self.test_case_name = test_case_name 696 697 def update(self, result: Result, error: Exception | None = None) -> None: 698 """Update the test case result. 699 700 This updates the result of the test case itself and doesn't affect 701 the results of the setup and teardown steps in any way. 702 703 Args: 704 result: The result of the test case. 705 error: The error that occurred in case of a failure. 706 """ 707 self.result = result 708 self.error = error 709 710 def _get_child_errors(self) -> list[Exception]: 711 if self.error: 712 return [self.error] 713 return [] 714 715 def to_dict(self) -> TestCaseResultDict: 716 """Convert the test case result into a dictionary. 717 718 The dictionary contains a test case name and the result name. 719 """ 720 return {"test_case_name": self.test_case_name, "result": self.result.name} 721 722 def add_result(self, results: dict[str, int]): 723 r"""Add the test case result to the results. 724 725 The base method goes through the hierarchy recursively and this method is here to stop 726 the recursion, as the :class:`TestCaseResult` are the leaves of the hierarchy tree. 727 728 Args: 729 results: The dictionary to which results will be collated. 730 """ 731 results[self.result.name] += 1 732 733 def _mark_results(self, result) -> None: 734 r"""Mark the result as `result`.""" 735 self.update(result) 736 737 def __bool__(self) -> bool: 738 """The test case passed only if setup, teardown and the test case itself passed.""" 739 return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result) 740 741 742class TextSummary: 743 """Generates and saves textual summaries of DTS run results. 744 745 The summary includes: 746 * Results of test cases, 747 * Compiler version of the DPDK build, 748 * DPDK version of the DPDK source tree, 749 * Overall summary of results when multiple test runs are present. 750 """ 751 752 _dict_result: DtsRunResultDict 753 _summary: dict[str, int | float] 754 _text: str 755 756 def __init__(self, dts_run_result: DTSResult): 757 """Initializes with a DTSResult object and converts it to a dictionary format. 758 759 Args: 760 dts_run_result: The DTS result. 761 """ 762 self._dict_result = dts_run_result.to_dict() 763 self._summary = self._dict_result["summary"] 764 self._text = "" 765 766 @property 767 def _outdent(self) -> str: 768 """Appropriate indentation based on multiple test run results.""" 769 return "\t" if len(self._dict_result["test_runs"]) > 1 else "" 770 771 def save(self, output_path: Path): 772 """Generate and save text statistics to a file. 773 774 Args: 775 output_path: The path where the text file will be saved. 776 """ 777 if self._dict_result["test_runs"]: 778 with open(f"{output_path}", "w") as fp: 779 self._add_test_runs_dict_decorator(self._add_test_run_dict) 780 fp.write(self._text) 781 782 def _add_test_runs_dict_decorator(self, func: Callable): 783 """Handles multiple test runs and appends results to the summary. 784 785 Adds headers for each test run and overall result when multiple 786 test runs are provided. 787 788 Args: 789 func: Function to process and add results from each test run. 790 """ 791 if len(self._dict_result["test_runs"]) > 1: 792 for idx, test_run_result in enumerate(self._dict_result["test_runs"]): 793 self._text += f"TEST_RUN_{idx}\n" 794 func(test_run_result) 795 796 self._add_overall_results() 797 else: 798 func(self._dict_result["test_runs"][0]) 799 800 def _add_test_run_dict(self, test_run_dict: TestRunResultDict): 801 """Adds the results and the test run attributes of a single test run to the summary. 802 803 Args: 804 test_run_dict: Dictionary containing the test run results. 805 """ 806 self._add_column( 807 DPDK_VERSION=test_run_dict["dpdk_version"], 808 COMPILER_VERSION=test_run_dict["compiler_version"], 809 **test_run_dict["summary"], 810 ) 811 self._text += "\n" 812 813 def _add_column(self, **rows): 814 """Formats and adds key-value pairs to the summary text. 815 816 Handles cases where values might be None by replacing them with "N/A". 817 818 Args: 819 **rows: Arbitrary key-value pairs representing the result data. 820 """ 821 rows = {k: "N/A" if v is None else v for k, v in rows.items()} 822 max_length = len(max(rows, key=len)) 823 for key, value in rows.items(): 824 self._text += f"{self._outdent}{key:<{max_length}} = {value}\n" 825 826 def _add_overall_results(self): 827 """Add overall summary of test runs.""" 828 self._text += "OVERALL\n" 829 self._add_column(**self._summary) 830 831 832class JsonResults: 833 """Save DTS run result in JSON format.""" 834 835 _dict_result: DtsRunResultDict 836 837 def __init__(self, dts_run_result: DTSResult): 838 """Initializes with a DTSResult object and converts it to a dictionary format. 839 840 Args: 841 dts_run_result: The DTS result. 842 """ 843 self._dict_result = dts_run_result.to_dict() 844 845 def save(self, output_path: Path): 846 """Save the result to a file as JSON. 847 848 Args: 849 output_path: The path where the JSON file will be saved. 850 """ 851 with open(f"{output_path}", "w") as fp: 852 json.dump(self._dict_result, fp, indent=4) 853