1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright(c) 2010-2021 Intel Corporation 3# Copyright(c) 2022-2023 PANTHEON.tech s.r.o. 4# Copyright(c) 2022 University of New Hampshire 5# Copyright(c) 2024 Arm Limited 6 7"""Environment variables and command line arguments parsing. 8 9This is a simple module utilizing the built-in argparse module to parse command line arguments, 10augment them with values from environment variables and make them available across the framework. 11 12The command line value takes precedence, followed by the environment variable value, 13followed by the default value defined in this module. 14 15The command line arguments along with the supported environment variables are: 16 17.. option:: --config-file 18.. envvar:: DTS_CFG_FILE 19 20 The path to the YAML test run configuration file. 21 22.. option:: --output-dir, --output 23.. envvar:: DTS_OUTPUT_DIR 24 25 The directory where DTS logs and results are saved. 26 27.. option:: --compile-timeout 28.. envvar:: DTS_COMPILE_TIMEOUT 29 30 The timeout for compiling DPDK. 31 32.. option:: -t, --timeout 33.. envvar:: DTS_TIMEOUT 34 35 The timeout for all DTS operation except for compiling DPDK. 36 37.. option:: -v, --verbose 38.. envvar:: DTS_VERBOSE 39 40 Set to any value to enable logging everything to the console. 41 42.. option:: --dpdk-tree 43.. envvar:: DTS_DPDK_TREE 44 45 The path to the DPDK source tree directory to test. Cannot be used in conjunction with 46 --tarball. 47 48.. option:: --tarball, --snapshot 49.. envvar:: DTS_DPDK_TARBALL 50 51 The path to the DPDK source tarball to test. DPDK must be contained in a folder with the same 52 name as the tarball file. Cannot be used in conjunction with --dpdk-tree. 53 54.. option:: --remote-source 55.. envvar:: DTS_REMOTE_SOURCE 56 57 Set this option if either the DPDK source tree or tarball to be used are located on the SUT 58 node. Can only be used with --dpdk-tree or --tarball. 59 60.. option:: --precompiled-build-dir 61.. envvar:: DTS_PRECOMPILED_BUILD_DIR 62 63 Define the subdirectory under the DPDK tree root directory or tarball where the pre-compiled 64 binaries are located. 65 66.. option:: --test-suite 67.. envvar:: DTS_TEST_SUITES 68 69 A test suite with test cases which may be specified multiple times. 70 In the environment variable, the suites are joined with a comma. 71 72.. option:: --re-run, --re_run 73.. envvar:: DTS_RERUN 74 75 Re-run each test case this many times in case of a failure. 76 77.. option:: --random-seed 78.. envvar:: DTS_RANDOM_SEED 79 80 The seed to use with the pseudo-random generator. If not specified, the configuration value is 81 used instead. If that's also not specified, a random seed is generated. 82 83The module provides one key module-level variable: 84 85Attributes: 86 SETTINGS: The module level variable storing framework-wide DTS settings. 87 88Typical usage example:: 89 90 from framework.settings import SETTINGS 91 foo = SETTINGS.foo 92""" 93 94import argparse 95import os 96import sys 97from argparse import Action, ArgumentDefaultsHelpFormatter, _get_action_name 98from dataclasses import dataclass, field 99from pathlib import Path 100from typing import Callable 101 102from pydantic import ValidationError 103 104from .config import ( 105 DPDKLocation, 106 LocalDPDKTarballLocation, 107 LocalDPDKTreeLocation, 108 RemoteDPDKTarballLocation, 109 RemoteDPDKTreeLocation, 110 TestSuiteConfig, 111) 112 113 114@dataclass(slots=True) 115class Settings: 116 """Default framework-wide user settings. 117 118 The defaults may be modified at the start of the run. 119 """ 120 121 #: 122 config_file_path: Path = Path(__file__).parent.parent.joinpath("conf.yaml") 123 #: 124 output_dir: str = "output" 125 #: 126 timeout: float = 15 127 #: 128 verbose: bool = False 129 #: 130 dpdk_location: DPDKLocation | None = None 131 #: 132 precompiled_build_dir: str | None = None 133 #: 134 compile_timeout: float = 1200 135 #: 136 test_suites: list[TestSuiteConfig] = field(default_factory=list) 137 #: 138 re_run: int = 0 139 #: 140 random_seed: int | None = None 141 142 143SETTINGS: Settings = Settings() 144 145 146#: Attribute name representing the env variable name to augment :class:`~argparse.Action` with. 147_ENV_VAR_NAME_ATTR = "env_var_name" 148#: Attribute name representing the value origin to augment :class:`~argparse.Action` with. 149_IS_FROM_ENV_ATTR = "is_from_env" 150 151#: The prefix to be added to all of the environment variables. 152_ENV_PREFIX = "DTS_" 153 154 155def _make_env_var_name(action: Action, env_var_name: str | None) -> str: 156 """Make and assign an environment variable name to the given action.""" 157 env_var_name = f"{_ENV_PREFIX}{env_var_name or action.dest.upper()}" 158 setattr(action, _ENV_VAR_NAME_ATTR, env_var_name) 159 return env_var_name 160 161 162def _get_env_var_name(action: Action) -> str | None: 163 """Get the environment variable name of the given action.""" 164 return getattr(action, _ENV_VAR_NAME_ATTR, None) 165 166 167def _set_is_from_env(action: Action) -> None: 168 """Make the environment the given action's value origin.""" 169 setattr(action, _IS_FROM_ENV_ATTR, True) 170 171 172def _is_from_env(action: Action) -> bool: 173 """Check if the given action's value originated from the environment.""" 174 return getattr(action, _IS_FROM_ENV_ATTR, False) 175 176 177def _is_action_in_args(action: Action) -> bool: 178 """Check if the action is invoked in the command line arguments.""" 179 for option in action.option_strings: 180 if option in sys.argv: 181 return True 182 return False 183 184 185def _add_env_var_to_action( 186 action: Action, 187 env_var_name: str | None = None, 188) -> None: 189 """Add an argument with an environment variable to the parser.""" 190 env_var_name = _make_env_var_name(action, env_var_name) 191 192 if not _is_action_in_args(action): 193 env_var_value = os.environ.get(env_var_name) 194 if env_var_value is not None: 195 _set_is_from_env(action) 196 sys.argv[1:0] = [action.format_usage(), env_var_value] 197 198 199class _DTSArgumentParser(argparse.ArgumentParser): 200 """ArgumentParser with a custom error message. 201 202 This custom version of ArgumentParser changes the error message to accurately reflect the origin 203 of the value of its arguments. If it was supplied through the command line nothing changes, but 204 if it was supplied as an environment variable this is correctly communicated. 205 """ 206 207 def find_action( 208 self, action_dest: str, filter_fn: Callable[[Action], bool] | None = None 209 ) -> Action | None: 210 """Find and return an action by its destination variable name. 211 212 Arguments: 213 action_dest: the destination variable name of the action to find. 214 filter_fn: if an action is found it is passed to this filter function, which must 215 return a boolean value. 216 """ 217 it = (action for action in self._actions if action.dest == action_dest) 218 action = next(it, None) 219 220 if action and filter_fn: 221 return action if filter_fn(action) else None 222 223 return action 224 225 def error(self, message): 226 """Augments :meth:`~argparse.ArgumentParser.error` with environment variable awareness.""" 227 for action in self._actions: 228 if _is_from_env(action): 229 action_name = _get_action_name(action) 230 env_var_name = _get_env_var_name(action) 231 env_var_value = os.environ.get(env_var_name) 232 233 message = message.replace( 234 f"argument {action_name}", 235 f"environment variable {env_var_name} (value: {env_var_value})", 236 ) 237 238 print(f"{self.prog}: error: {message}\n", file=sys.stderr) 239 self.exit(2, "For help and usage, " "run the command with the --help flag.\n") 240 241 242class _EnvVarHelpFormatter(ArgumentDefaultsHelpFormatter): 243 """Custom formatter to add environment variables to the help page.""" 244 245 def _get_help_string(self, action): 246 """Overrides :meth:`ArgumentDefaultsHelpFormatter._get_help_string`.""" 247 help = super()._get_help_string(action) 248 249 env_var_name = _get_env_var_name(action) 250 if env_var_name is not None: 251 help = f"[{env_var_name}] {help}" 252 253 env_var_value = os.environ.get(env_var_name) 254 if env_var_value is not None: 255 help = f"{help} (env value: {env_var_value})" 256 257 return help 258 259 260def _required_with_one_of(parser: _DTSArgumentParser, action: Action, *required_dests: str) -> None: 261 """Verify that `action` is listed together with at least one of `required_dests`. 262 263 Verify that when `action` is among the command-line arguments or 264 environment variables, at least one of `required_dests` is also among 265 the command-line arguments or environment variables. 266 267 Args: 268 parser: The custom ArgumentParser object which contains `action`. 269 action: The action to be verified. 270 *required_dests: Destination variable names of the required arguments. 271 272 Raises: 273 argparse.ArgumentTypeError: When none of the required_dest are defined. 274 275 Example: 276 We have ``--option1`` and we only want it to be a passed alongside 277 either ``--option2`` or ``--option3`` (meaning if ``--option1`` is 278 passed without either ``--option2`` or ``--option3``, that's an error). 279 280 parser = _DTSArgumentParser() 281 option1_arg = parser.add_argument('--option1', dest='option1', action='store_true') 282 option2_arg = parser.add_argument('--option2', dest='option2', action='store_true') 283 option2_arg = parser.add_argument('--option3', dest='option3', action='store_true') 284 285 _required_with_one_of(parser, option1_arg, 'option2', 'option3') 286 """ 287 if _is_action_in_args(action): 288 for required_dest in required_dests: 289 required_action = parser.find_action(required_dest) 290 if required_action is None: 291 continue 292 293 if _is_action_in_args(required_action): 294 return None 295 296 raise argparse.ArgumentTypeError( 297 f"The '{action.dest}' is required at least with one of '{', '.join(required_dests)}'." 298 ) 299 300 301def _get_parser() -> _DTSArgumentParser: 302 """Create the argument parser for DTS. 303 304 Command line options take precedence over environment variables, which in turn take precedence 305 over default values. 306 307 Returns: 308 _DTSArgumentParser: The configured argument parser with defined options. 309 """ 310 parser = _DTSArgumentParser( 311 description="Run DPDK test suites. All options may be specified with the environment " 312 "variables provided in brackets. Command line arguments have higher priority.", 313 formatter_class=_EnvVarHelpFormatter, 314 allow_abbrev=False, 315 ) 316 317 action = parser.add_argument( 318 "--config-file", 319 default=SETTINGS.config_file_path, 320 type=Path, 321 help="The configuration file that describes the test cases, SUTs and DPDK build configs.", 322 metavar="FILE_PATH", 323 dest="config_file_path", 324 ) 325 _add_env_var_to_action(action, "CFG_FILE") 326 327 action = parser.add_argument( 328 "--output-dir", 329 "--output", 330 default=SETTINGS.output_dir, 331 help="Output directory where DTS logs and results are saved.", 332 metavar="DIR_PATH", 333 ) 334 _add_env_var_to_action(action) 335 336 action = parser.add_argument( 337 "-t", 338 "--timeout", 339 default=SETTINGS.timeout, 340 type=float, 341 help="The default timeout for all DTS operations except for compiling DPDK.", 342 metavar="SECONDS", 343 ) 344 _add_env_var_to_action(action) 345 346 action = parser.add_argument( 347 "-v", 348 "--verbose", 349 action="store_true", 350 default=SETTINGS.verbose, 351 help="Specify to enable verbose output, logging all messages to the console.", 352 ) 353 _add_env_var_to_action(action) 354 355 dpdk_build = parser.add_argument_group( 356 "DPDK Build Options", 357 description="Arguments in this group (and subgroup) will be applied to a " 358 "DPDKLocation when the DPDK tree, tarball or revision will be provided, " 359 "other arguments like remote source and build dir are optional. A DPDKLocation " 360 "from settings are used instead of from config if construct successful.", 361 ) 362 363 dpdk_source = dpdk_build.add_mutually_exclusive_group() 364 action = dpdk_source.add_argument( 365 "--dpdk-tree", 366 help="The path to the DPDK source tree directory to test. Cannot be used in conjunction " 367 "with --tarball.", 368 metavar="DIR_PATH", 369 dest="dpdk_tree_path", 370 ) 371 _add_env_var_to_action(action, "DPDK_TREE") 372 373 action = dpdk_source.add_argument( 374 "--tarball", 375 "--snapshot", 376 help="The path to the DPDK source tarball to test. DPDK must be contained in a folder with " 377 "the same name as the tarball file. Cannot be used in conjunction with --dpdk-tree.", 378 metavar="FILE_PATH", 379 dest="dpdk_tarball_path", 380 ) 381 _add_env_var_to_action(action, "DPDK_TARBALL") 382 383 action = dpdk_build.add_argument( 384 "--remote-source", 385 action="store_true", 386 default=False, 387 help="Set this option if either the DPDK source tree or tarball to be used are located on " 388 "the SUT node. Can only be used with --dpdk-tree or --tarball.", 389 ) 390 _add_env_var_to_action(action) 391 _required_with_one_of(parser, action, "dpdk_tarball_path", "dpdk_tree_path") 392 393 action = dpdk_build.add_argument( 394 "--precompiled-build-dir", 395 help="Define the subdirectory under the DPDK tree root directory or tarball where the " 396 "pre-compiled binaries are located.", 397 metavar="DIR_NAME", 398 ) 399 _add_env_var_to_action(action) 400 401 action = parser.add_argument( 402 "--compile-timeout", 403 default=SETTINGS.compile_timeout, 404 type=float, 405 help="The timeout for compiling DPDK.", 406 metavar="SECONDS", 407 ) 408 _add_env_var_to_action(action) 409 410 action = parser.add_argument( 411 "--test-suite", 412 action="append", 413 nargs="+", 414 metavar=("TEST_SUITE", "TEST_CASES"), 415 default=SETTINGS.test_suites, 416 help="A list containing a test suite with test cases. " 417 "The first parameter is the test suite name, and the rest are test case names, " 418 "which are optional. May be specified multiple times. To specify multiple test suites in " 419 "the environment variable, join the lists with a comma. " 420 "Examples: " 421 "--test-suite suite case case --test-suite suite case ... | " 422 "DTS_TEST_SUITES='suite case case, suite case, ...' | " 423 "--test-suite suite --test-suite suite case ... | " 424 "DTS_TEST_SUITES='suite, suite case, ...'", 425 dest="test_suites", 426 ) 427 _add_env_var_to_action(action) 428 429 action = parser.add_argument( 430 "--re-run", 431 "--re_run", 432 default=SETTINGS.re_run, 433 type=int, 434 help="Re-run each test case the specified number of times if a test failure occurs.", 435 metavar="N_TIMES", 436 ) 437 _add_env_var_to_action(action, "RERUN") 438 439 action = parser.add_argument( 440 "--random-seed", 441 type=int, 442 help="The seed to use with the pseudo-random generator. If not specified, the configuration" 443 " value is used instead. If that's also not specified, a random seed is generated.", 444 metavar="NUMBER", 445 ) 446 _add_env_var_to_action(action) 447 448 return parser 449 450 451def _process_dpdk_location( 452 parser: _DTSArgumentParser, 453 dpdk_tree: str | None, 454 tarball: str | None, 455 remote: bool, 456) -> DPDKLocation | None: 457 """Process and validate DPDK build arguments. 458 459 Ensures that either `dpdk_tree` or `tarball` is provided. Validate existence and format of 460 `dpdk_tree` or `tarball` on local filesystem, if `remote` is False. Constructs and returns 461 any valid :class:`DPDKLocation` with the provided parameters if validation is successful. 462 463 Args: 464 dpdk_tree: The path to the DPDK source tree directory. 465 tarball: The path to the DPDK tarball. 466 remote: If :data:`True`, `dpdk_tree` or `tarball` is located on the SUT node, instead of the 467 execution host. 468 469 Returns: 470 A DPDK location if construction is successful, otherwise None. 471 """ 472 if dpdk_tree: 473 action = parser.find_action("dpdk_tree", _is_from_env) 474 475 try: 476 if remote: 477 return RemoteDPDKTreeLocation.model_validate({"dpdk_tree": dpdk_tree}) 478 else: 479 return LocalDPDKTreeLocation.model_validate({"dpdk_tree": dpdk_tree}) 480 except ValidationError as e: 481 print( 482 "An error has occurred while validating the DPDK tree supplied in the " 483 f"{'environment variable' if action else 'arguments'}:", 484 file=sys.stderr, 485 ) 486 print(e, file=sys.stderr) 487 sys.exit(1) 488 489 if tarball: 490 action = parser.find_action("tarball", _is_from_env) 491 492 try: 493 if remote: 494 return RemoteDPDKTarballLocation.model_validate({"tarball": tarball}) 495 else: 496 return LocalDPDKTarballLocation.model_validate({"tarball": tarball}) 497 except ValidationError as e: 498 print( 499 "An error has occurred while validating the DPDK tarball supplied in the " 500 f"{'environment variable' if action else 'arguments'}:", 501 file=sys.stderr, 502 ) 503 print(e, file=sys.stderr) 504 sys.exit(1) 505 506 return None 507 508 509def _process_test_suites( 510 parser: _DTSArgumentParser, args: list[list[str]] 511) -> list[TestSuiteConfig]: 512 """Process the given argument to a list of :class:`TestSuiteConfig` to execute. 513 514 Args: 515 args: The arguments to process. The args is a string from an environment variable 516 or a list of from the user input containing tests suites with tests cases, 517 each of which is a list of [test_suite, test_case, test_case, ...]. 518 519 Returns: 520 A list of test suite configurations to execute. 521 """ 522 action = parser.find_action("test_suites", _is_from_env) 523 if action: 524 # Environment variable in the form of "SUITE1 CASE1 CASE2, SUITE2 CASE1, SUITE3, ..." 525 args = [suite_with_cases.split() for suite_with_cases in args[0][0].split(",")] 526 527 try: 528 return [ 529 TestSuiteConfig(test_suite=test_suite, test_cases=test_cases) 530 for [test_suite, *test_cases] in args 531 ] 532 except ValidationError as e: 533 print( 534 "An error has occurred while validating the test suites supplied in the " 535 f"{'environment variable' if action else 'arguments'}:", 536 file=sys.stderr, 537 ) 538 print(e, file=sys.stderr) 539 sys.exit(1) 540 541 542def get_settings() -> Settings: 543 """Create new settings with inputs from the user. 544 545 The inputs are taken from the command line and from environment variables. 546 547 Returns: 548 The new settings object. 549 """ 550 parser = _get_parser() 551 552 args = parser.parse_args() 553 554 args.dpdk_location = _process_dpdk_location( 555 parser, args.dpdk_tree_path, args.dpdk_tarball_path, args.remote_source 556 ) 557 args.test_suites = _process_test_suites(parser, args.test_suites) 558 559 kwargs = {k: v for k, v in vars(args).items() if hasattr(SETTINGS, k)} 560 return Settings(**kwargs) 561