1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright(c) 2010-2014 Intel Corporation 3# Copyright(c) 2023 PANTHEON.tech s.r.o. 4# Copyright(c) 2023 University of New Hampshire 5# Copyright(c) 2024 Arm Limited 6 7"""System under test (DPDK + hardware) node. 8 9A system under test (SUT) is the combination of DPDK 10and the hardware we're testing with DPDK (NICs, crypto and other devices). 11An SUT node is where this SUT runs. 12""" 13 14 15import os 16import time 17from dataclasses import dataclass 18from pathlib import Path, PurePath 19 20from framework.config import ( 21 DPDKBuildConfiguration, 22 DPDKBuildOptionsConfiguration, 23 DPDKPrecompiledBuildConfiguration, 24 DPDKUncompiledBuildConfiguration, 25 LocalDPDKTarballLocation, 26 LocalDPDKTreeLocation, 27 RemoteDPDKTarballLocation, 28 RemoteDPDKTreeLocation, 29 SutNodeConfiguration, 30 TestRunConfiguration, 31) 32from framework.exception import ConfigurationError, RemoteFileNotFoundError 33from framework.params.eal import EalParams 34from framework.remote_session.remote_session import CommandResult 35from framework.utils import MesonArgs, TarCompressionFormat 36 37from .node import Node 38from .os_session import OSSession, OSSessionInfo 39from .virtual_device import VirtualDevice 40 41 42@dataclass(slots=True, frozen=True) 43class DPDKBuildInfo: 44 """Various versions and other information about a DPDK build. 45 46 Attributes: 47 dpdk_version: The DPDK version that was built. 48 compiler_version: The version of the compiler used to build DPDK. 49 """ 50 51 dpdk_version: str | None 52 compiler_version: str | None 53 54 55class SutNode(Node): 56 """The system under test node. 57 58 The SUT node extends :class:`Node` with DPDK specific features: 59 60 * Managing DPDK source tree on the remote SUT, 61 * Building the DPDK from source or using a pre-built version, 62 * Gathering of DPDK build info, 63 * The running of DPDK apps, interactively or one-time execution, 64 * DPDK apps cleanup. 65 66 Building DPDK from source uses `build` configuration inside `dpdk_build` of configuration. 67 68 Attributes: 69 config: The SUT node configuration. 70 virtual_devices: The virtual devices used on the node. 71 """ 72 73 config: SutNodeConfiguration 74 virtual_devices: list[VirtualDevice] 75 dpdk_prefix_list: list[str] 76 dpdk_timestamp: str 77 _env_vars: dict 78 _remote_tmp_dir: PurePath 79 __remote_dpdk_tree_path: str | PurePath | None 80 _remote_dpdk_build_dir: PurePath | None 81 _app_compile_timeout: float 82 _dpdk_kill_session: OSSession | None 83 _dpdk_version: str | None 84 _node_info: OSSessionInfo | None 85 _compiler_version: str | None 86 _path_to_devbind_script: PurePath | None 87 _ports_bound_to_dpdk: bool 88 89 def __init__(self, node_config: SutNodeConfiguration): 90 """Extend the constructor with SUT node specifics. 91 92 Args: 93 node_config: The SUT node's test run configuration. 94 """ 95 super().__init__(node_config) 96 self.virtual_devices = [] 97 self.dpdk_prefix_list = [] 98 self._env_vars = {} 99 self._remote_tmp_dir = self.main_session.get_remote_tmp_dir() 100 self.__remote_dpdk_tree_path = None 101 self._remote_dpdk_build_dir = None 102 self._app_compile_timeout = 90 103 self._dpdk_kill_session = None 104 self.dpdk_timestamp = ( 105 f"{str(os.getpid())}_{time.strftime('%Y%m%d%H%M%S', time.localtime())}" 106 ) 107 self._dpdk_version = None 108 self._node_info = None 109 self._compiler_version = None 110 self._path_to_devbind_script = None 111 self._ports_bound_to_dpdk = False 112 self._logger.info(f"Created node: {self.name}") 113 114 @property 115 def _remote_dpdk_tree_path(self) -> str | PurePath: 116 """The remote DPDK tree path.""" 117 if self.__remote_dpdk_tree_path: 118 return self.__remote_dpdk_tree_path 119 120 self._logger.warning( 121 "Failed to get remote dpdk tree path because we don't know the " 122 "location on the SUT node." 123 ) 124 return "" 125 126 @property 127 def remote_dpdk_build_dir(self) -> str | PurePath: 128 """The remote DPDK build dir path.""" 129 if self._remote_dpdk_build_dir: 130 return self._remote_dpdk_build_dir 131 132 self._logger.warning( 133 "Failed to get remote dpdk build dir because we don't know the " 134 "location on the SUT node." 135 ) 136 return "" 137 138 @property 139 def dpdk_version(self) -> str | None: 140 """Last built DPDK version.""" 141 if self._dpdk_version is None: 142 self._dpdk_version = self.main_session.get_dpdk_version(self._remote_dpdk_tree_path) 143 return self._dpdk_version 144 145 @property 146 def node_info(self) -> OSSessionInfo: 147 """Additional node information.""" 148 if self._node_info is None: 149 self._node_info = self.main_session.get_node_info() 150 return self._node_info 151 152 @property 153 def compiler_version(self) -> str | None: 154 """The node's compiler version.""" 155 if self._compiler_version is None: 156 self._logger.warning("The `compiler_version` is None because a pre-built DPDK is used.") 157 158 return self._compiler_version 159 160 @compiler_version.setter 161 def compiler_version(self, value: str) -> None: 162 """Set the `compiler_version` used on the SUT node. 163 164 Args: 165 value: The node's compiler version. 166 """ 167 self._compiler_version = value 168 169 @property 170 def path_to_devbind_script(self) -> PurePath | str: 171 """The path to the dpdk-devbind.py script on the node.""" 172 if self._path_to_devbind_script is None: 173 self._path_to_devbind_script = self.main_session.join_remote_path( 174 self._remote_dpdk_tree_path, "usertools", "dpdk-devbind.py" 175 ) 176 return self._path_to_devbind_script 177 178 def get_dpdk_build_info(self) -> DPDKBuildInfo: 179 """Get additional DPDK build information. 180 181 Returns: 182 The DPDK build information, 183 """ 184 return DPDKBuildInfo(dpdk_version=self.dpdk_version, compiler_version=self.compiler_version) 185 186 def set_up_test_run( 187 self, 188 test_run_config: TestRunConfiguration, 189 dpdk_build_config: DPDKBuildConfiguration, 190 ) -> None: 191 """Extend the test run setup with vdev config and DPDK build set up. 192 193 This method extends the setup process by configuring virtual devices and preparing the DPDK 194 environment based on the provided configuration. 195 196 Args: 197 test_run_config: A test run configuration according to which 198 the setup steps will be taken. 199 dpdk_build_config: The build configuration of DPDK. 200 """ 201 super().set_up_test_run(test_run_config, dpdk_build_config) 202 for vdev in test_run_config.system_under_test_node.vdevs: 203 self.virtual_devices.append(VirtualDevice(vdev)) 204 self._set_up_dpdk(dpdk_build_config) 205 206 def tear_down_test_run(self) -> None: 207 """Extend the test run teardown with virtual device teardown and DPDK teardown.""" 208 super().tear_down_test_run() 209 self.virtual_devices = [] 210 self._tear_down_dpdk() 211 212 def _set_up_dpdk( 213 self, 214 dpdk_build_config: DPDKBuildConfiguration, 215 ) -> None: 216 """Set up DPDK the SUT node and bind ports. 217 218 DPDK setup includes setting all internals needed for the build, the copying of DPDK 219 sources and then building DPDK or using the exist ones from the `dpdk_location`. The drivers 220 are bound to those that DPDK needs. 221 222 Args: 223 dpdk_build_config: A DPDK build configuration to test. 224 """ 225 match dpdk_build_config.dpdk_location: 226 case RemoteDPDKTreeLocation(dpdk_tree=dpdk_tree): 227 self._set_remote_dpdk_tree_path(dpdk_tree) 228 case LocalDPDKTreeLocation(dpdk_tree=dpdk_tree): 229 self._copy_dpdk_tree(dpdk_tree) 230 case RemoteDPDKTarballLocation(tarball=tarball): 231 self._validate_remote_dpdk_tarball(tarball) 232 self._prepare_and_extract_dpdk_tarball(tarball) 233 case LocalDPDKTarballLocation(tarball=tarball): 234 remote_tarball = self._copy_dpdk_tarball_to_remote(tarball) 235 self._prepare_and_extract_dpdk_tarball(remote_tarball) 236 237 match dpdk_build_config: 238 case DPDKPrecompiledBuildConfiguration(precompiled_build_dir=build_dir): 239 self._set_remote_dpdk_build_dir(build_dir) 240 case DPDKUncompiledBuildConfiguration(build_options=build_options): 241 self._configure_dpdk_build(build_options) 242 self._build_dpdk() 243 244 self.bind_ports_to_driver() 245 246 def _tear_down_dpdk(self) -> None: 247 """Reset DPDK variables and bind port driver to the OS driver.""" 248 self._env_vars = {} 249 self.__remote_dpdk_tree_path = None 250 self._remote_dpdk_build_dir = None 251 self._dpdk_version = None 252 self.compiler_version = None 253 self.bind_ports_to_driver(for_dpdk=False) 254 255 def _set_remote_dpdk_tree_path(self, dpdk_tree: PurePath): 256 """Set the path to the remote DPDK source tree based on the provided DPDK location. 257 258 Verify DPDK source tree existence on the SUT node, if exists sets the 259 `_remote_dpdk_tree_path` property, otherwise sets nothing. 260 261 Args: 262 dpdk_tree: The path to the DPDK source tree directory. 263 264 Raises: 265 RemoteFileNotFoundError: If the DPDK source tree is expected to be on the SUT node but 266 is not found. 267 """ 268 if not self.main_session.remote_path_exists(dpdk_tree): 269 raise RemoteFileNotFoundError( 270 f"Remote DPDK source tree '{dpdk_tree}' not found in SUT node." 271 ) 272 if not self.main_session.is_remote_dir(dpdk_tree): 273 raise ConfigurationError(f"Remote DPDK source tree '{dpdk_tree}' must be a directory.") 274 275 self.__remote_dpdk_tree_path = dpdk_tree 276 277 def _copy_dpdk_tree(self, dpdk_tree_path: Path) -> None: 278 """Copy the DPDK source tree to the SUT. 279 280 Args: 281 dpdk_tree_path: The path to DPDK source tree on local filesystem. 282 """ 283 self._logger.info( 284 f"Copying DPDK source tree to SUT: '{dpdk_tree_path}' into '{self._remote_tmp_dir}'." 285 ) 286 self.main_session.copy_dir_to( 287 dpdk_tree_path, 288 self._remote_tmp_dir, 289 exclude=[".git", "*.o"], 290 compress_format=TarCompressionFormat.gzip, 291 ) 292 293 self.__remote_dpdk_tree_path = self.main_session.join_remote_path( 294 self._remote_tmp_dir, PurePath(dpdk_tree_path).name 295 ) 296 297 def _validate_remote_dpdk_tarball(self, dpdk_tarball: PurePath) -> None: 298 """Validate the DPDK tarball on the SUT node. 299 300 Args: 301 dpdk_tarball: The path to the DPDK tarball on the SUT node. 302 303 Raises: 304 RemoteFileNotFoundError: If the `dpdk_tarball` is expected to be on the SUT node but is 305 not found. 306 ConfigurationError: If the `dpdk_tarball` is a valid path but not a valid tar archive. 307 """ 308 if not self.main_session.remote_path_exists(dpdk_tarball): 309 raise RemoteFileNotFoundError(f"Remote DPDK tarball '{dpdk_tarball}' not found in SUT.") 310 if not self.main_session.is_remote_tarfile(dpdk_tarball): 311 raise ConfigurationError(f"Remote DPDK tarball '{dpdk_tarball}' must be a tar archive.") 312 313 def _copy_dpdk_tarball_to_remote(self, dpdk_tarball: Path) -> PurePath: 314 """Copy the local DPDK tarball to the SUT node. 315 316 Args: 317 dpdk_tarball: The local path to the DPDK tarball. 318 319 Returns: 320 The path of the copied tarball on the SUT node. 321 """ 322 self._logger.info( 323 f"Copying DPDK tarball to SUT: '{dpdk_tarball}' into '{self._remote_tmp_dir}'." 324 ) 325 self.main_session.copy_to(dpdk_tarball, self._remote_tmp_dir) 326 return self.main_session.join_remote_path(self._remote_tmp_dir, dpdk_tarball.name) 327 328 def _prepare_and_extract_dpdk_tarball(self, remote_tarball_path: PurePath) -> None: 329 """Prepare the remote DPDK tree path and extract the tarball. 330 331 This method extracts the remote tarball and sets the `_remote_dpdk_tree_path` property to 332 the path of the extracted DPDK tree on the SUT node. 333 334 Args: 335 remote_tarball_path: The path to the DPDK tarball on the SUT node. 336 """ 337 338 def remove_tarball_suffix(remote_tarball_path: PurePath) -> PurePath: 339 """Remove the tarball suffix from the path. 340 341 Args: 342 remote_tarball_path: The path to the remote tarball. 343 344 Returns: 345 The path without the tarball suffix. 346 """ 347 if len(remote_tarball_path.suffixes) > 1: 348 if remote_tarball_path.suffixes[-2] == ".tar": 349 suffixes_to_remove = "".join(remote_tarball_path.suffixes[-2:]) 350 return PurePath(str(remote_tarball_path).replace(suffixes_to_remove, "")) 351 return remote_tarball_path.with_suffix("") 352 353 tarball_top_dir = self.main_session.get_tarball_top_dir(remote_tarball_path) 354 self.__remote_dpdk_tree_path = self.main_session.join_remote_path( 355 remote_tarball_path.parent, 356 tarball_top_dir or remove_tarball_suffix(remote_tarball_path), 357 ) 358 359 self._logger.info( 360 "Extracting DPDK tarball on SUT: " 361 f"'{remote_tarball_path}' into '{self._remote_dpdk_tree_path}'." 362 ) 363 self.main_session.extract_remote_tarball( 364 remote_tarball_path, 365 self._remote_dpdk_tree_path, 366 ) 367 368 def _set_remote_dpdk_build_dir(self, build_dir: str): 369 """Set the `remote_dpdk_build_dir` on the SUT. 370 371 Check existence on the SUT node and sets the 372 `remote_dpdk_build_dir` property by joining the `_remote_dpdk_tree_path` and `build_dir`. 373 Otherwise, sets nothing. 374 375 Args: 376 build_dir: DPDK has been pre-built and the build directory is located 377 in a subdirectory of `dpdk_tree` or `tarball` root directory. 378 379 Raises: 380 RemoteFileNotFoundError: If the `build_dir` is expected but does not exist on the SUT 381 node. 382 """ 383 remote_dpdk_build_dir = self.main_session.join_remote_path( 384 self._remote_dpdk_tree_path, build_dir 385 ) 386 if not self.main_session.remote_path_exists(remote_dpdk_build_dir): 387 raise RemoteFileNotFoundError( 388 f"Remote DPDK build dir '{remote_dpdk_build_dir}' not found in SUT node." 389 ) 390 391 self._remote_dpdk_build_dir = PurePath(remote_dpdk_build_dir) 392 393 def _configure_dpdk_build(self, dpdk_build_config: DPDKBuildOptionsConfiguration) -> None: 394 """Populate common environment variables and set the DPDK build related properties. 395 396 This method sets `compiler_version` for additional information and `remote_dpdk_build_dir` 397 from DPDK build config name. 398 399 Args: 400 dpdk_build_config: A DPDK build configuration to test. 401 """ 402 self._env_vars = {} 403 self._env_vars.update(self.main_session.get_dpdk_build_env_vars(dpdk_build_config.arch)) 404 if compiler_wrapper := dpdk_build_config.compiler_wrapper: 405 self._env_vars["CC"] = f"'{compiler_wrapper} {dpdk_build_config.compiler.name}'" 406 else: 407 self._env_vars["CC"] = dpdk_build_config.compiler.name 408 409 self.compiler_version = self.main_session.get_compiler_version( 410 dpdk_build_config.compiler.name 411 ) 412 413 self._remote_dpdk_build_dir = self.main_session.join_remote_path( 414 self._remote_dpdk_tree_path, dpdk_build_config.name 415 ) 416 417 def _build_dpdk(self) -> None: 418 """Build DPDK. 419 420 Uses the already configured DPDK build configuration. Assumes that the 421 `_remote_dpdk_tree_path` has already been set on the SUT node. 422 """ 423 self.main_session.build_dpdk( 424 self._env_vars, 425 MesonArgs(default_library="static", enable_kmods=True, libdir="lib"), 426 self._remote_dpdk_tree_path, 427 self.remote_dpdk_build_dir, 428 ) 429 430 def build_dpdk_app(self, app_name: str, **meson_dpdk_args: str | bool) -> PurePath: 431 """Build one or all DPDK apps. 432 433 Requires DPDK to be already built on the SUT node. 434 435 Args: 436 app_name: The name of the DPDK app to build. 437 When `app_name` is ``all``, build all example apps. 438 meson_dpdk_args: The arguments found in ``meson_options.txt`` in root DPDK directory. 439 Do not use ``-D`` with them. 440 441 Returns: 442 The directory path of the built app. If building all apps, return 443 the path to the examples directory (where all apps reside). 444 """ 445 self.main_session.build_dpdk( 446 self._env_vars, 447 MesonArgs(examples=app_name, **meson_dpdk_args), # type: ignore [arg-type] 448 # ^^ https://github.com/python/mypy/issues/11583 449 self._remote_dpdk_tree_path, 450 self.remote_dpdk_build_dir, 451 rebuild=True, 452 timeout=self._app_compile_timeout, 453 ) 454 455 if app_name == "all": 456 return self.main_session.join_remote_path(self.remote_dpdk_build_dir, "examples") 457 return self.main_session.join_remote_path( 458 self.remote_dpdk_build_dir, "examples", f"dpdk-{app_name}" 459 ) 460 461 def kill_cleanup_dpdk_apps(self) -> None: 462 """Kill all dpdk applications on the SUT, then clean up hugepages.""" 463 if self._dpdk_kill_session and self._dpdk_kill_session.is_alive(): 464 # we can use the session if it exists and responds 465 self._dpdk_kill_session.kill_cleanup_dpdk_apps(self.dpdk_prefix_list) 466 else: 467 # otherwise, we need to (re)create it 468 self._dpdk_kill_session = self.create_session("dpdk_kill") 469 self.dpdk_prefix_list = [] 470 471 def run_dpdk_app( 472 self, app_path: PurePath, eal_params: EalParams, timeout: float = 30 473 ) -> CommandResult: 474 """Run DPDK application on the remote node. 475 476 The application is not run interactively - the command that starts the application 477 is executed and then the call waits for it to finish execution. 478 479 Args: 480 app_path: The remote path to the DPDK application. 481 eal_params: EAL parameters to run the DPDK application with. 482 timeout: Wait at most this long in seconds for `command` execution to complete. 483 484 Returns: 485 The result of the DPDK app execution. 486 """ 487 return self.main_session.send_command( 488 f"{app_path} {eal_params}", timeout, privileged=True, verify=True 489 ) 490 491 def bind_ports_to_driver(self, for_dpdk: bool = True) -> None: 492 """Bind all ports on the SUT to a driver. 493 494 Args: 495 for_dpdk: If :data:`True`, binds ports to os_driver_for_dpdk. 496 If :data:`False`, binds to os_driver. 497 """ 498 if self._ports_bound_to_dpdk == for_dpdk: 499 return 500 501 for port in self.ports: 502 driver = port.os_driver_for_dpdk if for_dpdk else port.os_driver 503 self.main_session.send_command( 504 f"{self.path_to_devbind_script} -b {driver} --force {port.pci}", 505 privileged=True, 506 verify=True, 507 ) 508 self._ports_bound_to_dpdk = for_dpdk 509