1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright(c) 2010-2014 Intel Corporation 3# Copyright(c) 2023 PANTHEON.tech s.r.o. 4# Copyright(c) 2023 University of New Hampshire 5 6"""System under test (DPDK + hardware) node. 7 8A system under test (SUT) is the combination of DPDK 9and the hardware we're testing with DPDK (NICs, crypto and other devices). 10An SUT node is where this SUT runs. 11""" 12 13 14import os 15import tarfile 16import time 17from pathlib import PurePath 18from typing import Type 19 20from framework.config import ( 21 BuildTargetConfiguration, 22 BuildTargetInfo, 23 NodeInfo, 24 SutNodeConfiguration, 25) 26from framework.remote_session import CommandResult 27from framework.settings import SETTINGS 28from framework.utils import MesonArgs 29 30from .cpu import LogicalCoreCount, LogicalCoreList 31from .node import Node 32from .os_session import InteractiveShellType, OSSession 33from .port import Port 34from .virtual_device import VirtualDevice 35 36 37class EalParameters(object): 38 """The environment abstraction layer parameters. 39 40 The string representation can be created by converting the instance to a string. 41 """ 42 43 def __init__( 44 self, 45 lcore_list: LogicalCoreList, 46 memory_channels: int, 47 prefix: str, 48 no_pci: bool, 49 vdevs: list[VirtualDevice], 50 ports: list[Port], 51 other_eal_param: str, 52 ): 53 """Initialize the parameters according to inputs. 54 55 Process the parameters into the format used on the command line. 56 57 Args: 58 lcore_list: The list of logical cores to use. 59 memory_channels: The number of memory channels to use. 60 prefix: Set the file prefix string with which to start DPDK, e.g.: ``prefix='vf'``. 61 no_pci: Switch to disable PCI bus e.g.: ``no_pci=True``. 62 vdevs: Virtual devices, e.g.:: 63 64 vdevs=[ 65 VirtualDevice('net_ring0'), 66 VirtualDevice('net_ring1') 67 ] 68 ports: The list of ports to allow. 69 other_eal_param: user defined DPDK EAL parameters, e.g.: 70 ``other_eal_param='--single-file-segments'`` 71 """ 72 self._lcore_list = f"-l {lcore_list}" 73 self._memory_channels = f"-n {memory_channels}" 74 self._prefix = prefix 75 if prefix: 76 self._prefix = f"--file-prefix={prefix}" 77 self._no_pci = "--no-pci" if no_pci else "" 78 self._vdevs = " ".join(f"--vdev {vdev}" for vdev in vdevs) 79 self._ports = " ".join(f"-a {port.pci}" for port in ports) 80 self._other_eal_param = other_eal_param 81 82 def __str__(self) -> str: 83 """Create the EAL string.""" 84 return ( 85 f"{self._lcore_list} " 86 f"{self._memory_channels} " 87 f"{self._prefix} " 88 f"{self._no_pci} " 89 f"{self._vdevs} " 90 f"{self._ports} " 91 f"{self._other_eal_param}" 92 ) 93 94 95class SutNode(Node): 96 """The system under test node. 97 98 The SUT node extends :class:`Node` with DPDK specific features: 99 100 * DPDK build, 101 * Gathering of DPDK build info, 102 * The running of DPDK apps, interactively or one-time execution, 103 * DPDK apps cleanup. 104 105 The :option:`--tarball` command line argument and the :envvar:`DTS_DPDK_TARBALL` 106 environment variable configure the path to the DPDK tarball 107 or the git commit ID, tag ID or tree ID to test. 108 109 Attributes: 110 config: The SUT node configuration 111 """ 112 113 config: SutNodeConfiguration 114 _dpdk_prefix_list: list[str] 115 _dpdk_timestamp: str 116 _build_target_config: BuildTargetConfiguration | None 117 _env_vars: dict 118 _remote_tmp_dir: PurePath 119 __remote_dpdk_dir: PurePath | None 120 _app_compile_timeout: float 121 _dpdk_kill_session: OSSession | None 122 _dpdk_version: str | None 123 _node_info: NodeInfo | None 124 _compiler_version: str | None 125 _path_to_devbind_script: PurePath | None 126 127 def __init__(self, node_config: SutNodeConfiguration): 128 """Extend the constructor with SUT node specifics. 129 130 Args: 131 node_config: The SUT node's test run configuration. 132 """ 133 super(SutNode, self).__init__(node_config) 134 self._dpdk_prefix_list = [] 135 self._build_target_config = None 136 self._env_vars = {} 137 self._remote_tmp_dir = self.main_session.get_remote_tmp_dir() 138 self.__remote_dpdk_dir = None 139 self._app_compile_timeout = 90 140 self._dpdk_kill_session = None 141 self._dpdk_timestamp = ( 142 f"{str(os.getpid())}_{time.strftime('%Y%m%d%H%M%S', time.localtime())}" 143 ) 144 self._dpdk_version = None 145 self._node_info = None 146 self._compiler_version = None 147 self._path_to_devbind_script = None 148 self._logger.info(f"Created node: {self.name}") 149 150 @property 151 def _remote_dpdk_dir(self) -> PurePath: 152 """The remote DPDK dir. 153 154 This internal property should be set after extracting the DPDK tarball. If it's not set, 155 that implies the DPDK setup step has been skipped, in which case we can guess where 156 a previous build was located. 157 """ 158 if self.__remote_dpdk_dir is None: 159 self.__remote_dpdk_dir = self._guess_dpdk_remote_dir() 160 return self.__remote_dpdk_dir 161 162 @_remote_dpdk_dir.setter 163 def _remote_dpdk_dir(self, value: PurePath) -> None: 164 self.__remote_dpdk_dir = value 165 166 @property 167 def remote_dpdk_build_dir(self) -> PurePath: 168 """The remote DPDK build directory. 169 170 This is the directory where DPDK was built. 171 We assume it was built in a subdirectory of the extracted tarball. 172 """ 173 if self._build_target_config: 174 return self.main_session.join_remote_path( 175 self._remote_dpdk_dir, self._build_target_config.name 176 ) 177 else: 178 return self.main_session.join_remote_path(self._remote_dpdk_dir, "build") 179 180 @property 181 def dpdk_version(self) -> str: 182 """Last built DPDK version.""" 183 if self._dpdk_version is None: 184 self._dpdk_version = self.main_session.get_dpdk_version(self._remote_dpdk_dir) 185 return self._dpdk_version 186 187 @property 188 def node_info(self) -> NodeInfo: 189 """Additional node information.""" 190 if self._node_info is None: 191 self._node_info = self.main_session.get_node_info() 192 return self._node_info 193 194 @property 195 def compiler_version(self) -> str: 196 """The node's compiler version.""" 197 if self._compiler_version is None: 198 if self._build_target_config is not None: 199 self._compiler_version = self.main_session.get_compiler_version( 200 self._build_target_config.compiler.name 201 ) 202 else: 203 self._logger.warning( 204 "Failed to get compiler version because _build_target_config is None." 205 ) 206 return "" 207 return self._compiler_version 208 209 @property 210 def path_to_devbind_script(self) -> PurePath: 211 """The path to the dpdk-devbind.py script on the node.""" 212 if self._path_to_devbind_script is None: 213 self._path_to_devbind_script = self.main_session.join_remote_path( 214 self._remote_dpdk_dir, "usertools", "dpdk-devbind.py" 215 ) 216 return self._path_to_devbind_script 217 218 def get_build_target_info(self) -> BuildTargetInfo: 219 """Get additional build target information. 220 221 Returns: 222 The build target information, 223 """ 224 return BuildTargetInfo( 225 dpdk_version=self.dpdk_version, compiler_version=self.compiler_version 226 ) 227 228 def _guess_dpdk_remote_dir(self) -> PurePath: 229 return self.main_session.guess_dpdk_remote_dir(self._remote_tmp_dir) 230 231 def _set_up_build_target(self, build_target_config: BuildTargetConfiguration) -> None: 232 """Setup DPDK on the SUT node. 233 234 Additional build target setup steps on top of those in :class:`Node`. 235 """ 236 # we want to ensure that dpdk_version and compiler_version is reset for new 237 # build targets 238 self._dpdk_version = None 239 self._compiler_version = None 240 self._configure_build_target(build_target_config) 241 self._copy_dpdk_tarball() 242 self._build_dpdk() 243 self.bind_ports_to_driver() 244 245 def _tear_down_build_target(self) -> None: 246 """Bind ports to the operating system drivers. 247 248 Additional build target teardown steps on top of those in :class:`Node`. 249 """ 250 self.bind_ports_to_driver(for_dpdk=False) 251 252 def _configure_build_target(self, build_target_config: BuildTargetConfiguration) -> None: 253 """Populate common environment variables and set build target config.""" 254 self._env_vars = {} 255 self._build_target_config = build_target_config 256 self._env_vars.update(self.main_session.get_dpdk_build_env_vars(build_target_config.arch)) 257 self._env_vars["CC"] = build_target_config.compiler.name 258 if build_target_config.compiler_wrapper: 259 self._env_vars["CC"] = ( 260 f"'{build_target_config.compiler_wrapper} {build_target_config.compiler.name}'" 261 ) # fmt: skip 262 263 @Node.skip_setup 264 def _copy_dpdk_tarball(self) -> None: 265 """Copy to and extract DPDK tarball on the SUT node.""" 266 self._logger.info("Copying DPDK tarball to SUT.") 267 self.main_session.copy_to(SETTINGS.dpdk_tarball_path, self._remote_tmp_dir) 268 269 # construct remote tarball path 270 # the basename is the same on local host and on remote Node 271 remote_tarball_path = self.main_session.join_remote_path( 272 self._remote_tmp_dir, os.path.basename(SETTINGS.dpdk_tarball_path) 273 ) 274 275 # construct remote path after extracting 276 with tarfile.open(SETTINGS.dpdk_tarball_path) as dpdk_tar: 277 dpdk_top_dir = dpdk_tar.getnames()[0] 278 self._remote_dpdk_dir = self.main_session.join_remote_path( 279 self._remote_tmp_dir, dpdk_top_dir 280 ) 281 282 self._logger.info( 283 f"Extracting DPDK tarball on SUT: " 284 f"'{remote_tarball_path}' into '{self._remote_dpdk_dir}'." 285 ) 286 # clean remote path where we're extracting 287 self.main_session.remove_remote_dir(self._remote_dpdk_dir) 288 289 # then extract to remote path 290 self.main_session.extract_remote_tarball(remote_tarball_path, self._remote_dpdk_dir) 291 292 @Node.skip_setup 293 def _build_dpdk(self) -> None: 294 """Build DPDK. 295 296 Uses the already configured target. Assumes that the tarball has 297 already been copied to and extracted on the SUT node. 298 """ 299 self.main_session.build_dpdk( 300 self._env_vars, 301 MesonArgs(default_library="static", enable_kmods=True, libdir="lib"), 302 self._remote_dpdk_dir, 303 self.remote_dpdk_build_dir, 304 ) 305 306 def build_dpdk_app(self, app_name: str, **meson_dpdk_args: str | bool) -> PurePath: 307 """Build one or all DPDK apps. 308 309 Requires DPDK to be already built on the SUT node. 310 311 Args: 312 app_name: The name of the DPDK app to build. 313 When `app_name` is ``all``, build all example apps. 314 meson_dpdk_args: The arguments found in ``meson_options.txt`` in root DPDK directory. 315 Do not use ``-D`` with them. 316 317 Returns: 318 The directory path of the built app. If building all apps, return 319 the path to the examples directory (where all apps reside). 320 """ 321 self.main_session.build_dpdk( 322 self._env_vars, 323 MesonArgs(examples=app_name, **meson_dpdk_args), # type: ignore [arg-type] 324 # ^^ https://github.com/python/mypy/issues/11583 325 self._remote_dpdk_dir, 326 self.remote_dpdk_build_dir, 327 rebuild=True, 328 timeout=self._app_compile_timeout, 329 ) 330 331 if app_name == "all": 332 return self.main_session.join_remote_path(self.remote_dpdk_build_dir, "examples") 333 return self.main_session.join_remote_path( 334 self.remote_dpdk_build_dir, "examples", f"dpdk-{app_name}" 335 ) 336 337 def kill_cleanup_dpdk_apps(self) -> None: 338 """Kill all dpdk applications on the SUT, then clean up hugepages.""" 339 if self._dpdk_kill_session and self._dpdk_kill_session.is_alive(): 340 # we can use the session if it exists and responds 341 self._dpdk_kill_session.kill_cleanup_dpdk_apps(self._dpdk_prefix_list) 342 else: 343 # otherwise, we need to (re)create it 344 self._dpdk_kill_session = self.create_session("dpdk_kill") 345 self._dpdk_prefix_list = [] 346 347 def create_eal_parameters( 348 self, 349 lcore_filter_specifier: LogicalCoreCount | LogicalCoreList = LogicalCoreCount(), 350 ascending_cores: bool = True, 351 prefix: str = "dpdk", 352 append_prefix_timestamp: bool = True, 353 no_pci: bool = False, 354 vdevs: list[VirtualDevice] | None = None, 355 ports: list[Port] | None = None, 356 other_eal_param: str = "", 357 ) -> "EalParameters": 358 """Compose the EAL parameters. 359 360 Process the list of cores and the DPDK prefix and pass that along with 361 the rest of the arguments. 362 363 Args: 364 lcore_filter_specifier: A number of lcores/cores/sockets to use 365 or a list of lcore ids to use. 366 The default will select one lcore for each of two cores 367 on one socket, in ascending order of core ids. 368 ascending_cores: Sort cores in ascending order (lowest to highest IDs). 369 If :data:`False`, sort in descending order. 370 prefix: Set the file prefix string with which to start DPDK, e.g.: ``prefix='vf'``. 371 append_prefix_timestamp: If :data:`True`, will append a timestamp to DPDK file prefix. 372 no_pci: Switch to disable PCI bus e.g.: ``no_pci=True``. 373 vdevs: Virtual devices, e.g.:: 374 375 vdevs=[ 376 VirtualDevice('net_ring0'), 377 VirtualDevice('net_ring1') 378 ] 379 ports: The list of ports to allow. If :data:`None`, all ports listed in `self.ports` 380 will be allowed. 381 other_eal_param: user defined DPDK EAL parameters, e.g.: 382 ``other_eal_param='--single-file-segments'``. 383 384 Returns: 385 An EAL param string, such as 386 ``-c 0xf -a 0000:88:00.0 --file-prefix=dpdk_1112_20190809143420``. 387 """ 388 lcore_list = LogicalCoreList(self.filter_lcores(lcore_filter_specifier, ascending_cores)) 389 390 if append_prefix_timestamp: 391 prefix = f"{prefix}_{self._dpdk_timestamp}" 392 prefix = self.main_session.get_dpdk_file_prefix(prefix) 393 if prefix: 394 self._dpdk_prefix_list.append(prefix) 395 396 if vdevs is None: 397 vdevs = [] 398 399 if ports is None: 400 ports = self.ports 401 402 return EalParameters( 403 lcore_list=lcore_list, 404 memory_channels=self.config.memory_channels, 405 prefix=prefix, 406 no_pci=no_pci, 407 vdevs=vdevs, 408 ports=ports, 409 other_eal_param=other_eal_param, 410 ) 411 412 def run_dpdk_app( 413 self, app_path: PurePath, eal_args: "EalParameters", timeout: float = 30 414 ) -> CommandResult: 415 """Run DPDK application on the remote node. 416 417 The application is not run interactively - the command that starts the application 418 is executed and then the call waits for it to finish execution. 419 420 Args: 421 app_path: The remote path to the DPDK application. 422 eal_args: EAL parameters to run the DPDK application with. 423 timeout: Wait at most this long in seconds for `command` execution to complete. 424 425 Returns: 426 The result of the DPDK app execution. 427 """ 428 return self.main_session.send_command( 429 f"{app_path} {eal_args}", timeout, privileged=True, verify=True 430 ) 431 432 def configure_ipv4_forwarding(self, enable: bool) -> None: 433 """Enable/disable IPv4 forwarding on the node. 434 435 Args: 436 enable: If :data:`True`, enable the forwarding, otherwise disable it. 437 """ 438 self.main_session.configure_ipv4_forwarding(enable) 439 440 def create_interactive_shell( 441 self, 442 shell_cls: Type[InteractiveShellType], 443 timeout: float = SETTINGS.timeout, 444 privileged: bool = False, 445 app_parameters: str = "", 446 eal_parameters: EalParameters | None = None, 447 ) -> InteractiveShellType: 448 """Extend the factory for interactive session handlers. 449 450 The extensions are SUT node specific: 451 452 * The default for `eal_parameters`, 453 * The interactive shell path `shell_cls.path` is prepended with path to the remote 454 DPDK build directory for DPDK apps. 455 456 Args: 457 shell_cls: The class of the shell. 458 timeout: Timeout for reading output from the SSH channel. If you are 459 reading from the buffer and don't receive any data within the timeout 460 it will throw an error. 461 privileged: Whether to run the shell with administrative privileges. 462 eal_parameters: List of EAL parameters to use to launch the app. If this 463 isn't provided or an empty string is passed, it will default to calling 464 :meth:`create_eal_parameters`. 465 app_parameters: Additional arguments to pass into the application on the 466 command-line. 467 468 Returns: 469 An instance of the desired interactive application shell. 470 """ 471 # We need to append the build directory and add EAL parameters for DPDK apps 472 if shell_cls.dpdk_app: 473 if not eal_parameters: 474 eal_parameters = self.create_eal_parameters() 475 app_parameters = f"{eal_parameters} -- {app_parameters}" 476 477 shell_cls.path = self.main_session.join_remote_path( 478 self.remote_dpdk_build_dir, shell_cls.path 479 ) 480 481 return super().create_interactive_shell(shell_cls, timeout, privileged, app_parameters) 482 483 def bind_ports_to_driver(self, for_dpdk: bool = True) -> None: 484 """Bind all ports on the SUT to a driver. 485 486 Args: 487 for_dpdk: If :data:`True`, binds ports to os_driver_for_dpdk. 488 If :data:`False`, binds to os_driver. 489 """ 490 for port in self.ports: 491 driver = port.os_driver_for_dpdk if for_dpdk else port.os_driver 492 self.main_session.send_command( 493 f"{self.path_to_devbind_script} -b {driver} --force {port.pci}", 494 privileged=True, 495 verify=True, 496 ) 497