1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright(c) 2010-2014 Intel Corporation 3# Copyright(c) 2023 PANTHEON.tech s.r.o. 4# Copyright(c) 2023 University of New Hampshire 5 6"""System under test (DPDK + hardware) node. 7 8A system under test (SUT) is the combination of DPDK 9and the hardware we're testing with DPDK (NICs, crypto and other devices). 10An SUT node is where this SUT runs. 11""" 12 13 14import os 15import tarfile 16import time 17from pathlib import PurePath 18from typing import Type 19 20from framework.config import ( 21 BuildTargetConfiguration, 22 BuildTargetInfo, 23 NodeInfo, 24 SutNodeConfiguration, 25) 26from framework.remote_session import CommandResult 27from framework.settings import SETTINGS 28from framework.utils import MesonArgs 29 30from .cpu import LogicalCoreCount, LogicalCoreList 31from .node import Node 32from .os_session import InteractiveShellType, OSSession 33from .virtual_device import VirtualDevice 34 35 36class EalParameters(object): 37 """The environment abstraction layer parameters. 38 39 The string representation can be created by converting the instance to a string. 40 """ 41 42 def __init__( 43 self, 44 lcore_list: LogicalCoreList, 45 memory_channels: int, 46 prefix: str, 47 no_pci: bool, 48 vdevs: list[VirtualDevice], 49 other_eal_param: str, 50 ): 51 """Initialize the parameters according to inputs. 52 53 Process the parameters into the format used on the command line. 54 55 Args: 56 lcore_list: The list of logical cores to use. 57 memory_channels: The number of memory channels to use. 58 prefix: Set the file prefix string with which to start DPDK, e.g.: ``prefix='vf'``. 59 no_pci: Switch to disable PCI bus e.g.: ``no_pci=True``. 60 vdevs: Virtual devices, e.g.:: 61 62 vdevs=[ 63 VirtualDevice('net_ring0'), 64 VirtualDevice('net_ring1') 65 ] 66 other_eal_param: user defined DPDK EAL parameters, e.g.: 67 ``other_eal_param='--single-file-segments'`` 68 """ 69 self._lcore_list = f"-l {lcore_list}" 70 self._memory_channels = f"-n {memory_channels}" 71 self._prefix = prefix 72 if prefix: 73 self._prefix = f"--file-prefix={prefix}" 74 self._no_pci = "--no-pci" if no_pci else "" 75 self._vdevs = " ".join(f"--vdev {vdev}" for vdev in vdevs) 76 self._other_eal_param = other_eal_param 77 78 def __str__(self) -> str: 79 """Create the EAL string.""" 80 return ( 81 f"{self._lcore_list} " 82 f"{self._memory_channels} " 83 f"{self._prefix} " 84 f"{self._no_pci} " 85 f"{self._vdevs} " 86 f"{self._other_eal_param}" 87 ) 88 89 90class SutNode(Node): 91 """The system under test node. 92 93 The SUT node extends :class:`Node` with DPDK specific features: 94 95 * DPDK build, 96 * Gathering of DPDK build info, 97 * The running of DPDK apps, interactively or one-time execution, 98 * DPDK apps cleanup. 99 100 The :option:`--tarball` command line argument and the :envvar:`DTS_DPDK_TARBALL` 101 environment variable configure the path to the DPDK tarball 102 or the git commit ID, tag ID or tree ID to test. 103 104 Attributes: 105 config: The SUT node configuration 106 """ 107 108 config: SutNodeConfiguration 109 _dpdk_prefix_list: list[str] 110 _dpdk_timestamp: str 111 _build_target_config: BuildTargetConfiguration | None 112 _env_vars: dict 113 _remote_tmp_dir: PurePath 114 __remote_dpdk_dir: PurePath | None 115 _app_compile_timeout: float 116 _dpdk_kill_session: OSSession | None 117 _dpdk_version: str | None 118 _node_info: NodeInfo | None 119 _compiler_version: str | None 120 _path_to_devbind_script: PurePath | None 121 122 def __init__(self, node_config: SutNodeConfiguration): 123 """Extend the constructor with SUT node specifics. 124 125 Args: 126 node_config: The SUT node's test run configuration. 127 """ 128 super(SutNode, self).__init__(node_config) 129 self._dpdk_prefix_list = [] 130 self._build_target_config = None 131 self._env_vars = {} 132 self._remote_tmp_dir = self.main_session.get_remote_tmp_dir() 133 self.__remote_dpdk_dir = None 134 self._app_compile_timeout = 90 135 self._dpdk_kill_session = None 136 self._dpdk_timestamp = ( 137 f"{str(os.getpid())}_{time.strftime('%Y%m%d%H%M%S', time.localtime())}" 138 ) 139 self._dpdk_version = None 140 self._node_info = None 141 self._compiler_version = None 142 self._path_to_devbind_script = None 143 self._logger.info(f"Created node: {self.name}") 144 145 @property 146 def _remote_dpdk_dir(self) -> PurePath: 147 """The remote DPDK dir. 148 149 This internal property should be set after extracting the DPDK tarball. If it's not set, 150 that implies the DPDK setup step has been skipped, in which case we can guess where 151 a previous build was located. 152 """ 153 if self.__remote_dpdk_dir is None: 154 self.__remote_dpdk_dir = self._guess_dpdk_remote_dir() 155 return self.__remote_dpdk_dir 156 157 @_remote_dpdk_dir.setter 158 def _remote_dpdk_dir(self, value: PurePath) -> None: 159 self.__remote_dpdk_dir = value 160 161 @property 162 def remote_dpdk_build_dir(self) -> PurePath: 163 """The remote DPDK build directory. 164 165 This is the directory where DPDK was built. 166 We assume it was built in a subdirectory of the extracted tarball. 167 """ 168 if self._build_target_config: 169 return self.main_session.join_remote_path( 170 self._remote_dpdk_dir, self._build_target_config.name 171 ) 172 else: 173 return self.main_session.join_remote_path(self._remote_dpdk_dir, "build") 174 175 @property 176 def dpdk_version(self) -> str: 177 """Last built DPDK version.""" 178 if self._dpdk_version is None: 179 self._dpdk_version = self.main_session.get_dpdk_version(self._remote_dpdk_dir) 180 return self._dpdk_version 181 182 @property 183 def node_info(self) -> NodeInfo: 184 """Additional node information.""" 185 if self._node_info is None: 186 self._node_info = self.main_session.get_node_info() 187 return self._node_info 188 189 @property 190 def compiler_version(self) -> str: 191 """The node's compiler version.""" 192 if self._compiler_version is None: 193 if self._build_target_config is not None: 194 self._compiler_version = self.main_session.get_compiler_version( 195 self._build_target_config.compiler.name 196 ) 197 else: 198 self._logger.warning( 199 "Failed to get compiler version because _build_target_config is None." 200 ) 201 return "" 202 return self._compiler_version 203 204 @property 205 def path_to_devbind_script(self) -> PurePath: 206 """The path to the dpdk-devbind.py script on the node.""" 207 if self._path_to_devbind_script is None: 208 self._path_to_devbind_script = self.main_session.join_remote_path( 209 self._remote_dpdk_dir, "usertools", "dpdk-devbind.py" 210 ) 211 return self._path_to_devbind_script 212 213 def get_build_target_info(self) -> BuildTargetInfo: 214 """Get additional build target information. 215 216 Returns: 217 The build target information, 218 """ 219 return BuildTargetInfo( 220 dpdk_version=self.dpdk_version, compiler_version=self.compiler_version 221 ) 222 223 def _guess_dpdk_remote_dir(self) -> PurePath: 224 return self.main_session.guess_dpdk_remote_dir(self._remote_tmp_dir) 225 226 def _set_up_build_target(self, build_target_config: BuildTargetConfiguration) -> None: 227 """Setup DPDK on the SUT node. 228 229 Additional build target setup steps on top of those in :class:`Node`. 230 """ 231 # we want to ensure that dpdk_version and compiler_version is reset for new 232 # build targets 233 self._dpdk_version = None 234 self._compiler_version = None 235 self._configure_build_target(build_target_config) 236 self._copy_dpdk_tarball() 237 self._build_dpdk() 238 self.bind_ports_to_driver() 239 240 def _tear_down_build_target(self) -> None: 241 """Bind ports to the operating system drivers. 242 243 Additional build target teardown steps on top of those in :class:`Node`. 244 """ 245 self.bind_ports_to_driver(for_dpdk=False) 246 247 def _configure_build_target(self, build_target_config: BuildTargetConfiguration) -> None: 248 """Populate common environment variables and set build target config.""" 249 self._env_vars = {} 250 self._build_target_config = build_target_config 251 self._env_vars.update(self.main_session.get_dpdk_build_env_vars(build_target_config.arch)) 252 self._env_vars["CC"] = build_target_config.compiler.name 253 if build_target_config.compiler_wrapper: 254 self._env_vars["CC"] = ( 255 f"'{build_target_config.compiler_wrapper} {build_target_config.compiler.name}'" 256 ) # fmt: skip 257 258 @Node.skip_setup 259 def _copy_dpdk_tarball(self) -> None: 260 """Copy to and extract DPDK tarball on the SUT node.""" 261 self._logger.info("Copying DPDK tarball to SUT.") 262 self.main_session.copy_to(SETTINGS.dpdk_tarball_path, self._remote_tmp_dir) 263 264 # construct remote tarball path 265 # the basename is the same on local host and on remote Node 266 remote_tarball_path = self.main_session.join_remote_path( 267 self._remote_tmp_dir, os.path.basename(SETTINGS.dpdk_tarball_path) 268 ) 269 270 # construct remote path after extracting 271 with tarfile.open(SETTINGS.dpdk_tarball_path) as dpdk_tar: 272 dpdk_top_dir = dpdk_tar.getnames()[0] 273 self._remote_dpdk_dir = self.main_session.join_remote_path( 274 self._remote_tmp_dir, dpdk_top_dir 275 ) 276 277 self._logger.info( 278 f"Extracting DPDK tarball on SUT: " 279 f"'{remote_tarball_path}' into '{self._remote_dpdk_dir}'." 280 ) 281 # clean remote path where we're extracting 282 self.main_session.remove_remote_dir(self._remote_dpdk_dir) 283 284 # then extract to remote path 285 self.main_session.extract_remote_tarball(remote_tarball_path, self._remote_dpdk_dir) 286 287 @Node.skip_setup 288 def _build_dpdk(self) -> None: 289 """Build DPDK. 290 291 Uses the already configured target. Assumes that the tarball has 292 already been copied to and extracted on the SUT node. 293 """ 294 self.main_session.build_dpdk( 295 self._env_vars, 296 MesonArgs(default_library="static", enable_kmods=True, libdir="lib"), 297 self._remote_dpdk_dir, 298 self.remote_dpdk_build_dir, 299 ) 300 301 def build_dpdk_app(self, app_name: str, **meson_dpdk_args: str | bool) -> PurePath: 302 """Build one or all DPDK apps. 303 304 Requires DPDK to be already built on the SUT node. 305 306 Args: 307 app_name: The name of the DPDK app to build. 308 When `app_name` is ``all``, build all example apps. 309 meson_dpdk_args: The arguments found in ``meson_options.txt`` in root DPDK directory. 310 Do not use ``-D`` with them. 311 312 Returns: 313 The directory path of the built app. If building all apps, return 314 the path to the examples directory (where all apps reside). 315 """ 316 self.main_session.build_dpdk( 317 self._env_vars, 318 MesonArgs(examples=app_name, **meson_dpdk_args), # type: ignore [arg-type] 319 # ^^ https://github.com/python/mypy/issues/11583 320 self._remote_dpdk_dir, 321 self.remote_dpdk_build_dir, 322 rebuild=True, 323 timeout=self._app_compile_timeout, 324 ) 325 326 if app_name == "all": 327 return self.main_session.join_remote_path(self.remote_dpdk_build_dir, "examples") 328 return self.main_session.join_remote_path( 329 self.remote_dpdk_build_dir, "examples", f"dpdk-{app_name}" 330 ) 331 332 def kill_cleanup_dpdk_apps(self) -> None: 333 """Kill all dpdk applications on the SUT, then clean up hugepages.""" 334 if self._dpdk_kill_session and self._dpdk_kill_session.is_alive(): 335 # we can use the session if it exists and responds 336 self._dpdk_kill_session.kill_cleanup_dpdk_apps(self._dpdk_prefix_list) 337 else: 338 # otherwise, we need to (re)create it 339 self._dpdk_kill_session = self.create_session("dpdk_kill") 340 self._dpdk_prefix_list = [] 341 342 def create_eal_parameters( 343 self, 344 lcore_filter_specifier: LogicalCoreCount | LogicalCoreList = LogicalCoreCount(), 345 ascending_cores: bool = True, 346 prefix: str = "dpdk", 347 append_prefix_timestamp: bool = True, 348 no_pci: bool = False, 349 vdevs: list[VirtualDevice] | None = None, 350 other_eal_param: str = "", 351 ) -> "EalParameters": 352 """Compose the EAL parameters. 353 354 Process the list of cores and the DPDK prefix and pass that along with 355 the rest of the arguments. 356 357 Args: 358 lcore_filter_specifier: A number of lcores/cores/sockets to use 359 or a list of lcore ids to use. 360 The default will select one lcore for each of two cores 361 on one socket, in ascending order of core ids. 362 ascending_cores: Sort cores in ascending order (lowest to highest IDs). 363 If :data:`False`, sort in descending order. 364 prefix: Set the file prefix string with which to start DPDK, e.g.: ``prefix='vf'``. 365 append_prefix_timestamp: If :data:`True`, will append a timestamp to DPDK file prefix. 366 no_pci: Switch to disable PCI bus e.g.: ``no_pci=True``. 367 vdevs: Virtual devices, e.g.:: 368 369 vdevs=[ 370 VirtualDevice('net_ring0'), 371 VirtualDevice('net_ring1') 372 ] 373 other_eal_param: user defined DPDK EAL parameters, e.g.: 374 ``other_eal_param='--single-file-segments'``. 375 376 Returns: 377 An EAL param string, such as 378 ``-c 0xf -a 0000:88:00.0 --file-prefix=dpdk_1112_20190809143420``. 379 """ 380 lcore_list = LogicalCoreList(self.filter_lcores(lcore_filter_specifier, ascending_cores)) 381 382 if append_prefix_timestamp: 383 prefix = f"{prefix}_{self._dpdk_timestamp}" 384 prefix = self.main_session.get_dpdk_file_prefix(prefix) 385 if prefix: 386 self._dpdk_prefix_list.append(prefix) 387 388 if vdevs is None: 389 vdevs = [] 390 391 return EalParameters( 392 lcore_list=lcore_list, 393 memory_channels=self.config.memory_channels, 394 prefix=prefix, 395 no_pci=no_pci, 396 vdevs=vdevs, 397 other_eal_param=other_eal_param, 398 ) 399 400 def run_dpdk_app( 401 self, app_path: PurePath, eal_args: "EalParameters", timeout: float = 30 402 ) -> CommandResult: 403 """Run DPDK application on the remote node. 404 405 The application is not run interactively - the command that starts the application 406 is executed and then the call waits for it to finish execution. 407 408 Args: 409 app_path: The remote path to the DPDK application. 410 eal_args: EAL parameters to run the DPDK application with. 411 timeout: Wait at most this long in seconds for `command` execution to complete. 412 413 Returns: 414 The result of the DPDK app execution. 415 """ 416 return self.main_session.send_command( 417 f"{app_path} {eal_args}", timeout, privileged=True, verify=True 418 ) 419 420 def configure_ipv4_forwarding(self, enable: bool) -> None: 421 """Enable/disable IPv4 forwarding on the node. 422 423 Args: 424 enable: If :data:`True`, enable the forwarding, otherwise disable it. 425 """ 426 self.main_session.configure_ipv4_forwarding(enable) 427 428 def create_interactive_shell( 429 self, 430 shell_cls: Type[InteractiveShellType], 431 timeout: float = SETTINGS.timeout, 432 privileged: bool = False, 433 eal_parameters: EalParameters | str | None = None, 434 ) -> InteractiveShellType: 435 """Extend the factory for interactive session handlers. 436 437 The extensions are SUT node specific: 438 439 * The default for `eal_parameters`, 440 * The interactive shell path `shell_cls.path` is prepended with path to the remote 441 DPDK build directory for DPDK apps. 442 443 Args: 444 shell_cls: The class of the shell. 445 timeout: Timeout for reading output from the SSH channel. If you are 446 reading from the buffer and don't receive any data within the timeout 447 it will throw an error. 448 privileged: Whether to run the shell with administrative privileges. 449 eal_parameters: List of EAL parameters to use to launch the app. If this 450 isn't provided or an empty string is passed, it will default to calling 451 :meth:`create_eal_parameters`. 452 453 Returns: 454 An instance of the desired interactive application shell. 455 """ 456 if not eal_parameters: 457 eal_parameters = self.create_eal_parameters() 458 459 # We need to append the build directory for DPDK apps 460 if shell_cls.dpdk_app: 461 shell_cls.path = self.main_session.join_remote_path( 462 self.remote_dpdk_build_dir, shell_cls.path 463 ) 464 465 return super().create_interactive_shell(shell_cls, timeout, privileged, str(eal_parameters)) 466 467 def bind_ports_to_driver(self, for_dpdk: bool = True) -> None: 468 """Bind all ports on the SUT to a driver. 469 470 Args: 471 for_dpdk: If :data:`True`, binds ports to os_driver_for_dpdk. 472 If :data:`False`, binds to os_driver. 473 """ 474 for port in self.ports: 475 driver = port.os_driver_for_dpdk if for_dpdk else port.os_driver 476 self.main_session.send_command( 477 f"{self.path_to_devbind_script} -b {driver} --force {port.pci}", 478 privileged=True, 479 verify=True, 480 ) 481