1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "env_internal.h" 9 10 #include "spdk/version.h" 11 #include "spdk/env_dpdk.h" 12 #include "spdk/log.h" 13 14 #include <rte_config.h> 15 #include <rte_eal.h> 16 #include <rte_errno.h> 17 #include <rte_vfio.h> 18 19 #define SPDK_ENV_DPDK_DEFAULT_NAME "spdk" 20 #define SPDK_ENV_DPDK_DEFAULT_SHM_ID -1 21 #define SPDK_ENV_DPDK_DEFAULT_MEM_SIZE -1 22 #define SPDK_ENV_DPDK_DEFAULT_MAIN_CORE -1 23 #define SPDK_ENV_DPDK_DEFAULT_MEM_CHANNEL -1 24 #define SPDK_ENV_DPDK_DEFAULT_CORE_MASK "0x1" 25 #define SPDK_ENV_DPDK_DEFAULT_BASE_VIRTADDR 0x200000000000 26 27 #if RTE_VERSION < RTE_VERSION_NUM(20, 11, 0, 0) 28 #define DPDK_ALLOW_PARAM "--pci-whitelist" 29 #define DPDK_BLOCK_PARAM "--pci-blacklist" 30 #define DPDK_MAIN_CORE_PARAM "--master-lcore" 31 #else 32 #define DPDK_ALLOW_PARAM "--allow" 33 #define DPDK_BLOCK_PARAM "--block" 34 #define DPDK_MAIN_CORE_PARAM "--main-lcore" 35 #endif 36 37 static char **g_eal_cmdline; 38 static int g_eal_cmdline_argcount; 39 static bool g_external_init = true; 40 41 static char * 42 _sprintf_alloc(const char *format, ...) 43 { 44 va_list args; 45 va_list args_copy; 46 char *buf; 47 size_t bufsize; 48 int rc; 49 50 va_start(args, format); 51 52 /* Try with a small buffer first. */ 53 bufsize = 32; 54 55 /* Limit maximum buffer size to something reasonable so we don't loop forever. */ 56 while (bufsize <= 1024 * 1024) { 57 buf = malloc(bufsize); 58 if (buf == NULL) { 59 va_end(args); 60 return NULL; 61 } 62 63 va_copy(args_copy, args); 64 rc = vsnprintf(buf, bufsize, format, args_copy); 65 va_end(args_copy); 66 67 /* 68 * If vsnprintf() returned a count within our current buffer size, we are done. 69 * The count does not include the \0 terminator, so rc == bufsize is not OK. 70 */ 71 if (rc >= 0 && (size_t)rc < bufsize) { 72 va_end(args); 73 return buf; 74 } 75 76 /* 77 * vsnprintf() should return the required space, but some libc versions do not 78 * implement this correctly, so just double the buffer size and try again. 79 * 80 * We don't need the data in buf, so rather than realloc(), use free() and malloc() 81 * again to avoid a copy. 82 */ 83 free(buf); 84 bufsize *= 2; 85 } 86 87 va_end(args); 88 return NULL; 89 } 90 91 void 92 spdk_env_opts_init(struct spdk_env_opts *opts) 93 { 94 if (!opts) { 95 return; 96 } 97 98 memset(opts, 0, sizeof(*opts)); 99 100 opts->name = SPDK_ENV_DPDK_DEFAULT_NAME; 101 opts->core_mask = SPDK_ENV_DPDK_DEFAULT_CORE_MASK; 102 opts->shm_id = SPDK_ENV_DPDK_DEFAULT_SHM_ID; 103 opts->mem_size = SPDK_ENV_DPDK_DEFAULT_MEM_SIZE; 104 opts->main_core = SPDK_ENV_DPDK_DEFAULT_MAIN_CORE; 105 opts->mem_channel = SPDK_ENV_DPDK_DEFAULT_MEM_CHANNEL; 106 opts->base_virtaddr = SPDK_ENV_DPDK_DEFAULT_BASE_VIRTADDR; 107 } 108 109 static void 110 free_args(char **args, int argcount) 111 { 112 int i; 113 114 if (args == NULL) { 115 return; 116 } 117 118 for (i = 0; i < argcount; i++) { 119 free(args[i]); 120 } 121 122 if (argcount) { 123 free(args); 124 } 125 } 126 127 static char ** 128 push_arg(char *args[], int *argcount, char *arg) 129 { 130 char **tmp; 131 132 if (arg == NULL) { 133 SPDK_ERRLOG("%s: NULL arg supplied\n", __func__); 134 free_args(args, *argcount); 135 return NULL; 136 } 137 138 tmp = realloc(args, sizeof(char *) * (*argcount + 1)); 139 if (tmp == NULL) { 140 free(arg); 141 free_args(args, *argcount); 142 return NULL; 143 } 144 145 tmp[*argcount] = arg; 146 (*argcount)++; 147 148 return tmp; 149 } 150 151 #if defined(__linux__) && defined(__x86_64__) 152 153 /* TODO: Can likely get this value from rlimits in the future */ 154 #define SPDK_IOMMU_VA_REQUIRED_WIDTH 48 155 #define VTD_CAP_MGAW_SHIFT 16 156 #define VTD_CAP_MGAW_MASK (0x3F << VTD_CAP_MGAW_SHIFT) 157 #define RD_AMD_CAP_VASIZE_SHIFT 15 158 #define RD_AMD_CAP_VASIZE_MASK (0x7F << RD_AMD_CAP_VASIZE_SHIFT) 159 160 static int 161 get_iommu_width(void) 162 { 163 int width = 0; 164 glob_t glob_results = {}; 165 166 /* Break * and / into separate strings to appease check_format.sh comment style check. */ 167 glob("/sys/devices/virtual/iommu/dmar*" "/intel-iommu/cap", 0, NULL, &glob_results); 168 glob("/sys/class/iommu/ivhd*" "/amd-iommu/cap", GLOB_APPEND, NULL, &glob_results); 169 170 for (size_t i = 0; i < glob_results.gl_pathc; i++) { 171 const char *filename = glob_results.gl_pathv[0]; 172 FILE *file = fopen(filename, "r"); 173 uint64_t cap_reg = 0; 174 175 if (file != NULL && fscanf(file, "%" PRIx64, &cap_reg) == 1) { 176 if (strstr(filename, "intel-iommu") != NULL) { 177 /* We have an Intel IOMMU */ 178 int mgaw = ((cap_reg & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1; 179 180 if (width == 0 || (mgaw > 0 && mgaw < width)) { 181 width = mgaw; 182 } 183 } else if (strstr(filename, "amd-iommu") != NULL) { 184 /* We have an AMD IOMMU */ 185 int mgaw = ((cap_reg & RD_AMD_CAP_VASIZE_MASK) >> RD_AMD_CAP_VASIZE_SHIFT) + 1; 186 187 if (width == 0 || (mgaw > 0 && mgaw < width)) { 188 width = mgaw; 189 } 190 } 191 } 192 193 fclose(file); 194 } 195 196 globfree(&glob_results); 197 return width; 198 } 199 200 #endif 201 202 static int 203 build_eal_cmdline(const struct spdk_env_opts *opts) 204 { 205 int argcount = 0; 206 char **args; 207 208 args = NULL; 209 210 /* set the program name */ 211 args = push_arg(args, &argcount, _sprintf_alloc("%s", opts->name)); 212 if (args == NULL) { 213 return -1; 214 } 215 216 /* disable shared configuration files when in single process mode. This allows for cleaner shutdown */ 217 if (opts->shm_id < 0) { 218 args = push_arg(args, &argcount, _sprintf_alloc("%s", "--no-shconf")); 219 if (args == NULL) { 220 return -1; 221 } 222 } 223 224 /* 225 * Set the coremask: 226 * 227 * - if it starts with '-', we presume it's literal EAL arguments such 228 * as --lcores. 229 * 230 * - if it starts with '[', we presume it's a core list to use with the 231 * -l option. 232 * 233 * - otherwise, it's a CPU mask of the form "0xff.." as expected by the 234 * -c option. 235 */ 236 if (opts->core_mask[0] == '-') { 237 args = push_arg(args, &argcount, _sprintf_alloc("%s", opts->core_mask)); 238 } else if (opts->core_mask[0] == '[') { 239 char *l_arg = _sprintf_alloc("-l %s", opts->core_mask + 1); 240 241 if (l_arg != NULL) { 242 int len = strlen(l_arg); 243 244 if (l_arg[len - 1] == ']') { 245 l_arg[len - 1] = '\0'; 246 } 247 } 248 args = push_arg(args, &argcount, l_arg); 249 } else { 250 args = push_arg(args, &argcount, _sprintf_alloc("-c %s", opts->core_mask)); 251 } 252 253 if (args == NULL) { 254 return -1; 255 } 256 257 /* set the memory channel number */ 258 if (opts->mem_channel > 0) { 259 args = push_arg(args, &argcount, _sprintf_alloc("-n %d", opts->mem_channel)); 260 if (args == NULL) { 261 return -1; 262 } 263 } 264 265 /* set the memory size */ 266 if (opts->mem_size >= 0) { 267 args = push_arg(args, &argcount, _sprintf_alloc("-m %d", opts->mem_size)); 268 if (args == NULL) { 269 return -1; 270 } 271 } 272 273 /* set the main core */ 274 if (opts->main_core > 0) { 275 args = push_arg(args, &argcount, _sprintf_alloc("%s=%d", 276 DPDK_MAIN_CORE_PARAM, opts->main_core)); 277 if (args == NULL) { 278 return -1; 279 } 280 } 281 282 /* set no pci if enabled */ 283 if (opts->no_pci) { 284 args = push_arg(args, &argcount, _sprintf_alloc("--no-pci")); 285 if (args == NULL) { 286 return -1; 287 } 288 } 289 290 /* create just one hugetlbfs file */ 291 if (opts->hugepage_single_segments) { 292 args = push_arg(args, &argcount, _sprintf_alloc("--single-file-segments")); 293 if (args == NULL) { 294 return -1; 295 } 296 } 297 298 /* unlink hugepages after initialization */ 299 /* Note: Automatically unlink hugepage when shm_id < 0, since it means we're not using 300 * multi-process so we don't need the hugepage links anymore. But we need to make sure 301 * we don't specify --huge-unlink implicitly if --single-file-segments was specified since 302 * DPDK doesn't support that. 303 */ 304 if (opts->unlink_hugepage || 305 (opts->shm_id < 0 && !opts->hugepage_single_segments)) { 306 args = push_arg(args, &argcount, _sprintf_alloc("--huge-unlink")); 307 if (args == NULL) { 308 return -1; 309 } 310 } 311 312 /* use a specific hugetlbfs mount */ 313 if (opts->hugedir) { 314 args = push_arg(args, &argcount, _sprintf_alloc("--huge-dir=%s", opts->hugedir)); 315 if (args == NULL) { 316 return -1; 317 } 318 } 319 320 if (opts->num_pci_addr) { 321 size_t i; 322 char bdf[32]; 323 struct spdk_pci_addr *pci_addr = 324 opts->pci_blocked ? opts->pci_blocked : opts->pci_allowed; 325 326 for (i = 0; i < opts->num_pci_addr; i++) { 327 spdk_pci_addr_fmt(bdf, 32, &pci_addr[i]); 328 args = push_arg(args, &argcount, _sprintf_alloc("%s=%s", 329 (opts->pci_blocked ? DPDK_BLOCK_PARAM : DPDK_ALLOW_PARAM), 330 bdf)); 331 if (args == NULL) { 332 return -1; 333 } 334 } 335 } 336 337 /* Lower default EAL loglevel to RTE_LOG_NOTICE - normal, but significant messages. 338 * This can be overridden by specifying the same option in opts->env_context 339 */ 340 args = push_arg(args, &argcount, strdup("--log-level=lib.eal:6")); 341 if (args == NULL) { 342 return -1; 343 } 344 345 /* Lower default CRYPTO loglevel to RTE_LOG_ERR to avoid a ton of init msgs. 346 * This can be overridden by specifying the same option in opts->env_context 347 */ 348 args = push_arg(args, &argcount, strdup("--log-level=lib.cryptodev:5")); 349 if (args == NULL) { 350 return -1; 351 } 352 353 /* `user1` log type is used by rte_vhost, which prints an INFO log for each received 354 * vhost user message. We don't want that. The same log type is also used by a couple 355 * of other DPDK libs, but none of which we make use right now. If necessary, this can 356 * be overridden via opts->env_context. 357 */ 358 args = push_arg(args, &argcount, strdup("--log-level=user1:6")); 359 if (args == NULL) { 360 return -1; 361 } 362 363 if (opts->env_context) { 364 char *ptr = strdup(opts->env_context); 365 char *tok = strtok(ptr, " \t"); 366 367 /* DPDK expects each argument as a separate string in the argv 368 * array, so we need to tokenize here in case the caller 369 * passed multiple arguments in the env_context string. 370 */ 371 while (tok != NULL) { 372 args = push_arg(args, &argcount, strdup(tok)); 373 tok = strtok(NULL, " \t"); 374 } 375 376 free(ptr); 377 } 378 379 #ifdef __linux__ 380 381 if (opts->iova_mode) { 382 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=%s", opts->iova_mode)); 383 if (args == NULL) { 384 return -1; 385 } 386 } else { 387 /* When using vfio with enable_unsafe_noiommu_mode=Y, we need iova-mode=pa, 388 * but DPDK guesses it should be iova-mode=va. Add a check and force 389 * iova-mode=pa here. */ 390 if (rte_vfio_noiommu_is_enabled()) { 391 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 392 if (args == NULL) { 393 return -1; 394 } 395 } 396 397 #if defined(__x86_64__) 398 /* DPDK by default guesses that it should be using iova-mode=va so that it can 399 * support running as an unprivileged user. However, some systems (especially 400 * virtual machines) don't have an IOMMU capable of handling the full virtual 401 * address space and DPDK doesn't currently catch that. Add a check in SPDK 402 * and force iova-mode=pa here. */ 403 if (get_iommu_width() < SPDK_IOMMU_VA_REQUIRED_WIDTH) { 404 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 405 if (args == NULL) { 406 return -1; 407 } 408 } 409 #elif defined(__PPC64__) 410 /* On Linux + PowerPC, DPDK doesn't support VA mode at all. Unfortunately, it doesn't correctly 411 * auto-detect at the moment, so we'll just force it here. */ 412 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 413 if (args == NULL) { 414 return -1; 415 } 416 #endif 417 } 418 419 420 /* Set the base virtual address - it must be an address that is not in the 421 * ASAN shadow region, otherwise ASAN-enabled builds will ignore the 422 * mmap hint. 423 * 424 * Ref: https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm 425 */ 426 args = push_arg(args, &argcount, _sprintf_alloc("--base-virtaddr=0x%" PRIx64, opts->base_virtaddr)); 427 if (args == NULL) { 428 return -1; 429 } 430 431 /* --match-allocation prevents DPDK from merging or splitting system memory allocations under the hood. 432 * This is critical for RDMA when attempting to use an rte_mempool based buffer pool. If DPDK merges two 433 * physically or IOVA contiguous memory regions, then when we go to allocate a buffer pool, it can split 434 * the memory for a buffer over two allocations meaning the buffer will be split over a memory region. 435 */ 436 if (!opts->env_context || strstr(opts->env_context, "--legacy-mem") == NULL) { 437 args = push_arg(args, &argcount, _sprintf_alloc("%s", "--match-allocations")); 438 if (args == NULL) { 439 return -1; 440 } 441 } 442 443 if (opts->shm_id < 0) { 444 args = push_arg(args, &argcount, _sprintf_alloc("--file-prefix=spdk_pid%d", 445 getpid())); 446 if (args == NULL) { 447 return -1; 448 } 449 } else { 450 args = push_arg(args, &argcount, _sprintf_alloc("--file-prefix=spdk%d", 451 opts->shm_id)); 452 if (args == NULL) { 453 return -1; 454 } 455 456 /* set the process type */ 457 args = push_arg(args, &argcount, _sprintf_alloc("--proc-type=auto")); 458 if (args == NULL) { 459 return -1; 460 } 461 } 462 463 /* --vfio-vf-token used for VF initialized by vfio_pci driver. */ 464 if (opts->vf_token) { 465 args = push_arg(args, &argcount, _sprintf_alloc("--vfio-vf-token=%s", 466 opts->vf_token)); 467 if (args == NULL) { 468 return -1; 469 } 470 } 471 #endif 472 473 g_eal_cmdline = args; 474 g_eal_cmdline_argcount = argcount; 475 return argcount; 476 } 477 478 int 479 spdk_env_dpdk_post_init(bool legacy_mem) 480 { 481 int rc; 482 483 rc = pci_env_init(); 484 if (rc < 0) { 485 SPDK_ERRLOG("pci_env_init() failed\n"); 486 return rc; 487 } 488 489 rc = mem_map_init(legacy_mem); 490 if (rc < 0) { 491 SPDK_ERRLOG("Failed to allocate mem_map\n"); 492 return rc; 493 } 494 495 rc = vtophys_init(); 496 if (rc < 0) { 497 SPDK_ERRLOG("Failed to initialize vtophys\n"); 498 return rc; 499 } 500 501 return 0; 502 } 503 504 void 505 spdk_env_dpdk_post_fini(void) 506 { 507 pci_env_fini(); 508 509 free_args(g_eal_cmdline, g_eal_cmdline_argcount); 510 g_eal_cmdline = NULL; 511 g_eal_cmdline_argcount = 0; 512 } 513 514 int 515 spdk_env_init(const struct spdk_env_opts *opts) 516 { 517 char **dpdk_args = NULL; 518 int i, rc; 519 int orig_optind; 520 bool legacy_mem; 521 522 /* If SPDK env has been initialized before, then only pci env requires 523 * reinitialization. 524 */ 525 if (g_external_init == false) { 526 if (opts != NULL) { 527 fprintf(stderr, "Invalid arguments to reinitialize SPDK env\n"); 528 return -EINVAL; 529 } 530 531 printf("Starting %s / %s reinitialization...\n", SPDK_VERSION_STRING, rte_version()); 532 pci_env_reinit(); 533 534 return 0; 535 } 536 537 if (opts == NULL) { 538 fprintf(stderr, "NULL arguments to initialize DPDK\n"); 539 return -EINVAL; 540 } 541 542 rc = build_eal_cmdline(opts); 543 if (rc < 0) { 544 SPDK_ERRLOG("Invalid arguments to initialize DPDK\n"); 545 return -EINVAL; 546 } 547 548 SPDK_PRINTF("Starting %s / %s initialization...\n", SPDK_VERSION_STRING, rte_version()); 549 SPDK_PRINTF("[ DPDK EAL parameters: "); 550 for (i = 0; i < g_eal_cmdline_argcount; i++) { 551 SPDK_PRINTF("%s ", g_eal_cmdline[i]); 552 } 553 SPDK_PRINTF("]\n"); 554 555 /* DPDK rearranges the array we pass to it, so make a copy 556 * before passing so we can still free the individual strings 557 * correctly. 558 */ 559 dpdk_args = calloc(g_eal_cmdline_argcount, sizeof(char *)); 560 if (dpdk_args == NULL) { 561 SPDK_ERRLOG("Failed to allocate dpdk_args\n"); 562 return -ENOMEM; 563 } 564 memcpy(dpdk_args, g_eal_cmdline, sizeof(char *) * g_eal_cmdline_argcount); 565 566 fflush(stdout); 567 orig_optind = optind; 568 optind = 1; 569 rc = rte_eal_init(g_eal_cmdline_argcount, dpdk_args); 570 optind = orig_optind; 571 572 free(dpdk_args); 573 574 if (rc < 0) { 575 if (rte_errno == EALREADY) { 576 SPDK_ERRLOG("DPDK already initialized\n"); 577 } else { 578 SPDK_ERRLOG("Failed to initialize DPDK\n"); 579 } 580 return -rte_errno; 581 } 582 583 legacy_mem = false; 584 if (opts->env_context && strstr(opts->env_context, "--legacy-mem") != NULL) { 585 legacy_mem = true; 586 } 587 588 rc = spdk_env_dpdk_post_init(legacy_mem); 589 if (rc == 0) { 590 g_external_init = false; 591 } 592 593 return rc; 594 } 595 596 /* We use priority 101 which is the highest priority level available 597 * to applications (the toolchains reserve 1 to 100 for internal usage). 598 * This ensures this destructor runs last, after any other destructors 599 * that might still need the environment up and running. 600 */ 601 __attribute__((destructor(101))) static void 602 dpdk_cleanup(void) 603 { 604 /* Only call rte_eal_cleanup if the SPDK env library called rte_eal_init. */ 605 if (!g_external_init) { 606 rte_eal_cleanup(); 607 } 608 } 609 610 void 611 spdk_env_fini(void) 612 { 613 spdk_env_dpdk_post_fini(); 614 } 615 616 bool 617 spdk_env_dpdk_external_init(void) 618 { 619 return g_external_init; 620 } 621