1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "env_internal.h" 9 10 #include "spdk/version.h" 11 #include "spdk/env_dpdk.h" 12 #include "spdk/log.h" 13 14 #include <rte_config.h> 15 #include <rte_eal.h> 16 #include <rte_errno.h> 17 #include <rte_vfio.h> 18 19 #define SPDK_ENV_DPDK_DEFAULT_NAME "spdk" 20 #define SPDK_ENV_DPDK_DEFAULT_SHM_ID -1 21 #define SPDK_ENV_DPDK_DEFAULT_MEM_SIZE -1 22 #define SPDK_ENV_DPDK_DEFAULT_MAIN_CORE -1 23 #define SPDK_ENV_DPDK_DEFAULT_MEM_CHANNEL -1 24 #define SPDK_ENV_DPDK_DEFAULT_CORE_MASK "0x1" 25 #define SPDK_ENV_DPDK_DEFAULT_BASE_VIRTADDR 0x200000000000 26 27 #if RTE_VERSION < RTE_VERSION_NUM(20, 11, 0, 0) 28 #define DPDK_ALLOW_PARAM "--pci-whitelist" 29 #define DPDK_BLOCK_PARAM "--pci-blacklist" 30 #define DPDK_MAIN_CORE_PARAM "--master-lcore" 31 #else 32 #define DPDK_ALLOW_PARAM "--allow" 33 #define DPDK_BLOCK_PARAM "--block" 34 #define DPDK_MAIN_CORE_PARAM "--main-lcore" 35 #endif 36 37 static char **g_eal_cmdline; 38 static int g_eal_cmdline_argcount; 39 static bool g_external_init = true; 40 41 static char * 42 _sprintf_alloc(const char *format, ...) 43 { 44 va_list args; 45 va_list args_copy; 46 char *buf; 47 size_t bufsize; 48 int rc; 49 50 va_start(args, format); 51 52 /* Try with a small buffer first. */ 53 bufsize = 32; 54 55 /* Limit maximum buffer size to something reasonable so we don't loop forever. */ 56 while (bufsize <= 1024 * 1024) { 57 buf = malloc(bufsize); 58 if (buf == NULL) { 59 va_end(args); 60 return NULL; 61 } 62 63 va_copy(args_copy, args); 64 rc = vsnprintf(buf, bufsize, format, args_copy); 65 va_end(args_copy); 66 67 /* 68 * If vsnprintf() returned a count within our current buffer size, we are done. 69 * The count does not include the \0 terminator, so rc == bufsize is not OK. 70 */ 71 if (rc >= 0 && (size_t)rc < bufsize) { 72 va_end(args); 73 return buf; 74 } 75 76 /* 77 * vsnprintf() should return the required space, but some libc versions do not 78 * implement this correctly, so just double the buffer size and try again. 79 * 80 * We don't need the data in buf, so rather than realloc(), use free() and malloc() 81 * again to avoid a copy. 82 */ 83 free(buf); 84 bufsize *= 2; 85 } 86 87 va_end(args); 88 return NULL; 89 } 90 91 void 92 spdk_env_opts_init(struct spdk_env_opts *opts) 93 { 94 if (!opts) { 95 return; 96 } 97 98 memset(opts, 0, sizeof(*opts)); 99 100 opts->name = SPDK_ENV_DPDK_DEFAULT_NAME; 101 opts->core_mask = SPDK_ENV_DPDK_DEFAULT_CORE_MASK; 102 opts->shm_id = SPDK_ENV_DPDK_DEFAULT_SHM_ID; 103 opts->mem_size = SPDK_ENV_DPDK_DEFAULT_MEM_SIZE; 104 opts->main_core = SPDK_ENV_DPDK_DEFAULT_MAIN_CORE; 105 opts->mem_channel = SPDK_ENV_DPDK_DEFAULT_MEM_CHANNEL; 106 opts->base_virtaddr = SPDK_ENV_DPDK_DEFAULT_BASE_VIRTADDR; 107 } 108 109 static void 110 free_args(char **args, int argcount) 111 { 112 int i; 113 114 if (args == NULL) { 115 return; 116 } 117 118 for (i = 0; i < argcount; i++) { 119 free(args[i]); 120 } 121 122 if (argcount) { 123 free(args); 124 } 125 } 126 127 static char ** 128 push_arg(char *args[], int *argcount, char *arg) 129 { 130 char **tmp; 131 132 if (arg == NULL) { 133 SPDK_ERRLOG("%s: NULL arg supplied\n", __func__); 134 free_args(args, *argcount); 135 return NULL; 136 } 137 138 tmp = realloc(args, sizeof(char *) * (*argcount + 1)); 139 if (tmp == NULL) { 140 free(arg); 141 free_args(args, *argcount); 142 return NULL; 143 } 144 145 tmp[*argcount] = arg; 146 (*argcount)++; 147 148 return tmp; 149 } 150 151 #if defined(__linux__) && defined(__x86_64__) 152 153 /* TODO: Can likely get this value from rlimits in the future */ 154 #define SPDK_IOMMU_VA_REQUIRED_WIDTH 48 155 #define VTD_CAP_MGAW_SHIFT 16 156 #define VTD_CAP_MGAW_MASK (0x3F << VTD_CAP_MGAW_SHIFT) 157 #define RD_AMD_CAP_VASIZE_SHIFT 15 158 #define RD_AMD_CAP_VASIZE_MASK (0x7F << RD_AMD_CAP_VASIZE_SHIFT) 159 160 static int 161 get_iommu_width(void) 162 { 163 int width = 0; 164 glob_t glob_results = {}; 165 166 /* Break * and / into separate strings to appease check_format.sh comment style check. */ 167 glob("/sys/devices/virtual/iommu/dmar*" "/intel-iommu/cap", 0, NULL, &glob_results); 168 glob("/sys/class/iommu/ivhd*" "/amd-iommu/cap", GLOB_APPEND, NULL, &glob_results); 169 170 for (size_t i = 0; i < glob_results.gl_pathc; i++) { 171 const char *filename = glob_results.gl_pathv[0]; 172 FILE *file = fopen(filename, "r"); 173 uint64_t cap_reg = 0; 174 175 if (file == NULL) { 176 continue; 177 } 178 179 if (fscanf(file, "%" PRIx64, &cap_reg) == 1) { 180 if (strstr(filename, "intel-iommu") != NULL) { 181 /* We have an Intel IOMMU */ 182 int mgaw = ((cap_reg & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1; 183 184 if (width == 0 || (mgaw > 0 && mgaw < width)) { 185 width = mgaw; 186 } 187 } else if (strstr(filename, "amd-iommu") != NULL) { 188 /* We have an AMD IOMMU */ 189 int mgaw = ((cap_reg & RD_AMD_CAP_VASIZE_MASK) >> RD_AMD_CAP_VASIZE_SHIFT) + 1; 190 191 if (width == 0 || (mgaw > 0 && mgaw < width)) { 192 width = mgaw; 193 } 194 } 195 } 196 197 fclose(file); 198 } 199 200 globfree(&glob_results); 201 return width; 202 } 203 204 #endif 205 206 static int 207 build_eal_cmdline(const struct spdk_env_opts *opts) 208 { 209 int argcount = 0; 210 char **args; 211 212 args = NULL; 213 214 /* set the program name */ 215 args = push_arg(args, &argcount, _sprintf_alloc("%s", opts->name)); 216 if (args == NULL) { 217 return -1; 218 } 219 220 /* disable shared configuration files when in single process mode. This allows for cleaner shutdown */ 221 if (opts->shm_id < 0) { 222 args = push_arg(args, &argcount, _sprintf_alloc("%s", "--no-shconf")); 223 if (args == NULL) { 224 return -1; 225 } 226 } 227 228 /* 229 * Set the coremask: 230 * 231 * - if it starts with '-', we presume it's literal EAL arguments such 232 * as --lcores. 233 * 234 * - if it starts with '[', we presume it's a core list to use with the 235 * -l option. 236 * 237 * - otherwise, it's a CPU mask of the form "0xff.." as expected by the 238 * -c option. 239 */ 240 if (opts->core_mask[0] == '-') { 241 args = push_arg(args, &argcount, _sprintf_alloc("%s", opts->core_mask)); 242 } else if (opts->core_mask[0] == '[') { 243 char *l_arg = _sprintf_alloc("-l %s", opts->core_mask + 1); 244 245 if (l_arg != NULL) { 246 int len = strlen(l_arg); 247 248 if (l_arg[len - 1] == ']') { 249 l_arg[len - 1] = '\0'; 250 } 251 } 252 args = push_arg(args, &argcount, l_arg); 253 } else { 254 args = push_arg(args, &argcount, _sprintf_alloc("-c %s", opts->core_mask)); 255 } 256 257 if (args == NULL) { 258 return -1; 259 } 260 261 /* set the memory channel number */ 262 if (opts->mem_channel > 0) { 263 args = push_arg(args, &argcount, _sprintf_alloc("-n %d", opts->mem_channel)); 264 if (args == NULL) { 265 return -1; 266 } 267 } 268 269 /* set the memory size */ 270 if (opts->mem_size >= 0) { 271 args = push_arg(args, &argcount, _sprintf_alloc("-m %d", opts->mem_size)); 272 if (args == NULL) { 273 return -1; 274 } 275 } 276 277 /* set the main core */ 278 if (opts->main_core > 0) { 279 args = push_arg(args, &argcount, _sprintf_alloc("%s=%d", 280 DPDK_MAIN_CORE_PARAM, opts->main_core)); 281 if (args == NULL) { 282 return -1; 283 } 284 } 285 286 /* set no pci if enabled */ 287 if (opts->no_pci) { 288 args = push_arg(args, &argcount, _sprintf_alloc("--no-pci")); 289 if (args == NULL) { 290 return -1; 291 } 292 } 293 294 /* create just one hugetlbfs file */ 295 if (opts->hugepage_single_segments) { 296 args = push_arg(args, &argcount, _sprintf_alloc("--single-file-segments")); 297 if (args == NULL) { 298 return -1; 299 } 300 } 301 302 /* unlink hugepages after initialization */ 303 /* Note: Automatically unlink hugepage when shm_id < 0, since it means we're not using 304 * multi-process so we don't need the hugepage links anymore. But we need to make sure 305 * we don't specify --huge-unlink implicitly if --single-file-segments was specified since 306 * DPDK doesn't support that. 307 */ 308 if (opts->unlink_hugepage || 309 (opts->shm_id < 0 && !opts->hugepage_single_segments)) { 310 args = push_arg(args, &argcount, _sprintf_alloc("--huge-unlink")); 311 if (args == NULL) { 312 return -1; 313 } 314 } 315 316 /* use a specific hugetlbfs mount */ 317 if (opts->hugedir) { 318 args = push_arg(args, &argcount, _sprintf_alloc("--huge-dir=%s", opts->hugedir)); 319 if (args == NULL) { 320 return -1; 321 } 322 } 323 324 if (opts->num_pci_addr) { 325 size_t i; 326 char bdf[32]; 327 struct spdk_pci_addr *pci_addr = 328 opts->pci_blocked ? opts->pci_blocked : opts->pci_allowed; 329 330 for (i = 0; i < opts->num_pci_addr; i++) { 331 spdk_pci_addr_fmt(bdf, 32, &pci_addr[i]); 332 args = push_arg(args, &argcount, _sprintf_alloc("%s=%s", 333 (opts->pci_blocked ? DPDK_BLOCK_PARAM : DPDK_ALLOW_PARAM), 334 bdf)); 335 if (args == NULL) { 336 return -1; 337 } 338 } 339 } 340 341 /* Lower default EAL loglevel to RTE_LOG_NOTICE - normal, but significant messages. 342 * This can be overridden by specifying the same option in opts->env_context 343 */ 344 args = push_arg(args, &argcount, strdup("--log-level=lib.eal:6")); 345 if (args == NULL) { 346 return -1; 347 } 348 349 /* Lower default CRYPTO loglevel to RTE_LOG_ERR to avoid a ton of init msgs. 350 * This can be overridden by specifying the same option in opts->env_context 351 */ 352 args = push_arg(args, &argcount, strdup("--log-level=lib.cryptodev:5")); 353 if (args == NULL) { 354 return -1; 355 } 356 357 /* `user1` log type is used by rte_vhost, which prints an INFO log for each received 358 * vhost user message. We don't want that. The same log type is also used by a couple 359 * of other DPDK libs, but none of which we make use right now. If necessary, this can 360 * be overridden via opts->env_context. 361 */ 362 args = push_arg(args, &argcount, strdup("--log-level=user1:6")); 363 if (args == NULL) { 364 return -1; 365 } 366 367 if (opts->env_context) { 368 char *ptr = strdup(opts->env_context); 369 char *tok = strtok(ptr, " \t"); 370 371 /* DPDK expects each argument as a separate string in the argv 372 * array, so we need to tokenize here in case the caller 373 * passed multiple arguments in the env_context string. 374 */ 375 while (tok != NULL) { 376 args = push_arg(args, &argcount, strdup(tok)); 377 tok = strtok(NULL, " \t"); 378 } 379 380 free(ptr); 381 } 382 383 #ifdef __linux__ 384 385 if (opts->iova_mode) { 386 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=%s", opts->iova_mode)); 387 if (args == NULL) { 388 return -1; 389 } 390 } else { 391 /* When using vfio with enable_unsafe_noiommu_mode=Y, we need iova-mode=pa, 392 * but DPDK guesses it should be iova-mode=va. Add a check and force 393 * iova-mode=pa here. */ 394 if (rte_vfio_noiommu_is_enabled()) { 395 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 396 if (args == NULL) { 397 return -1; 398 } 399 } 400 401 #if defined(__x86_64__) 402 /* DPDK by default guesses that it should be using iova-mode=va so that it can 403 * support running as an unprivileged user. However, some systems (especially 404 * virtual machines) don't have an IOMMU capable of handling the full virtual 405 * address space and DPDK doesn't currently catch that. Add a check in SPDK 406 * and force iova-mode=pa here. */ 407 if (get_iommu_width() < SPDK_IOMMU_VA_REQUIRED_WIDTH) { 408 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 409 if (args == NULL) { 410 return -1; 411 } 412 } 413 #elif defined(__PPC64__) 414 /* On Linux + PowerPC, DPDK doesn't support VA mode at all. Unfortunately, it doesn't correctly 415 * auto-detect at the moment, so we'll just force it here. */ 416 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 417 if (args == NULL) { 418 return -1; 419 } 420 #endif 421 } 422 423 424 /* Set the base virtual address - it must be an address that is not in the 425 * ASAN shadow region, otherwise ASAN-enabled builds will ignore the 426 * mmap hint. 427 * 428 * Ref: https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm 429 */ 430 args = push_arg(args, &argcount, _sprintf_alloc("--base-virtaddr=0x%" PRIx64, opts->base_virtaddr)); 431 if (args == NULL) { 432 return -1; 433 } 434 435 /* --match-allocation prevents DPDK from merging or splitting system memory allocations under the hood. 436 * This is critical for RDMA when attempting to use an rte_mempool based buffer pool. If DPDK merges two 437 * physically or IOVA contiguous memory regions, then when we go to allocate a buffer pool, it can split 438 * the memory for a buffer over two allocations meaning the buffer will be split over a memory region. 439 */ 440 if (!opts->env_context || strstr(opts->env_context, "--legacy-mem") == NULL) { 441 args = push_arg(args, &argcount, _sprintf_alloc("%s", "--match-allocations")); 442 if (args == NULL) { 443 return -1; 444 } 445 } 446 447 if (opts->shm_id < 0) { 448 args = push_arg(args, &argcount, _sprintf_alloc("--file-prefix=spdk_pid%d", 449 getpid())); 450 if (args == NULL) { 451 return -1; 452 } 453 } else { 454 args = push_arg(args, &argcount, _sprintf_alloc("--file-prefix=spdk%d", 455 opts->shm_id)); 456 if (args == NULL) { 457 return -1; 458 } 459 460 /* set the process type */ 461 args = push_arg(args, &argcount, _sprintf_alloc("--proc-type=auto")); 462 if (args == NULL) { 463 return -1; 464 } 465 } 466 467 /* --vfio-vf-token used for VF initialized by vfio_pci driver. */ 468 if (opts->vf_token) { 469 args = push_arg(args, &argcount, _sprintf_alloc("--vfio-vf-token=%s", 470 opts->vf_token)); 471 if (args == NULL) { 472 return -1; 473 } 474 } 475 #endif 476 477 g_eal_cmdline = args; 478 g_eal_cmdline_argcount = argcount; 479 return argcount; 480 } 481 482 int 483 spdk_env_dpdk_post_init(bool legacy_mem) 484 { 485 int rc; 486 487 rc = pci_env_init(); 488 if (rc < 0) { 489 SPDK_ERRLOG("pci_env_init() failed\n"); 490 return rc; 491 } 492 493 rc = mem_map_init(legacy_mem); 494 if (rc < 0) { 495 SPDK_ERRLOG("Failed to allocate mem_map\n"); 496 return rc; 497 } 498 499 rc = vtophys_init(); 500 if (rc < 0) { 501 SPDK_ERRLOG("Failed to initialize vtophys\n"); 502 return rc; 503 } 504 505 return 0; 506 } 507 508 void 509 spdk_env_dpdk_post_fini(void) 510 { 511 pci_env_fini(); 512 513 free_args(g_eal_cmdline, g_eal_cmdline_argcount); 514 g_eal_cmdline = NULL; 515 g_eal_cmdline_argcount = 0; 516 } 517 518 int 519 spdk_env_init(const struct spdk_env_opts *opts) 520 { 521 char **dpdk_args = NULL; 522 int i, rc; 523 int orig_optind; 524 bool legacy_mem; 525 526 /* If SPDK env has been initialized before, then only pci env requires 527 * reinitialization. 528 */ 529 if (g_external_init == false) { 530 if (opts != NULL) { 531 fprintf(stderr, "Invalid arguments to reinitialize SPDK env\n"); 532 return -EINVAL; 533 } 534 535 printf("Starting %s / %s reinitialization...\n", SPDK_VERSION_STRING, rte_version()); 536 pci_env_reinit(); 537 538 return 0; 539 } 540 541 if (opts == NULL) { 542 fprintf(stderr, "NULL arguments to initialize DPDK\n"); 543 return -EINVAL; 544 } 545 546 rc = build_eal_cmdline(opts); 547 if (rc < 0) { 548 SPDK_ERRLOG("Invalid arguments to initialize DPDK\n"); 549 return -EINVAL; 550 } 551 552 SPDK_PRINTF("Starting %s / %s initialization...\n", SPDK_VERSION_STRING, rte_version()); 553 SPDK_PRINTF("[ DPDK EAL parameters: "); 554 for (i = 0; i < g_eal_cmdline_argcount; i++) { 555 SPDK_PRINTF("%s ", g_eal_cmdline[i]); 556 } 557 SPDK_PRINTF("]\n"); 558 559 /* DPDK rearranges the array we pass to it, so make a copy 560 * before passing so we can still free the individual strings 561 * correctly. 562 */ 563 dpdk_args = calloc(g_eal_cmdline_argcount, sizeof(char *)); 564 if (dpdk_args == NULL) { 565 SPDK_ERRLOG("Failed to allocate dpdk_args\n"); 566 return -ENOMEM; 567 } 568 memcpy(dpdk_args, g_eal_cmdline, sizeof(char *) * g_eal_cmdline_argcount); 569 570 fflush(stdout); 571 orig_optind = optind; 572 optind = 1; 573 rc = rte_eal_init(g_eal_cmdline_argcount, dpdk_args); 574 optind = orig_optind; 575 576 free(dpdk_args); 577 578 if (rc < 0) { 579 if (rte_errno == EALREADY) { 580 SPDK_ERRLOG("DPDK already initialized\n"); 581 } else { 582 SPDK_ERRLOG("Failed to initialize DPDK\n"); 583 } 584 return -rte_errno; 585 } 586 587 legacy_mem = false; 588 if (opts->env_context && strstr(opts->env_context, "--legacy-mem") != NULL) { 589 legacy_mem = true; 590 } 591 592 rc = spdk_env_dpdk_post_init(legacy_mem); 593 if (rc == 0) { 594 g_external_init = false; 595 } 596 597 return rc; 598 } 599 600 /* We use priority 101 which is the highest priority level available 601 * to applications (the toolchains reserve 1 to 100 for internal usage). 602 * This ensures this destructor runs last, after any other destructors 603 * that might still need the environment up and running. 604 */ 605 __attribute__((destructor(101))) static void 606 dpdk_cleanup(void) 607 { 608 /* Only call rte_eal_cleanup if the SPDK env library called rte_eal_init. */ 609 if (!g_external_init) { 610 rte_eal_cleanup(); 611 } 612 } 613 614 void 615 spdk_env_fini(void) 616 { 617 spdk_env_dpdk_post_fini(); 618 } 619 620 bool 621 spdk_env_dpdk_external_init(void) 622 { 623 return g_external_init; 624 } 625