1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "env_internal.h" 9 10 #include "spdk/version.h" 11 #include "spdk/env_dpdk.h" 12 #include "spdk/log.h" 13 14 #include <rte_config.h> 15 #include <rte_eal.h> 16 #include <rte_errno.h> 17 #include <rte_vfio.h> 18 19 #define SPDK_ENV_DPDK_DEFAULT_NAME "spdk" 20 #define SPDK_ENV_DPDK_DEFAULT_SHM_ID -1 21 #define SPDK_ENV_DPDK_DEFAULT_MEM_SIZE -1 22 #define SPDK_ENV_DPDK_DEFAULT_MAIN_CORE -1 23 #define SPDK_ENV_DPDK_DEFAULT_MEM_CHANNEL -1 24 #define SPDK_ENV_DPDK_DEFAULT_CORE_MASK "0x1" 25 #define SPDK_ENV_DPDK_DEFAULT_BASE_VIRTADDR 0x200000000000 26 27 #if RTE_VERSION < RTE_VERSION_NUM(20, 11, 0, 0) 28 #define DPDK_ALLOW_PARAM "--pci-whitelist" 29 #define DPDK_BLOCK_PARAM "--pci-blacklist" 30 #define DPDK_MAIN_CORE_PARAM "--master-lcore" 31 #else 32 #define DPDK_ALLOW_PARAM "--allow" 33 #define DPDK_BLOCK_PARAM "--block" 34 #define DPDK_MAIN_CORE_PARAM "--main-lcore" 35 #endif 36 37 static char **g_eal_cmdline; 38 static int g_eal_cmdline_argcount; 39 static bool g_external_init = true; 40 41 static char * 42 _sprintf_alloc(const char *format, ...) 43 { 44 va_list args; 45 va_list args_copy; 46 char *buf; 47 size_t bufsize; 48 int rc; 49 50 va_start(args, format); 51 52 /* Try with a small buffer first. */ 53 bufsize = 32; 54 55 /* Limit maximum buffer size to something reasonable so we don't loop forever. */ 56 while (bufsize <= 1024 * 1024) { 57 buf = malloc(bufsize); 58 if (buf == NULL) { 59 va_end(args); 60 return NULL; 61 } 62 63 va_copy(args_copy, args); 64 rc = vsnprintf(buf, bufsize, format, args_copy); 65 va_end(args_copy); 66 67 /* 68 * If vsnprintf() returned a count within our current buffer size, we are done. 69 * The count does not include the \0 terminator, so rc == bufsize is not OK. 70 */ 71 if (rc >= 0 && (size_t)rc < bufsize) { 72 va_end(args); 73 return buf; 74 } 75 76 /* 77 * vsnprintf() should return the required space, but some libc versions do not 78 * implement this correctly, so just double the buffer size and try again. 79 * 80 * We don't need the data in buf, so rather than realloc(), use free() and malloc() 81 * again to avoid a copy. 82 */ 83 free(buf); 84 bufsize *= 2; 85 } 86 87 va_end(args); 88 return NULL; 89 } 90 91 void 92 spdk_env_opts_init(struct spdk_env_opts *opts) 93 { 94 if (!opts) { 95 return; 96 } 97 98 memset(opts, 0, sizeof(*opts)); 99 100 opts->name = SPDK_ENV_DPDK_DEFAULT_NAME; 101 opts->core_mask = SPDK_ENV_DPDK_DEFAULT_CORE_MASK; 102 opts->shm_id = SPDK_ENV_DPDK_DEFAULT_SHM_ID; 103 opts->mem_size = SPDK_ENV_DPDK_DEFAULT_MEM_SIZE; 104 opts->main_core = SPDK_ENV_DPDK_DEFAULT_MAIN_CORE; 105 opts->mem_channel = SPDK_ENV_DPDK_DEFAULT_MEM_CHANNEL; 106 opts->base_virtaddr = SPDK_ENV_DPDK_DEFAULT_BASE_VIRTADDR; 107 } 108 109 static void 110 free_args(char **args, int argcount) 111 { 112 int i; 113 114 if (args == NULL) { 115 return; 116 } 117 118 for (i = 0; i < argcount; i++) { 119 free(args[i]); 120 } 121 122 if (argcount) { 123 free(args); 124 } 125 } 126 127 static char ** 128 push_arg(char *args[], int *argcount, char *arg) 129 { 130 char **tmp; 131 132 if (arg == NULL) { 133 SPDK_ERRLOG("%s: NULL arg supplied\n", __func__); 134 free_args(args, *argcount); 135 return NULL; 136 } 137 138 tmp = realloc(args, sizeof(char *) * (*argcount + 1)); 139 if (tmp == NULL) { 140 free(arg); 141 free_args(args, *argcount); 142 return NULL; 143 } 144 145 tmp[*argcount] = arg; 146 (*argcount)++; 147 148 return tmp; 149 } 150 151 #if defined(__linux__) && defined(__x86_64__) 152 153 /* TODO: Can likely get this value from rlimits in the future */ 154 #define SPDK_IOMMU_VA_REQUIRED_WIDTH 48 155 #define VTD_CAP_MGAW_SHIFT 16 156 #define VTD_CAP_MGAW_MASK (0x3F << VTD_CAP_MGAW_SHIFT) 157 #define RD_AMD_CAP_VASIZE_SHIFT 15 158 #define RD_AMD_CAP_VASIZE_MASK (0x7F << RD_AMD_CAP_VASIZE_SHIFT) 159 160 static int 161 get_iommu_width(void) 162 { 163 int width = 0; 164 glob_t glob_results = {}; 165 166 /* Break * and / into separate strings to appease check_format.sh comment style check. */ 167 glob("/sys/devices/virtual/iommu/dmar*" "/intel-iommu/cap", 0, NULL, &glob_results); 168 glob("/sys/class/iommu/ivhd*" "/amd-iommu/cap", GLOB_APPEND, NULL, &glob_results); 169 170 for (size_t i = 0; i < glob_results.gl_pathc; i++) { 171 const char *filename = glob_results.gl_pathv[0]; 172 FILE *file = fopen(filename, "r"); 173 uint64_t cap_reg = 0; 174 175 if (file == NULL) { 176 continue; 177 } 178 179 if (fscanf(file, "%" PRIx64, &cap_reg) == 1) { 180 if (strstr(filename, "intel-iommu") != NULL) { 181 /* We have an Intel IOMMU */ 182 int mgaw = ((cap_reg & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1; 183 184 if (width == 0 || (mgaw > 0 && mgaw < width)) { 185 width = mgaw; 186 } 187 } else if (strstr(filename, "amd-iommu") != NULL) { 188 /* We have an AMD IOMMU */ 189 int mgaw = ((cap_reg & RD_AMD_CAP_VASIZE_MASK) >> RD_AMD_CAP_VASIZE_SHIFT) + 1; 190 191 if (width == 0 || (mgaw > 0 && mgaw < width)) { 192 width = mgaw; 193 } 194 } 195 } 196 197 fclose(file); 198 } 199 200 globfree(&glob_results); 201 return width; 202 } 203 204 #endif 205 206 static int 207 build_eal_cmdline(const struct spdk_env_opts *opts) 208 { 209 int argcount = 0; 210 char **args; 211 212 args = NULL; 213 214 /* set the program name */ 215 args = push_arg(args, &argcount, _sprintf_alloc("%s", opts->name)); 216 if (args == NULL) { 217 return -1; 218 } 219 220 /* disable shared configuration files when in single process mode. This allows for cleaner shutdown */ 221 if (opts->shm_id < 0) { 222 args = push_arg(args, &argcount, _sprintf_alloc("%s", "--no-shconf")); 223 if (args == NULL) { 224 return -1; 225 } 226 } 227 228 /* 229 * Set the coremask: 230 * 231 * - if it starts with '-', we presume it's literal EAL arguments such 232 * as --lcores. 233 * 234 * - if it starts with '[', we presume it's a core list to use with the 235 * -l option. 236 * 237 * - otherwise, it's a CPU mask of the form "0xff.." as expected by the 238 * -c option. 239 */ 240 if (opts->core_mask[0] == '-') { 241 args = push_arg(args, &argcount, _sprintf_alloc("%s", opts->core_mask)); 242 } else if (opts->core_mask[0] == '[') { 243 char *l_arg = _sprintf_alloc("-l %s", opts->core_mask + 1); 244 245 if (l_arg != NULL) { 246 int len = strlen(l_arg); 247 248 if (l_arg[len - 1] == ']') { 249 l_arg[len - 1] = '\0'; 250 } 251 } 252 args = push_arg(args, &argcount, l_arg); 253 } else { 254 args = push_arg(args, &argcount, _sprintf_alloc("-c %s", opts->core_mask)); 255 } 256 257 if (args == NULL) { 258 return -1; 259 } 260 261 /* set the memory channel number */ 262 if (opts->mem_channel > 0) { 263 args = push_arg(args, &argcount, _sprintf_alloc("-n %d", opts->mem_channel)); 264 if (args == NULL) { 265 return -1; 266 } 267 } 268 269 /* set the memory size */ 270 if (opts->mem_size >= 0) { 271 args = push_arg(args, &argcount, _sprintf_alloc("-m %d", opts->mem_size)); 272 if (args == NULL) { 273 return -1; 274 } 275 } 276 277 /* set the main core */ 278 if (opts->main_core > 0) { 279 args = push_arg(args, &argcount, _sprintf_alloc("%s=%d", 280 DPDK_MAIN_CORE_PARAM, opts->main_core)); 281 if (args == NULL) { 282 return -1; 283 } 284 } 285 286 /* set no pci if enabled */ 287 if (opts->no_pci) { 288 args = push_arg(args, &argcount, _sprintf_alloc("--no-pci")); 289 if (args == NULL) { 290 return -1; 291 } 292 } 293 294 if (opts->env_context && strstr(opts->env_context, "--no-huge") != NULL) { 295 if (opts->hugepage_single_segments || opts->unlink_hugepage || opts->hugedir) { 296 fprintf(stderr, "--no-huge invalid with other hugepage options\n"); 297 free_args(args, argcount); 298 return -1; 299 } 300 } else { 301 /* create just one hugetlbfs file */ 302 if (opts->hugepage_single_segments) { 303 args = push_arg(args, &argcount, _sprintf_alloc("--single-file-segments")); 304 if (args == NULL) { 305 return -1; 306 } 307 } 308 309 /* unlink hugepages after initialization */ 310 /* Note: Automatically unlink hugepage when shm_id < 0, since it means we're not using 311 * multi-process so we don't need the hugepage links anymore. But we need to make sure 312 * we don't specify --huge-unlink implicitly if --single-file-segments was specified since 313 * DPDK doesn't support that. 314 */ 315 if (opts->unlink_hugepage || 316 (opts->shm_id < 0 && !opts->hugepage_single_segments)) { 317 args = push_arg(args, &argcount, _sprintf_alloc("--huge-unlink")); 318 if (args == NULL) { 319 return -1; 320 } 321 } 322 323 /* use a specific hugetlbfs mount */ 324 if (opts->hugedir) { 325 args = push_arg(args, &argcount, _sprintf_alloc("--huge-dir=%s", opts->hugedir)); 326 if (args == NULL) { 327 return -1; 328 } 329 } 330 } 331 332 if (opts->num_pci_addr) { 333 size_t i; 334 char bdf[32]; 335 struct spdk_pci_addr *pci_addr = 336 opts->pci_blocked ? opts->pci_blocked : opts->pci_allowed; 337 338 for (i = 0; i < opts->num_pci_addr; i++) { 339 spdk_pci_addr_fmt(bdf, 32, &pci_addr[i]); 340 args = push_arg(args, &argcount, _sprintf_alloc("%s=%s", 341 (opts->pci_blocked ? DPDK_BLOCK_PARAM : DPDK_ALLOW_PARAM), 342 bdf)); 343 if (args == NULL) { 344 return -1; 345 } 346 } 347 } 348 349 /* Lower default EAL loglevel to RTE_LOG_NOTICE - normal, but significant messages. 350 * This can be overridden by specifying the same option in opts->env_context 351 */ 352 args = push_arg(args, &argcount, strdup("--log-level=lib.eal:6")); 353 if (args == NULL) { 354 return -1; 355 } 356 357 /* Lower default CRYPTO loglevel to RTE_LOG_ERR to avoid a ton of init msgs. 358 * This can be overridden by specifying the same option in opts->env_context 359 */ 360 args = push_arg(args, &argcount, strdup("--log-level=lib.cryptodev:5")); 361 if (args == NULL) { 362 return -1; 363 } 364 365 /* `user1` log type is used by rte_vhost, which prints an INFO log for each received 366 * vhost user message. We don't want that. The same log type is also used by a couple 367 * of other DPDK libs, but none of which we make use right now. If necessary, this can 368 * be overridden via opts->env_context. 369 */ 370 args = push_arg(args, &argcount, strdup("--log-level=user1:6")); 371 if (args == NULL) { 372 return -1; 373 } 374 375 if (opts->env_context) { 376 char *ptr = strdup(opts->env_context); 377 char *tok = strtok(ptr, " \t"); 378 379 /* DPDK expects each argument as a separate string in the argv 380 * array, so we need to tokenize here in case the caller 381 * passed multiple arguments in the env_context string. 382 */ 383 while (tok != NULL) { 384 args = push_arg(args, &argcount, strdup(tok)); 385 tok = strtok(NULL, " \t"); 386 } 387 388 free(ptr); 389 } 390 391 #ifdef __linux__ 392 393 if (opts->iova_mode) { 394 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=%s", opts->iova_mode)); 395 if (args == NULL) { 396 return -1; 397 } 398 } else { 399 /* When using vfio with enable_unsafe_noiommu_mode=Y, we need iova-mode=pa, 400 * but DPDK guesses it should be iova-mode=va. Add a check and force 401 * iova-mode=pa here. */ 402 if (rte_vfio_noiommu_is_enabled()) { 403 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 404 if (args == NULL) { 405 return -1; 406 } 407 } 408 409 #if defined(__x86_64__) 410 /* DPDK by default guesses that it should be using iova-mode=va so that it can 411 * support running as an unprivileged user. However, some systems (especially 412 * virtual machines) don't have an IOMMU capable of handling the full virtual 413 * address space and DPDK doesn't currently catch that. Add a check in SPDK 414 * and force iova-mode=pa here. */ 415 if (get_iommu_width() < SPDK_IOMMU_VA_REQUIRED_WIDTH) { 416 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 417 if (args == NULL) { 418 return -1; 419 } 420 } 421 #elif defined(__PPC64__) 422 /* On Linux + PowerPC, DPDK doesn't support VA mode at all. Unfortunately, it doesn't correctly 423 * auto-detect at the moment, so we'll just force it here. */ 424 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 425 if (args == NULL) { 426 return -1; 427 } 428 #endif 429 } 430 431 432 /* Set the base virtual address - it must be an address that is not in the 433 * ASAN shadow region, otherwise ASAN-enabled builds will ignore the 434 * mmap hint. 435 * 436 * Ref: https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm 437 */ 438 args = push_arg(args, &argcount, _sprintf_alloc("--base-virtaddr=0x%" PRIx64, opts->base_virtaddr)); 439 if (args == NULL) { 440 return -1; 441 } 442 443 /* --match-allocation prevents DPDK from merging or splitting system memory allocations under the hood. 444 * This is critical for RDMA when attempting to use an rte_mempool based buffer pool. If DPDK merges two 445 * physically or IOVA contiguous memory regions, then when we go to allocate a buffer pool, it can split 446 * the memory for a buffer over two allocations meaning the buffer will be split over a memory region. 447 */ 448 if (!opts->env_context || strstr(opts->env_context, "--legacy-mem") == NULL) { 449 args = push_arg(args, &argcount, _sprintf_alloc("%s", "--match-allocations")); 450 if (args == NULL) { 451 return -1; 452 } 453 } 454 455 if (opts->shm_id < 0) { 456 args = push_arg(args, &argcount, _sprintf_alloc("--file-prefix=spdk_pid%d", 457 getpid())); 458 if (args == NULL) { 459 return -1; 460 } 461 } else { 462 args = push_arg(args, &argcount, _sprintf_alloc("--file-prefix=spdk%d", 463 opts->shm_id)); 464 if (args == NULL) { 465 return -1; 466 } 467 468 /* set the process type */ 469 args = push_arg(args, &argcount, _sprintf_alloc("--proc-type=auto")); 470 if (args == NULL) { 471 return -1; 472 } 473 } 474 475 /* --vfio-vf-token used for VF initialized by vfio_pci driver. */ 476 if (opts->vf_token) { 477 args = push_arg(args, &argcount, _sprintf_alloc("--vfio-vf-token=%s", 478 opts->vf_token)); 479 if (args == NULL) { 480 return -1; 481 } 482 } 483 #endif 484 485 g_eal_cmdline = args; 486 g_eal_cmdline_argcount = argcount; 487 return argcount; 488 } 489 490 int 491 spdk_env_dpdk_post_init(bool legacy_mem) 492 { 493 int rc; 494 495 rc = pci_env_init(); 496 if (rc < 0) { 497 SPDK_ERRLOG("pci_env_init() failed\n"); 498 return rc; 499 } 500 501 rc = mem_map_init(legacy_mem); 502 if (rc < 0) { 503 SPDK_ERRLOG("Failed to allocate mem_map\n"); 504 return rc; 505 } 506 507 rc = vtophys_init(); 508 if (rc < 0) { 509 SPDK_ERRLOG("Failed to initialize vtophys\n"); 510 return rc; 511 } 512 513 return 0; 514 } 515 516 void 517 spdk_env_dpdk_post_fini(void) 518 { 519 pci_env_fini(); 520 521 free_args(g_eal_cmdline, g_eal_cmdline_argcount); 522 g_eal_cmdline = NULL; 523 g_eal_cmdline_argcount = 0; 524 } 525 526 int 527 spdk_env_init(const struct spdk_env_opts *opts) 528 { 529 char **dpdk_args = NULL; 530 char *args_print = NULL, *args_tmp = NULL; 531 int i, rc; 532 int orig_optind; 533 bool legacy_mem; 534 535 /* If SPDK env has been initialized before, then only pci env requires 536 * reinitialization. 537 */ 538 if (g_external_init == false) { 539 if (opts != NULL) { 540 fprintf(stderr, "Invalid arguments to reinitialize SPDK env\n"); 541 return -EINVAL; 542 } 543 544 printf("Starting %s / %s reinitialization...\n", SPDK_VERSION_STRING, rte_version()); 545 pci_env_reinit(); 546 547 return 0; 548 } 549 550 if (opts == NULL) { 551 fprintf(stderr, "NULL arguments to initialize DPDK\n"); 552 return -EINVAL; 553 } 554 555 rc = build_eal_cmdline(opts); 556 if (rc < 0) { 557 SPDK_ERRLOG("Invalid arguments to initialize DPDK\n"); 558 return -EINVAL; 559 } 560 561 SPDK_PRINTF("Starting %s / %s initialization...\n", SPDK_VERSION_STRING, rte_version()); 562 563 args_print = _sprintf_alloc("[ DPDK EAL parameters: "); 564 if (args_print == NULL) { 565 return -ENOMEM; 566 } 567 for (i = 0; i < g_eal_cmdline_argcount; i++) { 568 args_tmp = args_print; 569 args_print = _sprintf_alloc("%s%s ", args_tmp, g_eal_cmdline[i]); 570 if (args_print == NULL) { 571 free(args_tmp); 572 return -ENOMEM; 573 } 574 free(args_tmp); 575 } 576 SPDK_PRINTF("%s]\n", args_print); 577 free(args_print); 578 579 /* DPDK rearranges the array we pass to it, so make a copy 580 * before passing so we can still free the individual strings 581 * correctly. 582 */ 583 dpdk_args = calloc(g_eal_cmdline_argcount, sizeof(char *)); 584 if (dpdk_args == NULL) { 585 SPDK_ERRLOG("Failed to allocate dpdk_args\n"); 586 return -ENOMEM; 587 } 588 memcpy(dpdk_args, g_eal_cmdline, sizeof(char *) * g_eal_cmdline_argcount); 589 590 fflush(stdout); 591 orig_optind = optind; 592 optind = 1; 593 rc = rte_eal_init(g_eal_cmdline_argcount, dpdk_args); 594 optind = orig_optind; 595 596 free(dpdk_args); 597 598 if (rc < 0) { 599 if (rte_errno == EALREADY) { 600 SPDK_ERRLOG("DPDK already initialized\n"); 601 } else { 602 SPDK_ERRLOG("Failed to initialize DPDK\n"); 603 } 604 return -rte_errno; 605 } 606 607 legacy_mem = false; 608 if (opts->env_context && strstr(opts->env_context, "--legacy-mem") != NULL) { 609 legacy_mem = true; 610 } 611 612 rc = spdk_env_dpdk_post_init(legacy_mem); 613 if (rc == 0) { 614 g_external_init = false; 615 } 616 617 return rc; 618 } 619 620 /* We use priority 101 which is the highest priority level available 621 * to applications (the toolchains reserve 1 to 100 for internal usage). 622 * This ensures this destructor runs last, after any other destructors 623 * that might still need the environment up and running. 624 */ 625 __attribute__((destructor(101))) static void 626 dpdk_cleanup(void) 627 { 628 /* Only call rte_eal_cleanup if the SPDK env library called rte_eal_init. */ 629 if (!g_external_init) { 630 rte_eal_cleanup(); 631 } 632 } 633 634 void 635 spdk_env_fini(void) 636 { 637 spdk_env_dpdk_post_fini(); 638 } 639 640 bool 641 spdk_env_dpdk_external_init(void) 642 { 643 return g_external_init; 644 } 645