1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "env_internal.h" 37 38 #include "spdk/version.h" 39 #include "spdk/env_dpdk.h" 40 #include "spdk/log.h" 41 42 #include <rte_config.h> 43 #include <rte_eal.h> 44 #include <rte_errno.h> 45 #include <rte_vfio.h> 46 47 #define SPDK_ENV_DPDK_DEFAULT_NAME "spdk" 48 #define SPDK_ENV_DPDK_DEFAULT_SHM_ID -1 49 #define SPDK_ENV_DPDK_DEFAULT_MEM_SIZE -1 50 #define SPDK_ENV_DPDK_DEFAULT_MAIN_CORE -1 51 #define SPDK_ENV_DPDK_DEFAULT_MEM_CHANNEL -1 52 #define SPDK_ENV_DPDK_DEFAULT_CORE_MASK "0x1" 53 #define SPDK_ENV_DPDK_DEFAULT_BASE_VIRTADDR 0x200000000000 54 55 #if RTE_VERSION < RTE_VERSION_NUM(20, 11, 0, 0) 56 #define DPDK_ALLOW_PARAM "--pci-whitelist" 57 #define DPDK_BLOCK_PARAM "--pci-blacklist" 58 #define DPDK_MAIN_CORE_PARAM "--master-lcore" 59 #else 60 #define DPDK_ALLOW_PARAM "--allow" 61 #define DPDK_BLOCK_PARAM "--block" 62 #define DPDK_MAIN_CORE_PARAM "--main-lcore" 63 #endif 64 65 static char **g_eal_cmdline; 66 static int g_eal_cmdline_argcount; 67 static bool g_external_init = true; 68 69 static char * 70 _sprintf_alloc(const char *format, ...) 71 { 72 va_list args; 73 va_list args_copy; 74 char *buf; 75 size_t bufsize; 76 int rc; 77 78 va_start(args, format); 79 80 /* Try with a small buffer first. */ 81 bufsize = 32; 82 83 /* Limit maximum buffer size to something reasonable so we don't loop forever. */ 84 while (bufsize <= 1024 * 1024) { 85 buf = malloc(bufsize); 86 if (buf == NULL) { 87 va_end(args); 88 return NULL; 89 } 90 91 va_copy(args_copy, args); 92 rc = vsnprintf(buf, bufsize, format, args_copy); 93 va_end(args_copy); 94 95 /* 96 * If vsnprintf() returned a count within our current buffer size, we are done. 97 * The count does not include the \0 terminator, so rc == bufsize is not OK. 98 */ 99 if (rc >= 0 && (size_t)rc < bufsize) { 100 va_end(args); 101 return buf; 102 } 103 104 /* 105 * vsnprintf() should return the required space, but some libc versions do not 106 * implement this correctly, so just double the buffer size and try again. 107 * 108 * We don't need the data in buf, so rather than realloc(), use free() and malloc() 109 * again to avoid a copy. 110 */ 111 free(buf); 112 bufsize *= 2; 113 } 114 115 va_end(args); 116 return NULL; 117 } 118 119 void 120 spdk_env_opts_init(struct spdk_env_opts *opts) 121 { 122 if (!opts) { 123 return; 124 } 125 126 memset(opts, 0, sizeof(*opts)); 127 128 opts->name = SPDK_ENV_DPDK_DEFAULT_NAME; 129 opts->core_mask = SPDK_ENV_DPDK_DEFAULT_CORE_MASK; 130 opts->shm_id = SPDK_ENV_DPDK_DEFAULT_SHM_ID; 131 opts->mem_size = SPDK_ENV_DPDK_DEFAULT_MEM_SIZE; 132 opts->main_core = SPDK_ENV_DPDK_DEFAULT_MAIN_CORE; 133 opts->mem_channel = SPDK_ENV_DPDK_DEFAULT_MEM_CHANNEL; 134 opts->base_virtaddr = SPDK_ENV_DPDK_DEFAULT_BASE_VIRTADDR; 135 } 136 137 static void 138 free_args(char **args, int argcount) 139 { 140 int i; 141 142 if (args == NULL) { 143 return; 144 } 145 146 for (i = 0; i < argcount; i++) { 147 free(args[i]); 148 } 149 150 if (argcount) { 151 free(args); 152 } 153 } 154 155 static char ** 156 push_arg(char *args[], int *argcount, char *arg) 157 { 158 char **tmp; 159 160 if (arg == NULL) { 161 SPDK_ERRLOG("%s: NULL arg supplied\n", __func__); 162 free_args(args, *argcount); 163 return NULL; 164 } 165 166 tmp = realloc(args, sizeof(char *) * (*argcount + 1)); 167 if (tmp == NULL) { 168 free(arg); 169 free_args(args, *argcount); 170 return NULL; 171 } 172 173 tmp[*argcount] = arg; 174 (*argcount)++; 175 176 return tmp; 177 } 178 179 #if defined(__linux__) && defined(__x86_64__) 180 181 /* TODO: Can likely get this value from rlimits in the future */ 182 #define SPDK_IOMMU_VA_REQUIRED_WIDTH 48 183 #define VTD_CAP_MGAW_SHIFT 16 184 #define VTD_CAP_MGAW_MASK (0x3F << VTD_CAP_MGAW_SHIFT) 185 186 static int 187 get_iommu_width(void) 188 { 189 DIR *dir; 190 FILE *file; 191 struct dirent *entry; 192 char mgaw_path[64]; 193 char buf[64]; 194 char *end; 195 long long int val; 196 int width, tmp; 197 198 dir = opendir("/sys/devices/virtual/iommu/"); 199 if (dir == NULL) { 200 return -EINVAL; 201 } 202 203 width = 0; 204 205 while ((entry = readdir(dir)) != NULL) { 206 /* Find directories named "dmar0", "dmar1", etc */ 207 if (strncmp(entry->d_name, "dmar", sizeof("dmar") - 1) != 0) { 208 continue; 209 } 210 211 tmp = snprintf(mgaw_path, sizeof(mgaw_path), "/sys/devices/virtual/iommu/%s/intel-iommu/cap", 212 entry->d_name); 213 if ((unsigned)tmp >= sizeof(mgaw_path)) { 214 continue; 215 } 216 217 file = fopen(mgaw_path, "r"); 218 if (file == NULL) { 219 continue; 220 } 221 222 if (fgets(buf, sizeof(buf), file) == NULL) { 223 fclose(file); 224 continue; 225 } 226 227 val = strtoll(buf, &end, 16); 228 if (val == LLONG_MIN || val == LLONG_MAX) { 229 fclose(file); 230 continue; 231 } 232 233 tmp = ((val & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1; 234 if (width == 0 || tmp < width) { 235 width = tmp; 236 } 237 238 fclose(file); 239 } 240 241 closedir(dir); 242 243 return width; 244 } 245 246 #endif 247 248 static int 249 build_eal_cmdline(const struct spdk_env_opts *opts) 250 { 251 int argcount = 0; 252 char **args; 253 254 args = NULL; 255 256 /* set the program name */ 257 args = push_arg(args, &argcount, _sprintf_alloc("%s", opts->name)); 258 if (args == NULL) { 259 return -1; 260 } 261 262 /* disable shared configuration files when in single process mode. This allows for cleaner shutdown */ 263 if (opts->shm_id < 0) { 264 args = push_arg(args, &argcount, _sprintf_alloc("%s", "--no-shconf")); 265 if (args == NULL) { 266 return -1; 267 } 268 } 269 270 /* 271 * Set the coremask: 272 * 273 * - if it starts with '-', we presume it's literal EAL arguments such 274 * as --lcores. 275 * 276 * - if it starts with '[', we presume it's a core list to use with the 277 * -l option. 278 * 279 * - otherwise, it's a CPU mask of the form "0xff.." as expected by the 280 * -c option. 281 */ 282 if (opts->core_mask[0] == '-') { 283 args = push_arg(args, &argcount, _sprintf_alloc("%s", opts->core_mask)); 284 } else if (opts->core_mask[0] == '[') { 285 char *l_arg = _sprintf_alloc("-l %s", opts->core_mask + 1); 286 287 if (l_arg != NULL) { 288 int len = strlen(l_arg); 289 290 if (l_arg[len - 1] == ']') { 291 l_arg[len - 1] = '\0'; 292 } 293 } 294 args = push_arg(args, &argcount, l_arg); 295 } else { 296 args = push_arg(args, &argcount, _sprintf_alloc("-c %s", opts->core_mask)); 297 } 298 299 if (args == NULL) { 300 return -1; 301 } 302 303 /* set the memory channel number */ 304 if (opts->mem_channel > 0) { 305 args = push_arg(args, &argcount, _sprintf_alloc("-n %d", opts->mem_channel)); 306 if (args == NULL) { 307 return -1; 308 } 309 } 310 311 /* set the memory size */ 312 if (opts->mem_size >= 0) { 313 args = push_arg(args, &argcount, _sprintf_alloc("-m %d", opts->mem_size)); 314 if (args == NULL) { 315 return -1; 316 } 317 } 318 319 /* set the main core */ 320 if (opts->main_core > 0) { 321 args = push_arg(args, &argcount, _sprintf_alloc("%s=%d", 322 DPDK_MAIN_CORE_PARAM, opts->main_core)); 323 if (args == NULL) { 324 return -1; 325 } 326 } 327 328 /* set no pci if enabled */ 329 if (opts->no_pci) { 330 args = push_arg(args, &argcount, _sprintf_alloc("--no-pci")); 331 if (args == NULL) { 332 return -1; 333 } 334 } 335 336 /* create just one hugetlbfs file */ 337 if (opts->hugepage_single_segments) { 338 args = push_arg(args, &argcount, _sprintf_alloc("--single-file-segments")); 339 if (args == NULL) { 340 return -1; 341 } 342 } 343 344 /* unlink hugepages after initialization */ 345 if (opts->unlink_hugepage) { 346 args = push_arg(args, &argcount, _sprintf_alloc("--huge-unlink")); 347 if (args == NULL) { 348 return -1; 349 } 350 } 351 352 /* use a specific hugetlbfs mount */ 353 if (opts->hugedir) { 354 args = push_arg(args, &argcount, _sprintf_alloc("--huge-dir=%s", opts->hugedir)); 355 if (args == NULL) { 356 return -1; 357 } 358 } 359 360 if (opts->num_pci_addr) { 361 size_t i; 362 char bdf[32]; 363 struct spdk_pci_addr *pci_addr = 364 opts->pci_blocked ? opts->pci_blocked : opts->pci_allowed; 365 366 for (i = 0; i < opts->num_pci_addr; i++) { 367 spdk_pci_addr_fmt(bdf, 32, &pci_addr[i]); 368 args = push_arg(args, &argcount, _sprintf_alloc("%s=%s", 369 (opts->pci_blocked ? DPDK_BLOCK_PARAM : DPDK_ALLOW_PARAM), 370 bdf)); 371 if (args == NULL) { 372 return -1; 373 } 374 } 375 } 376 377 /* Lower default EAL loglevel to RTE_LOG_NOTICE - normal, but significant messages. 378 * This can be overridden by specifying the same option in opts->env_context 379 */ 380 args = push_arg(args, &argcount, strdup("--log-level=lib.eal:6")); 381 if (args == NULL) { 382 return -1; 383 } 384 385 /* Lower default CRYPTO loglevel to RTE_LOG_ERR to avoid a ton of init msgs. 386 * This can be overridden by specifying the same option in opts->env_context 387 */ 388 args = push_arg(args, &argcount, strdup("--log-level=lib.cryptodev:5")); 389 if (args == NULL) { 390 return -1; 391 } 392 393 /* `user1` log type is used by rte_vhost, which prints an INFO log for each received 394 * vhost user message. We don't want that. The same log type is also used by a couple 395 * of other DPDK libs, but none of which we make use right now. If necessary, this can 396 * be overridden via opts->env_context. 397 */ 398 args = push_arg(args, &argcount, strdup("--log-level=user1:6")); 399 if (args == NULL) { 400 return -1; 401 } 402 403 if (opts->env_context) { 404 char *ptr = strdup(opts->env_context); 405 char *tok = strtok(ptr, " \t"); 406 407 /* DPDK expects each argument as a separate string in the argv 408 * array, so we need to tokenize here in case the caller 409 * passed multiple arguments in the env_context string. 410 */ 411 while (tok != NULL) { 412 args = push_arg(args, &argcount, strdup(tok)); 413 tok = strtok(NULL, " \t"); 414 } 415 416 free(ptr); 417 } 418 419 #ifdef __linux__ 420 421 if (opts->iova_mode) { 422 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=%s", opts->iova_mode)); 423 if (args == NULL) { 424 return -1; 425 } 426 } else { 427 /* When using vfio with enable_unsafe_noiommu_mode=Y, we need iova-mode=pa, 428 * but DPDK guesses it should be iova-mode=va. Add a check and force 429 * iova-mode=pa here. */ 430 if (rte_vfio_noiommu_is_enabled()) { 431 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 432 if (args == NULL) { 433 return -1; 434 } 435 } 436 437 #if defined(__x86_64__) 438 /* DPDK by default guesses that it should be using iova-mode=va so that it can 439 * support running as an unprivileged user. However, some systems (especially 440 * virtual machines) don't have an IOMMU capable of handling the full virtual 441 * address space and DPDK doesn't currently catch that. Add a check in SPDK 442 * and force iova-mode=pa here. */ 443 if (get_iommu_width() < SPDK_IOMMU_VA_REQUIRED_WIDTH) { 444 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 445 if (args == NULL) { 446 return -1; 447 } 448 } 449 #elif defined(__PPC64__) 450 /* On Linux + PowerPC, DPDK doesn't support VA mode at all. Unfortunately, it doesn't correctly 451 * auto-detect at the moment, so we'll just force it here. */ 452 args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); 453 if (args == NULL) { 454 return -1; 455 } 456 #endif 457 } 458 459 460 /* Set the base virtual address - it must be an address that is not in the 461 * ASAN shadow region, otherwise ASAN-enabled builds will ignore the 462 * mmap hint. 463 * 464 * Ref: https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm 465 */ 466 args = push_arg(args, &argcount, _sprintf_alloc("--base-virtaddr=0x%" PRIx64, opts->base_virtaddr)); 467 if (args == NULL) { 468 return -1; 469 } 470 471 /* --match-allocation prevents DPDK from merging or splitting system memory allocations under the hood. 472 * This is critical for RDMA when attempting to use an rte_mempool based buffer pool. If DPDK merges two 473 * physically or IOVA contiguous memory regions, then when we go to allocate a buffer pool, it can split 474 * the memory for a buffer over two allocations meaning the buffer will be split over a memory region. 475 */ 476 if (!opts->env_context || strstr(opts->env_context, "--legacy-mem") == NULL) { 477 args = push_arg(args, &argcount, _sprintf_alloc("%s", "--match-allocations")); 478 if (args == NULL) { 479 return -1; 480 } 481 } 482 483 if (opts->shm_id < 0) { 484 args = push_arg(args, &argcount, _sprintf_alloc("--file-prefix=spdk_pid%d", 485 getpid())); 486 if (args == NULL) { 487 return -1; 488 } 489 } else { 490 args = push_arg(args, &argcount, _sprintf_alloc("--file-prefix=spdk%d", 491 opts->shm_id)); 492 if (args == NULL) { 493 return -1; 494 } 495 496 /* set the process type */ 497 args = push_arg(args, &argcount, _sprintf_alloc("--proc-type=auto")); 498 if (args == NULL) { 499 return -1; 500 } 501 } 502 #endif 503 504 g_eal_cmdline = args; 505 g_eal_cmdline_argcount = argcount; 506 return argcount; 507 } 508 509 int 510 spdk_env_dpdk_post_init(bool legacy_mem) 511 { 512 int rc; 513 514 pci_env_init(); 515 516 rc = mem_map_init(legacy_mem); 517 if (rc < 0) { 518 SPDK_ERRLOG("Failed to allocate mem_map\n"); 519 return rc; 520 } 521 522 rc = vtophys_init(); 523 if (rc < 0) { 524 SPDK_ERRLOG("Failed to initialize vtophys\n"); 525 return rc; 526 } 527 528 return 0; 529 } 530 531 void 532 spdk_env_dpdk_post_fini(void) 533 { 534 pci_env_fini(); 535 536 free_args(g_eal_cmdline, g_eal_cmdline_argcount); 537 g_eal_cmdline = NULL; 538 g_eal_cmdline_argcount = 0; 539 } 540 541 int 542 spdk_env_init(const struct spdk_env_opts *opts) 543 { 544 char **dpdk_args = NULL; 545 int i, rc; 546 int orig_optind; 547 bool legacy_mem; 548 549 /* If SPDK env has been initialized before, then only pci env requires 550 * reinitialization. 551 */ 552 if (g_external_init == false) { 553 if (opts != NULL) { 554 fprintf(stderr, "Invalid arguments to reinitialize SPDK env\n"); 555 return -EINVAL; 556 } 557 558 printf("Starting %s / %s reinitialization...\n", SPDK_VERSION_STRING, rte_version()); 559 pci_env_reinit(); 560 561 return 0; 562 } 563 564 if (opts == NULL) { 565 fprintf(stderr, "NULL arguments to initialize DPDK\n"); 566 return -EINVAL; 567 } 568 569 rc = build_eal_cmdline(opts); 570 if (rc < 0) { 571 SPDK_ERRLOG("Invalid arguments to initialize DPDK\n"); 572 return -EINVAL; 573 } 574 575 SPDK_PRINTF("Starting %s / %s initialization...\n", SPDK_VERSION_STRING, rte_version()); 576 SPDK_PRINTF("[ DPDK EAL parameters: "); 577 for (i = 0; i < g_eal_cmdline_argcount; i++) { 578 SPDK_PRINTF("%s ", g_eal_cmdline[i]); 579 } 580 SPDK_PRINTF("]\n"); 581 582 /* DPDK rearranges the array we pass to it, so make a copy 583 * before passing so we can still free the individual strings 584 * correctly. 585 */ 586 dpdk_args = calloc(g_eal_cmdline_argcount, sizeof(char *)); 587 if (dpdk_args == NULL) { 588 SPDK_ERRLOG("Failed to allocate dpdk_args\n"); 589 return -ENOMEM; 590 } 591 memcpy(dpdk_args, g_eal_cmdline, sizeof(char *) * g_eal_cmdline_argcount); 592 593 fflush(stdout); 594 orig_optind = optind; 595 optind = 1; 596 rc = rte_eal_init(g_eal_cmdline_argcount, dpdk_args); 597 optind = orig_optind; 598 599 free(dpdk_args); 600 601 if (rc < 0) { 602 if (rte_errno == EALREADY) { 603 SPDK_ERRLOG("DPDK already initialized\n"); 604 } else { 605 SPDK_ERRLOG("Failed to initialize DPDK\n"); 606 } 607 return -rte_errno; 608 } 609 610 legacy_mem = false; 611 if (opts->env_context && strstr(opts->env_context, "--legacy-mem") != NULL) { 612 legacy_mem = true; 613 } 614 615 rc = spdk_env_dpdk_post_init(legacy_mem); 616 if (rc == 0) { 617 g_external_init = false; 618 } 619 620 return rc; 621 } 622 623 void 624 spdk_env_fini(void) 625 { 626 spdk_env_dpdk_post_fini(); 627 } 628 629 bool 630 spdk_env_dpdk_external_init(void) 631 { 632 return g_external_init; 633 } 634