1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk/version.h" 9 10 #include "spdk_internal/event.h" 11 12 #include "spdk/assert.h" 13 #include "spdk/env.h" 14 #include "spdk/init.h" 15 #include "spdk/log.h" 16 #include "spdk/thread.h" 17 #include "spdk/trace.h" 18 #include "spdk/string.h" 19 #include "spdk/scheduler.h" 20 #include "spdk/rpc.h" 21 #include "spdk/util.h" 22 #include "spdk/file.h" 23 #include "event_internal.h" 24 25 #define SPDK_APP_DEFAULT_LOG_LEVEL SPDK_LOG_NOTICE 26 #define SPDK_APP_DEFAULT_LOG_PRINT_LEVEL SPDK_LOG_INFO 27 #define SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES SPDK_DEFAULT_NUM_TRACE_ENTRIES 28 29 #define SPDK_APP_DPDK_DEFAULT_MEM_SIZE -1 30 #define SPDK_APP_DPDK_DEFAULT_MAIN_CORE -1 31 #define SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL -1 32 #define SPDK_APP_DPDK_DEFAULT_CORE_MASK "0x1" 33 #define SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR 0x200000000000 34 #define SPDK_APP_DEFAULT_CORE_LIMIT 0x140000000 /* 5 GiB */ 35 36 /* For core counts <= 63, the message memory pool size is set to 37 * SPDK_DEFAULT_MSG_MEMPOOL_SIZE. 38 * For core counts > 63, the message memory pool size is dependend on 39 * number of cores. Per core, it is calculated as SPDK_MSG_MEMPOOL_CACHE_SIZE 40 * multiplied by factor of 4 to have space for multiple spdk threads running 41 * on single core (e.g iscsi + nvmf + vhost ). */ 42 #define SPDK_APP_PER_CORE_MSG_MEMPOOL_SIZE (4 * SPDK_MSG_MEMPOOL_CACHE_SIZE) 43 44 #define MAX_CPU_CORES 128 45 46 struct spdk_app { 47 void *json_data; 48 size_t json_data_size; 49 bool json_config_ignore_errors; 50 bool stopped; 51 const char *rpc_addr; 52 const char **rpc_allowlist; 53 FILE *rpc_log_file; 54 enum spdk_log_level rpc_log_level; 55 int shm_id; 56 spdk_app_shutdown_cb shutdown_cb; 57 int rc; 58 }; 59 60 static struct spdk_app g_spdk_app; 61 static spdk_msg_fn g_start_fn = NULL; 62 static void *g_start_arg = NULL; 63 static bool g_delay_subsystem_init = false; 64 static bool g_shutdown_sig_received = false; 65 static char *g_executable_name; 66 static struct spdk_app_opts g_default_opts; 67 static bool g_disable_cpumask_locks = false; 68 69 static int g_core_locks[MAX_CPU_CORES]; 70 71 static struct { 72 uint64_t irq; 73 uint64_t usr; 74 uint64_t sys; 75 } g_initial_stat[MAX_CPU_CORES]; 76 77 int 78 spdk_app_get_shm_id(void) 79 { 80 return g_spdk_app.shm_id; 81 } 82 83 /* append one empty option to indicate the end of the array */ 84 static const struct option g_cmdline_options[] = { 85 #define CONFIG_FILE_OPT_IDX 'c' 86 {"config", required_argument, NULL, CONFIG_FILE_OPT_IDX}, 87 #define LIMIT_COREDUMP_OPT_IDX 'd' 88 {"limit-coredump", no_argument, NULL, LIMIT_COREDUMP_OPT_IDX}, 89 #define TPOINT_GROUP_OPT_IDX 'e' 90 {"tpoint-group", required_argument, NULL, TPOINT_GROUP_OPT_IDX}, 91 #define SINGLE_FILE_SEGMENTS_OPT_IDX 'g' 92 {"single-file-segments", no_argument, NULL, SINGLE_FILE_SEGMENTS_OPT_IDX}, 93 #define HELP_OPT_IDX 'h' 94 {"help", no_argument, NULL, HELP_OPT_IDX}, 95 #define SHM_ID_OPT_IDX 'i' 96 {"shm-id", required_argument, NULL, SHM_ID_OPT_IDX}, 97 #define CPUMASK_OPT_IDX 'm' 98 {"cpumask", required_argument, NULL, CPUMASK_OPT_IDX}, 99 #define MEM_CHANNELS_OPT_IDX 'n' 100 {"mem-channels", required_argument, NULL, MEM_CHANNELS_OPT_IDX}, 101 #define MAIN_CORE_OPT_IDX 'p' 102 {"main-core", required_argument, NULL, MAIN_CORE_OPT_IDX}, 103 #define RPC_SOCKET_OPT_IDX 'r' 104 {"rpc-socket", required_argument, NULL, RPC_SOCKET_OPT_IDX}, 105 #define MEM_SIZE_OPT_IDX 's' 106 {"mem-size", required_argument, NULL, MEM_SIZE_OPT_IDX}, 107 #define NO_PCI_OPT_IDX 'u' 108 {"no-pci", no_argument, NULL, NO_PCI_OPT_IDX}, 109 #define VERSION_OPT_IDX 'v' 110 {"version", no_argument, NULL, VERSION_OPT_IDX}, 111 #define PCI_BLOCKED_OPT_IDX 'B' 112 {"pci-blocked", required_argument, NULL, PCI_BLOCKED_OPT_IDX}, 113 #define LOGFLAG_OPT_IDX 'L' 114 {"logflag", required_argument, NULL, LOGFLAG_OPT_IDX}, 115 #define HUGE_UNLINK_OPT_IDX 'R' 116 {"huge-unlink", no_argument, NULL, HUGE_UNLINK_OPT_IDX}, 117 #define PCI_ALLOWED_OPT_IDX 'A' 118 {"pci-allowed", required_argument, NULL, PCI_ALLOWED_OPT_IDX}, 119 #define INTERRUPT_MODE_OPT_IDX 256 120 {"interrupt-mode", no_argument, NULL, INTERRUPT_MODE_OPT_IDX}, 121 #define SILENCE_NOTICELOG_OPT_IDX 257 122 {"silence-noticelog", no_argument, NULL, SILENCE_NOTICELOG_OPT_IDX}, 123 #define WAIT_FOR_RPC_OPT_IDX 258 124 {"wait-for-rpc", no_argument, NULL, WAIT_FOR_RPC_OPT_IDX}, 125 #define HUGE_DIR_OPT_IDX 259 126 {"huge-dir", required_argument, NULL, HUGE_DIR_OPT_IDX}, 127 #define NUM_TRACE_ENTRIES_OPT_IDX 260 128 {"num-trace-entries", required_argument, NULL, NUM_TRACE_ENTRIES_OPT_IDX}, 129 #define JSON_CONFIG_OPT_IDX 262 130 {"json", required_argument, NULL, JSON_CONFIG_OPT_IDX}, 131 #define JSON_CONFIG_IGNORE_INIT_ERRORS_IDX 263 132 {"json-ignore-init-errors", no_argument, NULL, JSON_CONFIG_IGNORE_INIT_ERRORS_IDX}, 133 #define IOVA_MODE_OPT_IDX 264 134 {"iova-mode", required_argument, NULL, IOVA_MODE_OPT_IDX}, 135 #define BASE_VIRTADDR_OPT_IDX 265 136 {"base-virtaddr", required_argument, NULL, BASE_VIRTADDR_OPT_IDX}, 137 #define ENV_CONTEXT_OPT_IDX 266 138 {"env-context", required_argument, NULL, ENV_CONTEXT_OPT_IDX}, 139 #define DISABLE_CPUMASK_LOCKS_OPT_IDX 267 140 {"disable-cpumask-locks", no_argument, NULL, DISABLE_CPUMASK_LOCKS_OPT_IDX}, 141 #define RPCS_ALLOWED_OPT_IDX 268 142 {"rpcs-allowed", required_argument, NULL, RPCS_ALLOWED_OPT_IDX}, 143 #define ENV_VF_TOKEN_OPT_IDX 269 144 {"vfio-vf-token", required_argument, NULL, ENV_VF_TOKEN_OPT_IDX}, 145 #define MSG_MEMPOOL_SIZE_OPT_IDX 270 146 {"msg-mempool-size", required_argument, NULL, MSG_MEMPOOL_SIZE_OPT_IDX}, 147 #define LCORES_OPT_IDX 271 148 {"lcores", required_argument, NULL, LCORES_OPT_IDX}, 149 #define NO_HUGE_OPT_IDX 272 150 {"no-huge", no_argument, NULL, NO_HUGE_OPT_IDX}, 151 #define NO_RPC_SERVER_OPT_IDX 273 152 {"no-rpc-server", no_argument, NULL, NO_RPC_SERVER_OPT_IDX}, 153 }; 154 155 static int 156 parse_proc_stat(unsigned int core, uint64_t *user, uint64_t *sys, uint64_t *irq) 157 { 158 FILE *f; 159 uint64_t i, soft_irq, cpu = 0; 160 int rc, found = 0; 161 162 f = fopen("/proc/stat", "r"); 163 if (!f) { 164 return -1; 165 } 166 167 for (i = 0; i <= core + 1; i++) { 168 /* scanf discards input with '*' in format, 169 * cpu;user;nice;system;idle;iowait;irq;softirq;steal;guest;guest_nice */ 170 rc = fscanf(f, "cpu%li %li %*i %li %*i %*i %li %li %*i %*i %*i\n", 171 &cpu, user, sys, irq, &soft_irq); 172 if (rc != 5) { 173 continue; 174 } 175 176 /* some cores can be disabled, list may not be in order */ 177 if (cpu == core) { 178 found = 1; 179 break; 180 } 181 } 182 183 *irq += soft_irq; 184 185 fclose(f); 186 return found ? 0 : -1; 187 } 188 189 static int 190 init_proc_stat(unsigned int core) 191 { 192 uint64_t usr, sys, irq; 193 194 if (core >= MAX_CPU_CORES) { 195 return -1; 196 } 197 198 if (parse_proc_stat(core, &usr, &sys, &irq) < 0) { 199 return -1; 200 } 201 202 g_initial_stat[core].irq = irq; 203 g_initial_stat[core].usr = usr; 204 g_initial_stat[core].sys = sys; 205 206 return 0; 207 } 208 209 int 210 app_get_proc_stat(unsigned int core, uint64_t *usr, uint64_t *sys, uint64_t *irq) 211 { 212 uint64_t _usr, _sys, _irq; 213 214 if (core >= MAX_CPU_CORES) { 215 return -1; 216 } 217 218 if (parse_proc_stat(core, &_usr, &_sys, &_irq) < 0) { 219 return -1; 220 } 221 222 *irq = _irq - g_initial_stat[core].irq; 223 *usr = _usr - g_initial_stat[core].usr; 224 *sys = _sys - g_initial_stat[core].sys; 225 226 return 0; 227 } 228 229 static void 230 app_start_shutdown(void *ctx) 231 { 232 if (g_spdk_app.shutdown_cb) { 233 g_spdk_app.shutdown_cb(); 234 g_spdk_app.shutdown_cb = NULL; 235 } else { 236 spdk_app_stop(0); 237 } 238 } 239 240 void 241 spdk_app_start_shutdown(void) 242 { 243 spdk_thread_send_critical_msg(spdk_thread_get_app_thread(), app_start_shutdown); 244 } 245 246 static void 247 __shutdown_signal(int signo) 248 { 249 if (!g_shutdown_sig_received) { 250 g_shutdown_sig_received = true; 251 spdk_app_start_shutdown(); 252 } 253 } 254 255 static int 256 app_opts_validate(const char *app_opts) 257 { 258 int i = 0, j; 259 260 for (i = 0; app_opts[i] != '\0'; i++) { 261 /* ignore getopt control characters */ 262 if (app_opts[i] == ':' || app_opts[i] == '+' || app_opts[i] == '-') { 263 continue; 264 } 265 266 for (j = 0; SPDK_APP_GETOPT_STRING[j] != '\0'; j++) { 267 if (app_opts[i] == SPDK_APP_GETOPT_STRING[j]) { 268 return app_opts[i]; 269 } 270 } 271 } 272 return 0; 273 } 274 275 static void 276 calculate_mempool_size(struct spdk_app_opts *opts, 277 struct spdk_app_opts *opts_user) 278 { 279 uint32_t core_count = spdk_env_get_core_count(); 280 281 if (!opts_user->msg_mempool_size) { 282 /* The user didn't specify msg_mempool_size, so let's calculate it. 283 Set the default (SPDK_DEFAULT_MSG_MEMPOOL_SIZE) if less than 284 64 cores, and use 4k per core otherwise */ 285 opts->msg_mempool_size = spdk_max(SPDK_DEFAULT_MSG_MEMPOOL_SIZE, 286 core_count * SPDK_APP_PER_CORE_MSG_MEMPOOL_SIZE); 287 } else { 288 opts->msg_mempool_size = opts_user->msg_mempool_size; 289 } 290 } 291 292 void 293 spdk_app_opts_init(struct spdk_app_opts *opts, size_t opts_size) 294 { 295 if (!opts) { 296 SPDK_ERRLOG("opts should not be NULL\n"); 297 return; 298 } 299 300 if (!opts_size) { 301 SPDK_ERRLOG("opts_size should not be zero value\n"); 302 return; 303 } 304 305 memset(opts, 0, opts_size); 306 opts->opts_size = opts_size; 307 308 #define SET_FIELD(field, value) \ 309 if (offsetof(struct spdk_app_opts, field) + sizeof(opts->field) <= opts_size) { \ 310 opts->field = value; \ 311 } \ 312 313 SET_FIELD(enable_coredump, true); 314 SET_FIELD(shm_id, -1); 315 SET_FIELD(mem_size, SPDK_APP_DPDK_DEFAULT_MEM_SIZE); 316 SET_FIELD(main_core, SPDK_APP_DPDK_DEFAULT_MAIN_CORE); 317 SET_FIELD(mem_channel, SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL); 318 SET_FIELD(base_virtaddr, SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR); 319 SET_FIELD(print_level, SPDK_APP_DEFAULT_LOG_PRINT_LEVEL); 320 SET_FIELD(rpc_addr, SPDK_DEFAULT_RPC_ADDR); 321 SET_FIELD(num_entries, SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES); 322 SET_FIELD(delay_subsystem_init, false); 323 SET_FIELD(disable_signal_handlers, false); 324 SET_FIELD(interrupt_mode, false); 325 /* Don't set msg_mempool_size here, it is set or calculated later */ 326 SET_FIELD(rpc_allowlist, NULL); 327 SET_FIELD(rpc_log_file, NULL); 328 SET_FIELD(rpc_log_level, SPDK_LOG_DISABLED); 329 #undef SET_FIELD 330 } 331 332 static int 333 app_setup_signal_handlers(struct spdk_app_opts *opts) 334 { 335 struct sigaction sigact; 336 sigset_t sigmask; 337 int rc; 338 339 sigemptyset(&sigmask); 340 memset(&sigact, 0, sizeof(sigact)); 341 sigemptyset(&sigact.sa_mask); 342 343 sigact.sa_handler = SIG_IGN; 344 rc = sigaction(SIGPIPE, &sigact, NULL); 345 if (rc < 0) { 346 SPDK_ERRLOG("sigaction(SIGPIPE) failed\n"); 347 return rc; 348 } 349 350 /* Install the same handler for SIGINT and SIGTERM */ 351 g_shutdown_sig_received = false; 352 sigact.sa_handler = __shutdown_signal; 353 rc = sigaction(SIGINT, &sigact, NULL); 354 if (rc < 0) { 355 SPDK_ERRLOG("sigaction(SIGINT) failed\n"); 356 return rc; 357 } 358 sigaddset(&sigmask, SIGINT); 359 360 rc = sigaction(SIGTERM, &sigact, NULL); 361 if (rc < 0) { 362 SPDK_ERRLOG("sigaction(SIGTERM) failed\n"); 363 return rc; 364 } 365 sigaddset(&sigmask, SIGTERM); 366 367 pthread_sigmask(SIG_UNBLOCK, &sigmask, NULL); 368 369 return 0; 370 } 371 372 static void 373 app_start_application(int rc, void *arg1) 374 { 375 assert(spdk_thread_is_app_thread(NULL)); 376 377 if (rc) { 378 SPDK_ERRLOG("Failed to load subsystems for RUNTIME state with code: %d\n", rc); 379 spdk_app_stop(rc); 380 return; 381 } 382 383 if (g_spdk_app.rpc_addr) { 384 spdk_rpc_server_resume(g_spdk_app.rpc_addr); 385 } 386 387 g_start_fn(g_start_arg); 388 } 389 390 static void 391 app_subsystem_init_done(int rc, void *arg1) 392 { 393 if (rc) { 394 SPDK_ERRLOG("Subsystem initialization failed with code: %d\n", rc); 395 spdk_app_stop(rc); 396 return; 397 } 398 399 spdk_rpc_set_allowlist(g_spdk_app.rpc_allowlist); 400 spdk_rpc_set_state(SPDK_RPC_RUNTIME); 401 402 if (g_spdk_app.json_data) { 403 /* Load SPDK_RPC_RUNTIME RPCs from config file */ 404 assert(spdk_rpc_get_state() == SPDK_RPC_RUNTIME); 405 spdk_subsystem_load_config(g_spdk_app.json_data, g_spdk_app.json_data_size, 406 app_start_application, NULL, 407 !g_spdk_app.json_config_ignore_errors); 408 } else { 409 app_start_application(0, NULL); 410 } 411 } 412 413 static void 414 app_do_spdk_subsystem_init(int rc, void *arg1) 415 { 416 struct spdk_rpc_opts opts; 417 418 if (rc) { 419 spdk_app_stop(rc); 420 return; 421 } 422 423 if (g_spdk_app.rpc_addr) { 424 opts.size = SPDK_SIZEOF(&opts, log_level); 425 opts.log_file = g_spdk_app.rpc_log_file; 426 opts.log_level = g_spdk_app.rpc_log_level; 427 428 rc = spdk_rpc_initialize(g_spdk_app.rpc_addr, &opts); 429 if (rc) { 430 spdk_app_stop(rc); 431 return; 432 } 433 if (g_delay_subsystem_init) { 434 return; 435 } 436 spdk_rpc_server_pause(g_spdk_app.rpc_addr); 437 } else { 438 SPDK_DEBUGLOG(app_rpc, "RPC server not started\n"); 439 } 440 spdk_subsystem_init(app_subsystem_init_done, NULL); 441 } 442 443 static int 444 app_opts_add_pci_addr(struct spdk_app_opts *opts, struct spdk_pci_addr **list, char *bdf) 445 { 446 struct spdk_pci_addr *tmp = *list; 447 size_t i = opts->num_pci_addr; 448 449 tmp = realloc(tmp, sizeof(*tmp) * (i + 1)); 450 if (tmp == NULL) { 451 SPDK_ERRLOG("realloc error\n"); 452 return -ENOMEM; 453 } 454 455 *list = tmp; 456 if (spdk_pci_addr_parse(*list + i, bdf) < 0) { 457 SPDK_ERRLOG("Invalid address %s\n", bdf); 458 return -EINVAL; 459 } 460 461 opts->num_pci_addr++; 462 return 0; 463 } 464 465 static int 466 app_setup_env(struct spdk_app_opts *opts) 467 { 468 struct spdk_env_opts env_opts = {}; 469 int rc; 470 471 if (opts == NULL) { 472 rc = spdk_env_init(NULL); 473 if (rc != 0) { 474 SPDK_ERRLOG("Unable to reinitialize SPDK env\n"); 475 } 476 477 return rc; 478 } 479 480 spdk_env_opts_init(&env_opts); 481 482 env_opts.name = opts->name; 483 env_opts.core_mask = opts->reactor_mask; 484 env_opts.lcore_map = opts->lcore_map; 485 env_opts.shm_id = opts->shm_id; 486 env_opts.mem_channel = opts->mem_channel; 487 env_opts.main_core = opts->main_core; 488 env_opts.mem_size = opts->mem_size; 489 env_opts.hugepage_single_segments = opts->hugepage_single_segments; 490 env_opts.unlink_hugepage = opts->unlink_hugepage; 491 env_opts.hugedir = opts->hugedir; 492 env_opts.no_pci = opts->no_pci; 493 env_opts.num_pci_addr = opts->num_pci_addr; 494 env_opts.pci_blocked = opts->pci_blocked; 495 env_opts.pci_allowed = opts->pci_allowed; 496 env_opts.base_virtaddr = opts->base_virtaddr; 497 env_opts.env_context = opts->env_context; 498 env_opts.iova_mode = opts->iova_mode; 499 env_opts.vf_token = opts->vf_token; 500 env_opts.no_huge = opts->no_huge; 501 502 rc = spdk_env_init(&env_opts); 503 free(env_opts.pci_blocked); 504 free(env_opts.pci_allowed); 505 506 if (rc < 0) { 507 SPDK_ERRLOG("Unable to initialize SPDK env\n"); 508 } 509 510 return rc; 511 } 512 513 static int 514 app_setup_trace(struct spdk_app_opts *opts) 515 { 516 char shm_name[64]; 517 uint64_t tpoint_group_mask, tpoint_mask = -1ULL; 518 char *end = NULL, *tpoint_group_mask_str, *tpoint_group_str = NULL; 519 char *tp_g_str, *tpoint_group, *tpoints; 520 bool error_found = false; 521 uint64_t group_id; 522 523 if (opts->shm_id >= 0) { 524 snprintf(shm_name, sizeof(shm_name), "/%s%s%d", opts->name, 525 SPDK_TRACE_SHM_NAME_BASE, opts->shm_id); 526 } else { 527 snprintf(shm_name, sizeof(shm_name), "/%s%spid%d", opts->name, 528 SPDK_TRACE_SHM_NAME_BASE, (int)getpid()); 529 } 530 531 if (spdk_trace_init(shm_name, opts->num_entries, 0) != 0) { 532 return -1; 533 } 534 535 if (opts->tpoint_group_mask == NULL) { 536 return 0; 537 } 538 539 tpoint_group_mask_str = strdup(opts->tpoint_group_mask); 540 if (tpoint_group_mask_str == NULL) { 541 SPDK_ERRLOG("Unable to get string of tpoint group mask from opts.\n"); 542 return -1; 543 } 544 /* Save a pointer to the original value of the tpoint group mask string 545 * to free later, because spdk_strsepq() modifies given char*. */ 546 tp_g_str = tpoint_group_mask_str; 547 while ((tpoint_group_str = spdk_strsepq(&tpoint_group_mask_str, ",")) != NULL) { 548 if (strchr(tpoint_group_str, ':')) { 549 /* Get the tpoint group mask */ 550 tpoint_group = spdk_strsepq(&tpoint_group_str, ":"); 551 /* Get the tpoint mask inside that group */ 552 tpoints = spdk_strsepq(&tpoint_group_str, ":"); 553 554 errno = 0; 555 tpoint_group_mask = strtoull(tpoint_group, &end, 16); 556 if (*end != '\0' || errno) { 557 tpoint_group_mask = spdk_trace_create_tpoint_group_mask(tpoint_group); 558 if (tpoint_group_mask == 0) { 559 error_found = true; 560 break; 561 } 562 } 563 /* Check if tpoint group mask has only one bit set. 564 * This is to avoid enabling individual tpoints in 565 * more than one tracepoint group at once. */ 566 if (!spdk_u64_is_pow2(tpoint_group_mask)) { 567 SPDK_ERRLOG("Tpoint group mask: %s contains multiple tpoint groups.\n", tpoint_group); 568 SPDK_ERRLOG("This is not supported, to prevent from activating tpoints by mistake.\n"); 569 error_found = true; 570 break; 571 } 572 573 errno = 0; 574 tpoint_mask = strtoull(tpoints, &end, 16); 575 if (*end != '\0' || errno) { 576 error_found = true; 577 break; 578 } 579 } else { 580 errno = 0; 581 tpoint_group_mask = strtoull(tpoint_group_str, &end, 16); 582 if (*end != '\0' || errno) { 583 tpoint_group_mask = spdk_trace_create_tpoint_group_mask(tpoint_group_str); 584 if (tpoint_group_mask == 0) { 585 error_found = true; 586 break; 587 } 588 } 589 tpoint_mask = -1ULL; 590 } 591 592 for (group_id = 0; group_id < SPDK_TRACE_MAX_GROUP_ID; ++group_id) { 593 if (tpoint_group_mask & (1 << group_id)) { 594 spdk_trace_set_tpoints(group_id, tpoint_mask); 595 } 596 } 597 } 598 599 if (error_found) { 600 SPDK_ERRLOG("invalid tpoint mask %s\n", opts->tpoint_group_mask); 601 free(tp_g_str); 602 return -1; 603 } else { 604 SPDK_NOTICELOG("Tracepoint Group Mask %s specified.\n", opts->tpoint_group_mask); 605 SPDK_NOTICELOG("Use 'spdk_trace -s %s %s %d' to capture a snapshot of events at runtime.\n", 606 opts->name, 607 opts->shm_id >= 0 ? "-i" : "-p", 608 opts->shm_id >= 0 ? opts->shm_id : getpid()); 609 #if defined(__linux__) 610 SPDK_NOTICELOG("'spdk_trace' without parameters will also work if this is the only\n"); 611 SPDK_NOTICELOG("SPDK application currently running.\n"); 612 SPDK_NOTICELOG("Or copy /dev/shm%s for offline analysis/debug.\n", shm_name); 613 #endif 614 } 615 free(tp_g_str); 616 617 return 0; 618 } 619 620 static void 621 bootstrap_fn(void *arg1) 622 { 623 spdk_rpc_set_allowlist(g_spdk_app.rpc_allowlist); 624 625 if (g_spdk_app.json_data) { 626 /* Load SPDK_RPC_STARTUP RPCs from config file */ 627 assert(spdk_rpc_get_state() == SPDK_RPC_STARTUP); 628 spdk_subsystem_load_config(g_spdk_app.json_data, g_spdk_app.json_data_size, 629 app_do_spdk_subsystem_init, NULL, 630 !g_spdk_app.json_config_ignore_errors); 631 } else { 632 app_do_spdk_subsystem_init(0, NULL); 633 } 634 } 635 636 static void 637 app_copy_opts(struct spdk_app_opts *opts, struct spdk_app_opts *opts_user, size_t opts_size) 638 { 639 spdk_app_opts_init(opts, sizeof(*opts)); 640 opts->opts_size = opts_size; 641 642 #define SET_FIELD(field) \ 643 if (offsetof(struct spdk_app_opts, field) + sizeof(opts->field) <= (opts->opts_size)) { \ 644 opts->field = opts_user->field; \ 645 } \ 646 647 SET_FIELD(name); 648 SET_FIELD(json_config_file); 649 SET_FIELD(json_config_ignore_errors); 650 SET_FIELD(rpc_addr); 651 SET_FIELD(reactor_mask); 652 SET_FIELD(lcore_map); 653 SET_FIELD(tpoint_group_mask); 654 SET_FIELD(shm_id); 655 SET_FIELD(shutdown_cb); 656 SET_FIELD(enable_coredump); 657 SET_FIELD(mem_channel); 658 SET_FIELD(main_core); 659 SET_FIELD(mem_size); 660 SET_FIELD(no_pci); 661 SET_FIELD(hugepage_single_segments); 662 SET_FIELD(unlink_hugepage); 663 SET_FIELD(no_huge); 664 SET_FIELD(hugedir); 665 SET_FIELD(print_level); 666 SET_FIELD(num_pci_addr); 667 SET_FIELD(pci_blocked); 668 SET_FIELD(pci_allowed); 669 SET_FIELD(iova_mode); 670 SET_FIELD(delay_subsystem_init); 671 SET_FIELD(num_entries); 672 SET_FIELD(env_context); 673 SET_FIELD(log); 674 SET_FIELD(base_virtaddr); 675 SET_FIELD(disable_signal_handlers); 676 SET_FIELD(interrupt_mode); 677 SET_FIELD(msg_mempool_size); 678 SET_FIELD(rpc_allowlist); 679 SET_FIELD(vf_token); 680 SET_FIELD(rpc_log_file); 681 SET_FIELD(rpc_log_level); 682 SET_FIELD(json_data); 683 SET_FIELD(json_data_size); 684 685 /* You should not remove this statement, but need to update the assert statement 686 * if you add a new field, and also add a corresponding SET_FIELD statement */ 687 SPDK_STATIC_ASSERT(sizeof(struct spdk_app_opts) == 252, "Incorrect size"); 688 689 #undef SET_FIELD 690 } 691 692 static int 693 unclaim_cpu_cores(uint32_t *failed_core) 694 { 695 char core_name[40]; 696 uint32_t i; 697 int rc; 698 699 for (i = 0; i < MAX_CPU_CORES; i++) { 700 if (g_core_locks[i] != -1) { 701 snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", i); 702 rc = close(g_core_locks[i]); 703 if (rc) { 704 SPDK_ERRLOG("Failed to close lock fd for core %d, errno: %d\n", i, errno); 705 goto error; 706 } 707 708 g_core_locks[i] = -1; 709 rc = unlink(core_name); 710 if (rc) { 711 SPDK_ERRLOG("Failed to unlink lock fd for core %d, errno: %d\n", i, errno); 712 goto error; 713 } 714 } 715 } 716 717 return 0; 718 719 error: 720 if (failed_core != NULL) { 721 /* Set number of core we failed to claim. */ 722 *failed_core = i; 723 } 724 return -1; 725 } 726 727 static int 728 claim_cpu_cores(uint32_t *failed_core) 729 { 730 char core_name[40]; 731 int core_fd, pid; 732 int *core_map; 733 uint32_t core; 734 735 struct flock core_lock = { 736 .l_type = F_WRLCK, 737 .l_whence = SEEK_SET, 738 .l_start = 0, 739 .l_len = 0, 740 }; 741 742 SPDK_ENV_FOREACH_CORE(core) { 743 if (g_core_locks[core] != -1) { 744 /* If this core is locked already, do not try lock it again. */ 745 continue; 746 } 747 748 snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", core); 749 core_fd = open(core_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 750 if (core_fd == -1) { 751 SPDK_ERRLOG("Could not open %s (%s).\n", core_name, spdk_strerror(errno)); 752 /* Return number of core we failed to claim. */ 753 goto error; 754 } 755 756 if (ftruncate(core_fd, sizeof(int)) != 0) { 757 SPDK_ERRLOG("Could not truncate %s (%s).\n", core_name, spdk_strerror(errno)); 758 close(core_fd); 759 goto error; 760 } 761 762 core_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE, MAP_SHARED, core_fd, 0); 763 if (core_map == MAP_FAILED) { 764 SPDK_ERRLOG("Could not mmap core %s (%s).\n", core_name, spdk_strerror(errno)); 765 close(core_fd); 766 goto error; 767 } 768 769 if (fcntl(core_fd, F_SETLK, &core_lock) != 0) { 770 pid = *core_map; 771 SPDK_ERRLOG("Cannot create lock on core %" PRIu32 ", probably process %d has claimed it.\n", 772 core, pid); 773 munmap(core_map, sizeof(int)); 774 close(core_fd); 775 goto error; 776 } 777 778 /* We write the PID to the core lock file so that other processes trying 779 * to claim the same core will know what process is holding the lock. */ 780 *core_map = (int)getpid(); 781 munmap(core_map, sizeof(int)); 782 g_core_locks[core] = core_fd; 783 /* Keep core_fd open to maintain the lock. */ 784 } 785 786 return 0; 787 788 error: 789 if (failed_core != NULL) { 790 /* Set number of core we failed to claim. */ 791 *failed_core = core; 792 } 793 unclaim_cpu_cores(NULL); 794 return -1; 795 } 796 797 int 798 spdk_app_start(struct spdk_app_opts *opts_user, spdk_msg_fn start_fn, 799 void *arg1) 800 { 801 int rc; 802 char *tty; 803 struct spdk_cpuset tmp_cpumask = {}; 804 static bool g_env_was_setup = false; 805 struct spdk_app_opts opts_local = {}; 806 struct spdk_app_opts *opts = &opts_local; 807 uint32_t i, core; 808 809 if (!opts_user) { 810 SPDK_ERRLOG("opts_user should not be NULL\n"); 811 return 1; 812 } 813 814 if (!opts_user->opts_size) { 815 SPDK_ERRLOG("The opts_size in opts_user structure should not be zero value\n"); 816 return 1; 817 } 818 819 if (opts_user->name == NULL) { 820 SPDK_ERRLOG("spdk_app_opts::name not specified\n"); 821 return 1; 822 } 823 824 app_copy_opts(opts, opts_user, opts_user->opts_size); 825 826 if (!start_fn) { 827 SPDK_ERRLOG("start_fn should not be NULL\n"); 828 return 1; 829 } 830 831 if (!opts->rpc_addr && opts->delay_subsystem_init) { 832 SPDK_ERRLOG("Cannot use '--wait-for-rpc' if no RPC server is going to be started.\n"); 833 return 1; 834 } 835 836 if (!(opts->lcore_map || opts->reactor_mask)) { 837 /* Set default CPU mask */ 838 opts->reactor_mask = SPDK_APP_DPDK_DEFAULT_CORE_MASK; 839 } 840 841 tty = ttyname(STDERR_FILENO); 842 if (opts->print_level > SPDK_LOG_WARN && 843 isatty(STDERR_FILENO) && 844 tty && 845 !strncmp(tty, "/dev/tty", strlen("/dev/tty"))) { 846 printf("Warning: printing stderr to console terminal without -q option specified.\n"); 847 printf("Suggest using --silence-noticelog to disable logging to stderr and\n"); 848 printf("monitor syslog, or redirect stderr to a file.\n"); 849 printf("(Delaying for 10 seconds...)\n"); 850 sleep(10); 851 } 852 853 spdk_log_set_print_level(opts->print_level); 854 855 #ifndef SPDK_NO_RLIMIT 856 if (opts->enable_coredump) { 857 struct rlimit core_limits; 858 859 core_limits.rlim_cur = core_limits.rlim_max = SPDK_APP_DEFAULT_CORE_LIMIT; 860 setrlimit(RLIMIT_CORE, &core_limits); 861 } 862 #endif 863 864 if (opts->interrupt_mode) { 865 spdk_interrupt_mode_enable(); 866 } 867 868 memset(&g_spdk_app, 0, sizeof(g_spdk_app)); 869 870 g_spdk_app.json_config_ignore_errors = opts->json_config_ignore_errors; 871 g_spdk_app.rpc_addr = opts->rpc_addr; 872 g_spdk_app.rpc_allowlist = opts->rpc_allowlist; 873 g_spdk_app.rpc_log_file = opts->rpc_log_file; 874 g_spdk_app.rpc_log_level = opts->rpc_log_level; 875 g_spdk_app.shm_id = opts->shm_id; 876 g_spdk_app.shutdown_cb = opts->shutdown_cb; 877 g_spdk_app.rc = 0; 878 g_spdk_app.stopped = false; 879 880 spdk_log_set_level(SPDK_APP_DEFAULT_LOG_LEVEL); 881 882 /* Pass NULL to app_setup_env if SPDK app has been set up, in order to 883 * indicate that this is a reinitialization. 884 */ 885 if (app_setup_env(g_env_was_setup ? NULL : opts) < 0) { 886 return 1; 887 } 888 889 /* Calculate mempool size now that the env layer has configured the core count 890 * for the application */ 891 calculate_mempool_size(opts, opts_user); 892 893 spdk_log_open(opts->log); 894 895 /* Initialize each lock to -1 to indicate "empty" status */ 896 for (i = 0; i < MAX_CPU_CORES; i++) { 897 g_core_locks[i] = -1; 898 } 899 900 if (!g_disable_cpumask_locks) { 901 if (claim_cpu_cores(NULL)) { 902 SPDK_ERRLOG("Unable to acquire lock on assigned core mask - exiting.\n"); 903 return 1; 904 } 905 } else { 906 SPDK_NOTICELOG("CPU core locks deactivated.\n"); 907 } 908 909 SPDK_NOTICELOG("Total cores available: %d\n", spdk_env_get_core_count()); 910 911 if ((rc = spdk_reactors_init(opts->msg_mempool_size)) != 0) { 912 SPDK_ERRLOG("Reactor Initialization failed: rc = %d\n", rc); 913 return 1; 914 } 915 916 spdk_cpuset_set_cpu(&tmp_cpumask, spdk_env_get_current_core(), true); 917 918 /* Now that the reactors have been initialized, we can create the app thread. */ 919 spdk_thread_create("app_thread", &tmp_cpumask); 920 if (!spdk_thread_get_app_thread()) { 921 SPDK_ERRLOG("Unable to create an spdk_thread for initialization\n"); 922 return 1; 923 } 924 925 SPDK_ENV_FOREACH_CORE(core) { 926 rc = init_proc_stat(core); 927 if (rc) { 928 SPDK_NOTICELOG("Unable to parse /proc/stat [core: %d].\n", core); 929 } 930 } 931 /* 932 * Disable and ignore trace setup if setting num_entries 933 * to be 0. 934 * 935 * Note the call to app_setup_trace() is located here 936 * ahead of app_setup_signal_handlers(). 937 * That's because there is not an easy/direct clean 938 * way of unwinding alloc'd resources that can occur 939 * in app_setup_signal_handlers(). 940 */ 941 if (opts->num_entries != 0 && app_setup_trace(opts) != 0) { 942 return 1; 943 } 944 945 if (!opts->disable_signal_handlers && app_setup_signal_handlers(opts) != 0) { 946 return 1; 947 } 948 949 g_delay_subsystem_init = opts->delay_subsystem_init; 950 g_start_fn = start_fn; 951 g_start_arg = arg1; 952 953 if (opts->json_config_file != NULL) { 954 if (opts->json_data) { 955 SPDK_ERRLOG("App opts json_config_file and json_data are mutually exclusive\n"); 956 return 1; 957 } 958 959 g_spdk_app.json_data = spdk_posix_file_load_from_name(opts->json_config_file, 960 &g_spdk_app.json_data_size); 961 if (!g_spdk_app.json_data) { 962 SPDK_ERRLOG("Read JSON configuration file %s failed: %s\n", 963 opts->json_config_file, spdk_strerror(errno)); 964 return 1; 965 } 966 } else if (opts->json_data) { 967 g_spdk_app.json_data = calloc(1, opts->json_data_size); 968 if (!g_spdk_app.json_data) { 969 SPDK_ERRLOG("Failed to allocate JSON data buffer\n"); 970 return 1; 971 } 972 973 memcpy(g_spdk_app.json_data, opts->json_data, opts->json_data_size); 974 g_spdk_app.json_data_size = opts->json_data_size; 975 } 976 977 spdk_thread_send_msg(spdk_thread_get_app_thread(), bootstrap_fn, NULL); 978 979 /* This blocks until spdk_app_stop is called */ 980 spdk_reactors_start(); 981 982 g_env_was_setup = true; 983 984 return g_spdk_app.rc; 985 } 986 987 void 988 spdk_app_fini(void) 989 { 990 spdk_trace_cleanup(); 991 spdk_reactors_fini(); 992 spdk_env_fini(); 993 spdk_log_close(); 994 unclaim_cpu_cores(NULL); 995 } 996 997 static void 998 subsystem_fini_done(void *arg1) 999 { 1000 spdk_rpc_finish(); 1001 spdk_reactors_stop(NULL); 1002 } 1003 1004 static void 1005 _start_subsystem_fini(void *arg1) 1006 { 1007 if (g_scheduling_in_progress) { 1008 spdk_thread_send_msg(spdk_thread_get_app_thread(), _start_subsystem_fini, NULL); 1009 return; 1010 } 1011 1012 spdk_subsystem_fini(subsystem_fini_done, NULL); 1013 } 1014 1015 static int 1016 log_deprecation_hits(void *ctx, struct spdk_deprecation *dep) 1017 { 1018 uint64_t hits = spdk_deprecation_get_hits(dep); 1019 1020 if (hits == 0) { 1021 return 0; 1022 } 1023 1024 SPDK_WARNLOG("%s: deprecation '%s' scheduled for removal in %s hit %" PRIu64 " times\n", 1025 spdk_deprecation_get_tag(dep), spdk_deprecation_get_description(dep), 1026 spdk_deprecation_get_remove_release(dep), hits); 1027 return 0; 1028 } 1029 1030 static void 1031 app_stop(void *arg1) 1032 { 1033 free(g_spdk_app.json_data); 1034 1035 if (g_spdk_app.rc == 0) { 1036 g_spdk_app.rc = (int)(intptr_t)arg1; 1037 } 1038 1039 if (g_spdk_app.stopped) { 1040 SPDK_NOTICELOG("spdk_app_stop called twice\n"); 1041 return; 1042 } 1043 1044 g_spdk_app.stopped = true; 1045 spdk_log_for_each_deprecation(NULL, log_deprecation_hits); 1046 _start_subsystem_fini(NULL); 1047 } 1048 1049 void 1050 spdk_app_stop(int rc) 1051 { 1052 if (rc) { 1053 SPDK_WARNLOG("spdk_app_stop'd on non-zero\n"); 1054 } 1055 1056 /* 1057 * We want to run spdk_subsystem_fini() from the same thread where spdk_subsystem_init() 1058 * was called. 1059 */ 1060 spdk_thread_send_msg(spdk_thread_get_app_thread(), app_stop, (void *)(intptr_t)rc); 1061 } 1062 1063 static void 1064 usage_memory_size(void) 1065 { 1066 #ifndef __linux__ 1067 if (g_default_opts.mem_size <= 0) { 1068 printf("all hugepage memory)\n"); 1069 } else 1070 #endif 1071 { 1072 printf("%dMB)\n", g_default_opts.mem_size >= 0 ? g_default_opts.mem_size : 0); 1073 } 1074 } 1075 1076 static void 1077 usage(void (*app_usage)(void)) 1078 { 1079 printf("%s [options]\n", g_executable_name); 1080 /* Keep entries inside categories roughly sorted by frequency of use. */ 1081 printf("\nCPU options:\n"); 1082 printf(" -m, --cpumask <mask or list> core mask (like 0xF) or core list of '[]' embraced for DPDK\n"); 1083 printf(" (like [0,1,10])\n"); 1084 printf(" --lcores <list> lcore to CPU mapping list. The list is in the format:\n"); 1085 printf(" <lcores[@CPUs]>[<,lcores[@CPUs]>...]\n"); 1086 printf(" lcores and cpus list are grouped by '(' and ')', e.g '--lcores \"(5-7)@(10-12)\"'\n"); 1087 printf(" Within the group, '-' is used for range separator,\n"); 1088 printf(" ',' is used for single number separator.\n"); 1089 printf(" '( )' can be omitted for single element group,\n"); 1090 printf(" '@' can be omitted if cpus and lcores have the same value\n"); 1091 printf(" --disable-cpumask-locks Disable CPU core lock files.\n"); 1092 printf(" --interrupt-mode set app to interrupt mode (Warning: CPU usage will be reduced only if all\n"); 1093 printf(" pollers in the app support interrupt mode)\n"); 1094 printf(" -p, --main-core <id> main (primary) core for DPDK\n"); 1095 1096 printf("\nConfiguration options:\n"); 1097 printf(" -c, --config, --json <config> JSON config file\n"); 1098 printf(" -r, --rpc-socket <path> RPC listen address (default %s)\n", SPDK_DEFAULT_RPC_ADDR); 1099 printf(" --no-rpc-server skip RPC server initialization. This option ignores '--rpc-socket' value.\n"); 1100 printf(" --wait-for-rpc wait for RPCs to initialize subsystems\n"); 1101 printf(" --rpcs-allowed comma-separated list of permitted RPCS\n"); 1102 printf(" --json-ignore-init-errors don't exit on invalid config entry\n"); 1103 1104 printf("\nMemory options:\n"); 1105 printf(" --iova-mode <pa/va> set IOVA mode ('pa' for IOVA_PA and 'va' for IOVA_VA)\n"); 1106 printf(" --base-virtaddr <addr> the base virtual address for DPDK (default: 0x200000000000)\n"); 1107 printf(" --huge-dir <path> use a specific hugetlbfs mount to reserve memory from\n"); 1108 printf(" -R, --huge-unlink unlink huge files after initialization\n"); 1109 printf(" -n, --mem-channels <num> number of memory channels used for DPDK\n"); 1110 printf(" -s, --mem-size <size> memory size in MB for DPDK (default: "); 1111 usage_memory_size(); 1112 printf(" --msg-mempool-size <size> global message memory pool size in count (default: %d)\n", 1113 SPDK_DEFAULT_MSG_MEMPOOL_SIZE); 1114 printf(" --no-huge run without using hugepages\n"); 1115 printf(" -i, --shm-id <id> shared memory ID (optional)\n"); 1116 printf(" -g, --single-file-segments force creating just one hugetlbfs file\n"); 1117 1118 printf("\nPCI options:\n"); 1119 printf(" -A, --pci-allowed <bdf> pci addr to allow (-B and -A cannot be used at the same time)\n"); 1120 printf(" -B, --pci-blocked <bdf> pci addr to block (can be used more than once)\n"); 1121 printf(" -u, --no-pci disable PCI access\n"); 1122 printf(" --vfio-vf-token VF token (UUID) shared between SR-IOV PF and VFs for vfio_pci driver\n"); 1123 1124 printf("\nLog options:\n"); 1125 spdk_log_usage(stdout, "-L"); 1126 printf(" --silence-noticelog disable notice level logging to stderr\n"); 1127 1128 printf("\nTrace options:\n"); 1129 printf(" --num-trace-entries <num> number of trace entries for each core, must be power of 2,\n"); 1130 printf(" setting 0 to disable trace (default %d)\n", 1131 SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES); 1132 printf(" Tracepoints vary in size and can use more than one trace entry.\n"); 1133 spdk_trace_mask_usage(stdout, "-e"); 1134 1135 printf("\nOther options:\n"); 1136 printf(" -h, --help show this usage\n"); 1137 printf(" -v, --version print SPDK version\n"); 1138 printf(" -d, --limit-coredump do not set max coredump size to RLIM_INFINITY\n"); 1139 printf(" --env-context Opaque context for use of the env implementation\n"); 1140 1141 if (app_usage) { 1142 printf("\nApplication specific:\n"); 1143 app_usage(); 1144 } 1145 } 1146 1147 spdk_app_parse_args_rvals_t 1148 spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts, 1149 const char *app_getopt_str, const struct option *app_long_opts, 1150 int (*app_parse)(int ch, char *arg), 1151 void (*app_usage)(void)) 1152 { 1153 int ch, rc, opt_idx, global_long_opts_len, app_long_opts_len; 1154 struct option *cmdline_options; 1155 char *cmdline_short_opts = NULL; 1156 char *shm_id_str = NULL; 1157 enum spdk_app_parse_args_rvals retval = SPDK_APP_PARSE_ARGS_FAIL; 1158 long int tmp; 1159 1160 memcpy(&g_default_opts, opts, sizeof(g_default_opts)); 1161 1162 if (opts->json_config_file && access(opts->json_config_file, R_OK) != 0) { 1163 SPDK_WARNLOG("Can't read JSON configuration file '%s'\n", opts->json_config_file); 1164 opts->json_config_file = NULL; 1165 } 1166 1167 if (app_long_opts == NULL) { 1168 app_long_opts_len = 0; 1169 } else { 1170 for (app_long_opts_len = 0; 1171 app_long_opts[app_long_opts_len].name != NULL; 1172 app_long_opts_len++); 1173 } 1174 1175 global_long_opts_len = SPDK_COUNTOF(g_cmdline_options); 1176 1177 cmdline_options = calloc(global_long_opts_len + app_long_opts_len + 1, sizeof(*cmdline_options)); 1178 if (!cmdline_options) { 1179 SPDK_ERRLOG("Out of memory\n"); 1180 return SPDK_APP_PARSE_ARGS_FAIL; 1181 } 1182 1183 memcpy(&cmdline_options[0], g_cmdline_options, sizeof(g_cmdline_options)); 1184 if (app_long_opts) { 1185 memcpy(&cmdline_options[global_long_opts_len], app_long_opts, 1186 app_long_opts_len * sizeof(*app_long_opts)); 1187 } 1188 1189 if (app_getopt_str != NULL) { 1190 ch = app_opts_validate(app_getopt_str); 1191 if (ch) { 1192 SPDK_ERRLOG("Duplicated option '%c' between app-specific command line parameter and generic spdk opts.\n", 1193 ch); 1194 goto out; 1195 } 1196 1197 if (!app_parse) { 1198 SPDK_ERRLOG("Parse function is required when app-specific command line parameters are provided.\n"); 1199 goto out; 1200 } 1201 } 1202 1203 cmdline_short_opts = spdk_sprintf_alloc("%s%s", app_getopt_str, SPDK_APP_GETOPT_STRING); 1204 if (!cmdline_short_opts) { 1205 SPDK_ERRLOG("Out of memory\n"); 1206 goto out; 1207 } 1208 1209 g_executable_name = argv[0]; 1210 1211 while ((ch = getopt_long(argc, argv, cmdline_short_opts, cmdline_options, &opt_idx)) != -1) { 1212 switch (ch) { 1213 case CONFIG_FILE_OPT_IDX: 1214 case JSON_CONFIG_OPT_IDX: 1215 opts->json_config_file = optarg; 1216 break; 1217 case JSON_CONFIG_IGNORE_INIT_ERRORS_IDX: 1218 opts->json_config_ignore_errors = true; 1219 break; 1220 case LIMIT_COREDUMP_OPT_IDX: 1221 opts->enable_coredump = false; 1222 break; 1223 case TPOINT_GROUP_OPT_IDX: 1224 opts->tpoint_group_mask = optarg; 1225 break; 1226 case SINGLE_FILE_SEGMENTS_OPT_IDX: 1227 opts->hugepage_single_segments = true; 1228 break; 1229 case HELP_OPT_IDX: 1230 usage(app_usage); 1231 retval = SPDK_APP_PARSE_ARGS_HELP; 1232 goto out; 1233 case SHM_ID_OPT_IDX: 1234 shm_id_str = optarg; 1235 /* a negative shm-id disables shared configuration file */ 1236 if (optarg[0] == '-') { 1237 shm_id_str++; 1238 } 1239 /* check if the positive value of provided shm_id can be parsed as 1240 * an integer 1241 */ 1242 opts->shm_id = spdk_strtol(shm_id_str, 0); 1243 if (opts->shm_id < 0) { 1244 SPDK_ERRLOG("Invalid shared memory ID %s\n", optarg); 1245 goto out; 1246 } 1247 if (optarg[0] == '-') { 1248 opts->shm_id = -opts->shm_id; 1249 } 1250 break; 1251 case CPUMASK_OPT_IDX: 1252 if (opts->lcore_map) { 1253 SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n"); 1254 goto out; 1255 } 1256 opts->reactor_mask = optarg; 1257 break; 1258 case LCORES_OPT_IDX: 1259 if (opts->reactor_mask) { 1260 SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n"); 1261 goto out; 1262 } 1263 opts->lcore_map = optarg; 1264 break; 1265 case DISABLE_CPUMASK_LOCKS_OPT_IDX: 1266 g_disable_cpumask_locks = true; 1267 break; 1268 case MEM_CHANNELS_OPT_IDX: 1269 opts->mem_channel = spdk_strtol(optarg, 0); 1270 if (opts->mem_channel < 0) { 1271 SPDK_ERRLOG("Invalid memory channel %s\n", optarg); 1272 goto out; 1273 } 1274 break; 1275 case MAIN_CORE_OPT_IDX: 1276 opts->main_core = spdk_strtol(optarg, 0); 1277 if (opts->main_core < 0) { 1278 SPDK_ERRLOG("Invalid main core %s\n", optarg); 1279 goto out; 1280 } 1281 break; 1282 case SILENCE_NOTICELOG_OPT_IDX: 1283 opts->print_level = SPDK_LOG_WARN; 1284 break; 1285 case RPC_SOCKET_OPT_IDX: 1286 opts->rpc_addr = optarg; 1287 break; 1288 case NO_RPC_SERVER_OPT_IDX: 1289 opts->rpc_addr = NULL; 1290 break; 1291 case MEM_SIZE_OPT_IDX: { 1292 uint64_t mem_size_mb; 1293 bool mem_size_has_prefix; 1294 1295 rc = spdk_parse_capacity(optarg, &mem_size_mb, &mem_size_has_prefix); 1296 if (rc != 0) { 1297 SPDK_ERRLOG("invalid memory pool size `-s %s`\n", optarg); 1298 usage(app_usage); 1299 goto out; 1300 } 1301 1302 if (mem_size_has_prefix) { 1303 /* the mem size is in MB by default, so if a prefix was 1304 * specified, we need to manually convert to MB. 1305 */ 1306 mem_size_mb /= 1024 * 1024; 1307 } 1308 1309 if (mem_size_mb > INT_MAX) { 1310 SPDK_ERRLOG("invalid memory pool size `-s %s`\n", optarg); 1311 usage(app_usage); 1312 goto out; 1313 } 1314 1315 opts->mem_size = (int) mem_size_mb; 1316 break; 1317 } 1318 case MSG_MEMPOOL_SIZE_OPT_IDX: 1319 tmp = spdk_strtol(optarg, 10); 1320 if (tmp <= 0) { 1321 SPDK_ERRLOG("Invalid message memory pool size %s\n", optarg); 1322 goto out; 1323 } 1324 1325 opts->msg_mempool_size = (size_t)tmp; 1326 break; 1327 1328 case NO_PCI_OPT_IDX: 1329 opts->no_pci = true; 1330 break; 1331 case WAIT_FOR_RPC_OPT_IDX: 1332 opts->delay_subsystem_init = true; 1333 break; 1334 case PCI_BLOCKED_OPT_IDX: 1335 if (opts->pci_allowed) { 1336 free(opts->pci_allowed); 1337 opts->pci_allowed = NULL; 1338 SPDK_ERRLOG("-B and -A cannot be used at the same time\n"); 1339 usage(app_usage); 1340 goto out; 1341 } 1342 1343 rc = app_opts_add_pci_addr(opts, &opts->pci_blocked, optarg); 1344 if (rc != 0) { 1345 free(opts->pci_blocked); 1346 opts->pci_blocked = NULL; 1347 goto out; 1348 } 1349 break; 1350 1351 case NO_HUGE_OPT_IDX: 1352 opts->no_huge = true; 1353 break; 1354 1355 case LOGFLAG_OPT_IDX: 1356 rc = spdk_log_set_flag(optarg); 1357 if (rc < 0) { 1358 SPDK_ERRLOG("unknown flag: %s\n", optarg); 1359 usage(app_usage); 1360 goto out; 1361 } 1362 #ifdef DEBUG 1363 opts->print_level = SPDK_LOG_DEBUG; 1364 #endif 1365 break; 1366 case HUGE_UNLINK_OPT_IDX: 1367 opts->unlink_hugepage = true; 1368 break; 1369 case PCI_ALLOWED_OPT_IDX: 1370 if (opts->pci_blocked) { 1371 free(opts->pci_blocked); 1372 opts->pci_blocked = NULL; 1373 SPDK_ERRLOG("-B and -W cannot be used at the same time\n"); 1374 usage(app_usage); 1375 goto out; 1376 } 1377 1378 rc = app_opts_add_pci_addr(opts, &opts->pci_allowed, optarg); 1379 if (rc != 0) { 1380 free(opts->pci_allowed); 1381 opts->pci_allowed = NULL; 1382 goto out; 1383 } 1384 break; 1385 case BASE_VIRTADDR_OPT_IDX: 1386 tmp = spdk_strtoll(optarg, 0); 1387 if (tmp <= 0) { 1388 SPDK_ERRLOG("Invalid base-virtaddr %s\n", optarg); 1389 usage(app_usage); 1390 goto out; 1391 } 1392 opts->base_virtaddr = (uint64_t)tmp; 1393 break; 1394 case HUGE_DIR_OPT_IDX: 1395 opts->hugedir = optarg; 1396 break; 1397 case IOVA_MODE_OPT_IDX: 1398 opts->iova_mode = optarg; 1399 break; 1400 case NUM_TRACE_ENTRIES_OPT_IDX: 1401 tmp = spdk_strtoll(optarg, 0); 1402 if (tmp < 0) { 1403 SPDK_ERRLOG("Invalid num-trace-entries %s\n", optarg); 1404 usage(app_usage); 1405 goto out; 1406 } 1407 opts->num_entries = (uint64_t)tmp; 1408 if (opts->num_entries > 0 && opts->num_entries & (opts->num_entries - 1)) { 1409 SPDK_ERRLOG("num-trace-entries must be power of 2\n"); 1410 usage(app_usage); 1411 goto out; 1412 } 1413 break; 1414 case ENV_CONTEXT_OPT_IDX: 1415 opts->env_context = optarg; 1416 break; 1417 case RPCS_ALLOWED_OPT_IDX: 1418 opts->rpc_allowlist = (const char **)spdk_strarray_from_string(optarg, ","); 1419 if (opts->rpc_allowlist == NULL) { 1420 SPDK_ERRLOG("Invalid --rpcs-allowed argument\n"); 1421 usage(app_usage); 1422 goto out; 1423 } 1424 break; 1425 case ENV_VF_TOKEN_OPT_IDX: 1426 opts->vf_token = optarg; 1427 break; 1428 case INTERRUPT_MODE_OPT_IDX: 1429 opts->interrupt_mode = true; 1430 break; 1431 case VERSION_OPT_IDX: 1432 printf(SPDK_VERSION_STRING"\n"); 1433 retval = SPDK_APP_PARSE_ARGS_HELP; 1434 goto out; 1435 case '?': 1436 /* 1437 * In the event getopt() above detects an option 1438 * in argv that is NOT in the getopt_str, 1439 * getopt() will return a '?' indicating failure. 1440 */ 1441 usage(app_usage); 1442 goto out; 1443 default: 1444 if (!app_parse) { 1445 SPDK_ERRLOG("Unsupported app-specific command line parameter '%c'.\n", ch); 1446 goto out; 1447 } 1448 1449 rc = app_parse(ch, optarg); 1450 if (rc) { 1451 SPDK_ERRLOG("Parsing app-specific command line parameter '%c' failed: %d\n", ch, rc); 1452 goto out; 1453 } 1454 } 1455 } 1456 1457 retval = SPDK_APP_PARSE_ARGS_SUCCESS; 1458 out: 1459 if (retval != SPDK_APP_PARSE_ARGS_SUCCESS) { 1460 free(opts->pci_blocked); 1461 opts->pci_blocked = NULL; 1462 free(opts->pci_allowed); 1463 opts->pci_allowed = NULL; 1464 spdk_strarray_free((char **)opts->rpc_allowlist); 1465 opts->rpc_allowlist = NULL; 1466 } 1467 free(cmdline_short_opts); 1468 free(cmdline_options); 1469 return retval; 1470 } 1471 1472 void 1473 spdk_app_usage(void) 1474 { 1475 if (g_executable_name == NULL) { 1476 SPDK_ERRLOG("%s not valid before calling spdk_app_parse_args()\n", __func__); 1477 return; 1478 } 1479 1480 usage(NULL); 1481 } 1482 1483 static void 1484 rpc_framework_start_init_cpl(int rc, void *arg1) 1485 { 1486 struct spdk_jsonrpc_request *request = arg1; 1487 1488 assert(spdk_thread_is_app_thread(NULL)); 1489 1490 if (rc) { 1491 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, 1492 "framework_initialization failed"); 1493 return; 1494 } 1495 1496 app_subsystem_init_done(0, NULL); 1497 1498 spdk_jsonrpc_send_bool_response(request, true); 1499 } 1500 1501 static void 1502 rpc_framework_start_init(struct spdk_jsonrpc_request *request, 1503 const struct spdk_json_val *params) 1504 { 1505 if (params != NULL) { 1506 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1507 "framework_start_init requires no parameters"); 1508 return; 1509 } 1510 1511 spdk_rpc_server_pause(g_spdk_app.rpc_addr); 1512 spdk_subsystem_init(rpc_framework_start_init_cpl, request); 1513 } 1514 SPDK_RPC_REGISTER("framework_start_init", rpc_framework_start_init, SPDK_RPC_STARTUP) 1515 1516 struct subsystem_init_poller_ctx { 1517 struct spdk_poller *init_poller; 1518 struct spdk_jsonrpc_request *request; 1519 }; 1520 1521 static int 1522 rpc_subsystem_init_poller_ctx(void *ctx) 1523 { 1524 struct subsystem_init_poller_ctx *poller_ctx = ctx; 1525 1526 if (spdk_rpc_get_state() == SPDK_RPC_RUNTIME) { 1527 spdk_jsonrpc_send_bool_response(poller_ctx->request, true); 1528 spdk_poller_unregister(&poller_ctx->init_poller); 1529 free(poller_ctx); 1530 } 1531 1532 return SPDK_POLLER_BUSY; 1533 } 1534 1535 static void 1536 rpc_framework_wait_init(struct spdk_jsonrpc_request *request, 1537 const struct spdk_json_val *params) 1538 { 1539 struct subsystem_init_poller_ctx *ctx; 1540 1541 if (spdk_rpc_get_state() == SPDK_RPC_RUNTIME) { 1542 spdk_jsonrpc_send_bool_response(request, true); 1543 } else { 1544 ctx = malloc(sizeof(struct subsystem_init_poller_ctx)); 1545 if (ctx == NULL) { 1546 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, 1547 "Unable to allocate memory for the request context\n"); 1548 return; 1549 } 1550 ctx->request = request; 1551 ctx->init_poller = SPDK_POLLER_REGISTER(rpc_subsystem_init_poller_ctx, ctx, 0); 1552 } 1553 } 1554 SPDK_RPC_REGISTER("framework_wait_init", rpc_framework_wait_init, 1555 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1556 1557 static void 1558 rpc_framework_disable_cpumask_locks(struct spdk_jsonrpc_request *request, 1559 const struct spdk_json_val *params) 1560 { 1561 char msg[128]; 1562 int rc; 1563 uint32_t failed_core; 1564 1565 if (params != NULL) { 1566 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1567 "framework_disable_cpumask_locks" 1568 "requires no arguments"); 1569 return; 1570 } 1571 1572 rc = unclaim_cpu_cores(&failed_core); 1573 if (rc) { 1574 snprintf(msg, sizeof(msg), "Failed to unclaim CPU core: %" PRIu32, failed_core); 1575 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg); 1576 return; 1577 } 1578 1579 spdk_jsonrpc_send_bool_response(request, true); 1580 } 1581 SPDK_RPC_REGISTER("framework_disable_cpumask_locks", rpc_framework_disable_cpumask_locks, 1582 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1583 1584 static void 1585 rpc_framework_enable_cpumask_locks(struct spdk_jsonrpc_request *request, 1586 const struct spdk_json_val *params) 1587 { 1588 char msg[128]; 1589 int rc; 1590 uint32_t failed_core; 1591 1592 if (params != NULL) { 1593 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1594 "framework_enable_cpumask_locks" 1595 "requires no arguments"); 1596 return; 1597 } 1598 1599 rc = claim_cpu_cores(&failed_core); 1600 if (rc) { 1601 snprintf(msg, sizeof(msg), "Failed to claim CPU core: %" PRIu32, failed_core); 1602 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg); 1603 return; 1604 } 1605 1606 spdk_jsonrpc_send_bool_response(request, true); 1607 } 1608 SPDK_RPC_REGISTER("framework_enable_cpumask_locks", rpc_framework_enable_cpumask_locks, 1609 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1610