1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk/version.h" 9 10 #include "spdk_internal/event.h" 11 12 #include "spdk/assert.h" 13 #include "spdk/env.h" 14 #include "spdk/init.h" 15 #include "spdk/log.h" 16 #include "spdk/thread.h" 17 #include "spdk/trace.h" 18 #include "spdk/string.h" 19 #include "spdk/scheduler.h" 20 #include "spdk/rpc.h" 21 #include "spdk/util.h" 22 #include "spdk/file.h" 23 #include "spdk/config.h" 24 #include "event_internal.h" 25 26 #define SPDK_APP_DEFAULT_LOG_LEVEL SPDK_LOG_NOTICE 27 #define SPDK_APP_DEFAULT_LOG_PRINT_LEVEL SPDK_LOG_INFO 28 #define SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES SPDK_DEFAULT_NUM_TRACE_ENTRIES 29 30 #define SPDK_APP_DPDK_DEFAULT_MEM_SIZE -1 31 #define SPDK_APP_DPDK_DEFAULT_MAIN_CORE -1 32 #define SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL -1 33 #define SPDK_APP_DPDK_DEFAULT_CORE_MASK "0x1" 34 #define SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR 0x200000000000 35 #define SPDK_APP_DEFAULT_CORE_LIMIT 0x140000000 /* 5 GiB */ 36 37 /* For core counts <= 63, the message memory pool size is set to 38 * SPDK_DEFAULT_MSG_MEMPOOL_SIZE. 39 * For core counts > 63, the message memory pool size is depended on 40 * number of cores. Per core, it is calculated as SPDK_MSG_MEMPOOL_CACHE_SIZE 41 * multiplied by factor of 4 to have space for multiple spdk threads running 42 * on single core (e.g iscsi + nvmf + vhost ). */ 43 #define SPDK_APP_PER_CORE_MSG_MEMPOOL_SIZE (4 * SPDK_MSG_MEMPOOL_CACHE_SIZE) 44 45 struct spdk_app { 46 void *json_data; 47 size_t json_data_size; 48 bool json_config_ignore_errors; 49 bool stopped; 50 const char *rpc_addr; 51 const char **rpc_allowlist; 52 FILE *rpc_log_file; 53 enum spdk_log_level rpc_log_level; 54 int shm_id; 55 spdk_app_shutdown_cb shutdown_cb; 56 int rc; 57 }; 58 59 static struct spdk_app g_spdk_app; 60 static spdk_msg_fn g_start_fn = NULL; 61 static void *g_start_arg = NULL; 62 static bool g_delay_subsystem_init = false; 63 static bool g_shutdown_sig_received = false; 64 static char *g_executable_name; 65 static struct spdk_app_opts g_default_opts; 66 67 static int g_core_locks[SPDK_CONFIG_MAX_LCORES]; 68 69 static struct { 70 uint64_t irq; 71 uint64_t usr; 72 uint64_t sys; 73 } g_initial_stat[SPDK_CONFIG_MAX_LCORES]; 74 75 int 76 spdk_app_get_shm_id(void) 77 { 78 return g_spdk_app.shm_id; 79 } 80 81 /* append one empty option to indicate the end of the array */ 82 static const struct option g_cmdline_options[] = { 83 #define CONFIG_FILE_OPT_IDX 'c' 84 {"config", required_argument, NULL, CONFIG_FILE_OPT_IDX}, 85 #define LIMIT_COREDUMP_OPT_IDX 'd' 86 {"limit-coredump", no_argument, NULL, LIMIT_COREDUMP_OPT_IDX}, 87 #define TPOINT_GROUP_OPT_IDX 'e' 88 {"tpoint-group", required_argument, NULL, TPOINT_GROUP_OPT_IDX}, 89 #define SINGLE_FILE_SEGMENTS_OPT_IDX 'g' 90 {"single-file-segments", no_argument, NULL, SINGLE_FILE_SEGMENTS_OPT_IDX}, 91 #define HELP_OPT_IDX 'h' 92 {"help", no_argument, NULL, HELP_OPT_IDX}, 93 #define SHM_ID_OPT_IDX 'i' 94 {"shm-id", required_argument, NULL, SHM_ID_OPT_IDX}, 95 #define CPUMASK_OPT_IDX 'm' 96 {"cpumask", required_argument, NULL, CPUMASK_OPT_IDX}, 97 #define MEM_CHANNELS_OPT_IDX 'n' 98 {"mem-channels", required_argument, NULL, MEM_CHANNELS_OPT_IDX}, 99 #define MAIN_CORE_OPT_IDX 'p' 100 {"main-core", required_argument, NULL, MAIN_CORE_OPT_IDX}, 101 #define RPC_SOCKET_OPT_IDX 'r' 102 {"rpc-socket", required_argument, NULL, RPC_SOCKET_OPT_IDX}, 103 #define MEM_SIZE_OPT_IDX 's' 104 {"mem-size", required_argument, NULL, MEM_SIZE_OPT_IDX}, 105 #define NO_PCI_OPT_IDX 'u' 106 {"no-pci", no_argument, NULL, NO_PCI_OPT_IDX}, 107 #define VERSION_OPT_IDX 'v' 108 {"version", no_argument, NULL, VERSION_OPT_IDX}, 109 #define PCI_BLOCKED_OPT_IDX 'B' 110 {"pci-blocked", required_argument, NULL, PCI_BLOCKED_OPT_IDX}, 111 #define LOGFLAG_OPT_IDX 'L' 112 {"logflag", required_argument, NULL, LOGFLAG_OPT_IDX}, 113 #define HUGE_UNLINK_OPT_IDX 'R' 114 {"huge-unlink", no_argument, NULL, HUGE_UNLINK_OPT_IDX}, 115 #define PCI_ALLOWED_OPT_IDX 'A' 116 {"pci-allowed", required_argument, NULL, PCI_ALLOWED_OPT_IDX}, 117 #define INTERRUPT_MODE_OPT_IDX 256 118 {"interrupt-mode", no_argument, NULL, INTERRUPT_MODE_OPT_IDX}, 119 #define SILENCE_NOTICELOG_OPT_IDX 257 120 {"silence-noticelog", no_argument, NULL, SILENCE_NOTICELOG_OPT_IDX}, 121 #define WAIT_FOR_RPC_OPT_IDX 258 122 {"wait-for-rpc", no_argument, NULL, WAIT_FOR_RPC_OPT_IDX}, 123 #define HUGE_DIR_OPT_IDX 259 124 {"huge-dir", required_argument, NULL, HUGE_DIR_OPT_IDX}, 125 #define NUM_TRACE_ENTRIES_OPT_IDX 260 126 {"num-trace-entries", required_argument, NULL, NUM_TRACE_ENTRIES_OPT_IDX}, 127 #define JSON_CONFIG_OPT_IDX 262 128 {"json", required_argument, NULL, JSON_CONFIG_OPT_IDX}, 129 #define JSON_CONFIG_IGNORE_INIT_ERRORS_IDX 263 130 {"json-ignore-init-errors", no_argument, NULL, JSON_CONFIG_IGNORE_INIT_ERRORS_IDX}, 131 #define IOVA_MODE_OPT_IDX 264 132 {"iova-mode", required_argument, NULL, IOVA_MODE_OPT_IDX}, 133 #define BASE_VIRTADDR_OPT_IDX 265 134 {"base-virtaddr", required_argument, NULL, BASE_VIRTADDR_OPT_IDX}, 135 #define ENV_CONTEXT_OPT_IDX 266 136 {"env-context", required_argument, NULL, ENV_CONTEXT_OPT_IDX}, 137 #define DISABLE_CPUMASK_LOCKS_OPT_IDX 267 138 {"disable-cpumask-locks", no_argument, NULL, DISABLE_CPUMASK_LOCKS_OPT_IDX}, 139 #define RPCS_ALLOWED_OPT_IDX 268 140 {"rpcs-allowed", required_argument, NULL, RPCS_ALLOWED_OPT_IDX}, 141 #define ENV_VF_TOKEN_OPT_IDX 269 142 {"vfio-vf-token", required_argument, NULL, ENV_VF_TOKEN_OPT_IDX}, 143 #define MSG_MEMPOOL_SIZE_OPT_IDX 270 144 {"msg-mempool-size", required_argument, NULL, MSG_MEMPOOL_SIZE_OPT_IDX}, 145 #define LCORES_OPT_IDX 271 146 {"lcores", required_argument, NULL, LCORES_OPT_IDX}, 147 #define NO_HUGE_OPT_IDX 272 148 {"no-huge", no_argument, NULL, NO_HUGE_OPT_IDX}, 149 #define NO_RPC_SERVER_OPT_IDX 273 150 {"no-rpc-server", no_argument, NULL, NO_RPC_SERVER_OPT_IDX}, 151 }; 152 153 static int 154 parse_proc_stat(unsigned int core, uint64_t *user, uint64_t *sys, uint64_t *irq) 155 { 156 FILE *f; 157 uint64_t i, soft_irq, cpu = 0; 158 int rc, found = 0; 159 160 f = fopen("/proc/stat", "r"); 161 if (!f) { 162 return -1; 163 } 164 165 for (i = 0; i <= core + 1; i++) { 166 /* scanf discards input with '*' in format, 167 * cpu;user;nice;system;idle;iowait;irq;softirq;steal;guest;guest_nice */ 168 rc = fscanf(f, "cpu%li %li %*i %li %*i %*i %li %li %*i %*i %*i\n", 169 &cpu, user, sys, irq, &soft_irq); 170 if (rc != 5) { 171 continue; 172 } 173 174 /* some cores can be disabled, list may not be in order */ 175 if (cpu == core) { 176 found = 1; 177 break; 178 } 179 } 180 181 *irq += soft_irq; 182 183 fclose(f); 184 return found ? 0 : -1; 185 } 186 187 static int 188 init_proc_stat(unsigned int core) 189 { 190 uint64_t usr, sys, irq; 191 192 if (core >= SPDK_CONFIG_MAX_LCORES) { 193 return -1; 194 } 195 196 if (parse_proc_stat(core, &usr, &sys, &irq) < 0) { 197 return -1; 198 } 199 200 g_initial_stat[core].irq = irq; 201 g_initial_stat[core].usr = usr; 202 g_initial_stat[core].sys = sys; 203 204 return 0; 205 } 206 207 int 208 app_get_proc_stat(unsigned int core, uint64_t *usr, uint64_t *sys, uint64_t *irq) 209 { 210 uint64_t _usr, _sys, _irq; 211 212 if (core >= SPDK_CONFIG_MAX_LCORES) { 213 return -1; 214 } 215 216 if (parse_proc_stat(core, &_usr, &_sys, &_irq) < 0) { 217 return -1; 218 } 219 220 *irq = _irq - g_initial_stat[core].irq; 221 *usr = _usr - g_initial_stat[core].usr; 222 *sys = _sys - g_initial_stat[core].sys; 223 224 return 0; 225 } 226 227 static void 228 app_start_shutdown(void *ctx) 229 { 230 if (g_spdk_app.shutdown_cb) { 231 g_spdk_app.shutdown_cb(); 232 g_spdk_app.shutdown_cb = NULL; 233 } else { 234 spdk_app_stop(0); 235 } 236 } 237 238 void 239 spdk_app_start_shutdown(void) 240 { 241 spdk_thread_send_critical_msg(spdk_thread_get_app_thread(), app_start_shutdown); 242 } 243 244 static void 245 __shutdown_signal(int signo) 246 { 247 if (!g_shutdown_sig_received) { 248 g_shutdown_sig_received = true; 249 spdk_app_start_shutdown(); 250 } 251 } 252 253 static int 254 app_opts_validate(const char *app_opts) 255 { 256 int i = 0, j; 257 258 for (i = 0; app_opts[i] != '\0'; i++) { 259 /* ignore getopt control characters */ 260 if (app_opts[i] == ':' || app_opts[i] == '+' || app_opts[i] == '-') { 261 continue; 262 } 263 264 for (j = 0; SPDK_APP_GETOPT_STRING[j] != '\0'; j++) { 265 if (app_opts[i] == SPDK_APP_GETOPT_STRING[j]) { 266 return app_opts[i]; 267 } 268 } 269 } 270 return 0; 271 } 272 273 static void 274 calculate_mempool_size(struct spdk_app_opts *opts, 275 struct spdk_app_opts *opts_user) 276 { 277 uint32_t core_count = spdk_env_get_core_count(); 278 279 if (!opts_user->msg_mempool_size) { 280 /* The user didn't specify msg_mempool_size, so let's calculate it. 281 Set the default (SPDK_DEFAULT_MSG_MEMPOOL_SIZE) if less than 282 64 cores, and use 4k per core otherwise */ 283 opts->msg_mempool_size = spdk_max(SPDK_DEFAULT_MSG_MEMPOOL_SIZE, 284 core_count * SPDK_APP_PER_CORE_MSG_MEMPOOL_SIZE); 285 } else { 286 opts->msg_mempool_size = opts_user->msg_mempool_size; 287 } 288 } 289 290 void 291 spdk_app_opts_init(struct spdk_app_opts *opts, size_t opts_size) 292 { 293 if (!opts) { 294 SPDK_ERRLOG("opts should not be NULL\n"); 295 return; 296 } 297 298 if (!opts_size) { 299 SPDK_ERRLOG("opts_size should not be zero value\n"); 300 return; 301 } 302 303 memset(opts, 0, opts_size); 304 opts->opts_size = opts_size; 305 306 #define SET_FIELD(field, value) \ 307 if (offsetof(struct spdk_app_opts, field) + sizeof(opts->field) <= opts_size) { \ 308 opts->field = value; \ 309 } \ 310 311 SET_FIELD(enable_coredump, true); 312 SET_FIELD(shm_id, -1); 313 SET_FIELD(mem_size, SPDK_APP_DPDK_DEFAULT_MEM_SIZE); 314 SET_FIELD(main_core, SPDK_APP_DPDK_DEFAULT_MAIN_CORE); 315 SET_FIELD(mem_channel, SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL); 316 SET_FIELD(base_virtaddr, SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR); 317 SET_FIELD(print_level, SPDK_APP_DEFAULT_LOG_PRINT_LEVEL); 318 SET_FIELD(rpc_addr, SPDK_DEFAULT_RPC_ADDR); 319 SET_FIELD(num_entries, SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES); 320 SET_FIELD(delay_subsystem_init, false); 321 SET_FIELD(disable_signal_handlers, false); 322 SET_FIELD(interrupt_mode, false); 323 /* Don't set msg_mempool_size here, it is set or calculated later */ 324 SET_FIELD(rpc_allowlist, NULL); 325 SET_FIELD(rpc_log_file, NULL); 326 SET_FIELD(rpc_log_level, SPDK_LOG_DISABLED); 327 SET_FIELD(disable_cpumask_locks, false); 328 #undef SET_FIELD 329 } 330 331 static int 332 app_setup_signal_handlers(struct spdk_app_opts *opts) 333 { 334 struct sigaction sigact; 335 sigset_t sigmask; 336 int rc; 337 338 sigemptyset(&sigmask); 339 memset(&sigact, 0, sizeof(sigact)); 340 sigemptyset(&sigact.sa_mask); 341 342 sigact.sa_handler = SIG_IGN; 343 rc = sigaction(SIGPIPE, &sigact, NULL); 344 if (rc < 0) { 345 SPDK_ERRLOG("sigaction(SIGPIPE) failed\n"); 346 return rc; 347 } 348 349 /* Install the same handler for SIGINT and SIGTERM */ 350 g_shutdown_sig_received = false; 351 sigact.sa_handler = __shutdown_signal; 352 rc = sigaction(SIGINT, &sigact, NULL); 353 if (rc < 0) { 354 SPDK_ERRLOG("sigaction(SIGINT) failed\n"); 355 return rc; 356 } 357 sigaddset(&sigmask, SIGINT); 358 359 rc = sigaction(SIGTERM, &sigact, NULL); 360 if (rc < 0) { 361 SPDK_ERRLOG("sigaction(SIGTERM) failed\n"); 362 return rc; 363 } 364 sigaddset(&sigmask, SIGTERM); 365 366 pthread_sigmask(SIG_UNBLOCK, &sigmask, NULL); 367 368 return 0; 369 } 370 371 static void 372 app_start_application(int rc, void *arg1) 373 { 374 assert(spdk_thread_is_app_thread(NULL)); 375 376 if (rc) { 377 SPDK_ERRLOG("Failed to load subsystems for RUNTIME state with code: %d\n", rc); 378 spdk_app_stop(rc); 379 return; 380 } 381 382 if (g_spdk_app.rpc_addr) { 383 spdk_rpc_server_resume(g_spdk_app.rpc_addr); 384 } 385 386 g_start_fn(g_start_arg); 387 } 388 389 static void 390 app_subsystem_init_done(int rc, void *arg1) 391 { 392 if (rc) { 393 SPDK_ERRLOG("Subsystem initialization failed with code: %d\n", rc); 394 spdk_app_stop(rc); 395 return; 396 } 397 398 spdk_rpc_set_allowlist(g_spdk_app.rpc_allowlist); 399 spdk_rpc_set_state(SPDK_RPC_RUNTIME); 400 401 if (g_spdk_app.json_data) { 402 /* Load SPDK_RPC_RUNTIME RPCs from config file */ 403 assert(spdk_rpc_get_state() == SPDK_RPC_RUNTIME); 404 spdk_subsystem_load_config(g_spdk_app.json_data, g_spdk_app.json_data_size, 405 app_start_application, NULL, 406 !g_spdk_app.json_config_ignore_errors); 407 free(g_spdk_app.json_data); 408 g_spdk_app.json_data = NULL; 409 } else { 410 app_start_application(0, NULL); 411 } 412 } 413 414 static void 415 app_do_spdk_subsystem_init(int rc, void *arg1) 416 { 417 struct spdk_rpc_opts opts; 418 419 if (rc) { 420 spdk_app_stop(rc); 421 return; 422 } 423 424 if (g_spdk_app.rpc_addr) { 425 opts.size = SPDK_SIZEOF(&opts, log_level); 426 opts.log_file = g_spdk_app.rpc_log_file; 427 opts.log_level = g_spdk_app.rpc_log_level; 428 429 rc = spdk_rpc_initialize(g_spdk_app.rpc_addr, &opts); 430 if (rc) { 431 spdk_app_stop(rc); 432 return; 433 } 434 if (g_delay_subsystem_init) { 435 return; 436 } 437 spdk_rpc_server_pause(g_spdk_app.rpc_addr); 438 } else { 439 SPDK_DEBUGLOG(app_rpc, "RPC server not started\n"); 440 } 441 spdk_subsystem_init(app_subsystem_init_done, NULL); 442 } 443 444 static int 445 app_opts_add_pci_addr(struct spdk_app_opts *opts, struct spdk_pci_addr **list, char *bdf) 446 { 447 struct spdk_pci_addr *tmp = *list; 448 size_t i = opts->num_pci_addr; 449 450 tmp = realloc(tmp, sizeof(*tmp) * (i + 1)); 451 if (tmp == NULL) { 452 SPDK_ERRLOG("realloc error\n"); 453 return -ENOMEM; 454 } 455 456 *list = tmp; 457 if (spdk_pci_addr_parse(*list + i, bdf) < 0) { 458 SPDK_ERRLOG("Invalid address %s\n", bdf); 459 return -EINVAL; 460 } 461 462 opts->num_pci_addr++; 463 return 0; 464 } 465 466 static int 467 app_setup_env(struct spdk_app_opts *opts) 468 { 469 struct spdk_env_opts env_opts = {}; 470 int rc; 471 472 if (opts == NULL) { 473 rc = spdk_env_init(NULL); 474 if (rc != 0) { 475 SPDK_ERRLOG("Unable to reinitialize SPDK env\n"); 476 } 477 478 return rc; 479 } 480 481 env_opts.opts_size = sizeof(env_opts); 482 spdk_env_opts_init(&env_opts); 483 484 env_opts.name = opts->name; 485 env_opts.core_mask = opts->reactor_mask; 486 env_opts.lcore_map = opts->lcore_map; 487 env_opts.shm_id = opts->shm_id; 488 env_opts.mem_channel = opts->mem_channel; 489 env_opts.main_core = opts->main_core; 490 env_opts.mem_size = opts->mem_size; 491 env_opts.hugepage_single_segments = opts->hugepage_single_segments; 492 env_opts.unlink_hugepage = opts->unlink_hugepage; 493 env_opts.hugedir = opts->hugedir; 494 env_opts.no_pci = opts->no_pci; 495 env_opts.num_pci_addr = opts->num_pci_addr; 496 env_opts.pci_blocked = opts->pci_blocked; 497 env_opts.pci_allowed = opts->pci_allowed; 498 env_opts.base_virtaddr = opts->base_virtaddr; 499 env_opts.env_context = opts->env_context; 500 env_opts.iova_mode = opts->iova_mode; 501 env_opts.vf_token = opts->vf_token; 502 env_opts.no_huge = opts->no_huge; 503 504 rc = spdk_env_init(&env_opts); 505 free(env_opts.pci_blocked); 506 free(env_opts.pci_allowed); 507 508 if (rc < 0) { 509 SPDK_ERRLOG("Unable to initialize SPDK env\n"); 510 } 511 512 return rc; 513 } 514 515 static int 516 app_setup_trace(struct spdk_app_opts *opts) 517 { 518 char shm_name[64]; 519 uint64_t tpoint_group_mask, tpoint_mask = -1ULL; 520 char *end = NULL, *tpoint_group_mask_str, *tpoint_group_str = NULL; 521 char *tp_g_str, *tpoint_group, *tpoints; 522 bool error_found = false; 523 uint64_t group_id; 524 525 if (opts->shm_id >= 0) { 526 snprintf(shm_name, sizeof(shm_name), "/%s%s%d", opts->name, 527 SPDK_TRACE_SHM_NAME_BASE, opts->shm_id); 528 } else { 529 snprintf(shm_name, sizeof(shm_name), "/%s%spid%d", opts->name, 530 SPDK_TRACE_SHM_NAME_BASE, (int)getpid()); 531 } 532 533 if (spdk_trace_init(shm_name, opts->num_entries, 0) != 0) { 534 return -1; 535 } 536 537 if (opts->tpoint_group_mask == NULL) { 538 return 0; 539 } 540 541 tpoint_group_mask_str = strdup(opts->tpoint_group_mask); 542 if (tpoint_group_mask_str == NULL) { 543 SPDK_ERRLOG("Unable to get string of tpoint group mask from opts.\n"); 544 return -1; 545 } 546 /* Save a pointer to the original value of the tpoint group mask string 547 * to free later, because spdk_strsepq() modifies given char*. */ 548 tp_g_str = tpoint_group_mask_str; 549 while ((tpoint_group_str = spdk_strsepq(&tpoint_group_mask_str, ",")) != NULL) { 550 if (strchr(tpoint_group_str, ':')) { 551 /* Get the tpoint group mask */ 552 tpoint_group = spdk_strsepq(&tpoint_group_str, ":"); 553 /* Get the tpoint mask inside that group */ 554 tpoints = spdk_strsepq(&tpoint_group_str, ":"); 555 556 errno = 0; 557 tpoint_group_mask = strtoull(tpoint_group, &end, 16); 558 if (*end != '\0' || errno) { 559 tpoint_group_mask = spdk_trace_create_tpoint_group_mask(tpoint_group); 560 if (tpoint_group_mask == 0) { 561 error_found = true; 562 break; 563 } 564 } 565 /* Check if tpoint group mask has only one bit set. 566 * This is to avoid enabling individual tpoints in 567 * more than one tracepoint group at once. */ 568 if (!spdk_u64_is_pow2(tpoint_group_mask)) { 569 SPDK_ERRLOG("Tpoint group mask: %s contains multiple tpoint groups.\n", tpoint_group); 570 SPDK_ERRLOG("This is not supported, to prevent from activating tpoints by mistake.\n"); 571 error_found = true; 572 break; 573 } 574 575 errno = 0; 576 tpoint_mask = strtoull(tpoints, &end, 16); 577 if (*end != '\0' || errno) { 578 error_found = true; 579 break; 580 } 581 } else { 582 errno = 0; 583 tpoint_group_mask = strtoull(tpoint_group_str, &end, 16); 584 if (*end != '\0' || errno) { 585 tpoint_group_mask = spdk_trace_create_tpoint_group_mask(tpoint_group_str); 586 if (tpoint_group_mask == 0) { 587 error_found = true; 588 break; 589 } 590 } 591 tpoint_mask = -1ULL; 592 } 593 594 for (group_id = 0; group_id < SPDK_TRACE_MAX_GROUP_ID; ++group_id) { 595 if (tpoint_group_mask & (1 << group_id)) { 596 spdk_trace_set_tpoints(group_id, tpoint_mask); 597 } 598 } 599 } 600 601 if (error_found) { 602 SPDK_ERRLOG("invalid tpoint mask %s\n", opts->tpoint_group_mask); 603 free(tp_g_str); 604 return -1; 605 } else { 606 SPDK_NOTICELOG("Tracepoint Group Mask %s specified.\n", opts->tpoint_group_mask); 607 SPDK_NOTICELOG("Use 'spdk_trace -s %s %s %d' to capture a snapshot of events at runtime.\n", 608 opts->name, 609 opts->shm_id >= 0 ? "-i" : "-p", 610 opts->shm_id >= 0 ? opts->shm_id : getpid()); 611 #if defined(__linux__) 612 SPDK_NOTICELOG("'spdk_trace' without parameters will also work if this is the only\n"); 613 SPDK_NOTICELOG("SPDK application currently running.\n"); 614 SPDK_NOTICELOG("Or copy /dev/shm%s for offline analysis/debug.\n", shm_name); 615 #endif 616 } 617 free(tp_g_str); 618 619 return 0; 620 } 621 622 static void 623 bootstrap_fn(void *arg1) 624 { 625 spdk_rpc_set_allowlist(g_spdk_app.rpc_allowlist); 626 627 if (g_spdk_app.json_data) { 628 /* Load SPDK_RPC_STARTUP RPCs from config file */ 629 assert(spdk_rpc_get_state() == SPDK_RPC_STARTUP); 630 spdk_subsystem_load_config(g_spdk_app.json_data, g_spdk_app.json_data_size, 631 app_do_spdk_subsystem_init, NULL, 632 !g_spdk_app.json_config_ignore_errors); 633 } else { 634 app_do_spdk_subsystem_init(0, NULL); 635 } 636 } 637 638 static void 639 app_copy_opts(struct spdk_app_opts *opts, struct spdk_app_opts *opts_user, size_t opts_size) 640 { 641 spdk_app_opts_init(opts, sizeof(*opts)); 642 opts->opts_size = opts_size; 643 644 #define SET_FIELD(field) \ 645 if (offsetof(struct spdk_app_opts, field) + sizeof(opts->field) <= (opts->opts_size)) { \ 646 opts->field = opts_user->field; \ 647 } \ 648 649 SET_FIELD(name); 650 SET_FIELD(json_config_file); 651 SET_FIELD(json_config_ignore_errors); 652 SET_FIELD(rpc_addr); 653 SET_FIELD(reactor_mask); 654 SET_FIELD(lcore_map); 655 SET_FIELD(tpoint_group_mask); 656 SET_FIELD(shm_id); 657 SET_FIELD(shutdown_cb); 658 SET_FIELD(enable_coredump); 659 SET_FIELD(mem_channel); 660 SET_FIELD(main_core); 661 SET_FIELD(mem_size); 662 SET_FIELD(no_pci); 663 SET_FIELD(hugepage_single_segments); 664 SET_FIELD(unlink_hugepage); 665 SET_FIELD(no_huge); 666 SET_FIELD(hugedir); 667 SET_FIELD(print_level); 668 SET_FIELD(num_pci_addr); 669 SET_FIELD(pci_blocked); 670 SET_FIELD(pci_allowed); 671 SET_FIELD(iova_mode); 672 SET_FIELD(delay_subsystem_init); 673 SET_FIELD(num_entries); 674 SET_FIELD(env_context); 675 SET_FIELD(log); 676 SET_FIELD(base_virtaddr); 677 SET_FIELD(disable_signal_handlers); 678 SET_FIELD(interrupt_mode); 679 SET_FIELD(msg_mempool_size); 680 SET_FIELD(rpc_allowlist); 681 SET_FIELD(vf_token); 682 SET_FIELD(rpc_log_file); 683 SET_FIELD(rpc_log_level); 684 SET_FIELD(json_data); 685 SET_FIELD(json_data_size); 686 SET_FIELD(disable_cpumask_locks); 687 688 /* You should not remove this statement, but need to update the assert statement 689 * if you add a new field, and also add a corresponding SET_FIELD statement */ 690 SPDK_STATIC_ASSERT(sizeof(struct spdk_app_opts) == 253, "Incorrect size"); 691 692 #undef SET_FIELD 693 } 694 695 static int 696 unclaim_cpu_cores(uint32_t *failed_core) 697 { 698 char core_name[40]; 699 uint32_t i; 700 int rc; 701 702 for (i = 0; i < SPDK_CONFIG_MAX_LCORES; i++) { 703 if (g_core_locks[i] != -1) { 704 snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", i); 705 rc = close(g_core_locks[i]); 706 if (rc) { 707 SPDK_ERRLOG("Failed to close lock fd for core %d, errno: %d\n", i, errno); 708 goto error; 709 } 710 711 g_core_locks[i] = -1; 712 rc = unlink(core_name); 713 if (rc) { 714 SPDK_ERRLOG("Failed to unlink lock fd for core %d, errno: %d\n", i, errno); 715 goto error; 716 } 717 } 718 } 719 720 return 0; 721 722 error: 723 if (failed_core != NULL) { 724 /* Set number of core we failed to claim. */ 725 *failed_core = i; 726 } 727 return -1; 728 } 729 730 static int 731 claim_cpu_cores(uint32_t *failed_core) 732 { 733 char core_name[40]; 734 int core_fd, pid; 735 int *core_map; 736 uint32_t core; 737 738 struct flock core_lock = { 739 .l_type = F_WRLCK, 740 .l_whence = SEEK_SET, 741 .l_start = 0, 742 .l_len = 0, 743 }; 744 745 SPDK_ENV_FOREACH_CORE(core) { 746 if (g_core_locks[core] != -1) { 747 /* If this core is locked already, do not try lock it again. */ 748 continue; 749 } 750 751 snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", core); 752 core_fd = open(core_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 753 if (core_fd == -1) { 754 SPDK_ERRLOG("Could not open %s (%s).\n", core_name, spdk_strerror(errno)); 755 /* Return number of core we failed to claim. */ 756 goto error; 757 } 758 759 if (ftruncate(core_fd, sizeof(int)) != 0) { 760 SPDK_ERRLOG("Could not truncate %s (%s).\n", core_name, spdk_strerror(errno)); 761 close(core_fd); 762 goto error; 763 } 764 765 core_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE, MAP_SHARED, core_fd, 0); 766 if (core_map == MAP_FAILED) { 767 SPDK_ERRLOG("Could not mmap core %s (%s).\n", core_name, spdk_strerror(errno)); 768 close(core_fd); 769 goto error; 770 } 771 772 if (fcntl(core_fd, F_SETLK, &core_lock) != 0) { 773 pid = *core_map; 774 SPDK_ERRLOG("Cannot create lock on core %" PRIu32 ", probably process %d has claimed it.\n", 775 core, pid); 776 munmap(core_map, sizeof(int)); 777 close(core_fd); 778 goto error; 779 } 780 781 /* We write the PID to the core lock file so that other processes trying 782 * to claim the same core will know what process is holding the lock. */ 783 *core_map = (int)getpid(); 784 munmap(core_map, sizeof(int)); 785 g_core_locks[core] = core_fd; 786 /* Keep core_fd open to maintain the lock. */ 787 } 788 789 return 0; 790 791 error: 792 if (failed_core != NULL) { 793 /* Set number of core we failed to claim. */ 794 *failed_core = core; 795 } 796 unclaim_cpu_cores(NULL); 797 return -1; 798 } 799 800 int 801 spdk_app_start(struct spdk_app_opts *opts_user, spdk_msg_fn start_fn, 802 void *arg1) 803 { 804 int rc; 805 char *tty; 806 struct spdk_cpuset tmp_cpumask = {}; 807 static bool g_env_was_setup = false; 808 struct spdk_app_opts opts_local = {}; 809 struct spdk_app_opts *opts = &opts_local; 810 uint32_t i, core; 811 812 if (!opts_user) { 813 SPDK_ERRLOG("opts_user should not be NULL\n"); 814 return 1; 815 } 816 817 if (!opts_user->opts_size) { 818 SPDK_ERRLOG("The opts_size in opts_user structure should not be zero value\n"); 819 return 1; 820 } 821 822 if (opts_user->name == NULL) { 823 SPDK_ERRLOG("spdk_app_opts::name not specified\n"); 824 return 1; 825 } 826 827 app_copy_opts(opts, opts_user, opts_user->opts_size); 828 829 if (!start_fn) { 830 SPDK_ERRLOG("start_fn should not be NULL\n"); 831 return 1; 832 } 833 834 if (!opts->rpc_addr && opts->delay_subsystem_init) { 835 SPDK_ERRLOG("Cannot use '--wait-for-rpc' if no RPC server is going to be started.\n"); 836 return 1; 837 } 838 839 if (!(opts->lcore_map || opts->reactor_mask)) { 840 /* Set default CPU mask */ 841 opts->reactor_mask = SPDK_APP_DPDK_DEFAULT_CORE_MASK; 842 } 843 844 tty = ttyname(STDERR_FILENO); 845 if (opts->print_level > SPDK_LOG_WARN && 846 isatty(STDERR_FILENO) && 847 tty && 848 !strncmp(tty, "/dev/tty", strlen("/dev/tty"))) { 849 printf("Warning: printing stderr to console terminal without -q option specified.\n"); 850 printf("Suggest using --silence-noticelog to disable logging to stderr and\n"); 851 printf("monitor syslog, or redirect stderr to a file.\n"); 852 printf("(Delaying for 10 seconds...)\n"); 853 sleep(10); 854 } 855 856 spdk_log_set_print_level(opts->print_level); 857 858 #ifndef SPDK_NO_RLIMIT 859 if (opts->enable_coredump) { 860 struct rlimit core_limits; 861 862 core_limits.rlim_cur = core_limits.rlim_max = SPDK_APP_DEFAULT_CORE_LIMIT; 863 setrlimit(RLIMIT_CORE, &core_limits); 864 } 865 #endif 866 867 if (opts->interrupt_mode) { 868 spdk_interrupt_mode_enable(); 869 } 870 871 memset(&g_spdk_app, 0, sizeof(g_spdk_app)); 872 873 g_spdk_app.json_config_ignore_errors = opts->json_config_ignore_errors; 874 g_spdk_app.rpc_addr = opts->rpc_addr; 875 g_spdk_app.rpc_allowlist = opts->rpc_allowlist; 876 g_spdk_app.rpc_log_file = opts->rpc_log_file; 877 g_spdk_app.rpc_log_level = opts->rpc_log_level; 878 g_spdk_app.shm_id = opts->shm_id; 879 g_spdk_app.shutdown_cb = opts->shutdown_cb; 880 g_spdk_app.rc = 0; 881 g_spdk_app.stopped = false; 882 883 spdk_log_set_level(SPDK_APP_DEFAULT_LOG_LEVEL); 884 885 /* Pass NULL to app_setup_env if SPDK app has been set up, in order to 886 * indicate that this is a reinitialization. 887 */ 888 if (app_setup_env(g_env_was_setup ? NULL : opts) < 0) { 889 return 1; 890 } 891 892 /* Calculate mempool size now that the env layer has configured the core count 893 * for the application */ 894 calculate_mempool_size(opts, opts_user); 895 896 spdk_log_open(opts->log); 897 898 /* Initialize each lock to -1 to indicate "empty" status */ 899 for (i = 0; i < SPDK_CONFIG_MAX_LCORES; i++) { 900 g_core_locks[i] = -1; 901 } 902 903 if (!opts->disable_cpumask_locks) { 904 if (claim_cpu_cores(NULL)) { 905 SPDK_ERRLOG("Unable to acquire lock on assigned core mask - exiting.\n"); 906 return 1; 907 } 908 } else { 909 SPDK_NOTICELOG("CPU core locks deactivated.\n"); 910 } 911 912 SPDK_NOTICELOG("Total cores available: %d\n", spdk_env_get_core_count()); 913 914 if ((rc = spdk_reactors_init(opts->msg_mempool_size)) != 0) { 915 SPDK_ERRLOG("Reactor Initialization failed: rc = %d\n", rc); 916 return 1; 917 } 918 919 spdk_cpuset_set_cpu(&tmp_cpumask, spdk_env_get_current_core(), true); 920 921 /* Now that the reactors have been initialized, we can create the app thread. */ 922 spdk_thread_create("app_thread", &tmp_cpumask); 923 if (!spdk_thread_get_app_thread()) { 924 SPDK_ERRLOG("Unable to create an spdk_thread for initialization\n"); 925 return 1; 926 } 927 928 SPDK_ENV_FOREACH_CORE(core) { 929 rc = init_proc_stat(core); 930 if (rc) { 931 SPDK_NOTICELOG("Unable to parse /proc/stat [core: %d].\n", core); 932 } 933 } 934 /* 935 * Disable and ignore trace setup if setting num_entries 936 * to be 0. 937 * 938 * Note the call to app_setup_trace() is located here 939 * ahead of app_setup_signal_handlers(). 940 * That's because there is not an easy/direct clean 941 * way of unwinding alloc'd resources that can occur 942 * in app_setup_signal_handlers(). 943 */ 944 if (opts->num_entries != 0 && app_setup_trace(opts) != 0) { 945 return 1; 946 } 947 948 if (!opts->disable_signal_handlers && app_setup_signal_handlers(opts) != 0) { 949 return 1; 950 } 951 952 g_delay_subsystem_init = opts->delay_subsystem_init; 953 g_start_fn = start_fn; 954 g_start_arg = arg1; 955 956 if (opts->json_config_file != NULL) { 957 if (opts->json_data) { 958 SPDK_ERRLOG("App opts json_config_file and json_data are mutually exclusive\n"); 959 return 1; 960 } 961 962 g_spdk_app.json_data = spdk_posix_file_load_from_name(opts->json_config_file, 963 &g_spdk_app.json_data_size); 964 if (!g_spdk_app.json_data) { 965 SPDK_ERRLOG("Read JSON configuration file %s failed: %s\n", 966 opts->json_config_file, spdk_strerror(errno)); 967 return 1; 968 } 969 } else if (opts->json_data) { 970 g_spdk_app.json_data = calloc(1, opts->json_data_size); 971 if (!g_spdk_app.json_data) { 972 SPDK_ERRLOG("Failed to allocate JSON data buffer\n"); 973 return 1; 974 } 975 976 memcpy(g_spdk_app.json_data, opts->json_data, opts->json_data_size); 977 g_spdk_app.json_data_size = opts->json_data_size; 978 } 979 980 spdk_thread_send_msg(spdk_thread_get_app_thread(), bootstrap_fn, NULL); 981 982 /* This blocks until spdk_app_stop is called */ 983 spdk_reactors_start(); 984 985 g_env_was_setup = true; 986 987 return g_spdk_app.rc; 988 } 989 990 void 991 spdk_app_fini(void) 992 { 993 spdk_trace_cleanup(); 994 spdk_reactors_fini(); 995 spdk_env_fini(); 996 spdk_log_close(); 997 unclaim_cpu_cores(NULL); 998 } 999 1000 static void 1001 subsystem_fini_done(void *arg1) 1002 { 1003 spdk_rpc_finish(); 1004 spdk_reactors_stop(NULL); 1005 } 1006 1007 static void 1008 _start_subsystem_fini(void *arg1) 1009 { 1010 if (g_scheduling_in_progress) { 1011 spdk_thread_send_msg(spdk_thread_get_app_thread(), _start_subsystem_fini, NULL); 1012 return; 1013 } 1014 1015 spdk_subsystem_fini(subsystem_fini_done, NULL); 1016 } 1017 1018 static int 1019 log_deprecation_hits(void *ctx, struct spdk_deprecation *dep) 1020 { 1021 uint64_t hits = spdk_deprecation_get_hits(dep); 1022 1023 if (hits == 0) { 1024 return 0; 1025 } 1026 1027 SPDK_WARNLOG("%s: deprecation '%s' scheduled for removal in %s hit %" PRIu64 " times\n", 1028 spdk_deprecation_get_tag(dep), spdk_deprecation_get_description(dep), 1029 spdk_deprecation_get_remove_release(dep), hits); 1030 return 0; 1031 } 1032 1033 static void 1034 app_stop(void *arg1) 1035 { 1036 if (g_spdk_app.rc == 0) { 1037 g_spdk_app.rc = (int)(intptr_t)arg1; 1038 } 1039 1040 if (g_spdk_app.stopped) { 1041 SPDK_NOTICELOG("spdk_app_stop called twice\n"); 1042 return; 1043 } 1044 1045 free(g_spdk_app.json_data); 1046 1047 g_spdk_app.stopped = true; 1048 spdk_log_for_each_deprecation(NULL, log_deprecation_hits); 1049 _start_subsystem_fini(NULL); 1050 } 1051 1052 void 1053 spdk_app_stop(int rc) 1054 { 1055 if (rc) { 1056 SPDK_WARNLOG("spdk_app_stop'd on non-zero\n"); 1057 } 1058 1059 /* 1060 * We want to run spdk_subsystem_fini() from the same thread where spdk_subsystem_init() 1061 * was called. 1062 */ 1063 spdk_thread_send_msg(spdk_thread_get_app_thread(), app_stop, (void *)(intptr_t)rc); 1064 } 1065 1066 static void 1067 usage_memory_size(void) 1068 { 1069 #ifndef __linux__ 1070 if (g_default_opts.mem_size <= 0) { 1071 printf("all hugepage memory)\n"); 1072 } else 1073 #endif 1074 { 1075 printf("%dMB)\n", g_default_opts.mem_size >= 0 ? g_default_opts.mem_size : 0); 1076 } 1077 } 1078 1079 static void 1080 usage(void (*app_usage)(void)) 1081 { 1082 printf("%s [options]\n", g_executable_name); 1083 /* Keep entries inside categories roughly sorted by frequency of use. */ 1084 printf("\nCPU options:\n"); 1085 printf(" -m, --cpumask <mask or list> core mask (like 0xF) or core list of '[]' embraced for DPDK\n"); 1086 printf(" (like [0,1,10])\n"); 1087 printf(" --lcores <list> lcore to CPU mapping list. The list is in the format:\n"); 1088 printf(" <lcores[@CPUs]>[<,lcores[@CPUs]>...]\n"); 1089 printf(" lcores and cpus list are grouped by '(' and ')', e.g '--lcores \"(5-7)@(10-12)\"'\n"); 1090 printf(" Within the group, '-' is used for range separator,\n"); 1091 printf(" ',' is used for single number separator.\n"); 1092 printf(" '( )' can be omitted for single element group,\n"); 1093 printf(" '@' can be omitted if cpus and lcores have the same value\n"); 1094 printf(" --disable-cpumask-locks Disable CPU core lock files.\n"); 1095 printf(" --interrupt-mode set app to interrupt mode (Warning: CPU usage will be reduced only if all\n"); 1096 printf(" pollers in the app support interrupt mode)\n"); 1097 printf(" -p, --main-core <id> main (primary) core for DPDK\n"); 1098 1099 printf("\nConfiguration options:\n"); 1100 printf(" -c, --config, --json <config> JSON config file\n"); 1101 printf(" -r, --rpc-socket <path> RPC listen address (default %s)\n", SPDK_DEFAULT_RPC_ADDR); 1102 printf(" --no-rpc-server skip RPC server initialization. This option ignores '--rpc-socket' value.\n"); 1103 printf(" --wait-for-rpc wait for RPCs to initialize subsystems\n"); 1104 printf(" --rpcs-allowed comma-separated list of permitted RPCS\n"); 1105 printf(" --json-ignore-init-errors don't exit on invalid config entry\n"); 1106 1107 printf("\nMemory options:\n"); 1108 printf(" --iova-mode <pa/va> set IOVA mode ('pa' for IOVA_PA and 'va' for IOVA_VA)\n"); 1109 printf(" --base-virtaddr <addr> the base virtual address for DPDK (default: 0x200000000000)\n"); 1110 printf(" --huge-dir <path> use a specific hugetlbfs mount to reserve memory from\n"); 1111 printf(" -R, --huge-unlink unlink huge files after initialization\n"); 1112 printf(" -n, --mem-channels <num> number of memory channels used for DPDK\n"); 1113 printf(" -s, --mem-size <size> memory size in MB for DPDK (default: "); 1114 usage_memory_size(); 1115 printf(" --msg-mempool-size <size> global message memory pool size in count (default: %d)\n", 1116 SPDK_DEFAULT_MSG_MEMPOOL_SIZE); 1117 printf(" --no-huge run without using hugepages\n"); 1118 printf(" -i, --shm-id <id> shared memory ID (optional)\n"); 1119 printf(" -g, --single-file-segments force creating just one hugetlbfs file\n"); 1120 1121 printf("\nPCI options:\n"); 1122 printf(" -A, --pci-allowed <bdf> pci addr to allow (-B and -A cannot be used at the same time)\n"); 1123 printf(" -B, --pci-blocked <bdf> pci addr to block (can be used more than once)\n"); 1124 printf(" -u, --no-pci disable PCI access\n"); 1125 printf(" --vfio-vf-token VF token (UUID) shared between SR-IOV PF and VFs for vfio_pci driver\n"); 1126 1127 printf("\nLog options:\n"); 1128 spdk_log_usage(stdout, "-L"); 1129 printf(" --silence-noticelog disable notice level logging to stderr\n"); 1130 1131 printf("\nTrace options:\n"); 1132 printf(" --num-trace-entries <num> number of trace entries for each core, must be power of 2,\n"); 1133 printf(" setting 0 to disable trace (default %d)\n", 1134 SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES); 1135 printf(" Tracepoints vary in size and can use more than one trace entry.\n"); 1136 spdk_trace_mask_usage(stdout, "-e"); 1137 1138 printf("\nOther options:\n"); 1139 printf(" -h, --help show this usage\n"); 1140 printf(" -v, --version print SPDK version\n"); 1141 printf(" -d, --limit-coredump do not set max coredump size to RLIM_INFINITY\n"); 1142 printf(" --env-context Opaque context for use of the env implementation\n"); 1143 1144 if (app_usage) { 1145 printf("\nApplication specific:\n"); 1146 app_usage(); 1147 } 1148 } 1149 1150 spdk_app_parse_args_rvals_t 1151 spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts, 1152 const char *app_getopt_str, const struct option *app_long_opts, 1153 int (*app_parse)(int ch, char *arg), 1154 void (*app_usage)(void)) 1155 { 1156 int ch, rc, opt_idx, global_long_opts_len, app_long_opts_len; 1157 struct option *cmdline_options; 1158 char *cmdline_short_opts = NULL; 1159 char *shm_id_str = NULL; 1160 enum spdk_app_parse_args_rvals retval = SPDK_APP_PARSE_ARGS_FAIL; 1161 long int tmp; 1162 1163 memcpy(&g_default_opts, opts, sizeof(g_default_opts)); 1164 1165 if (opts->json_config_file && access(opts->json_config_file, R_OK) != 0) { 1166 SPDK_WARNLOG("Can't read JSON configuration file '%s'\n", opts->json_config_file); 1167 opts->json_config_file = NULL; 1168 } 1169 1170 if (app_long_opts == NULL) { 1171 app_long_opts_len = 0; 1172 } else { 1173 for (app_long_opts_len = 0; 1174 app_long_opts[app_long_opts_len].name != NULL; 1175 app_long_opts_len++); 1176 } 1177 1178 global_long_opts_len = SPDK_COUNTOF(g_cmdline_options); 1179 1180 cmdline_options = calloc(global_long_opts_len + app_long_opts_len + 1, sizeof(*cmdline_options)); 1181 if (!cmdline_options) { 1182 SPDK_ERRLOG("Out of memory\n"); 1183 return SPDK_APP_PARSE_ARGS_FAIL; 1184 } 1185 1186 memcpy(&cmdline_options[0], g_cmdline_options, sizeof(g_cmdline_options)); 1187 if (app_long_opts) { 1188 memcpy(&cmdline_options[global_long_opts_len], app_long_opts, 1189 app_long_opts_len * sizeof(*app_long_opts)); 1190 } 1191 1192 if (app_getopt_str != NULL) { 1193 ch = app_opts_validate(app_getopt_str); 1194 if (ch) { 1195 SPDK_ERRLOG("Duplicated option '%c' between app-specific command line parameter and generic spdk opts.\n", 1196 ch); 1197 goto out; 1198 } 1199 1200 if (!app_parse) { 1201 SPDK_ERRLOG("Parse function is required when app-specific command line parameters are provided.\n"); 1202 goto out; 1203 } 1204 } 1205 1206 cmdline_short_opts = spdk_sprintf_alloc("%s%s", app_getopt_str, SPDK_APP_GETOPT_STRING); 1207 if (!cmdline_short_opts) { 1208 SPDK_ERRLOG("Out of memory\n"); 1209 goto out; 1210 } 1211 1212 g_executable_name = argv[0]; 1213 1214 while ((ch = getopt_long(argc, argv, cmdline_short_opts, cmdline_options, &opt_idx)) != -1) { 1215 switch (ch) { 1216 case CONFIG_FILE_OPT_IDX: 1217 case JSON_CONFIG_OPT_IDX: 1218 opts->json_config_file = optarg; 1219 break; 1220 case JSON_CONFIG_IGNORE_INIT_ERRORS_IDX: 1221 opts->json_config_ignore_errors = true; 1222 break; 1223 case LIMIT_COREDUMP_OPT_IDX: 1224 opts->enable_coredump = false; 1225 break; 1226 case TPOINT_GROUP_OPT_IDX: 1227 opts->tpoint_group_mask = optarg; 1228 break; 1229 case SINGLE_FILE_SEGMENTS_OPT_IDX: 1230 opts->hugepage_single_segments = true; 1231 break; 1232 case HELP_OPT_IDX: 1233 usage(app_usage); 1234 retval = SPDK_APP_PARSE_ARGS_HELP; 1235 goto out; 1236 case SHM_ID_OPT_IDX: 1237 shm_id_str = optarg; 1238 /* a negative shm-id disables shared configuration file */ 1239 if (optarg[0] == '-') { 1240 shm_id_str++; 1241 } 1242 /* check if the positive value of provided shm_id can be parsed as 1243 * an integer 1244 */ 1245 opts->shm_id = spdk_strtol(shm_id_str, 0); 1246 if (opts->shm_id < 0) { 1247 SPDK_ERRLOG("Invalid shared memory ID %s\n", optarg); 1248 goto out; 1249 } 1250 if (optarg[0] == '-') { 1251 opts->shm_id = -opts->shm_id; 1252 } 1253 break; 1254 case CPUMASK_OPT_IDX: 1255 if (opts->lcore_map) { 1256 SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n"); 1257 goto out; 1258 } 1259 opts->reactor_mask = optarg; 1260 break; 1261 case LCORES_OPT_IDX: 1262 if (opts->reactor_mask) { 1263 SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n"); 1264 goto out; 1265 } 1266 opts->lcore_map = optarg; 1267 break; 1268 case DISABLE_CPUMASK_LOCKS_OPT_IDX: 1269 opts->disable_cpumask_locks = true; 1270 break; 1271 case MEM_CHANNELS_OPT_IDX: 1272 opts->mem_channel = spdk_strtol(optarg, 0); 1273 if (opts->mem_channel < 0) { 1274 SPDK_ERRLOG("Invalid memory channel %s\n", optarg); 1275 goto out; 1276 } 1277 break; 1278 case MAIN_CORE_OPT_IDX: 1279 opts->main_core = spdk_strtol(optarg, 0); 1280 if (opts->main_core < 0) { 1281 SPDK_ERRLOG("Invalid main core %s\n", optarg); 1282 goto out; 1283 } 1284 break; 1285 case SILENCE_NOTICELOG_OPT_IDX: 1286 opts->print_level = SPDK_LOG_WARN; 1287 break; 1288 case RPC_SOCKET_OPT_IDX: 1289 opts->rpc_addr = optarg; 1290 break; 1291 case NO_RPC_SERVER_OPT_IDX: 1292 opts->rpc_addr = NULL; 1293 break; 1294 case MEM_SIZE_OPT_IDX: { 1295 uint64_t mem_size_mb; 1296 bool mem_size_has_prefix; 1297 1298 rc = spdk_parse_capacity(optarg, &mem_size_mb, &mem_size_has_prefix); 1299 if (rc != 0) { 1300 SPDK_ERRLOG("invalid memory pool size `-s %s`\n", optarg); 1301 usage(app_usage); 1302 goto out; 1303 } 1304 1305 if (mem_size_has_prefix) { 1306 /* the mem size is in MB by default, so if a prefix was 1307 * specified, we need to manually convert to MB. 1308 */ 1309 mem_size_mb /= 1024 * 1024; 1310 } 1311 1312 if (mem_size_mb > INT_MAX) { 1313 SPDK_ERRLOG("invalid memory pool size `-s %s`\n", optarg); 1314 usage(app_usage); 1315 goto out; 1316 } 1317 1318 opts->mem_size = (int) mem_size_mb; 1319 break; 1320 } 1321 case MSG_MEMPOOL_SIZE_OPT_IDX: 1322 tmp = spdk_strtol(optarg, 10); 1323 if (tmp <= 0) { 1324 SPDK_ERRLOG("Invalid message memory pool size %s\n", optarg); 1325 goto out; 1326 } 1327 1328 opts->msg_mempool_size = (size_t)tmp; 1329 break; 1330 1331 case NO_PCI_OPT_IDX: 1332 opts->no_pci = true; 1333 break; 1334 case WAIT_FOR_RPC_OPT_IDX: 1335 opts->delay_subsystem_init = true; 1336 break; 1337 case PCI_BLOCKED_OPT_IDX: 1338 if (opts->pci_allowed) { 1339 free(opts->pci_allowed); 1340 opts->pci_allowed = NULL; 1341 SPDK_ERRLOG("-B and -A cannot be used at the same time\n"); 1342 usage(app_usage); 1343 goto out; 1344 } 1345 1346 rc = app_opts_add_pci_addr(opts, &opts->pci_blocked, optarg); 1347 if (rc != 0) { 1348 free(opts->pci_blocked); 1349 opts->pci_blocked = NULL; 1350 goto out; 1351 } 1352 break; 1353 1354 case NO_HUGE_OPT_IDX: 1355 opts->no_huge = true; 1356 break; 1357 1358 case LOGFLAG_OPT_IDX: 1359 rc = spdk_log_set_flag(optarg); 1360 if (rc < 0) { 1361 SPDK_ERRLOG("unknown flag: %s\n", optarg); 1362 usage(app_usage); 1363 goto out; 1364 } 1365 #ifdef DEBUG 1366 opts->print_level = SPDK_LOG_DEBUG; 1367 #endif 1368 break; 1369 case HUGE_UNLINK_OPT_IDX: 1370 opts->unlink_hugepage = true; 1371 break; 1372 case PCI_ALLOWED_OPT_IDX: 1373 if (opts->pci_blocked) { 1374 free(opts->pci_blocked); 1375 opts->pci_blocked = NULL; 1376 SPDK_ERRLOG("-B and -W cannot be used at the same time\n"); 1377 usage(app_usage); 1378 goto out; 1379 } 1380 1381 rc = app_opts_add_pci_addr(opts, &opts->pci_allowed, optarg); 1382 if (rc != 0) { 1383 free(opts->pci_allowed); 1384 opts->pci_allowed = NULL; 1385 goto out; 1386 } 1387 break; 1388 case BASE_VIRTADDR_OPT_IDX: 1389 tmp = spdk_strtoll(optarg, 0); 1390 if (tmp <= 0) { 1391 SPDK_ERRLOG("Invalid base-virtaddr %s\n", optarg); 1392 usage(app_usage); 1393 goto out; 1394 } 1395 opts->base_virtaddr = (uint64_t)tmp; 1396 break; 1397 case HUGE_DIR_OPT_IDX: 1398 opts->hugedir = optarg; 1399 break; 1400 case IOVA_MODE_OPT_IDX: 1401 opts->iova_mode = optarg; 1402 break; 1403 case NUM_TRACE_ENTRIES_OPT_IDX: 1404 tmp = spdk_strtoll(optarg, 0); 1405 if (tmp < 0) { 1406 SPDK_ERRLOG("Invalid num-trace-entries %s\n", optarg); 1407 usage(app_usage); 1408 goto out; 1409 } 1410 opts->num_entries = (uint64_t)tmp; 1411 if (opts->num_entries > 0 && opts->num_entries & (opts->num_entries - 1)) { 1412 SPDK_ERRLOG("num-trace-entries must be power of 2\n"); 1413 usage(app_usage); 1414 goto out; 1415 } 1416 break; 1417 case ENV_CONTEXT_OPT_IDX: 1418 opts->env_context = optarg; 1419 break; 1420 case RPCS_ALLOWED_OPT_IDX: 1421 opts->rpc_allowlist = (const char **)spdk_strarray_from_string(optarg, ","); 1422 if (opts->rpc_allowlist == NULL) { 1423 SPDK_ERRLOG("Invalid --rpcs-allowed argument\n"); 1424 usage(app_usage); 1425 goto out; 1426 } 1427 break; 1428 case ENV_VF_TOKEN_OPT_IDX: 1429 opts->vf_token = optarg; 1430 break; 1431 case INTERRUPT_MODE_OPT_IDX: 1432 opts->interrupt_mode = true; 1433 break; 1434 case VERSION_OPT_IDX: 1435 printf(SPDK_VERSION_STRING"\n"); 1436 retval = SPDK_APP_PARSE_ARGS_HELP; 1437 goto out; 1438 case '?': 1439 /* 1440 * In the event getopt() above detects an option 1441 * in argv that is NOT in the getopt_str, 1442 * getopt() will return a '?' indicating failure. 1443 */ 1444 usage(app_usage); 1445 goto out; 1446 default: 1447 if (!app_parse) { 1448 SPDK_ERRLOG("Unsupported app-specific command line parameter '%c'.\n", ch); 1449 goto out; 1450 } 1451 1452 rc = app_parse(ch, optarg); 1453 if (rc) { 1454 SPDK_ERRLOG("Parsing app-specific command line parameter '%c' failed: %d\n", ch, rc); 1455 goto out; 1456 } 1457 } 1458 } 1459 1460 retval = SPDK_APP_PARSE_ARGS_SUCCESS; 1461 out: 1462 if (retval != SPDK_APP_PARSE_ARGS_SUCCESS) { 1463 free(opts->pci_blocked); 1464 opts->pci_blocked = NULL; 1465 free(opts->pci_allowed); 1466 opts->pci_allowed = NULL; 1467 spdk_strarray_free((char **)opts->rpc_allowlist); 1468 opts->rpc_allowlist = NULL; 1469 } 1470 free(cmdline_short_opts); 1471 free(cmdline_options); 1472 return retval; 1473 } 1474 1475 void 1476 spdk_app_usage(void) 1477 { 1478 if (g_executable_name == NULL) { 1479 SPDK_ERRLOG("%s not valid before calling spdk_app_parse_args()\n", __func__); 1480 return; 1481 } 1482 1483 usage(NULL); 1484 } 1485 1486 static void 1487 rpc_framework_start_init_cpl(int rc, void *arg1) 1488 { 1489 struct spdk_jsonrpc_request *request = arg1; 1490 1491 assert(spdk_thread_is_app_thread(NULL)); 1492 1493 if (rc) { 1494 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, 1495 "framework_initialization failed"); 1496 return; 1497 } 1498 1499 app_subsystem_init_done(0, NULL); 1500 1501 spdk_jsonrpc_send_bool_response(request, true); 1502 } 1503 1504 static void 1505 rpc_framework_start_init(struct spdk_jsonrpc_request *request, 1506 const struct spdk_json_val *params) 1507 { 1508 if (params != NULL) { 1509 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1510 "framework_start_init requires no parameters"); 1511 return; 1512 } 1513 1514 spdk_rpc_server_pause(g_spdk_app.rpc_addr); 1515 spdk_subsystem_init(rpc_framework_start_init_cpl, request); 1516 } 1517 SPDK_RPC_REGISTER("framework_start_init", rpc_framework_start_init, SPDK_RPC_STARTUP) 1518 1519 struct subsystem_init_poller_ctx { 1520 struct spdk_poller *init_poller; 1521 struct spdk_jsonrpc_request *request; 1522 }; 1523 1524 static int 1525 rpc_subsystem_init_poller_ctx(void *ctx) 1526 { 1527 struct subsystem_init_poller_ctx *poller_ctx = ctx; 1528 1529 if (spdk_rpc_get_state() == SPDK_RPC_RUNTIME) { 1530 spdk_jsonrpc_send_bool_response(poller_ctx->request, true); 1531 spdk_poller_unregister(&poller_ctx->init_poller); 1532 free(poller_ctx); 1533 } 1534 1535 return SPDK_POLLER_BUSY; 1536 } 1537 1538 static void 1539 rpc_framework_wait_init(struct spdk_jsonrpc_request *request, 1540 const struct spdk_json_val *params) 1541 { 1542 struct subsystem_init_poller_ctx *ctx; 1543 1544 if (spdk_rpc_get_state() == SPDK_RPC_RUNTIME) { 1545 spdk_jsonrpc_send_bool_response(request, true); 1546 } else { 1547 ctx = malloc(sizeof(struct subsystem_init_poller_ctx)); 1548 if (ctx == NULL) { 1549 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, 1550 "Unable to allocate memory for the request context\n"); 1551 return; 1552 } 1553 ctx->request = request; 1554 ctx->init_poller = SPDK_POLLER_REGISTER(rpc_subsystem_init_poller_ctx, ctx, 0); 1555 } 1556 } 1557 SPDK_RPC_REGISTER("framework_wait_init", rpc_framework_wait_init, 1558 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1559 1560 static void 1561 rpc_framework_disable_cpumask_locks(struct spdk_jsonrpc_request *request, 1562 const struct spdk_json_val *params) 1563 { 1564 char msg[128]; 1565 int rc; 1566 uint32_t failed_core; 1567 1568 if (params != NULL) { 1569 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1570 "framework_disable_cpumask_locks" 1571 "requires no arguments"); 1572 return; 1573 } 1574 1575 rc = unclaim_cpu_cores(&failed_core); 1576 if (rc) { 1577 snprintf(msg, sizeof(msg), "Failed to unclaim CPU core: %" PRIu32, failed_core); 1578 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg); 1579 return; 1580 } 1581 1582 spdk_jsonrpc_send_bool_response(request, true); 1583 } 1584 SPDK_RPC_REGISTER("framework_disable_cpumask_locks", rpc_framework_disable_cpumask_locks, 1585 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1586 1587 static void 1588 rpc_framework_enable_cpumask_locks(struct spdk_jsonrpc_request *request, 1589 const struct spdk_json_val *params) 1590 { 1591 char msg[128]; 1592 int rc; 1593 uint32_t failed_core; 1594 1595 if (params != NULL) { 1596 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1597 "framework_enable_cpumask_locks" 1598 "requires no arguments"); 1599 return; 1600 } 1601 1602 rc = claim_cpu_cores(&failed_core); 1603 if (rc) { 1604 snprintf(msg, sizeof(msg), "Failed to claim CPU core: %" PRIu32, failed_core); 1605 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg); 1606 return; 1607 } 1608 1609 spdk_jsonrpc_send_bool_response(request, true); 1610 } 1611 SPDK_RPC_REGISTER("framework_enable_cpumask_locks", rpc_framework_enable_cpumask_locks, 1612 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1613