1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk/version.h" 9 10 #include "spdk_internal/event.h" 11 12 #include "spdk/assert.h" 13 #include "spdk/env.h" 14 #include "spdk/init.h" 15 #include "spdk/log.h" 16 #include "spdk/thread.h" 17 #include "spdk/trace.h" 18 #include "spdk/string.h" 19 #include "spdk/scheduler.h" 20 #include "spdk/rpc.h" 21 #include "spdk/util.h" 22 #include "spdk/file.h" 23 #include "spdk/config.h" 24 #include "event_internal.h" 25 26 #define SPDK_APP_DEFAULT_LOG_LEVEL SPDK_LOG_NOTICE 27 #define SPDK_APP_DEFAULT_LOG_PRINT_LEVEL SPDK_LOG_INFO 28 #define SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES SPDK_DEFAULT_NUM_TRACE_ENTRIES 29 30 #define SPDK_APP_DPDK_DEFAULT_MEM_SIZE -1 31 #define SPDK_APP_DPDK_DEFAULT_MAIN_CORE -1 32 #define SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL -1 33 #define SPDK_APP_DPDK_DEFAULT_CORE_MASK "0x1" 34 #define SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR 0x200000000000 35 #define SPDK_APP_DEFAULT_CORE_LIMIT 0x140000000 /* 5 GiB */ 36 37 /* For core counts <= 63, the message memory pool size is set to 38 * SPDK_DEFAULT_MSG_MEMPOOL_SIZE. 39 * For core counts > 63, the message memory pool size is depended on 40 * number of cores. Per core, it is calculated as SPDK_MSG_MEMPOOL_CACHE_SIZE 41 * multiplied by factor of 4 to have space for multiple spdk threads running 42 * on single core (e.g iscsi + nvmf + vhost ). */ 43 #define SPDK_APP_PER_CORE_MSG_MEMPOOL_SIZE (4 * SPDK_MSG_MEMPOOL_CACHE_SIZE) 44 45 struct spdk_app { 46 void *json_data; 47 size_t json_data_size; 48 bool json_config_ignore_errors; 49 bool stopped; 50 const char *rpc_addr; 51 const char **rpc_allowlist; 52 FILE *rpc_log_file; 53 enum spdk_log_level rpc_log_level; 54 int shm_id; 55 spdk_app_shutdown_cb shutdown_cb; 56 int rc; 57 }; 58 59 static struct spdk_app g_spdk_app; 60 static spdk_msg_fn g_start_fn = NULL; 61 static void *g_start_arg = NULL; 62 static bool g_delay_subsystem_init = false; 63 static bool g_shutdown_sig_received = false; 64 static char *g_executable_name; 65 static struct spdk_app_opts g_default_opts; 66 67 static int g_core_locks[SPDK_CONFIG_MAX_LCORES]; 68 69 static struct { 70 uint64_t irq; 71 uint64_t usr; 72 uint64_t sys; 73 } g_initial_stat[SPDK_CONFIG_MAX_LCORES]; 74 75 int 76 spdk_app_get_shm_id(void) 77 { 78 return g_spdk_app.shm_id; 79 } 80 81 /* append one empty option to indicate the end of the array */ 82 static const struct option g_cmdline_options[] = { 83 #define CONFIG_FILE_OPT_IDX 'c' 84 {"config", required_argument, NULL, CONFIG_FILE_OPT_IDX}, 85 #define LIMIT_COREDUMP_OPT_IDX 'd' 86 {"limit-coredump", no_argument, NULL, LIMIT_COREDUMP_OPT_IDX}, 87 #define TPOINT_GROUP_OPT_IDX 'e' 88 {"tpoint-group", required_argument, NULL, TPOINT_GROUP_OPT_IDX}, 89 #define SINGLE_FILE_SEGMENTS_OPT_IDX 'g' 90 {"single-file-segments", no_argument, NULL, SINGLE_FILE_SEGMENTS_OPT_IDX}, 91 #define HELP_OPT_IDX 'h' 92 {"help", no_argument, NULL, HELP_OPT_IDX}, 93 #define SHM_ID_OPT_IDX 'i' 94 {"shm-id", required_argument, NULL, SHM_ID_OPT_IDX}, 95 #define CPUMASK_OPT_IDX 'm' 96 {"cpumask", required_argument, NULL, CPUMASK_OPT_IDX}, 97 #define MEM_CHANNELS_OPT_IDX 'n' 98 {"mem-channels", required_argument, NULL, MEM_CHANNELS_OPT_IDX}, 99 #define MAIN_CORE_OPT_IDX 'p' 100 {"main-core", required_argument, NULL, MAIN_CORE_OPT_IDX}, 101 #define RPC_SOCKET_OPT_IDX 'r' 102 {"rpc-socket", required_argument, NULL, RPC_SOCKET_OPT_IDX}, 103 #define MEM_SIZE_OPT_IDX 's' 104 {"mem-size", required_argument, NULL, MEM_SIZE_OPT_IDX}, 105 #define NO_PCI_OPT_IDX 'u' 106 {"no-pci", no_argument, NULL, NO_PCI_OPT_IDX}, 107 #define VERSION_OPT_IDX 'v' 108 {"version", no_argument, NULL, VERSION_OPT_IDX}, 109 #define PCI_BLOCKED_OPT_IDX 'B' 110 {"pci-blocked", required_argument, NULL, PCI_BLOCKED_OPT_IDX}, 111 #define LOGFLAG_OPT_IDX 'L' 112 {"logflag", required_argument, NULL, LOGFLAG_OPT_IDX}, 113 #define HUGE_UNLINK_OPT_IDX 'R' 114 {"huge-unlink", no_argument, NULL, HUGE_UNLINK_OPT_IDX}, 115 #define PCI_ALLOWED_OPT_IDX 'A' 116 {"pci-allowed", required_argument, NULL, PCI_ALLOWED_OPT_IDX}, 117 #define INTERRUPT_MODE_OPT_IDX 256 118 {"interrupt-mode", no_argument, NULL, INTERRUPT_MODE_OPT_IDX}, 119 #define SILENCE_NOTICELOG_OPT_IDX 257 120 {"silence-noticelog", no_argument, NULL, SILENCE_NOTICELOG_OPT_IDX}, 121 #define WAIT_FOR_RPC_OPT_IDX 258 122 {"wait-for-rpc", no_argument, NULL, WAIT_FOR_RPC_OPT_IDX}, 123 #define HUGE_DIR_OPT_IDX 259 124 {"huge-dir", required_argument, NULL, HUGE_DIR_OPT_IDX}, 125 #define NUM_TRACE_ENTRIES_OPT_IDX 260 126 {"num-trace-entries", required_argument, NULL, NUM_TRACE_ENTRIES_OPT_IDX}, 127 #define JSON_CONFIG_OPT_IDX 262 128 {"json", required_argument, NULL, JSON_CONFIG_OPT_IDX}, 129 #define JSON_CONFIG_IGNORE_INIT_ERRORS_IDX 263 130 {"json-ignore-init-errors", no_argument, NULL, JSON_CONFIG_IGNORE_INIT_ERRORS_IDX}, 131 #define IOVA_MODE_OPT_IDX 264 132 {"iova-mode", required_argument, NULL, IOVA_MODE_OPT_IDX}, 133 #define BASE_VIRTADDR_OPT_IDX 265 134 {"base-virtaddr", required_argument, NULL, BASE_VIRTADDR_OPT_IDX}, 135 #define ENV_CONTEXT_OPT_IDX 266 136 {"env-context", required_argument, NULL, ENV_CONTEXT_OPT_IDX}, 137 #define DISABLE_CPUMASK_LOCKS_OPT_IDX 267 138 {"disable-cpumask-locks", no_argument, NULL, DISABLE_CPUMASK_LOCKS_OPT_IDX}, 139 #define RPCS_ALLOWED_OPT_IDX 268 140 {"rpcs-allowed", required_argument, NULL, RPCS_ALLOWED_OPT_IDX}, 141 #define ENV_VF_TOKEN_OPT_IDX 269 142 {"vfio-vf-token", required_argument, NULL, ENV_VF_TOKEN_OPT_IDX}, 143 #define MSG_MEMPOOL_SIZE_OPT_IDX 270 144 {"msg-mempool-size", required_argument, NULL, MSG_MEMPOOL_SIZE_OPT_IDX}, 145 #define LCORES_OPT_IDX 271 146 {"lcores", required_argument, NULL, LCORES_OPT_IDX}, 147 #define NO_HUGE_OPT_IDX 272 148 {"no-huge", no_argument, NULL, NO_HUGE_OPT_IDX}, 149 #define NO_RPC_SERVER_OPT_IDX 273 150 {"no-rpc-server", no_argument, NULL, NO_RPC_SERVER_OPT_IDX}, 151 }; 152 153 static int 154 parse_proc_stat(unsigned int core, uint64_t *user, uint64_t *sys, uint64_t *irq) 155 { 156 FILE *f; 157 uint64_t i, soft_irq, cpu = 0; 158 int rc, found = 0; 159 160 f = fopen("/proc/stat", "r"); 161 if (!f) { 162 return -1; 163 } 164 165 for (i = 0; i <= core + 1; i++) { 166 /* scanf discards input with '*' in format, 167 * cpu;user;nice;system;idle;iowait;irq;softirq;steal;guest;guest_nice */ 168 rc = fscanf(f, "cpu%li %li %*i %li %*i %*i %li %li %*i %*i %*i\n", 169 &cpu, user, sys, irq, &soft_irq); 170 if (rc != 5) { 171 continue; 172 } 173 174 /* some cores can be disabled, list may not be in order */ 175 if (cpu == core) { 176 found = 1; 177 break; 178 } 179 } 180 181 *irq += soft_irq; 182 183 fclose(f); 184 return found ? 0 : -1; 185 } 186 187 static int 188 init_proc_stat(unsigned int core) 189 { 190 uint64_t usr, sys, irq; 191 192 if (core >= SPDK_CONFIG_MAX_LCORES) { 193 return -1; 194 } 195 196 if (parse_proc_stat(core, &usr, &sys, &irq) < 0) { 197 return -1; 198 } 199 200 g_initial_stat[core].irq = irq; 201 g_initial_stat[core].usr = usr; 202 g_initial_stat[core].sys = sys; 203 204 return 0; 205 } 206 207 int 208 app_get_proc_stat(unsigned int core, uint64_t *usr, uint64_t *sys, uint64_t *irq) 209 { 210 uint64_t _usr, _sys, _irq; 211 212 if (core >= SPDK_CONFIG_MAX_LCORES) { 213 return -1; 214 } 215 216 if (parse_proc_stat(core, &_usr, &_sys, &_irq) < 0) { 217 return -1; 218 } 219 220 *irq = _irq - g_initial_stat[core].irq; 221 *usr = _usr - g_initial_stat[core].usr; 222 *sys = _sys - g_initial_stat[core].sys; 223 224 return 0; 225 } 226 227 static void 228 app_start_shutdown(void *ctx) 229 { 230 if (g_spdk_app.shutdown_cb) { 231 g_spdk_app.shutdown_cb(); 232 g_spdk_app.shutdown_cb = NULL; 233 } else { 234 spdk_app_stop(0); 235 } 236 } 237 238 void 239 spdk_app_start_shutdown(void) 240 { 241 spdk_thread_send_critical_msg(spdk_thread_get_app_thread(), app_start_shutdown); 242 } 243 244 static void 245 __shutdown_signal(int signo) 246 { 247 if (!g_shutdown_sig_received) { 248 g_shutdown_sig_received = true; 249 spdk_app_start_shutdown(); 250 } 251 } 252 253 static int 254 app_opts_validate(const char *app_opts) 255 { 256 int i = 0, j; 257 258 for (i = 0; app_opts[i] != '\0'; i++) { 259 /* ignore getopt control characters */ 260 if (app_opts[i] == ':' || app_opts[i] == '+' || app_opts[i] == '-') { 261 continue; 262 } 263 264 for (j = 0; SPDK_APP_GETOPT_STRING[j] != '\0'; j++) { 265 if (app_opts[i] == SPDK_APP_GETOPT_STRING[j]) { 266 return app_opts[i]; 267 } 268 } 269 } 270 return 0; 271 } 272 273 static void 274 calculate_mempool_size(struct spdk_app_opts *opts, 275 struct spdk_app_opts *opts_user) 276 { 277 uint32_t core_count = spdk_env_get_core_count(); 278 279 if (!opts_user->msg_mempool_size) { 280 /* The user didn't specify msg_mempool_size, so let's calculate it. 281 Set the default (SPDK_DEFAULT_MSG_MEMPOOL_SIZE) if less than 282 64 cores, and use 4k per core otherwise */ 283 opts->msg_mempool_size = spdk_max(SPDK_DEFAULT_MSG_MEMPOOL_SIZE, 284 core_count * SPDK_APP_PER_CORE_MSG_MEMPOOL_SIZE); 285 } else { 286 opts->msg_mempool_size = opts_user->msg_mempool_size; 287 } 288 } 289 290 void 291 spdk_app_opts_init(struct spdk_app_opts *opts, size_t opts_size) 292 { 293 if (!opts) { 294 SPDK_ERRLOG("opts should not be NULL\n"); 295 return; 296 } 297 298 if (!opts_size) { 299 SPDK_ERRLOG("opts_size should not be zero value\n"); 300 return; 301 } 302 303 memset(opts, 0, opts_size); 304 opts->opts_size = opts_size; 305 306 #define SET_FIELD(field, value) \ 307 if (offsetof(struct spdk_app_opts, field) + sizeof(opts->field) <= opts_size) { \ 308 opts->field = value; \ 309 } \ 310 311 SET_FIELD(enable_coredump, true); 312 SET_FIELD(shm_id, -1); 313 SET_FIELD(mem_size, SPDK_APP_DPDK_DEFAULT_MEM_SIZE); 314 SET_FIELD(main_core, SPDK_APP_DPDK_DEFAULT_MAIN_CORE); 315 SET_FIELD(mem_channel, SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL); 316 SET_FIELD(base_virtaddr, SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR); 317 SET_FIELD(print_level, SPDK_APP_DEFAULT_LOG_PRINT_LEVEL); 318 SET_FIELD(rpc_addr, SPDK_DEFAULT_RPC_ADDR); 319 SET_FIELD(num_entries, SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES); 320 SET_FIELD(delay_subsystem_init, false); 321 SET_FIELD(disable_signal_handlers, false); 322 SET_FIELD(interrupt_mode, false); 323 /* Don't set msg_mempool_size here, it is set or calculated later */ 324 SET_FIELD(rpc_allowlist, NULL); 325 SET_FIELD(rpc_log_file, NULL); 326 SET_FIELD(rpc_log_level, SPDK_LOG_DISABLED); 327 SET_FIELD(disable_cpumask_locks, false); 328 #undef SET_FIELD 329 } 330 331 static int 332 app_setup_signal_handlers(struct spdk_app_opts *opts) 333 { 334 struct sigaction sigact; 335 sigset_t sigmask; 336 int rc; 337 338 sigemptyset(&sigmask); 339 memset(&sigact, 0, sizeof(sigact)); 340 sigemptyset(&sigact.sa_mask); 341 342 sigact.sa_handler = SIG_IGN; 343 rc = sigaction(SIGPIPE, &sigact, NULL); 344 if (rc < 0) { 345 SPDK_ERRLOG("sigaction(SIGPIPE) failed\n"); 346 return rc; 347 } 348 349 /* Install the same handler for SIGINT and SIGTERM */ 350 g_shutdown_sig_received = false; 351 sigact.sa_handler = __shutdown_signal; 352 rc = sigaction(SIGINT, &sigact, NULL); 353 if (rc < 0) { 354 SPDK_ERRLOG("sigaction(SIGINT) failed\n"); 355 return rc; 356 } 357 sigaddset(&sigmask, SIGINT); 358 359 rc = sigaction(SIGTERM, &sigact, NULL); 360 if (rc < 0) { 361 SPDK_ERRLOG("sigaction(SIGTERM) failed\n"); 362 return rc; 363 } 364 sigaddset(&sigmask, SIGTERM); 365 366 pthread_sigmask(SIG_UNBLOCK, &sigmask, NULL); 367 368 return 0; 369 } 370 371 static void 372 app_start_application(int rc, void *arg1) 373 { 374 assert(spdk_thread_is_app_thread(NULL)); 375 376 if (rc) { 377 SPDK_ERRLOG("Failed to load subsystems for RUNTIME state with code: %d\n", rc); 378 spdk_app_stop(rc); 379 return; 380 } 381 382 if (g_spdk_app.rpc_addr) { 383 spdk_rpc_server_resume(g_spdk_app.rpc_addr); 384 } 385 386 g_start_fn(g_start_arg); 387 } 388 389 static void 390 app_subsystem_init_done(int rc, void *arg1) 391 { 392 if (rc) { 393 SPDK_ERRLOG("Subsystem initialization failed with code: %d\n", rc); 394 spdk_app_stop(rc); 395 return; 396 } 397 398 spdk_rpc_set_allowlist(g_spdk_app.rpc_allowlist); 399 spdk_rpc_set_state(SPDK_RPC_RUNTIME); 400 401 if (g_spdk_app.json_data) { 402 /* Load SPDK_RPC_RUNTIME RPCs from config file */ 403 assert(spdk_rpc_get_state() == SPDK_RPC_RUNTIME); 404 spdk_subsystem_load_config(g_spdk_app.json_data, g_spdk_app.json_data_size, 405 app_start_application, NULL, 406 !g_spdk_app.json_config_ignore_errors); 407 } else { 408 app_start_application(0, NULL); 409 } 410 } 411 412 static void 413 app_do_spdk_subsystem_init(int rc, void *arg1) 414 { 415 struct spdk_rpc_opts opts; 416 417 if (rc) { 418 spdk_app_stop(rc); 419 return; 420 } 421 422 if (g_spdk_app.rpc_addr) { 423 opts.size = SPDK_SIZEOF(&opts, log_level); 424 opts.log_file = g_spdk_app.rpc_log_file; 425 opts.log_level = g_spdk_app.rpc_log_level; 426 427 rc = spdk_rpc_initialize(g_spdk_app.rpc_addr, &opts); 428 if (rc) { 429 spdk_app_stop(rc); 430 return; 431 } 432 if (g_delay_subsystem_init) { 433 return; 434 } 435 spdk_rpc_server_pause(g_spdk_app.rpc_addr); 436 } else { 437 SPDK_DEBUGLOG(app_rpc, "RPC server not started\n"); 438 } 439 spdk_subsystem_init(app_subsystem_init_done, NULL); 440 } 441 442 static int 443 app_opts_add_pci_addr(struct spdk_app_opts *opts, struct spdk_pci_addr **list, char *bdf) 444 { 445 struct spdk_pci_addr *tmp = *list; 446 size_t i = opts->num_pci_addr; 447 448 tmp = realloc(tmp, sizeof(*tmp) * (i + 1)); 449 if (tmp == NULL) { 450 SPDK_ERRLOG("realloc error\n"); 451 return -ENOMEM; 452 } 453 454 *list = tmp; 455 if (spdk_pci_addr_parse(*list + i, bdf) < 0) { 456 SPDK_ERRLOG("Invalid address %s\n", bdf); 457 return -EINVAL; 458 } 459 460 opts->num_pci_addr++; 461 return 0; 462 } 463 464 static int 465 app_setup_env(struct spdk_app_opts *opts) 466 { 467 struct spdk_env_opts env_opts = {}; 468 int rc; 469 470 if (opts == NULL) { 471 rc = spdk_env_init(NULL); 472 if (rc != 0) { 473 SPDK_ERRLOG("Unable to reinitialize SPDK env\n"); 474 } 475 476 return rc; 477 } 478 479 env_opts.opts_size = sizeof(env_opts); 480 spdk_env_opts_init(&env_opts); 481 482 env_opts.name = opts->name; 483 env_opts.core_mask = opts->reactor_mask; 484 env_opts.lcore_map = opts->lcore_map; 485 env_opts.shm_id = opts->shm_id; 486 env_opts.mem_channel = opts->mem_channel; 487 env_opts.main_core = opts->main_core; 488 env_opts.mem_size = opts->mem_size; 489 env_opts.hugepage_single_segments = opts->hugepage_single_segments; 490 env_opts.unlink_hugepage = opts->unlink_hugepage; 491 env_opts.hugedir = opts->hugedir; 492 env_opts.no_pci = opts->no_pci; 493 env_opts.num_pci_addr = opts->num_pci_addr; 494 env_opts.pci_blocked = opts->pci_blocked; 495 env_opts.pci_allowed = opts->pci_allowed; 496 env_opts.base_virtaddr = opts->base_virtaddr; 497 env_opts.env_context = opts->env_context; 498 env_opts.iova_mode = opts->iova_mode; 499 env_opts.vf_token = opts->vf_token; 500 env_opts.no_huge = opts->no_huge; 501 502 rc = spdk_env_init(&env_opts); 503 free(env_opts.pci_blocked); 504 free(env_opts.pci_allowed); 505 506 if (rc < 0) { 507 SPDK_ERRLOG("Unable to initialize SPDK env\n"); 508 } 509 510 return rc; 511 } 512 513 static int 514 app_setup_trace(struct spdk_app_opts *opts) 515 { 516 char shm_name[64]; 517 uint64_t tpoint_group_mask, tpoint_mask = -1ULL; 518 char *end = NULL, *tpoint_group_mask_str, *tpoint_group_str = NULL; 519 char *tp_g_str, *tpoint_group, *tpoints; 520 bool error_found = false; 521 uint64_t group_id; 522 523 if (opts->shm_id >= 0) { 524 snprintf(shm_name, sizeof(shm_name), "/%s%s%d", opts->name, 525 SPDK_TRACE_SHM_NAME_BASE, opts->shm_id); 526 } else { 527 snprintf(shm_name, sizeof(shm_name), "/%s%spid%d", opts->name, 528 SPDK_TRACE_SHM_NAME_BASE, (int)getpid()); 529 } 530 531 if (spdk_trace_init(shm_name, opts->num_entries, 0) != 0) { 532 return -1; 533 } 534 535 if (opts->tpoint_group_mask == NULL) { 536 return 0; 537 } 538 539 tpoint_group_mask_str = strdup(opts->tpoint_group_mask); 540 if (tpoint_group_mask_str == NULL) { 541 SPDK_ERRLOG("Unable to get string of tpoint group mask from opts.\n"); 542 return -1; 543 } 544 /* Save a pointer to the original value of the tpoint group mask string 545 * to free later, because spdk_strsepq() modifies given char*. */ 546 tp_g_str = tpoint_group_mask_str; 547 while ((tpoint_group_str = spdk_strsepq(&tpoint_group_mask_str, ",")) != NULL) { 548 if (strchr(tpoint_group_str, ':')) { 549 /* Get the tpoint group mask */ 550 tpoint_group = spdk_strsepq(&tpoint_group_str, ":"); 551 /* Get the tpoint mask inside that group */ 552 tpoints = spdk_strsepq(&tpoint_group_str, ":"); 553 554 errno = 0; 555 tpoint_group_mask = strtoull(tpoint_group, &end, 16); 556 if (*end != '\0' || errno) { 557 tpoint_group_mask = spdk_trace_create_tpoint_group_mask(tpoint_group); 558 if (tpoint_group_mask == 0) { 559 error_found = true; 560 break; 561 } 562 } 563 /* Check if tpoint group mask has only one bit set. 564 * This is to avoid enabling individual tpoints in 565 * more than one tracepoint group at once. */ 566 if (!spdk_u64_is_pow2(tpoint_group_mask)) { 567 SPDK_ERRLOG("Tpoint group mask: %s contains multiple tpoint groups.\n", tpoint_group); 568 SPDK_ERRLOG("This is not supported, to prevent from activating tpoints by mistake.\n"); 569 error_found = true; 570 break; 571 } 572 573 errno = 0; 574 tpoint_mask = strtoull(tpoints, &end, 16); 575 if (*end != '\0' || errno) { 576 error_found = true; 577 break; 578 } 579 } else { 580 errno = 0; 581 tpoint_group_mask = strtoull(tpoint_group_str, &end, 16); 582 if (*end != '\0' || errno) { 583 tpoint_group_mask = spdk_trace_create_tpoint_group_mask(tpoint_group_str); 584 if (tpoint_group_mask == 0) { 585 error_found = true; 586 break; 587 } 588 } 589 tpoint_mask = -1ULL; 590 } 591 592 for (group_id = 0; group_id < SPDK_TRACE_MAX_GROUP_ID; ++group_id) { 593 if (tpoint_group_mask & (1 << group_id)) { 594 spdk_trace_set_tpoints(group_id, tpoint_mask); 595 } 596 } 597 } 598 599 if (error_found) { 600 SPDK_ERRLOG("invalid tpoint mask %s\n", opts->tpoint_group_mask); 601 free(tp_g_str); 602 return -1; 603 } else { 604 SPDK_NOTICELOG("Tracepoint Group Mask %s specified.\n", opts->tpoint_group_mask); 605 SPDK_NOTICELOG("Use 'spdk_trace -s %s %s %d' to capture a snapshot of events at runtime.\n", 606 opts->name, 607 opts->shm_id >= 0 ? "-i" : "-p", 608 opts->shm_id >= 0 ? opts->shm_id : getpid()); 609 #if defined(__linux__) 610 SPDK_NOTICELOG("'spdk_trace' without parameters will also work if this is the only\n"); 611 SPDK_NOTICELOG("SPDK application currently running.\n"); 612 SPDK_NOTICELOG("Or copy /dev/shm%s for offline analysis/debug.\n", shm_name); 613 #endif 614 } 615 free(tp_g_str); 616 617 return 0; 618 } 619 620 static void 621 bootstrap_fn(void *arg1) 622 { 623 spdk_rpc_set_allowlist(g_spdk_app.rpc_allowlist); 624 625 if (g_spdk_app.json_data) { 626 /* Load SPDK_RPC_STARTUP RPCs from config file */ 627 assert(spdk_rpc_get_state() == SPDK_RPC_STARTUP); 628 spdk_subsystem_load_config(g_spdk_app.json_data, g_spdk_app.json_data_size, 629 app_do_spdk_subsystem_init, NULL, 630 !g_spdk_app.json_config_ignore_errors); 631 } else { 632 app_do_spdk_subsystem_init(0, NULL); 633 } 634 } 635 636 static void 637 app_copy_opts(struct spdk_app_opts *opts, struct spdk_app_opts *opts_user, size_t opts_size) 638 { 639 spdk_app_opts_init(opts, sizeof(*opts)); 640 opts->opts_size = opts_size; 641 642 #define SET_FIELD(field) \ 643 if (offsetof(struct spdk_app_opts, field) + sizeof(opts->field) <= (opts->opts_size)) { \ 644 opts->field = opts_user->field; \ 645 } \ 646 647 SET_FIELD(name); 648 SET_FIELD(json_config_file); 649 SET_FIELD(json_config_ignore_errors); 650 SET_FIELD(rpc_addr); 651 SET_FIELD(reactor_mask); 652 SET_FIELD(lcore_map); 653 SET_FIELD(tpoint_group_mask); 654 SET_FIELD(shm_id); 655 SET_FIELD(shutdown_cb); 656 SET_FIELD(enable_coredump); 657 SET_FIELD(mem_channel); 658 SET_FIELD(main_core); 659 SET_FIELD(mem_size); 660 SET_FIELD(no_pci); 661 SET_FIELD(hugepage_single_segments); 662 SET_FIELD(unlink_hugepage); 663 SET_FIELD(no_huge); 664 SET_FIELD(hugedir); 665 SET_FIELD(print_level); 666 SET_FIELD(num_pci_addr); 667 SET_FIELD(pci_blocked); 668 SET_FIELD(pci_allowed); 669 SET_FIELD(iova_mode); 670 SET_FIELD(delay_subsystem_init); 671 SET_FIELD(num_entries); 672 SET_FIELD(env_context); 673 SET_FIELD(log); 674 SET_FIELD(base_virtaddr); 675 SET_FIELD(disable_signal_handlers); 676 SET_FIELD(interrupt_mode); 677 SET_FIELD(msg_mempool_size); 678 SET_FIELD(rpc_allowlist); 679 SET_FIELD(vf_token); 680 SET_FIELD(rpc_log_file); 681 SET_FIELD(rpc_log_level); 682 SET_FIELD(json_data); 683 SET_FIELD(json_data_size); 684 SET_FIELD(disable_cpumask_locks); 685 686 /* You should not remove this statement, but need to update the assert statement 687 * if you add a new field, and also add a corresponding SET_FIELD statement */ 688 SPDK_STATIC_ASSERT(sizeof(struct spdk_app_opts) == 253, "Incorrect size"); 689 690 #undef SET_FIELD 691 } 692 693 static int 694 unclaim_cpu_cores(uint32_t *failed_core) 695 { 696 char core_name[40]; 697 uint32_t i; 698 int rc; 699 700 for (i = 0; i < SPDK_CONFIG_MAX_LCORES; i++) { 701 if (g_core_locks[i] != -1) { 702 snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", i); 703 rc = close(g_core_locks[i]); 704 if (rc) { 705 SPDK_ERRLOG("Failed to close lock fd for core %d, errno: %d\n", i, errno); 706 goto error; 707 } 708 709 g_core_locks[i] = -1; 710 rc = unlink(core_name); 711 if (rc) { 712 SPDK_ERRLOG("Failed to unlink lock fd for core %d, errno: %d\n", i, errno); 713 goto error; 714 } 715 } 716 } 717 718 return 0; 719 720 error: 721 if (failed_core != NULL) { 722 /* Set number of core we failed to claim. */ 723 *failed_core = i; 724 } 725 return -1; 726 } 727 728 static int 729 claim_cpu_cores(uint32_t *failed_core) 730 { 731 char core_name[40]; 732 int core_fd, pid; 733 int *core_map; 734 uint32_t core; 735 736 struct flock core_lock = { 737 .l_type = F_WRLCK, 738 .l_whence = SEEK_SET, 739 .l_start = 0, 740 .l_len = 0, 741 }; 742 743 SPDK_ENV_FOREACH_CORE(core) { 744 if (g_core_locks[core] != -1) { 745 /* If this core is locked already, do not try lock it again. */ 746 continue; 747 } 748 749 snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", core); 750 core_fd = open(core_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 751 if (core_fd == -1) { 752 SPDK_ERRLOG("Could not open %s (%s).\n", core_name, spdk_strerror(errno)); 753 /* Return number of core we failed to claim. */ 754 goto error; 755 } 756 757 if (ftruncate(core_fd, sizeof(int)) != 0) { 758 SPDK_ERRLOG("Could not truncate %s (%s).\n", core_name, spdk_strerror(errno)); 759 close(core_fd); 760 goto error; 761 } 762 763 core_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE, MAP_SHARED, core_fd, 0); 764 if (core_map == MAP_FAILED) { 765 SPDK_ERRLOG("Could not mmap core %s (%s).\n", core_name, spdk_strerror(errno)); 766 close(core_fd); 767 goto error; 768 } 769 770 if (fcntl(core_fd, F_SETLK, &core_lock) != 0) { 771 pid = *core_map; 772 SPDK_ERRLOG("Cannot create lock on core %" PRIu32 ", probably process %d has claimed it.\n", 773 core, pid); 774 munmap(core_map, sizeof(int)); 775 close(core_fd); 776 goto error; 777 } 778 779 /* We write the PID to the core lock file so that other processes trying 780 * to claim the same core will know what process is holding the lock. */ 781 *core_map = (int)getpid(); 782 munmap(core_map, sizeof(int)); 783 g_core_locks[core] = core_fd; 784 /* Keep core_fd open to maintain the lock. */ 785 } 786 787 return 0; 788 789 error: 790 if (failed_core != NULL) { 791 /* Set number of core we failed to claim. */ 792 *failed_core = core; 793 } 794 unclaim_cpu_cores(NULL); 795 return -1; 796 } 797 798 int 799 spdk_app_start(struct spdk_app_opts *opts_user, spdk_msg_fn start_fn, 800 void *arg1) 801 { 802 int rc; 803 char *tty; 804 struct spdk_cpuset tmp_cpumask = {}; 805 static bool g_env_was_setup = false; 806 struct spdk_app_opts opts_local = {}; 807 struct spdk_app_opts *opts = &opts_local; 808 uint32_t i, core; 809 810 if (!opts_user) { 811 SPDK_ERRLOG("opts_user should not be NULL\n"); 812 return 1; 813 } 814 815 if (!opts_user->opts_size) { 816 SPDK_ERRLOG("The opts_size in opts_user structure should not be zero value\n"); 817 return 1; 818 } 819 820 if (opts_user->name == NULL) { 821 SPDK_ERRLOG("spdk_app_opts::name not specified\n"); 822 return 1; 823 } 824 825 app_copy_opts(opts, opts_user, opts_user->opts_size); 826 827 if (!start_fn) { 828 SPDK_ERRLOG("start_fn should not be NULL\n"); 829 return 1; 830 } 831 832 if (!opts->rpc_addr && opts->delay_subsystem_init) { 833 SPDK_ERRLOG("Cannot use '--wait-for-rpc' if no RPC server is going to be started.\n"); 834 return 1; 835 } 836 837 if (!(opts->lcore_map || opts->reactor_mask)) { 838 /* Set default CPU mask */ 839 opts->reactor_mask = SPDK_APP_DPDK_DEFAULT_CORE_MASK; 840 } 841 842 tty = ttyname(STDERR_FILENO); 843 if (opts->print_level > SPDK_LOG_WARN && 844 isatty(STDERR_FILENO) && 845 tty && 846 !strncmp(tty, "/dev/tty", strlen("/dev/tty"))) { 847 printf("Warning: printing stderr to console terminal without -q option specified.\n"); 848 printf("Suggest using --silence-noticelog to disable logging to stderr and\n"); 849 printf("monitor syslog, or redirect stderr to a file.\n"); 850 printf("(Delaying for 10 seconds...)\n"); 851 sleep(10); 852 } 853 854 spdk_log_set_print_level(opts->print_level); 855 856 #ifndef SPDK_NO_RLIMIT 857 if (opts->enable_coredump) { 858 struct rlimit core_limits; 859 860 core_limits.rlim_cur = core_limits.rlim_max = SPDK_APP_DEFAULT_CORE_LIMIT; 861 setrlimit(RLIMIT_CORE, &core_limits); 862 } 863 #endif 864 865 if (opts->interrupt_mode) { 866 spdk_interrupt_mode_enable(); 867 } 868 869 memset(&g_spdk_app, 0, sizeof(g_spdk_app)); 870 871 g_spdk_app.json_config_ignore_errors = opts->json_config_ignore_errors; 872 g_spdk_app.rpc_addr = opts->rpc_addr; 873 g_spdk_app.rpc_allowlist = opts->rpc_allowlist; 874 g_spdk_app.rpc_log_file = opts->rpc_log_file; 875 g_spdk_app.rpc_log_level = opts->rpc_log_level; 876 g_spdk_app.shm_id = opts->shm_id; 877 g_spdk_app.shutdown_cb = opts->shutdown_cb; 878 g_spdk_app.rc = 0; 879 g_spdk_app.stopped = false; 880 881 spdk_log_set_level(SPDK_APP_DEFAULT_LOG_LEVEL); 882 883 /* Pass NULL to app_setup_env if SPDK app has been set up, in order to 884 * indicate that this is a reinitialization. 885 */ 886 if (app_setup_env(g_env_was_setup ? NULL : opts) < 0) { 887 return 1; 888 } 889 890 /* Calculate mempool size now that the env layer has configured the core count 891 * for the application */ 892 calculate_mempool_size(opts, opts_user); 893 894 spdk_log_open(opts->log); 895 896 /* Initialize each lock to -1 to indicate "empty" status */ 897 for (i = 0; i < SPDK_CONFIG_MAX_LCORES; i++) { 898 g_core_locks[i] = -1; 899 } 900 901 if (!opts->disable_cpumask_locks) { 902 if (claim_cpu_cores(NULL)) { 903 SPDK_ERRLOG("Unable to acquire lock on assigned core mask - exiting.\n"); 904 return 1; 905 } 906 } else { 907 SPDK_NOTICELOG("CPU core locks deactivated.\n"); 908 } 909 910 SPDK_NOTICELOG("Total cores available: %d\n", spdk_env_get_core_count()); 911 912 if ((rc = spdk_reactors_init(opts->msg_mempool_size)) != 0) { 913 SPDK_ERRLOG("Reactor Initialization failed: rc = %d\n", rc); 914 return 1; 915 } 916 917 spdk_cpuset_set_cpu(&tmp_cpumask, spdk_env_get_current_core(), true); 918 919 /* Now that the reactors have been initialized, we can create the app thread. */ 920 spdk_thread_create("app_thread", &tmp_cpumask); 921 if (!spdk_thread_get_app_thread()) { 922 SPDK_ERRLOG("Unable to create an spdk_thread for initialization\n"); 923 return 1; 924 } 925 926 SPDK_ENV_FOREACH_CORE(core) { 927 rc = init_proc_stat(core); 928 if (rc) { 929 SPDK_NOTICELOG("Unable to parse /proc/stat [core: %d].\n", core); 930 } 931 } 932 /* 933 * Disable and ignore trace setup if setting num_entries 934 * to be 0. 935 * 936 * Note the call to app_setup_trace() is located here 937 * ahead of app_setup_signal_handlers(). 938 * That's because there is not an easy/direct clean 939 * way of unwinding alloc'd resources that can occur 940 * in app_setup_signal_handlers(). 941 */ 942 if (opts->num_entries != 0 && app_setup_trace(opts) != 0) { 943 return 1; 944 } 945 946 if (!opts->disable_signal_handlers && app_setup_signal_handlers(opts) != 0) { 947 return 1; 948 } 949 950 g_delay_subsystem_init = opts->delay_subsystem_init; 951 g_start_fn = start_fn; 952 g_start_arg = arg1; 953 954 if (opts->json_config_file != NULL) { 955 if (opts->json_data) { 956 SPDK_ERRLOG("App opts json_config_file and json_data are mutually exclusive\n"); 957 return 1; 958 } 959 960 g_spdk_app.json_data = spdk_posix_file_load_from_name(opts->json_config_file, 961 &g_spdk_app.json_data_size); 962 if (!g_spdk_app.json_data) { 963 SPDK_ERRLOG("Read JSON configuration file %s failed: %s\n", 964 opts->json_config_file, spdk_strerror(errno)); 965 return 1; 966 } 967 } else if (opts->json_data) { 968 g_spdk_app.json_data = calloc(1, opts->json_data_size); 969 if (!g_spdk_app.json_data) { 970 SPDK_ERRLOG("Failed to allocate JSON data buffer\n"); 971 return 1; 972 } 973 974 memcpy(g_spdk_app.json_data, opts->json_data, opts->json_data_size); 975 g_spdk_app.json_data_size = opts->json_data_size; 976 } 977 978 spdk_thread_send_msg(spdk_thread_get_app_thread(), bootstrap_fn, NULL); 979 980 /* This blocks until spdk_app_stop is called */ 981 spdk_reactors_start(); 982 983 g_env_was_setup = true; 984 985 return g_spdk_app.rc; 986 } 987 988 void 989 spdk_app_fini(void) 990 { 991 spdk_trace_cleanup(); 992 spdk_reactors_fini(); 993 spdk_env_fini(); 994 spdk_log_close(); 995 unclaim_cpu_cores(NULL); 996 } 997 998 static void 999 subsystem_fini_done(void *arg1) 1000 { 1001 spdk_rpc_finish(); 1002 spdk_reactors_stop(NULL); 1003 } 1004 1005 static void 1006 _start_subsystem_fini(void *arg1) 1007 { 1008 if (g_scheduling_in_progress) { 1009 spdk_thread_send_msg(spdk_thread_get_app_thread(), _start_subsystem_fini, NULL); 1010 return; 1011 } 1012 1013 spdk_subsystem_fini(subsystem_fini_done, NULL); 1014 } 1015 1016 static int 1017 log_deprecation_hits(void *ctx, struct spdk_deprecation *dep) 1018 { 1019 uint64_t hits = spdk_deprecation_get_hits(dep); 1020 1021 if (hits == 0) { 1022 return 0; 1023 } 1024 1025 SPDK_WARNLOG("%s: deprecation '%s' scheduled for removal in %s hit %" PRIu64 " times\n", 1026 spdk_deprecation_get_tag(dep), spdk_deprecation_get_description(dep), 1027 spdk_deprecation_get_remove_release(dep), hits); 1028 return 0; 1029 } 1030 1031 static void 1032 app_stop(void *arg1) 1033 { 1034 free(g_spdk_app.json_data); 1035 1036 if (g_spdk_app.rc == 0) { 1037 g_spdk_app.rc = (int)(intptr_t)arg1; 1038 } 1039 1040 if (g_spdk_app.stopped) { 1041 SPDK_NOTICELOG("spdk_app_stop called twice\n"); 1042 return; 1043 } 1044 1045 g_spdk_app.stopped = true; 1046 spdk_log_for_each_deprecation(NULL, log_deprecation_hits); 1047 _start_subsystem_fini(NULL); 1048 } 1049 1050 void 1051 spdk_app_stop(int rc) 1052 { 1053 if (rc) { 1054 SPDK_WARNLOG("spdk_app_stop'd on non-zero\n"); 1055 } 1056 1057 /* 1058 * We want to run spdk_subsystem_fini() from the same thread where spdk_subsystem_init() 1059 * was called. 1060 */ 1061 spdk_thread_send_msg(spdk_thread_get_app_thread(), app_stop, (void *)(intptr_t)rc); 1062 } 1063 1064 static void 1065 usage_memory_size(void) 1066 { 1067 #ifndef __linux__ 1068 if (g_default_opts.mem_size <= 0) { 1069 printf("all hugepage memory)\n"); 1070 } else 1071 #endif 1072 { 1073 printf("%dMB)\n", g_default_opts.mem_size >= 0 ? g_default_opts.mem_size : 0); 1074 } 1075 } 1076 1077 static void 1078 usage(void (*app_usage)(void)) 1079 { 1080 printf("%s [options]\n", g_executable_name); 1081 /* Keep entries inside categories roughly sorted by frequency of use. */ 1082 printf("\nCPU options:\n"); 1083 printf(" -m, --cpumask <mask or list> core mask (like 0xF) or core list of '[]' embraced for DPDK\n"); 1084 printf(" (like [0,1,10])\n"); 1085 printf(" --lcores <list> lcore to CPU mapping list. The list is in the format:\n"); 1086 printf(" <lcores[@CPUs]>[<,lcores[@CPUs]>...]\n"); 1087 printf(" lcores and cpus list are grouped by '(' and ')', e.g '--lcores \"(5-7)@(10-12)\"'\n"); 1088 printf(" Within the group, '-' is used for range separator,\n"); 1089 printf(" ',' is used for single number separator.\n"); 1090 printf(" '( )' can be omitted for single element group,\n"); 1091 printf(" '@' can be omitted if cpus and lcores have the same value\n"); 1092 printf(" --disable-cpumask-locks Disable CPU core lock files.\n"); 1093 printf(" --interrupt-mode set app to interrupt mode (Warning: CPU usage will be reduced only if all\n"); 1094 printf(" pollers in the app support interrupt mode)\n"); 1095 printf(" -p, --main-core <id> main (primary) core for DPDK\n"); 1096 1097 printf("\nConfiguration options:\n"); 1098 printf(" -c, --config, --json <config> JSON config file\n"); 1099 printf(" -r, --rpc-socket <path> RPC listen address (default %s)\n", SPDK_DEFAULT_RPC_ADDR); 1100 printf(" --no-rpc-server skip RPC server initialization. This option ignores '--rpc-socket' value.\n"); 1101 printf(" --wait-for-rpc wait for RPCs to initialize subsystems\n"); 1102 printf(" --rpcs-allowed comma-separated list of permitted RPCS\n"); 1103 printf(" --json-ignore-init-errors don't exit on invalid config entry\n"); 1104 1105 printf("\nMemory options:\n"); 1106 printf(" --iova-mode <pa/va> set IOVA mode ('pa' for IOVA_PA and 'va' for IOVA_VA)\n"); 1107 printf(" --base-virtaddr <addr> the base virtual address for DPDK (default: 0x200000000000)\n"); 1108 printf(" --huge-dir <path> use a specific hugetlbfs mount to reserve memory from\n"); 1109 printf(" -R, --huge-unlink unlink huge files after initialization\n"); 1110 printf(" -n, --mem-channels <num> number of memory channels used for DPDK\n"); 1111 printf(" -s, --mem-size <size> memory size in MB for DPDK (default: "); 1112 usage_memory_size(); 1113 printf(" --msg-mempool-size <size> global message memory pool size in count (default: %d)\n", 1114 SPDK_DEFAULT_MSG_MEMPOOL_SIZE); 1115 printf(" --no-huge run without using hugepages\n"); 1116 printf(" -i, --shm-id <id> shared memory ID (optional)\n"); 1117 printf(" -g, --single-file-segments force creating just one hugetlbfs file\n"); 1118 1119 printf("\nPCI options:\n"); 1120 printf(" -A, --pci-allowed <bdf> pci addr to allow (-B and -A cannot be used at the same time)\n"); 1121 printf(" -B, --pci-blocked <bdf> pci addr to block (can be used more than once)\n"); 1122 printf(" -u, --no-pci disable PCI access\n"); 1123 printf(" --vfio-vf-token VF token (UUID) shared between SR-IOV PF and VFs for vfio_pci driver\n"); 1124 1125 printf("\nLog options:\n"); 1126 spdk_log_usage(stdout, "-L"); 1127 printf(" --silence-noticelog disable notice level logging to stderr\n"); 1128 1129 printf("\nTrace options:\n"); 1130 printf(" --num-trace-entries <num> number of trace entries for each core, must be power of 2,\n"); 1131 printf(" setting 0 to disable trace (default %d)\n", 1132 SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES); 1133 printf(" Tracepoints vary in size and can use more than one trace entry.\n"); 1134 spdk_trace_mask_usage(stdout, "-e"); 1135 1136 printf("\nOther options:\n"); 1137 printf(" -h, --help show this usage\n"); 1138 printf(" -v, --version print SPDK version\n"); 1139 printf(" -d, --limit-coredump do not set max coredump size to RLIM_INFINITY\n"); 1140 printf(" --env-context Opaque context for use of the env implementation\n"); 1141 1142 if (app_usage) { 1143 printf("\nApplication specific:\n"); 1144 app_usage(); 1145 } 1146 } 1147 1148 spdk_app_parse_args_rvals_t 1149 spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts, 1150 const char *app_getopt_str, const struct option *app_long_opts, 1151 int (*app_parse)(int ch, char *arg), 1152 void (*app_usage)(void)) 1153 { 1154 int ch, rc, opt_idx, global_long_opts_len, app_long_opts_len; 1155 struct option *cmdline_options; 1156 char *cmdline_short_opts = NULL; 1157 char *shm_id_str = NULL; 1158 enum spdk_app_parse_args_rvals retval = SPDK_APP_PARSE_ARGS_FAIL; 1159 long int tmp; 1160 1161 memcpy(&g_default_opts, opts, sizeof(g_default_opts)); 1162 1163 if (opts->json_config_file && access(opts->json_config_file, R_OK) != 0) { 1164 SPDK_WARNLOG("Can't read JSON configuration file '%s'\n", opts->json_config_file); 1165 opts->json_config_file = NULL; 1166 } 1167 1168 if (app_long_opts == NULL) { 1169 app_long_opts_len = 0; 1170 } else { 1171 for (app_long_opts_len = 0; 1172 app_long_opts[app_long_opts_len].name != NULL; 1173 app_long_opts_len++); 1174 } 1175 1176 global_long_opts_len = SPDK_COUNTOF(g_cmdline_options); 1177 1178 cmdline_options = calloc(global_long_opts_len + app_long_opts_len + 1, sizeof(*cmdline_options)); 1179 if (!cmdline_options) { 1180 SPDK_ERRLOG("Out of memory\n"); 1181 return SPDK_APP_PARSE_ARGS_FAIL; 1182 } 1183 1184 memcpy(&cmdline_options[0], g_cmdline_options, sizeof(g_cmdline_options)); 1185 if (app_long_opts) { 1186 memcpy(&cmdline_options[global_long_opts_len], app_long_opts, 1187 app_long_opts_len * sizeof(*app_long_opts)); 1188 } 1189 1190 if (app_getopt_str != NULL) { 1191 ch = app_opts_validate(app_getopt_str); 1192 if (ch) { 1193 SPDK_ERRLOG("Duplicated option '%c' between app-specific command line parameter and generic spdk opts.\n", 1194 ch); 1195 goto out; 1196 } 1197 1198 if (!app_parse) { 1199 SPDK_ERRLOG("Parse function is required when app-specific command line parameters are provided.\n"); 1200 goto out; 1201 } 1202 } 1203 1204 cmdline_short_opts = spdk_sprintf_alloc("%s%s", app_getopt_str, SPDK_APP_GETOPT_STRING); 1205 if (!cmdline_short_opts) { 1206 SPDK_ERRLOG("Out of memory\n"); 1207 goto out; 1208 } 1209 1210 g_executable_name = argv[0]; 1211 1212 while ((ch = getopt_long(argc, argv, cmdline_short_opts, cmdline_options, &opt_idx)) != -1) { 1213 switch (ch) { 1214 case CONFIG_FILE_OPT_IDX: 1215 case JSON_CONFIG_OPT_IDX: 1216 opts->json_config_file = optarg; 1217 break; 1218 case JSON_CONFIG_IGNORE_INIT_ERRORS_IDX: 1219 opts->json_config_ignore_errors = true; 1220 break; 1221 case LIMIT_COREDUMP_OPT_IDX: 1222 opts->enable_coredump = false; 1223 break; 1224 case TPOINT_GROUP_OPT_IDX: 1225 opts->tpoint_group_mask = optarg; 1226 break; 1227 case SINGLE_FILE_SEGMENTS_OPT_IDX: 1228 opts->hugepage_single_segments = true; 1229 break; 1230 case HELP_OPT_IDX: 1231 usage(app_usage); 1232 retval = SPDK_APP_PARSE_ARGS_HELP; 1233 goto out; 1234 case SHM_ID_OPT_IDX: 1235 shm_id_str = optarg; 1236 /* a negative shm-id disables shared configuration file */ 1237 if (optarg[0] == '-') { 1238 shm_id_str++; 1239 } 1240 /* check if the positive value of provided shm_id can be parsed as 1241 * an integer 1242 */ 1243 opts->shm_id = spdk_strtol(shm_id_str, 0); 1244 if (opts->shm_id < 0) { 1245 SPDK_ERRLOG("Invalid shared memory ID %s\n", optarg); 1246 goto out; 1247 } 1248 if (optarg[0] == '-') { 1249 opts->shm_id = -opts->shm_id; 1250 } 1251 break; 1252 case CPUMASK_OPT_IDX: 1253 if (opts->lcore_map) { 1254 SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n"); 1255 goto out; 1256 } 1257 opts->reactor_mask = optarg; 1258 break; 1259 case LCORES_OPT_IDX: 1260 if (opts->reactor_mask) { 1261 SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n"); 1262 goto out; 1263 } 1264 opts->lcore_map = optarg; 1265 break; 1266 case DISABLE_CPUMASK_LOCKS_OPT_IDX: 1267 opts->disable_cpumask_locks = true; 1268 break; 1269 case MEM_CHANNELS_OPT_IDX: 1270 opts->mem_channel = spdk_strtol(optarg, 0); 1271 if (opts->mem_channel < 0) { 1272 SPDK_ERRLOG("Invalid memory channel %s\n", optarg); 1273 goto out; 1274 } 1275 break; 1276 case MAIN_CORE_OPT_IDX: 1277 opts->main_core = spdk_strtol(optarg, 0); 1278 if (opts->main_core < 0) { 1279 SPDK_ERRLOG("Invalid main core %s\n", optarg); 1280 goto out; 1281 } 1282 break; 1283 case SILENCE_NOTICELOG_OPT_IDX: 1284 opts->print_level = SPDK_LOG_WARN; 1285 break; 1286 case RPC_SOCKET_OPT_IDX: 1287 opts->rpc_addr = optarg; 1288 break; 1289 case NO_RPC_SERVER_OPT_IDX: 1290 opts->rpc_addr = NULL; 1291 break; 1292 case MEM_SIZE_OPT_IDX: { 1293 uint64_t mem_size_mb; 1294 bool mem_size_has_prefix; 1295 1296 rc = spdk_parse_capacity(optarg, &mem_size_mb, &mem_size_has_prefix); 1297 if (rc != 0) { 1298 SPDK_ERRLOG("invalid memory pool size `-s %s`\n", optarg); 1299 usage(app_usage); 1300 goto out; 1301 } 1302 1303 if (mem_size_has_prefix) { 1304 /* the mem size is in MB by default, so if a prefix was 1305 * specified, we need to manually convert to MB. 1306 */ 1307 mem_size_mb /= 1024 * 1024; 1308 } 1309 1310 if (mem_size_mb > INT_MAX) { 1311 SPDK_ERRLOG("invalid memory pool size `-s %s`\n", optarg); 1312 usage(app_usage); 1313 goto out; 1314 } 1315 1316 opts->mem_size = (int) mem_size_mb; 1317 break; 1318 } 1319 case MSG_MEMPOOL_SIZE_OPT_IDX: 1320 tmp = spdk_strtol(optarg, 10); 1321 if (tmp <= 0) { 1322 SPDK_ERRLOG("Invalid message memory pool size %s\n", optarg); 1323 goto out; 1324 } 1325 1326 opts->msg_mempool_size = (size_t)tmp; 1327 break; 1328 1329 case NO_PCI_OPT_IDX: 1330 opts->no_pci = true; 1331 break; 1332 case WAIT_FOR_RPC_OPT_IDX: 1333 opts->delay_subsystem_init = true; 1334 break; 1335 case PCI_BLOCKED_OPT_IDX: 1336 if (opts->pci_allowed) { 1337 free(opts->pci_allowed); 1338 opts->pci_allowed = NULL; 1339 SPDK_ERRLOG("-B and -A cannot be used at the same time\n"); 1340 usage(app_usage); 1341 goto out; 1342 } 1343 1344 rc = app_opts_add_pci_addr(opts, &opts->pci_blocked, optarg); 1345 if (rc != 0) { 1346 free(opts->pci_blocked); 1347 opts->pci_blocked = NULL; 1348 goto out; 1349 } 1350 break; 1351 1352 case NO_HUGE_OPT_IDX: 1353 opts->no_huge = true; 1354 break; 1355 1356 case LOGFLAG_OPT_IDX: 1357 rc = spdk_log_set_flag(optarg); 1358 if (rc < 0) { 1359 SPDK_ERRLOG("unknown flag: %s\n", optarg); 1360 usage(app_usage); 1361 goto out; 1362 } 1363 #ifdef DEBUG 1364 opts->print_level = SPDK_LOG_DEBUG; 1365 #endif 1366 break; 1367 case HUGE_UNLINK_OPT_IDX: 1368 opts->unlink_hugepage = true; 1369 break; 1370 case PCI_ALLOWED_OPT_IDX: 1371 if (opts->pci_blocked) { 1372 free(opts->pci_blocked); 1373 opts->pci_blocked = NULL; 1374 SPDK_ERRLOG("-B and -W cannot be used at the same time\n"); 1375 usage(app_usage); 1376 goto out; 1377 } 1378 1379 rc = app_opts_add_pci_addr(opts, &opts->pci_allowed, optarg); 1380 if (rc != 0) { 1381 free(opts->pci_allowed); 1382 opts->pci_allowed = NULL; 1383 goto out; 1384 } 1385 break; 1386 case BASE_VIRTADDR_OPT_IDX: 1387 tmp = spdk_strtoll(optarg, 0); 1388 if (tmp <= 0) { 1389 SPDK_ERRLOG("Invalid base-virtaddr %s\n", optarg); 1390 usage(app_usage); 1391 goto out; 1392 } 1393 opts->base_virtaddr = (uint64_t)tmp; 1394 break; 1395 case HUGE_DIR_OPT_IDX: 1396 opts->hugedir = optarg; 1397 break; 1398 case IOVA_MODE_OPT_IDX: 1399 opts->iova_mode = optarg; 1400 break; 1401 case NUM_TRACE_ENTRIES_OPT_IDX: 1402 tmp = spdk_strtoll(optarg, 0); 1403 if (tmp < 0) { 1404 SPDK_ERRLOG("Invalid num-trace-entries %s\n", optarg); 1405 usage(app_usage); 1406 goto out; 1407 } 1408 opts->num_entries = (uint64_t)tmp; 1409 if (opts->num_entries > 0 && opts->num_entries & (opts->num_entries - 1)) { 1410 SPDK_ERRLOG("num-trace-entries must be power of 2\n"); 1411 usage(app_usage); 1412 goto out; 1413 } 1414 break; 1415 case ENV_CONTEXT_OPT_IDX: 1416 opts->env_context = optarg; 1417 break; 1418 case RPCS_ALLOWED_OPT_IDX: 1419 opts->rpc_allowlist = (const char **)spdk_strarray_from_string(optarg, ","); 1420 if (opts->rpc_allowlist == NULL) { 1421 SPDK_ERRLOG("Invalid --rpcs-allowed argument\n"); 1422 usage(app_usage); 1423 goto out; 1424 } 1425 break; 1426 case ENV_VF_TOKEN_OPT_IDX: 1427 opts->vf_token = optarg; 1428 break; 1429 case INTERRUPT_MODE_OPT_IDX: 1430 opts->interrupt_mode = true; 1431 break; 1432 case VERSION_OPT_IDX: 1433 printf(SPDK_VERSION_STRING"\n"); 1434 retval = SPDK_APP_PARSE_ARGS_HELP; 1435 goto out; 1436 case '?': 1437 /* 1438 * In the event getopt() above detects an option 1439 * in argv that is NOT in the getopt_str, 1440 * getopt() will return a '?' indicating failure. 1441 */ 1442 usage(app_usage); 1443 goto out; 1444 default: 1445 if (!app_parse) { 1446 SPDK_ERRLOG("Unsupported app-specific command line parameter '%c'.\n", ch); 1447 goto out; 1448 } 1449 1450 rc = app_parse(ch, optarg); 1451 if (rc) { 1452 SPDK_ERRLOG("Parsing app-specific command line parameter '%c' failed: %d\n", ch, rc); 1453 goto out; 1454 } 1455 } 1456 } 1457 1458 retval = SPDK_APP_PARSE_ARGS_SUCCESS; 1459 out: 1460 if (retval != SPDK_APP_PARSE_ARGS_SUCCESS) { 1461 free(opts->pci_blocked); 1462 opts->pci_blocked = NULL; 1463 free(opts->pci_allowed); 1464 opts->pci_allowed = NULL; 1465 spdk_strarray_free((char **)opts->rpc_allowlist); 1466 opts->rpc_allowlist = NULL; 1467 } 1468 free(cmdline_short_opts); 1469 free(cmdline_options); 1470 return retval; 1471 } 1472 1473 void 1474 spdk_app_usage(void) 1475 { 1476 if (g_executable_name == NULL) { 1477 SPDK_ERRLOG("%s not valid before calling spdk_app_parse_args()\n", __func__); 1478 return; 1479 } 1480 1481 usage(NULL); 1482 } 1483 1484 static void 1485 rpc_framework_start_init_cpl(int rc, void *arg1) 1486 { 1487 struct spdk_jsonrpc_request *request = arg1; 1488 1489 assert(spdk_thread_is_app_thread(NULL)); 1490 1491 if (rc) { 1492 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, 1493 "framework_initialization failed"); 1494 return; 1495 } 1496 1497 app_subsystem_init_done(0, NULL); 1498 1499 spdk_jsonrpc_send_bool_response(request, true); 1500 } 1501 1502 static void 1503 rpc_framework_start_init(struct spdk_jsonrpc_request *request, 1504 const struct spdk_json_val *params) 1505 { 1506 if (params != NULL) { 1507 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1508 "framework_start_init requires no parameters"); 1509 return; 1510 } 1511 1512 spdk_rpc_server_pause(g_spdk_app.rpc_addr); 1513 spdk_subsystem_init(rpc_framework_start_init_cpl, request); 1514 } 1515 SPDK_RPC_REGISTER("framework_start_init", rpc_framework_start_init, SPDK_RPC_STARTUP) 1516 1517 struct subsystem_init_poller_ctx { 1518 struct spdk_poller *init_poller; 1519 struct spdk_jsonrpc_request *request; 1520 }; 1521 1522 static int 1523 rpc_subsystem_init_poller_ctx(void *ctx) 1524 { 1525 struct subsystem_init_poller_ctx *poller_ctx = ctx; 1526 1527 if (spdk_rpc_get_state() == SPDK_RPC_RUNTIME) { 1528 spdk_jsonrpc_send_bool_response(poller_ctx->request, true); 1529 spdk_poller_unregister(&poller_ctx->init_poller); 1530 free(poller_ctx); 1531 } 1532 1533 return SPDK_POLLER_BUSY; 1534 } 1535 1536 static void 1537 rpc_framework_wait_init(struct spdk_jsonrpc_request *request, 1538 const struct spdk_json_val *params) 1539 { 1540 struct subsystem_init_poller_ctx *ctx; 1541 1542 if (spdk_rpc_get_state() == SPDK_RPC_RUNTIME) { 1543 spdk_jsonrpc_send_bool_response(request, true); 1544 } else { 1545 ctx = malloc(sizeof(struct subsystem_init_poller_ctx)); 1546 if (ctx == NULL) { 1547 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, 1548 "Unable to allocate memory for the request context\n"); 1549 return; 1550 } 1551 ctx->request = request; 1552 ctx->init_poller = SPDK_POLLER_REGISTER(rpc_subsystem_init_poller_ctx, ctx, 0); 1553 } 1554 } 1555 SPDK_RPC_REGISTER("framework_wait_init", rpc_framework_wait_init, 1556 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1557 1558 static void 1559 rpc_framework_disable_cpumask_locks(struct spdk_jsonrpc_request *request, 1560 const struct spdk_json_val *params) 1561 { 1562 char msg[128]; 1563 int rc; 1564 uint32_t failed_core; 1565 1566 if (params != NULL) { 1567 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1568 "framework_disable_cpumask_locks" 1569 "requires no arguments"); 1570 return; 1571 } 1572 1573 rc = unclaim_cpu_cores(&failed_core); 1574 if (rc) { 1575 snprintf(msg, sizeof(msg), "Failed to unclaim CPU core: %" PRIu32, failed_core); 1576 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg); 1577 return; 1578 } 1579 1580 spdk_jsonrpc_send_bool_response(request, true); 1581 } 1582 SPDK_RPC_REGISTER("framework_disable_cpumask_locks", rpc_framework_disable_cpumask_locks, 1583 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1584 1585 static void 1586 rpc_framework_enable_cpumask_locks(struct spdk_jsonrpc_request *request, 1587 const struct spdk_json_val *params) 1588 { 1589 char msg[128]; 1590 int rc; 1591 uint32_t failed_core; 1592 1593 if (params != NULL) { 1594 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1595 "framework_enable_cpumask_locks" 1596 "requires no arguments"); 1597 return; 1598 } 1599 1600 rc = claim_cpu_cores(&failed_core); 1601 if (rc) { 1602 snprintf(msg, sizeof(msg), "Failed to claim CPU core: %" PRIu32, failed_core); 1603 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg); 1604 return; 1605 } 1606 1607 spdk_jsonrpc_send_bool_response(request, true); 1608 } 1609 SPDK_RPC_REGISTER("framework_enable_cpumask_locks", rpc_framework_enable_cpumask_locks, 1610 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1611