1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk/version.h" 9 10 #include "spdk_internal/event.h" 11 12 #include "spdk/assert.h" 13 #include "spdk/env.h" 14 #include "spdk/init.h" 15 #include "spdk/log.h" 16 #include "spdk/thread.h" 17 #include "spdk/trace.h" 18 #include "spdk/string.h" 19 #include "spdk/scheduler.h" 20 #include "spdk/rpc.h" 21 #include "spdk/util.h" 22 #include "spdk/file.h" 23 #include "spdk/config.h" 24 #include "event_internal.h" 25 26 #define SPDK_APP_DEFAULT_LOG_LEVEL SPDK_LOG_NOTICE 27 #define SPDK_APP_DEFAULT_LOG_PRINT_LEVEL SPDK_LOG_INFO 28 #define SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES SPDK_DEFAULT_NUM_TRACE_ENTRIES 29 30 #define SPDK_APP_DPDK_DEFAULT_MEM_SIZE -1 31 #define SPDK_APP_DPDK_DEFAULT_MAIN_CORE -1 32 #define SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL -1 33 #define SPDK_APP_DPDK_DEFAULT_CORE_MASK "0x1" 34 #define SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR 0x200000000000 35 #define SPDK_APP_DEFAULT_CORE_LIMIT 0x140000000 /* 5 GiB */ 36 37 /* For core counts <= 63, the message memory pool size is set to 38 * SPDK_DEFAULT_MSG_MEMPOOL_SIZE. 39 * For core counts > 63, the message memory pool size is depended on 40 * number of cores. Per core, it is calculated as SPDK_MSG_MEMPOOL_CACHE_SIZE 41 * multiplied by factor of 4 to have space for multiple spdk threads running 42 * on single core (e.g iscsi + nvmf + vhost ). */ 43 #define SPDK_APP_PER_CORE_MSG_MEMPOOL_SIZE (4 * SPDK_MSG_MEMPOOL_CACHE_SIZE) 44 45 struct spdk_app { 46 void *json_data; 47 size_t json_data_size; 48 bool json_config_ignore_errors; 49 bool stopped; 50 const char *rpc_addr; 51 const char **rpc_allowlist; 52 FILE *rpc_log_file; 53 enum spdk_log_level rpc_log_level; 54 int shm_id; 55 spdk_app_shutdown_cb shutdown_cb; 56 int rc; 57 }; 58 59 static struct spdk_app g_spdk_app; 60 static spdk_msg_fn g_start_fn = NULL; 61 static void *g_start_arg = NULL; 62 static bool g_delay_subsystem_init = false; 63 static bool g_shutdown_sig_received = false; 64 static char *g_executable_name; 65 static struct spdk_app_opts g_default_opts; 66 static int g_core_locks[SPDK_CONFIG_MAX_LCORES]; 67 68 static struct { 69 uint64_t irq; 70 uint64_t usr; 71 uint64_t sys; 72 } g_initial_stat[SPDK_CONFIG_MAX_LCORES]; 73 74 int 75 spdk_app_get_shm_id(void) 76 { 77 return g_spdk_app.shm_id; 78 } 79 80 /* append one empty option to indicate the end of the array */ 81 static const struct option g_cmdline_options[] = { 82 #define CONFIG_FILE_OPT_IDX 'c' 83 {"config", required_argument, NULL, CONFIG_FILE_OPT_IDX}, 84 #define LIMIT_COREDUMP_OPT_IDX 'd' 85 {"limit-coredump", no_argument, NULL, LIMIT_COREDUMP_OPT_IDX}, 86 #define TPOINT_GROUP_OPT_IDX 'e' 87 {"tpoint-group", required_argument, NULL, TPOINT_GROUP_OPT_IDX}, 88 #define SINGLE_FILE_SEGMENTS_OPT_IDX 'g' 89 {"single-file-segments", no_argument, NULL, SINGLE_FILE_SEGMENTS_OPT_IDX}, 90 #define HELP_OPT_IDX 'h' 91 {"help", no_argument, NULL, HELP_OPT_IDX}, 92 #define SHM_ID_OPT_IDX 'i' 93 {"shm-id", required_argument, NULL, SHM_ID_OPT_IDX}, 94 #define CPUMASK_OPT_IDX 'm' 95 {"cpumask", required_argument, NULL, CPUMASK_OPT_IDX}, 96 #define MEM_CHANNELS_OPT_IDX 'n' 97 {"mem-channels", required_argument, NULL, MEM_CHANNELS_OPT_IDX}, 98 #define MAIN_CORE_OPT_IDX 'p' 99 {"main-core", required_argument, NULL, MAIN_CORE_OPT_IDX}, 100 #define RPC_SOCKET_OPT_IDX 'r' 101 {"rpc-socket", required_argument, NULL, RPC_SOCKET_OPT_IDX}, 102 #define MEM_SIZE_OPT_IDX 's' 103 {"mem-size", required_argument, NULL, MEM_SIZE_OPT_IDX}, 104 #define NO_PCI_OPT_IDX 'u' 105 {"no-pci", no_argument, NULL, NO_PCI_OPT_IDX}, 106 #define VERSION_OPT_IDX 'v' 107 {"version", no_argument, NULL, VERSION_OPT_IDX}, 108 #define PCI_BLOCKED_OPT_IDX 'B' 109 {"pci-blocked", required_argument, NULL, PCI_BLOCKED_OPT_IDX}, 110 #define LOGFLAG_OPT_IDX 'L' 111 {"logflag", required_argument, NULL, LOGFLAG_OPT_IDX}, 112 #define HUGE_UNLINK_OPT_IDX 'R' 113 {"huge-unlink", no_argument, NULL, HUGE_UNLINK_OPT_IDX}, 114 #define PCI_ALLOWED_OPT_IDX 'A' 115 {"pci-allowed", required_argument, NULL, PCI_ALLOWED_OPT_IDX}, 116 #define INTERRUPT_MODE_OPT_IDX 256 117 {"interrupt-mode", no_argument, NULL, INTERRUPT_MODE_OPT_IDX}, 118 #define SILENCE_NOTICELOG_OPT_IDX 257 119 {"silence-noticelog", no_argument, NULL, SILENCE_NOTICELOG_OPT_IDX}, 120 #define WAIT_FOR_RPC_OPT_IDX 258 121 {"wait-for-rpc", no_argument, NULL, WAIT_FOR_RPC_OPT_IDX}, 122 #define HUGE_DIR_OPT_IDX 259 123 {"huge-dir", required_argument, NULL, HUGE_DIR_OPT_IDX}, 124 #define NUM_TRACE_ENTRIES_OPT_IDX 260 125 {"num-trace-entries", required_argument, NULL, NUM_TRACE_ENTRIES_OPT_IDX}, 126 #define JSON_CONFIG_OPT_IDX 262 127 {"json", required_argument, NULL, JSON_CONFIG_OPT_IDX}, 128 #define JSON_CONFIG_IGNORE_INIT_ERRORS_IDX 263 129 {"json-ignore-init-errors", no_argument, NULL, JSON_CONFIG_IGNORE_INIT_ERRORS_IDX}, 130 #define IOVA_MODE_OPT_IDX 264 131 {"iova-mode", required_argument, NULL, IOVA_MODE_OPT_IDX}, 132 #define BASE_VIRTADDR_OPT_IDX 265 133 {"base-virtaddr", required_argument, NULL, BASE_VIRTADDR_OPT_IDX}, 134 #define ENV_CONTEXT_OPT_IDX 266 135 {"env-context", required_argument, NULL, ENV_CONTEXT_OPT_IDX}, 136 #define DISABLE_CPUMASK_LOCKS_OPT_IDX 267 137 {"disable-cpumask-locks", no_argument, NULL, DISABLE_CPUMASK_LOCKS_OPT_IDX}, 138 #define RPCS_ALLOWED_OPT_IDX 268 139 {"rpcs-allowed", required_argument, NULL, RPCS_ALLOWED_OPT_IDX}, 140 #define ENV_VF_TOKEN_OPT_IDX 269 141 {"vfio-vf-token", required_argument, NULL, ENV_VF_TOKEN_OPT_IDX}, 142 #define MSG_MEMPOOL_SIZE_OPT_IDX 270 143 {"msg-mempool-size", required_argument, NULL, MSG_MEMPOOL_SIZE_OPT_IDX}, 144 #define LCORES_OPT_IDX 271 145 {"lcores", required_argument, NULL, LCORES_OPT_IDX}, 146 #define NO_HUGE_OPT_IDX 272 147 {"no-huge", no_argument, NULL, NO_HUGE_OPT_IDX}, 148 #define NO_RPC_SERVER_OPT_IDX 273 149 {"no-rpc-server", no_argument, NULL, NO_RPC_SERVER_OPT_IDX}, 150 #define ENFORCE_NUMA_OPT_IDX 274 151 {"enforce-numa", no_argument, NULL, ENFORCE_NUMA_OPT_IDX}, 152 }; 153 154 static int 155 parse_proc_stat(unsigned int core, uint64_t *user, uint64_t *sys, uint64_t *irq) 156 { 157 FILE *f; 158 uint64_t i, soft_irq, cpu = 0; 159 int rc, found = 0; 160 161 f = fopen("/proc/stat", "r"); 162 if (!f) { 163 return -1; 164 } 165 166 for (i = 0; i <= core + 1; i++) { 167 /* scanf discards input with '*' in format, 168 * cpu;user;nice;system;idle;iowait;irq;softirq;steal;guest;guest_nice */ 169 rc = fscanf(f, "cpu%li %li %*i %li %*i %*i %li %li %*i %*i %*i\n", 170 &cpu, user, sys, irq, &soft_irq); 171 if (rc != 5) { 172 continue; 173 } 174 175 /* some cores can be disabled, list may not be in order */ 176 if (cpu == core) { 177 found = 1; 178 break; 179 } 180 } 181 182 *irq += soft_irq; 183 184 fclose(f); 185 return found ? 0 : -1; 186 } 187 188 static int 189 init_proc_stat(unsigned int core) 190 { 191 uint64_t usr, sys, irq; 192 193 if (core >= SPDK_CONFIG_MAX_LCORES) { 194 return -1; 195 } 196 197 if (parse_proc_stat(core, &usr, &sys, &irq) < 0) { 198 return -1; 199 } 200 201 g_initial_stat[core].irq = irq; 202 g_initial_stat[core].usr = usr; 203 g_initial_stat[core].sys = sys; 204 205 return 0; 206 } 207 208 int 209 app_get_proc_stat(unsigned int core, uint64_t *usr, uint64_t *sys, uint64_t *irq) 210 { 211 uint64_t _usr, _sys, _irq; 212 213 if (core >= SPDK_CONFIG_MAX_LCORES) { 214 return -1; 215 } 216 217 if (parse_proc_stat(core, &_usr, &_sys, &_irq) < 0) { 218 return -1; 219 } 220 221 *irq = _irq - g_initial_stat[core].irq; 222 *usr = _usr - g_initial_stat[core].usr; 223 *sys = _sys - g_initial_stat[core].sys; 224 225 return 0; 226 } 227 228 static void 229 app_start_shutdown(void *ctx) 230 { 231 if (g_spdk_app.shutdown_cb) { 232 g_spdk_app.shutdown_cb(); 233 g_spdk_app.shutdown_cb = NULL; 234 } else { 235 spdk_app_stop(0); 236 } 237 } 238 239 void 240 spdk_app_start_shutdown(void) 241 { 242 spdk_thread_send_critical_msg(spdk_thread_get_app_thread(), app_start_shutdown); 243 } 244 245 static void 246 __shutdown_signal(int signo) 247 { 248 if (!g_shutdown_sig_received) { 249 g_shutdown_sig_received = true; 250 spdk_app_start_shutdown(); 251 } 252 } 253 254 static int 255 app_opts_validate(const char *app_opts) 256 { 257 int i = 0, j; 258 259 for (i = 0; app_opts[i] != '\0'; i++) { 260 /* ignore getopt control characters */ 261 if (app_opts[i] == ':' || app_opts[i] == '+' || app_opts[i] == '-') { 262 continue; 263 } 264 265 for (j = 0; SPDK_APP_GETOPT_STRING[j] != '\0'; j++) { 266 if (app_opts[i] == SPDK_APP_GETOPT_STRING[j]) { 267 return app_opts[i]; 268 } 269 } 270 } 271 return 0; 272 } 273 274 static void 275 calculate_mempool_size(struct spdk_app_opts *opts, 276 struct spdk_app_opts *opts_user) 277 { 278 uint32_t core_count = spdk_env_get_core_count(); 279 280 if (!opts_user->msg_mempool_size) { 281 /* The user didn't specify msg_mempool_size, so let's calculate it. 282 Set the default (SPDK_DEFAULT_MSG_MEMPOOL_SIZE) if less than 283 64 cores, and use 4k per core otherwise */ 284 opts->msg_mempool_size = spdk_max(SPDK_DEFAULT_MSG_MEMPOOL_SIZE, 285 core_count * SPDK_APP_PER_CORE_MSG_MEMPOOL_SIZE); 286 } else { 287 opts->msg_mempool_size = opts_user->msg_mempool_size; 288 } 289 } 290 291 void 292 spdk_app_opts_init(struct spdk_app_opts *opts, size_t opts_size) 293 { 294 if (!opts) { 295 SPDK_ERRLOG("opts should not be NULL\n"); 296 return; 297 } 298 299 if (!opts_size) { 300 SPDK_ERRLOG("opts_size should not be zero value\n"); 301 return; 302 } 303 304 memset(opts, 0, opts_size); 305 opts->opts_size = opts_size; 306 307 #define SET_FIELD(field, value) \ 308 if (offsetof(struct spdk_app_opts, field) + sizeof(opts->field) <= opts_size) { \ 309 opts->field = value; \ 310 } \ 311 312 SET_FIELD(enable_coredump, true); 313 SET_FIELD(shm_id, -1); 314 SET_FIELD(mem_size, SPDK_APP_DPDK_DEFAULT_MEM_SIZE); 315 SET_FIELD(main_core, SPDK_APP_DPDK_DEFAULT_MAIN_CORE); 316 SET_FIELD(mem_channel, SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL); 317 SET_FIELD(base_virtaddr, SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR); 318 SET_FIELD(print_level, SPDK_APP_DEFAULT_LOG_PRINT_LEVEL); 319 SET_FIELD(rpc_addr, SPDK_DEFAULT_RPC_ADDR); 320 SET_FIELD(num_entries, SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES); 321 SET_FIELD(delay_subsystem_init, false); 322 SET_FIELD(disable_signal_handlers, false); 323 SET_FIELD(interrupt_mode, false); 324 SET_FIELD(enforce_numa, false); 325 /* Don't set msg_mempool_size here, it is set or calculated later */ 326 SET_FIELD(rpc_allowlist, NULL); 327 SET_FIELD(rpc_log_file, NULL); 328 SET_FIELD(rpc_log_level, SPDK_LOG_DISABLED); 329 SET_FIELD(disable_cpumask_locks, false); 330 #undef SET_FIELD 331 } 332 333 static int 334 app_setup_signal_handlers(struct spdk_app_opts *opts) 335 { 336 struct sigaction sigact; 337 sigset_t sigmask; 338 int rc; 339 340 sigemptyset(&sigmask); 341 memset(&sigact, 0, sizeof(sigact)); 342 sigemptyset(&sigact.sa_mask); 343 344 sigact.sa_handler = SIG_IGN; 345 rc = sigaction(SIGPIPE, &sigact, NULL); 346 if (rc < 0) { 347 SPDK_ERRLOG("sigaction(SIGPIPE) failed\n"); 348 return rc; 349 } 350 351 /* Install the same handler for SIGINT and SIGTERM */ 352 g_shutdown_sig_received = false; 353 sigact.sa_handler = __shutdown_signal; 354 rc = sigaction(SIGINT, &sigact, NULL); 355 if (rc < 0) { 356 SPDK_ERRLOG("sigaction(SIGINT) failed\n"); 357 return rc; 358 } 359 sigaddset(&sigmask, SIGINT); 360 361 rc = sigaction(SIGTERM, &sigact, NULL); 362 if (rc < 0) { 363 SPDK_ERRLOG("sigaction(SIGTERM) failed\n"); 364 return rc; 365 } 366 sigaddset(&sigmask, SIGTERM); 367 368 pthread_sigmask(SIG_UNBLOCK, &sigmask, NULL); 369 370 return 0; 371 } 372 373 static void 374 app_start_application(int rc, void *arg1) 375 { 376 assert(spdk_thread_is_app_thread(NULL)); 377 378 if (rc) { 379 SPDK_ERRLOG("Failed to load subsystems for RUNTIME state with code: %d\n", rc); 380 spdk_app_stop(rc); 381 return; 382 } 383 384 if (g_spdk_app.rpc_addr) { 385 spdk_rpc_server_resume(g_spdk_app.rpc_addr); 386 } 387 388 g_start_fn(g_start_arg); 389 } 390 391 static void 392 app_subsystem_init_done(int rc, void *arg1) 393 { 394 if (rc) { 395 SPDK_ERRLOG("Subsystem initialization failed with code: %d\n", rc); 396 spdk_app_stop(rc); 397 return; 398 } 399 400 spdk_rpc_set_allowlist(g_spdk_app.rpc_allowlist); 401 spdk_rpc_set_state(SPDK_RPC_RUNTIME); 402 403 if (g_spdk_app.json_data) { 404 /* Load SPDK_RPC_RUNTIME RPCs from config file */ 405 assert(spdk_rpc_get_state() == SPDK_RPC_RUNTIME); 406 spdk_subsystem_load_config(g_spdk_app.json_data, g_spdk_app.json_data_size, 407 app_start_application, NULL, 408 !g_spdk_app.json_config_ignore_errors); 409 free(g_spdk_app.json_data); 410 g_spdk_app.json_data = NULL; 411 } else { 412 app_start_application(0, NULL); 413 } 414 } 415 416 static void 417 app_do_spdk_subsystem_init(int rc, void *arg1) 418 { 419 struct spdk_rpc_opts opts; 420 421 if (rc) { 422 spdk_app_stop(rc); 423 return; 424 } 425 426 if (g_spdk_app.rpc_addr) { 427 opts.size = SPDK_SIZEOF(&opts, log_level); 428 opts.log_file = g_spdk_app.rpc_log_file; 429 opts.log_level = g_spdk_app.rpc_log_level; 430 431 rc = spdk_rpc_initialize(g_spdk_app.rpc_addr, &opts); 432 if (rc) { 433 spdk_app_stop(rc); 434 return; 435 } 436 if (g_delay_subsystem_init) { 437 return; 438 } 439 spdk_rpc_server_pause(g_spdk_app.rpc_addr); 440 } else { 441 SPDK_DEBUGLOG(app_rpc, "RPC server not started\n"); 442 } 443 spdk_subsystem_init(app_subsystem_init_done, NULL); 444 } 445 446 static int 447 app_opts_add_pci_addr(struct spdk_app_opts *opts, struct spdk_pci_addr **list, char *bdf) 448 { 449 struct spdk_pci_addr *tmp = *list; 450 size_t i = opts->num_pci_addr; 451 452 tmp = realloc(tmp, sizeof(*tmp) * (i + 1)); 453 if (tmp == NULL) { 454 SPDK_ERRLOG("realloc error\n"); 455 return -ENOMEM; 456 } 457 458 *list = tmp; 459 if (spdk_pci_addr_parse(*list + i, bdf) < 0) { 460 SPDK_ERRLOG("Invalid address %s\n", bdf); 461 return -EINVAL; 462 } 463 464 opts->num_pci_addr++; 465 return 0; 466 } 467 468 static int 469 app_setup_env(struct spdk_app_opts *opts) 470 { 471 struct spdk_env_opts env_opts = {}; 472 int rc; 473 474 if (opts == NULL) { 475 rc = spdk_env_init(NULL); 476 if (rc != 0) { 477 SPDK_ERRLOG("Unable to reinitialize SPDK env\n"); 478 } 479 480 return rc; 481 } 482 483 env_opts.opts_size = sizeof(env_opts); 484 spdk_env_opts_init(&env_opts); 485 486 env_opts.name = opts->name; 487 env_opts.core_mask = opts->reactor_mask; 488 env_opts.lcore_map = opts->lcore_map; 489 env_opts.shm_id = opts->shm_id; 490 env_opts.mem_channel = opts->mem_channel; 491 env_opts.main_core = opts->main_core; 492 env_opts.mem_size = opts->mem_size; 493 env_opts.hugepage_single_segments = opts->hugepage_single_segments; 494 env_opts.unlink_hugepage = opts->unlink_hugepage; 495 env_opts.hugedir = opts->hugedir; 496 env_opts.no_pci = opts->no_pci; 497 env_opts.num_pci_addr = opts->num_pci_addr; 498 env_opts.pci_blocked = opts->pci_blocked; 499 env_opts.pci_allowed = opts->pci_allowed; 500 env_opts.base_virtaddr = opts->base_virtaddr; 501 env_opts.env_context = opts->env_context; 502 env_opts.iova_mode = opts->iova_mode; 503 env_opts.vf_token = opts->vf_token; 504 env_opts.no_huge = opts->no_huge; 505 env_opts.enforce_numa = opts->enforce_numa; 506 507 rc = spdk_env_init(&env_opts); 508 free(env_opts.pci_blocked); 509 free(env_opts.pci_allowed); 510 511 if (rc < 0) { 512 SPDK_ERRLOG("Unable to initialize SPDK env\n"); 513 if (getuid() != 0) { 514 SPDK_ERRLOG("You may need to run as root\n"); 515 } 516 } 517 518 return rc; 519 } 520 521 static int 522 app_setup_trace(struct spdk_app_opts *opts) 523 { 524 char shm_name[64]; 525 uint64_t tpoint_group_mask, tpoint_mask = -1ULL; 526 char *end = NULL, *tpoint_group_mask_str, *tpoint_group_str = NULL; 527 char *tp_g_str, *tpoint_group, *tpoints; 528 bool error_found = false; 529 uint64_t group_id; 530 531 if (opts->shm_id >= 0) { 532 snprintf(shm_name, sizeof(shm_name), "/%s%s%d", opts->name, 533 SPDK_TRACE_SHM_NAME_BASE, opts->shm_id); 534 } else { 535 snprintf(shm_name, sizeof(shm_name), "/%s%spid%d", opts->name, 536 SPDK_TRACE_SHM_NAME_BASE, (int)getpid()); 537 } 538 539 if (spdk_trace_init(shm_name, opts->num_entries, 0) != 0) { 540 return -1; 541 } 542 543 if (opts->tpoint_group_mask == NULL) { 544 return 0; 545 } 546 547 tpoint_group_mask_str = strdup(opts->tpoint_group_mask); 548 if (tpoint_group_mask_str == NULL) { 549 SPDK_ERRLOG("Unable to get string of tpoint group mask from opts.\n"); 550 return -1; 551 } 552 /* Save a pointer to the original value of the tpoint group mask string 553 * to free later, because spdk_strsepq() modifies given char*. */ 554 tp_g_str = tpoint_group_mask_str; 555 while ((tpoint_group_str = spdk_strsepq(&tpoint_group_mask_str, ",")) != NULL) { 556 if (strchr(tpoint_group_str, ':')) { 557 /* Get the tpoint group mask */ 558 tpoint_group = spdk_strsepq(&tpoint_group_str, ":"); 559 /* Get the tpoint mask inside that group */ 560 tpoints = spdk_strsepq(&tpoint_group_str, ":"); 561 562 errno = 0; 563 tpoint_group_mask = strtoull(tpoint_group, &end, 16); 564 if (*end != '\0' || errno) { 565 tpoint_group_mask = spdk_trace_create_tpoint_group_mask(tpoint_group); 566 if (tpoint_group_mask == 0) { 567 error_found = true; 568 break; 569 } 570 } 571 /* Check if tpoint group mask has only one bit set. 572 * This is to avoid enabling individual tpoints in 573 * more than one tracepoint group at once. */ 574 if (!spdk_u64_is_pow2(tpoint_group_mask)) { 575 SPDK_ERRLOG("Tpoint group mask: %s contains multiple tpoint groups.\n", tpoint_group); 576 SPDK_ERRLOG("This is not supported, to prevent from activating tpoints by mistake.\n"); 577 error_found = true; 578 break; 579 } 580 581 errno = 0; 582 tpoint_mask = strtoull(tpoints, &end, 16); 583 if (*end != '\0' || errno) { 584 error_found = true; 585 break; 586 } 587 } else { 588 errno = 0; 589 tpoint_group_mask = strtoull(tpoint_group_str, &end, 16); 590 if (*end != '\0' || errno) { 591 tpoint_group_mask = spdk_trace_create_tpoint_group_mask(tpoint_group_str); 592 if (tpoint_group_mask == 0) { 593 error_found = true; 594 break; 595 } 596 } 597 tpoint_mask = -1ULL; 598 } 599 600 for (group_id = 0; group_id < SPDK_TRACE_MAX_GROUP_ID; ++group_id) { 601 if (tpoint_group_mask & (1 << group_id)) { 602 spdk_trace_set_tpoints(group_id, tpoint_mask); 603 } 604 } 605 } 606 607 if (error_found) { 608 SPDK_ERRLOG("invalid tpoint mask %s\n", opts->tpoint_group_mask); 609 free(tp_g_str); 610 return -1; 611 } else { 612 SPDK_NOTICELOG("Tracepoint Group Mask %s specified.\n", opts->tpoint_group_mask); 613 SPDK_NOTICELOG("Use 'spdk_trace -s %s %s %d' to capture a snapshot of events at runtime.\n", 614 opts->name, 615 opts->shm_id >= 0 ? "-i" : "-p", 616 opts->shm_id >= 0 ? opts->shm_id : getpid()); 617 #if defined(__linux__) 618 SPDK_NOTICELOG("'spdk_trace' without parameters will also work if this is the only\n"); 619 SPDK_NOTICELOG("SPDK application currently running.\n"); 620 SPDK_NOTICELOG("Or copy /dev/shm%s for offline analysis/debug.\n", shm_name); 621 #endif 622 } 623 free(tp_g_str); 624 625 return 0; 626 } 627 628 static void 629 bootstrap_fn(void *arg1) 630 { 631 spdk_rpc_set_allowlist(g_spdk_app.rpc_allowlist); 632 633 if (g_spdk_app.json_data) { 634 /* Load SPDK_RPC_STARTUP RPCs from config file */ 635 assert(spdk_rpc_get_state() == SPDK_RPC_STARTUP); 636 spdk_subsystem_load_config(g_spdk_app.json_data, g_spdk_app.json_data_size, 637 app_do_spdk_subsystem_init, NULL, 638 !g_spdk_app.json_config_ignore_errors); 639 } else { 640 app_do_spdk_subsystem_init(0, NULL); 641 } 642 } 643 644 static void 645 app_copy_opts(struct spdk_app_opts *opts, struct spdk_app_opts *opts_user, size_t opts_size) 646 { 647 spdk_app_opts_init(opts, sizeof(*opts)); 648 opts->opts_size = opts_size; 649 650 #define SET_FIELD(field) \ 651 if (offsetof(struct spdk_app_opts, field) + sizeof(opts->field) <= (opts->opts_size)) { \ 652 opts->field = opts_user->field; \ 653 } \ 654 655 SET_FIELD(name); 656 SET_FIELD(json_config_file); 657 SET_FIELD(json_config_ignore_errors); 658 SET_FIELD(rpc_addr); 659 SET_FIELD(reactor_mask); 660 SET_FIELD(lcore_map); 661 SET_FIELD(tpoint_group_mask); 662 SET_FIELD(shm_id); 663 SET_FIELD(shutdown_cb); 664 SET_FIELD(enable_coredump); 665 SET_FIELD(mem_channel); 666 SET_FIELD(main_core); 667 SET_FIELD(mem_size); 668 SET_FIELD(no_pci); 669 SET_FIELD(hugepage_single_segments); 670 SET_FIELD(unlink_hugepage); 671 SET_FIELD(no_huge); 672 SET_FIELD(hugedir); 673 SET_FIELD(print_level); 674 SET_FIELD(num_pci_addr); 675 SET_FIELD(pci_blocked); 676 SET_FIELD(pci_allowed); 677 SET_FIELD(iova_mode); 678 SET_FIELD(delay_subsystem_init); 679 SET_FIELD(num_entries); 680 SET_FIELD(env_context); 681 SET_FIELD(log); 682 SET_FIELD(base_virtaddr); 683 SET_FIELD(disable_signal_handlers); 684 SET_FIELD(interrupt_mode); 685 SET_FIELD(enforce_numa); 686 SET_FIELD(msg_mempool_size); 687 SET_FIELD(rpc_allowlist); 688 SET_FIELD(vf_token); 689 SET_FIELD(rpc_log_file); 690 SET_FIELD(rpc_log_level); 691 SET_FIELD(json_data); 692 SET_FIELD(json_data_size); 693 SET_FIELD(disable_cpumask_locks); 694 695 /* You should not remove this statement, but need to update the assert statement 696 * if you add a new field, and also add a corresponding SET_FIELD statement */ 697 SPDK_STATIC_ASSERT(sizeof(struct spdk_app_opts) == 253, "Incorrect size"); 698 699 #undef SET_FIELD 700 } 701 702 static int 703 unclaim_cpu_cores(uint32_t *failed_core) 704 { 705 char core_name[40]; 706 uint32_t i; 707 int rc; 708 709 for (i = 0; i < SPDK_CONFIG_MAX_LCORES; i++) { 710 if (g_core_locks[i] != -1 && g_core_locks[i] != 0) { 711 snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", i); 712 rc = close(g_core_locks[i]); 713 if (rc) { 714 SPDK_ERRLOG("Failed to close lock fd for core %d, errno: %d\n", i, errno); 715 goto error; 716 } 717 718 g_core_locks[i] = -1; 719 rc = unlink(core_name); 720 if (rc) { 721 SPDK_ERRLOG("Failed to unlink lock fd for core %d, errno: %d\n", i, errno); 722 goto error; 723 } 724 } 725 } 726 727 return 0; 728 729 error: 730 if (failed_core != NULL) { 731 /* Set number of core we failed to claim. */ 732 *failed_core = i; 733 } 734 return -1; 735 } 736 737 static int 738 claim_cpu_cores(uint32_t *failed_core) 739 { 740 char core_name[40]; 741 int core_fd, pid; 742 int *core_map; 743 uint32_t core; 744 745 struct flock core_lock = { 746 .l_type = F_WRLCK, 747 .l_whence = SEEK_SET, 748 .l_start = 0, 749 .l_len = 0, 750 }; 751 752 SPDK_ENV_FOREACH_CORE(core) { 753 if (g_core_locks[core] != -1) { 754 /* If this core is locked already, do not try lock it again. */ 755 continue; 756 } 757 758 snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", core); 759 core_fd = open(core_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 760 if (core_fd == -1) { 761 SPDK_ERRLOG("Could not open %s (%s).\n", core_name, spdk_strerror(errno)); 762 /* Return number of core we failed to claim. */ 763 goto error; 764 } 765 766 if (ftruncate(core_fd, sizeof(int)) != 0) { 767 SPDK_ERRLOG("Could not truncate %s (%s).\n", core_name, spdk_strerror(errno)); 768 close(core_fd); 769 goto error; 770 } 771 772 core_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE, MAP_SHARED, core_fd, 0); 773 if (core_map == MAP_FAILED) { 774 SPDK_ERRLOG("Could not mmap core %s (%s).\n", core_name, spdk_strerror(errno)); 775 close(core_fd); 776 goto error; 777 } 778 779 if (fcntl(core_fd, F_SETLK, &core_lock) != 0) { 780 pid = *core_map; 781 SPDK_ERRLOG("Cannot create lock on core %" PRIu32 ", probably process %d has claimed it.\n", 782 core, pid); 783 munmap(core_map, sizeof(int)); 784 close(core_fd); 785 goto error; 786 } 787 788 /* We write the PID to the core lock file so that other processes trying 789 * to claim the same core will know what process is holding the lock. */ 790 *core_map = (int)getpid(); 791 munmap(core_map, sizeof(int)); 792 g_core_locks[core] = core_fd; 793 /* Keep core_fd open to maintain the lock. */ 794 } 795 796 return 0; 797 798 error: 799 if (failed_core != NULL) { 800 /* Set number of core we failed to claim. */ 801 *failed_core = core; 802 } 803 unclaim_cpu_cores(NULL); 804 return -1; 805 } 806 807 int 808 spdk_app_start(struct spdk_app_opts *opts_user, spdk_msg_fn start_fn, 809 void *arg1) 810 { 811 int rc; 812 char *tty; 813 struct spdk_cpuset tmp_cpumask = {}; 814 static bool g_env_was_setup = false; 815 struct spdk_app_opts opts_local = {}; 816 struct spdk_app_opts *opts = &opts_local; 817 uint32_t i, core; 818 819 if (!opts_user) { 820 SPDK_ERRLOG("opts_user should not be NULL\n"); 821 return 1; 822 } 823 824 if (!opts_user->opts_size) { 825 SPDK_ERRLOG("The opts_size in opts_user structure should not be zero value\n"); 826 return 1; 827 } 828 829 if (opts_user->name == NULL) { 830 SPDK_ERRLOG("spdk_app_opts::name not specified\n"); 831 return 1; 832 } 833 834 app_copy_opts(opts, opts_user, opts_user->opts_size); 835 836 if (!start_fn) { 837 SPDK_ERRLOG("start_fn should not be NULL\n"); 838 return 1; 839 } 840 841 if (!opts->rpc_addr && opts->delay_subsystem_init) { 842 SPDK_ERRLOG("Cannot use '--wait-for-rpc' if no RPC server is going to be started.\n"); 843 return 1; 844 } 845 846 if (!(opts->lcore_map || opts->reactor_mask)) { 847 /* Set default CPU mask */ 848 opts->reactor_mask = SPDK_APP_DPDK_DEFAULT_CORE_MASK; 849 } 850 851 tty = ttyname(STDERR_FILENO); 852 if (opts->print_level > SPDK_LOG_WARN && 853 isatty(STDERR_FILENO) && 854 tty && 855 !strncmp(tty, "/dev/tty", strlen("/dev/tty"))) { 856 printf("Warning: printing stderr to console terminal without -q option specified.\n"); 857 printf("Suggest using --silence-noticelog to disable logging to stderr and\n"); 858 printf("monitor syslog, or redirect stderr to a file.\n"); 859 printf("(Delaying for 10 seconds...)\n"); 860 sleep(10); 861 } 862 863 spdk_log_set_print_level(opts->print_level); 864 865 #ifndef SPDK_NO_RLIMIT 866 if (opts->enable_coredump) { 867 struct rlimit core_limits; 868 869 core_limits.rlim_cur = core_limits.rlim_max = SPDK_APP_DEFAULT_CORE_LIMIT; 870 setrlimit(RLIMIT_CORE, &core_limits); 871 } 872 #endif 873 874 if (opts->interrupt_mode) { 875 spdk_interrupt_mode_enable(); 876 } 877 878 memset(&g_spdk_app, 0, sizeof(g_spdk_app)); 879 880 g_spdk_app.json_config_ignore_errors = opts->json_config_ignore_errors; 881 g_spdk_app.rpc_addr = opts->rpc_addr; 882 g_spdk_app.rpc_allowlist = opts->rpc_allowlist; 883 g_spdk_app.rpc_log_file = opts->rpc_log_file; 884 g_spdk_app.rpc_log_level = opts->rpc_log_level; 885 g_spdk_app.shm_id = opts->shm_id; 886 g_spdk_app.shutdown_cb = opts->shutdown_cb; 887 g_spdk_app.rc = 0; 888 g_spdk_app.stopped = false; 889 890 spdk_log_set_level(SPDK_APP_DEFAULT_LOG_LEVEL); 891 892 /* Pass NULL to app_setup_env if SPDK app has been set up, in order to 893 * indicate that this is a reinitialization. 894 */ 895 if (app_setup_env(g_env_was_setup ? NULL : opts) < 0) { 896 return 1; 897 } 898 899 /* Calculate mempool size now that the env layer has configured the core count 900 * for the application */ 901 calculate_mempool_size(opts, opts_user); 902 903 spdk_log_open(opts->log); 904 905 /* Initialize each lock to -1 to indicate "empty" status */ 906 for (i = 0; i < SPDK_CONFIG_MAX_LCORES; i++) { 907 g_core_locks[i] = -1; 908 } 909 910 if (!opts->disable_cpumask_locks) { 911 if (claim_cpu_cores(NULL)) { 912 SPDK_ERRLOG("Unable to acquire lock on assigned core mask - exiting.\n"); 913 return 1; 914 } 915 } else { 916 SPDK_NOTICELOG("CPU core locks deactivated.\n"); 917 } 918 919 SPDK_NOTICELOG("Total cores available: %d\n", spdk_env_get_core_count()); 920 921 if ((rc = spdk_reactors_init(opts->msg_mempool_size)) != 0) { 922 SPDK_ERRLOG("Reactor Initialization failed: rc = %d\n", rc); 923 return 1; 924 } 925 926 spdk_cpuset_set_cpu(&tmp_cpumask, spdk_env_get_current_core(), true); 927 928 /* 929 * Disable and ignore trace setup if setting num_entries 930 * to be 0. 931 * 932 * Note the call to app_setup_trace() is located here 933 * ahead of app_setup_signal_handlers(). 934 * That's because there is not an easy/direct clean 935 * way of unwinding alloc'd resources that can occur 936 * in app_setup_signal_handlers(). 937 */ 938 if (opts->num_entries != 0 && app_setup_trace(opts) != 0) { 939 return 1; 940 } 941 942 /* Now that the reactors have been initialized, we can create the app thread. */ 943 spdk_thread_create("app_thread", &tmp_cpumask); 944 if (!spdk_thread_get_app_thread()) { 945 SPDK_ERRLOG("Unable to create an spdk_thread for initialization\n"); 946 return 1; 947 } 948 949 SPDK_ENV_FOREACH_CORE(core) { 950 rc = init_proc_stat(core); 951 if (rc) { 952 SPDK_NOTICELOG("Unable to parse /proc/stat [core: %d].\n", core); 953 } 954 } 955 956 if (!opts->disable_signal_handlers && app_setup_signal_handlers(opts) != 0) { 957 return 1; 958 } 959 960 g_delay_subsystem_init = opts->delay_subsystem_init; 961 g_start_fn = start_fn; 962 g_start_arg = arg1; 963 964 if (opts->json_config_file != NULL) { 965 if (opts->json_data) { 966 SPDK_ERRLOG("App opts json_config_file and json_data are mutually exclusive\n"); 967 return 1; 968 } 969 970 g_spdk_app.json_data = spdk_posix_file_load_from_name(opts->json_config_file, 971 &g_spdk_app.json_data_size); 972 if (!g_spdk_app.json_data) { 973 SPDK_ERRLOG("Read JSON configuration file %s failed: %s\n", 974 opts->json_config_file, spdk_strerror(errno)); 975 return 1; 976 } 977 } else if (opts->json_data) { 978 g_spdk_app.json_data = calloc(1, opts->json_data_size); 979 if (!g_spdk_app.json_data) { 980 SPDK_ERRLOG("Failed to allocate JSON data buffer\n"); 981 return 1; 982 } 983 984 memcpy(g_spdk_app.json_data, opts->json_data, opts->json_data_size); 985 g_spdk_app.json_data_size = opts->json_data_size; 986 } 987 988 spdk_thread_send_msg(spdk_thread_get_app_thread(), bootstrap_fn, NULL); 989 990 /* This blocks until spdk_app_stop is called */ 991 spdk_reactors_start(); 992 993 g_env_was_setup = true; 994 995 return g_spdk_app.rc; 996 } 997 998 void 999 spdk_app_fini(void) 1000 { 1001 spdk_trace_cleanup(); 1002 spdk_reactors_fini(); 1003 spdk_env_fini(); 1004 spdk_log_close(); 1005 unclaim_cpu_cores(NULL); 1006 } 1007 1008 static void 1009 subsystem_fini_done(void *arg1) 1010 { 1011 spdk_rpc_finish(); 1012 spdk_reactors_stop(NULL); 1013 } 1014 1015 static void 1016 _start_subsystem_fini(void *arg1) 1017 { 1018 if (g_scheduling_in_progress) { 1019 spdk_thread_send_msg(spdk_thread_get_app_thread(), _start_subsystem_fini, NULL); 1020 return; 1021 } 1022 1023 spdk_subsystem_fini(subsystem_fini_done, NULL); 1024 } 1025 1026 static int 1027 log_deprecation_hits(void *ctx, struct spdk_deprecation *dep) 1028 { 1029 uint64_t hits = spdk_deprecation_get_hits(dep); 1030 1031 if (hits == 0) { 1032 return 0; 1033 } 1034 1035 SPDK_WARNLOG("%s: deprecation '%s' scheduled for removal in %s hit %" PRIu64 " times\n", 1036 spdk_deprecation_get_tag(dep), spdk_deprecation_get_description(dep), 1037 spdk_deprecation_get_remove_release(dep), hits); 1038 return 0; 1039 } 1040 1041 static void 1042 app_stop(void *arg1) 1043 { 1044 if (g_spdk_app.rc == 0) { 1045 g_spdk_app.rc = (int)(intptr_t)arg1; 1046 } 1047 1048 if (g_spdk_app.stopped) { 1049 SPDK_NOTICELOG("spdk_app_stop called twice\n"); 1050 return; 1051 } 1052 1053 free(g_spdk_app.json_data); 1054 1055 g_spdk_app.stopped = true; 1056 spdk_log_for_each_deprecation(NULL, log_deprecation_hits); 1057 _start_subsystem_fini(NULL); 1058 } 1059 1060 void 1061 spdk_app_stop(int rc) 1062 { 1063 if (rc) { 1064 SPDK_WARNLOG("spdk_app_stop'd on non-zero\n"); 1065 } 1066 1067 /* 1068 * We want to run spdk_subsystem_fini() from the same thread where spdk_subsystem_init() 1069 * was called. 1070 */ 1071 spdk_thread_send_msg(spdk_thread_get_app_thread(), app_stop, (void *)(intptr_t)rc); 1072 } 1073 1074 static void 1075 usage_memory_size(void) 1076 { 1077 #ifndef __linux__ 1078 if (g_default_opts.mem_size <= 0) { 1079 printf("all hugepage memory)\n"); 1080 } else 1081 #endif 1082 { 1083 printf("%dMB)\n", g_default_opts.mem_size >= 0 ? g_default_opts.mem_size : 0); 1084 } 1085 } 1086 1087 static void 1088 usage(void (*app_usage)(void)) 1089 { 1090 printf("%s [options]\n", g_executable_name); 1091 /* Keep entries inside categories roughly sorted by frequency of use. */ 1092 printf("\nCPU options:\n"); 1093 printf(" -m, --cpumask <mask or list> core mask (like 0xF) or core list of '[]' embraced for DPDK\n"); 1094 printf(" (like [0,1,10])\n"); 1095 printf(" --lcores <list> lcore to CPU mapping list. The list is in the format:\n"); 1096 printf(" <lcores[@CPUs]>[<,lcores[@CPUs]>...]\n"); 1097 printf(" lcores and cpus list are grouped by '(' and ')', e.g '--lcores \"(5-7)@(10-12)\"'\n"); 1098 printf(" Within the group, '-' is used for range separator,\n"); 1099 printf(" ',' is used for single number separator.\n"); 1100 printf(" '( )' can be omitted for single element group,\n"); 1101 printf(" '@' can be omitted if cpus and lcores have the same value\n"); 1102 printf(" --disable-cpumask-locks Disable CPU core lock files.\n"); 1103 printf(" --interrupt-mode set app to interrupt mode (Warning: CPU usage will be reduced only if all\n"); 1104 printf(" pollers in the app support interrupt mode)\n"); 1105 printf(" -p, --main-core <id> main (primary) core for DPDK\n"); 1106 1107 printf("\nConfiguration options:\n"); 1108 printf(" -c, --config, --json <config> JSON config file\n"); 1109 printf(" -r, --rpc-socket <path> RPC listen address (default %s)\n", SPDK_DEFAULT_RPC_ADDR); 1110 printf(" --no-rpc-server skip RPC server initialization. This option ignores '--rpc-socket' value.\n"); 1111 printf(" --wait-for-rpc wait for RPCs to initialize subsystems\n"); 1112 printf(" --rpcs-allowed comma-separated list of permitted RPCS\n"); 1113 printf(" --json-ignore-init-errors don't exit on invalid config entry\n"); 1114 1115 printf("\nMemory options:\n"); 1116 printf(" --iova-mode <pa/va> set IOVA mode ('pa' for IOVA_PA and 'va' for IOVA_VA)\n"); 1117 printf(" --base-virtaddr <addr> the base virtual address for DPDK (default: 0x200000000000)\n"); 1118 printf(" --huge-dir <path> use a specific hugetlbfs mount to reserve memory from\n"); 1119 printf(" -R, --huge-unlink unlink huge files after initialization\n"); 1120 printf(" -n, --mem-channels <num> number of memory channels used for DPDK\n"); 1121 printf(" -s, --mem-size <size> memory size in MB for DPDK (default: "); 1122 usage_memory_size(); 1123 printf(" --msg-mempool-size <size> global message memory pool size in count (default: %d)\n", 1124 SPDK_DEFAULT_MSG_MEMPOOL_SIZE); 1125 printf(" --no-huge run without using hugepages\n"); 1126 printf(" --enforce-numa enforce NUMA allocations from the specified NUMA node\n"); 1127 printf(" -i, --shm-id <id> shared memory ID (optional)\n"); 1128 printf(" -g, --single-file-segments force creating just one hugetlbfs file\n"); 1129 1130 printf("\nPCI options:\n"); 1131 printf(" -A, --pci-allowed <bdf> pci addr to allow (-B and -A cannot be used at the same time)\n"); 1132 printf(" -B, --pci-blocked <bdf> pci addr to block (can be used more than once)\n"); 1133 printf(" -u, --no-pci disable PCI access\n"); 1134 printf(" --vfio-vf-token VF token (UUID) shared between SR-IOV PF and VFs for vfio_pci driver\n"); 1135 1136 printf("\nLog options:\n"); 1137 spdk_log_usage(stdout, "-L"); 1138 printf(" --silence-noticelog disable notice level logging to stderr\n"); 1139 1140 printf("\nTrace options:\n"); 1141 printf(" --num-trace-entries <num> number of trace entries for each core, must be power of 2,\n"); 1142 printf(" setting 0 to disable trace (default %d)\n", 1143 SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES); 1144 printf(" Tracepoints vary in size and can use more than one trace entry.\n"); 1145 spdk_trace_mask_usage(stdout, "-e"); 1146 1147 printf("\nOther options:\n"); 1148 printf(" -h, --help show this usage\n"); 1149 printf(" -v, --version print SPDK version\n"); 1150 printf(" -d, --limit-coredump do not set max coredump size to RLIM_INFINITY\n"); 1151 printf(" --env-context Opaque context for use of the env implementation\n"); 1152 1153 if (app_usage) { 1154 printf("\nApplication specific:\n"); 1155 app_usage(); 1156 } 1157 } 1158 1159 spdk_app_parse_args_rvals_t 1160 spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts, 1161 const char *app_getopt_str, const struct option *app_long_opts, 1162 int (*app_parse)(int ch, char *arg), 1163 void (*app_usage)(void)) 1164 { 1165 int ch, rc, opt_idx, global_long_opts_len, app_long_opts_len; 1166 struct option *cmdline_options; 1167 char *cmdline_short_opts = NULL; 1168 char *shm_id_str = NULL; 1169 enum spdk_app_parse_args_rvals retval = SPDK_APP_PARSE_ARGS_FAIL; 1170 long int tmp; 1171 1172 memcpy(&g_default_opts, opts, sizeof(g_default_opts)); 1173 1174 if (opts->json_config_file && access(opts->json_config_file, R_OK) != 0) { 1175 SPDK_WARNLOG("Can't read JSON configuration file '%s'\n", opts->json_config_file); 1176 opts->json_config_file = NULL; 1177 } 1178 1179 if (app_long_opts == NULL) { 1180 app_long_opts_len = 0; 1181 } else { 1182 for (app_long_opts_len = 0; 1183 app_long_opts[app_long_opts_len].name != NULL; 1184 app_long_opts_len++); 1185 } 1186 1187 global_long_opts_len = SPDK_COUNTOF(g_cmdline_options); 1188 1189 cmdline_options = calloc(global_long_opts_len + app_long_opts_len + 1, sizeof(*cmdline_options)); 1190 if (!cmdline_options) { 1191 SPDK_ERRLOG("Out of memory\n"); 1192 return SPDK_APP_PARSE_ARGS_FAIL; 1193 } 1194 1195 memcpy(&cmdline_options[0], g_cmdline_options, sizeof(g_cmdline_options)); 1196 if (app_long_opts) { 1197 memcpy(&cmdline_options[global_long_opts_len], app_long_opts, 1198 app_long_opts_len * sizeof(*app_long_opts)); 1199 } 1200 1201 if (app_getopt_str != NULL) { 1202 ch = app_opts_validate(app_getopt_str); 1203 if (ch) { 1204 SPDK_ERRLOG("Duplicated option '%c' between app-specific command line parameter and generic spdk opts.\n", 1205 ch); 1206 goto out; 1207 } 1208 1209 if (!app_parse) { 1210 SPDK_ERRLOG("Parse function is required when app-specific command line parameters are provided.\n"); 1211 goto out; 1212 } 1213 } 1214 1215 cmdline_short_opts = spdk_sprintf_alloc("%s%s", app_getopt_str, SPDK_APP_GETOPT_STRING); 1216 if (!cmdline_short_opts) { 1217 SPDK_ERRLOG("Out of memory\n"); 1218 goto out; 1219 } 1220 1221 g_executable_name = argv[0]; 1222 1223 while ((ch = getopt_long(argc, argv, cmdline_short_opts, cmdline_options, &opt_idx)) != -1) { 1224 switch (ch) { 1225 case CONFIG_FILE_OPT_IDX: 1226 case JSON_CONFIG_OPT_IDX: 1227 opts->json_config_file = optarg; 1228 break; 1229 case JSON_CONFIG_IGNORE_INIT_ERRORS_IDX: 1230 opts->json_config_ignore_errors = true; 1231 break; 1232 case LIMIT_COREDUMP_OPT_IDX: 1233 opts->enable_coredump = false; 1234 break; 1235 case TPOINT_GROUP_OPT_IDX: 1236 opts->tpoint_group_mask = optarg; 1237 break; 1238 case SINGLE_FILE_SEGMENTS_OPT_IDX: 1239 opts->hugepage_single_segments = true; 1240 break; 1241 case HELP_OPT_IDX: 1242 usage(app_usage); 1243 retval = SPDK_APP_PARSE_ARGS_HELP; 1244 goto out; 1245 case SHM_ID_OPT_IDX: 1246 shm_id_str = optarg; 1247 /* a negative shm-id disables shared configuration file */ 1248 if (optarg[0] == '-') { 1249 shm_id_str++; 1250 } 1251 /* check if the positive value of provided shm_id can be parsed as 1252 * an integer 1253 */ 1254 opts->shm_id = spdk_strtol(shm_id_str, 0); 1255 if (opts->shm_id < 0) { 1256 SPDK_ERRLOG("Invalid shared memory ID %s\n", optarg); 1257 goto out; 1258 } 1259 if (optarg[0] == '-') { 1260 opts->shm_id = -opts->shm_id; 1261 } 1262 break; 1263 case CPUMASK_OPT_IDX: 1264 if (opts->lcore_map) { 1265 SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n"); 1266 goto out; 1267 } 1268 opts->reactor_mask = optarg; 1269 break; 1270 case LCORES_OPT_IDX: 1271 if (opts->reactor_mask) { 1272 SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n"); 1273 goto out; 1274 } 1275 opts->lcore_map = optarg; 1276 break; 1277 case DISABLE_CPUMASK_LOCKS_OPT_IDX: 1278 opts->disable_cpumask_locks = true; 1279 break; 1280 case MEM_CHANNELS_OPT_IDX: 1281 opts->mem_channel = spdk_strtol(optarg, 0); 1282 if (opts->mem_channel < 0) { 1283 SPDK_ERRLOG("Invalid memory channel %s\n", optarg); 1284 goto out; 1285 } 1286 break; 1287 case MAIN_CORE_OPT_IDX: 1288 opts->main_core = spdk_strtol(optarg, 0); 1289 if (opts->main_core < 0) { 1290 SPDK_ERRLOG("Invalid main core %s\n", optarg); 1291 goto out; 1292 } 1293 break; 1294 case SILENCE_NOTICELOG_OPT_IDX: 1295 opts->print_level = SPDK_LOG_WARN; 1296 break; 1297 case RPC_SOCKET_OPT_IDX: 1298 opts->rpc_addr = optarg; 1299 break; 1300 case NO_RPC_SERVER_OPT_IDX: 1301 opts->rpc_addr = NULL; 1302 break; 1303 case ENFORCE_NUMA_OPT_IDX: 1304 opts->enforce_numa = true; 1305 break; 1306 case MEM_SIZE_OPT_IDX: { 1307 uint64_t mem_size_mb; 1308 bool mem_size_has_prefix; 1309 1310 rc = spdk_parse_capacity(optarg, &mem_size_mb, &mem_size_has_prefix); 1311 if (rc != 0) { 1312 SPDK_ERRLOG("invalid memory pool size `-s %s`\n", optarg); 1313 usage(app_usage); 1314 goto out; 1315 } 1316 1317 if (mem_size_has_prefix) { 1318 /* the mem size is in MB by default, so if a prefix was 1319 * specified, we need to manually convert to MB. 1320 */ 1321 mem_size_mb /= 1024 * 1024; 1322 } 1323 1324 if (mem_size_mb > INT_MAX) { 1325 SPDK_ERRLOG("invalid memory pool size `-s %s`\n", optarg); 1326 usage(app_usage); 1327 goto out; 1328 } 1329 1330 opts->mem_size = (int) mem_size_mb; 1331 break; 1332 } 1333 case MSG_MEMPOOL_SIZE_OPT_IDX: 1334 tmp = spdk_strtol(optarg, 10); 1335 if (tmp <= 0) { 1336 SPDK_ERRLOG("Invalid message memory pool size %s\n", optarg); 1337 goto out; 1338 } 1339 1340 opts->msg_mempool_size = (size_t)tmp; 1341 break; 1342 1343 case NO_PCI_OPT_IDX: 1344 opts->no_pci = true; 1345 break; 1346 case WAIT_FOR_RPC_OPT_IDX: 1347 opts->delay_subsystem_init = true; 1348 break; 1349 case PCI_BLOCKED_OPT_IDX: 1350 if (opts->pci_allowed) { 1351 free(opts->pci_allowed); 1352 opts->pci_allowed = NULL; 1353 SPDK_ERRLOG("-B and -A cannot be used at the same time\n"); 1354 usage(app_usage); 1355 goto out; 1356 } 1357 1358 rc = app_opts_add_pci_addr(opts, &opts->pci_blocked, optarg); 1359 if (rc != 0) { 1360 free(opts->pci_blocked); 1361 opts->pci_blocked = NULL; 1362 goto out; 1363 } 1364 break; 1365 1366 case NO_HUGE_OPT_IDX: 1367 opts->no_huge = true; 1368 break; 1369 1370 case LOGFLAG_OPT_IDX: 1371 rc = spdk_log_set_flag(optarg); 1372 if (rc < 0) { 1373 SPDK_ERRLOG("unknown flag: %s\n", optarg); 1374 usage(app_usage); 1375 goto out; 1376 } 1377 #ifdef DEBUG 1378 opts->print_level = SPDK_LOG_DEBUG; 1379 #endif 1380 break; 1381 case HUGE_UNLINK_OPT_IDX: 1382 opts->unlink_hugepage = true; 1383 break; 1384 case PCI_ALLOWED_OPT_IDX: 1385 if (opts->pci_blocked) { 1386 free(opts->pci_blocked); 1387 opts->pci_blocked = NULL; 1388 SPDK_ERRLOG("-B and -W cannot be used at the same time\n"); 1389 usage(app_usage); 1390 goto out; 1391 } 1392 1393 rc = app_opts_add_pci_addr(opts, &opts->pci_allowed, optarg); 1394 if (rc != 0) { 1395 free(opts->pci_allowed); 1396 opts->pci_allowed = NULL; 1397 goto out; 1398 } 1399 break; 1400 case BASE_VIRTADDR_OPT_IDX: 1401 tmp = spdk_strtoll(optarg, 0); 1402 if (tmp <= 0) { 1403 SPDK_ERRLOG("Invalid base-virtaddr %s\n", optarg); 1404 usage(app_usage); 1405 goto out; 1406 } 1407 opts->base_virtaddr = (uint64_t)tmp; 1408 break; 1409 case HUGE_DIR_OPT_IDX: 1410 opts->hugedir = optarg; 1411 break; 1412 case IOVA_MODE_OPT_IDX: 1413 opts->iova_mode = optarg; 1414 break; 1415 case NUM_TRACE_ENTRIES_OPT_IDX: 1416 tmp = spdk_strtoll(optarg, 0); 1417 if (tmp < 0) { 1418 SPDK_ERRLOG("Invalid num-trace-entries %s\n", optarg); 1419 usage(app_usage); 1420 goto out; 1421 } 1422 opts->num_entries = (uint64_t)tmp; 1423 if (opts->num_entries > 0 && opts->num_entries & (opts->num_entries - 1)) { 1424 SPDK_ERRLOG("num-trace-entries must be power of 2\n"); 1425 usage(app_usage); 1426 goto out; 1427 } 1428 break; 1429 case ENV_CONTEXT_OPT_IDX: 1430 opts->env_context = optarg; 1431 break; 1432 case RPCS_ALLOWED_OPT_IDX: 1433 opts->rpc_allowlist = (const char **)spdk_strarray_from_string(optarg, ","); 1434 if (opts->rpc_allowlist == NULL) { 1435 SPDK_ERRLOG("Invalid --rpcs-allowed argument\n"); 1436 usage(app_usage); 1437 goto out; 1438 } 1439 break; 1440 case ENV_VF_TOKEN_OPT_IDX: 1441 opts->vf_token = optarg; 1442 break; 1443 case INTERRUPT_MODE_OPT_IDX: 1444 opts->interrupt_mode = true; 1445 break; 1446 case VERSION_OPT_IDX: 1447 printf(SPDK_VERSION_STRING"\n"); 1448 retval = SPDK_APP_PARSE_ARGS_HELP; 1449 goto out; 1450 case '?': 1451 /* 1452 * In the event getopt() above detects an option 1453 * in argv that is NOT in the getopt_str, 1454 * getopt() will return a '?' indicating failure. 1455 */ 1456 usage(app_usage); 1457 goto out; 1458 default: 1459 if (!app_parse) { 1460 SPDK_ERRLOG("Unsupported app-specific command line parameter '%c'.\n", ch); 1461 goto out; 1462 } 1463 1464 rc = app_parse(ch, optarg); 1465 if (rc) { 1466 SPDK_ERRLOG("Parsing app-specific command line parameter '%c' failed: %d\n", ch, rc); 1467 goto out; 1468 } 1469 } 1470 } 1471 1472 retval = SPDK_APP_PARSE_ARGS_SUCCESS; 1473 out: 1474 if (retval != SPDK_APP_PARSE_ARGS_SUCCESS) { 1475 free(opts->pci_blocked); 1476 opts->pci_blocked = NULL; 1477 free(opts->pci_allowed); 1478 opts->pci_allowed = NULL; 1479 spdk_strarray_free((char **)opts->rpc_allowlist); 1480 opts->rpc_allowlist = NULL; 1481 } 1482 free(cmdline_short_opts); 1483 free(cmdline_options); 1484 return retval; 1485 } 1486 1487 void 1488 spdk_app_usage(void) 1489 { 1490 if (g_executable_name == NULL) { 1491 SPDK_ERRLOG("%s not valid before calling spdk_app_parse_args()\n", __func__); 1492 return; 1493 } 1494 1495 usage(NULL); 1496 } 1497 1498 static void 1499 rpc_framework_start_init_cpl(int rc, void *arg1) 1500 { 1501 struct spdk_jsonrpc_request *request = arg1; 1502 1503 assert(spdk_thread_is_app_thread(NULL)); 1504 1505 if (rc) { 1506 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, 1507 "framework_initialization failed"); 1508 return; 1509 } 1510 1511 app_subsystem_init_done(0, NULL); 1512 1513 spdk_jsonrpc_send_bool_response(request, true); 1514 } 1515 1516 static void 1517 rpc_framework_start_init(struct spdk_jsonrpc_request *request, 1518 const struct spdk_json_val *params) 1519 { 1520 if (params != NULL) { 1521 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1522 "framework_start_init requires no parameters"); 1523 return; 1524 } 1525 1526 spdk_rpc_server_pause(g_spdk_app.rpc_addr); 1527 spdk_subsystem_init(rpc_framework_start_init_cpl, request); 1528 } 1529 SPDK_RPC_REGISTER("framework_start_init", rpc_framework_start_init, SPDK_RPC_STARTUP) 1530 1531 struct subsystem_init_poller_ctx { 1532 struct spdk_poller *init_poller; 1533 struct spdk_jsonrpc_request *request; 1534 }; 1535 1536 static int 1537 rpc_subsystem_init_poller_ctx(void *ctx) 1538 { 1539 struct subsystem_init_poller_ctx *poller_ctx = ctx; 1540 1541 if (spdk_rpc_get_state() == SPDK_RPC_RUNTIME) { 1542 spdk_jsonrpc_send_bool_response(poller_ctx->request, true); 1543 spdk_poller_unregister(&poller_ctx->init_poller); 1544 free(poller_ctx); 1545 } 1546 1547 return SPDK_POLLER_BUSY; 1548 } 1549 1550 static void 1551 rpc_framework_wait_init(struct spdk_jsonrpc_request *request, 1552 const struct spdk_json_val *params) 1553 { 1554 struct subsystem_init_poller_ctx *ctx; 1555 1556 if (spdk_rpc_get_state() == SPDK_RPC_RUNTIME) { 1557 spdk_jsonrpc_send_bool_response(request, true); 1558 } else { 1559 ctx = malloc(sizeof(struct subsystem_init_poller_ctx)); 1560 if (ctx == NULL) { 1561 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, 1562 "Unable to allocate memory for the request context\n"); 1563 return; 1564 } 1565 ctx->request = request; 1566 ctx->init_poller = SPDK_POLLER_REGISTER(rpc_subsystem_init_poller_ctx, ctx, 0); 1567 } 1568 } 1569 SPDK_RPC_REGISTER("framework_wait_init", rpc_framework_wait_init, 1570 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1571 1572 static void 1573 rpc_framework_disable_cpumask_locks(struct spdk_jsonrpc_request *request, 1574 const struct spdk_json_val *params) 1575 { 1576 char msg[128]; 1577 int rc; 1578 uint32_t failed_core; 1579 1580 if (params != NULL) { 1581 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1582 "framework_disable_cpumask_locks" 1583 "requires no arguments"); 1584 return; 1585 } 1586 1587 rc = unclaim_cpu_cores(&failed_core); 1588 if (rc) { 1589 snprintf(msg, sizeof(msg), "Failed to unclaim CPU core: %" PRIu32, failed_core); 1590 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg); 1591 return; 1592 } 1593 1594 spdk_jsonrpc_send_bool_response(request, true); 1595 } 1596 SPDK_RPC_REGISTER("framework_disable_cpumask_locks", rpc_framework_disable_cpumask_locks, 1597 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1598 1599 static void 1600 rpc_framework_enable_cpumask_locks(struct spdk_jsonrpc_request *request, 1601 const struct spdk_json_val *params) 1602 { 1603 char msg[128]; 1604 int rc; 1605 uint32_t failed_core; 1606 1607 if (params != NULL) { 1608 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, 1609 "framework_enable_cpumask_locks" 1610 "requires no arguments"); 1611 return; 1612 } 1613 1614 rc = claim_cpu_cores(&failed_core); 1615 if (rc) { 1616 snprintf(msg, sizeof(msg), "Failed to claim CPU core: %" PRIu32, failed_core); 1617 spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg); 1618 return; 1619 } 1620 1621 spdk_jsonrpc_send_bool_response(request, true); 1622 } 1623 SPDK_RPC_REGISTER("framework_enable_cpumask_locks", rpc_framework_enable_cpumask_locks, 1624 SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) 1625