1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/event.h" 35 36 #include <assert.h> 37 #include <stdio.h> 38 #include <stdlib.h> 39 #include <sys/mman.h> 40 #include <sys/resource.h> 41 #include <sys/stat.h> 42 #include <sys/types.h> 43 #include <unistd.h> 44 #include <errno.h> 45 #include <string.h> 46 #include <fcntl.h> 47 #include <signal.h> 48 49 #include <rte_config.h> 50 #include <rte_lcore.h> 51 52 #include "spdk/log.h" 53 #include "spdk/conf.h" 54 #include "spdk/trace.h" 55 56 #include "reactor.h" 57 #include "subsystem.h" 58 59 /* Add enough here to append ".pid" plus 2 digit instance ID */ 60 #define SPDK_APP_PIDFILE_MAX_LENGTH 40 61 #define SPDK_APP_PIDFILE_PREFIX "/var/run" 62 63 struct spdk_app { 64 struct spdk_conf *config; 65 char pidfile[SPDK_APP_PIDFILE_MAX_LENGTH]; 66 int instance_id; 67 spdk_app_shutdown_cb shutdown_cb; 68 int rc; 69 }; 70 71 static struct spdk_app g_spdk_app; 72 static spdk_event_t g_shutdown_event = NULL; 73 74 static int spdk_app_write_pidfile(void); 75 static void spdk_app_remove_pidfile(void); 76 77 int 78 spdk_app_get_instance_id(void) 79 { 80 return g_spdk_app.instance_id; 81 } 82 83 /* Global section */ 84 #define GLOBAL_CONFIG_TMPL \ 85 "# Configuration file\n" \ 86 "#\n" \ 87 "# Please write all parameters using ASCII.\n" \ 88 "# The parameter must be quoted if it includes whitespace.\n" \ 89 "#\n" \ 90 "# Configuration syntax:\n" \ 91 "# Spaces at head of line are deleted, other spaces are as separator\n" \ 92 "# Lines starting with '#' are comments and not evaluated.\n" \ 93 "# Lines ending with '\\' are concatenated with the next line.\n" \ 94 "# Bracketed keys are section keys grouping the following value keys.\n" \ 95 "# Number of section key is used as a tag number.\n" \ 96 "# Ex. [TargetNode1] = TargetNode section key with tag number 1\n" \ 97 "[Global]\n" \ 98 " Comment \"Global section\"\n" \ 99 "\n" \ 100 " # Users can restrict work items to only run on certain cores by\n" \ 101 " # specifying a ReactorMask. Default is to allow work items to run\n" \ 102 " # on all cores. Core 0 must be set in the mask if one is specified.\n" \ 103 " # Default: 0xFFFF (cores 0-15)\n" \ 104 " ReactorMask \"0x%" PRIX64 "\"\n" \ 105 "\n" \ 106 " # Tracepoint group mask for spdk trace buffers\n" \ 107 " # Default: 0x0 (all tracepoint groups disabled)\n" \ 108 " # Set to 0xFFFFFFFFFFFFFFFF to enable all tracepoint groups.\n" \ 109 " TpointGroupMask \"0x%" PRIX64 "\"\n" \ 110 "\n" \ 111 " # syslog facility\n" \ 112 " LogFacility \"%s\"\n" \ 113 "\n" 114 115 static void 116 spdk_app_config_dump_global_section(FILE *fp) 117 { 118 if (NULL == fp) 119 return; 120 121 /* FIXME - lookup log facility and put it in place of "local7" below */ 122 fprintf(fp, GLOBAL_CONFIG_TMPL, 123 spdk_app_get_core_mask(), spdk_trace_get_tpoint_group_mask(), 124 "local7"); 125 } 126 127 int 128 spdk_app_get_running_config(char **config_str, char *name) 129 { 130 FILE *fp = NULL; 131 int fd = -1; 132 long length = 0, ret = 0; 133 char vbuf[BUFSIZ]; 134 char config_template[64]; 135 136 snprintf(config_template, sizeof(config_template), "/tmp/%s.XXXXXX", name); 137 /* Create temporary file to hold config */ 138 fd = mkstemp(config_template); 139 if (fd == -1) { 140 fprintf(stderr, "mkstemp failed\n"); 141 return -1; 142 } 143 fp = fdopen(fd, "wb+"); 144 if (NULL == fp) { 145 fprintf(stderr, "error opening tmpfile fd = %d\n", fd); 146 return -1; 147 } 148 149 /* Buffered IO */ 150 setvbuf(fp, vbuf, _IOFBF, BUFSIZ); 151 152 spdk_app_config_dump_global_section(fp); 153 spdk_subsystem_config(fp); 154 155 length = ftell(fp); 156 157 *config_str = malloc(length + 1); 158 if (!*config_str) { 159 perror("config_str"); 160 fclose(fp); 161 return -1; 162 } 163 fseek(fp, 0, SEEK_SET); 164 ret = fread(*config_str, sizeof(char), length, fp); 165 if (ret < length) 166 fprintf(stderr, "%s: warning - short read\n", __func__); 167 fclose(fp); 168 (*config_str)[length] = '\0'; 169 170 return 0; 171 } 172 173 static const char * 174 spdk_get_log_facility(struct spdk_conf *config) 175 { 176 struct spdk_conf_section *sp; 177 const char *logfacility; 178 179 sp = spdk_conf_find_section(config, "Global"); 180 if (sp == NULL) { 181 return SPDK_APP_DEFAULT_LOG_FACILITY; 182 } 183 184 logfacility = spdk_conf_section_get_val(sp, "LogFacility"); 185 if (logfacility == NULL) { 186 return SPDK_APP_DEFAULT_LOG_FACILITY; 187 } 188 189 return logfacility; 190 } 191 192 void 193 spdk_app_start_shutdown(void) 194 { 195 if (g_shutdown_event != NULL) { 196 spdk_event_call(g_shutdown_event); 197 g_shutdown_event = NULL; 198 } 199 } 200 201 static void 202 __shutdown_signal(int signo) 203 { 204 spdk_app_start_shutdown(); 205 } 206 207 static void 208 __shutdown_event_cb(spdk_event_t event) 209 { 210 g_spdk_app.shutdown_cb(); 211 } 212 213 void 214 spdk_app_opts_init(struct spdk_app_opts *opts) 215 { 216 if (!opts) 217 return; 218 219 memset(opts, 0, sizeof(*opts)); 220 221 opts->enable_coredump = true; 222 opts->instance_id = -1; 223 opts->dpdk_mem_size = -1; 224 opts->dpdk_master_core = SPDK_APP_DPDK_DEFAULT_MASTER_CORE; 225 opts->dpdk_mem_channel = SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL; 226 opts->reactor_mask = NULL; 227 opts->max_delay_us = 0; 228 } 229 230 void 231 spdk_app_init(struct spdk_app_opts *opts) 232 { 233 struct spdk_conf *config; 234 struct spdk_conf_section *sp; 235 struct sigaction sigact; 236 sigset_t signew; 237 char shm_name[64]; 238 int rc; 239 uint64_t tpoint_group_mask; 240 char *end; 241 242 if (opts->enable_coredump) { 243 struct rlimit core_limits; 244 245 core_limits.rlim_cur = core_limits.rlim_max = RLIM_INFINITY; 246 setrlimit(RLIMIT_CORE, &core_limits); 247 } 248 249 config = spdk_conf_allocate(); 250 assert(config != NULL); 251 if (opts->config_file) { 252 rc = spdk_conf_read(config, opts->config_file); 253 if (rc != 0) { 254 fprintf(stderr, "Could not read config file %s\n", opts->config_file); 255 exit(EXIT_FAILURE); 256 } 257 if (spdk_conf_first_section(config) == NULL) { 258 fprintf(stderr, "Invalid config file %s\n", opts->config_file); 259 exit(EXIT_FAILURE); 260 } 261 } 262 spdk_conf_set_as_default(config); 263 264 if (opts->instance_id == -1) { 265 sp = spdk_conf_find_section(config, "Global"); 266 if (sp != NULL) { 267 opts->instance_id = spdk_conf_section_get_intval(sp, "InstanceID"); 268 } 269 } 270 271 if (opts->instance_id < 0) { 272 opts->instance_id = 0; 273 } 274 275 memset(&g_spdk_app, 0, sizeof(g_spdk_app)); 276 g_spdk_app.config = config; 277 g_spdk_app.instance_id = opts->instance_id; 278 g_spdk_app.shutdown_cb = opts->shutdown_cb; 279 snprintf(g_spdk_app.pidfile, sizeof(g_spdk_app.pidfile), "%s/%s.pid.%d", 280 SPDK_APP_PIDFILE_PREFIX, opts->name, opts->instance_id); 281 spdk_app_write_pidfile(); 282 283 /* open log files */ 284 if (opts->log_facility == NULL) { 285 opts->log_facility = spdk_get_log_facility(g_spdk_app.config); 286 if (opts->log_facility == NULL) { 287 fprintf(stderr, "NULL logfacility\n"); 288 spdk_conf_free(g_spdk_app.config); 289 exit(EXIT_FAILURE); 290 } 291 } 292 rc = spdk_set_log_facility(opts->log_facility); 293 if (rc < 0) { 294 fprintf(stderr, "log facility error\n"); 295 spdk_conf_free(g_spdk_app.config); 296 exit(EXIT_FAILURE); 297 } 298 299 rc = spdk_set_log_priority(SPDK_APP_DEFAULT_LOG_PRIORITY); 300 if (rc < 0) { 301 fprintf(stderr, "log priority error\n"); 302 spdk_conf_free(g_spdk_app.config); 303 exit(EXIT_FAILURE); 304 } 305 spdk_open_log(); 306 307 if (opts->reactor_mask == NULL) { 308 sp = spdk_conf_find_section(g_spdk_app.config, "Global"); 309 if (sp != NULL) { 310 if (spdk_conf_section_get_val(sp, "ReactorMask")) { 311 opts->reactor_mask = spdk_conf_section_get_val(sp, "ReactorMask"); 312 } else { 313 opts->reactor_mask = SPDK_APP_DPDK_DEFAULT_CORE_MASK; 314 } 315 } else { 316 opts->reactor_mask = SPDK_APP_DPDK_DEFAULT_CORE_MASK; 317 } 318 } 319 320 spdk_dpdk_framework_init(opts); 321 322 /* 323 * If mask not specified on command line or in configuration file, 324 * reactor_mask will be 0x1 which will enable core 0 to run one 325 * reactor. 326 */ 327 if (spdk_reactors_init(opts->reactor_mask, opts->max_delay_us)) { 328 fprintf(stderr, "Invalid reactor mask.\n"); 329 exit(EXIT_FAILURE); 330 } 331 332 /* setup signal handler thread */ 333 pthread_sigmask(SIG_SETMASK, NULL, &signew); 334 335 memset(&sigact, 0, sizeof(sigact)); 336 sigact.sa_handler = SIG_IGN; 337 sigemptyset(&sigact.sa_mask); 338 rc = sigaction(SIGPIPE, &sigact, NULL); 339 if (rc < 0) { 340 SPDK_ERRLOG("sigaction(SIGPIPE) failed\n"); 341 exit(EXIT_FAILURE); 342 } 343 344 if (opts->shutdown_cb != NULL) { 345 g_shutdown_event = spdk_event_allocate(rte_lcore_id(), __shutdown_event_cb, 346 NULL, NULL, NULL); 347 348 sigact.sa_handler = __shutdown_signal; 349 sigemptyset(&sigact.sa_mask); 350 rc = sigaction(SIGINT, &sigact, NULL); 351 if (rc < 0) { 352 SPDK_ERRLOG("sigaction(SIGINT) failed\n"); 353 exit(EXIT_FAILURE); 354 } 355 sigaddset(&signew, SIGINT); 356 357 sigact.sa_handler = __shutdown_signal; 358 sigemptyset(&sigact.sa_mask); 359 rc = sigaction(SIGTERM, &sigact, NULL); 360 if (rc < 0) { 361 SPDK_ERRLOG("sigaction(SIGTERM) failed\n"); 362 exit(EXIT_FAILURE); 363 } 364 sigaddset(&signew, SIGTERM); 365 } 366 367 if (opts->usr1_handler != NULL) { 368 sigact.sa_handler = opts->usr1_handler; 369 sigemptyset(&sigact.sa_mask); 370 rc = sigaction(SIGUSR1, &sigact, NULL); 371 if (rc < 0) { 372 SPDK_ERRLOG("sigaction(SIGUSR1) failed\n"); 373 exit(EXIT_FAILURE); 374 } 375 sigaddset(&signew, SIGUSR1); 376 } 377 378 sigaddset(&signew, SIGQUIT); 379 sigaddset(&signew, SIGHUP); 380 pthread_sigmask(SIG_SETMASK, &signew, NULL); 381 382 snprintf(shm_name, sizeof(shm_name), "/%s_trace.%d", opts->name, opts->instance_id); 383 spdk_trace_init(shm_name); 384 385 if (opts->tpoint_group_mask == NULL) { 386 sp = spdk_conf_find_section(g_spdk_app.config, "Global"); 387 if (sp != NULL) { 388 opts->tpoint_group_mask = spdk_conf_section_get_val(sp, "TpointGroupMask"); 389 } 390 } 391 392 if (opts->tpoint_group_mask != NULL) { 393 errno = 0; 394 tpoint_group_mask = strtoull(opts->tpoint_group_mask, &end, 16); 395 if (*end != '\0' || errno) { 396 SPDK_ERRLOG("invalid tpoint mask %s\n", opts->tpoint_group_mask); 397 } else { 398 spdk_trace_set_tpoint_group_mask(tpoint_group_mask); 399 } 400 } 401 402 rc = spdk_subsystem_init(); 403 if (rc < 0) { 404 SPDK_ERRLOG("spdk_subsystem_init() failed\n"); 405 exit(EXIT_FAILURE); 406 } 407 } 408 409 int 410 spdk_app_fini(void) 411 { 412 int rc; 413 414 rc = spdk_subsystem_fini(); 415 spdk_trace_cleanup(); 416 spdk_app_remove_pidfile(); 417 spdk_conf_free(g_spdk_app.config); 418 spdk_close_log(); 419 420 return rc; 421 } 422 423 int 424 spdk_app_start(spdk_event_fn start_fn, void *arg1, void *arg2) 425 { 426 spdk_event_t event; 427 428 g_spdk_app.rc = 0; 429 430 event = spdk_event_allocate(rte_get_master_lcore(), start_fn, 431 arg1, arg2, NULL); 432 /* Queues up the event, but can't run it until the reactors start */ 433 spdk_event_call(event); 434 435 /* This blocks until spdk_app_stop is called */ 436 spdk_reactors_start(); 437 438 return g_spdk_app.rc; 439 } 440 441 void 442 spdk_app_stop(int rc) 443 { 444 spdk_reactors_stop(); 445 g_spdk_app.rc = rc; 446 } 447 448 static int 449 spdk_app_write_pidfile(void) 450 { 451 FILE *fp; 452 pid_t pid; 453 struct flock lock = { 454 .l_type = F_WRLCK, 455 .l_whence = SEEK_SET, 456 .l_start = 0, 457 .l_len = 0, 458 }; 459 460 fp = fopen(g_spdk_app.pidfile, "w"); 461 if (fp == NULL) { 462 SPDK_ERRLOG("pidfile open error %d\n", errno); 463 return -1; 464 } 465 466 if (fcntl(fileno(fp), F_SETLK, &lock) != 0) { 467 fprintf(stderr, "Cannot create lock on file %s, probably you" 468 " should use different instance id\n", g_spdk_app.pidfile); 469 exit(EXIT_FAILURE); 470 } 471 472 pid = getpid(); 473 fprintf(fp, "%d\n", (int)pid); 474 fclose(fp); 475 return 0; 476 } 477 478 static void 479 spdk_app_remove_pidfile(void) 480 { 481 int rc; 482 483 rc = remove(g_spdk_app.pidfile); 484 if (rc != 0) { 485 SPDK_ERRLOG("pidfile remove error %d\n", errno); 486 /* ignore error */ 487 } 488 } 489