1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "event_nvmf.h" 35 36 #include "spdk/bdev.h" 37 #include "spdk/event.h" 38 #include "spdk/thread.h" 39 #include "spdk/log.h" 40 #include "spdk/nvme.h" 41 #include "spdk/nvmf_cmd.h" 42 #include "spdk/util.h" 43 44 enum nvmf_tgt_state { 45 NVMF_TGT_INIT_NONE = 0, 46 NVMF_TGT_INIT_CREATE_TARGET, 47 NVMF_TGT_INIT_CREATE_POLL_GROUPS, 48 NVMF_TGT_INIT_START_SUBSYSTEMS, 49 NVMF_TGT_RUNNING, 50 NVMF_TGT_FINI_STOP_SUBSYSTEMS, 51 NVMF_TGT_FINI_DESTROY_POLL_GROUPS, 52 NVMF_TGT_FINI_FREE_RESOURCES, 53 NVMF_TGT_STOPPED, 54 NVMF_TGT_ERROR, 55 }; 56 57 struct nvmf_tgt_poll_group { 58 struct spdk_nvmf_poll_group *group; 59 struct spdk_thread *thread; 60 TAILQ_ENTRY(nvmf_tgt_poll_group) link; 61 }; 62 63 struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf = { 64 .acceptor_poll_rate = ACCEPT_TIMEOUT_US, 65 .admin_passthru.identify_ctrlr = false 66 }; 67 68 struct spdk_nvmf_tgt *g_spdk_nvmf_tgt = NULL; 69 uint32_t g_spdk_nvmf_tgt_max_subsystems = 0; 70 71 static enum nvmf_tgt_state g_tgt_state; 72 73 static struct spdk_thread *g_tgt_init_thread = NULL; 74 static struct spdk_thread *g_tgt_fini_thread = NULL; 75 76 static TAILQ_HEAD(, nvmf_tgt_poll_group) g_poll_groups = TAILQ_HEAD_INITIALIZER(g_poll_groups); 77 static size_t g_num_poll_groups = 0; 78 79 static void nvmf_tgt_advance_state(void); 80 81 static void 82 nvmf_shutdown_cb(void *arg1) 83 { 84 /* Still in initialization state, defer shutdown operation */ 85 if (g_tgt_state < NVMF_TGT_RUNNING) { 86 spdk_thread_send_msg(spdk_get_thread(), nvmf_shutdown_cb, NULL); 87 return; 88 } else if (g_tgt_state != NVMF_TGT_RUNNING && g_tgt_state != NVMF_TGT_ERROR) { 89 /* Already in Shutdown status, ignore the signal */ 90 return; 91 } 92 93 if (g_tgt_state == NVMF_TGT_ERROR) { 94 /* Parse configuration error */ 95 g_tgt_state = NVMF_TGT_FINI_FREE_RESOURCES; 96 } else { 97 g_tgt_state = NVMF_TGT_FINI_STOP_SUBSYSTEMS; 98 } 99 nvmf_tgt_advance_state(); 100 } 101 102 static void 103 nvmf_subsystem_fini(void) 104 { 105 nvmf_shutdown_cb(NULL); 106 } 107 108 static void 109 _nvmf_tgt_destroy_poll_group_done(void *ctx) 110 { 111 assert(g_num_poll_groups > 0); 112 113 if (--g_num_poll_groups == 0) { 114 g_tgt_state = NVMF_TGT_FINI_FREE_RESOURCES; 115 nvmf_tgt_advance_state(); 116 } 117 } 118 119 static void 120 nvmf_tgt_destroy_poll_group_done(void *cb_arg, int status) 121 { 122 struct nvmf_tgt_poll_group *pg = cb_arg; 123 124 free(pg); 125 126 spdk_thread_send_msg(g_tgt_fini_thread, _nvmf_tgt_destroy_poll_group_done, NULL); 127 128 spdk_thread_exit(spdk_get_thread()); 129 } 130 131 static void 132 nvmf_tgt_destroy_poll_group(void *ctx) 133 { 134 struct nvmf_tgt_poll_group *pg = ctx; 135 136 spdk_nvmf_poll_group_destroy(pg->group, nvmf_tgt_destroy_poll_group_done, pg); 137 } 138 139 static void 140 nvmf_tgt_destroy_poll_groups(void) 141 { 142 struct nvmf_tgt_poll_group *pg, *tpg; 143 144 g_tgt_fini_thread = spdk_get_thread(); 145 assert(g_tgt_fini_thread != NULL); 146 147 TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tpg) { 148 TAILQ_REMOVE(&g_poll_groups, pg, link); 149 spdk_thread_send_msg(pg->thread, nvmf_tgt_destroy_poll_group, pg); 150 } 151 } 152 153 static void 154 nvmf_tgt_create_poll_group_done(void *ctx) 155 { 156 struct nvmf_tgt_poll_group *pg = ctx; 157 158 TAILQ_INSERT_TAIL(&g_poll_groups, pg, link); 159 160 assert(g_num_poll_groups < spdk_env_get_core_count()); 161 162 if (++g_num_poll_groups == spdk_env_get_core_count()) { 163 g_tgt_state = NVMF_TGT_INIT_START_SUBSYSTEMS; 164 nvmf_tgt_advance_state(); 165 } 166 } 167 168 static void 169 nvmf_tgt_create_poll_group(void *ctx) 170 { 171 struct nvmf_tgt_poll_group *pg; 172 173 pg = calloc(1, sizeof(*pg)); 174 if (!pg) { 175 SPDK_ERRLOG("Not enough memory to allocate poll groups\n"); 176 spdk_app_stop(-ENOMEM); 177 return; 178 } 179 180 pg->thread = spdk_get_thread(); 181 pg->group = spdk_nvmf_poll_group_create(g_spdk_nvmf_tgt); 182 183 spdk_thread_send_msg(g_tgt_init_thread, nvmf_tgt_create_poll_group_done, pg); 184 } 185 186 static void 187 nvmf_tgt_create_poll_groups(void) 188 { 189 struct spdk_cpuset tmp_cpumask = {}; 190 uint32_t i; 191 char thread_name[32]; 192 struct spdk_thread *thread; 193 194 g_tgt_init_thread = spdk_get_thread(); 195 assert(g_tgt_init_thread != NULL); 196 197 SPDK_ENV_FOREACH_CORE(i) { 198 spdk_cpuset_zero(&tmp_cpumask); 199 spdk_cpuset_set_cpu(&tmp_cpumask, i, true); 200 snprintf(thread_name, sizeof(thread_name), "nvmf_tgt_poll_group_%u", i); 201 202 thread = spdk_thread_create(thread_name, &tmp_cpumask); 203 assert(thread != NULL); 204 205 spdk_thread_send_msg(thread, nvmf_tgt_create_poll_group, NULL); 206 } 207 } 208 209 static void 210 nvmf_tgt_subsystem_started(struct spdk_nvmf_subsystem *subsystem, 211 void *cb_arg, int status) 212 { 213 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 214 int rc; 215 216 if (subsystem) { 217 rc = spdk_nvmf_subsystem_start(subsystem, nvmf_tgt_subsystem_started, NULL); 218 if (rc) { 219 g_tgt_state = NVMF_TGT_FINI_STOP_SUBSYSTEMS; 220 SPDK_ERRLOG("Unable to start NVMe-oF subsystem. Stopping app.\n"); 221 nvmf_tgt_advance_state(); 222 } 223 return; 224 } 225 226 g_tgt_state = NVMF_TGT_RUNNING; 227 nvmf_tgt_advance_state(); 228 } 229 230 static void 231 nvmf_tgt_subsystem_stopped(struct spdk_nvmf_subsystem *subsystem, 232 void *cb_arg, int status) 233 { 234 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 235 int rc; 236 237 if (subsystem) { 238 rc = spdk_nvmf_subsystem_stop(subsystem, nvmf_tgt_subsystem_stopped, NULL); 239 if (rc) { 240 SPDK_ERRLOG("Unable to stop NVMe-oF subsystem. Trying others.\n"); 241 nvmf_tgt_subsystem_stopped(subsystem, NULL, 0); 242 } 243 return; 244 } 245 246 g_tgt_state = NVMF_TGT_FINI_DESTROY_POLL_GROUPS; 247 nvmf_tgt_advance_state(); 248 } 249 250 static void 251 nvmf_tgt_destroy_done(void *ctx, int status) 252 { 253 g_tgt_state = NVMF_TGT_STOPPED; 254 255 nvmf_tgt_advance_state(); 256 } 257 258 static int 259 nvmf_add_discovery_subsystem(void) 260 { 261 struct spdk_nvmf_subsystem *subsystem; 262 263 subsystem = spdk_nvmf_subsystem_create(g_spdk_nvmf_tgt, SPDK_NVMF_DISCOVERY_NQN, 264 SPDK_NVMF_SUBTYPE_DISCOVERY, 0); 265 if (subsystem == NULL) { 266 SPDK_ERRLOG("Failed creating discovery nvmf library subsystem\n"); 267 return -1; 268 } 269 270 spdk_nvmf_subsystem_set_allow_any_host(subsystem, true); 271 272 return 0; 273 } 274 275 static int 276 nvmf_tgt_create_target(void) 277 { 278 struct spdk_nvmf_target_opts opts = { 279 .name = "nvmf_tgt" 280 }; 281 282 opts.max_subsystems = g_spdk_nvmf_tgt_max_subsystems; 283 opts.acceptor_poll_rate = g_spdk_nvmf_tgt_conf.acceptor_poll_rate; 284 g_spdk_nvmf_tgt = spdk_nvmf_tgt_create(&opts); 285 if (!g_spdk_nvmf_tgt) { 286 SPDK_ERRLOG("spdk_nvmf_tgt_create() failed\n"); 287 return -1; 288 } 289 290 if (nvmf_add_discovery_subsystem() != 0) { 291 SPDK_ERRLOG("nvmf_add_discovery_subsystem failed\n"); 292 return -1; 293 } 294 295 return 0; 296 } 297 298 static void 299 fixup_identify_ctrlr(struct spdk_nvmf_request *req) 300 { 301 uint32_t length; 302 int rc; 303 struct spdk_nvme_ctrlr_data *nvme_cdata; 304 struct spdk_nvme_ctrlr_data nvmf_cdata = {}; 305 struct spdk_nvmf_ctrlr *ctrlr = spdk_nvmf_request_get_ctrlr(req); 306 struct spdk_nvme_cpl *rsp = spdk_nvmf_request_get_response(req); 307 308 /* This is the identify data from the NVMe drive */ 309 spdk_nvmf_request_get_data(req, (void **)&nvme_cdata, &length); 310 311 /* Get the NVMF identify data */ 312 rc = spdk_nvmf_ctrlr_identify_ctrlr(ctrlr, &nvmf_cdata); 313 if (rc != SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 314 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 315 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 316 return; 317 } 318 319 /* Fixup NVMF identify data with NVMe identify data */ 320 321 /* Serial Number (SN) */ 322 memcpy(&nvmf_cdata.sn[0], &nvme_cdata->sn[0], sizeof(nvmf_cdata.sn)); 323 /* Model Number (MN) */ 324 memcpy(&nvmf_cdata.mn[0], &nvme_cdata->mn[0], sizeof(nvmf_cdata.mn)); 325 /* Firmware Revision (FR) */ 326 memcpy(&nvmf_cdata.fr[0], &nvme_cdata->fr[0], sizeof(nvmf_cdata.fr)); 327 /* IEEE OUI Identifier (IEEE) */ 328 memcpy(&nvmf_cdata.ieee[0], &nvme_cdata->ieee[0], sizeof(nvmf_cdata.ieee)); 329 /* FRU Globally Unique Identifier (FGUID) */ 330 331 /* Copy the fixed up data back to the response */ 332 memcpy(nvme_cdata, &nvmf_cdata, length); 333 } 334 335 static int 336 nvmf_custom_identify_hdlr(struct spdk_nvmf_request *req) 337 { 338 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req); 339 struct spdk_bdev *bdev; 340 struct spdk_bdev_desc *desc; 341 struct spdk_io_channel *ch; 342 struct spdk_nvmf_subsystem *subsys; 343 int rc; 344 345 if (cmd->cdw10_bits.identify.cns != SPDK_NVME_IDENTIFY_CTRLR) { 346 return -1; /* continue */ 347 } 348 349 subsys = spdk_nvmf_request_get_subsystem(req); 350 if (subsys == NULL) { 351 return -1; 352 } 353 354 /* Only procss this request if it has exactly one namespace */ 355 if (spdk_nvmf_subsystem_get_max_nsid(subsys) != 1) { 356 return -1; 357 } 358 359 /* Forward to first namespace if it supports NVME admin commands */ 360 rc = spdk_nvmf_request_get_bdev(1, req, &bdev, &desc, &ch); 361 if (rc) { 362 /* No bdev found for this namespace. Continue. */ 363 return -1; 364 } 365 366 if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_NVME_ADMIN)) { 367 return -1; 368 } 369 370 return spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(bdev, desc, ch, req, fixup_identify_ctrlr); 371 } 372 373 static void 374 nvmf_tgt_advance_state(void) 375 { 376 enum nvmf_tgt_state prev_state; 377 int rc = -1; 378 int ret; 379 380 do { 381 prev_state = g_tgt_state; 382 383 switch (g_tgt_state) { 384 case NVMF_TGT_INIT_NONE: { 385 g_tgt_state = NVMF_TGT_INIT_CREATE_TARGET; 386 break; 387 } 388 case NVMF_TGT_INIT_CREATE_TARGET: 389 ret = nvmf_tgt_create_target(); 390 g_tgt_state = (ret == 0) ? NVMF_TGT_INIT_CREATE_POLL_GROUPS : NVMF_TGT_ERROR; 391 break; 392 case NVMF_TGT_INIT_CREATE_POLL_GROUPS: 393 if (g_spdk_nvmf_tgt_conf.admin_passthru.identify_ctrlr) { 394 SPDK_NOTICELOG("Custom identify ctrlr handler enabled\n"); 395 spdk_nvmf_set_custom_admin_cmd_hdlr(SPDK_NVME_OPC_IDENTIFY, nvmf_custom_identify_hdlr); 396 } 397 /* Create poll group threads, and send a message to each thread 398 * and create a poll group. 399 */ 400 nvmf_tgt_create_poll_groups(); 401 break; 402 case NVMF_TGT_INIT_START_SUBSYSTEMS: { 403 struct spdk_nvmf_subsystem *subsystem; 404 405 subsystem = spdk_nvmf_subsystem_get_first(g_spdk_nvmf_tgt); 406 407 if (subsystem) { 408 ret = spdk_nvmf_subsystem_start(subsystem, nvmf_tgt_subsystem_started, NULL); 409 if (ret) { 410 SPDK_ERRLOG("Unable to start NVMe-oF subsystem. Stopping app.\n"); 411 g_tgt_state = NVMF_TGT_FINI_STOP_SUBSYSTEMS; 412 } 413 } else { 414 g_tgt_state = NVMF_TGT_RUNNING; 415 } 416 break; 417 } 418 case NVMF_TGT_RUNNING: 419 spdk_subsystem_init_next(0); 420 break; 421 case NVMF_TGT_FINI_STOP_SUBSYSTEMS: { 422 struct spdk_nvmf_subsystem *subsystem; 423 424 subsystem = spdk_nvmf_subsystem_get_first(g_spdk_nvmf_tgt); 425 426 if (subsystem) { 427 ret = spdk_nvmf_subsystem_stop(subsystem, nvmf_tgt_subsystem_stopped, NULL); 428 if (ret) { 429 nvmf_tgt_subsystem_stopped(subsystem, NULL, 0); 430 } 431 } else { 432 g_tgt_state = NVMF_TGT_FINI_DESTROY_POLL_GROUPS; 433 } 434 break; 435 } 436 case NVMF_TGT_FINI_DESTROY_POLL_GROUPS: 437 /* Send a message to each poll group thread, and terminate the thread */ 438 nvmf_tgt_destroy_poll_groups(); 439 break; 440 case NVMF_TGT_FINI_FREE_RESOURCES: 441 spdk_nvmf_tgt_destroy(g_spdk_nvmf_tgt, nvmf_tgt_destroy_done, NULL); 442 break; 443 case NVMF_TGT_STOPPED: 444 spdk_subsystem_fini_next(); 445 return; 446 case NVMF_TGT_ERROR: 447 spdk_subsystem_init_next(rc); 448 return; 449 } 450 451 } while (g_tgt_state != prev_state); 452 } 453 454 static void 455 nvmf_subsystem_init(void) 456 { 457 g_tgt_state = NVMF_TGT_INIT_NONE; 458 nvmf_tgt_advance_state(); 459 } 460 461 static void 462 nvmf_subsystem_write_config_json(struct spdk_json_write_ctx *w) 463 { 464 spdk_json_write_array_begin(w); 465 466 spdk_json_write_object_begin(w); 467 spdk_json_write_named_string(w, "method", "nvmf_set_config"); 468 469 spdk_json_write_named_object_begin(w, "params"); 470 spdk_json_write_named_uint32(w, "acceptor_poll_rate", g_spdk_nvmf_tgt_conf.acceptor_poll_rate); 471 spdk_json_write_named_object_begin(w, "admin_cmd_passthru"); 472 spdk_json_write_named_bool(w, "identify_ctrlr", 473 g_spdk_nvmf_tgt_conf.admin_passthru.identify_ctrlr); 474 spdk_json_write_object_end(w); 475 spdk_json_write_object_end(w); 476 spdk_json_write_object_end(w); 477 478 spdk_nvmf_tgt_write_config_json(w, g_spdk_nvmf_tgt); 479 spdk_json_write_array_end(w); 480 } 481 482 static struct spdk_subsystem g_spdk_subsystem_nvmf = { 483 .name = "nvmf", 484 .init = nvmf_subsystem_init, 485 .fini = nvmf_subsystem_fini, 486 .write_config_json = nvmf_subsystem_write_config_json, 487 }; 488 489 SPDK_SUBSYSTEM_REGISTER(g_spdk_subsystem_nvmf) 490 SPDK_SUBSYSTEM_DEPEND(nvmf, bdev) 491 SPDK_SUBSYSTEM_DEPEND(nvmf, sock) 492