1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "event_nvmf.h" 35 36 #include "spdk/bdev.h" 37 #include "spdk/event.h" 38 #include "spdk/thread.h" 39 #include "spdk/log.h" 40 #include "spdk/nvme.h" 41 #include "spdk/nvmf_cmd.h" 42 #include "spdk/util.h" 43 44 enum nvmf_tgt_state { 45 NVMF_TGT_INIT_NONE = 0, 46 NVMF_TGT_INIT_PARSE_CONFIG, 47 NVMF_TGT_INIT_CREATE_POLL_GROUPS, 48 NVMF_TGT_INIT_START_SUBSYSTEMS, 49 NVMF_TGT_INIT_START_ACCEPTOR, 50 NVMF_TGT_RUNNING, 51 NVMF_TGT_FINI_STOP_SUBSYSTEMS, 52 NVMF_TGT_FINI_DESTROY_POLL_GROUPS, 53 NVMF_TGT_FINI_STOP_ACCEPTOR, 54 NVMF_TGT_FINI_FREE_RESOURCES, 55 NVMF_TGT_STOPPED, 56 NVMF_TGT_ERROR, 57 }; 58 59 struct nvmf_tgt_poll_group { 60 struct spdk_nvmf_poll_group *group; 61 struct spdk_thread *thread; 62 TAILQ_ENTRY(nvmf_tgt_poll_group) link; 63 }; 64 65 struct spdk_nvmf_tgt *g_spdk_nvmf_tgt = NULL; 66 67 static enum nvmf_tgt_state g_tgt_state; 68 69 static struct spdk_thread *g_tgt_init_thread = NULL; 70 static struct spdk_thread *g_tgt_fini_thread = NULL; 71 72 static TAILQ_HEAD(, nvmf_tgt_poll_group) g_poll_groups = TAILQ_HEAD_INITIALIZER(g_poll_groups); 73 static size_t g_num_poll_groups = 0; 74 75 static struct spdk_poller *g_acceptor_poller = NULL; 76 77 static void nvmf_tgt_advance_state(void); 78 79 static void 80 nvmf_shutdown_cb(void *arg1) 81 { 82 /* Still in initialization state, defer shutdown operation */ 83 if (g_tgt_state < NVMF_TGT_RUNNING) { 84 spdk_thread_send_msg(spdk_get_thread(), nvmf_shutdown_cb, NULL); 85 return; 86 } else if (g_tgt_state != NVMF_TGT_RUNNING && g_tgt_state != NVMF_TGT_ERROR) { 87 /* Already in Shutdown status, ignore the signal */ 88 return; 89 } 90 91 if (g_tgt_state == NVMF_TGT_ERROR) { 92 /* Parse configuration error */ 93 g_tgt_state = NVMF_TGT_FINI_FREE_RESOURCES; 94 } else { 95 g_tgt_state = NVMF_TGT_FINI_STOP_SUBSYSTEMS; 96 } 97 nvmf_tgt_advance_state(); 98 } 99 100 static void 101 nvmf_subsystem_fini(void) 102 { 103 nvmf_shutdown_cb(NULL); 104 } 105 106 static int 107 acceptor_poll(void *arg) 108 { 109 struct spdk_nvmf_tgt *tgt = arg; 110 uint32_t count; 111 112 count = spdk_nvmf_tgt_accept(tgt); 113 114 if (count > 0) { 115 return SPDK_POLLER_BUSY; 116 } else { 117 return SPDK_POLLER_IDLE; 118 } 119 } 120 121 static void 122 _nvmf_tgt_destroy_poll_group_done(void *ctx) 123 { 124 assert(g_num_poll_groups > 0); 125 126 if (--g_num_poll_groups == 0) { 127 g_tgt_state = NVMF_TGT_FINI_STOP_ACCEPTOR; 128 nvmf_tgt_advance_state(); 129 } 130 } 131 132 static void 133 nvmf_tgt_destroy_poll_group_done(void *cb_arg, int status) 134 { 135 struct nvmf_tgt_poll_group *pg = cb_arg; 136 137 free(pg); 138 139 spdk_thread_send_msg(g_tgt_fini_thread, _nvmf_tgt_destroy_poll_group_done, NULL); 140 141 spdk_thread_exit(spdk_get_thread()); 142 } 143 144 static void 145 nvmf_tgt_destroy_poll_group(void *ctx) 146 { 147 struct nvmf_tgt_poll_group *pg = ctx; 148 149 spdk_nvmf_poll_group_destroy(pg->group, nvmf_tgt_destroy_poll_group_done, pg); 150 } 151 152 static void 153 nvmf_tgt_destroy_poll_groups(void) 154 { 155 struct nvmf_tgt_poll_group *pg, *tpg; 156 157 g_tgt_fini_thread = spdk_get_thread(); 158 assert(g_tgt_fini_thread != NULL); 159 160 TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tpg) { 161 TAILQ_REMOVE(&g_poll_groups, pg, link); 162 spdk_thread_send_msg(pg->thread, nvmf_tgt_destroy_poll_group, pg); 163 } 164 } 165 166 static void 167 nvmf_tgt_create_poll_group_done(void *ctx) 168 { 169 struct nvmf_tgt_poll_group *pg = ctx; 170 171 TAILQ_INSERT_TAIL(&g_poll_groups, pg, link); 172 173 assert(g_num_poll_groups < spdk_env_get_core_count()); 174 175 if (++g_num_poll_groups == spdk_env_get_core_count()) { 176 g_tgt_state = NVMF_TGT_INIT_START_SUBSYSTEMS; 177 nvmf_tgt_advance_state(); 178 } 179 } 180 181 static void 182 nvmf_tgt_create_poll_group(void *ctx) 183 { 184 struct nvmf_tgt_poll_group *pg; 185 186 pg = calloc(1, sizeof(*pg)); 187 if (!pg) { 188 SPDK_ERRLOG("Not enough memory to allocate poll groups\n"); 189 spdk_app_stop(-ENOMEM); 190 return; 191 } 192 193 pg->thread = spdk_get_thread(); 194 pg->group = spdk_nvmf_poll_group_create(g_spdk_nvmf_tgt); 195 196 spdk_thread_send_msg(g_tgt_init_thread, nvmf_tgt_create_poll_group_done, pg); 197 } 198 199 static void 200 nvmf_tgt_create_poll_groups(void) 201 { 202 struct spdk_cpuset tmp_cpumask = {}; 203 uint32_t i; 204 char thread_name[32]; 205 struct spdk_thread *thread; 206 207 g_tgt_init_thread = spdk_get_thread(); 208 assert(g_tgt_init_thread != NULL); 209 210 SPDK_ENV_FOREACH_CORE(i) { 211 spdk_cpuset_zero(&tmp_cpumask); 212 spdk_cpuset_set_cpu(&tmp_cpumask, i, true); 213 snprintf(thread_name, sizeof(thread_name), "nvmf_tgt_poll_group_%u", i); 214 215 thread = spdk_thread_create(thread_name, &tmp_cpumask); 216 assert(thread != NULL); 217 218 spdk_thread_send_msg(thread, nvmf_tgt_create_poll_group, NULL); 219 } 220 } 221 222 static void 223 nvmf_tgt_subsystem_started(struct spdk_nvmf_subsystem *subsystem, 224 void *cb_arg, int status) 225 { 226 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 227 228 if (subsystem) { 229 spdk_nvmf_subsystem_start(subsystem, nvmf_tgt_subsystem_started, NULL); 230 return; 231 } 232 233 g_tgt_state = NVMF_TGT_INIT_START_ACCEPTOR; 234 nvmf_tgt_advance_state(); 235 } 236 237 static void 238 nvmf_tgt_subsystem_stopped(struct spdk_nvmf_subsystem *subsystem, 239 void *cb_arg, int status) 240 { 241 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 242 243 if (subsystem) { 244 spdk_nvmf_subsystem_stop(subsystem, nvmf_tgt_subsystem_stopped, NULL); 245 return; 246 } 247 248 g_tgt_state = NVMF_TGT_FINI_DESTROY_POLL_GROUPS; 249 nvmf_tgt_advance_state(); 250 } 251 252 static void 253 nvmf_tgt_destroy_done(void *ctx, int status) 254 { 255 g_tgt_state = NVMF_TGT_STOPPED; 256 257 free(g_spdk_nvmf_tgt_conf); 258 g_spdk_nvmf_tgt_conf = NULL; 259 nvmf_tgt_advance_state(); 260 } 261 262 static void 263 nvmf_tgt_parse_conf_done(int status) 264 { 265 g_tgt_state = (status == 0) ? NVMF_TGT_INIT_CREATE_POLL_GROUPS : NVMF_TGT_ERROR; 266 nvmf_tgt_advance_state(); 267 } 268 269 static void 270 nvmf_tgt_parse_conf_start(void *ctx) 271 { 272 if (nvmf_parse_conf(nvmf_tgt_parse_conf_done)) { 273 SPDK_ERRLOG("nvmf_parse_conf() failed\n"); 274 g_tgt_state = NVMF_TGT_ERROR; 275 nvmf_tgt_advance_state(); 276 } 277 } 278 279 static void 280 fixup_identify_ctrlr(struct spdk_nvmf_request *req) 281 { 282 uint32_t length; 283 int rc; 284 struct spdk_nvme_ctrlr_data *nvme_cdata; 285 struct spdk_nvme_ctrlr_data nvmf_cdata = {}; 286 struct spdk_nvmf_ctrlr *ctrlr = spdk_nvmf_request_get_ctrlr(req); 287 struct spdk_nvme_cpl *rsp = spdk_nvmf_request_get_response(req); 288 289 /* This is the identify data from the NVMe drive */ 290 spdk_nvmf_request_get_data(req, (void **)&nvme_cdata, &length); 291 292 /* Get the NVMF identify data */ 293 rc = spdk_nvmf_ctrlr_identify_ctrlr(ctrlr, &nvmf_cdata); 294 if (rc != SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 295 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 296 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 297 return; 298 } 299 300 /* Fixup NVMF identify data with NVMe identify data */ 301 302 /* Serial Number (SN) */ 303 memcpy(&nvmf_cdata.sn[0], &nvme_cdata->sn[0], sizeof(nvmf_cdata.sn)); 304 /* Model Number (MN) */ 305 memcpy(&nvmf_cdata.mn[0], &nvme_cdata->mn[0], sizeof(nvmf_cdata.mn)); 306 /* Firmware Revision (FR) */ 307 memcpy(&nvmf_cdata.fr[0], &nvme_cdata->fr[0], sizeof(nvmf_cdata.fr)); 308 /* IEEE OUI Identifier (IEEE) */ 309 memcpy(&nvmf_cdata.ieee[0], &nvme_cdata->ieee[0], sizeof(nvmf_cdata.ieee)); 310 /* FRU Globally Unique Identifier (FGUID) */ 311 312 /* Copy the fixed up data back to the response */ 313 memcpy(nvme_cdata, &nvmf_cdata, length); 314 } 315 316 static int 317 nvmf_custom_identify_hdlr(struct spdk_nvmf_request *req) 318 { 319 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req); 320 struct spdk_bdev *bdev; 321 struct spdk_bdev_desc *desc; 322 struct spdk_io_channel *ch; 323 struct spdk_nvmf_subsystem *subsys; 324 int rc; 325 326 if (cmd->cdw10_bits.identify.cns != SPDK_NVME_IDENTIFY_CTRLR) { 327 return -1; /* continue */ 328 } 329 330 subsys = spdk_nvmf_request_get_subsystem(req); 331 if (subsys == NULL) { 332 return -1; 333 } 334 335 /* Only procss this request if it has exactly one namespace */ 336 if (spdk_nvmf_subsystem_get_max_nsid(subsys) != 1) { 337 return -1; 338 } 339 340 /* Forward to first namespace if it supports NVME admin commands */ 341 rc = spdk_nvmf_request_get_bdev(1, req, &bdev, &desc, &ch); 342 if (rc) { 343 /* No bdev found for this namespace. Continue. */ 344 return -1; 345 } 346 347 if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_NVME_ADMIN)) { 348 return -1; 349 } 350 351 return spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(bdev, desc, ch, req, fixup_identify_ctrlr); 352 } 353 354 static void 355 nvmf_tgt_advance_state(void) 356 { 357 enum nvmf_tgt_state prev_state; 358 int rc = -1; 359 360 do { 361 prev_state = g_tgt_state; 362 363 switch (g_tgt_state) { 364 case NVMF_TGT_INIT_NONE: { 365 g_tgt_state = NVMF_TGT_INIT_PARSE_CONFIG; 366 break; 367 } 368 case NVMF_TGT_INIT_PARSE_CONFIG: 369 /* Send message to self to call parse conf func. 370 * Prevents it from possibly performing cb before getting 371 * out of this function, which causes problems. */ 372 spdk_thread_send_msg(spdk_get_thread(), nvmf_tgt_parse_conf_start, NULL); 373 break; 374 case NVMF_TGT_INIT_CREATE_POLL_GROUPS: 375 /* Config parsed */ 376 if (g_spdk_nvmf_tgt_conf->admin_passthru.identify_ctrlr) { 377 SPDK_NOTICELOG("Custom identify ctrlr handler enabled\n"); 378 spdk_nvmf_set_custom_admin_cmd_hdlr(SPDK_NVME_OPC_IDENTIFY, nvmf_custom_identify_hdlr); 379 } 380 /* Create poll group threads, and send a message to each thread 381 * and create a poll group. 382 */ 383 nvmf_tgt_create_poll_groups(); 384 break; 385 case NVMF_TGT_INIT_START_SUBSYSTEMS: { 386 struct spdk_nvmf_subsystem *subsystem; 387 388 subsystem = spdk_nvmf_subsystem_get_first(g_spdk_nvmf_tgt); 389 390 if (subsystem) { 391 spdk_nvmf_subsystem_start(subsystem, nvmf_tgt_subsystem_started, NULL); 392 } else { 393 g_tgt_state = NVMF_TGT_INIT_START_ACCEPTOR; 394 } 395 break; 396 } 397 case NVMF_TGT_INIT_START_ACCEPTOR: 398 g_acceptor_poller = SPDK_POLLER_REGISTER(acceptor_poll, g_spdk_nvmf_tgt, 399 g_spdk_nvmf_tgt_conf->acceptor_poll_rate); 400 g_tgt_state = NVMF_TGT_RUNNING; 401 break; 402 case NVMF_TGT_RUNNING: 403 spdk_subsystem_init_next(0); 404 break; 405 case NVMF_TGT_FINI_STOP_SUBSYSTEMS: { 406 struct spdk_nvmf_subsystem *subsystem; 407 408 subsystem = spdk_nvmf_subsystem_get_first(g_spdk_nvmf_tgt); 409 410 if (subsystem) { 411 spdk_nvmf_subsystem_stop(subsystem, nvmf_tgt_subsystem_stopped, NULL); 412 } else { 413 g_tgt_state = NVMF_TGT_FINI_DESTROY_POLL_GROUPS; 414 } 415 break; 416 } 417 case NVMF_TGT_FINI_DESTROY_POLL_GROUPS: 418 /* Send a message to each poll group thread, and terminate the thread */ 419 nvmf_tgt_destroy_poll_groups(); 420 break; 421 case NVMF_TGT_FINI_STOP_ACCEPTOR: 422 spdk_poller_unregister(&g_acceptor_poller); 423 g_tgt_state = NVMF_TGT_FINI_FREE_RESOURCES; 424 break; 425 case NVMF_TGT_FINI_FREE_RESOURCES: 426 spdk_nvmf_tgt_destroy(g_spdk_nvmf_tgt, nvmf_tgt_destroy_done, NULL); 427 break; 428 case NVMF_TGT_STOPPED: 429 spdk_subsystem_fini_next(); 430 return; 431 case NVMF_TGT_ERROR: 432 spdk_subsystem_init_next(rc); 433 return; 434 } 435 436 } while (g_tgt_state != prev_state); 437 } 438 439 static void 440 nvmf_subsystem_init(void) 441 { 442 g_tgt_state = NVMF_TGT_INIT_NONE; 443 nvmf_tgt_advance_state(); 444 } 445 446 static void 447 nvmf_subsystem_write_config_json(struct spdk_json_write_ctx *w) 448 { 449 spdk_json_write_array_begin(w); 450 451 spdk_json_write_object_begin(w); 452 spdk_json_write_named_string(w, "method", "nvmf_set_config"); 453 454 spdk_json_write_named_object_begin(w, "params"); 455 spdk_json_write_named_uint32(w, "acceptor_poll_rate", g_spdk_nvmf_tgt_conf->acceptor_poll_rate); 456 spdk_json_write_named_object_begin(w, "admin_cmd_passthru"); 457 spdk_json_write_named_bool(w, "identify_ctrlr", 458 g_spdk_nvmf_tgt_conf->admin_passthru.identify_ctrlr); 459 spdk_json_write_object_end(w); 460 spdk_json_write_object_end(w); 461 spdk_json_write_object_end(w); 462 463 spdk_nvmf_tgt_write_config_json(w, g_spdk_nvmf_tgt); 464 spdk_json_write_array_end(w); 465 } 466 467 static struct spdk_subsystem g_spdk_subsystem_nvmf = { 468 .name = "nvmf", 469 .init = nvmf_subsystem_init, 470 .fini = nvmf_subsystem_fini, 471 .write_config_json = nvmf_subsystem_write_config_json, 472 }; 473 474 SPDK_SUBSYSTEM_REGISTER(g_spdk_subsystem_nvmf) 475 SPDK_SUBSYSTEM_DEPEND(nvmf, bdev) 476 SPDK_SUBSYSTEM_DEPEND(nvmf, sock) 477