1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "event_nvmf.h" 35 36 #include "spdk/bdev.h" 37 #include "spdk/event.h" 38 #include "spdk/thread.h" 39 #include "spdk/log.h" 40 #include "spdk/nvme.h" 41 #include "spdk/nvmf_cmd.h" 42 #include "spdk/util.h" 43 44 enum nvmf_tgt_state { 45 NVMF_TGT_INIT_NONE = 0, 46 NVMF_TGT_INIT_PARSE_CONFIG, 47 NVMF_TGT_INIT_CREATE_POLL_GROUPS, 48 NVMF_TGT_INIT_START_SUBSYSTEMS, 49 NVMF_TGT_RUNNING, 50 NVMF_TGT_FINI_STOP_SUBSYSTEMS, 51 NVMF_TGT_FINI_DESTROY_POLL_GROUPS, 52 NVMF_TGT_FINI_FREE_RESOURCES, 53 NVMF_TGT_STOPPED, 54 NVMF_TGT_ERROR, 55 }; 56 57 struct nvmf_tgt_poll_group { 58 struct spdk_nvmf_poll_group *group; 59 struct spdk_thread *thread; 60 TAILQ_ENTRY(nvmf_tgt_poll_group) link; 61 }; 62 63 struct spdk_nvmf_tgt *g_spdk_nvmf_tgt = NULL; 64 65 static enum nvmf_tgt_state g_tgt_state; 66 67 static struct spdk_thread *g_tgt_init_thread = NULL; 68 static struct spdk_thread *g_tgt_fini_thread = NULL; 69 70 static TAILQ_HEAD(, nvmf_tgt_poll_group) g_poll_groups = TAILQ_HEAD_INITIALIZER(g_poll_groups); 71 static size_t g_num_poll_groups = 0; 72 73 static void nvmf_tgt_advance_state(void); 74 75 static void 76 nvmf_shutdown_cb(void *arg1) 77 { 78 /* Still in initialization state, defer shutdown operation */ 79 if (g_tgt_state < NVMF_TGT_RUNNING) { 80 spdk_thread_send_msg(spdk_get_thread(), nvmf_shutdown_cb, NULL); 81 return; 82 } else if (g_tgt_state != NVMF_TGT_RUNNING && g_tgt_state != NVMF_TGT_ERROR) { 83 /* Already in Shutdown status, ignore the signal */ 84 return; 85 } 86 87 if (g_tgt_state == NVMF_TGT_ERROR) { 88 /* Parse configuration error */ 89 g_tgt_state = NVMF_TGT_FINI_FREE_RESOURCES; 90 } else { 91 g_tgt_state = NVMF_TGT_FINI_STOP_SUBSYSTEMS; 92 } 93 nvmf_tgt_advance_state(); 94 } 95 96 static void 97 nvmf_subsystem_fini(void) 98 { 99 nvmf_shutdown_cb(NULL); 100 } 101 102 static void 103 _nvmf_tgt_destroy_poll_group_done(void *ctx) 104 { 105 assert(g_num_poll_groups > 0); 106 107 if (--g_num_poll_groups == 0) { 108 g_tgt_state = NVMF_TGT_FINI_FREE_RESOURCES; 109 nvmf_tgt_advance_state(); 110 } 111 } 112 113 static void 114 nvmf_tgt_destroy_poll_group_done(void *cb_arg, int status) 115 { 116 struct nvmf_tgt_poll_group *pg = cb_arg; 117 118 free(pg); 119 120 spdk_thread_send_msg(g_tgt_fini_thread, _nvmf_tgt_destroy_poll_group_done, NULL); 121 122 spdk_thread_exit(spdk_get_thread()); 123 } 124 125 static void 126 nvmf_tgt_destroy_poll_group(void *ctx) 127 { 128 struct nvmf_tgt_poll_group *pg = ctx; 129 130 spdk_nvmf_poll_group_destroy(pg->group, nvmf_tgt_destroy_poll_group_done, pg); 131 } 132 133 static void 134 nvmf_tgt_destroy_poll_groups(void) 135 { 136 struct nvmf_tgt_poll_group *pg, *tpg; 137 138 g_tgt_fini_thread = spdk_get_thread(); 139 assert(g_tgt_fini_thread != NULL); 140 141 TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tpg) { 142 TAILQ_REMOVE(&g_poll_groups, pg, link); 143 spdk_thread_send_msg(pg->thread, nvmf_tgt_destroy_poll_group, pg); 144 } 145 } 146 147 static void 148 nvmf_tgt_create_poll_group_done(void *ctx) 149 { 150 struct nvmf_tgt_poll_group *pg = ctx; 151 152 TAILQ_INSERT_TAIL(&g_poll_groups, pg, link); 153 154 assert(g_num_poll_groups < spdk_env_get_core_count()); 155 156 if (++g_num_poll_groups == spdk_env_get_core_count()) { 157 g_tgt_state = NVMF_TGT_INIT_START_SUBSYSTEMS; 158 nvmf_tgt_advance_state(); 159 } 160 } 161 162 static void 163 nvmf_tgt_create_poll_group(void *ctx) 164 { 165 struct nvmf_tgt_poll_group *pg; 166 167 pg = calloc(1, sizeof(*pg)); 168 if (!pg) { 169 SPDK_ERRLOG("Not enough memory to allocate poll groups\n"); 170 spdk_app_stop(-ENOMEM); 171 return; 172 } 173 174 pg->thread = spdk_get_thread(); 175 pg->group = spdk_nvmf_poll_group_create(g_spdk_nvmf_tgt); 176 177 spdk_thread_send_msg(g_tgt_init_thread, nvmf_tgt_create_poll_group_done, pg); 178 } 179 180 static void 181 nvmf_tgt_create_poll_groups(void) 182 { 183 struct spdk_cpuset tmp_cpumask = {}; 184 uint32_t i; 185 char thread_name[32]; 186 struct spdk_thread *thread; 187 188 g_tgt_init_thread = spdk_get_thread(); 189 assert(g_tgt_init_thread != NULL); 190 191 SPDK_ENV_FOREACH_CORE(i) { 192 spdk_cpuset_zero(&tmp_cpumask); 193 spdk_cpuset_set_cpu(&tmp_cpumask, i, true); 194 snprintf(thread_name, sizeof(thread_name), "nvmf_tgt_poll_group_%u", i); 195 196 thread = spdk_thread_create(thread_name, &tmp_cpumask); 197 assert(thread != NULL); 198 199 spdk_thread_send_msg(thread, nvmf_tgt_create_poll_group, NULL); 200 } 201 } 202 203 static void 204 nvmf_tgt_subsystem_started(struct spdk_nvmf_subsystem *subsystem, 205 void *cb_arg, int status) 206 { 207 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 208 int rc; 209 210 if (subsystem) { 211 rc = spdk_nvmf_subsystem_start(subsystem, nvmf_tgt_subsystem_started, NULL); 212 if (rc) { 213 g_tgt_state = NVMF_TGT_FINI_STOP_SUBSYSTEMS; 214 SPDK_ERRLOG("Unable to start NVMe-oF subsystem. Stopping app.\n"); 215 nvmf_tgt_advance_state(); 216 } 217 return; 218 } 219 220 g_tgt_state = NVMF_TGT_RUNNING; 221 nvmf_tgt_advance_state(); 222 } 223 224 static void 225 nvmf_tgt_subsystem_stopped(struct spdk_nvmf_subsystem *subsystem, 226 void *cb_arg, int status) 227 { 228 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 229 int rc; 230 231 if (subsystem) { 232 rc = spdk_nvmf_subsystem_stop(subsystem, nvmf_tgt_subsystem_stopped, NULL); 233 if (rc) { 234 SPDK_ERRLOG("Unable to stop NVMe-oF subsystem. Trying others.\n"); 235 nvmf_tgt_subsystem_stopped(subsystem, NULL, 0); 236 } 237 return; 238 } 239 240 g_tgt_state = NVMF_TGT_FINI_DESTROY_POLL_GROUPS; 241 nvmf_tgt_advance_state(); 242 } 243 244 static void 245 nvmf_tgt_destroy_done(void *ctx, int status) 246 { 247 g_tgt_state = NVMF_TGT_STOPPED; 248 249 free(g_spdk_nvmf_tgt_conf); 250 g_spdk_nvmf_tgt_conf = NULL; 251 nvmf_tgt_advance_state(); 252 } 253 254 static void 255 nvmf_tgt_parse_conf_done(int status) 256 { 257 g_tgt_state = (status == 0) ? NVMF_TGT_INIT_CREATE_POLL_GROUPS : NVMF_TGT_ERROR; 258 nvmf_tgt_advance_state(); 259 } 260 261 static void 262 nvmf_tgt_parse_conf_start(void *ctx) 263 { 264 if (nvmf_parse_conf(nvmf_tgt_parse_conf_done)) { 265 SPDK_ERRLOG("nvmf_parse_conf() failed\n"); 266 g_tgt_state = NVMF_TGT_ERROR; 267 nvmf_tgt_advance_state(); 268 } 269 } 270 271 static void 272 fixup_identify_ctrlr(struct spdk_nvmf_request *req) 273 { 274 uint32_t length; 275 int rc; 276 struct spdk_nvme_ctrlr_data *nvme_cdata; 277 struct spdk_nvme_ctrlr_data nvmf_cdata = {}; 278 struct spdk_nvmf_ctrlr *ctrlr = spdk_nvmf_request_get_ctrlr(req); 279 struct spdk_nvme_cpl *rsp = spdk_nvmf_request_get_response(req); 280 281 /* This is the identify data from the NVMe drive */ 282 spdk_nvmf_request_get_data(req, (void **)&nvme_cdata, &length); 283 284 /* Get the NVMF identify data */ 285 rc = spdk_nvmf_ctrlr_identify_ctrlr(ctrlr, &nvmf_cdata); 286 if (rc != SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 287 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 288 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 289 return; 290 } 291 292 /* Fixup NVMF identify data with NVMe identify data */ 293 294 /* Serial Number (SN) */ 295 memcpy(&nvmf_cdata.sn[0], &nvme_cdata->sn[0], sizeof(nvmf_cdata.sn)); 296 /* Model Number (MN) */ 297 memcpy(&nvmf_cdata.mn[0], &nvme_cdata->mn[0], sizeof(nvmf_cdata.mn)); 298 /* Firmware Revision (FR) */ 299 memcpy(&nvmf_cdata.fr[0], &nvme_cdata->fr[0], sizeof(nvmf_cdata.fr)); 300 /* IEEE OUI Identifier (IEEE) */ 301 memcpy(&nvmf_cdata.ieee[0], &nvme_cdata->ieee[0], sizeof(nvmf_cdata.ieee)); 302 /* FRU Globally Unique Identifier (FGUID) */ 303 304 /* Copy the fixed up data back to the response */ 305 memcpy(nvme_cdata, &nvmf_cdata, length); 306 } 307 308 static int 309 nvmf_custom_identify_hdlr(struct spdk_nvmf_request *req) 310 { 311 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req); 312 struct spdk_bdev *bdev; 313 struct spdk_bdev_desc *desc; 314 struct spdk_io_channel *ch; 315 struct spdk_nvmf_subsystem *subsys; 316 int rc; 317 318 if (cmd->cdw10_bits.identify.cns != SPDK_NVME_IDENTIFY_CTRLR) { 319 return -1; /* continue */ 320 } 321 322 subsys = spdk_nvmf_request_get_subsystem(req); 323 if (subsys == NULL) { 324 return -1; 325 } 326 327 /* Only procss this request if it has exactly one namespace */ 328 if (spdk_nvmf_subsystem_get_max_nsid(subsys) != 1) { 329 return -1; 330 } 331 332 /* Forward to first namespace if it supports NVME admin commands */ 333 rc = spdk_nvmf_request_get_bdev(1, req, &bdev, &desc, &ch); 334 if (rc) { 335 /* No bdev found for this namespace. Continue. */ 336 return -1; 337 } 338 339 if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_NVME_ADMIN)) { 340 return -1; 341 } 342 343 return spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(bdev, desc, ch, req, fixup_identify_ctrlr); 344 } 345 346 static void 347 nvmf_tgt_advance_state(void) 348 { 349 enum nvmf_tgt_state prev_state; 350 int rc = -1; 351 int ret; 352 353 do { 354 prev_state = g_tgt_state; 355 356 switch (g_tgt_state) { 357 case NVMF_TGT_INIT_NONE: { 358 g_tgt_state = NVMF_TGT_INIT_PARSE_CONFIG; 359 break; 360 } 361 case NVMF_TGT_INIT_PARSE_CONFIG: 362 /* Send message to self to call parse conf func. 363 * Prevents it from possibly performing cb before getting 364 * out of this function, which causes problems. */ 365 spdk_thread_send_msg(spdk_get_thread(), nvmf_tgt_parse_conf_start, NULL); 366 break; 367 case NVMF_TGT_INIT_CREATE_POLL_GROUPS: 368 /* Config parsed */ 369 if (g_spdk_nvmf_tgt_conf->admin_passthru.identify_ctrlr) { 370 SPDK_NOTICELOG("Custom identify ctrlr handler enabled\n"); 371 spdk_nvmf_set_custom_admin_cmd_hdlr(SPDK_NVME_OPC_IDENTIFY, nvmf_custom_identify_hdlr); 372 } 373 /* Create poll group threads, and send a message to each thread 374 * and create a poll group. 375 */ 376 nvmf_tgt_create_poll_groups(); 377 break; 378 case NVMF_TGT_INIT_START_SUBSYSTEMS: { 379 struct spdk_nvmf_subsystem *subsystem; 380 381 subsystem = spdk_nvmf_subsystem_get_first(g_spdk_nvmf_tgt); 382 383 if (subsystem) { 384 ret = spdk_nvmf_subsystem_start(subsystem, nvmf_tgt_subsystem_started, NULL); 385 if (ret) { 386 SPDK_ERRLOG("Unable to start NVMe-oF subsystem. Stopping app.\n"); 387 g_tgt_state = NVMF_TGT_FINI_STOP_SUBSYSTEMS; 388 } 389 } else { 390 g_tgt_state = NVMF_TGT_RUNNING; 391 } 392 break; 393 } 394 case NVMF_TGT_RUNNING: 395 spdk_subsystem_init_next(0); 396 break; 397 case NVMF_TGT_FINI_STOP_SUBSYSTEMS: { 398 struct spdk_nvmf_subsystem *subsystem; 399 400 subsystem = spdk_nvmf_subsystem_get_first(g_spdk_nvmf_tgt); 401 402 if (subsystem) { 403 ret = spdk_nvmf_subsystem_stop(subsystem, nvmf_tgt_subsystem_stopped, NULL); 404 if (ret) { 405 nvmf_tgt_subsystem_stopped(subsystem, NULL, 0); 406 } 407 } else { 408 g_tgt_state = NVMF_TGT_FINI_DESTROY_POLL_GROUPS; 409 } 410 break; 411 } 412 case NVMF_TGT_FINI_DESTROY_POLL_GROUPS: 413 /* Send a message to each poll group thread, and terminate the thread */ 414 nvmf_tgt_destroy_poll_groups(); 415 break; 416 case NVMF_TGT_FINI_FREE_RESOURCES: 417 spdk_nvmf_tgt_destroy(g_spdk_nvmf_tgt, nvmf_tgt_destroy_done, NULL); 418 break; 419 case NVMF_TGT_STOPPED: 420 spdk_subsystem_fini_next(); 421 return; 422 case NVMF_TGT_ERROR: 423 spdk_subsystem_init_next(rc); 424 return; 425 } 426 427 } while (g_tgt_state != prev_state); 428 } 429 430 static void 431 nvmf_subsystem_init(void) 432 { 433 g_tgt_state = NVMF_TGT_INIT_NONE; 434 nvmf_tgt_advance_state(); 435 } 436 437 static void 438 nvmf_subsystem_write_config_json(struct spdk_json_write_ctx *w) 439 { 440 spdk_json_write_array_begin(w); 441 442 spdk_json_write_object_begin(w); 443 spdk_json_write_named_string(w, "method", "nvmf_set_config"); 444 445 spdk_json_write_named_object_begin(w, "params"); 446 spdk_json_write_named_uint32(w, "acceptor_poll_rate", g_spdk_nvmf_tgt_conf->acceptor_poll_rate); 447 spdk_json_write_named_object_begin(w, "admin_cmd_passthru"); 448 spdk_json_write_named_bool(w, "identify_ctrlr", 449 g_spdk_nvmf_tgt_conf->admin_passthru.identify_ctrlr); 450 spdk_json_write_object_end(w); 451 spdk_json_write_object_end(w); 452 spdk_json_write_object_end(w); 453 454 spdk_nvmf_tgt_write_config_json(w, g_spdk_nvmf_tgt); 455 spdk_json_write_array_end(w); 456 } 457 458 static struct spdk_subsystem g_spdk_subsystem_nvmf = { 459 .name = "nvmf", 460 .init = nvmf_subsystem_init, 461 .fini = nvmf_subsystem_fini, 462 .write_config_json = nvmf_subsystem_write_config_json, 463 }; 464 465 SPDK_SUBSYSTEM_REGISTER(g_spdk_subsystem_nvmf) 466 SPDK_SUBSYSTEM_DEPEND(nvmf, bdev) 467 SPDK_SUBSYSTEM_DEPEND(nvmf, sock) 468