1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019, 2020 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "nvmf_internal.h" 37 #include "transport.h" 38 39 #include "spdk/bit_array.h" 40 #include "spdk/endian.h" 41 #include "spdk/thread.h" 42 #include "spdk/nvme_spec.h" 43 #include "spdk/nvmf_cmd.h" 44 #include "spdk/string.h" 45 #include "spdk/util.h" 46 #include "spdk/version.h" 47 #include "spdk/log.h" 48 #include "spdk_internal/usdt.h" 49 50 #define MIN_KEEP_ALIVE_TIMEOUT_IN_MS 10000 51 #define NVMF_DISC_KATO_IN_MS 120000 52 #define KAS_TIME_UNIT_IN_MS 100 53 #define KAS_DEFAULT_VALUE (MIN_KEEP_ALIVE_TIMEOUT_IN_MS / KAS_TIME_UNIT_IN_MS) 54 55 /* 56 * Report the SPDK version as the firmware revision. 57 * SPDK_VERSION_STRING won't fit into FR (only 8 bytes), so try to fit the most important parts. 58 */ 59 #define FW_VERSION SPDK_VERSION_MAJOR_STRING SPDK_VERSION_MINOR_STRING SPDK_VERSION_PATCH_STRING 60 61 #define ANA_TRANSITION_TIME_IN_SEC 10 62 63 /* 64 * Support for custom admin command handlers 65 */ 66 struct spdk_nvmf_custom_admin_cmd { 67 spdk_nvmf_custom_cmd_hdlr hdlr; 68 uint32_t nsid; /* nsid to forward */ 69 }; 70 71 static struct spdk_nvmf_custom_admin_cmd g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_MAX_OPC + 1]; 72 73 static void _nvmf_request_complete(void *ctx); 74 75 static inline void 76 nvmf_invalid_connect_response(struct spdk_nvmf_fabric_connect_rsp *rsp, 77 uint8_t iattr, uint16_t ipo) 78 { 79 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 80 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 81 rsp->status_code_specific.invalid.iattr = iattr; 82 rsp->status_code_specific.invalid.ipo = ipo; 83 } 84 85 #define SPDK_NVMF_INVALID_CONNECT_CMD(rsp, field) \ 86 nvmf_invalid_connect_response(rsp, 0, offsetof(struct spdk_nvmf_fabric_connect_cmd, field)) 87 #define SPDK_NVMF_INVALID_CONNECT_DATA(rsp, field) \ 88 nvmf_invalid_connect_response(rsp, 1, offsetof(struct spdk_nvmf_fabric_connect_data, field)) 89 90 91 static void 92 nvmf_ctrlr_stop_keep_alive_timer(struct spdk_nvmf_ctrlr *ctrlr) 93 { 94 if (!ctrlr) { 95 SPDK_ERRLOG("Controller is NULL\n"); 96 return; 97 } 98 99 if (ctrlr->keep_alive_poller == NULL) { 100 return; 101 } 102 103 SPDK_DEBUGLOG(nvmf, "Stop keep alive poller\n"); 104 spdk_poller_unregister(&ctrlr->keep_alive_poller); 105 } 106 107 static void 108 nvmf_ctrlr_stop_association_timer(struct spdk_nvmf_ctrlr *ctrlr) 109 { 110 if (!ctrlr) { 111 SPDK_ERRLOG("Controller is NULL\n"); 112 assert(false); 113 return; 114 } 115 116 if (ctrlr->association_timer == NULL) { 117 return; 118 } 119 120 SPDK_DEBUGLOG(nvmf, "Stop association timer\n"); 121 spdk_poller_unregister(&ctrlr->association_timer); 122 } 123 124 static void 125 nvmf_ctrlr_disconnect_qpairs_done(struct spdk_io_channel_iter *i, int status) 126 { 127 if (status == 0) { 128 SPDK_DEBUGLOG(nvmf, "ctrlr disconnect qpairs complete successfully\n"); 129 } else { 130 SPDK_ERRLOG("Fail to disconnect ctrlr qpairs\n"); 131 } 132 } 133 134 static int 135 _nvmf_ctrlr_disconnect_qpairs_on_pg(struct spdk_io_channel_iter *i, bool include_admin) 136 { 137 int rc = 0; 138 struct spdk_nvmf_ctrlr *ctrlr; 139 struct spdk_nvmf_qpair *qpair, *temp_qpair; 140 struct spdk_io_channel *ch; 141 struct spdk_nvmf_poll_group *group; 142 143 ctrlr = spdk_io_channel_iter_get_ctx(i); 144 ch = spdk_io_channel_iter_get_channel(i); 145 group = spdk_io_channel_get_ctx(ch); 146 147 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, temp_qpair) { 148 if (qpair->ctrlr == ctrlr && (include_admin || !nvmf_qpair_is_admin_queue(qpair))) { 149 rc = spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 150 if (rc) { 151 SPDK_ERRLOG("Qpair disconnect failed\n"); 152 return rc; 153 } 154 } 155 } 156 157 return rc; 158 } 159 160 static void 161 nvmf_ctrlr_disconnect_qpairs_on_pg(struct spdk_io_channel_iter *i) 162 { 163 spdk_for_each_channel_continue(i, _nvmf_ctrlr_disconnect_qpairs_on_pg(i, true)); 164 } 165 166 static void 167 nvmf_ctrlr_disconnect_io_qpairs_on_pg(struct spdk_io_channel_iter *i) 168 { 169 spdk_for_each_channel_continue(i, _nvmf_ctrlr_disconnect_qpairs_on_pg(i, false)); 170 } 171 172 static int 173 nvmf_ctrlr_keep_alive_poll(void *ctx) 174 { 175 uint64_t keep_alive_timeout_tick; 176 uint64_t now = spdk_get_ticks(); 177 struct spdk_nvmf_ctrlr *ctrlr = ctx; 178 179 SPDK_DEBUGLOG(nvmf, "Polling ctrlr keep alive timeout\n"); 180 181 /* If the Keep alive feature is in use and the timer expires */ 182 keep_alive_timeout_tick = ctrlr->last_keep_alive_tick + 183 ctrlr->feat.keep_alive_timer.bits.kato * spdk_get_ticks_hz() / UINT64_C(1000); 184 if (now > keep_alive_timeout_tick) { 185 SPDK_NOTICELOG("Disconnecting host %s from subsystem %s due to keep alive timeout.\n", 186 ctrlr->hostnqn, ctrlr->subsys->subnqn); 187 /* set the Controller Fatal Status bit to '1' */ 188 if (ctrlr->vcprop.csts.bits.cfs == 0) { 189 ctrlr->vcprop.csts.bits.cfs = 1; 190 191 /* 192 * disconnect qpairs, terminate Transport connection 193 * destroy ctrlr, break the host to controller association 194 * disconnect qpairs with qpair->ctrlr == ctrlr 195 */ 196 spdk_for_each_channel(ctrlr->subsys->tgt, 197 nvmf_ctrlr_disconnect_qpairs_on_pg, 198 ctrlr, 199 nvmf_ctrlr_disconnect_qpairs_done); 200 } 201 } 202 203 return SPDK_POLLER_BUSY; 204 } 205 206 static void 207 nvmf_ctrlr_start_keep_alive_timer(struct spdk_nvmf_ctrlr *ctrlr) 208 { 209 if (!ctrlr) { 210 SPDK_ERRLOG("Controller is NULL\n"); 211 return; 212 } 213 214 /* if cleared to 0 then the Keep Alive Timer is disabled */ 215 if (ctrlr->feat.keep_alive_timer.bits.kato != 0) { 216 217 ctrlr->last_keep_alive_tick = spdk_get_ticks(); 218 219 SPDK_DEBUGLOG(nvmf, "Ctrlr add keep alive poller\n"); 220 ctrlr->keep_alive_poller = SPDK_POLLER_REGISTER(nvmf_ctrlr_keep_alive_poll, ctrlr, 221 ctrlr->feat.keep_alive_timer.bits.kato * 1000); 222 } 223 } 224 225 static void 226 ctrlr_add_qpair_and_update_rsp(struct spdk_nvmf_qpair *qpair, 227 struct spdk_nvmf_ctrlr *ctrlr, 228 struct spdk_nvmf_fabric_connect_rsp *rsp) 229 { 230 assert(ctrlr->admin_qpair->group->thread == spdk_get_thread()); 231 232 /* check if we would exceed ctrlr connection limit */ 233 if (qpair->qid >= spdk_bit_array_capacity(ctrlr->qpair_mask)) { 234 SPDK_ERRLOG("Requested QID %u but Max QID is %u\n", 235 qpair->qid, spdk_bit_array_capacity(ctrlr->qpair_mask) - 1); 236 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 237 rsp->status.sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER; 238 return; 239 } 240 241 if (spdk_bit_array_get(ctrlr->qpair_mask, qpair->qid)) { 242 SPDK_ERRLOG("Got I/O connect with duplicate QID %u\n", qpair->qid); 243 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 244 rsp->status.sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER; 245 return; 246 } 247 248 qpair->ctrlr = ctrlr; 249 spdk_bit_array_set(ctrlr->qpair_mask, qpair->qid); 250 251 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 252 rsp->status_code_specific.success.cntlid = ctrlr->cntlid; 253 SPDK_DEBUGLOG(nvmf, "connect capsule response: cntlid = 0x%04x\n", 254 rsp->status_code_specific.success.cntlid); 255 256 SPDK_DTRACE_PROBE4(nvmf_ctrlr_add_qpair, qpair, qpair->qid, ctrlr->subsys->subnqn, 257 ctrlr->hostnqn); 258 } 259 260 static void 261 _nvmf_ctrlr_add_admin_qpair(void *ctx) 262 { 263 struct spdk_nvmf_request *req = ctx; 264 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 265 struct spdk_nvmf_qpair *qpair = req->qpair; 266 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 267 268 ctrlr->admin_qpair = qpair; 269 ctrlr->association_timeout = qpair->transport->opts.association_timeout; 270 nvmf_ctrlr_start_keep_alive_timer(ctrlr); 271 ctrlr_add_qpair_and_update_rsp(qpair, ctrlr, rsp); 272 _nvmf_request_complete(req); 273 } 274 275 static void 276 _nvmf_subsystem_add_ctrlr(void *ctx) 277 { 278 struct spdk_nvmf_request *req = ctx; 279 struct spdk_nvmf_qpair *qpair = req->qpair; 280 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 281 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 282 283 if (nvmf_subsystem_add_ctrlr(ctrlr->subsys, ctrlr)) { 284 SPDK_ERRLOG("Unable to add controller to subsystem\n"); 285 spdk_bit_array_free(&ctrlr->qpair_mask); 286 free(ctrlr); 287 qpair->ctrlr = NULL; 288 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 289 spdk_nvmf_request_complete(req); 290 return; 291 } 292 293 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_add_admin_qpair, req); 294 } 295 296 static void 297 nvmf_ctrlr_cdata_init(struct spdk_nvmf_transport *transport, struct spdk_nvmf_subsystem *subsystem, 298 struct spdk_nvmf_ctrlr_data *cdata) 299 { 300 cdata->kas = KAS_DEFAULT_VALUE; 301 cdata->sgls.supported = 1; 302 cdata->sgls.keyed_sgl = 1; 303 cdata->sgls.sgl_offset = 1; 304 cdata->nvmf_specific.ioccsz = sizeof(struct spdk_nvme_cmd) / 16; 305 cdata->nvmf_specific.ioccsz += transport->opts.in_capsule_data_size / 16; 306 cdata->nvmf_specific.iorcsz = sizeof(struct spdk_nvme_cpl) / 16; 307 cdata->nvmf_specific.icdoff = 0; /* offset starts directly after SQE */ 308 cdata->nvmf_specific.ctrattr.ctrlr_model = SPDK_NVMF_CTRLR_MODEL_DYNAMIC; 309 cdata->nvmf_specific.msdbd = 1; 310 311 if (transport->ops->cdata_init) { 312 transport->ops->cdata_init(transport, subsystem, cdata); 313 } 314 } 315 316 static struct spdk_nvmf_ctrlr * 317 nvmf_ctrlr_create(struct spdk_nvmf_subsystem *subsystem, 318 struct spdk_nvmf_request *req, 319 struct spdk_nvmf_fabric_connect_cmd *connect_cmd, 320 struct spdk_nvmf_fabric_connect_data *connect_data) 321 { 322 struct spdk_nvmf_ctrlr *ctrlr; 323 struct spdk_nvmf_transport *transport; 324 struct spdk_nvme_transport_id listen_trid = {}; 325 326 ctrlr = calloc(1, sizeof(*ctrlr)); 327 if (ctrlr == NULL) { 328 SPDK_ERRLOG("Memory allocation failed\n"); 329 return NULL; 330 } 331 332 STAILQ_INIT(&ctrlr->async_events); 333 TAILQ_INIT(&ctrlr->log_head); 334 ctrlr->subsys = subsystem; 335 ctrlr->thread = req->qpair->group->thread; 336 ctrlr->disconnect_in_progress = false; 337 338 transport = req->qpair->transport; 339 ctrlr->qpair_mask = spdk_bit_array_create(transport->opts.max_qpairs_per_ctrlr); 340 if (!ctrlr->qpair_mask) { 341 SPDK_ERRLOG("Failed to allocate controller qpair mask\n"); 342 goto err_qpair_mask; 343 } 344 345 nvmf_ctrlr_cdata_init(transport, subsystem, &ctrlr->cdata); 346 347 /* 348 * KAS: This field indicates the granularity of the Keep Alive Timer in 100ms units. 349 * If this field is cleared to 0h, then Keep Alive is not supported. 350 */ 351 if (ctrlr->cdata.kas) { 352 ctrlr->feat.keep_alive_timer.bits.kato = spdk_divide_round_up(connect_cmd->kato, 353 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS) * 354 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS; 355 } 356 357 ctrlr->feat.async_event_configuration.bits.ns_attr_notice = 1; 358 if (ctrlr->subsys->flags.ana_reporting) { 359 ctrlr->feat.async_event_configuration.bits.ana_change_notice = 1; 360 } 361 ctrlr->feat.volatile_write_cache.bits.wce = 1; 362 363 if (ctrlr->subsys->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 364 /* 365 * If keep-alive timeout is not set, discovery controllers use some 366 * arbitrary high value in order to cleanup stale discovery sessions 367 * 368 * From the 1.0a nvme-of spec: 369 * "The Keep Alive command is reserved for 370 * Discovery controllers. A transport may specify a 371 * fixed Discovery controller activity timeout value 372 * (e.g., 2 minutes). If no commands are received 373 * by a Discovery controller within that time 374 * period, the controller may perform the 375 * actions for Keep Alive Timer expiration". 376 * 377 * From the 1.1 nvme-of spec: 378 * "A host requests an explicit persistent connection 379 * to a Discovery controller and Asynchronous Event Notifications from 380 * the Discovery controller on that persistent connection by specifying 381 * a non-zero Keep Alive Timer value in the Connect command." 382 * 383 * In case non-zero KATO is used, we enable discovery_log_change_notice 384 * otherwise we disable it and use default discovery controller KATO. 385 * KATO is in millisecond. 386 */ 387 if (ctrlr->feat.keep_alive_timer.bits.kato == 0) { 388 ctrlr->feat.keep_alive_timer.bits.kato = NVMF_DISC_KATO_IN_MS; 389 ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice = 0; 390 } else { 391 ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice = 1; 392 } 393 } 394 395 /* Subtract 1 for admin queue, 1 for 0's based */ 396 ctrlr->feat.number_of_queues.bits.ncqr = transport->opts.max_qpairs_per_ctrlr - 1 - 397 1; 398 ctrlr->feat.number_of_queues.bits.nsqr = transport->opts.max_qpairs_per_ctrlr - 1 - 399 1; 400 401 spdk_uuid_copy(&ctrlr->hostid, (struct spdk_uuid *)connect_data->hostid); 402 memcpy(ctrlr->hostnqn, connect_data->hostnqn, sizeof(ctrlr->hostnqn)); 403 404 ctrlr->vcprop.cap.raw = 0; 405 ctrlr->vcprop.cap.bits.cqr = 1; /* NVMe-oF specification required */ 406 ctrlr->vcprop.cap.bits.mqes = transport->opts.max_queue_depth - 407 1; /* max queue depth */ 408 ctrlr->vcprop.cap.bits.ams = 0; /* optional arb mechanisms */ 409 ctrlr->vcprop.cap.bits.to = 1; /* ready timeout - 500 msec units */ 410 ctrlr->vcprop.cap.bits.dstrd = 0; /* fixed to 0 for NVMe-oF */ 411 ctrlr->vcprop.cap.bits.css = SPDK_NVME_CAP_CSS_NVM; /* NVM command set */ 412 ctrlr->vcprop.cap.bits.mpsmin = 0; /* 2 ^ (12 + mpsmin) == 4k */ 413 ctrlr->vcprop.cap.bits.mpsmax = 0; /* 2 ^ (12 + mpsmax) == 4k */ 414 415 /* Version Supported: 1.3 */ 416 ctrlr->vcprop.vs.bits.mjr = 1; 417 ctrlr->vcprop.vs.bits.mnr = 3; 418 ctrlr->vcprop.vs.bits.ter = 0; 419 420 ctrlr->vcprop.cc.raw = 0; 421 ctrlr->vcprop.cc.bits.en = 0; /* Init controller disabled */ 422 423 ctrlr->vcprop.csts.raw = 0; 424 ctrlr->vcprop.csts.bits.rdy = 0; /* Init controller as not ready */ 425 426 SPDK_DEBUGLOG(nvmf, "cap 0x%" PRIx64 "\n", ctrlr->vcprop.cap.raw); 427 SPDK_DEBUGLOG(nvmf, "vs 0x%x\n", ctrlr->vcprop.vs.raw); 428 SPDK_DEBUGLOG(nvmf, "cc 0x%x\n", ctrlr->vcprop.cc.raw); 429 SPDK_DEBUGLOG(nvmf, "csts 0x%x\n", ctrlr->vcprop.csts.raw); 430 431 ctrlr->dif_insert_or_strip = transport->opts.dif_insert_or_strip; 432 433 if (ctrlr->subsys->subtype == SPDK_NVMF_SUBTYPE_NVME) { 434 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &listen_trid) != 0) { 435 SPDK_ERRLOG("Could not get listener transport ID\n"); 436 goto err_listener; 437 } 438 439 ctrlr->listener = nvmf_subsystem_find_listener(ctrlr->subsys, &listen_trid); 440 if (!ctrlr->listener) { 441 SPDK_ERRLOG("Listener was not found\n"); 442 goto err_listener; 443 } 444 } 445 446 req->qpair->ctrlr = ctrlr; 447 spdk_thread_send_msg(subsystem->thread, _nvmf_subsystem_add_ctrlr, req); 448 449 return ctrlr; 450 err_listener: 451 spdk_bit_array_free(&ctrlr->qpair_mask); 452 err_qpair_mask: 453 free(ctrlr); 454 return NULL; 455 } 456 457 static void 458 _nvmf_ctrlr_destruct(void *ctx) 459 { 460 struct spdk_nvmf_ctrlr *ctrlr = ctx; 461 struct spdk_nvmf_reservation_log *log, *log_tmp; 462 struct spdk_nvmf_async_event_completion *event, *event_tmp; 463 464 if (ctrlr->disconnect_in_progress) { 465 SPDK_ERRLOG("freeing ctrlr with disconnect in progress\n"); 466 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_destruct, ctrlr); 467 return; 468 } 469 470 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 471 nvmf_ctrlr_stop_association_timer(ctrlr); 472 spdk_bit_array_free(&ctrlr->qpair_mask); 473 474 TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) { 475 TAILQ_REMOVE(&ctrlr->log_head, log, link); 476 free(log); 477 } 478 STAILQ_FOREACH_SAFE(event, &ctrlr->async_events, link, event_tmp) { 479 STAILQ_REMOVE(&ctrlr->async_events, event, spdk_nvmf_async_event_completion, link); 480 free(event); 481 } 482 free(ctrlr); 483 } 484 485 void 486 nvmf_ctrlr_destruct(struct spdk_nvmf_ctrlr *ctrlr) 487 { 488 nvmf_subsystem_remove_ctrlr(ctrlr->subsys, ctrlr); 489 490 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_destruct, ctrlr); 491 } 492 493 static void 494 nvmf_ctrlr_add_io_qpair(void *ctx) 495 { 496 struct spdk_nvmf_request *req = ctx; 497 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 498 struct spdk_nvmf_qpair *qpair = req->qpair; 499 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 500 struct spdk_nvmf_qpair *admin_qpair = ctrlr->admin_qpair; 501 502 /* Unit test will check qpair->ctrlr after calling spdk_nvmf_ctrlr_connect. 503 * For error case, the value should be NULL. So set it to NULL at first. 504 */ 505 qpair->ctrlr = NULL; 506 507 /* Make sure the controller is not being destroyed. */ 508 if (ctrlr->in_destruct) { 509 SPDK_ERRLOG("Got I/O connect while ctrlr was being destroyed.\n"); 510 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 511 goto end; 512 } 513 514 if (ctrlr->subsys->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 515 SPDK_ERRLOG("I/O connect not allowed on discovery controller\n"); 516 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 517 goto end; 518 } 519 520 if (!ctrlr->vcprop.cc.bits.en) { 521 SPDK_ERRLOG("Got I/O connect before ctrlr was enabled\n"); 522 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 523 goto end; 524 } 525 526 if (1u << ctrlr->vcprop.cc.bits.iosqes != sizeof(struct spdk_nvme_cmd)) { 527 SPDK_ERRLOG("Got I/O connect with invalid IOSQES %u\n", 528 ctrlr->vcprop.cc.bits.iosqes); 529 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 530 goto end; 531 } 532 533 if (1u << ctrlr->vcprop.cc.bits.iocqes != sizeof(struct spdk_nvme_cpl)) { 534 SPDK_ERRLOG("Got I/O connect with invalid IOCQES %u\n", 535 ctrlr->vcprop.cc.bits.iocqes); 536 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 537 goto end; 538 } 539 540 if (admin_qpair->state != SPDK_NVMF_QPAIR_ACTIVE || admin_qpair->group == NULL) { 541 /* There is a chance that admin qpair is being destroyed at this moment due to e.g. 542 * expired keep alive timer. Part of the qpair destruction process is change of qpair's 543 * state to DEACTIVATING and removing it from poll group */ 544 SPDK_ERRLOG("Inactive admin qpair (state %d, group %p)\n", admin_qpair->state, admin_qpair->group); 545 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 546 goto end; 547 } 548 549 ctrlr_add_qpair_and_update_rsp(qpair, ctrlr, rsp); 550 end: 551 spdk_nvmf_request_complete(req); 552 } 553 554 static void 555 _nvmf_ctrlr_add_io_qpair(void *ctx) 556 { 557 struct spdk_nvmf_request *req = ctx; 558 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 559 struct spdk_nvmf_fabric_connect_data *data = req->data; 560 struct spdk_nvmf_ctrlr *ctrlr; 561 struct spdk_nvmf_qpair *qpair = req->qpair; 562 struct spdk_nvmf_qpair *admin_qpair; 563 struct spdk_nvmf_tgt *tgt = qpair->transport->tgt; 564 struct spdk_nvmf_subsystem *subsystem; 565 struct spdk_nvme_transport_id listen_trid = {}; 566 const struct spdk_nvmf_subsystem_listener *listener; 567 568 SPDK_DEBUGLOG(nvmf, "Connect I/O Queue for controller id 0x%x\n", data->cntlid); 569 570 subsystem = spdk_nvmf_tgt_find_subsystem(tgt, data->subnqn); 571 /* We already checked this in spdk_nvmf_ctrlr_connect */ 572 assert(subsystem != NULL); 573 574 ctrlr = nvmf_subsystem_get_ctrlr(subsystem, data->cntlid); 575 if (ctrlr == NULL) { 576 SPDK_ERRLOG("Unknown controller ID 0x%x\n", data->cntlid); 577 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, cntlid); 578 spdk_nvmf_request_complete(req); 579 return; 580 } 581 582 /* fail before passing a message to the controller thread. */ 583 if (ctrlr->in_destruct) { 584 SPDK_ERRLOG("Got I/O connect while ctrlr was being destroyed.\n"); 585 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 586 spdk_nvmf_request_complete(req); 587 return; 588 } 589 590 /* If ANA reporting is enabled, check if I/O connect is on the same listener. */ 591 if (subsystem->flags.ana_reporting) { 592 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &listen_trid) != 0) { 593 SPDK_ERRLOG("Could not get listener transport ID\n"); 594 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 595 spdk_nvmf_request_complete(req); 596 return; 597 } 598 599 listener = nvmf_subsystem_find_listener(subsystem, &listen_trid); 600 if (listener != ctrlr->listener) { 601 SPDK_ERRLOG("I/O connect is on a listener different from admin connect\n"); 602 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 603 spdk_nvmf_request_complete(req); 604 return; 605 } 606 } 607 608 admin_qpair = ctrlr->admin_qpair; 609 if (admin_qpair->state != SPDK_NVMF_QPAIR_ACTIVE || admin_qpair->group == NULL) { 610 /* There is a chance that admin qpair is being destroyed at this moment due to e.g. 611 * expired keep alive timer. Part of the qpair destruction process is change of qpair's 612 * state to DEACTIVATING and removing it from poll group */ 613 SPDK_ERRLOG("Inactive admin qpair (state %d, group %p)\n", admin_qpair->state, admin_qpair->group); 614 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 615 spdk_nvmf_request_complete(req); 616 return; 617 } 618 qpair->ctrlr = ctrlr; 619 spdk_thread_send_msg(admin_qpair->group->thread, nvmf_ctrlr_add_io_qpair, req); 620 } 621 622 static bool 623 nvmf_qpair_access_allowed(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_subsystem *subsystem, 624 const char *hostnqn) 625 { 626 struct spdk_nvme_transport_id listen_trid = {}; 627 628 if (!spdk_nvmf_subsystem_host_allowed(subsystem, hostnqn)) { 629 SPDK_ERRLOG("Subsystem '%s' does not allow host '%s'\n", subsystem->subnqn, hostnqn); 630 return false; 631 } 632 633 if (spdk_nvmf_qpair_get_listen_trid(qpair, &listen_trid)) { 634 SPDK_ERRLOG("Subsystem '%s' is unable to enforce access control due to an internal error.\n", 635 subsystem->subnqn); 636 return false; 637 } 638 639 if (!spdk_nvmf_subsystem_listener_allowed(subsystem, &listen_trid)) { 640 SPDK_ERRLOG("Subsystem '%s' does not allow host '%s' to connect at this address.\n", 641 subsystem->subnqn, hostnqn); 642 return false; 643 } 644 645 return true; 646 } 647 648 static int 649 _nvmf_ctrlr_connect(struct spdk_nvmf_request *req) 650 { 651 struct spdk_nvmf_fabric_connect_data *data = req->data; 652 struct spdk_nvmf_fabric_connect_cmd *cmd = &req->cmd->connect_cmd; 653 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 654 struct spdk_nvmf_qpair *qpair = req->qpair; 655 struct spdk_nvmf_transport *transport = qpair->transport; 656 struct spdk_nvmf_ctrlr *ctrlr; 657 struct spdk_nvmf_subsystem *subsystem; 658 659 SPDK_DEBUGLOG(nvmf, "recfmt 0x%x qid %u sqsize %u\n", 660 cmd->recfmt, cmd->qid, cmd->sqsize); 661 662 SPDK_DEBUGLOG(nvmf, "Connect data:\n"); 663 SPDK_DEBUGLOG(nvmf, " cntlid: 0x%04x\n", data->cntlid); 664 SPDK_DEBUGLOG(nvmf, " hostid: %08x-%04x-%04x-%02x%02x-%04x%08x ***\n", 665 ntohl(*(uint32_t *)&data->hostid[0]), 666 ntohs(*(uint16_t *)&data->hostid[4]), 667 ntohs(*(uint16_t *)&data->hostid[6]), 668 data->hostid[8], 669 data->hostid[9], 670 ntohs(*(uint16_t *)&data->hostid[10]), 671 ntohl(*(uint32_t *)&data->hostid[12])); 672 SPDK_DEBUGLOG(nvmf, " subnqn: \"%s\"\n", data->subnqn); 673 SPDK_DEBUGLOG(nvmf, " hostnqn: \"%s\"\n", data->hostnqn); 674 675 subsystem = spdk_nvmf_tgt_find_subsystem(transport->tgt, data->subnqn); 676 if (!subsystem) { 677 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 678 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 679 } 680 681 if (cmd->recfmt != 0) { 682 SPDK_ERRLOG("Connect command unsupported RECFMT %u\n", cmd->recfmt); 683 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 684 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INCOMPATIBLE_FORMAT; 685 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 686 } 687 688 /* 689 * SQSIZE is a 0-based value, so it must be at least 1 (minimum queue depth is 2) and 690 * strictly less than max_aq_depth (admin queues) or max_queue_depth (io queues). 691 */ 692 if (cmd->sqsize == 0) { 693 SPDK_ERRLOG("Invalid SQSIZE = 0\n"); 694 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 695 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 696 } 697 698 if (cmd->qid == 0) { 699 if (cmd->sqsize >= transport->opts.max_aq_depth) { 700 SPDK_ERRLOG("Invalid SQSIZE for admin queue %u (min 1, max %u)\n", 701 cmd->sqsize, transport->opts.max_aq_depth - 1); 702 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 703 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 704 } 705 } else if (cmd->sqsize >= transport->opts.max_queue_depth) { 706 SPDK_ERRLOG("Invalid SQSIZE %u (min 1, max %u)\n", 707 cmd->sqsize, transport->opts.max_queue_depth - 1); 708 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 709 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 710 } 711 712 qpair->sq_head_max = cmd->sqsize; 713 qpair->qid = cmd->qid; 714 715 if (0 == qpair->qid) { 716 qpair->group->stat.admin_qpairs++; 717 qpair->group->stat.current_admin_qpairs++; 718 } else { 719 qpair->group->stat.io_qpairs++; 720 qpair->group->stat.current_io_qpairs++; 721 } 722 723 if (cmd->qid == 0) { 724 SPDK_DEBUGLOG(nvmf, "Connect Admin Queue for controller ID 0x%x\n", data->cntlid); 725 726 if (data->cntlid != 0xFFFF) { 727 /* This NVMf target only supports dynamic mode. */ 728 SPDK_ERRLOG("The NVMf target only supports dynamic mode (CNTLID = 0x%x).\n", data->cntlid); 729 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, cntlid); 730 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 731 } 732 733 /* Establish a new ctrlr */ 734 ctrlr = nvmf_ctrlr_create(subsystem, req, cmd, data); 735 if (!ctrlr) { 736 SPDK_ERRLOG("nvmf_ctrlr_create() failed\n"); 737 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 738 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 739 } else { 740 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 741 } 742 } else { 743 spdk_thread_send_msg(subsystem->thread, _nvmf_ctrlr_add_io_qpair, req); 744 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 745 } 746 } 747 748 static inline bool 749 nvmf_request_is_fabric_connect(struct spdk_nvmf_request *req) 750 { 751 return req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC && 752 req->cmd->nvmf_cmd.fctype == SPDK_NVMF_FABRIC_COMMAND_CONNECT; 753 } 754 755 static struct spdk_nvmf_subsystem_poll_group * 756 nvmf_subsystem_pg_from_connect_cmd(struct spdk_nvmf_request *req) 757 { 758 struct spdk_nvmf_fabric_connect_data *data; 759 struct spdk_nvmf_subsystem *subsystem; 760 struct spdk_nvmf_tgt *tgt; 761 762 assert(nvmf_request_is_fabric_connect(req)); 763 assert(req->qpair->ctrlr == NULL); 764 765 data = req->data; 766 tgt = req->qpair->transport->tgt; 767 768 subsystem = spdk_nvmf_tgt_find_subsystem(tgt, data->subnqn); 769 if (subsystem == NULL) { 770 return NULL; 771 } 772 773 return &req->qpair->group->sgroups[subsystem->id]; 774 } 775 776 int 777 spdk_nvmf_ctrlr_connect(struct spdk_nvmf_request *req) 778 { 779 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 780 struct spdk_nvmf_qpair *qpair = req->qpair; 781 struct spdk_nvmf_subsystem_poll_group *sgroup; 782 enum spdk_nvmf_request_exec_status status; 783 784 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 785 if (!sgroup) { 786 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 787 status = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 788 goto out; 789 } 790 791 sgroup->mgmt_io_outstanding++; 792 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 793 794 status = _nvmf_ctrlr_connect(req); 795 796 out: 797 if (status == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 798 _nvmf_request_complete(req); 799 } 800 801 return status; 802 } 803 804 static int nvmf_ctrlr_cmd_connect(struct spdk_nvmf_request *req); 805 806 static int 807 retry_connect(void *arg) 808 { 809 struct spdk_nvmf_request *req = arg; 810 struct spdk_nvmf_subsystem_poll_group *sgroup; 811 int rc; 812 813 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 814 sgroup->mgmt_io_outstanding++; 815 spdk_poller_unregister(&req->poller); 816 rc = nvmf_ctrlr_cmd_connect(req); 817 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 818 _nvmf_request_complete(req); 819 } 820 return SPDK_POLLER_BUSY; 821 } 822 823 static int 824 nvmf_ctrlr_cmd_connect(struct spdk_nvmf_request *req) 825 { 826 struct spdk_nvmf_fabric_connect_data *data = req->data; 827 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 828 struct spdk_nvmf_transport *transport = req->qpair->transport; 829 struct spdk_nvmf_subsystem *subsystem; 830 831 if (req->length < sizeof(struct spdk_nvmf_fabric_connect_data)) { 832 SPDK_ERRLOG("Connect command data length 0x%x too small\n", req->length); 833 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 834 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 835 } 836 837 subsystem = spdk_nvmf_tgt_find_subsystem(transport->tgt, data->subnqn); 838 if (!subsystem) { 839 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 840 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 841 } 842 843 if ((subsystem->state == SPDK_NVMF_SUBSYSTEM_INACTIVE) || 844 (subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSING) || 845 (subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSED) || 846 (subsystem->state == SPDK_NVMF_SUBSYSTEM_DEACTIVATING)) { 847 struct spdk_nvmf_subsystem_poll_group *sgroup; 848 849 if (req->timeout_tsc == 0) { 850 /* We will only retry the request up to 1 second. */ 851 req->timeout_tsc = spdk_get_ticks() + spdk_get_ticks_hz(); 852 } else if (spdk_get_ticks() > req->timeout_tsc) { 853 SPDK_ERRLOG("Subsystem '%s' was not ready for 1 second\n", subsystem->subnqn); 854 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 855 rsp->status.sc = SPDK_NVMF_FABRIC_SC_CONTROLLER_BUSY; 856 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 857 } 858 859 /* Subsystem is not ready to handle a connect. Use a poller to retry it 860 * again later. Decrement the mgmt_io_outstanding to avoid the 861 * subsystem waiting for this command to complete before unpausing. 862 */ 863 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 864 sgroup->mgmt_io_outstanding--; 865 SPDK_DEBUGLOG(nvmf, "Subsystem '%s' is not ready for connect, retrying...\n", subsystem->subnqn); 866 req->poller = SPDK_POLLER_REGISTER(retry_connect, req, 100); 867 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 868 } 869 870 /* Ensure that hostnqn is null terminated */ 871 if (!memchr(data->hostnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 872 SPDK_ERRLOG("Connect HOSTNQN is not null terminated\n"); 873 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, hostnqn); 874 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 875 } 876 877 if (!nvmf_qpair_access_allowed(req->qpair, subsystem, data->hostnqn)) { 878 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 879 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_HOST; 880 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 881 } 882 883 return _nvmf_ctrlr_connect(req); 884 } 885 886 static int 887 nvmf_ctrlr_association_remove(void *ctx) 888 { 889 struct spdk_nvmf_ctrlr *ctrlr = ctx; 890 int rc; 891 892 SPDK_DEBUGLOG(nvmf, "Disconnecting host from subsystem %s due to association timeout.\n", 893 ctrlr->subsys->subnqn); 894 895 rc = spdk_nvmf_qpair_disconnect(ctrlr->admin_qpair, NULL, NULL); 896 if (rc < 0) { 897 SPDK_ERRLOG("Fail to disconnect admin ctrlr qpair\n"); 898 assert(false); 899 } 900 901 nvmf_ctrlr_stop_association_timer(ctrlr); 902 return 1; 903 } 904 905 static void 906 nvmf_ctrlr_cc_shn_done(struct spdk_io_channel_iter *i, int status) 907 { 908 struct spdk_nvmf_ctrlr *ctrlr = spdk_io_channel_iter_get_ctx(i); 909 910 if (status < 0) { 911 SPDK_ERRLOG("Fail to disconnect io ctrlr qpairs\n"); 912 assert(false); 913 } 914 915 ctrlr->vcprop.csts.bits.shst = SPDK_NVME_SHST_COMPLETE; 916 917 /* After CC.EN transitions to 0 (due to shutdown or reset), the association 918 * between the host and controller shall be preserved for at least 2 minutes */ 919 if (ctrlr->association_timer) { 920 SPDK_DEBUGLOG(nvmf, "Association timer already set\n"); 921 nvmf_ctrlr_stop_association_timer(ctrlr); 922 } 923 if (ctrlr->association_timeout) { 924 ctrlr->association_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_association_remove, ctrlr, 925 ctrlr->association_timeout * 1000); 926 } 927 ctrlr->disconnect_in_progress = false; 928 } 929 930 static void 931 nvmf_ctrlr_cc_reset_done(struct spdk_io_channel_iter *i, int status) 932 { 933 struct spdk_nvmf_ctrlr *ctrlr = spdk_io_channel_iter_get_ctx(i); 934 935 if (status < 0) { 936 SPDK_ERRLOG("Fail to disconnect io ctrlr qpairs\n"); 937 assert(false); 938 } 939 940 /* Only a subset of the registers are cleared out on a reset */ 941 ctrlr->vcprop.cc.raw = 0; 942 ctrlr->vcprop.csts.raw = 0; 943 944 /* After CC.EN transitions to 0 (due to shutdown or reset), the association 945 * between the host and controller shall be preserved for at least 2 minutes */ 946 if (ctrlr->association_timer) { 947 SPDK_DEBUGLOG(nvmf, "Association timer already set\n"); 948 nvmf_ctrlr_stop_association_timer(ctrlr); 949 } 950 if (ctrlr->association_timeout) { 951 ctrlr->association_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_association_remove, ctrlr, 952 ctrlr->association_timeout * 1000); 953 } 954 ctrlr->disconnect_in_progress = false; 955 } 956 957 const struct spdk_nvmf_registers * 958 spdk_nvmf_ctrlr_get_regs(struct spdk_nvmf_ctrlr *ctrlr) 959 { 960 return &ctrlr->vcprop; 961 } 962 963 static uint64_t 964 nvmf_prop_get_cap(struct spdk_nvmf_ctrlr *ctrlr) 965 { 966 return ctrlr->vcprop.cap.raw; 967 } 968 969 static uint64_t 970 nvmf_prop_get_vs(struct spdk_nvmf_ctrlr *ctrlr) 971 { 972 return ctrlr->vcprop.vs.raw; 973 } 974 975 static uint64_t 976 nvmf_prop_get_cc(struct spdk_nvmf_ctrlr *ctrlr) 977 { 978 return ctrlr->vcprop.cc.raw; 979 } 980 981 static bool 982 nvmf_prop_set_cc(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 983 { 984 union spdk_nvme_cc_register cc, diff; 985 986 cc.raw = value; 987 988 SPDK_DEBUGLOG(nvmf, "cur CC: 0x%08x\n", ctrlr->vcprop.cc.raw); 989 SPDK_DEBUGLOG(nvmf, "new CC: 0x%08x\n", cc.raw); 990 991 /* 992 * Calculate which bits changed between the current and new CC. 993 * Mark each bit as 0 once it is handled to determine if any unhandled bits were changed. 994 */ 995 diff.raw = cc.raw ^ ctrlr->vcprop.cc.raw; 996 997 if (diff.bits.en) { 998 if (cc.bits.en) { 999 SPDK_DEBUGLOG(nvmf, "Property Set CC Enable!\n"); 1000 nvmf_ctrlr_stop_association_timer(ctrlr); 1001 1002 ctrlr->vcprop.cc.bits.en = 1; 1003 ctrlr->vcprop.csts.bits.rdy = 1; 1004 } else { 1005 SPDK_DEBUGLOG(nvmf, "Property Set CC Disable!\n"); 1006 ctrlr->vcprop.cc.bits.en = 0; 1007 ctrlr->disconnect_in_progress = true; 1008 spdk_for_each_channel(ctrlr->subsys->tgt, 1009 nvmf_ctrlr_disconnect_io_qpairs_on_pg, 1010 ctrlr, 1011 nvmf_ctrlr_cc_reset_done); 1012 } 1013 diff.bits.en = 0; 1014 } 1015 1016 if (diff.bits.shn) { 1017 if (cc.bits.shn == SPDK_NVME_SHN_NORMAL || 1018 cc.bits.shn == SPDK_NVME_SHN_ABRUPT) { 1019 SPDK_DEBUGLOG(nvmf, "Property Set CC Shutdown %u%ub!\n", 1020 cc.bits.shn >> 1, cc.bits.shn & 1); 1021 ctrlr->vcprop.cc.bits.shn = cc.bits.shn; 1022 ctrlr->disconnect_in_progress = true; 1023 spdk_for_each_channel(ctrlr->subsys->tgt, 1024 nvmf_ctrlr_disconnect_io_qpairs_on_pg, 1025 ctrlr, 1026 nvmf_ctrlr_cc_shn_done); 1027 1028 /* From the time a shutdown is initiated the controller shall disable 1029 * Keep Alive timer */ 1030 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 1031 } else if (cc.bits.shn == 0) { 1032 ctrlr->vcprop.cc.bits.shn = 0; 1033 } else { 1034 SPDK_ERRLOG("Prop Set CC: Invalid SHN value %u%ub\n", 1035 cc.bits.shn >> 1, cc.bits.shn & 1); 1036 return false; 1037 } 1038 diff.bits.shn = 0; 1039 } 1040 1041 if (diff.bits.iosqes) { 1042 SPDK_DEBUGLOG(nvmf, "Prop Set IOSQES = %u (%u bytes)\n", 1043 cc.bits.iosqes, 1u << cc.bits.iosqes); 1044 ctrlr->vcprop.cc.bits.iosqes = cc.bits.iosqes; 1045 diff.bits.iosqes = 0; 1046 } 1047 1048 if (diff.bits.iocqes) { 1049 SPDK_DEBUGLOG(nvmf, "Prop Set IOCQES = %u (%u bytes)\n", 1050 cc.bits.iocqes, 1u << cc.bits.iocqes); 1051 ctrlr->vcprop.cc.bits.iocqes = cc.bits.iocqes; 1052 diff.bits.iocqes = 0; 1053 } 1054 1055 if (diff.bits.ams) { 1056 SPDK_ERRLOG("Arbitration Mechanism Selected (AMS) 0x%x not supported!\n", cc.bits.ams); 1057 return false; 1058 } 1059 1060 if (diff.bits.mps) { 1061 SPDK_ERRLOG("Memory Page Size (MPS) %u KiB not supported!\n", (1 << (2 + cc.bits.mps))); 1062 return false; 1063 } 1064 1065 if (diff.bits.css) { 1066 SPDK_ERRLOG("I/O Command Set Selected (CSS) 0x%x not supported!\n", cc.bits.css); 1067 return false; 1068 } 1069 1070 if (diff.raw != 0) { 1071 /* Print an error message, but don't fail the command in this case. 1072 * If we did want to fail in this case, we'd need to ensure we acted 1073 * on no other bits or the initiator gets confused. */ 1074 SPDK_ERRLOG("Prop Set CC toggled reserved bits 0x%x!\n", diff.raw); 1075 } 1076 1077 return true; 1078 } 1079 1080 static uint64_t 1081 nvmf_prop_get_csts(struct spdk_nvmf_ctrlr *ctrlr) 1082 { 1083 return ctrlr->vcprop.csts.raw; 1084 } 1085 1086 static uint64_t 1087 nvmf_prop_get_aqa(struct spdk_nvmf_ctrlr *ctrlr) 1088 { 1089 return ctrlr->vcprop.aqa.raw; 1090 } 1091 1092 static bool 1093 nvmf_prop_set_aqa(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1094 { 1095 union spdk_nvme_aqa_register aqa; 1096 1097 aqa.raw = value; 1098 1099 if (aqa.bits.asqs < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES - 1 || 1100 aqa.bits.acqs < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES - 1 || 1101 aqa.bits.reserved1 != 0 || aqa.bits.reserved2 != 0) { 1102 return false; 1103 } 1104 1105 ctrlr->vcprop.aqa.raw = value; 1106 1107 return true; 1108 } 1109 1110 static uint64_t 1111 nvmf_prop_get_asq(struct spdk_nvmf_ctrlr *ctrlr) 1112 { 1113 return ctrlr->vcprop.asq; 1114 } 1115 1116 static bool 1117 nvmf_prop_set_asq_lower(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1118 { 1119 ctrlr->vcprop.asq = (ctrlr->vcprop.asq & (0xFFFFFFFFULL << 32ULL)) | value; 1120 1121 return true; 1122 } 1123 1124 static bool 1125 nvmf_prop_set_asq_upper(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1126 { 1127 ctrlr->vcprop.asq = (ctrlr->vcprop.asq & 0xFFFFFFFFULL) | ((uint64_t)value << 32ULL); 1128 1129 return true; 1130 } 1131 1132 static uint64_t 1133 nvmf_prop_get_acq(struct spdk_nvmf_ctrlr *ctrlr) 1134 { 1135 return ctrlr->vcprop.acq; 1136 } 1137 1138 static bool 1139 nvmf_prop_set_acq_lower(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1140 { 1141 ctrlr->vcprop.acq = (ctrlr->vcprop.acq & (0xFFFFFFFFULL << 32ULL)) | value; 1142 1143 return true; 1144 } 1145 1146 static bool 1147 nvmf_prop_set_acq_upper(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1148 { 1149 ctrlr->vcprop.acq = (ctrlr->vcprop.acq & 0xFFFFFFFFULL) | ((uint64_t)value << 32ULL); 1150 1151 return true; 1152 } 1153 1154 struct nvmf_prop { 1155 uint32_t ofst; 1156 uint8_t size; 1157 char name[11]; 1158 uint64_t (*get_cb)(struct spdk_nvmf_ctrlr *ctrlr); 1159 bool (*set_cb)(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value); 1160 bool (*set_upper_cb)(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value); 1161 }; 1162 1163 #define PROP(field, size, get_cb, set_cb, set_upper_cb) \ 1164 { \ 1165 offsetof(struct spdk_nvme_registers, field), \ 1166 size, \ 1167 #field, \ 1168 get_cb, set_cb, set_upper_cb \ 1169 } 1170 1171 static const struct nvmf_prop nvmf_props[] = { 1172 PROP(cap, 8, nvmf_prop_get_cap, NULL, NULL), 1173 PROP(vs, 4, nvmf_prop_get_vs, NULL, NULL), 1174 PROP(cc, 4, nvmf_prop_get_cc, nvmf_prop_set_cc, NULL), 1175 PROP(csts, 4, nvmf_prop_get_csts, NULL, NULL), 1176 PROP(aqa, 4, nvmf_prop_get_aqa, nvmf_prop_set_aqa, NULL), 1177 PROP(asq, 8, nvmf_prop_get_asq, nvmf_prop_set_asq_lower, nvmf_prop_set_asq_upper), 1178 PROP(acq, 8, nvmf_prop_get_acq, nvmf_prop_set_acq_lower, nvmf_prop_set_acq_upper), 1179 }; 1180 1181 static const struct nvmf_prop * 1182 find_prop(uint32_t ofst, uint8_t size) 1183 { 1184 size_t i; 1185 1186 for (i = 0; i < SPDK_COUNTOF(nvmf_props); i++) { 1187 const struct nvmf_prop *prop = &nvmf_props[i]; 1188 1189 if ((ofst >= prop->ofst) && (ofst + size <= prop->ofst + prop->size)) { 1190 return prop; 1191 } 1192 } 1193 1194 return NULL; 1195 } 1196 1197 static int 1198 nvmf_property_get(struct spdk_nvmf_request *req) 1199 { 1200 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1201 struct spdk_nvmf_fabric_prop_get_cmd *cmd = &req->cmd->prop_get_cmd; 1202 struct spdk_nvmf_fabric_prop_get_rsp *response = &req->rsp->prop_get_rsp; 1203 const struct nvmf_prop *prop; 1204 uint8_t size; 1205 1206 response->status.sc = 0; 1207 response->value.u64 = 0; 1208 1209 SPDK_DEBUGLOG(nvmf, "size %d, offset 0x%x\n", 1210 cmd->attrib.size, cmd->ofst); 1211 1212 switch (cmd->attrib.size) { 1213 case SPDK_NVMF_PROP_SIZE_4: 1214 size = 4; 1215 break; 1216 case SPDK_NVMF_PROP_SIZE_8: 1217 size = 8; 1218 break; 1219 default: 1220 SPDK_ERRLOG("Invalid size value %d\n", cmd->attrib.size); 1221 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1222 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1223 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1224 } 1225 1226 prop = find_prop(cmd->ofst, size); 1227 if (prop == NULL || prop->get_cb == NULL) { 1228 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1229 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1230 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1231 } 1232 1233 SPDK_DEBUGLOG(nvmf, "name: %s\n", prop->name); 1234 1235 response->value.u64 = prop->get_cb(ctrlr); 1236 1237 if (size != prop->size) { 1238 /* The size must be 4 and the prop->size is 8. Figure out which part of the property to read. */ 1239 assert(size == 4); 1240 assert(prop->size == 8); 1241 1242 if (cmd->ofst == prop->ofst) { 1243 /* Keep bottom 4 bytes only */ 1244 response->value.u64 &= 0xFFFFFFFF; 1245 } else { 1246 /* Keep top 4 bytes only */ 1247 response->value.u64 >>= 32; 1248 } 1249 } 1250 1251 SPDK_DEBUGLOG(nvmf, "response value: 0x%" PRIx64 "\n", response->value.u64); 1252 1253 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1254 } 1255 1256 static int 1257 nvmf_property_set(struct spdk_nvmf_request *req) 1258 { 1259 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1260 struct spdk_nvmf_fabric_prop_set_cmd *cmd = &req->cmd->prop_set_cmd; 1261 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1262 const struct nvmf_prop *prop; 1263 uint64_t value; 1264 uint8_t size; 1265 bool ret; 1266 1267 SPDK_DEBUGLOG(nvmf, "size %d, offset 0x%x, value 0x%" PRIx64 "\n", 1268 cmd->attrib.size, cmd->ofst, cmd->value.u64); 1269 1270 switch (cmd->attrib.size) { 1271 case SPDK_NVMF_PROP_SIZE_4: 1272 size = 4; 1273 break; 1274 case SPDK_NVMF_PROP_SIZE_8: 1275 size = 8; 1276 break; 1277 default: 1278 SPDK_ERRLOG("Invalid size value %d\n", cmd->attrib.size); 1279 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1280 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1281 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1282 } 1283 1284 prop = find_prop(cmd->ofst, size); 1285 if (prop == NULL || prop->set_cb == NULL) { 1286 SPDK_ERRLOG("Invalid offset 0x%x\n", cmd->ofst); 1287 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1288 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1289 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1290 } 1291 1292 SPDK_DEBUGLOG(nvmf, "name: %s\n", prop->name); 1293 1294 value = cmd->value.u64; 1295 1296 if (prop->size == 4) { 1297 ret = prop->set_cb(ctrlr, (uint32_t)value); 1298 } else if (size != prop->size) { 1299 /* The size must be 4 and the prop->size is 8. Figure out which part of the property to write. */ 1300 assert(size == 4); 1301 assert(prop->size == 8); 1302 1303 if (cmd->ofst == prop->ofst) { 1304 ret = prop->set_cb(ctrlr, (uint32_t)value); 1305 } else { 1306 ret = prop->set_upper_cb(ctrlr, (uint32_t)value); 1307 } 1308 } else { 1309 ret = prop->set_cb(ctrlr, (uint32_t)value); 1310 if (ret) { 1311 ret = prop->set_upper_cb(ctrlr, (uint32_t)(value >> 32)); 1312 } 1313 } 1314 1315 if (!ret) { 1316 SPDK_ERRLOG("prop set_cb failed\n"); 1317 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1318 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1319 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1320 } 1321 1322 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1323 } 1324 1325 static int 1326 nvmf_ctrlr_set_features_arbitration(struct spdk_nvmf_request *req) 1327 { 1328 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1329 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1330 1331 SPDK_DEBUGLOG(nvmf, "Set Features - Arbitration (cdw11 = 0x%0x)\n", cmd->cdw11); 1332 1333 ctrlr->feat.arbitration.raw = cmd->cdw11; 1334 ctrlr->feat.arbitration.bits.reserved = 0; 1335 1336 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1337 } 1338 1339 static int 1340 nvmf_ctrlr_set_features_power_management(struct spdk_nvmf_request *req) 1341 { 1342 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1343 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1344 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1345 1346 SPDK_DEBUGLOG(nvmf, "Set Features - Power Management (cdw11 = 0x%0x)\n", cmd->cdw11); 1347 1348 /* Only PS = 0 is allowed, since we report NPSS = 0 */ 1349 if (cmd->cdw11_bits.feat_power_management.bits.ps != 0) { 1350 SPDK_ERRLOG("Invalid power state %u\n", cmd->cdw11_bits.feat_power_management.bits.ps); 1351 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1352 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1353 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1354 } 1355 1356 ctrlr->feat.power_management.raw = cmd->cdw11; 1357 ctrlr->feat.power_management.bits.reserved = 0; 1358 1359 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1360 } 1361 1362 static bool 1363 temp_threshold_opts_valid(const union spdk_nvme_feat_temperature_threshold *opts) 1364 { 1365 /* 1366 * Valid TMPSEL values: 1367 * 0000b - 1000b: temperature sensors 1368 * 1111b: set all implemented temperature sensors 1369 */ 1370 if (opts->bits.tmpsel >= 9 && opts->bits.tmpsel != 15) { 1371 /* 1001b - 1110b: reserved */ 1372 SPDK_ERRLOG("Invalid TMPSEL %u\n", opts->bits.tmpsel); 1373 return false; 1374 } 1375 1376 /* 1377 * Valid THSEL values: 1378 * 00b: over temperature threshold 1379 * 01b: under temperature threshold 1380 */ 1381 if (opts->bits.thsel > 1) { 1382 /* 10b - 11b: reserved */ 1383 SPDK_ERRLOG("Invalid THSEL %u\n", opts->bits.thsel); 1384 return false; 1385 } 1386 1387 return true; 1388 } 1389 1390 static int 1391 nvmf_ctrlr_set_features_temperature_threshold(struct spdk_nvmf_request *req) 1392 { 1393 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1394 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1395 1396 SPDK_DEBUGLOG(nvmf, "Set Features - Temperature Threshold (cdw11 = 0x%0x)\n", cmd->cdw11); 1397 1398 if (!temp_threshold_opts_valid(&cmd->cdw11_bits.feat_temp_threshold)) { 1399 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1400 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1401 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1402 } 1403 1404 /* TODO: no sensors implemented - ignore new values */ 1405 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1406 } 1407 1408 static int 1409 nvmf_ctrlr_get_features_temperature_threshold(struct spdk_nvmf_request *req) 1410 { 1411 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1412 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1413 1414 SPDK_DEBUGLOG(nvmf, "Get Features - Temperature Threshold (cdw11 = 0x%0x)\n", cmd->cdw11); 1415 1416 if (!temp_threshold_opts_valid(&cmd->cdw11_bits.feat_temp_threshold)) { 1417 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1418 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1419 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1420 } 1421 1422 /* TODO: no sensors implemented - return 0 for all thresholds */ 1423 rsp->cdw0 = 0; 1424 1425 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1426 } 1427 1428 static int 1429 nvmf_ctrlr_set_features_error_recovery(struct spdk_nvmf_request *req) 1430 { 1431 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1432 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1433 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1434 1435 SPDK_DEBUGLOG(nvmf, "Set Features - Error Recovery (cdw11 = 0x%0x)\n", cmd->cdw11); 1436 1437 if (cmd->cdw11_bits.feat_error_recovery.bits.dulbe) { 1438 /* 1439 * Host is not allowed to set this bit, since we don't advertise it in 1440 * Identify Namespace. 1441 */ 1442 SPDK_ERRLOG("Host set unsupported DULBE bit\n"); 1443 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1444 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1445 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1446 } 1447 1448 ctrlr->feat.error_recovery.raw = cmd->cdw11; 1449 ctrlr->feat.error_recovery.bits.reserved = 0; 1450 1451 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1452 } 1453 1454 static int 1455 nvmf_ctrlr_set_features_volatile_write_cache(struct spdk_nvmf_request *req) 1456 { 1457 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1458 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1459 1460 SPDK_DEBUGLOG(nvmf, "Set Features - Volatile Write Cache (cdw11 = 0x%0x)\n", cmd->cdw11); 1461 1462 ctrlr->feat.volatile_write_cache.raw = cmd->cdw11; 1463 ctrlr->feat.volatile_write_cache.bits.reserved = 0; 1464 1465 SPDK_DEBUGLOG(nvmf, "Set Features - Volatile Write Cache %s\n", 1466 ctrlr->feat.volatile_write_cache.bits.wce ? "Enabled" : "Disabled"); 1467 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1468 } 1469 1470 static int 1471 nvmf_ctrlr_set_features_write_atomicity(struct spdk_nvmf_request *req) 1472 { 1473 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1474 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1475 1476 SPDK_DEBUGLOG(nvmf, "Set Features - Write Atomicity (cdw11 = 0x%0x)\n", cmd->cdw11); 1477 1478 ctrlr->feat.write_atomicity.raw = cmd->cdw11; 1479 ctrlr->feat.write_atomicity.bits.reserved = 0; 1480 1481 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1482 } 1483 1484 static int 1485 nvmf_ctrlr_set_features_host_identifier(struct spdk_nvmf_request *req) 1486 { 1487 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1488 1489 SPDK_ERRLOG("Set Features - Host Identifier not allowed\n"); 1490 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 1491 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1492 } 1493 1494 static int 1495 nvmf_ctrlr_get_features_host_identifier(struct spdk_nvmf_request *req) 1496 { 1497 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1498 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1499 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1500 1501 SPDK_DEBUGLOG(nvmf, "Get Features - Host Identifier\n"); 1502 1503 if (!cmd->cdw11_bits.feat_host_identifier.bits.exhid) { 1504 /* NVMe over Fabrics requires EXHID=1 (128-bit/16-byte host ID) */ 1505 SPDK_ERRLOG("Get Features - Host Identifier with EXHID=0 not allowed\n"); 1506 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1507 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1508 } 1509 1510 if (req->data == NULL || req->length < sizeof(ctrlr->hostid)) { 1511 SPDK_ERRLOG("Invalid data buffer for Get Features - Host Identifier\n"); 1512 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1513 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1514 } 1515 1516 spdk_uuid_copy((struct spdk_uuid *)req->data, &ctrlr->hostid); 1517 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1518 } 1519 1520 static int 1521 nvmf_ctrlr_get_features_reservation_notification_mask(struct spdk_nvmf_request *req) 1522 { 1523 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1524 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1525 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1526 struct spdk_nvmf_ns *ns; 1527 1528 SPDK_DEBUGLOG(nvmf, "get Features - Reservation Notificaton Mask\n"); 1529 1530 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1531 SPDK_ERRLOG("get Features - Invalid Namespace ID\n"); 1532 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1533 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1534 } 1535 1536 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, cmd->nsid); 1537 if (ns == NULL) { 1538 SPDK_ERRLOG("Set Features - Invalid Namespace ID\n"); 1539 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1540 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1541 } 1542 rsp->cdw0 = ns->mask; 1543 1544 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1545 } 1546 1547 static int 1548 nvmf_ctrlr_set_features_reservation_notification_mask(struct spdk_nvmf_request *req) 1549 { 1550 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1551 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 1552 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1553 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1554 struct spdk_nvmf_ns *ns; 1555 1556 SPDK_DEBUGLOG(nvmf, "Set Features - Reservation Notificaton Mask\n"); 1557 1558 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1559 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 1560 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 1561 ns->mask = cmd->cdw11; 1562 } 1563 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1564 } 1565 1566 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, cmd->nsid); 1567 if (ns == NULL) { 1568 SPDK_ERRLOG("Set Features - Invalid Namespace ID\n"); 1569 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1570 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1571 } 1572 ns->mask = cmd->cdw11; 1573 1574 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1575 } 1576 1577 static int 1578 nvmf_ctrlr_get_features_reservation_persistence(struct spdk_nvmf_request *req) 1579 { 1580 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1581 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1582 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1583 struct spdk_nvmf_ns *ns; 1584 1585 SPDK_DEBUGLOG(nvmf, "Get Features - Reservation Persistence\n"); 1586 1587 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, cmd->nsid); 1588 /* NSID with SPDK_NVME_GLOBAL_NS_TAG (=0xffffffff) also included */ 1589 if (ns == NULL) { 1590 SPDK_ERRLOG("Get Features - Invalid Namespace ID\n"); 1591 response->status.sct = SPDK_NVME_SCT_GENERIC; 1592 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1593 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1594 } 1595 1596 response->cdw0 = ns->ptpl_activated; 1597 1598 response->status.sct = SPDK_NVME_SCT_GENERIC; 1599 response->status.sc = SPDK_NVME_SC_SUCCESS; 1600 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1601 } 1602 1603 static int 1604 nvmf_ctrlr_set_features_reservation_persistence(struct spdk_nvmf_request *req) 1605 { 1606 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1607 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1608 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1609 struct spdk_nvmf_ns *ns; 1610 bool ptpl; 1611 1612 SPDK_DEBUGLOG(nvmf, "Set Features - Reservation Persistence\n"); 1613 1614 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, cmd->nsid); 1615 ptpl = cmd->cdw11_bits.feat_rsv_persistence.bits.ptpl; 1616 1617 if (cmd->nsid != SPDK_NVME_GLOBAL_NS_TAG && ns && ns->ptpl_file) { 1618 ns->ptpl_activated = ptpl; 1619 } else if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1620 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns && ns->ptpl_file; 1621 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 1622 ns->ptpl_activated = ptpl; 1623 } 1624 } else { 1625 SPDK_ERRLOG("Set Features - Invalid Namespace ID or Reservation Configuration\n"); 1626 response->status.sct = SPDK_NVME_SCT_GENERIC; 1627 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1628 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1629 } 1630 1631 /* TODO: Feature not changeable for now */ 1632 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1633 response->status.sc = SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE; 1634 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1635 } 1636 1637 static int 1638 nvmf_ctrlr_set_features_host_behavior_support(struct spdk_nvmf_request *req) 1639 { 1640 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1641 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1642 struct spdk_nvme_host_behavior *host_behavior; 1643 1644 SPDK_DEBUGLOG(nvmf, "Set Features - Host Behavior Support\n"); 1645 if (req->iovcnt != 1) { 1646 SPDK_ERRLOG("Host Behavior Support invalid iovcnt: %d\n", req->iovcnt); 1647 response->status.sct = SPDK_NVME_SCT_GENERIC; 1648 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1649 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1650 } 1651 if (req->iov[0].iov_len != sizeof(struct spdk_nvme_host_behavior)) { 1652 SPDK_ERRLOG("Host Behavior Support invalid iov_len: %zd\n", req->iov[0].iov_len); 1653 response->status.sct = SPDK_NVME_SCT_GENERIC; 1654 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1655 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1656 } 1657 1658 host_behavior = (struct spdk_nvme_host_behavior *)req->iov[0].iov_base; 1659 if (host_behavior->acre == 0) { 1660 ctrlr->acre_enabled = false; 1661 } else if (host_behavior->acre == 1) { 1662 ctrlr->acre_enabled = true; 1663 } else { 1664 SPDK_ERRLOG("Host Behavior Support invalid acre: 0x%02x\n", host_behavior->acre); 1665 response->status.sct = SPDK_NVME_SCT_GENERIC; 1666 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1667 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1668 } 1669 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1670 } 1671 1672 static int 1673 nvmf_ctrlr_set_features_keep_alive_timer(struct spdk_nvmf_request *req) 1674 { 1675 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1676 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1677 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1678 1679 SPDK_DEBUGLOG(nvmf, "Set Features - Keep Alive Timer (%u ms)\n", cmd->cdw11); 1680 1681 /* 1682 * if attempts to disable keep alive by setting kato to 0h 1683 * a status value of keep alive invalid shall be returned 1684 */ 1685 if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato == 0) { 1686 rsp->status.sc = SPDK_NVME_SC_KEEP_ALIVE_INVALID; 1687 } else if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato < MIN_KEEP_ALIVE_TIMEOUT_IN_MS) { 1688 ctrlr->feat.keep_alive_timer.bits.kato = MIN_KEEP_ALIVE_TIMEOUT_IN_MS; 1689 } else { 1690 /* round up to milliseconds */ 1691 ctrlr->feat.keep_alive_timer.bits.kato = spdk_divide_round_up( 1692 cmd->cdw11_bits.feat_keep_alive_timer.bits.kato, 1693 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS) * 1694 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS; 1695 } 1696 1697 /* 1698 * if change the keep alive timeout value successfully 1699 * update the keep alive poller. 1700 */ 1701 if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato != 0) { 1702 if (ctrlr->keep_alive_poller != NULL) { 1703 spdk_poller_unregister(&ctrlr->keep_alive_poller); 1704 } 1705 ctrlr->keep_alive_poller = SPDK_POLLER_REGISTER(nvmf_ctrlr_keep_alive_poll, ctrlr, 1706 ctrlr->feat.keep_alive_timer.bits.kato * 1000); 1707 } 1708 1709 SPDK_DEBUGLOG(nvmf, "Set Features - Keep Alive Timer set to %u ms\n", 1710 ctrlr->feat.keep_alive_timer.bits.kato); 1711 1712 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1713 } 1714 1715 static int 1716 nvmf_ctrlr_set_features_number_of_queues(struct spdk_nvmf_request *req) 1717 { 1718 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1719 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1720 uint32_t count; 1721 1722 SPDK_DEBUGLOG(nvmf, "Set Features - Number of Queues, cdw11 0x%x\n", 1723 req->cmd->nvme_cmd.cdw11); 1724 1725 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1726 /* verify that the controller is ready to process commands */ 1727 if (count > 1) { 1728 SPDK_DEBUGLOG(nvmf, "Queue pairs already active!\n"); 1729 rsp->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 1730 } else { 1731 /* 1732 * Ignore the value requested by the host - 1733 * always return the pre-configured value based on max_qpairs_allowed. 1734 */ 1735 rsp->cdw0 = ctrlr->feat.number_of_queues.raw; 1736 } 1737 1738 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1739 } 1740 1741 static int 1742 nvmf_ctrlr_set_features_async_event_configuration(struct spdk_nvmf_request *req) 1743 { 1744 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1745 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1746 1747 SPDK_DEBUGLOG(nvmf, "Set Features - Async Event Configuration, cdw11 0x%08x\n", 1748 cmd->cdw11); 1749 ctrlr->feat.async_event_configuration.raw = cmd->cdw11; 1750 ctrlr->feat.async_event_configuration.bits.reserved1 = 0; 1751 ctrlr->feat.async_event_configuration.bits.reserved2 = 0; 1752 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1753 } 1754 1755 static int 1756 nvmf_ctrlr_async_event_request(struct spdk_nvmf_request *req) 1757 { 1758 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1759 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1760 struct spdk_nvmf_subsystem_poll_group *sgroup; 1761 struct spdk_nvmf_async_event_completion *pending_event; 1762 1763 SPDK_DEBUGLOG(nvmf, "Async Event Request\n"); 1764 1765 /* AER cmd is an exception */ 1766 sgroup = &req->qpair->group->sgroups[ctrlr->subsys->id]; 1767 assert(sgroup != NULL); 1768 sgroup->mgmt_io_outstanding--; 1769 1770 /* Four asynchronous events are supported for now */ 1771 if (ctrlr->nr_aer_reqs >= NVMF_MAX_ASYNC_EVENTS) { 1772 SPDK_DEBUGLOG(nvmf, "AERL exceeded\n"); 1773 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1774 rsp->status.sc = SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED; 1775 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1776 } 1777 1778 if (!STAILQ_EMPTY(&ctrlr->async_events)) { 1779 pending_event = STAILQ_FIRST(&ctrlr->async_events); 1780 rsp->cdw0 = pending_event->event.raw; 1781 STAILQ_REMOVE(&ctrlr->async_events, pending_event, spdk_nvmf_async_event_completion, link); 1782 free(pending_event); 1783 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1784 } 1785 1786 ctrlr->aer_req[ctrlr->nr_aer_reqs++] = req; 1787 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 1788 } 1789 1790 struct copy_iovs_ctx { 1791 struct iovec *iovs; 1792 int iovcnt; 1793 int cur_iov_idx; 1794 size_t cur_iov_offset; 1795 }; 1796 1797 static void 1798 _init_copy_iovs_ctx(struct copy_iovs_ctx *copy_ctx, struct iovec *iovs, int iovcnt) 1799 { 1800 int iov_idx = 0; 1801 struct iovec *iov; 1802 1803 copy_ctx->iovs = iovs; 1804 copy_ctx->iovcnt = iovcnt; 1805 copy_ctx->cur_iov_idx = 0; 1806 copy_ctx->cur_iov_offset = 0; 1807 1808 while (iov_idx < copy_ctx->iovcnt) { 1809 iov = ©_ctx->iovs[iov_idx]; 1810 memset(iov->iov_base, 0, iov->iov_len); 1811 iov_idx++; 1812 } 1813 } 1814 1815 static size_t 1816 _copy_buf_to_iovs(struct copy_iovs_ctx *copy_ctx, const void *buf, size_t buf_len) 1817 { 1818 size_t len, iov_remain_len, copied_len = 0; 1819 struct iovec *iov; 1820 1821 if (buf_len == 0) { 1822 return 0; 1823 } 1824 1825 while (copy_ctx->cur_iov_idx < copy_ctx->iovcnt) { 1826 iov = ©_ctx->iovs[copy_ctx->cur_iov_idx]; 1827 iov_remain_len = iov->iov_len - copy_ctx->cur_iov_offset; 1828 if (iov_remain_len == 0) { 1829 copy_ctx->cur_iov_idx++; 1830 copy_ctx->cur_iov_offset = 0; 1831 continue; 1832 } 1833 1834 len = spdk_min(iov_remain_len, buf_len - copied_len); 1835 memcpy((char *)iov->iov_base + copy_ctx->cur_iov_offset, 1836 (const char *)buf + copied_len, 1837 len); 1838 copied_len += len; 1839 copy_ctx->cur_iov_offset += len; 1840 1841 if (buf_len == copied_len) { 1842 return copied_len; 1843 } 1844 } 1845 1846 return copied_len; 1847 } 1848 1849 static void 1850 nvmf_get_firmware_slot_log_page(struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length) 1851 { 1852 struct spdk_nvme_firmware_page fw_page; 1853 size_t copy_len; 1854 struct copy_iovs_ctx copy_ctx; 1855 1856 _init_copy_iovs_ctx(©_ctx, iovs, iovcnt); 1857 1858 memset(&fw_page, 0, sizeof(fw_page)); 1859 fw_page.afi.active_slot = 1; 1860 fw_page.afi.next_reset_slot = 0; 1861 spdk_strcpy_pad(fw_page.revision[0], FW_VERSION, sizeof(fw_page.revision[0]), ' '); 1862 1863 if (offset < sizeof(fw_page)) { 1864 copy_len = spdk_min(sizeof(fw_page) - offset, length); 1865 if (copy_len > 0) { 1866 _copy_buf_to_iovs(©_ctx, (const char *)&fw_page + offset, copy_len); 1867 } 1868 } 1869 } 1870 1871 /* 1872 * Asynchronous Event Mask Bit 1873 */ 1874 enum spdk_nvme_async_event_mask_bit { 1875 /* Mask Namespace Change Notificaton */ 1876 SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT = 0, 1877 /* Mask Asymmetric Namespace Access Change Notification */ 1878 SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT = 1, 1879 /* Mask Discovery Log Change Notification */ 1880 SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT = 2, 1881 /* Mask Reservation Log Page Available Notification */ 1882 SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT = 3, 1883 1884 /* 4 - 63 Reserved */ 1885 }; 1886 1887 static inline void 1888 nvmf_ctrlr_unmask_aen(struct spdk_nvmf_ctrlr *ctrlr, 1889 enum spdk_nvme_async_event_mask_bit mask) 1890 { 1891 ctrlr->notice_aen_mask &= ~(1 << mask); 1892 } 1893 1894 static inline bool 1895 nvmf_ctrlr_mask_aen(struct spdk_nvmf_ctrlr *ctrlr, 1896 enum spdk_nvme_async_event_mask_bit mask) 1897 { 1898 if (ctrlr->notice_aen_mask & (1 << mask)) { 1899 return false; 1900 } else { 1901 ctrlr->notice_aen_mask |= (1 << mask); 1902 return true; 1903 } 1904 } 1905 1906 #define SPDK_NVMF_ANA_DESC_SIZE (sizeof(struct spdk_nvme_ana_group_descriptor) + \ 1907 sizeof(uint32_t)) 1908 static void 1909 nvmf_get_ana_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iovcnt, 1910 uint64_t offset, uint32_t length, uint32_t rae) 1911 { 1912 struct spdk_nvme_ana_page ana_hdr; 1913 char _ana_desc[SPDK_NVMF_ANA_DESC_SIZE]; 1914 struct spdk_nvme_ana_group_descriptor *ana_desc; 1915 size_t copy_len, copied_len; 1916 uint32_t num_ns = 0; 1917 struct spdk_nvmf_ns *ns; 1918 struct copy_iovs_ctx copy_ctx; 1919 1920 _init_copy_iovs_ctx(©_ctx, iovs, iovcnt); 1921 1922 if (length == 0) { 1923 return; 1924 } 1925 1926 if (offset >= sizeof(ana_hdr)) { 1927 offset -= sizeof(ana_hdr); 1928 } else { 1929 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 1930 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 1931 num_ns++; 1932 } 1933 1934 memset(&ana_hdr, 0, sizeof(ana_hdr)); 1935 1936 ana_hdr.num_ana_group_desc = num_ns; 1937 /* TODO: Support Change Count. */ 1938 ana_hdr.change_count = 0; 1939 1940 copy_len = spdk_min(sizeof(ana_hdr) - offset, length); 1941 copied_len = _copy_buf_to_iovs(©_ctx, (const char *)&ana_hdr + offset, copy_len); 1942 assert(copied_len == copy_len); 1943 length -= copied_len; 1944 offset = 0; 1945 } 1946 1947 if (length == 0) { 1948 return; 1949 } 1950 1951 ana_desc = (void *)_ana_desc; 1952 1953 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 1954 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 1955 if (offset >= SPDK_NVMF_ANA_DESC_SIZE) { 1956 offset -= SPDK_NVMF_ANA_DESC_SIZE; 1957 continue; 1958 } 1959 1960 memset(ana_desc, 0, SPDK_NVMF_ANA_DESC_SIZE); 1961 1962 ana_desc->ana_group_id = ns->nsid; 1963 ana_desc->num_of_nsid = 1; 1964 ana_desc->ana_state = ctrlr->listener->ana_state; 1965 ana_desc->nsid[0] = ns->nsid; 1966 /* TODO: Support Change Count. */ 1967 ana_desc->change_count = 0; 1968 1969 copy_len = spdk_min(SPDK_NVMF_ANA_DESC_SIZE - offset, length); 1970 copied_len = _copy_buf_to_iovs(©_ctx, (const char *)ana_desc + offset, copy_len); 1971 assert(copied_len == copy_len); 1972 length -= copied_len; 1973 offset = 0; 1974 1975 if (length == 0) { 1976 goto done; 1977 } 1978 } 1979 1980 done: 1981 if (!rae) { 1982 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT); 1983 } 1984 } 1985 1986 void 1987 nvmf_ctrlr_ns_changed(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid) 1988 { 1989 uint16_t max_changes = SPDK_COUNTOF(ctrlr->changed_ns_list.ns_list); 1990 uint16_t i; 1991 bool found = false; 1992 1993 for (i = 0; i < ctrlr->changed_ns_list_count; i++) { 1994 if (ctrlr->changed_ns_list.ns_list[i] == nsid) { 1995 /* nsid is already in the list */ 1996 found = true; 1997 break; 1998 } 1999 } 2000 2001 if (!found) { 2002 if (ctrlr->changed_ns_list_count == max_changes) { 2003 /* Out of space - set first entry to FFFFFFFFh and zero-fill the rest. */ 2004 ctrlr->changed_ns_list.ns_list[0] = 0xFFFFFFFFu; 2005 for (i = 1; i < max_changes; i++) { 2006 ctrlr->changed_ns_list.ns_list[i] = 0; 2007 } 2008 } else { 2009 ctrlr->changed_ns_list.ns_list[ctrlr->changed_ns_list_count++] = nsid; 2010 } 2011 } 2012 } 2013 2014 static void 2015 nvmf_get_changed_ns_list_log_page(struct spdk_nvmf_ctrlr *ctrlr, 2016 struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length, uint32_t rae) 2017 { 2018 size_t copy_length; 2019 struct copy_iovs_ctx copy_ctx; 2020 2021 _init_copy_iovs_ctx(©_ctx, iovs, iovcnt); 2022 2023 if (offset < sizeof(ctrlr->changed_ns_list)) { 2024 copy_length = spdk_min(length, sizeof(ctrlr->changed_ns_list) - offset); 2025 if (copy_length) { 2026 _copy_buf_to_iovs(©_ctx, (char *)&ctrlr->changed_ns_list + offset, copy_length); 2027 } 2028 } 2029 2030 /* Clear log page each time it is read */ 2031 ctrlr->changed_ns_list_count = 0; 2032 memset(&ctrlr->changed_ns_list, 0, sizeof(ctrlr->changed_ns_list)); 2033 2034 if (!rae) { 2035 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT); 2036 } 2037 } 2038 2039 /* The structure can be modified if we provide support for other commands in future */ 2040 static const struct spdk_nvme_cmds_and_effect_log_page g_cmds_and_effect_log_page = { 2041 .admin_cmds_supported = { 2042 /* CSUPP, LBCC, NCC, NIC, CCC, CSE */ 2043 /* Get Log Page */ 2044 [SPDK_NVME_OPC_GET_LOG_PAGE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2045 /* Identify */ 2046 [SPDK_NVME_OPC_IDENTIFY] = {1, 0, 0, 0, 0, 0, 0, 0}, 2047 /* Abort */ 2048 [SPDK_NVME_OPC_ABORT] = {1, 0, 0, 0, 0, 0, 0, 0}, 2049 /* Set Features */ 2050 [SPDK_NVME_OPC_SET_FEATURES] = {1, 0, 0, 0, 0, 0, 0, 0}, 2051 /* Get Features */ 2052 [SPDK_NVME_OPC_GET_FEATURES] = {1, 0, 0, 0, 0, 0, 0, 0}, 2053 /* Async Event Request */ 2054 [SPDK_NVME_OPC_ASYNC_EVENT_REQUEST] = {1, 0, 0, 0, 0, 0, 0, 0}, 2055 /* Keep Alive */ 2056 [SPDK_NVME_OPC_KEEP_ALIVE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2057 }, 2058 .io_cmds_supported = { 2059 /* FLUSH */ 2060 [SPDK_NVME_OPC_FLUSH] = {1, 1, 0, 0, 0, 0, 0, 0}, 2061 /* WRITE */ 2062 [SPDK_NVME_OPC_WRITE] = {1, 1, 0, 0, 0, 0, 0, 0}, 2063 /* READ */ 2064 [SPDK_NVME_OPC_READ] = {1, 0, 0, 0, 0, 0, 0, 0}, 2065 /* WRITE ZEROES */ 2066 [SPDK_NVME_OPC_WRITE_ZEROES] = {1, 1, 0, 0, 0, 0, 0, 0}, 2067 /* DATASET MANAGEMENT */ 2068 [SPDK_NVME_OPC_DATASET_MANAGEMENT] = {1, 1, 0, 0, 0, 0, 0, 0}, 2069 /* COMPARE */ 2070 [SPDK_NVME_OPC_COMPARE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2071 }, 2072 }; 2073 2074 static void 2075 nvmf_get_cmds_and_effects_log_page(struct iovec *iovs, int iovcnt, 2076 uint64_t offset, uint32_t length) 2077 { 2078 uint32_t page_size = sizeof(struct spdk_nvme_cmds_and_effect_log_page); 2079 size_t copy_len = 0; 2080 struct copy_iovs_ctx copy_ctx; 2081 2082 _init_copy_iovs_ctx(©_ctx, iovs, iovcnt); 2083 2084 if (offset < page_size) { 2085 copy_len = spdk_min(page_size - offset, length); 2086 _copy_buf_to_iovs(©_ctx, (char *)(&g_cmds_and_effect_log_page) + offset, copy_len); 2087 } 2088 } 2089 2090 static void 2091 nvmf_get_reservation_notification_log_page(struct spdk_nvmf_ctrlr *ctrlr, 2092 struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length, uint32_t rae) 2093 { 2094 uint32_t unit_log_len, avail_log_len, next_pos, copy_len; 2095 struct spdk_nvmf_reservation_log *log, *log_tmp; 2096 struct copy_iovs_ctx copy_ctx; 2097 2098 _init_copy_iovs_ctx(©_ctx, iovs, iovcnt); 2099 2100 unit_log_len = sizeof(struct spdk_nvme_reservation_notification_log); 2101 /* No available log, return zeroed log pages */ 2102 if (!ctrlr->num_avail_log_pages) { 2103 return; 2104 } 2105 2106 avail_log_len = ctrlr->num_avail_log_pages * unit_log_len; 2107 if (offset >= avail_log_len) { 2108 return; 2109 } 2110 2111 next_pos = 0; 2112 TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) { 2113 TAILQ_REMOVE(&ctrlr->log_head, log, link); 2114 ctrlr->num_avail_log_pages--; 2115 2116 next_pos += unit_log_len; 2117 if (next_pos > offset) { 2118 copy_len = spdk_min(next_pos - offset, length); 2119 _copy_buf_to_iovs(©_ctx, &log->log, copy_len); 2120 length -= copy_len; 2121 offset += copy_len; 2122 } 2123 free(log); 2124 2125 if (length == 0) { 2126 break; 2127 } 2128 } 2129 2130 if (!rae) { 2131 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT); 2132 } 2133 return; 2134 } 2135 2136 static int 2137 nvmf_ctrlr_get_log_page(struct spdk_nvmf_request *req) 2138 { 2139 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2140 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2141 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2142 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 2143 uint64_t offset, len; 2144 uint32_t rae, numdl, numdu; 2145 uint8_t lid; 2146 2147 if (req->data == NULL) { 2148 SPDK_ERRLOG("get log command with no buffer\n"); 2149 response->status.sct = SPDK_NVME_SCT_GENERIC; 2150 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2151 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2152 } 2153 2154 offset = (uint64_t)cmd->cdw12 | ((uint64_t)cmd->cdw13 << 32); 2155 if (offset & 3) { 2156 SPDK_ERRLOG("Invalid log page offset 0x%" PRIx64 "\n", offset); 2157 response->status.sct = SPDK_NVME_SCT_GENERIC; 2158 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2159 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2160 } 2161 2162 rae = cmd->cdw10_bits.get_log_page.rae; 2163 numdl = cmd->cdw10_bits.get_log_page.numdl; 2164 numdu = cmd->cdw11_bits.get_log_page.numdu; 2165 len = ((numdu << 16) + numdl + (uint64_t)1) * 4; 2166 if (len > req->length) { 2167 SPDK_ERRLOG("Get log page: len (%" PRIu64 ") > buf size (%u)\n", 2168 len, req->length); 2169 response->status.sct = SPDK_NVME_SCT_GENERIC; 2170 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2171 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2172 } 2173 2174 lid = cmd->cdw10_bits.get_log_page.lid; 2175 SPDK_DEBUGLOG(nvmf, "Get log page: LID=0x%02X offset=0x%" PRIx64 " len=0x%" PRIx64 " rae=%u\n", 2176 lid, offset, len, rae); 2177 2178 if (subsystem->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 2179 switch (lid) { 2180 case SPDK_NVME_LOG_DISCOVERY: 2181 nvmf_get_discovery_log_page(subsystem->tgt, ctrlr->hostnqn, req->iov, req->iovcnt, offset, 2182 len); 2183 if (!rae) { 2184 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT); 2185 } 2186 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2187 default: 2188 goto invalid_log_page; 2189 } 2190 } else { 2191 switch (lid) { 2192 case SPDK_NVME_LOG_ERROR: 2193 case SPDK_NVME_LOG_HEALTH_INFORMATION: 2194 /* TODO: actually fill out log page data */ 2195 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2196 case SPDK_NVME_LOG_FIRMWARE_SLOT: 2197 nvmf_get_firmware_slot_log_page(req->iov, req->iovcnt, offset, len); 2198 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2199 case SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS: 2200 if (subsystem->flags.ana_reporting) { 2201 nvmf_get_ana_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2202 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2203 } else { 2204 goto invalid_log_page; 2205 } 2206 case SPDK_NVME_LOG_COMMAND_EFFECTS_LOG: 2207 nvmf_get_cmds_and_effects_log_page(req->iov, req->iovcnt, offset, len); 2208 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2209 case SPDK_NVME_LOG_CHANGED_NS_LIST: 2210 nvmf_get_changed_ns_list_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2211 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2212 case SPDK_NVME_LOG_RESERVATION_NOTIFICATION: 2213 nvmf_get_reservation_notification_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2214 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2215 default: 2216 goto invalid_log_page; 2217 } 2218 } 2219 2220 invalid_log_page: 2221 SPDK_DEBUGLOG(nvmf, "Unsupported Get Log Page 0x%02X\n", lid); 2222 response->status.sct = SPDK_NVME_SCT_GENERIC; 2223 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2224 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2225 } 2226 2227 int 2228 spdk_nvmf_ctrlr_identify_ns(struct spdk_nvmf_ctrlr *ctrlr, 2229 struct spdk_nvme_cmd *cmd, 2230 struct spdk_nvme_cpl *rsp, 2231 struct spdk_nvme_ns_data *nsdata) 2232 { 2233 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2234 struct spdk_nvmf_ns *ns; 2235 uint32_t max_num_blocks; 2236 2237 if (cmd->nsid == 0 || cmd->nsid > subsystem->max_nsid) { 2238 SPDK_ERRLOG("Identify Namespace for invalid NSID %u\n", cmd->nsid); 2239 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2240 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 2241 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2242 } 2243 2244 ns = _nvmf_subsystem_get_ns(subsystem, cmd->nsid); 2245 if (ns == NULL || ns->bdev == NULL) { 2246 /* 2247 * Inactive namespaces should return a zero filled data structure. 2248 * The data buffer is already zeroed by nvmf_ctrlr_process_admin_cmd(), 2249 * so we can just return early here. 2250 */ 2251 SPDK_DEBUGLOG(nvmf, "Identify Namespace for inactive NSID %u\n", cmd->nsid); 2252 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2253 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2254 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2255 } 2256 2257 nvmf_bdev_ctrlr_identify_ns(ns, nsdata, ctrlr->dif_insert_or_strip); 2258 2259 /* Due to bug in the Linux kernel NVMe driver we have to set noiob no larger than mdts */ 2260 max_num_blocks = ctrlr->admin_qpair->transport->opts.max_io_size / 2261 (1U << nsdata->lbaf[nsdata->flbas.format].lbads); 2262 if (nsdata->noiob > max_num_blocks) { 2263 nsdata->noiob = max_num_blocks; 2264 } 2265 2266 if (subsystem->flags.ana_reporting) { 2267 /* ANA group ID matches NSID. */ 2268 nsdata->anagrpid = ns->nsid; 2269 2270 if (ctrlr->listener->ana_state == SPDK_NVME_ANA_INACCESSIBLE_STATE || 2271 ctrlr->listener->ana_state == SPDK_NVME_ANA_PERSISTENT_LOSS_STATE) { 2272 nsdata->nuse = 0; 2273 } 2274 } 2275 2276 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2277 } 2278 2279 static void 2280 nvmf_ctrlr_populate_oacs(struct spdk_nvmf_ctrlr *ctrlr, 2281 struct spdk_nvme_ctrlr_data *cdata) 2282 { 2283 cdata->oacs.virtualization_management = 2284 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_VIRTUALIZATION_MANAGEMENT].hdlr != NULL; 2285 cdata->oacs.nvme_mi = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NVME_MI_SEND].hdlr != NULL 2286 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NVME_MI_RECEIVE].hdlr != NULL; 2287 cdata->oacs.directives = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DIRECTIVE_SEND].hdlr != NULL 2288 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DIRECTIVE_RECEIVE].hdlr != NULL; 2289 cdata->oacs.device_self_test = 2290 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DEVICE_SELF_TEST].hdlr != NULL; 2291 cdata->oacs.ns_manage = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NS_MANAGEMENT].hdlr != NULL 2292 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NS_ATTACHMENT].hdlr != NULL; 2293 cdata->oacs.firmware = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD].hdlr != 2294 NULL 2295 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FIRMWARE_COMMIT].hdlr != NULL; 2296 cdata->oacs.format = 2297 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FORMAT_NVM].hdlr != NULL; 2298 cdata->oacs.security = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_SECURITY_SEND].hdlr != NULL 2299 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_SECURITY_RECEIVE].hdlr != NULL; 2300 cdata->oacs.get_lba_status = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_GET_LBA_STATUS].hdlr != 2301 NULL; 2302 } 2303 2304 int 2305 spdk_nvmf_ctrlr_identify_ctrlr(struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvme_ctrlr_data *cdata) 2306 { 2307 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2308 struct spdk_nvmf_transport *transport = ctrlr->admin_qpair->transport; 2309 2310 /* 2311 * Common fields for discovery and NVM subsystems 2312 */ 2313 spdk_strcpy_pad(cdata->fr, FW_VERSION, sizeof(cdata->fr), ' '); 2314 assert((transport->opts.max_io_size % 4096) == 0); 2315 cdata->mdts = spdk_u32log2(transport->opts.max_io_size / 4096); 2316 cdata->cntlid = ctrlr->cntlid; 2317 cdata->ver = ctrlr->vcprop.vs; 2318 cdata->aerl = NVMF_MAX_ASYNC_EVENTS - 1; 2319 cdata->lpa.edlp = 1; 2320 cdata->elpe = 127; 2321 cdata->maxcmd = transport->opts.max_queue_depth; 2322 cdata->sgls = ctrlr->cdata.sgls; 2323 cdata->fuses.compare_and_write = 1; 2324 cdata->acwu = 1; 2325 if (subsystem->flags.ana_reporting) { 2326 cdata->mnan = subsystem->max_nsid; 2327 } 2328 spdk_strcpy_pad(cdata->subnqn, subsystem->subnqn, sizeof(cdata->subnqn), '\0'); 2329 2330 SPDK_DEBUGLOG(nvmf, "ctrlr data: maxcmd 0x%x\n", cdata->maxcmd); 2331 SPDK_DEBUGLOG(nvmf, "sgls data: 0x%x\n", from_le32(&cdata->sgls)); 2332 2333 2334 if (subsystem->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 2335 /* 2336 * NVM Discovery subsystem fields 2337 */ 2338 cdata->oaes.discovery_log_change_notices = 1; 2339 } else { 2340 /* 2341 * NVM subsystem fields (reserved for discovery subsystems) 2342 */ 2343 spdk_strcpy_pad(cdata->mn, spdk_nvmf_subsystem_get_mn(subsystem), sizeof(cdata->mn), ' '); 2344 spdk_strcpy_pad(cdata->sn, spdk_nvmf_subsystem_get_sn(subsystem), sizeof(cdata->sn), ' '); 2345 cdata->kas = ctrlr->cdata.kas; 2346 2347 cdata->rab = 6; 2348 cdata->cmic.multi_port = 1; 2349 cdata->cmic.multi_ctrlr = 1; 2350 if (subsystem->flags.ana_reporting) { 2351 /* Asymmetric Namespace Access Reporting is supported. */ 2352 cdata->cmic.ana_reporting = 1; 2353 } 2354 cdata->oaes.ns_attribute_notices = 1; 2355 if (subsystem->flags.ana_reporting) { 2356 cdata->oaes.ana_change_notices = 1; 2357 } 2358 cdata->ctratt.host_id_exhid_supported = 1; 2359 /* TODO: Concurrent execution of multiple abort commands. */ 2360 cdata->acl = 0; 2361 cdata->frmw.slot1_ro = 1; 2362 cdata->frmw.num_slots = 1; 2363 2364 cdata->lpa.celp = 1; /* Command Effects log page supported */ 2365 2366 cdata->sqes.min = 6; 2367 cdata->sqes.max = 6; 2368 cdata->cqes.min = 4; 2369 cdata->cqes.max = 4; 2370 cdata->nn = subsystem->max_nsid; 2371 cdata->vwc.present = 1; 2372 cdata->vwc.flush_broadcast = SPDK_NVME_FLUSH_BROADCAST_NOT_SUPPORTED; 2373 2374 cdata->nvmf_specific = ctrlr->cdata.nvmf_specific; 2375 2376 cdata->oncs.dsm = nvmf_ctrlr_dsm_supported(ctrlr); 2377 cdata->oncs.write_zeroes = nvmf_ctrlr_write_zeroes_supported(ctrlr); 2378 cdata->oncs.reservations = 1; 2379 if (subsystem->flags.ana_reporting) { 2380 cdata->anatt = ANA_TRANSITION_TIME_IN_SEC; 2381 /* ANA Change state is not used, and ANA Persistent Loss state 2382 * is not supported for now. 2383 */ 2384 cdata->anacap.ana_optimized_state = 1; 2385 cdata->anacap.ana_non_optimized_state = 1; 2386 cdata->anacap.ana_inaccessible_state = 1; 2387 /* ANAGRPID does not change while namespace is attached to controller */ 2388 cdata->anacap.no_change_anagrpid = 1; 2389 cdata->anagrpmax = subsystem->max_nsid; 2390 cdata->nanagrpid = subsystem->max_nsid; 2391 } 2392 2393 nvmf_ctrlr_populate_oacs(ctrlr, cdata); 2394 2395 assert(subsystem->tgt != NULL); 2396 cdata->crdt[0] = subsystem->tgt->crdt[0]; 2397 cdata->crdt[1] = subsystem->tgt->crdt[1]; 2398 cdata->crdt[2] = subsystem->tgt->crdt[2]; 2399 2400 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: ioccsz 0x%x\n", 2401 cdata->nvmf_specific.ioccsz); 2402 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: iorcsz 0x%x\n", 2403 cdata->nvmf_specific.iorcsz); 2404 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: icdoff 0x%x\n", 2405 cdata->nvmf_specific.icdoff); 2406 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: ctrattr 0x%x\n", 2407 *(uint8_t *)&cdata->nvmf_specific.ctrattr); 2408 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: msdbd 0x%x\n", 2409 cdata->nvmf_specific.msdbd); 2410 } 2411 2412 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2413 } 2414 2415 static int 2416 nvmf_ctrlr_identify_active_ns_list(struct spdk_nvmf_subsystem *subsystem, 2417 struct spdk_nvme_cmd *cmd, 2418 struct spdk_nvme_cpl *rsp, 2419 struct spdk_nvme_ns_list *ns_list) 2420 { 2421 struct spdk_nvmf_ns *ns; 2422 uint32_t count = 0; 2423 2424 if (cmd->nsid >= 0xfffffffeUL) { 2425 SPDK_ERRLOG("Identify Active Namespace List with invalid NSID %u\n", cmd->nsid); 2426 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 2427 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2428 } 2429 2430 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 2431 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 2432 if (ns->opts.nsid <= cmd->nsid) { 2433 continue; 2434 } 2435 2436 ns_list->ns_list[count++] = ns->opts.nsid; 2437 if (count == SPDK_COUNTOF(ns_list->ns_list)) { 2438 break; 2439 } 2440 } 2441 2442 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2443 } 2444 2445 static void 2446 _add_ns_id_desc(void **buf_ptr, size_t *buf_remain, 2447 enum spdk_nvme_nidt type, 2448 const void *data, size_t data_size) 2449 { 2450 struct spdk_nvme_ns_id_desc *desc; 2451 size_t desc_size = sizeof(*desc) + data_size; 2452 2453 /* 2454 * These should never fail in practice, since all valid NS ID descriptors 2455 * should be defined so that they fit in the available 4096-byte buffer. 2456 */ 2457 assert(data_size > 0); 2458 assert(data_size <= UINT8_MAX); 2459 assert(desc_size < *buf_remain); 2460 if (data_size == 0 || data_size > UINT8_MAX || desc_size > *buf_remain) { 2461 return; 2462 } 2463 2464 desc = *buf_ptr; 2465 desc->nidt = type; 2466 desc->nidl = data_size; 2467 memcpy(desc->nid, data, data_size); 2468 2469 *buf_ptr += desc_size; 2470 *buf_remain -= desc_size; 2471 } 2472 2473 static int 2474 nvmf_ctrlr_identify_ns_id_descriptor_list( 2475 struct spdk_nvmf_subsystem *subsystem, 2476 struct spdk_nvme_cmd *cmd, 2477 struct spdk_nvme_cpl *rsp, 2478 void *id_desc_list, size_t id_desc_list_size) 2479 { 2480 struct spdk_nvmf_ns *ns; 2481 size_t buf_remain = id_desc_list_size; 2482 void *buf_ptr = id_desc_list; 2483 2484 ns = _nvmf_subsystem_get_ns(subsystem, cmd->nsid); 2485 if (ns == NULL || ns->bdev == NULL) { 2486 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2487 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 2488 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2489 } 2490 2491 #define ADD_ID_DESC(type, data, size) \ 2492 do { \ 2493 if (!spdk_mem_all_zero(data, size)) { \ 2494 _add_ns_id_desc(&buf_ptr, &buf_remain, type, data, size); \ 2495 } \ 2496 } while (0) 2497 2498 ADD_ID_DESC(SPDK_NVME_NIDT_EUI64, ns->opts.eui64, sizeof(ns->opts.eui64)); 2499 ADD_ID_DESC(SPDK_NVME_NIDT_NGUID, ns->opts.nguid, sizeof(ns->opts.nguid)); 2500 ADD_ID_DESC(SPDK_NVME_NIDT_UUID, &ns->opts.uuid, sizeof(ns->opts.uuid)); 2501 2502 /* 2503 * The list is automatically 0-terminated because controller to host buffers in 2504 * admin commands always get zeroed in nvmf_ctrlr_process_admin_cmd(). 2505 */ 2506 2507 #undef ADD_ID_DESC 2508 2509 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2510 } 2511 2512 static int 2513 nvmf_ctrlr_identify(struct spdk_nvmf_request *req) 2514 { 2515 uint8_t cns; 2516 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2517 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2518 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 2519 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2520 2521 if (req->data == NULL || req->length < 4096) { 2522 SPDK_ERRLOG("identify command with invalid buffer\n"); 2523 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2524 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2525 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2526 } 2527 2528 cns = cmd->cdw10_bits.identify.cns; 2529 2530 if (subsystem->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY && 2531 cns != SPDK_NVME_IDENTIFY_CTRLR) { 2532 /* Discovery controllers only support Identify Controller */ 2533 goto invalid_cns; 2534 } 2535 2536 switch (cns) { 2537 case SPDK_NVME_IDENTIFY_NS: 2538 return spdk_nvmf_ctrlr_identify_ns(ctrlr, cmd, rsp, req->data); 2539 case SPDK_NVME_IDENTIFY_CTRLR: 2540 return spdk_nvmf_ctrlr_identify_ctrlr(ctrlr, req->data); 2541 case SPDK_NVME_IDENTIFY_ACTIVE_NS_LIST: 2542 return nvmf_ctrlr_identify_active_ns_list(subsystem, cmd, rsp, req->data); 2543 case SPDK_NVME_IDENTIFY_NS_ID_DESCRIPTOR_LIST: 2544 return nvmf_ctrlr_identify_ns_id_descriptor_list(subsystem, cmd, rsp, req->data, req->length); 2545 default: 2546 goto invalid_cns; 2547 } 2548 2549 invalid_cns: 2550 SPDK_ERRLOG("Identify command with unsupported CNS 0x%02x\n", cns); 2551 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2552 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2553 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2554 } 2555 2556 static bool 2557 nvmf_qpair_abort_aer(struct spdk_nvmf_qpair *qpair, uint16_t cid) 2558 { 2559 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 2560 struct spdk_nvmf_request *req; 2561 int i; 2562 2563 if (!nvmf_qpair_is_admin_queue(qpair)) { 2564 return false; 2565 } 2566 2567 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 2568 if (ctrlr->aer_req[i]->cmd->nvme_cmd.cid == cid) { 2569 SPDK_DEBUGLOG(nvmf, "Aborting AER request\n"); 2570 req = ctrlr->aer_req[i]; 2571 ctrlr->aer_req[i] = NULL; 2572 ctrlr->nr_aer_reqs--; 2573 2574 /* Move the last req to the aborting position for making aer_reqs 2575 * in continuous 2576 */ 2577 if (i < ctrlr->nr_aer_reqs) { 2578 ctrlr->aer_req[i] = ctrlr->aer_req[ctrlr->nr_aer_reqs]; 2579 ctrlr->aer_req[ctrlr->nr_aer_reqs] = NULL; 2580 } 2581 2582 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 2583 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 2584 _nvmf_request_complete(req); 2585 return true; 2586 } 2587 } 2588 2589 return false; 2590 } 2591 2592 static void 2593 nvmf_qpair_abort_request(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_request *req) 2594 { 2595 uint16_t cid = req->cmd->nvme_cmd.cdw10_bits.abort.cid; 2596 2597 if (nvmf_qpair_abort_aer(qpair, cid)) { 2598 SPDK_DEBUGLOG(nvmf, "abort ctrlr=%p sqid=%u cid=%u successful\n", 2599 qpair->ctrlr, qpair->qid, cid); 2600 req->rsp->nvme_cpl.cdw0 &= ~1U; /* Command successfully aborted */ 2601 2602 spdk_nvmf_request_complete(req); 2603 return; 2604 } 2605 2606 nvmf_transport_qpair_abort_request(qpair, req); 2607 } 2608 2609 static void 2610 nvmf_ctrlr_abort_done(struct spdk_io_channel_iter *i, int status) 2611 { 2612 struct spdk_nvmf_request *req = spdk_io_channel_iter_get_ctx(i); 2613 2614 if (status == 0) { 2615 /* There was no qpair whose ID matches SQID of the abort command. 2616 * Hence call _nvmf_request_complete() here. 2617 */ 2618 _nvmf_request_complete(req); 2619 } 2620 } 2621 2622 static void 2623 nvmf_ctrlr_abort_on_pg(struct spdk_io_channel_iter *i) 2624 { 2625 struct spdk_nvmf_request *req = spdk_io_channel_iter_get_ctx(i); 2626 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 2627 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 2628 uint16_t sqid = req->cmd->nvme_cmd.cdw10_bits.abort.sqid; 2629 struct spdk_nvmf_qpair *qpair; 2630 2631 TAILQ_FOREACH(qpair, &group->qpairs, link) { 2632 if (qpair->ctrlr == req->qpair->ctrlr && qpair->qid == sqid) { 2633 /* Found the qpair */ 2634 2635 nvmf_qpair_abort_request(qpair, req); 2636 2637 /* Return -1 for the status so the iteration across threads stops. */ 2638 spdk_for_each_channel_continue(i, -1); 2639 return; 2640 } 2641 } 2642 2643 spdk_for_each_channel_continue(i, 0); 2644 } 2645 2646 static int 2647 nvmf_ctrlr_abort(struct spdk_nvmf_request *req) 2648 { 2649 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 2650 2651 rsp->cdw0 = 1U; /* Command not aborted */ 2652 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2653 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2654 2655 /* Send a message to each poll group, searching for this ctrlr, sqid, and command. */ 2656 spdk_for_each_channel(req->qpair->ctrlr->subsys->tgt, 2657 nvmf_ctrlr_abort_on_pg, 2658 req, 2659 nvmf_ctrlr_abort_done 2660 ); 2661 2662 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 2663 } 2664 2665 int 2666 nvmf_ctrlr_abort_request(struct spdk_nvmf_request *req) 2667 { 2668 struct spdk_nvmf_request *req_to_abort = req->req_to_abort; 2669 struct spdk_bdev *bdev; 2670 struct spdk_bdev_desc *desc; 2671 struct spdk_io_channel *ch; 2672 int rc; 2673 2674 assert(req_to_abort != NULL); 2675 2676 if (g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_ABORT].hdlr && 2677 nvmf_qpair_is_admin_queue(req_to_abort->qpair)) { 2678 return g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_ABORT].hdlr(req); 2679 } 2680 2681 rc = spdk_nvmf_request_get_bdev(req_to_abort->cmd->nvme_cmd.nsid, req_to_abort, 2682 &bdev, &desc, &ch); 2683 if (rc != 0) { 2684 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2685 } 2686 2687 return spdk_nvmf_bdev_ctrlr_abort_cmd(bdev, desc, ch, req, req_to_abort); 2688 } 2689 2690 static int 2691 get_features_generic(struct spdk_nvmf_request *req, uint32_t cdw0) 2692 { 2693 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 2694 2695 rsp->cdw0 = cdw0; 2696 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2697 } 2698 2699 /* we have to use the typedef in the function declaration to appease astyle. */ 2700 typedef enum spdk_nvme_path_status_code spdk_nvme_path_status_code_t; 2701 2702 static spdk_nvme_path_status_code_t 2703 _nvme_ana_state_to_path_status(enum spdk_nvme_ana_state ana_state) 2704 { 2705 switch (ana_state) { 2706 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 2707 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_INACCESSIBLE; 2708 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 2709 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS; 2710 case SPDK_NVME_ANA_CHANGE_STATE: 2711 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_TRANSITION; 2712 default: 2713 return SPDK_NVME_SC_INTERNAL_PATH_ERROR; 2714 } 2715 } 2716 2717 static int 2718 nvmf_ctrlr_get_features(struct spdk_nvmf_request *req) 2719 { 2720 uint8_t feature; 2721 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2722 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2723 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 2724 enum spdk_nvme_ana_state ana_state; 2725 2726 feature = cmd->cdw10_bits.get_features.fid; 2727 2728 if (ctrlr->subsys->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 2729 /* 2730 * Features supported by Discovery controller 2731 */ 2732 switch (feature) { 2733 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 2734 return get_features_generic(req, ctrlr->feat.keep_alive_timer.raw); 2735 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 2736 return get_features_generic(req, ctrlr->feat.async_event_configuration.raw); 2737 default: 2738 SPDK_DEBUGLOG(nvmf, "Get Features command with unsupported feature ID 0x%02x\n", feature); 2739 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2740 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2741 } 2742 } 2743 /* 2744 * Process Get Features command for non-discovery controller 2745 */ 2746 ana_state = ctrlr->listener->ana_state; 2747 switch (ana_state) { 2748 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 2749 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 2750 case SPDK_NVME_ANA_CHANGE_STATE: 2751 switch (feature) { 2752 case SPDK_NVME_FEAT_ERROR_RECOVERY: 2753 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 2754 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 2755 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 2756 response->status.sct = SPDK_NVME_SCT_PATH; 2757 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 2758 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2759 default: 2760 break; 2761 } 2762 break; 2763 default: 2764 break; 2765 } 2766 2767 switch (feature) { 2768 case SPDK_NVME_FEAT_ARBITRATION: 2769 return get_features_generic(req, ctrlr->feat.arbitration.raw); 2770 case SPDK_NVME_FEAT_POWER_MANAGEMENT: 2771 return get_features_generic(req, ctrlr->feat.power_management.raw); 2772 case SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD: 2773 return nvmf_ctrlr_get_features_temperature_threshold(req); 2774 case SPDK_NVME_FEAT_ERROR_RECOVERY: 2775 return get_features_generic(req, ctrlr->feat.error_recovery.raw); 2776 case SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE: 2777 return get_features_generic(req, ctrlr->feat.volatile_write_cache.raw); 2778 case SPDK_NVME_FEAT_NUMBER_OF_QUEUES: 2779 return get_features_generic(req, ctrlr->feat.number_of_queues.raw); 2780 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 2781 return get_features_generic(req, ctrlr->feat.write_atomicity.raw); 2782 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 2783 return get_features_generic(req, ctrlr->feat.async_event_configuration.raw); 2784 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 2785 return get_features_generic(req, ctrlr->feat.keep_alive_timer.raw); 2786 case SPDK_NVME_FEAT_HOST_IDENTIFIER: 2787 return nvmf_ctrlr_get_features_host_identifier(req); 2788 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 2789 return nvmf_ctrlr_get_features_reservation_notification_mask(req); 2790 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 2791 return nvmf_ctrlr_get_features_reservation_persistence(req); 2792 default: 2793 SPDK_ERRLOG("Get Features command with unsupported feature ID 0x%02x\n", feature); 2794 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2795 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2796 } 2797 } 2798 2799 static int 2800 nvmf_ctrlr_set_features(struct spdk_nvmf_request *req) 2801 { 2802 uint8_t feature, save; 2803 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2804 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2805 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 2806 enum spdk_nvme_ana_state ana_state; 2807 /* 2808 * Features are not saveable by the controller as indicated by 2809 * ONCS field of the Identify Controller data. 2810 * */ 2811 save = cmd->cdw10_bits.set_features.sv; 2812 if (save) { 2813 response->status.sc = SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE; 2814 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 2815 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2816 } 2817 2818 feature = cmd->cdw10_bits.set_features.fid; 2819 2820 if (ctrlr->subsys->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 2821 /* 2822 * Features supported by Discovery controller 2823 */ 2824 switch (feature) { 2825 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 2826 return nvmf_ctrlr_set_features_keep_alive_timer(req); 2827 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 2828 return nvmf_ctrlr_set_features_async_event_configuration(req); 2829 default: 2830 SPDK_ERRLOG("Set Features command with unsupported feature ID 0x%02x\n", feature); 2831 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2832 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2833 } 2834 } 2835 /* 2836 * Process Set Features command for non-discovery controller 2837 */ 2838 ana_state = ctrlr->listener->ana_state; 2839 switch (ana_state) { 2840 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 2841 case SPDK_NVME_ANA_CHANGE_STATE: 2842 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 2843 response->status.sct = SPDK_NVME_SCT_PATH; 2844 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 2845 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2846 } else { 2847 switch (feature) { 2848 case SPDK_NVME_FEAT_ERROR_RECOVERY: 2849 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 2850 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 2851 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 2852 response->status.sct = SPDK_NVME_SCT_PATH; 2853 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 2854 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2855 default: 2856 break; 2857 } 2858 } 2859 break; 2860 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 2861 response->status.sct = SPDK_NVME_SCT_PATH; 2862 response->status.sc = SPDK_NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS; 2863 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2864 default: 2865 break; 2866 } 2867 2868 switch (feature) { 2869 case SPDK_NVME_FEAT_ARBITRATION: 2870 return nvmf_ctrlr_set_features_arbitration(req); 2871 case SPDK_NVME_FEAT_POWER_MANAGEMENT: 2872 return nvmf_ctrlr_set_features_power_management(req); 2873 case SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD: 2874 return nvmf_ctrlr_set_features_temperature_threshold(req); 2875 case SPDK_NVME_FEAT_ERROR_RECOVERY: 2876 return nvmf_ctrlr_set_features_error_recovery(req); 2877 case SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE: 2878 return nvmf_ctrlr_set_features_volatile_write_cache(req); 2879 case SPDK_NVME_FEAT_NUMBER_OF_QUEUES: 2880 return nvmf_ctrlr_set_features_number_of_queues(req); 2881 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 2882 return nvmf_ctrlr_set_features_write_atomicity(req); 2883 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 2884 return nvmf_ctrlr_set_features_async_event_configuration(req); 2885 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 2886 return nvmf_ctrlr_set_features_keep_alive_timer(req); 2887 case SPDK_NVME_FEAT_HOST_IDENTIFIER: 2888 return nvmf_ctrlr_set_features_host_identifier(req); 2889 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 2890 return nvmf_ctrlr_set_features_reservation_notification_mask(req); 2891 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 2892 return nvmf_ctrlr_set_features_reservation_persistence(req); 2893 case SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 2894 return nvmf_ctrlr_set_features_host_behavior_support(req); 2895 default: 2896 SPDK_ERRLOG("Set Features command with unsupported feature ID 0x%02x\n", feature); 2897 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2898 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2899 } 2900 } 2901 2902 static int 2903 nvmf_ctrlr_keep_alive(struct spdk_nvmf_request *req) 2904 { 2905 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2906 2907 SPDK_DEBUGLOG(nvmf, "Keep Alive\n"); 2908 /* 2909 * To handle keep alive just clear or reset the 2910 * ctrlr based keep alive duration counter. 2911 * When added, a separate timer based process 2912 * will monitor if the time since last recorded 2913 * keep alive has exceeded the max duration and 2914 * take appropriate action. 2915 */ 2916 ctrlr->last_keep_alive_tick = spdk_get_ticks(); 2917 2918 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2919 } 2920 2921 int 2922 nvmf_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req) 2923 { 2924 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2925 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2926 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 2927 int rc; 2928 2929 if (ctrlr == NULL) { 2930 SPDK_ERRLOG("Admin command sent before CONNECT\n"); 2931 response->status.sct = SPDK_NVME_SCT_GENERIC; 2932 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 2933 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2934 } 2935 2936 if (ctrlr->vcprop.cc.bits.en != 1) { 2937 SPDK_ERRLOG("Admin command sent to disabled controller\n"); 2938 response->status.sct = SPDK_NVME_SCT_GENERIC; 2939 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 2940 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2941 } 2942 2943 if (req->data && spdk_nvme_opc_get_data_transfer(cmd->opc) == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 2944 memset(req->data, 0, req->length); 2945 } 2946 2947 if (ctrlr->subsys->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 2948 /* Discovery controllers only support these admin OPS. */ 2949 switch (cmd->opc) { 2950 case SPDK_NVME_OPC_IDENTIFY: 2951 case SPDK_NVME_OPC_GET_LOG_PAGE: 2952 case SPDK_NVME_OPC_KEEP_ALIVE: 2953 case SPDK_NVME_OPC_SET_FEATURES: 2954 case SPDK_NVME_OPC_GET_FEATURES: 2955 case SPDK_NVME_OPC_ASYNC_EVENT_REQUEST: 2956 break; 2957 default: 2958 goto invalid_opcode; 2959 } 2960 } 2961 2962 /* Call a custom adm cmd handler if set. Aborts are handled in a different path (see nvmf_passthru_admin_cmd) */ 2963 if (g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].hdlr && cmd->opc != SPDK_NVME_OPC_ABORT) { 2964 rc = g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].hdlr(req); 2965 if (rc >= SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 2966 /* The handler took care of this commmand */ 2967 return rc; 2968 } 2969 } 2970 2971 switch (cmd->opc) { 2972 case SPDK_NVME_OPC_GET_LOG_PAGE: 2973 return nvmf_ctrlr_get_log_page(req); 2974 case SPDK_NVME_OPC_IDENTIFY: 2975 return nvmf_ctrlr_identify(req); 2976 case SPDK_NVME_OPC_ABORT: 2977 return nvmf_ctrlr_abort(req); 2978 case SPDK_NVME_OPC_GET_FEATURES: 2979 return nvmf_ctrlr_get_features(req); 2980 case SPDK_NVME_OPC_SET_FEATURES: 2981 return nvmf_ctrlr_set_features(req); 2982 case SPDK_NVME_OPC_ASYNC_EVENT_REQUEST: 2983 return nvmf_ctrlr_async_event_request(req); 2984 case SPDK_NVME_OPC_KEEP_ALIVE: 2985 return nvmf_ctrlr_keep_alive(req); 2986 2987 case SPDK_NVME_OPC_CREATE_IO_SQ: 2988 case SPDK_NVME_OPC_CREATE_IO_CQ: 2989 case SPDK_NVME_OPC_DELETE_IO_SQ: 2990 case SPDK_NVME_OPC_DELETE_IO_CQ: 2991 /* Create and Delete I/O CQ/SQ not allowed in NVMe-oF */ 2992 goto invalid_opcode; 2993 2994 default: 2995 goto invalid_opcode; 2996 } 2997 2998 invalid_opcode: 2999 SPDK_ERRLOG("Unsupported admin opcode 0x%x\n", cmd->opc); 3000 response->status.sct = SPDK_NVME_SCT_GENERIC; 3001 response->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3002 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3003 } 3004 3005 static int 3006 nvmf_ctrlr_process_fabrics_cmd(struct spdk_nvmf_request *req) 3007 { 3008 struct spdk_nvmf_qpair *qpair = req->qpair; 3009 struct spdk_nvmf_capsule_cmd *cap_hdr; 3010 3011 cap_hdr = &req->cmd->nvmf_cmd; 3012 3013 if (qpair->ctrlr == NULL) { 3014 /* No ctrlr established yet; the only valid command is Connect */ 3015 if (cap_hdr->fctype == SPDK_NVMF_FABRIC_COMMAND_CONNECT) { 3016 return nvmf_ctrlr_cmd_connect(req); 3017 } else { 3018 SPDK_DEBUGLOG(nvmf, "Got fctype 0x%x, expected Connect\n", 3019 cap_hdr->fctype); 3020 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3021 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 3022 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3023 } 3024 } else if (nvmf_qpair_is_admin_queue(qpair)) { 3025 /* 3026 * Controller session is established, and this is an admin queue. 3027 * Disallow Connect and allow other fabrics commands. 3028 */ 3029 switch (cap_hdr->fctype) { 3030 case SPDK_NVMF_FABRIC_COMMAND_PROPERTY_SET: 3031 return nvmf_property_set(req); 3032 case SPDK_NVMF_FABRIC_COMMAND_PROPERTY_GET: 3033 return nvmf_property_get(req); 3034 default: 3035 SPDK_DEBUGLOG(nvmf, "unknown fctype 0x%02x\n", 3036 cap_hdr->fctype); 3037 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3038 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3039 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3040 } 3041 } else { 3042 /* Controller session is established, and this is an I/O queue */ 3043 /* For now, no I/O-specific Fabrics commands are implemented (other than Connect) */ 3044 SPDK_DEBUGLOG(nvmf, "Unexpected I/O fctype 0x%x\n", cap_hdr->fctype); 3045 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3046 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3047 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3048 } 3049 } 3050 3051 static inline int 3052 nvmf_ctrlr_async_event_notification(struct spdk_nvmf_ctrlr *ctrlr, 3053 union spdk_nvme_async_event_completion *event) 3054 { 3055 struct spdk_nvmf_request *req; 3056 struct spdk_nvme_cpl *rsp; 3057 3058 assert(ctrlr->nr_aer_reqs > 0); 3059 3060 req = ctrlr->aer_req[--ctrlr->nr_aer_reqs]; 3061 rsp = &req->rsp->nvme_cpl; 3062 3063 rsp->cdw0 = event->raw; 3064 3065 _nvmf_request_complete(req); 3066 ctrlr->aer_req[ctrlr->nr_aer_reqs] = NULL; 3067 3068 return 0; 3069 } 3070 3071 static inline void 3072 nvmf_ctrlr_queue_pending_async_event(struct spdk_nvmf_ctrlr *ctrlr, 3073 union spdk_nvme_async_event_completion *event) 3074 { 3075 struct spdk_nvmf_async_event_completion *nvmf_event; 3076 3077 nvmf_event = calloc(1, sizeof(*nvmf_event)); 3078 if (!nvmf_event) { 3079 SPDK_ERRLOG("Alloc nvmf event failed, ignore the event\n"); 3080 return; 3081 } 3082 nvmf_event->event.raw = event->raw; 3083 STAILQ_INSERT_TAIL(&ctrlr->async_events, nvmf_event, link); 3084 } 3085 3086 int 3087 nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr) 3088 { 3089 union spdk_nvme_async_event_completion event = {0}; 3090 3091 /* Users may disable the event notification */ 3092 if (!ctrlr->feat.async_event_configuration.bits.ns_attr_notice) { 3093 return 0; 3094 } 3095 3096 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT)) { 3097 return 0; 3098 } 3099 3100 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 3101 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED; 3102 event.bits.log_page_identifier = SPDK_NVME_LOG_CHANGED_NS_LIST; 3103 3104 /* If there is no outstanding AER request, queue the event. Then 3105 * if an AER is later submitted, this event can be sent as a 3106 * response. 3107 */ 3108 if (ctrlr->nr_aer_reqs == 0) { 3109 nvmf_ctrlr_queue_pending_async_event(ctrlr, &event); 3110 return 0; 3111 } 3112 3113 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 3114 } 3115 3116 int 3117 nvmf_ctrlr_async_event_ana_change_notice(struct spdk_nvmf_ctrlr *ctrlr) 3118 { 3119 union spdk_nvme_async_event_completion event = {0}; 3120 3121 /* Users may disable the event notification */ 3122 if (!ctrlr->feat.async_event_configuration.bits.ana_change_notice) { 3123 return 0; 3124 } 3125 3126 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT)) { 3127 return 0; 3128 } 3129 3130 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 3131 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_ANA_CHANGE; 3132 event.bits.log_page_identifier = SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS; 3133 3134 /* If there is no outstanding AER request, queue the event. Then 3135 * if an AER is later submitted, this event can be sent as a 3136 * response. 3137 */ 3138 if (ctrlr->nr_aer_reqs == 0) { 3139 nvmf_ctrlr_queue_pending_async_event(ctrlr, &event); 3140 return 0; 3141 } 3142 3143 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 3144 } 3145 3146 void 3147 nvmf_ctrlr_async_event_reservation_notification(struct spdk_nvmf_ctrlr *ctrlr) 3148 { 3149 union spdk_nvme_async_event_completion event = {0}; 3150 3151 if (!ctrlr->num_avail_log_pages) { 3152 return; 3153 } 3154 3155 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT)) { 3156 return; 3157 } 3158 3159 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_IO; 3160 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL; 3161 event.bits.log_page_identifier = SPDK_NVME_LOG_RESERVATION_NOTIFICATION; 3162 3163 /* If there is no outstanding AER request, queue the event. Then 3164 * if an AER is later submitted, this event can be sent as a 3165 * response. 3166 */ 3167 if (ctrlr->nr_aer_reqs == 0) { 3168 nvmf_ctrlr_queue_pending_async_event(ctrlr, &event); 3169 return; 3170 } 3171 3172 nvmf_ctrlr_async_event_notification(ctrlr, &event); 3173 } 3174 3175 int 3176 nvmf_ctrlr_async_event_discovery_log_change_notice(struct spdk_nvmf_ctrlr *ctrlr) 3177 { 3178 union spdk_nvme_async_event_completion event = {0}; 3179 3180 /* Users may disable the event notification manually or 3181 * it may not be enabled due to keep alive timeout 3182 * not being set in connect command to discovery controller. 3183 */ 3184 if (!ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice) { 3185 return 0; 3186 } 3187 3188 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT)) { 3189 return 0; 3190 } 3191 3192 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 3193 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE; 3194 event.bits.log_page_identifier = SPDK_NVME_LOG_DISCOVERY; 3195 3196 /* If there is no outstanding AER request, queue the event. Then 3197 * if an AER is later submitted, this event can be sent as a 3198 * response. 3199 */ 3200 if (ctrlr->nr_aer_reqs == 0) { 3201 nvmf_ctrlr_queue_pending_async_event(ctrlr, &event); 3202 return 0; 3203 } 3204 3205 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 3206 } 3207 3208 void 3209 nvmf_qpair_free_aer(struct spdk_nvmf_qpair *qpair) 3210 { 3211 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 3212 int i; 3213 3214 if (!nvmf_qpair_is_admin_queue(qpair)) { 3215 return; 3216 } 3217 3218 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 3219 spdk_nvmf_request_free(ctrlr->aer_req[i]); 3220 ctrlr->aer_req[i] = NULL; 3221 } 3222 3223 ctrlr->nr_aer_reqs = 0; 3224 } 3225 3226 void 3227 nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr) 3228 { 3229 struct spdk_nvmf_request *req; 3230 int i; 3231 3232 if (!ctrlr->nr_aer_reqs) { 3233 return; 3234 } 3235 3236 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 3237 req = ctrlr->aer_req[i]; 3238 3239 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3240 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 3241 _nvmf_request_complete(req); 3242 3243 ctrlr->aer_req[i] = NULL; 3244 } 3245 3246 ctrlr->nr_aer_reqs = 0; 3247 } 3248 3249 static void 3250 _nvmf_ctrlr_add_reservation_log(void *ctx) 3251 { 3252 struct spdk_nvmf_reservation_log *log = (struct spdk_nvmf_reservation_log *)ctx; 3253 struct spdk_nvmf_ctrlr *ctrlr = log->ctrlr; 3254 3255 ctrlr->log_page_count++; 3256 3257 /* Maximum number of queued log pages is 255 */ 3258 if (ctrlr->num_avail_log_pages == 0xff) { 3259 struct spdk_nvmf_reservation_log *entry; 3260 entry = TAILQ_LAST(&ctrlr->log_head, log_page_head); 3261 entry->log.log_page_count = ctrlr->log_page_count; 3262 free(log); 3263 return; 3264 } 3265 3266 log->log.log_page_count = ctrlr->log_page_count; 3267 log->log.num_avail_log_pages = ctrlr->num_avail_log_pages++; 3268 TAILQ_INSERT_TAIL(&ctrlr->log_head, log, link); 3269 3270 nvmf_ctrlr_async_event_reservation_notification(ctrlr); 3271 } 3272 3273 void 3274 nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr, 3275 struct spdk_nvmf_ns *ns, 3276 enum spdk_nvme_reservation_notification_log_page_type type) 3277 { 3278 struct spdk_nvmf_reservation_log *log; 3279 3280 switch (type) { 3281 case SPDK_NVME_RESERVATION_LOG_PAGE_EMPTY: 3282 return; 3283 case SPDK_NVME_REGISTRATION_PREEMPTED: 3284 if (ns->mask & SPDK_NVME_REGISTRATION_PREEMPTED_MASK) { 3285 return; 3286 } 3287 break; 3288 case SPDK_NVME_RESERVATION_RELEASED: 3289 if (ns->mask & SPDK_NVME_RESERVATION_RELEASED_MASK) { 3290 return; 3291 } 3292 break; 3293 case SPDK_NVME_RESERVATION_PREEMPTED: 3294 if (ns->mask & SPDK_NVME_RESERVATION_PREEMPTED_MASK) { 3295 return; 3296 } 3297 break; 3298 default: 3299 return; 3300 } 3301 3302 log = calloc(1, sizeof(*log)); 3303 if (!log) { 3304 SPDK_ERRLOG("Alloc log page failed, ignore the log\n"); 3305 return; 3306 } 3307 log->ctrlr = ctrlr; 3308 log->log.type = type; 3309 log->log.nsid = ns->nsid; 3310 3311 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_add_reservation_log, log); 3312 } 3313 3314 /* Check from subsystem poll group's namespace information data structure */ 3315 static bool 3316 nvmf_ns_info_ctrlr_is_registrant(struct spdk_nvmf_subsystem_pg_ns_info *ns_info, 3317 struct spdk_nvmf_ctrlr *ctrlr) 3318 { 3319 uint32_t i; 3320 3321 for (i = 0; i < SPDK_NVMF_MAX_NUM_REGISTRANTS; i++) { 3322 if (!spdk_uuid_compare(&ns_info->reg_hostid[i], &ctrlr->hostid)) { 3323 return true; 3324 } 3325 } 3326 3327 return false; 3328 } 3329 3330 /* 3331 * Check the NVMe command is permitted or not for current controller(Host). 3332 */ 3333 static int 3334 nvmf_ns_reservation_request_check(struct spdk_nvmf_subsystem_pg_ns_info *ns_info, 3335 struct spdk_nvmf_ctrlr *ctrlr, 3336 struct spdk_nvmf_request *req) 3337 { 3338 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3339 enum spdk_nvme_reservation_type rtype = ns_info->rtype; 3340 uint8_t status = SPDK_NVME_SC_SUCCESS; 3341 uint8_t racqa; 3342 bool is_registrant; 3343 3344 /* No valid reservation */ 3345 if (!rtype) { 3346 return 0; 3347 } 3348 3349 is_registrant = nvmf_ns_info_ctrlr_is_registrant(ns_info, ctrlr); 3350 /* All registrants type and current ctrlr is a valid registrant */ 3351 if ((rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE_ALL_REGS || 3352 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_ALL_REGS) && is_registrant) { 3353 return 0; 3354 } else if (!spdk_uuid_compare(&ns_info->holder_id, &ctrlr->hostid)) { 3355 return 0; 3356 } 3357 3358 /* Non-holder for current controller */ 3359 switch (cmd->opc) { 3360 case SPDK_NVME_OPC_READ: 3361 case SPDK_NVME_OPC_COMPARE: 3362 if (rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) { 3363 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 3364 goto exit; 3365 } 3366 if ((rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_REG_ONLY || 3367 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_ALL_REGS) && !is_registrant) { 3368 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 3369 } 3370 break; 3371 case SPDK_NVME_OPC_FLUSH: 3372 case SPDK_NVME_OPC_WRITE: 3373 case SPDK_NVME_OPC_WRITE_UNCORRECTABLE: 3374 case SPDK_NVME_OPC_WRITE_ZEROES: 3375 case SPDK_NVME_OPC_DATASET_MANAGEMENT: 3376 if (rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE || 3377 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) { 3378 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 3379 goto exit; 3380 } 3381 if (!is_registrant) { 3382 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 3383 } 3384 break; 3385 case SPDK_NVME_OPC_RESERVATION_ACQUIRE: 3386 racqa = cmd->cdw10_bits.resv_acquire.racqa; 3387 if (racqa == SPDK_NVME_RESERVE_ACQUIRE) { 3388 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 3389 goto exit; 3390 } 3391 if (!is_registrant) { 3392 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 3393 } 3394 break; 3395 case SPDK_NVME_OPC_RESERVATION_RELEASE: 3396 if (!is_registrant) { 3397 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 3398 } 3399 break; 3400 default: 3401 break; 3402 } 3403 3404 exit: 3405 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3406 req->rsp->nvme_cpl.status.sc = status; 3407 if (status == SPDK_NVME_SC_RESERVATION_CONFLICT) { 3408 return -EPERM; 3409 } 3410 3411 return 0; 3412 } 3413 3414 static int 3415 nvmf_ctrlr_process_io_fused_cmd(struct spdk_nvmf_request *req, struct spdk_bdev *bdev, 3416 struct spdk_bdev_desc *desc, struct spdk_io_channel *ch) 3417 { 3418 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3419 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3420 struct spdk_nvmf_request *first_fused_req = req->qpair->first_fused_req; 3421 int rc; 3422 3423 if (cmd->fuse == SPDK_NVME_CMD_FUSE_FIRST) { 3424 /* first fused operation (should be compare) */ 3425 if (first_fused_req != NULL) { 3426 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 3427 3428 SPDK_ERRLOG("Wrong sequence of fused operations\n"); 3429 3430 /* abort req->qpair->first_fused_request and continue with new fused command */ 3431 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 3432 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 3433 _nvmf_request_complete(first_fused_req); 3434 } else if (cmd->opc != SPDK_NVME_OPC_COMPARE) { 3435 SPDK_ERRLOG("Wrong op code of fused operations\n"); 3436 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3437 rsp->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3438 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3439 } 3440 3441 req->qpair->first_fused_req = req; 3442 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 3443 } else if (cmd->fuse == SPDK_NVME_CMD_FUSE_SECOND) { 3444 /* second fused operation (should be write) */ 3445 if (first_fused_req == NULL) { 3446 SPDK_ERRLOG("Wrong sequence of fused operations\n"); 3447 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3448 rsp->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 3449 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3450 } else if (cmd->opc != SPDK_NVME_OPC_WRITE) { 3451 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 3452 3453 SPDK_ERRLOG("Wrong op code of fused operations\n"); 3454 3455 /* abort req->qpair->first_fused_request and fail current command */ 3456 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 3457 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 3458 _nvmf_request_complete(first_fused_req); 3459 3460 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3461 rsp->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3462 req->qpair->first_fused_req = NULL; 3463 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3464 } 3465 3466 /* save request of first command to generate response later */ 3467 req->first_fused_req = first_fused_req; 3468 req->qpair->first_fused_req = NULL; 3469 } else { 3470 SPDK_ERRLOG("Invalid fused command fuse field.\n"); 3471 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3472 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3473 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3474 } 3475 3476 rc = nvmf_bdev_ctrlr_compare_and_write_cmd(bdev, desc, ch, req->first_fused_req, req); 3477 3478 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 3479 if (spdk_nvme_cpl_is_error(rsp)) { 3480 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 3481 3482 fused_response->status = rsp->status; 3483 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3484 rsp->status.sc = SPDK_NVME_SC_ABORTED_FAILED_FUSED; 3485 /* Complete first of fused commands. Second will be completed by upper layer */ 3486 _nvmf_request_complete(first_fused_req); 3487 req->first_fused_req = NULL; 3488 } 3489 } 3490 3491 return rc; 3492 } 3493 3494 int 3495 nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) 3496 { 3497 uint32_t nsid; 3498 struct spdk_nvmf_ns *ns; 3499 struct spdk_bdev *bdev; 3500 struct spdk_bdev_desc *desc; 3501 struct spdk_io_channel *ch; 3502 struct spdk_nvmf_poll_group *group = req->qpair->group; 3503 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3504 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3505 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 3506 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 3507 enum spdk_nvme_ana_state ana_state; 3508 3509 /* pre-set response details for this command */ 3510 response->status.sc = SPDK_NVME_SC_SUCCESS; 3511 nsid = cmd->nsid; 3512 3513 if (spdk_unlikely(ctrlr == NULL)) { 3514 SPDK_ERRLOG("I/O command sent before CONNECT\n"); 3515 response->status.sct = SPDK_NVME_SCT_GENERIC; 3516 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 3517 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3518 } 3519 3520 if (spdk_unlikely(ctrlr->vcprop.cc.bits.en != 1)) { 3521 SPDK_ERRLOG("I/O command sent to disabled controller\n"); 3522 response->status.sct = SPDK_NVME_SCT_GENERIC; 3523 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 3524 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3525 } 3526 3527 /* It will be lower overhead to check if ANA state is optimized or 3528 * non-optimized. 3529 */ 3530 ana_state = ctrlr->listener->ana_state; 3531 if (spdk_unlikely(ana_state != SPDK_NVME_ANA_OPTIMIZED_STATE && 3532 ana_state != SPDK_NVME_ANA_NON_OPTIMIZED_STATE)) { 3533 SPDK_DEBUGLOG(nvmf, "Fail I/O command due to ANA state %d\n", 3534 ana_state); 3535 response->status.sct = SPDK_NVME_SCT_PATH; 3536 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 3537 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3538 } 3539 3540 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, nsid); 3541 if (ns == NULL || ns->bdev == NULL) { 3542 SPDK_DEBUGLOG(nvmf, "Unsuccessful query for nsid %u\n", cmd->nsid); 3543 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 3544 response->status.dnr = 1; 3545 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3546 } 3547 3548 /* scan-build falsely reporting dereference of null pointer */ 3549 assert(group != NULL && group->sgroups != NULL); 3550 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[nsid - 1]; 3551 if (nvmf_ns_reservation_request_check(ns_info, ctrlr, req)) { 3552 SPDK_DEBUGLOG(nvmf, "Reservation Conflict for nsid %u, opcode %u\n", 3553 cmd->nsid, cmd->opc); 3554 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3555 } 3556 3557 bdev = ns->bdev; 3558 desc = ns->desc; 3559 ch = ns_info->channel; 3560 3561 if (spdk_unlikely(cmd->fuse & SPDK_NVME_CMD_FUSE_MASK)) { 3562 return nvmf_ctrlr_process_io_fused_cmd(req, bdev, desc, ch); 3563 } else if (spdk_unlikely(req->qpair->first_fused_req != NULL)) { 3564 struct spdk_nvme_cpl *fused_response = &req->qpair->first_fused_req->rsp->nvme_cpl; 3565 3566 SPDK_ERRLOG("Expected second of fused commands - failing first of fused commands\n"); 3567 3568 /* abort req->qpair->first_fused_request and continue with new command */ 3569 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 3570 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 3571 _nvmf_request_complete(req->qpair->first_fused_req); 3572 req->qpair->first_fused_req = NULL; 3573 } 3574 3575 switch (cmd->opc) { 3576 case SPDK_NVME_OPC_READ: 3577 return nvmf_bdev_ctrlr_read_cmd(bdev, desc, ch, req); 3578 case SPDK_NVME_OPC_WRITE: 3579 return nvmf_bdev_ctrlr_write_cmd(bdev, desc, ch, req); 3580 case SPDK_NVME_OPC_COMPARE: 3581 return nvmf_bdev_ctrlr_compare_cmd(bdev, desc, ch, req); 3582 case SPDK_NVME_OPC_WRITE_ZEROES: 3583 return nvmf_bdev_ctrlr_write_zeroes_cmd(bdev, desc, ch, req); 3584 case SPDK_NVME_OPC_FLUSH: 3585 return nvmf_bdev_ctrlr_flush_cmd(bdev, desc, ch, req); 3586 case SPDK_NVME_OPC_DATASET_MANAGEMENT: 3587 return nvmf_bdev_ctrlr_dsm_cmd(bdev, desc, ch, req); 3588 case SPDK_NVME_OPC_RESERVATION_REGISTER: 3589 case SPDK_NVME_OPC_RESERVATION_ACQUIRE: 3590 case SPDK_NVME_OPC_RESERVATION_RELEASE: 3591 case SPDK_NVME_OPC_RESERVATION_REPORT: 3592 spdk_thread_send_msg(ctrlr->subsys->thread, nvmf_ns_reservation_request, req); 3593 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 3594 default: 3595 return nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req); 3596 } 3597 } 3598 3599 static void 3600 nvmf_qpair_request_cleanup(struct spdk_nvmf_qpair *qpair) 3601 { 3602 if (qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING) { 3603 assert(qpair->state_cb != NULL); 3604 3605 if (TAILQ_EMPTY(&qpair->outstanding)) { 3606 qpair->state_cb(qpair->state_cb_arg, 0); 3607 } 3608 } 3609 } 3610 3611 int 3612 spdk_nvmf_request_free(struct spdk_nvmf_request *req) 3613 { 3614 struct spdk_nvmf_qpair *qpair = req->qpair; 3615 3616 TAILQ_REMOVE(&qpair->outstanding, req, link); 3617 if (nvmf_transport_req_free(req)) { 3618 SPDK_ERRLOG("Unable to free transport level request resources.\n"); 3619 } 3620 3621 nvmf_qpair_request_cleanup(qpair); 3622 3623 return 0; 3624 } 3625 3626 static void 3627 _nvmf_request_complete(void *ctx) 3628 { 3629 struct spdk_nvmf_request *req = ctx; 3630 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3631 struct spdk_nvmf_qpair *qpair; 3632 struct spdk_nvmf_subsystem_poll_group *sgroup = NULL; 3633 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 3634 bool is_aer = false; 3635 uint32_t nsid; 3636 bool paused; 3637 uint8_t opcode; 3638 3639 rsp->sqid = 0; 3640 rsp->status.p = 0; 3641 rsp->cid = req->cmd->nvme_cmd.cid; 3642 nsid = req->cmd->nvme_cmd.nsid; 3643 opcode = req->cmd->nvmf_cmd.opcode; 3644 3645 qpair = req->qpair; 3646 if (qpair->ctrlr) { 3647 sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id]; 3648 assert(sgroup != NULL); 3649 is_aer = req->cmd->nvme_cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST; 3650 3651 /* 3652 * Set the crd value. 3653 * If the the IO has any error, and dnr (DoNotRetry) is not 1, 3654 * and ACRE is enabled, we will set the crd to 1 to select the first CRDT. 3655 */ 3656 if (spdk_nvme_cpl_is_error(rsp) && 3657 rsp->status.dnr == 0 && 3658 qpair->ctrlr->acre_enabled) { 3659 rsp->status.crd = 1; 3660 } 3661 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) { 3662 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 3663 } 3664 3665 if (SPDK_DEBUGLOG_FLAG_ENABLED("nvmf")) { 3666 spdk_nvme_print_completion(qpair->qid, rsp); 3667 } 3668 3669 TAILQ_REMOVE(&qpair->outstanding, req, link); 3670 if (nvmf_transport_req_complete(req)) { 3671 SPDK_ERRLOG("Transport request completion error!\n"); 3672 } 3673 3674 /* AER cmd is an exception */ 3675 if (sgroup && !is_aer) { 3676 if (spdk_unlikely(opcode == SPDK_NVME_OPC_FABRIC || 3677 nvmf_qpair_is_admin_queue(qpair))) { 3678 assert(sgroup->mgmt_io_outstanding > 0); 3679 sgroup->mgmt_io_outstanding--; 3680 } else { 3681 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 3682 if (spdk_likely(nsid - 1 < sgroup->num_ns)) { 3683 sgroup->ns_info[nsid - 1].io_outstanding--; 3684 } 3685 } 3686 3687 if (spdk_unlikely(sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSING && 3688 sgroup->mgmt_io_outstanding == 0)) { 3689 paused = true; 3690 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 3691 ns_info = &sgroup->ns_info[nsid]; 3692 3693 if (ns_info->state == SPDK_NVMF_SUBSYSTEM_PAUSING && 3694 ns_info->io_outstanding > 0) { 3695 paused = false; 3696 break; 3697 } 3698 } 3699 3700 if (paused) { 3701 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 3702 sgroup->cb_fn(sgroup->cb_arg, 0); 3703 sgroup->cb_fn = NULL; 3704 sgroup->cb_arg = NULL; 3705 } 3706 } 3707 3708 } 3709 3710 nvmf_qpair_request_cleanup(qpair); 3711 } 3712 3713 int 3714 spdk_nvmf_request_complete(struct spdk_nvmf_request *req) 3715 { 3716 struct spdk_nvmf_qpair *qpair = req->qpair; 3717 3718 if (spdk_likely(qpair->group->thread == spdk_get_thread())) { 3719 _nvmf_request_complete(req); 3720 } else { 3721 spdk_thread_send_msg(qpair->group->thread, 3722 _nvmf_request_complete, req); 3723 } 3724 3725 return 0; 3726 } 3727 3728 void 3729 spdk_nvmf_request_exec_fabrics(struct spdk_nvmf_request *req) 3730 { 3731 struct spdk_nvmf_qpair *qpair = req->qpair; 3732 struct spdk_nvmf_subsystem_poll_group *sgroup = NULL; 3733 enum spdk_nvmf_request_exec_status status; 3734 3735 if (qpair->ctrlr) { 3736 sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id]; 3737 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) { 3738 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 3739 } 3740 3741 assert(sgroup != NULL); 3742 sgroup->mgmt_io_outstanding++; 3743 3744 /* Place the request on the outstanding list so we can keep track of it */ 3745 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 3746 3747 assert(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC); 3748 status = nvmf_ctrlr_process_fabrics_cmd(req); 3749 3750 if (status == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 3751 _nvmf_request_complete(req); 3752 } 3753 } 3754 3755 void 3756 spdk_nvmf_request_exec(struct spdk_nvmf_request *req) 3757 { 3758 struct spdk_nvmf_qpair *qpair = req->qpair; 3759 struct spdk_nvmf_subsystem_poll_group *sgroup = NULL; 3760 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 3761 enum spdk_nvmf_request_exec_status status; 3762 uint32_t nsid; 3763 3764 if (qpair->ctrlr) { 3765 sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id]; 3766 assert(sgroup != NULL); 3767 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) { 3768 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 3769 } 3770 3771 /* Check if the subsystem is paused (if there is a subsystem) */ 3772 if (sgroup != NULL) { 3773 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC || 3774 nvmf_qpair_is_admin_queue(qpair))) { 3775 if (sgroup->state != SPDK_NVMF_SUBSYSTEM_ACTIVE) { 3776 /* The subsystem is not currently active. Queue this request. */ 3777 TAILQ_INSERT_TAIL(&sgroup->queued, req, link); 3778 return; 3779 } 3780 sgroup->mgmt_io_outstanding++; 3781 } else { 3782 nsid = req->cmd->nvme_cmd.nsid; 3783 3784 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 3785 if (spdk_unlikely(nsid - 1 >= sgroup->num_ns)) { 3786 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3787 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 3788 req->rsp->nvme_cpl.status.dnr = 1; 3789 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 3790 _nvmf_request_complete(req); 3791 return; 3792 } 3793 3794 ns_info = &sgroup->ns_info[nsid - 1]; 3795 if (ns_info->channel == NULL) { 3796 /* This can can happen if host sends I/O to a namespace that is 3797 * in the process of being added, but before the full addition 3798 * process is complete. Report invalid namespace in that case. 3799 */ 3800 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3801 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 3802 req->rsp->nvme_cpl.status.dnr = 1; 3803 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 3804 ns_info->io_outstanding++; 3805 _nvmf_request_complete(req); 3806 return; 3807 } 3808 3809 if (ns_info->state != SPDK_NVMF_SUBSYSTEM_ACTIVE) { 3810 /* The namespace is not currently active. Queue this request. */ 3811 TAILQ_INSERT_TAIL(&sgroup->queued, req, link); 3812 return; 3813 } 3814 ns_info->io_outstanding++; 3815 } 3816 } 3817 3818 if (qpair->state != SPDK_NVMF_QPAIR_ACTIVE) { 3819 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3820 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 3821 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 3822 _nvmf_request_complete(req); 3823 return; 3824 } 3825 3826 if (SPDK_DEBUGLOG_FLAG_ENABLED("nvmf")) { 3827 spdk_nvme_print_command(qpair->qid, &req->cmd->nvme_cmd); 3828 } 3829 3830 /* Place the request on the outstanding list so we can keep track of it */ 3831 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 3832 3833 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC)) { 3834 status = nvmf_ctrlr_process_fabrics_cmd(req); 3835 } else if (spdk_unlikely(nvmf_qpair_is_admin_queue(qpair))) { 3836 status = nvmf_ctrlr_process_admin_cmd(req); 3837 } else { 3838 status = nvmf_ctrlr_process_io_cmd(req); 3839 } 3840 3841 if (status == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 3842 _nvmf_request_complete(req); 3843 } 3844 } 3845 3846 static bool 3847 nvmf_ctrlr_get_dif_ctx(struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvme_cmd *cmd, 3848 struct spdk_dif_ctx *dif_ctx) 3849 { 3850 struct spdk_nvmf_ns *ns; 3851 struct spdk_bdev *bdev; 3852 3853 if (ctrlr == NULL || cmd == NULL) { 3854 return false; 3855 } 3856 3857 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, cmd->nsid); 3858 if (ns == NULL || ns->bdev == NULL) { 3859 return false; 3860 } 3861 3862 bdev = ns->bdev; 3863 3864 switch (cmd->opc) { 3865 case SPDK_NVME_OPC_READ: 3866 case SPDK_NVME_OPC_WRITE: 3867 case SPDK_NVME_OPC_COMPARE: 3868 return nvmf_bdev_ctrlr_get_dif_ctx(bdev, cmd, dif_ctx); 3869 default: 3870 break; 3871 } 3872 3873 return false; 3874 } 3875 3876 bool 3877 spdk_nvmf_request_get_dif_ctx(struct spdk_nvmf_request *req, struct spdk_dif_ctx *dif_ctx) 3878 { 3879 struct spdk_nvmf_qpair *qpair = req->qpair; 3880 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 3881 3882 if (spdk_likely(ctrlr == NULL || !ctrlr->dif_insert_or_strip)) { 3883 return false; 3884 } 3885 3886 if (spdk_unlikely(qpair->state != SPDK_NVMF_QPAIR_ACTIVE)) { 3887 return false; 3888 } 3889 3890 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC)) { 3891 return false; 3892 } 3893 3894 if (spdk_unlikely(nvmf_qpair_is_admin_queue(qpair))) { 3895 return false; 3896 } 3897 3898 return nvmf_ctrlr_get_dif_ctx(ctrlr, &req->cmd->nvme_cmd, dif_ctx); 3899 } 3900 3901 void 3902 spdk_nvmf_set_custom_admin_cmd_hdlr(uint8_t opc, spdk_nvmf_custom_cmd_hdlr hdlr) 3903 { 3904 g_nvmf_custom_admin_cmd_hdlrs[opc].hdlr = hdlr; 3905 } 3906 3907 static int 3908 nvmf_passthru_admin_cmd(struct spdk_nvmf_request *req) 3909 { 3910 struct spdk_bdev *bdev; 3911 struct spdk_bdev_desc *desc; 3912 struct spdk_io_channel *ch; 3913 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req); 3914 struct spdk_nvme_cpl *response = spdk_nvmf_request_get_response(req); 3915 uint32_t bdev_nsid; 3916 int rc; 3917 3918 if (g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].nsid == 0) { 3919 bdev_nsid = cmd->nsid; 3920 } else { 3921 bdev_nsid = g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].nsid; 3922 } 3923 3924 rc = spdk_nvmf_request_get_bdev(bdev_nsid, req, &bdev, &desc, &ch); 3925 if (rc) { 3926 response->status.sct = SPDK_NVME_SCT_GENERIC; 3927 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 3928 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3929 } 3930 return spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(bdev, desc, ch, req, NULL); 3931 } 3932 3933 void 3934 spdk_nvmf_set_passthru_admin_cmd(uint8_t opc, uint32_t forward_nsid) 3935 { 3936 g_nvmf_custom_admin_cmd_hdlrs[opc].hdlr = nvmf_passthru_admin_cmd; 3937 g_nvmf_custom_admin_cmd_hdlrs[opc].nsid = forward_nsid; 3938 } 3939 3940 int 3941 spdk_nvmf_request_get_bdev(uint32_t nsid, struct spdk_nvmf_request *req, 3942 struct spdk_bdev **bdev, struct spdk_bdev_desc **desc, struct spdk_io_channel **ch) 3943 { 3944 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3945 struct spdk_nvmf_ns *ns; 3946 struct spdk_nvmf_poll_group *group = req->qpair->group; 3947 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 3948 3949 *bdev = NULL; 3950 *desc = NULL; 3951 *ch = NULL; 3952 3953 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, nsid); 3954 if (ns == NULL || ns->bdev == NULL) { 3955 return -EINVAL; 3956 } 3957 3958 assert(group != NULL && group->sgroups != NULL); 3959 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[nsid - 1]; 3960 *bdev = ns->bdev; 3961 *desc = ns->desc; 3962 *ch = ns_info->channel; 3963 3964 return 0; 3965 } 3966 3967 struct spdk_nvmf_ctrlr *spdk_nvmf_request_get_ctrlr(struct spdk_nvmf_request *req) 3968 { 3969 return req->qpair->ctrlr; 3970 } 3971 3972 struct spdk_nvme_cmd *spdk_nvmf_request_get_cmd(struct spdk_nvmf_request *req) 3973 { 3974 return &req->cmd->nvme_cmd; 3975 } 3976 3977 struct spdk_nvme_cpl *spdk_nvmf_request_get_response(struct spdk_nvmf_request *req) 3978 { 3979 return &req->rsp->nvme_cpl; 3980 } 3981 3982 struct spdk_nvmf_subsystem *spdk_nvmf_request_get_subsystem(struct spdk_nvmf_request *req) 3983 { 3984 return req->qpair->ctrlr->subsys; 3985 } 3986 3987 void spdk_nvmf_request_get_data(struct spdk_nvmf_request *req, void **data, uint32_t *length) 3988 { 3989 *data = req->data; 3990 *length = req->length; 3991 } 3992 3993 struct spdk_nvmf_subsystem *spdk_nvmf_ctrlr_get_subsystem(struct spdk_nvmf_ctrlr *ctrlr) 3994 { 3995 return ctrlr->subsys; 3996 } 3997 3998 uint16_t spdk_nvmf_ctrlr_get_id(struct spdk_nvmf_ctrlr *ctrlr) 3999 { 4000 return ctrlr->cntlid; 4001 } 4002 4003 struct spdk_nvmf_request *spdk_nvmf_request_get_req_to_abort(struct spdk_nvmf_request *req) 4004 { 4005 return req->req_to_abort; 4006 } 4007