1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019, 2020 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "nvmf_internal.h" 10 #include "transport.h" 11 12 #include "spdk/bdev.h" 13 #include "spdk/bdev_zone.h" 14 #include "spdk/bit_array.h" 15 #include "spdk/endian.h" 16 #include "spdk/thread.h" 17 #include "spdk/nvme_spec.h" 18 #include "spdk/nvmf_cmd.h" 19 #include "spdk/string.h" 20 #include "spdk/util.h" 21 #include "spdk/version.h" 22 #include "spdk/log.h" 23 #include "spdk_internal/usdt.h" 24 25 #define MIN_KEEP_ALIVE_TIMEOUT_IN_MS 10000 26 #define NVMF_DISC_KATO_IN_MS 120000 27 #define KAS_TIME_UNIT_IN_MS 100 28 #define KAS_DEFAULT_VALUE (MIN_KEEP_ALIVE_TIMEOUT_IN_MS / KAS_TIME_UNIT_IN_MS) 29 30 #define NVMF_CC_RESET_SHN_TIMEOUT_IN_MS 10000 31 32 #define NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS (NVMF_CC_RESET_SHN_TIMEOUT_IN_MS + 5000) 33 34 #define DUPLICATE_QID_RETRY_US 100 35 36 /* 37 * Report the SPDK version as the firmware revision. 38 * SPDK_VERSION_STRING won't fit into FR (only 8 bytes), so try to fit the most important parts. 39 */ 40 #define FW_VERSION SPDK_VERSION_MAJOR_STRING SPDK_VERSION_MINOR_STRING SPDK_VERSION_PATCH_STRING 41 42 #define ANA_TRANSITION_TIME_IN_SEC 10 43 44 #define NVMF_ABORT_COMMAND_LIMIT 3 45 46 /* 47 * Support for custom admin command handlers 48 */ 49 struct spdk_nvmf_custom_admin_cmd { 50 spdk_nvmf_custom_cmd_hdlr hdlr; 51 uint32_t nsid; /* nsid to forward */ 52 }; 53 54 static struct spdk_nvmf_custom_admin_cmd g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_MAX_OPC + 1]; 55 56 static void _nvmf_request_complete(void *ctx); 57 int nvmf_passthru_admin_cmd_for_ctrlr(struct spdk_nvmf_request *req, struct spdk_nvmf_ctrlr *ctrlr); 58 59 static inline void 60 nvmf_invalid_connect_response(struct spdk_nvmf_fabric_connect_rsp *rsp, 61 uint8_t iattr, uint16_t ipo) 62 { 63 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 64 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 65 rsp->status_code_specific.invalid.iattr = iattr; 66 rsp->status_code_specific.invalid.ipo = ipo; 67 } 68 69 #define SPDK_NVMF_INVALID_CONNECT_CMD(rsp, field) \ 70 nvmf_invalid_connect_response(rsp, 0, offsetof(struct spdk_nvmf_fabric_connect_cmd, field)) 71 #define SPDK_NVMF_INVALID_CONNECT_DATA(rsp, field) \ 72 nvmf_invalid_connect_response(rsp, 1, offsetof(struct spdk_nvmf_fabric_connect_data, field)) 73 74 75 static void 76 nvmf_ctrlr_stop_keep_alive_timer(struct spdk_nvmf_ctrlr *ctrlr) 77 { 78 if (!ctrlr) { 79 SPDK_ERRLOG("Controller is NULL\n"); 80 return; 81 } 82 83 if (ctrlr->keep_alive_poller == NULL) { 84 return; 85 } 86 87 SPDK_DEBUGLOG(nvmf, "Stop keep alive poller\n"); 88 spdk_poller_unregister(&ctrlr->keep_alive_poller); 89 } 90 91 static void 92 nvmf_ctrlr_stop_association_timer(struct spdk_nvmf_ctrlr *ctrlr) 93 { 94 if (!ctrlr) { 95 SPDK_ERRLOG("Controller is NULL\n"); 96 assert(false); 97 return; 98 } 99 100 if (ctrlr->association_timer == NULL) { 101 return; 102 } 103 104 SPDK_DEBUGLOG(nvmf, "Stop association timer\n"); 105 spdk_poller_unregister(&ctrlr->association_timer); 106 } 107 108 static void 109 nvmf_ctrlr_disconnect_qpairs_done(struct spdk_io_channel_iter *i, int status) 110 { 111 if (status == 0) { 112 SPDK_DEBUGLOG(nvmf, "ctrlr disconnect qpairs complete successfully\n"); 113 } else { 114 SPDK_ERRLOG("Fail to disconnect ctrlr qpairs\n"); 115 } 116 } 117 118 static int 119 _nvmf_ctrlr_disconnect_qpairs_on_pg(struct spdk_io_channel_iter *i, bool include_admin) 120 { 121 int rc = 0; 122 struct spdk_nvmf_ctrlr *ctrlr; 123 struct spdk_nvmf_qpair *qpair, *temp_qpair; 124 struct spdk_io_channel *ch; 125 struct spdk_nvmf_poll_group *group; 126 127 ctrlr = spdk_io_channel_iter_get_ctx(i); 128 ch = spdk_io_channel_iter_get_channel(i); 129 group = spdk_io_channel_get_ctx(ch); 130 131 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, temp_qpair) { 132 if (qpair->ctrlr == ctrlr && (include_admin || !nvmf_qpair_is_admin_queue(qpair))) { 133 rc = spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 134 if (rc) { 135 if (rc == -EINPROGRESS) { 136 rc = 0; 137 } else { 138 SPDK_ERRLOG("Qpair disconnect failed\n"); 139 return rc; 140 } 141 } 142 } 143 } 144 145 return rc; 146 } 147 148 static void 149 nvmf_ctrlr_disconnect_qpairs_on_pg(struct spdk_io_channel_iter *i) 150 { 151 spdk_for_each_channel_continue(i, _nvmf_ctrlr_disconnect_qpairs_on_pg(i, true)); 152 } 153 154 static void 155 nvmf_ctrlr_disconnect_io_qpairs_on_pg(struct spdk_io_channel_iter *i) 156 { 157 spdk_for_each_channel_continue(i, _nvmf_ctrlr_disconnect_qpairs_on_pg(i, false)); 158 } 159 160 static int 161 nvmf_ctrlr_keep_alive_poll(void *ctx) 162 { 163 uint64_t keep_alive_timeout_tick; 164 uint64_t now = spdk_get_ticks(); 165 struct spdk_nvmf_ctrlr *ctrlr = ctx; 166 167 if (ctrlr->in_destruct) { 168 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 169 return SPDK_POLLER_IDLE; 170 } 171 172 SPDK_DEBUGLOG(nvmf, "Polling ctrlr keep alive timeout\n"); 173 174 /* If the Keep alive feature is in use and the timer expires */ 175 keep_alive_timeout_tick = ctrlr->last_keep_alive_tick + 176 ctrlr->feat.keep_alive_timer.bits.kato * spdk_get_ticks_hz() / UINT64_C(1000); 177 if (now > keep_alive_timeout_tick) { 178 SPDK_NOTICELOG("Disconnecting host %s from subsystem %s due to keep alive timeout.\n", 179 ctrlr->hostnqn, ctrlr->subsys->subnqn); 180 /* set the Controller Fatal Status bit to '1' */ 181 if (ctrlr->vcprop.csts.bits.cfs == 0) { 182 nvmf_ctrlr_set_fatal_status(ctrlr); 183 184 /* 185 * disconnect qpairs, terminate Transport connection 186 * destroy ctrlr, break the host to controller association 187 * disconnect qpairs with qpair->ctrlr == ctrlr 188 */ 189 spdk_for_each_channel(ctrlr->subsys->tgt, 190 nvmf_ctrlr_disconnect_qpairs_on_pg, 191 ctrlr, 192 nvmf_ctrlr_disconnect_qpairs_done); 193 return SPDK_POLLER_BUSY; 194 } 195 } 196 197 return SPDK_POLLER_IDLE; 198 } 199 200 static void 201 nvmf_ctrlr_start_keep_alive_timer(struct spdk_nvmf_ctrlr *ctrlr) 202 { 203 if (!ctrlr) { 204 SPDK_ERRLOG("Controller is NULL\n"); 205 return; 206 } 207 208 /* if cleared to 0 then the Keep Alive Timer is disabled */ 209 if (ctrlr->feat.keep_alive_timer.bits.kato != 0) { 210 211 ctrlr->last_keep_alive_tick = spdk_get_ticks(); 212 213 SPDK_DEBUGLOG(nvmf, "Ctrlr add keep alive poller\n"); 214 ctrlr->keep_alive_poller = SPDK_POLLER_REGISTER(nvmf_ctrlr_keep_alive_poll, ctrlr, 215 ctrlr->feat.keep_alive_timer.bits.kato * 1000); 216 } 217 } 218 219 static int _retry_qid_check(void *ctx); 220 221 static void 222 ctrlr_add_qpair_and_send_rsp(struct spdk_nvmf_qpair *qpair, 223 struct spdk_nvmf_ctrlr *ctrlr, 224 struct spdk_nvmf_request *req) 225 { 226 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 227 228 assert(ctrlr->admin_qpair->group->thread == spdk_get_thread()); 229 230 if (spdk_bit_array_get(ctrlr->qpair_mask, qpair->qid)) { 231 if (qpair->connect_req != NULL) { 232 SPDK_ERRLOG("Got I/O connect with duplicate QID %u\n", qpair->qid); 233 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 234 rsp->status.sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER; 235 qpair->connect_req = NULL; 236 qpair->ctrlr = NULL; 237 spdk_nvmf_request_complete(req); 238 } else { 239 SPDK_WARNLOG("Duplicate QID detected, re-check in %dus\n", 240 DUPLICATE_QID_RETRY_US); 241 qpair->connect_req = req; 242 /* Set qpair->ctrlr here so that we'll have it when the poller expires. */ 243 qpair->ctrlr = ctrlr; 244 req->poller = SPDK_POLLER_REGISTER(_retry_qid_check, qpair, 245 DUPLICATE_QID_RETRY_US); 246 } 247 return; 248 } 249 250 qpair->ctrlr = ctrlr; 251 spdk_bit_array_set(ctrlr->qpair_mask, qpair->qid); 252 253 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 254 rsp->status_code_specific.success.cntlid = ctrlr->cntlid; 255 SPDK_DEBUGLOG(nvmf, "connect capsule response: cntlid = 0x%04x\n", 256 rsp->status_code_specific.success.cntlid); 257 spdk_nvmf_request_complete(req); 258 259 SPDK_DTRACE_PROBE4_TICKS(nvmf_ctrlr_add_qpair, qpair, qpair->qid, ctrlr->subsys->subnqn, 260 ctrlr->hostnqn); 261 } 262 263 static int 264 _retry_qid_check(void *ctx) 265 { 266 struct spdk_nvmf_qpair *qpair = ctx; 267 struct spdk_nvmf_request *req = qpair->connect_req; 268 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 269 270 spdk_poller_unregister(&req->poller); 271 ctrlr_add_qpair_and_send_rsp(qpair, ctrlr, req); 272 return SPDK_POLLER_BUSY; 273 } 274 275 static void 276 _nvmf_ctrlr_add_admin_qpair(void *ctx) 277 { 278 struct spdk_nvmf_request *req = ctx; 279 struct spdk_nvmf_qpair *qpair = req->qpair; 280 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 281 282 ctrlr->admin_qpair = qpair; 283 ctrlr->association_timeout = qpair->transport->opts.association_timeout; 284 nvmf_ctrlr_start_keep_alive_timer(ctrlr); 285 ctrlr_add_qpair_and_send_rsp(qpair, ctrlr, req); 286 } 287 288 static void 289 _nvmf_subsystem_add_ctrlr(void *ctx) 290 { 291 struct spdk_nvmf_request *req = ctx; 292 struct spdk_nvmf_qpair *qpair = req->qpair; 293 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 294 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 295 296 if (nvmf_subsystem_add_ctrlr(ctrlr->subsys, ctrlr)) { 297 SPDK_ERRLOG("Unable to add controller to subsystem\n"); 298 spdk_bit_array_free(&ctrlr->qpair_mask); 299 free(ctrlr); 300 qpair->ctrlr = NULL; 301 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 302 spdk_nvmf_request_complete(req); 303 return; 304 } 305 306 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_add_admin_qpair, req); 307 } 308 309 static void 310 nvmf_ctrlr_cdata_init(struct spdk_nvmf_transport *transport, struct spdk_nvmf_subsystem *subsystem, 311 struct spdk_nvmf_ctrlr_data *cdata) 312 { 313 cdata->aerl = SPDK_NVMF_MAX_ASYNC_EVENTS - 1; 314 cdata->kas = KAS_DEFAULT_VALUE; 315 cdata->vid = SPDK_PCI_VID_INTEL; 316 cdata->ssvid = SPDK_PCI_VID_INTEL; 317 /* INTEL OUI */ 318 cdata->ieee[0] = 0xe4; 319 cdata->ieee[1] = 0xd2; 320 cdata->ieee[2] = 0x5c; 321 cdata->oncs.compare = 1; 322 cdata->oncs.reservations = 1; 323 cdata->fuses.compare_and_write = 1; 324 cdata->oncs.copy = 1; 325 cdata->sgls.supported = 1; 326 cdata->sgls.keyed_sgl = 1; 327 cdata->sgls.sgl_offset = 1; 328 cdata->nvmf_specific.ioccsz = sizeof(struct spdk_nvme_cmd) / 16; 329 cdata->nvmf_specific.ioccsz += transport->opts.in_capsule_data_size / 16; 330 cdata->nvmf_specific.iorcsz = sizeof(struct spdk_nvme_cpl) / 16; 331 cdata->nvmf_specific.icdoff = 0; /* offset starts directly after SQE */ 332 cdata->nvmf_specific.ctrattr.ctrlr_model = SPDK_NVMF_CTRLR_MODEL_DYNAMIC; 333 cdata->nvmf_specific.msdbd = 1; 334 335 if (transport->ops->cdata_init) { 336 transport->ops->cdata_init(transport, subsystem, cdata); 337 } 338 } 339 340 static bool 341 nvmf_subsys_has_multi_iocs(struct spdk_nvmf_subsystem *subsystem) 342 { 343 struct spdk_nvmf_ns *ns; 344 uint32_t i; 345 346 for (i = 0; i < subsystem->max_nsid; i++) { 347 ns = subsystem->ns[i]; 348 if (ns && ns->bdev && spdk_bdev_is_zoned(ns->bdev)) { 349 return true; 350 } 351 } 352 return false; 353 } 354 355 static struct spdk_nvmf_ctrlr * 356 nvmf_ctrlr_create(struct spdk_nvmf_subsystem *subsystem, 357 struct spdk_nvmf_request *req, 358 struct spdk_nvmf_fabric_connect_cmd *connect_cmd, 359 struct spdk_nvmf_fabric_connect_data *connect_data) 360 { 361 struct spdk_nvmf_ctrlr *ctrlr; 362 struct spdk_nvmf_transport *transport = req->qpair->transport; 363 struct spdk_nvme_transport_id listen_trid = {}; 364 bool subsys_has_multi_iocs = false; 365 366 ctrlr = calloc(1, sizeof(*ctrlr)); 367 if (ctrlr == NULL) { 368 SPDK_ERRLOG("Memory allocation failed\n"); 369 return NULL; 370 } 371 372 if (spdk_nvme_trtype_is_fabrics(transport->ops->type)) { 373 ctrlr->dynamic_ctrlr = true; 374 } else { 375 ctrlr->cntlid = connect_data->cntlid; 376 } 377 378 SPDK_DTRACE_PROBE3_TICKS(nvmf_ctrlr_create, ctrlr, subsystem->subnqn, 379 spdk_thread_get_id(req->qpair->group->thread)); 380 381 STAILQ_INIT(&ctrlr->async_events); 382 TAILQ_INIT(&ctrlr->log_head); 383 ctrlr->subsys = subsystem; 384 ctrlr->thread = req->qpair->group->thread; 385 ctrlr->disconnect_in_progress = false; 386 387 ctrlr->qpair_mask = spdk_bit_array_create(transport->opts.max_qpairs_per_ctrlr); 388 if (!ctrlr->qpair_mask) { 389 SPDK_ERRLOG("Failed to allocate controller qpair mask\n"); 390 goto err_qpair_mask; 391 } 392 393 nvmf_ctrlr_cdata_init(transport, subsystem, &ctrlr->cdata); 394 395 /* 396 * KAS: This field indicates the granularity of the Keep Alive Timer in 100ms units. 397 * If this field is cleared to 0h, then Keep Alive is not supported. 398 */ 399 if (ctrlr->cdata.kas) { 400 ctrlr->feat.keep_alive_timer.bits.kato = spdk_divide_round_up(connect_cmd->kato, 401 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS) * 402 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS; 403 } 404 405 ctrlr->feat.async_event_configuration.bits.ns_attr_notice = 1; 406 if (ctrlr->subsys->flags.ana_reporting) { 407 ctrlr->feat.async_event_configuration.bits.ana_change_notice = 1; 408 } 409 ctrlr->feat.volatile_write_cache.bits.wce = 1; 410 /* Coalescing Disable */ 411 ctrlr->feat.interrupt_vector_configuration.bits.cd = 1; 412 413 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 414 /* 415 * If keep-alive timeout is not set, discovery controllers use some 416 * arbitrary high value in order to cleanup stale discovery sessions 417 * 418 * From the 1.0a nvme-of spec: 419 * "The Keep Alive command is reserved for 420 * Discovery controllers. A transport may specify a 421 * fixed Discovery controller activity timeout value 422 * (e.g., 2 minutes). If no commands are received 423 * by a Discovery controller within that time 424 * period, the controller may perform the 425 * actions for Keep Alive Timer expiration". 426 * 427 * From the 1.1 nvme-of spec: 428 * "A host requests an explicit persistent connection 429 * to a Discovery controller and Asynchronous Event Notifications from 430 * the Discovery controller on that persistent connection by specifying 431 * a non-zero Keep Alive Timer value in the Connect command." 432 * 433 * In case non-zero KATO is used, we enable discovery_log_change_notice 434 * otherwise we disable it and use default discovery controller KATO. 435 * KATO is in millisecond. 436 */ 437 if (ctrlr->feat.keep_alive_timer.bits.kato == 0) { 438 ctrlr->feat.keep_alive_timer.bits.kato = NVMF_DISC_KATO_IN_MS; 439 ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice = 0; 440 } else { 441 ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice = 1; 442 } 443 } 444 445 /* Subtract 1 for admin queue, 1 for 0's based */ 446 ctrlr->feat.number_of_queues.bits.ncqr = transport->opts.max_qpairs_per_ctrlr - 1 - 447 1; 448 ctrlr->feat.number_of_queues.bits.nsqr = transport->opts.max_qpairs_per_ctrlr - 1 - 449 1; 450 451 spdk_uuid_copy(&ctrlr->hostid, (struct spdk_uuid *)connect_data->hostid); 452 memcpy(ctrlr->hostnqn, connect_data->hostnqn, sizeof(ctrlr->hostnqn)); 453 454 ctrlr->vcprop.cap.raw = 0; 455 ctrlr->vcprop.cap.bits.cqr = 1; /* NVMe-oF specification required */ 456 ctrlr->vcprop.cap.bits.mqes = transport->opts.max_queue_depth - 457 1; /* max queue depth */ 458 ctrlr->vcprop.cap.bits.ams = 0; /* optional arb mechanisms */ 459 /* ready timeout - 500 msec units */ 460 ctrlr->vcprop.cap.bits.to = NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS / 500; 461 ctrlr->vcprop.cap.bits.dstrd = 0; /* fixed to 0 for NVMe-oF */ 462 subsys_has_multi_iocs = nvmf_subsys_has_multi_iocs(subsystem); 463 if (subsys_has_multi_iocs) { 464 ctrlr->vcprop.cap.bits.css = 465 SPDK_NVME_CAP_CSS_IOCS; /* One or more I/O command sets supported */ 466 } else { 467 ctrlr->vcprop.cap.bits.css = SPDK_NVME_CAP_CSS_NVM; /* NVM command set */ 468 } 469 470 ctrlr->vcprop.cap.bits.mpsmin = 0; /* 2 ^ (12 + mpsmin) == 4k */ 471 ctrlr->vcprop.cap.bits.mpsmax = 0; /* 2 ^ (12 + mpsmax) == 4k */ 472 473 /* Version Supported: 1.3 */ 474 ctrlr->vcprop.vs.bits.mjr = 1; 475 ctrlr->vcprop.vs.bits.mnr = 3; 476 ctrlr->vcprop.vs.bits.ter = 0; 477 478 ctrlr->vcprop.cc.raw = 0; 479 ctrlr->vcprop.cc.bits.en = 0; /* Init controller disabled */ 480 if (subsys_has_multi_iocs) { 481 ctrlr->vcprop.cc.bits.css = 482 SPDK_NVME_CC_CSS_IOCS; /* All supported I/O Command Sets */ 483 } 484 485 ctrlr->vcprop.csts.raw = 0; 486 ctrlr->vcprop.csts.bits.rdy = 0; /* Init controller as not ready */ 487 488 SPDK_DEBUGLOG(nvmf, "cap 0x%" PRIx64 "\n", ctrlr->vcprop.cap.raw); 489 SPDK_DEBUGLOG(nvmf, "vs 0x%x\n", ctrlr->vcprop.vs.raw); 490 SPDK_DEBUGLOG(nvmf, "cc 0x%x\n", ctrlr->vcprop.cc.raw); 491 SPDK_DEBUGLOG(nvmf, "csts 0x%x\n", ctrlr->vcprop.csts.raw); 492 493 ctrlr->dif_insert_or_strip = transport->opts.dif_insert_or_strip; 494 495 if (ctrlr->subsys->subtype == SPDK_NVMF_SUBTYPE_NVME) { 496 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &listen_trid) != 0) { 497 SPDK_ERRLOG("Could not get listener transport ID\n"); 498 goto err_listener; 499 } 500 501 ctrlr->listener = nvmf_subsystem_find_listener(ctrlr->subsys, &listen_trid); 502 if (!ctrlr->listener) { 503 SPDK_ERRLOG("Listener was not found\n"); 504 goto err_listener; 505 } 506 } 507 508 req->qpair->ctrlr = ctrlr; 509 spdk_thread_send_msg(subsystem->thread, _nvmf_subsystem_add_ctrlr, req); 510 511 return ctrlr; 512 err_listener: 513 spdk_bit_array_free(&ctrlr->qpair_mask); 514 err_qpair_mask: 515 free(ctrlr); 516 return NULL; 517 } 518 519 static void 520 _nvmf_ctrlr_destruct(void *ctx) 521 { 522 struct spdk_nvmf_ctrlr *ctrlr = ctx; 523 struct spdk_nvmf_reservation_log *log, *log_tmp; 524 struct spdk_nvmf_async_event_completion *event, *event_tmp; 525 526 SPDK_DTRACE_PROBE3_TICKS(nvmf_ctrlr_destruct, ctrlr, ctrlr->subsys->subnqn, 527 spdk_thread_get_id(ctrlr->thread)); 528 529 assert(spdk_get_thread() == ctrlr->thread); 530 assert(ctrlr->in_destruct); 531 532 SPDK_DEBUGLOG(nvmf, "Destroy ctrlr 0x%hx\n", ctrlr->cntlid); 533 if (ctrlr->disconnect_in_progress) { 534 SPDK_ERRLOG("freeing ctrlr with disconnect in progress\n"); 535 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_destruct, ctrlr); 536 return; 537 } 538 539 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 540 nvmf_ctrlr_stop_association_timer(ctrlr); 541 spdk_bit_array_free(&ctrlr->qpair_mask); 542 543 TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) { 544 TAILQ_REMOVE(&ctrlr->log_head, log, link); 545 free(log); 546 } 547 STAILQ_FOREACH_SAFE(event, &ctrlr->async_events, link, event_tmp) { 548 STAILQ_REMOVE(&ctrlr->async_events, event, spdk_nvmf_async_event_completion, link); 549 free(event); 550 } 551 free(ctrlr); 552 } 553 554 void 555 nvmf_ctrlr_destruct(struct spdk_nvmf_ctrlr *ctrlr) 556 { 557 nvmf_subsystem_remove_ctrlr(ctrlr->subsys, ctrlr); 558 559 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_destruct, ctrlr); 560 } 561 562 static void 563 nvmf_ctrlr_add_io_qpair(void *ctx) 564 { 565 struct spdk_nvmf_request *req = ctx; 566 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 567 struct spdk_nvmf_qpair *qpair = req->qpair; 568 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 569 struct spdk_nvmf_qpair *admin_qpair = ctrlr->admin_qpair; 570 571 SPDK_DTRACE_PROBE4_TICKS(nvmf_ctrlr_add_io_qpair, ctrlr, req->qpair, req->qpair->qid, 572 spdk_thread_get_id(ctrlr->thread)); 573 574 /* Unit test will check qpair->ctrlr after calling spdk_nvmf_ctrlr_connect. 575 * For error case, the value should be NULL. So set it to NULL at first. 576 */ 577 qpair->ctrlr = NULL; 578 579 /* Make sure the controller is not being destroyed. */ 580 if (ctrlr->in_destruct) { 581 SPDK_ERRLOG("Got I/O connect while ctrlr was being destroyed.\n"); 582 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 583 goto end; 584 } 585 586 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 587 SPDK_ERRLOG("I/O connect not allowed on discovery controller\n"); 588 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 589 goto end; 590 } 591 592 if (!ctrlr->vcprop.cc.bits.en) { 593 SPDK_ERRLOG("Got I/O connect before ctrlr was enabled\n"); 594 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 595 goto end; 596 } 597 598 if (1u << ctrlr->vcprop.cc.bits.iosqes != sizeof(struct spdk_nvme_cmd)) { 599 SPDK_ERRLOG("Got I/O connect with invalid IOSQES %u\n", 600 ctrlr->vcprop.cc.bits.iosqes); 601 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 602 goto end; 603 } 604 605 if (1u << ctrlr->vcprop.cc.bits.iocqes != sizeof(struct spdk_nvme_cpl)) { 606 SPDK_ERRLOG("Got I/O connect with invalid IOCQES %u\n", 607 ctrlr->vcprop.cc.bits.iocqes); 608 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 609 goto end; 610 } 611 612 if (admin_qpair->state != SPDK_NVMF_QPAIR_ACTIVE || admin_qpair->group == NULL) { 613 /* There is a chance that admin qpair is being destroyed at this moment due to e.g. 614 * expired keep alive timer. Part of the qpair destruction process is change of qpair's 615 * state to DEACTIVATING and removing it from poll group */ 616 SPDK_ERRLOG("Inactive admin qpair (state %d, group %p)\n", admin_qpair->state, admin_qpair->group); 617 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 618 goto end; 619 } 620 621 /* check if we would exceed ctrlr connection limit */ 622 if (qpair->qid >= spdk_bit_array_capacity(ctrlr->qpair_mask)) { 623 SPDK_ERRLOG("Requested QID %u but Max QID is %u\n", 624 qpair->qid, spdk_bit_array_capacity(ctrlr->qpair_mask) - 1); 625 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 626 rsp->status.sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER; 627 goto end; 628 } 629 630 ctrlr_add_qpair_and_send_rsp(qpair, ctrlr, req); 631 return; 632 end: 633 spdk_nvmf_request_complete(req); 634 } 635 636 static void 637 _nvmf_ctrlr_add_io_qpair(void *ctx) 638 { 639 struct spdk_nvmf_request *req = ctx; 640 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 641 struct spdk_nvmf_fabric_connect_data *data; 642 struct spdk_nvmf_ctrlr *ctrlr; 643 struct spdk_nvmf_qpair *qpair = req->qpair; 644 struct spdk_nvmf_qpair *admin_qpair; 645 struct spdk_nvmf_tgt *tgt = qpair->transport->tgt; 646 struct spdk_nvmf_subsystem *subsystem; 647 struct spdk_nvme_transport_id listen_trid = {}; 648 const struct spdk_nvmf_subsystem_listener *listener; 649 650 assert(req->iovcnt == 1); 651 652 data = req->iov[0].iov_base; 653 654 SPDK_DEBUGLOG(nvmf, "Connect I/O Queue for controller id 0x%x\n", data->cntlid); 655 656 subsystem = spdk_nvmf_tgt_find_subsystem(tgt, data->subnqn); 657 /* We already checked this in spdk_nvmf_ctrlr_connect */ 658 assert(subsystem != NULL); 659 660 ctrlr = nvmf_subsystem_get_ctrlr(subsystem, data->cntlid); 661 if (ctrlr == NULL) { 662 SPDK_ERRLOG("Unknown controller ID 0x%x\n", data->cntlid); 663 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, cntlid); 664 spdk_nvmf_request_complete(req); 665 return; 666 } 667 668 /* fail before passing a message to the controller thread. */ 669 if (ctrlr->in_destruct) { 670 SPDK_ERRLOG("Got I/O connect while ctrlr was being destroyed.\n"); 671 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 672 spdk_nvmf_request_complete(req); 673 return; 674 } 675 676 /* If ANA reporting is enabled, check if I/O connect is on the same listener. */ 677 if (subsystem->flags.ana_reporting) { 678 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &listen_trid) != 0) { 679 SPDK_ERRLOG("Could not get listener transport ID\n"); 680 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 681 spdk_nvmf_request_complete(req); 682 return; 683 } 684 685 listener = nvmf_subsystem_find_listener(subsystem, &listen_trid); 686 if (listener != ctrlr->listener) { 687 SPDK_ERRLOG("I/O connect is on a listener different from admin connect\n"); 688 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 689 spdk_nvmf_request_complete(req); 690 return; 691 } 692 } 693 694 admin_qpair = ctrlr->admin_qpair; 695 if (admin_qpair->state != SPDK_NVMF_QPAIR_ACTIVE || admin_qpair->group == NULL) { 696 /* There is a chance that admin qpair is being destroyed at this moment due to e.g. 697 * expired keep alive timer. Part of the qpair destruction process is change of qpair's 698 * state to DEACTIVATING and removing it from poll group */ 699 SPDK_ERRLOG("Inactive admin qpair (state %d, group %p)\n", admin_qpair->state, admin_qpair->group); 700 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 701 spdk_nvmf_request_complete(req); 702 return; 703 } 704 qpair->ctrlr = ctrlr; 705 spdk_thread_send_msg(admin_qpair->group->thread, nvmf_ctrlr_add_io_qpair, req); 706 } 707 708 static bool 709 nvmf_qpair_access_allowed(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_subsystem *subsystem, 710 const char *hostnqn) 711 { 712 struct spdk_nvme_transport_id listen_trid = {}; 713 714 if (!spdk_nvmf_subsystem_host_allowed(subsystem, hostnqn)) { 715 SPDK_ERRLOG("Subsystem '%s' does not allow host '%s'\n", subsystem->subnqn, hostnqn); 716 return false; 717 } 718 719 if (spdk_nvmf_qpair_get_listen_trid(qpair, &listen_trid)) { 720 SPDK_ERRLOG("Subsystem '%s' is unable to enforce access control due to an internal error.\n", 721 subsystem->subnqn); 722 return false; 723 } 724 725 if (!spdk_nvmf_subsystem_listener_allowed(subsystem, &listen_trid)) { 726 SPDK_ERRLOG("Subsystem '%s' does not allow host '%s' to connect at this address.\n", 727 subsystem->subnqn, hostnqn); 728 return false; 729 } 730 731 return true; 732 } 733 734 static int 735 _nvmf_ctrlr_connect(struct spdk_nvmf_request *req) 736 { 737 struct spdk_nvmf_fabric_connect_data *data = req->iov[0].iov_base; 738 struct spdk_nvmf_fabric_connect_cmd *cmd = &req->cmd->connect_cmd; 739 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 740 struct spdk_nvmf_qpair *qpair = req->qpair; 741 struct spdk_nvmf_transport *transport = qpair->transport; 742 struct spdk_nvmf_ctrlr *ctrlr; 743 struct spdk_nvmf_subsystem *subsystem; 744 745 SPDK_DEBUGLOG(nvmf, "recfmt 0x%x qid %u sqsize %u\n", 746 cmd->recfmt, cmd->qid, cmd->sqsize); 747 748 SPDK_DEBUGLOG(nvmf, "Connect data:\n"); 749 SPDK_DEBUGLOG(nvmf, " cntlid: 0x%04x\n", data->cntlid); 750 SPDK_DEBUGLOG(nvmf, " hostid: %08x-%04x-%04x-%02x%02x-%04x%08x ***\n", 751 ntohl(*(uint32_t *)&data->hostid[0]), 752 ntohs(*(uint16_t *)&data->hostid[4]), 753 ntohs(*(uint16_t *)&data->hostid[6]), 754 data->hostid[8], 755 data->hostid[9], 756 ntohs(*(uint16_t *)&data->hostid[10]), 757 ntohl(*(uint32_t *)&data->hostid[12])); 758 SPDK_DEBUGLOG(nvmf, " subnqn: \"%s\"\n", data->subnqn); 759 SPDK_DEBUGLOG(nvmf, " hostnqn: \"%s\"\n", data->hostnqn); 760 761 subsystem = spdk_nvmf_tgt_find_subsystem(transport->tgt, data->subnqn); 762 if (!subsystem) { 763 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 764 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 765 } 766 767 if (cmd->recfmt != 0) { 768 SPDK_ERRLOG("Connect command unsupported RECFMT %u\n", cmd->recfmt); 769 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 770 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INCOMPATIBLE_FORMAT; 771 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 772 } 773 774 /* 775 * SQSIZE is a 0-based value, so it must be at least 1 (minimum queue depth is 2) and 776 * strictly less than max_aq_depth (admin queues) or max_queue_depth (io queues). 777 */ 778 if (cmd->sqsize == 0) { 779 SPDK_ERRLOG("Invalid SQSIZE = 0\n"); 780 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 781 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 782 } 783 784 if (cmd->qid == 0) { 785 if (cmd->sqsize >= transport->opts.max_aq_depth) { 786 SPDK_ERRLOG("Invalid SQSIZE for admin queue %u (min 1, max %u)\n", 787 cmd->sqsize, transport->opts.max_aq_depth - 1); 788 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 789 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 790 } 791 } else if (cmd->sqsize >= transport->opts.max_queue_depth) { 792 SPDK_ERRLOG("Invalid SQSIZE %u (min 1, max %u)\n", 793 cmd->sqsize, transport->opts.max_queue_depth - 1); 794 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 795 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 796 } 797 798 qpair->sq_head_max = cmd->sqsize; 799 qpair->qid = cmd->qid; 800 qpair->connect_received = true; 801 802 pthread_mutex_lock(&qpair->group->mutex); 803 qpair->group->current_unassociated_qpairs--; 804 pthread_mutex_unlock(&qpair->group->mutex); 805 806 if (0 == qpair->qid) { 807 qpair->group->stat.admin_qpairs++; 808 qpair->group->stat.current_admin_qpairs++; 809 } else { 810 qpair->group->stat.io_qpairs++; 811 qpair->group->stat.current_io_qpairs++; 812 } 813 814 if (cmd->qid == 0) { 815 SPDK_DEBUGLOG(nvmf, "Connect Admin Queue for controller ID 0x%x\n", data->cntlid); 816 817 if (spdk_nvme_trtype_is_fabrics(transport->ops->type) && data->cntlid != 0xFFFF) { 818 /* This NVMf target only supports dynamic mode. */ 819 SPDK_ERRLOG("The NVMf target only supports dynamic mode (CNTLID = 0x%x).\n", data->cntlid); 820 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, cntlid); 821 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 822 } 823 824 /* Establish a new ctrlr */ 825 ctrlr = nvmf_ctrlr_create(subsystem, req, cmd, data); 826 if (!ctrlr) { 827 SPDK_ERRLOG("nvmf_ctrlr_create() failed\n"); 828 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 829 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 830 } else { 831 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 832 } 833 } else { 834 spdk_thread_send_msg(subsystem->thread, _nvmf_ctrlr_add_io_qpair, req); 835 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 836 } 837 } 838 839 static inline bool 840 nvmf_request_is_fabric_connect(struct spdk_nvmf_request *req) 841 { 842 return req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC && 843 req->cmd->nvmf_cmd.fctype == SPDK_NVMF_FABRIC_COMMAND_CONNECT; 844 } 845 846 static struct spdk_nvmf_subsystem_poll_group * 847 nvmf_subsystem_pg_from_connect_cmd(struct spdk_nvmf_request *req) 848 { 849 struct spdk_nvmf_fabric_connect_data *data; 850 struct spdk_nvmf_subsystem *subsystem; 851 struct spdk_nvmf_tgt *tgt; 852 853 assert(nvmf_request_is_fabric_connect(req)); 854 assert(req->qpair->ctrlr == NULL); 855 assert(req->iovcnt == 1); 856 857 data = req->iov[0].iov_base; 858 tgt = req->qpair->transport->tgt; 859 860 subsystem = spdk_nvmf_tgt_find_subsystem(tgt, data->subnqn); 861 if (subsystem == NULL) { 862 return NULL; 863 } 864 865 return &req->qpair->group->sgroups[subsystem->id]; 866 } 867 868 int 869 spdk_nvmf_ctrlr_connect(struct spdk_nvmf_request *req) 870 { 871 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 872 struct spdk_nvmf_subsystem_poll_group *sgroup; 873 struct spdk_nvmf_qpair *qpair = req->qpair; 874 enum spdk_nvmf_request_exec_status status; 875 876 if (req->iovcnt > 1) { 877 SPDK_ERRLOG("Connect command invalid iovcnt: %d\n", req->iovcnt); 878 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 879 status = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 880 goto out; 881 } 882 883 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 884 if (!sgroup) { 885 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 886 status = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 887 goto out; 888 } 889 890 sgroup->mgmt_io_outstanding++; 891 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 892 893 status = _nvmf_ctrlr_connect(req); 894 895 out: 896 if (status == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 897 _nvmf_request_complete(req); 898 } 899 900 return status; 901 } 902 903 static int nvmf_ctrlr_cmd_connect(struct spdk_nvmf_request *req); 904 905 static int 906 retry_connect(void *arg) 907 { 908 struct spdk_nvmf_request *req = arg; 909 struct spdk_nvmf_subsystem_poll_group *sgroup; 910 int rc; 911 912 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 913 /* subsystem may be deleted during the retry interval, so we need to check sgroup */ 914 if (sgroup != NULL) { 915 sgroup->mgmt_io_outstanding++; 916 } 917 spdk_poller_unregister(&req->poller); 918 rc = nvmf_ctrlr_cmd_connect(req); 919 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 920 _nvmf_request_complete(req); 921 } 922 return SPDK_POLLER_BUSY; 923 } 924 925 static int 926 nvmf_ctrlr_cmd_connect(struct spdk_nvmf_request *req) 927 { 928 struct spdk_nvmf_fabric_connect_data *data = req->iov[0].iov_base; 929 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 930 struct spdk_nvmf_transport *transport = req->qpair->transport; 931 struct spdk_nvmf_subsystem *subsystem; 932 933 if (req->length < sizeof(struct spdk_nvmf_fabric_connect_data)) { 934 SPDK_ERRLOG("Connect command data length 0x%x too small\n", req->length); 935 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 936 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 937 } 938 939 if (req->iovcnt > 1) { 940 SPDK_ERRLOG("Connect command invalid iovcnt: %d\n", req->iovcnt); 941 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 942 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 943 } 944 945 subsystem = spdk_nvmf_tgt_find_subsystem(transport->tgt, data->subnqn); 946 if (!subsystem) { 947 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 948 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 949 } 950 951 if ((subsystem->state == SPDK_NVMF_SUBSYSTEM_INACTIVE) || 952 (subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSING) || 953 (subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSED) || 954 (subsystem->state == SPDK_NVMF_SUBSYSTEM_DEACTIVATING)) { 955 struct spdk_nvmf_subsystem_poll_group *sgroup; 956 957 if (req->timeout_tsc == 0) { 958 /* We will only retry the request up to 1 second. */ 959 req->timeout_tsc = spdk_get_ticks() + spdk_get_ticks_hz(); 960 } else if (spdk_get_ticks() > req->timeout_tsc) { 961 SPDK_ERRLOG("Subsystem '%s' was not ready for 1 second\n", subsystem->subnqn); 962 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 963 rsp->status.sc = SPDK_NVMF_FABRIC_SC_CONTROLLER_BUSY; 964 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 965 } 966 967 /* Subsystem is not ready to handle a connect. Use a poller to retry it 968 * again later. Decrement the mgmt_io_outstanding to avoid the 969 * subsystem waiting for this command to complete before unpausing. 970 */ 971 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 972 assert(sgroup != NULL); 973 sgroup->mgmt_io_outstanding--; 974 SPDK_DEBUGLOG(nvmf, "Subsystem '%s' is not ready for connect, retrying...\n", subsystem->subnqn); 975 req->poller = SPDK_POLLER_REGISTER(retry_connect, req, 100); 976 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 977 } 978 979 /* Ensure that hostnqn is null terminated */ 980 if (!memchr(data->hostnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 981 SPDK_ERRLOG("Connect HOSTNQN is not null terminated\n"); 982 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, hostnqn); 983 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 984 } 985 986 if (!nvmf_qpair_access_allowed(req->qpair, subsystem, data->hostnqn)) { 987 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 988 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_HOST; 989 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 990 } 991 992 return _nvmf_ctrlr_connect(req); 993 } 994 995 static int 996 nvmf_ctrlr_association_remove(void *ctx) 997 { 998 struct spdk_nvmf_ctrlr *ctrlr = ctx; 999 int rc; 1000 1001 nvmf_ctrlr_stop_association_timer(ctrlr); 1002 1003 if (ctrlr->in_destruct) { 1004 return SPDK_POLLER_IDLE; 1005 } 1006 SPDK_DEBUGLOG(nvmf, "Disconnecting host from subsystem %s due to association timeout.\n", 1007 ctrlr->subsys->subnqn); 1008 1009 if (ctrlr->admin_qpair) { 1010 rc = spdk_nvmf_qpair_disconnect(ctrlr->admin_qpair, NULL, NULL); 1011 if (rc < 0 && rc != -EINPROGRESS) { 1012 SPDK_ERRLOG("Fail to disconnect admin ctrlr qpair\n"); 1013 assert(false); 1014 } 1015 } 1016 1017 return SPDK_POLLER_BUSY; 1018 } 1019 1020 static int 1021 _nvmf_ctrlr_cc_reset_shn_done(void *ctx) 1022 { 1023 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1024 uint64_t now = spdk_get_ticks(); 1025 uint32_t count; 1026 1027 if (ctrlr->cc_timer) { 1028 spdk_poller_unregister(&ctrlr->cc_timer); 1029 } 1030 1031 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1032 SPDK_DEBUGLOG(nvmf, "ctrlr %p active queue count %u\n", ctrlr, count); 1033 1034 if (count > 1) { 1035 if (now < ctrlr->cc_timeout_tsc) { 1036 /* restart cc timer */ 1037 ctrlr->cc_timer = SPDK_POLLER_REGISTER(_nvmf_ctrlr_cc_reset_shn_done, ctrlr, 100 * 1000); 1038 return SPDK_POLLER_IDLE; 1039 } else { 1040 /* controller fatal status */ 1041 SPDK_WARNLOG("IO timeout, ctrlr %p is in fatal status\n", ctrlr); 1042 nvmf_ctrlr_set_fatal_status(ctrlr); 1043 } 1044 } 1045 1046 spdk_poller_unregister(&ctrlr->cc_timeout_timer); 1047 1048 if (ctrlr->disconnect_is_shn) { 1049 ctrlr->vcprop.csts.bits.shst = SPDK_NVME_SHST_COMPLETE; 1050 ctrlr->disconnect_is_shn = false; 1051 } else { 1052 /* Only a subset of the registers are cleared out on a reset */ 1053 ctrlr->vcprop.cc.raw = 0; 1054 ctrlr->vcprop.csts.raw = 0; 1055 } 1056 1057 /* After CC.EN transitions to 0 (due to shutdown or reset), the association 1058 * between the host and controller shall be preserved for at least 2 minutes */ 1059 if (ctrlr->association_timer) { 1060 SPDK_DEBUGLOG(nvmf, "Association timer already set\n"); 1061 nvmf_ctrlr_stop_association_timer(ctrlr); 1062 } 1063 if (ctrlr->association_timeout) { 1064 ctrlr->association_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_association_remove, ctrlr, 1065 ctrlr->association_timeout * 1000); 1066 } 1067 ctrlr->disconnect_in_progress = false; 1068 return SPDK_POLLER_BUSY; 1069 } 1070 1071 static void 1072 nvmf_ctrlr_cc_reset_shn_done(struct spdk_io_channel_iter *i, int status) 1073 { 1074 struct spdk_nvmf_ctrlr *ctrlr = spdk_io_channel_iter_get_ctx(i); 1075 1076 if (status < 0) { 1077 SPDK_ERRLOG("Fail to disconnect io ctrlr qpairs\n"); 1078 assert(false); 1079 } 1080 1081 _nvmf_ctrlr_cc_reset_shn_done((void *)ctrlr); 1082 } 1083 1084 static void 1085 nvmf_bdev_complete_reset(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1086 { 1087 SPDK_NOTICELOG("Resetting bdev done with %s\n", success ? "success" : "failure"); 1088 1089 spdk_bdev_free_io(bdev_io); 1090 } 1091 1092 1093 static int 1094 nvmf_ctrlr_cc_timeout(void *ctx) 1095 { 1096 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1097 struct spdk_nvmf_poll_group *group = ctrlr->admin_qpair->group; 1098 struct spdk_nvmf_ns *ns; 1099 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1100 1101 assert(group != NULL && group->sgroups != NULL); 1102 spdk_poller_unregister(&ctrlr->cc_timeout_timer); 1103 SPDK_DEBUGLOG(nvmf, "Ctrlr %p reset or shutdown timeout\n", ctrlr); 1104 1105 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 1106 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 1107 if (ns->bdev == NULL) { 1108 continue; 1109 } 1110 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[ns->opts.nsid - 1]; 1111 SPDK_NOTICELOG("Ctrlr %p resetting NSID %u\n", ctrlr, ns->opts.nsid); 1112 spdk_bdev_reset(ns->desc, ns_info->channel, nvmf_bdev_complete_reset, NULL); 1113 } 1114 1115 return SPDK_POLLER_BUSY; 1116 } 1117 1118 const struct spdk_nvmf_registers * 1119 spdk_nvmf_ctrlr_get_regs(struct spdk_nvmf_ctrlr *ctrlr) 1120 { 1121 return &ctrlr->vcprop; 1122 } 1123 1124 void 1125 nvmf_ctrlr_set_fatal_status(struct spdk_nvmf_ctrlr *ctrlr) 1126 { 1127 ctrlr->vcprop.csts.bits.cfs = 1; 1128 } 1129 1130 static uint64_t 1131 nvmf_prop_get_cap(struct spdk_nvmf_ctrlr *ctrlr) 1132 { 1133 return ctrlr->vcprop.cap.raw; 1134 } 1135 1136 static uint64_t 1137 nvmf_prop_get_vs(struct spdk_nvmf_ctrlr *ctrlr) 1138 { 1139 return ctrlr->vcprop.vs.raw; 1140 } 1141 1142 static uint64_t 1143 nvmf_prop_get_cc(struct spdk_nvmf_ctrlr *ctrlr) 1144 { 1145 return ctrlr->vcprop.cc.raw; 1146 } 1147 1148 static bool 1149 nvmf_prop_set_cc(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1150 { 1151 union spdk_nvme_cc_register cc, diff; 1152 uint32_t cc_timeout_ms; 1153 1154 cc.raw = value; 1155 1156 SPDK_DEBUGLOG(nvmf, "cur CC: 0x%08x\n", ctrlr->vcprop.cc.raw); 1157 SPDK_DEBUGLOG(nvmf, "new CC: 0x%08x\n", cc.raw); 1158 1159 /* 1160 * Calculate which bits changed between the current and new CC. 1161 * Mark each bit as 0 once it is handled to determine if any unhandled bits were changed. 1162 */ 1163 diff.raw = cc.raw ^ ctrlr->vcprop.cc.raw; 1164 1165 if (diff.bits.en) { 1166 if (cc.bits.en) { 1167 SPDK_DEBUGLOG(nvmf, "Property Set CC Enable!\n"); 1168 nvmf_ctrlr_stop_association_timer(ctrlr); 1169 1170 ctrlr->vcprop.cc.bits.en = 1; 1171 ctrlr->vcprop.csts.bits.rdy = 1; 1172 } else { 1173 SPDK_DEBUGLOG(nvmf, "Property Set CC Disable!\n"); 1174 if (ctrlr->disconnect_in_progress) { 1175 SPDK_DEBUGLOG(nvmf, "Disconnect in progress\n"); 1176 return true; 1177 } 1178 1179 ctrlr->cc_timeout_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_cc_timeout, ctrlr, 1180 NVMF_CC_RESET_SHN_TIMEOUT_IN_MS * 1000); 1181 /* Make sure cc_timeout_ms is between cc_timeout_timer and Host reset/shutdown timeout */ 1182 cc_timeout_ms = (NVMF_CC_RESET_SHN_TIMEOUT_IN_MS + NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS) / 2; 1183 ctrlr->cc_timeout_tsc = spdk_get_ticks() + cc_timeout_ms * spdk_get_ticks_hz() / (uint64_t)1000; 1184 1185 ctrlr->vcprop.cc.bits.en = 0; 1186 ctrlr->disconnect_in_progress = true; 1187 ctrlr->disconnect_is_shn = false; 1188 spdk_for_each_channel(ctrlr->subsys->tgt, 1189 nvmf_ctrlr_disconnect_io_qpairs_on_pg, 1190 ctrlr, 1191 nvmf_ctrlr_cc_reset_shn_done); 1192 } 1193 diff.bits.en = 0; 1194 } 1195 1196 if (diff.bits.shn) { 1197 if (cc.bits.shn == SPDK_NVME_SHN_NORMAL || 1198 cc.bits.shn == SPDK_NVME_SHN_ABRUPT) { 1199 SPDK_DEBUGLOG(nvmf, "Property Set CC Shutdown %u%ub!\n", 1200 cc.bits.shn >> 1, cc.bits.shn & 1); 1201 if (ctrlr->disconnect_in_progress) { 1202 SPDK_DEBUGLOG(nvmf, "Disconnect in progress\n"); 1203 return true; 1204 } 1205 1206 ctrlr->cc_timeout_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_cc_timeout, ctrlr, 1207 NVMF_CC_RESET_SHN_TIMEOUT_IN_MS * 1000); 1208 /* Make sure cc_timeout_ms is between cc_timeout_timer and Host reset/shutdown timeout */ 1209 cc_timeout_ms = (NVMF_CC_RESET_SHN_TIMEOUT_IN_MS + NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS) / 2; 1210 ctrlr->cc_timeout_tsc = spdk_get_ticks() + cc_timeout_ms * spdk_get_ticks_hz() / (uint64_t)1000; 1211 1212 ctrlr->vcprop.cc.bits.shn = cc.bits.shn; 1213 ctrlr->disconnect_in_progress = true; 1214 ctrlr->disconnect_is_shn = true; 1215 spdk_for_each_channel(ctrlr->subsys->tgt, 1216 nvmf_ctrlr_disconnect_io_qpairs_on_pg, 1217 ctrlr, 1218 nvmf_ctrlr_cc_reset_shn_done); 1219 1220 /* From the time a shutdown is initiated the controller shall disable 1221 * Keep Alive timer */ 1222 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 1223 } else if (cc.bits.shn == 0) { 1224 ctrlr->vcprop.cc.bits.shn = 0; 1225 } else { 1226 SPDK_ERRLOG("Prop Set CC: Invalid SHN value %u%ub\n", 1227 cc.bits.shn >> 1, cc.bits.shn & 1); 1228 return false; 1229 } 1230 diff.bits.shn = 0; 1231 } 1232 1233 if (diff.bits.iosqes) { 1234 SPDK_DEBUGLOG(nvmf, "Prop Set IOSQES = %u (%u bytes)\n", 1235 cc.bits.iosqes, 1u << cc.bits.iosqes); 1236 ctrlr->vcprop.cc.bits.iosqes = cc.bits.iosqes; 1237 diff.bits.iosqes = 0; 1238 } 1239 1240 if (diff.bits.iocqes) { 1241 SPDK_DEBUGLOG(nvmf, "Prop Set IOCQES = %u (%u bytes)\n", 1242 cc.bits.iocqes, 1u << cc.bits.iocqes); 1243 ctrlr->vcprop.cc.bits.iocqes = cc.bits.iocqes; 1244 diff.bits.iocqes = 0; 1245 } 1246 1247 if (diff.bits.ams) { 1248 SPDK_ERRLOG("Arbitration Mechanism Selected (AMS) 0x%x not supported!\n", cc.bits.ams); 1249 return false; 1250 } 1251 1252 if (diff.bits.mps) { 1253 SPDK_ERRLOG("Memory Page Size (MPS) %u KiB not supported!\n", (1 << (2 + cc.bits.mps))); 1254 return false; 1255 } 1256 1257 if (diff.bits.css) { 1258 if (cc.bits.css > SPDK_NVME_CC_CSS_IOCS) { 1259 SPDK_ERRLOG("I/O Command Set Selected (CSS) 0x%x not supported!\n", cc.bits.css); 1260 return false; 1261 } 1262 diff.bits.css = 0; 1263 } 1264 1265 if (diff.raw != 0) { 1266 /* Print an error message, but don't fail the command in this case. 1267 * If we did want to fail in this case, we'd need to ensure we acted 1268 * on no other bits or the initiator gets confused. */ 1269 SPDK_ERRLOG("Prop Set CC toggled reserved bits 0x%x!\n", diff.raw); 1270 } 1271 1272 return true; 1273 } 1274 1275 static uint64_t 1276 nvmf_prop_get_csts(struct spdk_nvmf_ctrlr *ctrlr) 1277 { 1278 return ctrlr->vcprop.csts.raw; 1279 } 1280 1281 static uint64_t 1282 nvmf_prop_get_aqa(struct spdk_nvmf_ctrlr *ctrlr) 1283 { 1284 return ctrlr->vcprop.aqa.raw; 1285 } 1286 1287 static bool 1288 nvmf_prop_set_aqa(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1289 { 1290 union spdk_nvme_aqa_register aqa; 1291 1292 aqa.raw = value; 1293 1294 /* 1295 * We don't need to explicitly check for maximum size, as the fields are 1296 * limited to 12 bits (4096). 1297 */ 1298 if (aqa.bits.asqs < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES - 1 || 1299 aqa.bits.acqs < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES - 1 || 1300 aqa.bits.reserved1 != 0 || aqa.bits.reserved2 != 0) { 1301 return false; 1302 } 1303 1304 ctrlr->vcprop.aqa.raw = value; 1305 1306 return true; 1307 } 1308 1309 static uint64_t 1310 nvmf_prop_get_asq(struct spdk_nvmf_ctrlr *ctrlr) 1311 { 1312 return ctrlr->vcprop.asq; 1313 } 1314 1315 static bool 1316 nvmf_prop_set_asq_lower(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1317 { 1318 ctrlr->vcprop.asq = (ctrlr->vcprop.asq & (0xFFFFFFFFULL << 32ULL)) | value; 1319 1320 return true; 1321 } 1322 1323 static bool 1324 nvmf_prop_set_asq_upper(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1325 { 1326 ctrlr->vcprop.asq = (ctrlr->vcprop.asq & 0xFFFFFFFFULL) | ((uint64_t)value << 32ULL); 1327 1328 return true; 1329 } 1330 1331 static uint64_t 1332 nvmf_prop_get_acq(struct spdk_nvmf_ctrlr *ctrlr) 1333 { 1334 return ctrlr->vcprop.acq; 1335 } 1336 1337 static bool 1338 nvmf_prop_set_acq_lower(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1339 { 1340 ctrlr->vcprop.acq = (ctrlr->vcprop.acq & (0xFFFFFFFFULL << 32ULL)) | value; 1341 1342 return true; 1343 } 1344 1345 static bool 1346 nvmf_prop_set_acq_upper(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1347 { 1348 ctrlr->vcprop.acq = (ctrlr->vcprop.acq & 0xFFFFFFFFULL) | ((uint64_t)value << 32ULL); 1349 1350 return true; 1351 } 1352 1353 struct nvmf_prop { 1354 uint32_t ofst; 1355 uint8_t size; 1356 char name[11]; 1357 uint64_t (*get_cb)(struct spdk_nvmf_ctrlr *ctrlr); 1358 bool (*set_cb)(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value); 1359 bool (*set_upper_cb)(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value); 1360 }; 1361 1362 #define PROP(field, size, get_cb, set_cb, set_upper_cb) \ 1363 { \ 1364 offsetof(struct spdk_nvme_registers, field), \ 1365 size, \ 1366 #field, \ 1367 get_cb, set_cb, set_upper_cb \ 1368 } 1369 1370 static const struct nvmf_prop nvmf_props[] = { 1371 PROP(cap, 8, nvmf_prop_get_cap, NULL, NULL), 1372 PROP(vs, 4, nvmf_prop_get_vs, NULL, NULL), 1373 PROP(cc, 4, nvmf_prop_get_cc, nvmf_prop_set_cc, NULL), 1374 PROP(csts, 4, nvmf_prop_get_csts, NULL, NULL), 1375 PROP(aqa, 4, nvmf_prop_get_aqa, nvmf_prop_set_aqa, NULL), 1376 PROP(asq, 8, nvmf_prop_get_asq, nvmf_prop_set_asq_lower, nvmf_prop_set_asq_upper), 1377 PROP(acq, 8, nvmf_prop_get_acq, nvmf_prop_set_acq_lower, nvmf_prop_set_acq_upper), 1378 }; 1379 1380 static const struct nvmf_prop * 1381 find_prop(uint32_t ofst, uint8_t size) 1382 { 1383 size_t i; 1384 1385 for (i = 0; i < SPDK_COUNTOF(nvmf_props); i++) { 1386 const struct nvmf_prop *prop = &nvmf_props[i]; 1387 1388 if ((ofst >= prop->ofst) && (ofst + size <= prop->ofst + prop->size)) { 1389 return prop; 1390 } 1391 } 1392 1393 return NULL; 1394 } 1395 1396 static int 1397 nvmf_property_get(struct spdk_nvmf_request *req) 1398 { 1399 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1400 struct spdk_nvmf_fabric_prop_get_cmd *cmd = &req->cmd->prop_get_cmd; 1401 struct spdk_nvmf_fabric_prop_get_rsp *response = &req->rsp->prop_get_rsp; 1402 const struct nvmf_prop *prop; 1403 uint8_t size; 1404 1405 response->status.sc = 0; 1406 response->value.u64 = 0; 1407 1408 SPDK_DEBUGLOG(nvmf, "size %d, offset 0x%x\n", 1409 cmd->attrib.size, cmd->ofst); 1410 1411 switch (cmd->attrib.size) { 1412 case SPDK_NVMF_PROP_SIZE_4: 1413 size = 4; 1414 break; 1415 case SPDK_NVMF_PROP_SIZE_8: 1416 size = 8; 1417 break; 1418 default: 1419 SPDK_DEBUGLOG(nvmf, "Invalid size value %d\n", cmd->attrib.size); 1420 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1421 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1422 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1423 } 1424 1425 prop = find_prop(cmd->ofst, size); 1426 if (prop == NULL || prop->get_cb == NULL) { 1427 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1428 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1429 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1430 } 1431 1432 SPDK_DEBUGLOG(nvmf, "name: %s\n", prop->name); 1433 1434 response->value.u64 = prop->get_cb(ctrlr); 1435 1436 if (size != prop->size) { 1437 /* The size must be 4 and the prop->size is 8. Figure out which part of the property to read. */ 1438 assert(size == 4); 1439 assert(prop->size == 8); 1440 1441 if (cmd->ofst == prop->ofst) { 1442 /* Keep bottom 4 bytes only */ 1443 response->value.u64 &= 0xFFFFFFFF; 1444 } else { 1445 /* Keep top 4 bytes only */ 1446 response->value.u64 >>= 32; 1447 } 1448 } 1449 1450 SPDK_DEBUGLOG(nvmf, "response value: 0x%" PRIx64 "\n", response->value.u64); 1451 1452 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1453 } 1454 1455 static int 1456 nvmf_property_set(struct spdk_nvmf_request *req) 1457 { 1458 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1459 struct spdk_nvmf_fabric_prop_set_cmd *cmd = &req->cmd->prop_set_cmd; 1460 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1461 const struct nvmf_prop *prop; 1462 uint64_t value; 1463 uint8_t size; 1464 bool ret; 1465 1466 SPDK_DEBUGLOG(nvmf, "size %d, offset 0x%x, value 0x%" PRIx64 "\n", 1467 cmd->attrib.size, cmd->ofst, cmd->value.u64); 1468 1469 switch (cmd->attrib.size) { 1470 case SPDK_NVMF_PROP_SIZE_4: 1471 size = 4; 1472 break; 1473 case SPDK_NVMF_PROP_SIZE_8: 1474 size = 8; 1475 break; 1476 default: 1477 SPDK_DEBUGLOG(nvmf, "Invalid size value %d\n", cmd->attrib.size); 1478 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1479 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1480 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1481 } 1482 1483 prop = find_prop(cmd->ofst, size); 1484 if (prop == NULL || prop->set_cb == NULL) { 1485 SPDK_INFOLOG(nvmf, "Invalid offset 0x%x\n", cmd->ofst); 1486 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1487 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1488 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1489 } 1490 1491 SPDK_DEBUGLOG(nvmf, "name: %s\n", prop->name); 1492 1493 value = cmd->value.u64; 1494 1495 if (prop->size == 4) { 1496 ret = prop->set_cb(ctrlr, (uint32_t)value); 1497 } else if (size != prop->size) { 1498 /* The size must be 4 and the prop->size is 8. Figure out which part of the property to write. */ 1499 assert(size == 4); 1500 assert(prop->size == 8); 1501 1502 if (cmd->ofst == prop->ofst) { 1503 ret = prop->set_cb(ctrlr, (uint32_t)value); 1504 } else { 1505 ret = prop->set_upper_cb(ctrlr, (uint32_t)value); 1506 } 1507 } else { 1508 ret = prop->set_cb(ctrlr, (uint32_t)value); 1509 if (ret) { 1510 ret = prop->set_upper_cb(ctrlr, (uint32_t)(value >> 32)); 1511 } 1512 } 1513 1514 if (!ret) { 1515 SPDK_ERRLOG("prop set_cb failed\n"); 1516 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1517 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1518 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1519 } 1520 1521 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1522 } 1523 1524 static int 1525 nvmf_ctrlr_set_features_arbitration(struct spdk_nvmf_request *req) 1526 { 1527 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1528 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1529 1530 SPDK_DEBUGLOG(nvmf, "Set Features - Arbitration (cdw11 = 0x%0x)\n", cmd->cdw11); 1531 1532 ctrlr->feat.arbitration.raw = cmd->cdw11; 1533 ctrlr->feat.arbitration.bits.reserved = 0; 1534 1535 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1536 } 1537 1538 static int 1539 nvmf_ctrlr_set_features_power_management(struct spdk_nvmf_request *req) 1540 { 1541 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1542 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1543 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1544 1545 SPDK_DEBUGLOG(nvmf, "Set Features - Power Management (cdw11 = 0x%0x)\n", cmd->cdw11); 1546 1547 /* Only PS = 0 is allowed, since we report NPSS = 0 */ 1548 if (cmd->cdw11_bits.feat_power_management.bits.ps != 0) { 1549 SPDK_ERRLOG("Invalid power state %u\n", cmd->cdw11_bits.feat_power_management.bits.ps); 1550 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1551 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1552 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1553 } 1554 1555 ctrlr->feat.power_management.raw = cmd->cdw11; 1556 ctrlr->feat.power_management.bits.reserved = 0; 1557 1558 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1559 } 1560 1561 static bool 1562 temp_threshold_opts_valid(const union spdk_nvme_feat_temperature_threshold *opts) 1563 { 1564 /* 1565 * Valid TMPSEL values: 1566 * 0000b - 1000b: temperature sensors 1567 * 1111b: set all implemented temperature sensors 1568 */ 1569 if (opts->bits.tmpsel >= 9 && opts->bits.tmpsel != 15) { 1570 /* 1001b - 1110b: reserved */ 1571 SPDK_ERRLOG("Invalid TMPSEL %u\n", opts->bits.tmpsel); 1572 return false; 1573 } 1574 1575 /* 1576 * Valid THSEL values: 1577 * 00b: over temperature threshold 1578 * 01b: under temperature threshold 1579 */ 1580 if (opts->bits.thsel > 1) { 1581 /* 10b - 11b: reserved */ 1582 SPDK_ERRLOG("Invalid THSEL %u\n", opts->bits.thsel); 1583 return false; 1584 } 1585 1586 return true; 1587 } 1588 1589 static int 1590 nvmf_ctrlr_set_features_temperature_threshold(struct spdk_nvmf_request *req) 1591 { 1592 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1593 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1594 1595 SPDK_DEBUGLOG(nvmf, "Set Features - Temperature Threshold (cdw11 = 0x%0x)\n", cmd->cdw11); 1596 1597 if (!temp_threshold_opts_valid(&cmd->cdw11_bits.feat_temp_threshold)) { 1598 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1599 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1600 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1601 } 1602 1603 /* TODO: no sensors implemented - ignore new values */ 1604 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1605 } 1606 1607 static int 1608 nvmf_ctrlr_get_features_temperature_threshold(struct spdk_nvmf_request *req) 1609 { 1610 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1611 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1612 1613 SPDK_DEBUGLOG(nvmf, "Get Features - Temperature Threshold (cdw11 = 0x%0x)\n", cmd->cdw11); 1614 1615 if (!temp_threshold_opts_valid(&cmd->cdw11_bits.feat_temp_threshold)) { 1616 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1617 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1618 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1619 } 1620 1621 /* TODO: no sensors implemented - return 0 for all thresholds */ 1622 rsp->cdw0 = 0; 1623 1624 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1625 } 1626 1627 static int 1628 nvmf_ctrlr_get_features_interrupt_vector_configuration(struct spdk_nvmf_request *req) 1629 { 1630 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1631 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1632 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1633 union spdk_nvme_feat_interrupt_vector_configuration iv_conf = {}; 1634 1635 SPDK_DEBUGLOG(nvmf, "Get Features - Interrupt Vector Configuration (cdw11 = 0x%0x)\n", cmd->cdw11); 1636 1637 iv_conf.bits.iv = cmd->cdw11_bits.feat_interrupt_vector_configuration.bits.iv; 1638 iv_conf.bits.cd = ctrlr->feat.interrupt_vector_configuration.bits.cd; 1639 rsp->cdw0 = iv_conf.raw; 1640 1641 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1642 } 1643 1644 static int 1645 nvmf_ctrlr_set_features_error_recovery(struct spdk_nvmf_request *req) 1646 { 1647 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1648 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1649 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1650 1651 SPDK_DEBUGLOG(nvmf, "Set Features - Error Recovery (cdw11 = 0x%0x)\n", cmd->cdw11); 1652 1653 if (cmd->cdw11_bits.feat_error_recovery.bits.dulbe) { 1654 /* 1655 * Host is not allowed to set this bit, since we don't advertise it in 1656 * Identify Namespace. 1657 */ 1658 SPDK_ERRLOG("Host set unsupported DULBE bit\n"); 1659 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1660 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1661 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1662 } 1663 1664 ctrlr->feat.error_recovery.raw = cmd->cdw11; 1665 ctrlr->feat.error_recovery.bits.reserved = 0; 1666 1667 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1668 } 1669 1670 static int 1671 nvmf_ctrlr_set_features_volatile_write_cache(struct spdk_nvmf_request *req) 1672 { 1673 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1674 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1675 1676 SPDK_DEBUGLOG(nvmf, "Set Features - Volatile Write Cache (cdw11 = 0x%0x)\n", cmd->cdw11); 1677 1678 ctrlr->feat.volatile_write_cache.raw = cmd->cdw11; 1679 ctrlr->feat.volatile_write_cache.bits.reserved = 0; 1680 1681 SPDK_DEBUGLOG(nvmf, "Set Features - Volatile Write Cache %s\n", 1682 ctrlr->feat.volatile_write_cache.bits.wce ? "Enabled" : "Disabled"); 1683 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1684 } 1685 1686 static int 1687 nvmf_ctrlr_set_features_write_atomicity(struct spdk_nvmf_request *req) 1688 { 1689 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1690 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1691 1692 SPDK_DEBUGLOG(nvmf, "Set Features - Write Atomicity (cdw11 = 0x%0x)\n", cmd->cdw11); 1693 1694 ctrlr->feat.write_atomicity.raw = cmd->cdw11; 1695 ctrlr->feat.write_atomicity.bits.reserved = 0; 1696 1697 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1698 } 1699 1700 static int 1701 nvmf_ctrlr_set_features_host_identifier(struct spdk_nvmf_request *req) 1702 { 1703 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1704 1705 SPDK_ERRLOG("Set Features - Host Identifier not allowed\n"); 1706 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 1707 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1708 } 1709 1710 static int 1711 nvmf_ctrlr_get_features_host_identifier(struct spdk_nvmf_request *req) 1712 { 1713 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1714 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1715 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1716 struct spdk_iov_xfer ix; 1717 1718 SPDK_DEBUGLOG(nvmf, "Get Features - Host Identifier\n"); 1719 1720 if (!cmd->cdw11_bits.feat_host_identifier.bits.exhid) { 1721 /* NVMe over Fabrics requires EXHID=1 (128-bit/16-byte host ID) */ 1722 SPDK_ERRLOG("Get Features - Host Identifier with EXHID=0 not allowed\n"); 1723 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1724 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1725 } 1726 1727 if (req->iovcnt < 1 || req->length < sizeof(ctrlr->hostid)) { 1728 SPDK_ERRLOG("Invalid data buffer for Get Features - Host Identifier\n"); 1729 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1730 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1731 } 1732 1733 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 1734 spdk_iov_xfer_from_buf(&ix, &ctrlr->hostid, sizeof(ctrlr->hostid)); 1735 1736 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1737 } 1738 1739 static int 1740 nvmf_ctrlr_get_features_reservation_notification_mask(struct spdk_nvmf_request *req) 1741 { 1742 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1743 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1744 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1745 struct spdk_nvmf_ns *ns; 1746 1747 SPDK_DEBUGLOG(nvmf, "get Features - Reservation Notification Mask\n"); 1748 1749 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1750 SPDK_ERRLOG("get Features - Invalid Namespace ID\n"); 1751 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1752 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1753 } 1754 1755 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, cmd->nsid); 1756 if (ns == NULL) { 1757 SPDK_ERRLOG("Set Features - Invalid Namespace ID\n"); 1758 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1759 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1760 } 1761 rsp->cdw0 = ns->mask; 1762 1763 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1764 } 1765 1766 static int 1767 nvmf_ctrlr_set_features_reservation_notification_mask(struct spdk_nvmf_request *req) 1768 { 1769 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1770 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 1771 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1772 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1773 struct spdk_nvmf_ns *ns; 1774 1775 SPDK_DEBUGLOG(nvmf, "Set Features - Reservation Notification Mask\n"); 1776 1777 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1778 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 1779 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 1780 ns->mask = cmd->cdw11; 1781 } 1782 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1783 } 1784 1785 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, cmd->nsid); 1786 if (ns == NULL) { 1787 SPDK_ERRLOG("Set Features - Invalid Namespace ID\n"); 1788 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1789 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1790 } 1791 ns->mask = cmd->cdw11; 1792 1793 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1794 } 1795 1796 static int 1797 nvmf_ctrlr_get_features_reservation_persistence(struct spdk_nvmf_request *req) 1798 { 1799 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1800 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1801 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1802 struct spdk_nvmf_ns *ns; 1803 1804 SPDK_DEBUGLOG(nvmf, "Get Features - Reservation Persistence\n"); 1805 1806 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, cmd->nsid); 1807 /* NSID with SPDK_NVME_GLOBAL_NS_TAG (=0xffffffff) also included */ 1808 if (ns == NULL) { 1809 SPDK_ERRLOG("Get Features - Invalid Namespace ID\n"); 1810 response->status.sct = SPDK_NVME_SCT_GENERIC; 1811 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1812 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1813 } 1814 1815 response->cdw0 = ns->ptpl_activated; 1816 1817 response->status.sct = SPDK_NVME_SCT_GENERIC; 1818 response->status.sc = SPDK_NVME_SC_SUCCESS; 1819 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1820 } 1821 1822 static int 1823 nvmf_ctrlr_set_features_reservation_persistence(struct spdk_nvmf_request *req) 1824 { 1825 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1826 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1827 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1828 struct spdk_nvmf_ns *ns; 1829 bool ptpl; 1830 1831 SPDK_DEBUGLOG(nvmf, "Set Features - Reservation Persistence\n"); 1832 1833 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, cmd->nsid); 1834 ptpl = cmd->cdw11_bits.feat_rsv_persistence.bits.ptpl; 1835 1836 if (cmd->nsid != SPDK_NVME_GLOBAL_NS_TAG && ns && ns->ptpl_file) { 1837 ns->ptpl_activated = ptpl; 1838 } else if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1839 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns && ns->ptpl_file; 1840 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 1841 ns->ptpl_activated = ptpl; 1842 } 1843 } else { 1844 SPDK_ERRLOG("Set Features - Invalid Namespace ID or Reservation Configuration\n"); 1845 response->status.sct = SPDK_NVME_SCT_GENERIC; 1846 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1847 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1848 } 1849 1850 /* TODO: Feature not changeable for now */ 1851 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1852 response->status.sc = SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE; 1853 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1854 } 1855 1856 static int 1857 nvmf_ctrlr_get_features_host_behavior_support(struct spdk_nvmf_request *req) 1858 { 1859 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1860 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1861 struct spdk_nvme_host_behavior host_behavior = {}; 1862 struct spdk_iov_xfer ix; 1863 1864 SPDK_DEBUGLOG(nvmf, "Get Features - Host Behavior Support\n"); 1865 1866 if (req->iovcnt < 1 || req->length < sizeof(struct spdk_nvme_host_behavior)) { 1867 SPDK_ERRLOG("invalid data buffer for Host Behavior Support\n"); 1868 response->status.sct = SPDK_NVME_SCT_GENERIC; 1869 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1870 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1871 } 1872 1873 host_behavior.acre = ctrlr->acre_enabled; 1874 1875 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 1876 spdk_iov_xfer_from_buf(&ix, &host_behavior, sizeof(host_behavior)); 1877 1878 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1879 } 1880 1881 static int 1882 nvmf_ctrlr_set_features_host_behavior_support(struct spdk_nvmf_request *req) 1883 { 1884 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1885 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1886 struct spdk_nvme_host_behavior *host_behavior; 1887 1888 SPDK_DEBUGLOG(nvmf, "Set Features - Host Behavior Support\n"); 1889 if (req->iovcnt != 1) { 1890 SPDK_ERRLOG("Host Behavior Support invalid iovcnt: %d\n", req->iovcnt); 1891 response->status.sct = SPDK_NVME_SCT_GENERIC; 1892 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1893 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1894 } 1895 if (req->iov[0].iov_len != sizeof(struct spdk_nvme_host_behavior)) { 1896 SPDK_ERRLOG("Host Behavior Support invalid iov_len: %zd\n", req->iov[0].iov_len); 1897 response->status.sct = SPDK_NVME_SCT_GENERIC; 1898 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1899 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1900 } 1901 1902 host_behavior = (struct spdk_nvme_host_behavior *)req->iov[0].iov_base; 1903 if (host_behavior->acre == 0) { 1904 ctrlr->acre_enabled = false; 1905 } else if (host_behavior->acre == 1) { 1906 ctrlr->acre_enabled = true; 1907 } else { 1908 SPDK_ERRLOG("Host Behavior Support invalid acre: 0x%02x\n", host_behavior->acre); 1909 response->status.sct = SPDK_NVME_SCT_GENERIC; 1910 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1911 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1912 } 1913 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1914 } 1915 1916 static int 1917 nvmf_ctrlr_set_features_keep_alive_timer(struct spdk_nvmf_request *req) 1918 { 1919 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1920 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1921 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1922 1923 SPDK_DEBUGLOG(nvmf, "Set Features - Keep Alive Timer (%u ms)\n", cmd->cdw11); 1924 1925 /* 1926 * if attempts to disable keep alive by setting kato to 0h 1927 * a status value of keep alive invalid shall be returned 1928 */ 1929 if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato == 0) { 1930 rsp->status.sc = SPDK_NVME_SC_KEEP_ALIVE_INVALID; 1931 } else if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato < MIN_KEEP_ALIVE_TIMEOUT_IN_MS) { 1932 ctrlr->feat.keep_alive_timer.bits.kato = MIN_KEEP_ALIVE_TIMEOUT_IN_MS; 1933 } else { 1934 /* round up to milliseconds */ 1935 ctrlr->feat.keep_alive_timer.bits.kato = spdk_divide_round_up( 1936 cmd->cdw11_bits.feat_keep_alive_timer.bits.kato, 1937 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS) * 1938 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS; 1939 } 1940 1941 /* 1942 * if change the keep alive timeout value successfully 1943 * update the keep alive poller. 1944 */ 1945 if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato != 0) { 1946 if (ctrlr->keep_alive_poller != NULL) { 1947 spdk_poller_unregister(&ctrlr->keep_alive_poller); 1948 } 1949 ctrlr->keep_alive_poller = SPDK_POLLER_REGISTER(nvmf_ctrlr_keep_alive_poll, ctrlr, 1950 ctrlr->feat.keep_alive_timer.bits.kato * 1000); 1951 } 1952 1953 SPDK_DEBUGLOG(nvmf, "Set Features - Keep Alive Timer set to %u ms\n", 1954 ctrlr->feat.keep_alive_timer.bits.kato); 1955 1956 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1957 } 1958 1959 static int 1960 nvmf_ctrlr_set_features_number_of_queues(struct spdk_nvmf_request *req) 1961 { 1962 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1963 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1964 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1965 uint32_t count; 1966 1967 SPDK_DEBUGLOG(nvmf, "Set Features - Number of Queues, cdw11 0x%x\n", 1968 req->cmd->nvme_cmd.cdw11); 1969 1970 if (cmd->cdw11_bits.feat_num_of_queues.bits.ncqr == UINT16_MAX || 1971 cmd->cdw11_bits.feat_num_of_queues.bits.nsqr == UINT16_MAX) { 1972 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1973 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1974 } 1975 1976 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1977 /* verify that the controller is ready to process commands */ 1978 if (count > 1) { 1979 SPDK_DEBUGLOG(nvmf, "Queue pairs already active!\n"); 1980 rsp->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 1981 } else { 1982 /* 1983 * Ignore the value requested by the host - 1984 * always return the pre-configured value based on max_qpairs_allowed. 1985 */ 1986 rsp->cdw0 = ctrlr->feat.number_of_queues.raw; 1987 } 1988 1989 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1990 } 1991 1992 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_ctrlr) == 4920, 1993 "Please check migration fields that need to be added or not"); 1994 1995 static void 1996 nvmf_ctrlr_migr_data_copy(struct spdk_nvmf_ctrlr_migr_data *data, 1997 const struct spdk_nvmf_ctrlr_migr_data *data_src, size_t data_size) 1998 { 1999 assert(data); 2000 assert(data_src); 2001 assert(data_size); 2002 2003 memcpy(&data->regs, &data_src->regs, spdk_min(data->regs_size, data_src->regs_size)); 2004 memcpy(&data->feat, &data_src->feat, spdk_min(data->feat_size, data_src->feat_size)); 2005 2006 #define SET_FIELD(field) \ 2007 if (offsetof(struct spdk_nvmf_ctrlr_migr_data, field) + sizeof(data->field) <= data_size) { \ 2008 data->field = data_src->field; \ 2009 } \ 2010 2011 SET_FIELD(cntlid); 2012 SET_FIELD(acre); 2013 SET_FIELD(num_aer_cids); 2014 SET_FIELD(num_async_events); 2015 SET_FIELD(notice_aen_mask); 2016 #undef SET_FIELD 2017 2018 #define SET_ARRAY(arr) \ 2019 if (offsetof(struct spdk_nvmf_ctrlr_migr_data, arr) + sizeof(data->arr) <= data_size) { \ 2020 memcpy(&data->arr, &data_src->arr, sizeof(data->arr)); \ 2021 } \ 2022 2023 SET_ARRAY(async_events); 2024 SET_ARRAY(aer_cids); 2025 #undef SET_ARRAY 2026 } 2027 2028 int 2029 spdk_nvmf_ctrlr_save_migr_data(struct spdk_nvmf_ctrlr *ctrlr, 2030 struct spdk_nvmf_ctrlr_migr_data *data) 2031 { 2032 struct spdk_nvmf_async_event_completion *event, *event_tmp; 2033 uint32_t i; 2034 struct spdk_nvmf_ctrlr_migr_data data_local = { 2035 .data_size = offsetof(struct spdk_nvmf_ctrlr_migr_data, unused), 2036 .regs_size = sizeof(struct spdk_nvmf_registers), 2037 .feat_size = sizeof(struct spdk_nvmf_ctrlr_feat) 2038 }; 2039 2040 assert(data->data_size <= sizeof(data_local)); 2041 assert(spdk_get_thread() == ctrlr->thread); 2042 2043 memcpy(&data_local.regs, &ctrlr->vcprop, sizeof(struct spdk_nvmf_registers)); 2044 memcpy(&data_local.feat, &ctrlr->feat, sizeof(struct spdk_nvmf_ctrlr_feat)); 2045 2046 data_local.cntlid = ctrlr->cntlid; 2047 data_local.acre = ctrlr->acre_enabled; 2048 data_local.num_aer_cids = ctrlr->nr_aer_reqs; 2049 2050 STAILQ_FOREACH_SAFE(event, &ctrlr->async_events, link, event_tmp) { 2051 if (data_local.num_async_events + 1 > SPDK_NVMF_MIGR_MAX_PENDING_AERS) { 2052 SPDK_ERRLOG("ctrlr %p has too many pending AERs\n", ctrlr); 2053 break; 2054 } 2055 2056 data_local.async_events[data_local.num_async_events++].raw = event->event.raw; 2057 } 2058 2059 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 2060 struct spdk_nvmf_request *req = ctrlr->aer_req[i]; 2061 data_local.aer_cids[i] = req->cmd->nvme_cmd.cid; 2062 } 2063 data_local.notice_aen_mask = ctrlr->notice_aen_mask; 2064 2065 nvmf_ctrlr_migr_data_copy(data, &data_local, spdk_min(data->data_size, data_local.data_size)); 2066 return 0; 2067 } 2068 2069 int 2070 spdk_nvmf_ctrlr_restore_migr_data(struct spdk_nvmf_ctrlr *ctrlr, 2071 const struct spdk_nvmf_ctrlr_migr_data *data) 2072 { 2073 uint32_t i; 2074 struct spdk_nvmf_ctrlr_migr_data data_local = { 2075 .data_size = offsetof(struct spdk_nvmf_ctrlr_migr_data, unused), 2076 .regs_size = sizeof(struct spdk_nvmf_registers), 2077 .feat_size = sizeof(struct spdk_nvmf_ctrlr_feat) 2078 }; 2079 2080 assert(data->data_size <= sizeof(data_local)); 2081 assert(spdk_get_thread() == ctrlr->thread); 2082 2083 /* local version of data should have defaults set before copy */ 2084 nvmf_ctrlr_migr_data_copy(&data_local, data, spdk_min(data->data_size, data_local.data_size)); 2085 memcpy(&ctrlr->vcprop, &data_local.regs, sizeof(struct spdk_nvmf_registers)); 2086 memcpy(&ctrlr->feat, &data_local.feat, sizeof(struct spdk_nvmf_ctrlr_feat)); 2087 2088 ctrlr->cntlid = data_local.cntlid; 2089 ctrlr->acre_enabled = data_local.acre; 2090 2091 for (i = 0; i < data_local.num_async_events; i++) { 2092 struct spdk_nvmf_async_event_completion *event; 2093 2094 event = calloc(1, sizeof(*event)); 2095 if (!event) { 2096 return -ENOMEM; 2097 } 2098 2099 event->event.raw = data_local.async_events[i].raw; 2100 STAILQ_INSERT_TAIL(&ctrlr->async_events, event, link); 2101 } 2102 ctrlr->notice_aen_mask = data_local.notice_aen_mask; 2103 2104 return 0; 2105 } 2106 2107 static int 2108 nvmf_ctrlr_set_features_async_event_configuration(struct spdk_nvmf_request *req) 2109 { 2110 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2111 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2112 2113 SPDK_DEBUGLOG(nvmf, "Set Features - Async Event Configuration, cdw11 0x%08x\n", 2114 cmd->cdw11); 2115 ctrlr->feat.async_event_configuration.raw = cmd->cdw11; 2116 ctrlr->feat.async_event_configuration.bits.reserved1 = 0; 2117 ctrlr->feat.async_event_configuration.bits.reserved2 = 0; 2118 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2119 } 2120 2121 static int 2122 nvmf_ctrlr_async_event_request(struct spdk_nvmf_request *req) 2123 { 2124 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2125 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 2126 struct spdk_nvmf_async_event_completion *pending_event; 2127 2128 SPDK_DEBUGLOG(nvmf, "Async Event Request\n"); 2129 2130 /* Four asynchronous events are supported for now */ 2131 if (ctrlr->nr_aer_reqs >= SPDK_NVMF_MAX_ASYNC_EVENTS) { 2132 SPDK_DEBUGLOG(nvmf, "AERL exceeded\n"); 2133 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 2134 rsp->status.sc = SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED; 2135 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2136 } 2137 2138 if (!STAILQ_EMPTY(&ctrlr->async_events)) { 2139 pending_event = STAILQ_FIRST(&ctrlr->async_events); 2140 rsp->cdw0 = pending_event->event.raw; 2141 STAILQ_REMOVE(&ctrlr->async_events, pending_event, spdk_nvmf_async_event_completion, link); 2142 free(pending_event); 2143 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2144 } 2145 2146 ctrlr->aer_req[ctrlr->nr_aer_reqs++] = req; 2147 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 2148 } 2149 2150 static void 2151 nvmf_get_firmware_slot_log_page(struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length) 2152 { 2153 struct spdk_nvme_firmware_page fw_page; 2154 size_t copy_len; 2155 struct spdk_iov_xfer ix; 2156 2157 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2158 2159 memset(&fw_page, 0, sizeof(fw_page)); 2160 fw_page.afi.active_slot = 1; 2161 fw_page.afi.next_reset_slot = 0; 2162 spdk_strcpy_pad(fw_page.revision[0], FW_VERSION, sizeof(fw_page.revision[0]), ' '); 2163 2164 if (offset < sizeof(fw_page)) { 2165 copy_len = spdk_min(sizeof(fw_page) - offset, length); 2166 if (copy_len > 0) { 2167 spdk_iov_xfer_from_buf(&ix, (const char *)&fw_page + offset, copy_len); 2168 } 2169 } 2170 } 2171 2172 /* 2173 * Asynchronous Event Mask Bit 2174 */ 2175 enum spdk_nvme_async_event_mask_bit { 2176 /* Mask Namespace Change Notification */ 2177 SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT = 0, 2178 /* Mask Asymmetric Namespace Access Change Notification */ 2179 SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT = 1, 2180 /* Mask Discovery Log Change Notification */ 2181 SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT = 2, 2182 /* Mask Reservation Log Page Available Notification */ 2183 SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT = 3, 2184 /* Mask Error Event */ 2185 SPDK_NVME_ASYNC_EVENT_ERROR_MASK_BIT = 4, 2186 /* 4 - 63 Reserved */ 2187 }; 2188 2189 static inline void 2190 nvmf_ctrlr_unmask_aen(struct spdk_nvmf_ctrlr *ctrlr, 2191 enum spdk_nvme_async_event_mask_bit mask) 2192 { 2193 ctrlr->notice_aen_mask &= ~(1 << mask); 2194 } 2195 2196 static inline bool 2197 nvmf_ctrlr_mask_aen(struct spdk_nvmf_ctrlr *ctrlr, 2198 enum spdk_nvme_async_event_mask_bit mask) 2199 { 2200 if (ctrlr->notice_aen_mask & (1 << mask)) { 2201 return false; 2202 } else { 2203 ctrlr->notice_aen_mask |= (1 << mask); 2204 return true; 2205 } 2206 } 2207 2208 /* we have to use the typedef in the function declaration to appease astyle. */ 2209 typedef enum spdk_nvme_ana_state spdk_nvme_ana_state_t; 2210 2211 static inline spdk_nvme_ana_state_t 2212 nvmf_ctrlr_get_ana_state(struct spdk_nvmf_ctrlr *ctrlr, uint32_t anagrpid) 2213 { 2214 if (!ctrlr->subsys->flags.ana_reporting) { 2215 return SPDK_NVME_ANA_OPTIMIZED_STATE; 2216 } 2217 2218 if (spdk_unlikely(ctrlr->listener == NULL)) { 2219 return SPDK_NVME_ANA_INACCESSIBLE_STATE; 2220 } 2221 2222 assert(anagrpid - 1 < ctrlr->subsys->max_nsid); 2223 return ctrlr->listener->ana_state[anagrpid - 1]; 2224 } 2225 2226 static spdk_nvme_ana_state_t 2227 nvmf_ctrlr_get_ana_state_from_nsid(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid) 2228 { 2229 struct spdk_nvmf_ns *ns; 2230 2231 /* We do not have NVM subsystem specific ANA state. Hence if NSID is either 2232 * SPDK_NVMF_GLOBAL_NS_TAG, invalid, or for inactive namespace, return 2233 * the optimized state. 2234 */ 2235 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, nsid); 2236 if (ns == NULL) { 2237 return SPDK_NVME_ANA_OPTIMIZED_STATE; 2238 } 2239 2240 return nvmf_ctrlr_get_ana_state(ctrlr, ns->anagrpid); 2241 } 2242 2243 static void 2244 nvmf_get_error_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iovcnt, 2245 uint64_t offset, uint32_t length, uint32_t rae) 2246 { 2247 if (!rae) { 2248 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ERROR_MASK_BIT); 2249 } 2250 2251 /* TODO: actually fill out log page data */ 2252 } 2253 2254 static void 2255 nvmf_get_ana_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iovcnt, 2256 uint64_t offset, uint32_t length, uint32_t rae) 2257 { 2258 struct spdk_nvme_ana_page ana_hdr; 2259 struct spdk_nvme_ana_group_descriptor ana_desc; 2260 size_t copy_len, copied_len; 2261 uint32_t num_anagrp = 0, anagrpid; 2262 struct spdk_nvmf_ns *ns; 2263 struct spdk_iov_xfer ix; 2264 2265 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2266 2267 if (length == 0) { 2268 goto done; 2269 } 2270 2271 if (offset >= sizeof(ana_hdr)) { 2272 offset -= sizeof(ana_hdr); 2273 } else { 2274 for (anagrpid = 1; anagrpid <= ctrlr->subsys->max_nsid; anagrpid++) { 2275 if (ctrlr->subsys->ana_group[anagrpid - 1] > 0) { 2276 num_anagrp++; 2277 } 2278 } 2279 2280 memset(&ana_hdr, 0, sizeof(ana_hdr)); 2281 2282 ana_hdr.num_ana_group_desc = num_anagrp; 2283 /* TODO: Support Change Count. */ 2284 ana_hdr.change_count = 0; 2285 2286 copy_len = spdk_min(sizeof(ana_hdr) - offset, length); 2287 copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ana_hdr + offset, copy_len); 2288 assert(copied_len == copy_len); 2289 length -= copied_len; 2290 offset = 0; 2291 } 2292 2293 if (length == 0) { 2294 goto done; 2295 } 2296 2297 for (anagrpid = 1; anagrpid <= ctrlr->subsys->max_nsid; anagrpid++) { 2298 if (ctrlr->subsys->ana_group[anagrpid - 1] == 0) { 2299 continue; 2300 } 2301 2302 if (offset >= sizeof(ana_desc)) { 2303 offset -= sizeof(ana_desc); 2304 } else { 2305 memset(&ana_desc, 0, sizeof(ana_desc)); 2306 2307 ana_desc.ana_group_id = anagrpid; 2308 ana_desc.num_of_nsid = ctrlr->subsys->ana_group[anagrpid - 1]; 2309 ana_desc.ana_state = nvmf_ctrlr_get_ana_state(ctrlr, anagrpid); 2310 2311 copy_len = spdk_min(sizeof(ana_desc) - offset, length); 2312 copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ana_desc + offset, 2313 copy_len); 2314 assert(copied_len == copy_len); 2315 length -= copied_len; 2316 offset = 0; 2317 2318 if (length == 0) { 2319 goto done; 2320 } 2321 } 2322 2323 /* TODO: Revisit here about O(n^2) cost if we have subsystem with 2324 * many namespaces in the future. 2325 */ 2326 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 2327 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 2328 if (ns->anagrpid != anagrpid) { 2329 continue; 2330 } 2331 2332 if (offset >= sizeof(uint32_t)) { 2333 offset -= sizeof(uint32_t); 2334 continue; 2335 } 2336 2337 copy_len = spdk_min(sizeof(uint32_t) - offset, length); 2338 copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ns->nsid + offset, 2339 copy_len); 2340 assert(copied_len == copy_len); 2341 length -= copied_len; 2342 offset = 0; 2343 2344 if (length == 0) { 2345 goto done; 2346 } 2347 } 2348 } 2349 2350 done: 2351 if (!rae) { 2352 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT); 2353 } 2354 } 2355 2356 void 2357 nvmf_ctrlr_ns_changed(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid) 2358 { 2359 uint16_t max_changes = SPDK_COUNTOF(ctrlr->changed_ns_list.ns_list); 2360 uint16_t i; 2361 bool found = false; 2362 2363 for (i = 0; i < ctrlr->changed_ns_list_count; i++) { 2364 if (ctrlr->changed_ns_list.ns_list[i] == nsid) { 2365 /* nsid is already in the list */ 2366 found = true; 2367 break; 2368 } 2369 } 2370 2371 if (!found) { 2372 if (ctrlr->changed_ns_list_count == max_changes) { 2373 /* Out of space - set first entry to FFFFFFFFh and zero-fill the rest. */ 2374 ctrlr->changed_ns_list.ns_list[0] = 0xFFFFFFFFu; 2375 for (i = 1; i < max_changes; i++) { 2376 ctrlr->changed_ns_list.ns_list[i] = 0; 2377 } 2378 } else { 2379 ctrlr->changed_ns_list.ns_list[ctrlr->changed_ns_list_count++] = nsid; 2380 } 2381 } 2382 } 2383 2384 static void 2385 nvmf_get_changed_ns_list_log_page(struct spdk_nvmf_ctrlr *ctrlr, 2386 struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length, uint32_t rae) 2387 { 2388 size_t copy_length; 2389 struct spdk_iov_xfer ix; 2390 2391 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2392 2393 if (offset < sizeof(ctrlr->changed_ns_list)) { 2394 copy_length = spdk_min(length, sizeof(ctrlr->changed_ns_list) - offset); 2395 if (copy_length) { 2396 spdk_iov_xfer_from_buf(&ix, (char *)&ctrlr->changed_ns_list + offset, copy_length); 2397 } 2398 } 2399 2400 /* Clear log page each time it is read */ 2401 ctrlr->changed_ns_list_count = 0; 2402 memset(&ctrlr->changed_ns_list, 0, sizeof(ctrlr->changed_ns_list)); 2403 2404 if (!rae) { 2405 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT); 2406 } 2407 } 2408 2409 /* The structure can be modified if we provide support for other commands in future */ 2410 static const struct spdk_nvme_cmds_and_effect_log_page g_cmds_and_effect_log_page = { 2411 .admin_cmds_supported = { 2412 /* CSUPP, LBCC, NCC, NIC, CCC, CSE */ 2413 /* Get Log Page */ 2414 [SPDK_NVME_OPC_GET_LOG_PAGE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2415 /* Identify */ 2416 [SPDK_NVME_OPC_IDENTIFY] = {1, 0, 0, 0, 0, 0, 0, 0}, 2417 /* Abort */ 2418 [SPDK_NVME_OPC_ABORT] = {1, 0, 0, 0, 0, 0, 0, 0}, 2419 /* Set Features */ 2420 [SPDK_NVME_OPC_SET_FEATURES] = {1, 0, 0, 0, 0, 0, 0, 0}, 2421 /* Get Features */ 2422 [SPDK_NVME_OPC_GET_FEATURES] = {1, 0, 0, 0, 0, 0, 0, 0}, 2423 /* Async Event Request */ 2424 [SPDK_NVME_OPC_ASYNC_EVENT_REQUEST] = {1, 0, 0, 0, 0, 0, 0, 0}, 2425 /* Keep Alive */ 2426 [SPDK_NVME_OPC_KEEP_ALIVE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2427 }, 2428 .io_cmds_supported = { 2429 /* FLUSH */ 2430 [SPDK_NVME_OPC_FLUSH] = {1, 1, 0, 0, 0, 0, 0, 0}, 2431 /* WRITE */ 2432 [SPDK_NVME_OPC_WRITE] = {1, 1, 0, 0, 0, 0, 0, 0}, 2433 /* READ */ 2434 [SPDK_NVME_OPC_READ] = {1, 0, 0, 0, 0, 0, 0, 0}, 2435 /* WRITE ZEROES */ 2436 [SPDK_NVME_OPC_WRITE_ZEROES] = {1, 1, 0, 0, 0, 0, 0, 0}, 2437 /* DATASET MANAGEMENT */ 2438 [SPDK_NVME_OPC_DATASET_MANAGEMENT] = {1, 1, 0, 0, 0, 0, 0, 0}, 2439 /* COMPARE */ 2440 [SPDK_NVME_OPC_COMPARE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2441 /* ZONE MANAGEMENT SEND */ 2442 [SPDK_NVME_OPC_ZONE_MGMT_SEND] = {1, 1, 0, 0, 0, 0, 0, 0}, 2443 /* ZONE MANAGEMENT RECEIVE */ 2444 [SPDK_NVME_OPC_ZONE_MGMT_RECV] = {1, 0, 0, 0, 0, 0, 0, 0}, 2445 /* COPY */ 2446 [SPDK_NVME_OPC_COPY] = {1, 1, 0, 0, 0, 0, 0, 0}, 2447 }, 2448 }; 2449 2450 static void 2451 nvmf_get_cmds_and_effects_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iovcnt, 2452 uint64_t offset, uint32_t length) 2453 { 2454 uint32_t page_size = sizeof(struct spdk_nvme_cmds_and_effect_log_page); 2455 size_t copy_len = 0; 2456 struct spdk_nvme_cmds_and_effect_log_page cmds_and_effect_log_page = g_cmds_and_effect_log_page; 2457 struct spdk_nvme_cmds_and_effect_entry csupp_and_lbcc_effect_entry = {1, 1, 0, 0, 0, 0, 0, 0}; 2458 struct spdk_iov_xfer ix; 2459 2460 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2461 2462 if (offset < page_size) { 2463 if (ctrlr->subsys->zone_append_supported) { 2464 cmds_and_effect_log_page.io_cmds_supported[SPDK_NVME_OPC_ZONE_APPEND] = 2465 csupp_and_lbcc_effect_entry; 2466 } 2467 copy_len = spdk_min(page_size - offset, length); 2468 spdk_iov_xfer_from_buf(&ix, (char *)(&cmds_and_effect_log_page) + offset, copy_len); 2469 } 2470 } 2471 2472 static void 2473 nvmf_get_reservation_notification_log_page(struct spdk_nvmf_ctrlr *ctrlr, 2474 struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length, uint32_t rae) 2475 { 2476 uint32_t unit_log_len, avail_log_len, next_pos, copy_len; 2477 struct spdk_nvmf_reservation_log *log, *log_tmp; 2478 struct spdk_iov_xfer ix; 2479 2480 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2481 2482 unit_log_len = sizeof(struct spdk_nvme_reservation_notification_log); 2483 /* No available log, return zeroed log pages */ 2484 if (!ctrlr->num_avail_log_pages) { 2485 return; 2486 } 2487 2488 avail_log_len = ctrlr->num_avail_log_pages * unit_log_len; 2489 if (offset >= avail_log_len) { 2490 return; 2491 } 2492 2493 next_pos = 0; 2494 TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) { 2495 TAILQ_REMOVE(&ctrlr->log_head, log, link); 2496 ctrlr->num_avail_log_pages--; 2497 2498 next_pos += unit_log_len; 2499 if (next_pos > offset) { 2500 copy_len = spdk_min(next_pos - offset, length); 2501 spdk_iov_xfer_from_buf(&ix, &log->log, copy_len); 2502 length -= copy_len; 2503 offset += copy_len; 2504 } 2505 free(log); 2506 2507 if (length == 0) { 2508 break; 2509 } 2510 } 2511 2512 if (!rae) { 2513 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT); 2514 } 2515 return; 2516 } 2517 2518 static int 2519 nvmf_ctrlr_get_log_page(struct spdk_nvmf_request *req) 2520 { 2521 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2522 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2523 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2524 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 2525 struct spdk_nvme_transport_id cmd_source_trid; 2526 uint64_t offset, len; 2527 uint32_t rae, numdl, numdu; 2528 uint8_t lid; 2529 2530 if (req->iovcnt < 1) { 2531 SPDK_DEBUGLOG(nvmf, "get log command with no buffer\n"); 2532 response->status.sct = SPDK_NVME_SCT_GENERIC; 2533 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2534 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2535 } 2536 2537 offset = (uint64_t)cmd->cdw12 | ((uint64_t)cmd->cdw13 << 32); 2538 if (offset & 3) { 2539 SPDK_ERRLOG("Invalid log page offset 0x%" PRIx64 "\n", offset); 2540 response->status.sct = SPDK_NVME_SCT_GENERIC; 2541 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2542 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2543 } 2544 2545 rae = cmd->cdw10_bits.get_log_page.rae; 2546 numdl = cmd->cdw10_bits.get_log_page.numdl; 2547 numdu = cmd->cdw11_bits.get_log_page.numdu; 2548 len = ((numdu << 16) + numdl + (uint64_t)1) * 4; 2549 if (len > req->length) { 2550 SPDK_ERRLOG("Get log page: len (%" PRIu64 ") > buf size (%u)\n", 2551 len, req->length); 2552 response->status.sct = SPDK_NVME_SCT_GENERIC; 2553 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2554 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2555 } 2556 2557 lid = cmd->cdw10_bits.get_log_page.lid; 2558 SPDK_DEBUGLOG(nvmf, "Get log page: LID=0x%02X offset=0x%" PRIx64 " len=0x%" PRIx64 " rae=%u\n", 2559 lid, offset, len, rae); 2560 2561 if (spdk_nvmf_subsystem_is_discovery(subsystem)) { 2562 switch (lid) { 2563 case SPDK_NVME_LOG_DISCOVERY: 2564 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &cmd_source_trid)) { 2565 SPDK_ERRLOG("Failed to get LOG_DISCOVERY source trid\n"); 2566 response->status.sct = SPDK_NVME_SCT_GENERIC; 2567 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 2568 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2569 } 2570 nvmf_get_discovery_log_page(subsystem->tgt, ctrlr->hostnqn, req->iov, req->iovcnt, 2571 offset, len, &cmd_source_trid); 2572 if (!rae) { 2573 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT); 2574 } 2575 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2576 default: 2577 goto invalid_log_page; 2578 } 2579 } else { 2580 if (offset > len) { 2581 SPDK_ERRLOG("Get log page: offset (%" PRIu64 ") > len (%" PRIu64 ")\n", 2582 offset, len); 2583 response->status.sct = SPDK_NVME_SCT_GENERIC; 2584 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2585 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2586 } 2587 2588 switch (lid) { 2589 case SPDK_NVME_LOG_ERROR: 2590 nvmf_get_error_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2591 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2592 case SPDK_NVME_LOG_HEALTH_INFORMATION: 2593 /* TODO: actually fill out log page data */ 2594 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2595 case SPDK_NVME_LOG_FIRMWARE_SLOT: 2596 nvmf_get_firmware_slot_log_page(req->iov, req->iovcnt, offset, len); 2597 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2598 case SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS: 2599 if (subsystem->flags.ana_reporting) { 2600 nvmf_get_ana_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2601 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2602 } else { 2603 goto invalid_log_page; 2604 } 2605 case SPDK_NVME_LOG_COMMAND_EFFECTS_LOG: 2606 nvmf_get_cmds_and_effects_log_page(ctrlr, req->iov, req->iovcnt, offset, len); 2607 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2608 case SPDK_NVME_LOG_CHANGED_NS_LIST: 2609 nvmf_get_changed_ns_list_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2610 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2611 case SPDK_NVME_LOG_RESERVATION_NOTIFICATION: 2612 nvmf_get_reservation_notification_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2613 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2614 default: 2615 goto invalid_log_page; 2616 } 2617 } 2618 2619 invalid_log_page: 2620 SPDK_INFOLOG(nvmf, "Unsupported Get Log Page 0x%02X\n", lid); 2621 response->status.sct = SPDK_NVME_SCT_GENERIC; 2622 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2623 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2624 } 2625 2626 static struct spdk_nvmf_ns * 2627 _nvmf_subsystem_get_ns_safe(struct spdk_nvmf_subsystem *subsystem, 2628 uint32_t nsid, 2629 struct spdk_nvme_cpl *rsp) 2630 { 2631 struct spdk_nvmf_ns *ns; 2632 if (nsid == 0 || nsid > subsystem->max_nsid) { 2633 SPDK_ERRLOG("Identify Namespace for invalid NSID %u\n", nsid); 2634 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2635 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 2636 return NULL; 2637 } 2638 2639 ns = _nvmf_subsystem_get_ns(subsystem, nsid); 2640 if (ns == NULL || ns->bdev == NULL) { 2641 /* 2642 * Inactive namespaces should return a zero filled data structure. 2643 * The data buffer is already zeroed by nvmf_ctrlr_process_admin_cmd(), 2644 * so we can just return early here. 2645 */ 2646 SPDK_DEBUGLOG(nvmf, "Identify Namespace for inactive NSID %u\n", nsid); 2647 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2648 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2649 return NULL; 2650 } 2651 return ns; 2652 } 2653 2654 int 2655 spdk_nvmf_ctrlr_identify_ns(struct spdk_nvmf_ctrlr *ctrlr, 2656 struct spdk_nvme_cmd *cmd, 2657 struct spdk_nvme_cpl *rsp, 2658 struct spdk_nvme_ns_data *nsdata) 2659 { 2660 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2661 struct spdk_nvmf_ns *ns; 2662 uint32_t max_num_blocks, format_index; 2663 enum spdk_nvme_ana_state ana_state; 2664 2665 ns = _nvmf_subsystem_get_ns_safe(subsystem, cmd->nsid, rsp); 2666 if (ns == NULL) { 2667 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2668 } 2669 2670 nvmf_bdev_ctrlr_identify_ns(ns, nsdata, ctrlr->dif_insert_or_strip); 2671 2672 assert(ctrlr->admin_qpair); 2673 2674 format_index = spdk_nvme_ns_get_format_index(nsdata); 2675 2676 /* Due to bug in the Linux kernel NVMe driver we have to set noiob no larger than mdts */ 2677 max_num_blocks = ctrlr->admin_qpair->transport->opts.max_io_size / 2678 (1U << nsdata->lbaf[format_index].lbads); 2679 if (nsdata->noiob > max_num_blocks) { 2680 nsdata->noiob = max_num_blocks; 2681 } 2682 2683 /* Set NOWS equal to Controller MDTS */ 2684 if (nsdata->nsfeat.optperf) { 2685 nsdata->nows = max_num_blocks - 1; 2686 } 2687 2688 if (subsystem->flags.ana_reporting) { 2689 assert(ns->anagrpid - 1 < subsystem->max_nsid); 2690 nsdata->anagrpid = ns->anagrpid; 2691 2692 ana_state = nvmf_ctrlr_get_ana_state(ctrlr, ns->anagrpid); 2693 if (ana_state == SPDK_NVME_ANA_INACCESSIBLE_STATE || 2694 ana_state == SPDK_NVME_ANA_PERSISTENT_LOSS_STATE) { 2695 nsdata->nuse = 0; 2696 } 2697 } 2698 2699 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2700 } 2701 2702 static void 2703 nvmf_ctrlr_populate_oacs(struct spdk_nvmf_ctrlr *ctrlr, 2704 struct spdk_nvme_ctrlr_data *cdata) 2705 { 2706 cdata->oacs = ctrlr->cdata.oacs; 2707 2708 cdata->oacs.virtualization_management = 2709 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_VIRTUALIZATION_MANAGEMENT].hdlr != NULL; 2710 cdata->oacs.nvme_mi = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NVME_MI_SEND].hdlr != NULL 2711 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NVME_MI_RECEIVE].hdlr != NULL; 2712 cdata->oacs.directives = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DIRECTIVE_SEND].hdlr != NULL 2713 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DIRECTIVE_RECEIVE].hdlr != NULL; 2714 cdata->oacs.device_self_test = 2715 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DEVICE_SELF_TEST].hdlr != NULL; 2716 cdata->oacs.ns_manage = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NS_MANAGEMENT].hdlr != NULL 2717 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NS_ATTACHMENT].hdlr != NULL; 2718 cdata->oacs.firmware = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD].hdlr != 2719 NULL 2720 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FIRMWARE_COMMIT].hdlr != NULL; 2721 cdata->oacs.format = 2722 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FORMAT_NVM].hdlr != NULL; 2723 cdata->oacs.security = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_SECURITY_SEND].hdlr != NULL 2724 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_SECURITY_RECEIVE].hdlr != NULL; 2725 cdata->oacs.get_lba_status = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_GET_LBA_STATUS].hdlr != 2726 NULL; 2727 } 2728 2729 int 2730 spdk_nvmf_ctrlr_identify_ctrlr(struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvme_ctrlr_data *cdata) 2731 { 2732 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2733 struct spdk_nvmf_transport *transport; 2734 2735 /* 2736 * Common fields for discovery and NVM subsystems 2737 */ 2738 assert(ctrlr->admin_qpair); 2739 transport = ctrlr->admin_qpair->transport; 2740 spdk_strcpy_pad(cdata->fr, FW_VERSION, sizeof(cdata->fr), ' '); 2741 assert((transport->opts.max_io_size % 4096) == 0); 2742 cdata->mdts = spdk_u32log2(transport->opts.max_io_size / 4096); 2743 cdata->cntlid = ctrlr->cntlid; 2744 cdata->ver = ctrlr->vcprop.vs; 2745 cdata->aerl = ctrlr->cdata.aerl; 2746 cdata->lpa.edlp = 1; 2747 cdata->elpe = 127; 2748 cdata->maxcmd = transport->opts.max_queue_depth; 2749 cdata->sgls = ctrlr->cdata.sgls; 2750 cdata->fuses = ctrlr->cdata.fuses; 2751 cdata->acwu = 0; /* ACWU is 0-based. */ 2752 if (subsystem->flags.ana_reporting) { 2753 cdata->mnan = subsystem->max_nsid; 2754 } 2755 spdk_strcpy_pad(cdata->subnqn, subsystem->subnqn, sizeof(cdata->subnqn), '\0'); 2756 2757 SPDK_DEBUGLOG(nvmf, "ctrlr data: maxcmd 0x%x\n", cdata->maxcmd); 2758 SPDK_DEBUGLOG(nvmf, "sgls data: 0x%x\n", from_le32(&cdata->sgls)); 2759 2760 2761 if (spdk_nvmf_subsystem_is_discovery(subsystem)) { 2762 /* 2763 * NVM Discovery subsystem fields 2764 */ 2765 cdata->oaes.discovery_log_change_notices = 1; 2766 } else { 2767 cdata->vid = ctrlr->cdata.vid; 2768 cdata->ssvid = ctrlr->cdata.ssvid; 2769 cdata->ieee[0] = ctrlr->cdata.ieee[0]; 2770 cdata->ieee[1] = ctrlr->cdata.ieee[1]; 2771 cdata->ieee[2] = ctrlr->cdata.ieee[2]; 2772 2773 /* 2774 * NVM subsystem fields (reserved for discovery subsystems) 2775 */ 2776 spdk_strcpy_pad(cdata->mn, spdk_nvmf_subsystem_get_mn(subsystem), sizeof(cdata->mn), ' '); 2777 spdk_strcpy_pad(cdata->sn, spdk_nvmf_subsystem_get_sn(subsystem), sizeof(cdata->sn), ' '); 2778 cdata->kas = ctrlr->cdata.kas; 2779 2780 cdata->rab = 6; 2781 cdata->cmic.multi_port = 1; 2782 cdata->cmic.multi_ctrlr = 1; 2783 cdata->oaes.ns_attribute_notices = 1; 2784 cdata->ctratt.host_id_exhid_supported = 1; 2785 /* We do not have any actual limitation to the number of abort commands. 2786 * We follow the recommendation by the NVMe specification. 2787 */ 2788 cdata->acl = NVMF_ABORT_COMMAND_LIMIT; 2789 cdata->frmw.slot1_ro = 1; 2790 cdata->frmw.num_slots = 1; 2791 2792 cdata->lpa.celp = 1; /* Command Effects log page supported */ 2793 2794 cdata->sqes.min = 6; 2795 cdata->sqes.max = 6; 2796 cdata->cqes.min = 4; 2797 cdata->cqes.max = 4; 2798 cdata->nn = subsystem->max_nsid; 2799 cdata->vwc.present = 1; 2800 cdata->vwc.flush_broadcast = SPDK_NVME_FLUSH_BROADCAST_NOT_SUPPORTED; 2801 2802 cdata->nvmf_specific = ctrlr->cdata.nvmf_specific; 2803 2804 cdata->oncs.compare = ctrlr->cdata.oncs.compare; 2805 cdata->oncs.dsm = nvmf_ctrlr_dsm_supported(ctrlr); 2806 cdata->oncs.write_zeroes = nvmf_ctrlr_write_zeroes_supported(ctrlr); 2807 cdata->oncs.reservations = ctrlr->cdata.oncs.reservations; 2808 cdata->oncs.copy = ctrlr->cdata.oncs.copy; 2809 cdata->ocfs.copy_format0 = cdata->oncs.copy; 2810 if (subsystem->flags.ana_reporting) { 2811 /* Asymmetric Namespace Access Reporting is supported. */ 2812 cdata->cmic.ana_reporting = 1; 2813 cdata->oaes.ana_change_notices = 1; 2814 2815 cdata->anatt = ANA_TRANSITION_TIME_IN_SEC; 2816 /* ANA Change state is not used, and ANA Persistent Loss state 2817 * is not supported for now. 2818 */ 2819 cdata->anacap.ana_optimized_state = 1; 2820 cdata->anacap.ana_non_optimized_state = 1; 2821 cdata->anacap.ana_inaccessible_state = 1; 2822 /* ANAGRPID does not change while namespace is attached to controller */ 2823 cdata->anacap.no_change_anagrpid = 1; 2824 cdata->anagrpmax = subsystem->max_nsid; 2825 cdata->nanagrpid = subsystem->max_nsid; 2826 } 2827 2828 nvmf_ctrlr_populate_oacs(ctrlr, cdata); 2829 2830 assert(subsystem->tgt != NULL); 2831 cdata->crdt[0] = subsystem->tgt->crdt[0]; 2832 cdata->crdt[1] = subsystem->tgt->crdt[1]; 2833 cdata->crdt[2] = subsystem->tgt->crdt[2]; 2834 2835 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: ioccsz 0x%x\n", 2836 cdata->nvmf_specific.ioccsz); 2837 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: iorcsz 0x%x\n", 2838 cdata->nvmf_specific.iorcsz); 2839 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: icdoff 0x%x\n", 2840 cdata->nvmf_specific.icdoff); 2841 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: ctrattr 0x%x\n", 2842 *(uint8_t *)&cdata->nvmf_specific.ctrattr); 2843 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: msdbd 0x%x\n", 2844 cdata->nvmf_specific.msdbd); 2845 } 2846 2847 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2848 } 2849 2850 static int 2851 nvmf_ns_identify_iocs_zns(struct spdk_nvmf_ns *ns, 2852 struct spdk_nvme_cmd *cmd, 2853 struct spdk_nvme_cpl *rsp, 2854 struct spdk_nvme_zns_ns_data *nsdata_zns) 2855 { 2856 nsdata_zns->zoc.variable_zone_capacity = 0; 2857 nsdata_zns->zoc.zone_active_excursions = 0; 2858 nsdata_zns->ozcs.read_across_zone_boundaries = 1; 2859 /* Underflowing the zero based mar and mor bdev helper results in the correct 2860 value of FFFFFFFFh. */ 2861 nsdata_zns->mar = spdk_bdev_get_max_active_zones(ns->bdev) - 1; 2862 nsdata_zns->mor = spdk_bdev_get_max_open_zones(ns->bdev) - 1; 2863 nsdata_zns->rrl = 0; 2864 nsdata_zns->frl = 0; 2865 nsdata_zns->lbafe[0].zsze = spdk_bdev_get_zone_size(ns->bdev); 2866 2867 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2868 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2869 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2870 } 2871 2872 int 2873 spdk_nvmf_ns_identify_iocs_specific(struct spdk_nvmf_ctrlr *ctrlr, 2874 struct spdk_nvme_cmd *cmd, 2875 struct spdk_nvme_cpl *rsp, 2876 void *nsdata, 2877 size_t nsdata_size) 2878 { 2879 uint8_t csi = cmd->cdw11_bits.identify.csi; 2880 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2881 struct spdk_nvmf_ns *ns = _nvmf_subsystem_get_ns_safe(subsystem, cmd->nsid, rsp); 2882 2883 memset(nsdata, 0, nsdata_size); 2884 2885 if (ns == NULL) { 2886 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2887 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 2888 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2889 } 2890 2891 switch (csi) { 2892 case SPDK_NVME_CSI_ZNS: 2893 return nvmf_ns_identify_iocs_zns(ns, cmd, rsp, nsdata); 2894 default: 2895 break; 2896 } 2897 2898 SPDK_DEBUGLOG(nvmf, 2899 "Returning zero filled struct for the iocs specific ns " 2900 "identify command and CSI 0x%02x\n", 2901 csi); 2902 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2903 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2904 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2905 } 2906 2907 static int 2908 nvmf_ctrlr_identify_iocs_zns(struct spdk_nvmf_ctrlr *ctrlr, 2909 struct spdk_nvme_cmd *cmd, 2910 struct spdk_nvme_cpl *rsp, 2911 struct spdk_nvme_zns_ctrlr_data *cdata_zns) 2912 { 2913 /* The unit of max_zone_append_size_kib is KiB. 2914 The unit of zasl is the minimum memory page size 2915 (2 ^ (12 + CAP.MPSMIN) KiB) 2916 and is reported as a power of two (2^n). */ 2917 cdata_zns->zasl = spdk_u64log2(ctrlr->subsys->max_zone_append_size_kib >> 2918 (12 + ctrlr->vcprop.cap.bits.mpsmin)); 2919 2920 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2921 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2922 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2923 } 2924 2925 int 2926 spdk_nvmf_ctrlr_identify_iocs_specific(struct spdk_nvmf_ctrlr *ctrlr, 2927 struct spdk_nvme_cmd *cmd, 2928 struct spdk_nvme_cpl *rsp, 2929 void *cdata, 2930 size_t cdata_size) 2931 { 2932 uint8_t csi = cmd->cdw11_bits.identify.csi; 2933 2934 memset(cdata, 0, cdata_size); 2935 2936 switch (csi) { 2937 case SPDK_NVME_CSI_ZNS: 2938 return nvmf_ctrlr_identify_iocs_zns(ctrlr, cmd, rsp, cdata); 2939 default: 2940 break; 2941 } 2942 2943 SPDK_DEBUGLOG(nvmf, 2944 "Returning zero filled struct for the iocs specific ctrlr " 2945 "identify command and CSI 0x%02x\n", 2946 csi); 2947 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2948 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2949 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2950 } 2951 2952 static int 2953 nvmf_ctrlr_identify_active_ns_list(struct spdk_nvmf_subsystem *subsystem, 2954 struct spdk_nvme_cmd *cmd, 2955 struct spdk_nvme_cpl *rsp, 2956 struct spdk_nvme_ns_list *ns_list) 2957 { 2958 struct spdk_nvmf_ns *ns; 2959 uint32_t count = 0; 2960 2961 if (cmd->nsid >= 0xfffffffeUL) { 2962 SPDK_ERRLOG("Identify Active Namespace List with invalid NSID %u\n", cmd->nsid); 2963 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 2964 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2965 } 2966 2967 memset(ns_list, 0, sizeof(*ns_list)); 2968 2969 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 2970 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 2971 if (ns->opts.nsid <= cmd->nsid) { 2972 continue; 2973 } 2974 2975 ns_list->ns_list[count++] = ns->opts.nsid; 2976 if (count == SPDK_COUNTOF(ns_list->ns_list)) { 2977 break; 2978 } 2979 } 2980 2981 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2982 } 2983 2984 static void 2985 _add_ns_id_desc(void **buf_ptr, size_t *buf_remain, 2986 enum spdk_nvme_nidt type, 2987 const void *data, size_t data_size) 2988 { 2989 struct spdk_nvme_ns_id_desc *desc; 2990 size_t desc_size = sizeof(*desc) + data_size; 2991 2992 /* 2993 * These should never fail in practice, since all valid NS ID descriptors 2994 * should be defined so that they fit in the available 4096-byte buffer. 2995 */ 2996 assert(data_size > 0); 2997 assert(data_size <= UINT8_MAX); 2998 assert(desc_size < *buf_remain); 2999 if (data_size == 0 || data_size > UINT8_MAX || desc_size > *buf_remain) { 3000 return; 3001 } 3002 3003 desc = *buf_ptr; 3004 desc->nidt = type; 3005 desc->nidl = data_size; 3006 memcpy(desc->nid, data, data_size); 3007 3008 *buf_ptr += desc_size; 3009 *buf_remain -= desc_size; 3010 } 3011 3012 static int 3013 nvmf_ctrlr_identify_ns_id_descriptor_list( 3014 struct spdk_nvmf_subsystem *subsystem, 3015 struct spdk_nvme_cmd *cmd, 3016 struct spdk_nvme_cpl *rsp, 3017 void *id_desc_list, size_t id_desc_list_size) 3018 { 3019 struct spdk_nvmf_ns *ns; 3020 size_t buf_remain = id_desc_list_size; 3021 void *buf_ptr = id_desc_list; 3022 3023 ns = _nvmf_subsystem_get_ns(subsystem, cmd->nsid); 3024 if (ns == NULL || ns->bdev == NULL) { 3025 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3026 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 3027 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3028 } 3029 3030 #define ADD_ID_DESC(type, data, size) \ 3031 do { \ 3032 if (!spdk_mem_all_zero(data, size)) { \ 3033 _add_ns_id_desc(&buf_ptr, &buf_remain, type, data, size); \ 3034 } \ 3035 } while (0) 3036 3037 ADD_ID_DESC(SPDK_NVME_NIDT_EUI64, ns->opts.eui64, sizeof(ns->opts.eui64)); 3038 ADD_ID_DESC(SPDK_NVME_NIDT_NGUID, ns->opts.nguid, sizeof(ns->opts.nguid)); 3039 ADD_ID_DESC(SPDK_NVME_NIDT_UUID, &ns->opts.uuid, sizeof(ns->opts.uuid)); 3040 ADD_ID_DESC(SPDK_NVME_NIDT_CSI, &ns->csi, sizeof(uint8_t)); 3041 3042 /* 3043 * The list is automatically 0-terminated, both in the temporary buffer 3044 * used by nvmf_ctrlr_identify(), and the eventual iov destination - 3045 * controller to host buffers in admin commands always get zeroed in 3046 * nvmf_ctrlr_process_admin_cmd(). 3047 */ 3048 3049 #undef ADD_ID_DESC 3050 3051 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3052 } 3053 3054 static int 3055 nvmf_ctrlr_identify_iocs(struct spdk_nvmf_ctrlr *ctrlr, 3056 struct spdk_nvme_cmd *cmd, 3057 struct spdk_nvme_cpl *rsp, 3058 void *cdata, size_t cdata_size) 3059 { 3060 struct spdk_nvme_iocs_vector *vector; 3061 struct spdk_nvmf_ns *ns; 3062 3063 if (cdata_size < sizeof(struct spdk_nvme_iocs_vector)) { 3064 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3065 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3066 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3067 } 3068 3069 /* For now we only support this command sent to the current 3070 * controller. 3071 */ 3072 if (cmd->cdw10_bits.identify.cntid != 0xFFFF && 3073 cmd->cdw10_bits.identify.cntid != ctrlr->cntlid) { 3074 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3075 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3076 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3077 } 3078 memset(cdata, 0, cdata_size); 3079 3080 vector = cdata; 3081 vector->nvm = 1; 3082 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 3083 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 3084 if (ns->bdev == NULL) { 3085 continue; 3086 } 3087 if (spdk_bdev_is_zoned(ns->bdev)) { 3088 vector->zns = 1; 3089 } 3090 } 3091 3092 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3093 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3094 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3095 } 3096 3097 static int 3098 nvmf_ctrlr_identify(struct spdk_nvmf_request *req) 3099 { 3100 uint8_t cns; 3101 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3102 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3103 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3104 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 3105 int ret = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3106 char tmpbuf[SPDK_NVME_IDENTIFY_BUFLEN] = ""; 3107 struct spdk_iov_xfer ix; 3108 3109 if (req->iovcnt < 1 || req->length < SPDK_NVME_IDENTIFY_BUFLEN) { 3110 SPDK_DEBUGLOG(nvmf, "identify command with invalid buffer\n"); 3111 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3112 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3113 return ret; 3114 } 3115 3116 cns = cmd->cdw10_bits.identify.cns; 3117 3118 if (spdk_nvmf_subsystem_is_discovery(subsystem) && 3119 cns != SPDK_NVME_IDENTIFY_CTRLR) { 3120 /* Discovery controllers only support Identify Controller */ 3121 goto invalid_cns; 3122 } 3123 3124 /* 3125 * We must use a temporary buffer: it's entirely possible the out buffer 3126 * is split across more than one IOV. 3127 */ 3128 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 3129 3130 SPDK_DEBUGLOG(nvmf, "Received identify command with CNS 0x%02x\n", cns); 3131 3132 switch (cns) { 3133 case SPDK_NVME_IDENTIFY_NS: 3134 ret = spdk_nvmf_ctrlr_identify_ns(ctrlr, cmd, rsp, (void *)&tmpbuf); 3135 break; 3136 case SPDK_NVME_IDENTIFY_CTRLR: 3137 ret = spdk_nvmf_ctrlr_identify_ctrlr(ctrlr, (void *)&tmpbuf); 3138 break; 3139 case SPDK_NVME_IDENTIFY_ACTIVE_NS_LIST: 3140 ret = nvmf_ctrlr_identify_active_ns_list(subsystem, cmd, rsp, (void *)&tmpbuf); 3141 break; 3142 case SPDK_NVME_IDENTIFY_NS_ID_DESCRIPTOR_LIST: 3143 ret = nvmf_ctrlr_identify_ns_id_descriptor_list(subsystem, cmd, rsp, 3144 tmpbuf, req->length); 3145 break; 3146 case SPDK_NVME_IDENTIFY_NS_IOCS: 3147 ret = spdk_nvmf_ns_identify_iocs_specific(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length); 3148 break; 3149 case SPDK_NVME_IDENTIFY_CTRLR_IOCS: 3150 ret = spdk_nvmf_ctrlr_identify_iocs_specific(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length); 3151 break; 3152 case SPDK_NVME_IDENTIFY_IOCS: 3153 ret = nvmf_ctrlr_identify_iocs(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length); 3154 break; 3155 default: 3156 goto invalid_cns; 3157 } 3158 3159 if (ret == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 3160 spdk_iov_xfer_from_buf(&ix, tmpbuf, sizeof(tmpbuf)); 3161 } 3162 3163 return ret; 3164 3165 invalid_cns: 3166 SPDK_DEBUGLOG(nvmf, "Identify command with unsupported CNS 0x%02x\n", cns); 3167 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3168 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3169 return ret; 3170 } 3171 3172 static bool 3173 nvmf_qpair_abort_aer(struct spdk_nvmf_qpair *qpair, uint16_t cid) 3174 { 3175 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 3176 struct spdk_nvmf_request *req; 3177 int i; 3178 3179 if (!nvmf_qpair_is_admin_queue(qpair)) { 3180 return false; 3181 } 3182 3183 assert(spdk_get_thread() == ctrlr->thread); 3184 3185 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 3186 if (ctrlr->aer_req[i]->cmd->nvme_cmd.cid == cid) { 3187 SPDK_DEBUGLOG(nvmf, "Aborting AER request\n"); 3188 req = ctrlr->aer_req[i]; 3189 ctrlr->aer_req[i] = NULL; 3190 ctrlr->nr_aer_reqs--; 3191 3192 /* Move the last req to the aborting position for making aer_reqs 3193 * in continuous 3194 */ 3195 if (i < ctrlr->nr_aer_reqs) { 3196 ctrlr->aer_req[i] = ctrlr->aer_req[ctrlr->nr_aer_reqs]; 3197 ctrlr->aer_req[ctrlr->nr_aer_reqs] = NULL; 3198 } 3199 3200 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3201 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 3202 _nvmf_request_complete(req); 3203 return true; 3204 } 3205 } 3206 3207 return false; 3208 } 3209 3210 void 3211 nvmf_qpair_abort_pending_zcopy_reqs(struct spdk_nvmf_qpair *qpair) 3212 { 3213 struct spdk_nvmf_request *req, *tmp; 3214 3215 TAILQ_FOREACH_SAFE(req, &qpair->outstanding, link, tmp) { 3216 if (req->zcopy_phase == NVMF_ZCOPY_PHASE_EXECUTE) { 3217 /* Zero-copy requests are kept on the outstanding queue from the moment 3218 * zcopy_start is sent until a zcopy_end callback is received. Therefore, 3219 * we can't remove them from the outstanding queue here, but need to rely on 3220 * the transport to do a zcopy_end to release their buffers and, in turn, 3221 * remove them from the queue. 3222 */ 3223 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3224 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 3225 nvmf_transport_req_free(req); 3226 } 3227 } 3228 } 3229 3230 static void 3231 nvmf_qpair_abort_request(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_request *req) 3232 { 3233 uint16_t cid = req->cmd->nvme_cmd.cdw10_bits.abort.cid; 3234 3235 if (nvmf_qpair_abort_aer(qpair, cid)) { 3236 SPDK_DEBUGLOG(nvmf, "abort ctrlr=%p sqid=%u cid=%u successful\n", 3237 qpair->ctrlr, qpair->qid, cid); 3238 req->rsp->nvme_cpl.cdw0 &= ~1U; /* Command successfully aborted */ 3239 3240 spdk_nvmf_request_complete(req); 3241 return; 3242 } 3243 3244 nvmf_transport_qpair_abort_request(qpair, req); 3245 } 3246 3247 static void 3248 nvmf_ctrlr_abort_done(struct spdk_io_channel_iter *i, int status) 3249 { 3250 struct spdk_nvmf_request *req = spdk_io_channel_iter_get_ctx(i); 3251 3252 if (status == 0) { 3253 /* There was no qpair whose ID matches SQID of the abort command. 3254 * Hence call _nvmf_request_complete() here. 3255 */ 3256 _nvmf_request_complete(req); 3257 } 3258 } 3259 3260 static void 3261 nvmf_ctrlr_abort_on_pg(struct spdk_io_channel_iter *i) 3262 { 3263 struct spdk_nvmf_request *req = spdk_io_channel_iter_get_ctx(i); 3264 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 3265 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 3266 uint16_t sqid = req->cmd->nvme_cmd.cdw10_bits.abort.sqid; 3267 struct spdk_nvmf_qpair *qpair; 3268 3269 TAILQ_FOREACH(qpair, &group->qpairs, link) { 3270 if (qpair->ctrlr == req->qpair->ctrlr && qpair->qid == sqid) { 3271 /* Found the qpair */ 3272 3273 nvmf_qpair_abort_request(qpair, req); 3274 3275 /* Return -1 for the status so the iteration across threads stops. */ 3276 spdk_for_each_channel_continue(i, -1); 3277 return; 3278 } 3279 } 3280 3281 spdk_for_each_channel_continue(i, 0); 3282 } 3283 3284 static int 3285 nvmf_ctrlr_abort(struct spdk_nvmf_request *req) 3286 { 3287 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3288 3289 rsp->cdw0 = 1U; /* Command not aborted */ 3290 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3291 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3292 3293 /* Send a message to each poll group, searching for this ctrlr, sqid, and command. */ 3294 spdk_for_each_channel(req->qpair->ctrlr->subsys->tgt, 3295 nvmf_ctrlr_abort_on_pg, 3296 req, 3297 nvmf_ctrlr_abort_done 3298 ); 3299 3300 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 3301 } 3302 3303 int 3304 nvmf_ctrlr_abort_request(struct spdk_nvmf_request *req) 3305 { 3306 struct spdk_nvmf_request *req_to_abort = req->req_to_abort; 3307 struct spdk_bdev *bdev; 3308 struct spdk_bdev_desc *desc; 3309 struct spdk_io_channel *ch; 3310 int rc; 3311 3312 assert(req_to_abort != NULL); 3313 3314 if (g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_ABORT].hdlr && 3315 nvmf_qpair_is_admin_queue(req_to_abort->qpair)) { 3316 return g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_ABORT].hdlr(req); 3317 } 3318 3319 rc = spdk_nvmf_request_get_bdev(req_to_abort->cmd->nvme_cmd.nsid, req_to_abort, 3320 &bdev, &desc, &ch); 3321 if (rc != 0) { 3322 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3323 } 3324 3325 return spdk_nvmf_bdev_ctrlr_abort_cmd(bdev, desc, ch, req, req_to_abort); 3326 } 3327 3328 static int 3329 get_features_generic(struct spdk_nvmf_request *req, uint32_t cdw0) 3330 { 3331 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3332 3333 rsp->cdw0 = cdw0; 3334 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3335 } 3336 3337 /* we have to use the typedef in the function declaration to appease astyle. */ 3338 typedef enum spdk_nvme_path_status_code spdk_nvme_path_status_code_t; 3339 3340 static spdk_nvme_path_status_code_t 3341 _nvme_ana_state_to_path_status(enum spdk_nvme_ana_state ana_state) 3342 { 3343 switch (ana_state) { 3344 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3345 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_INACCESSIBLE; 3346 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3347 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS; 3348 case SPDK_NVME_ANA_CHANGE_STATE: 3349 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_TRANSITION; 3350 default: 3351 return SPDK_NVME_SC_INTERNAL_PATH_ERROR; 3352 } 3353 } 3354 3355 static int 3356 nvmf_ctrlr_get_features(struct spdk_nvmf_request *req) 3357 { 3358 uint8_t feature; 3359 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3360 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3361 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 3362 enum spdk_nvme_ana_state ana_state; 3363 3364 feature = cmd->cdw10_bits.get_features.fid; 3365 3366 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 3367 /* 3368 * Features supported by Discovery controller 3369 */ 3370 switch (feature) { 3371 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3372 return get_features_generic(req, ctrlr->feat.keep_alive_timer.raw); 3373 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3374 return get_features_generic(req, ctrlr->feat.async_event_configuration.raw); 3375 default: 3376 SPDK_INFOLOG(nvmf, "Get Features command with unsupported feature ID 0x%02x\n", feature); 3377 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3378 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3379 } 3380 } 3381 /* 3382 * Process Get Features command for non-discovery controller 3383 */ 3384 ana_state = nvmf_ctrlr_get_ana_state_from_nsid(ctrlr, cmd->nsid); 3385 switch (ana_state) { 3386 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3387 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3388 case SPDK_NVME_ANA_CHANGE_STATE: 3389 switch (feature) { 3390 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3391 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3392 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3393 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3394 response->status.sct = SPDK_NVME_SCT_PATH; 3395 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 3396 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3397 default: 3398 break; 3399 } 3400 break; 3401 default: 3402 break; 3403 } 3404 3405 switch (feature) { 3406 case SPDK_NVME_FEAT_ARBITRATION: 3407 return get_features_generic(req, ctrlr->feat.arbitration.raw); 3408 case SPDK_NVME_FEAT_POWER_MANAGEMENT: 3409 return get_features_generic(req, ctrlr->feat.power_management.raw); 3410 case SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD: 3411 return nvmf_ctrlr_get_features_temperature_threshold(req); 3412 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3413 return get_features_generic(req, ctrlr->feat.error_recovery.raw); 3414 case SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE: 3415 return get_features_generic(req, ctrlr->feat.volatile_write_cache.raw); 3416 case SPDK_NVME_FEAT_NUMBER_OF_QUEUES: 3417 return get_features_generic(req, ctrlr->feat.number_of_queues.raw); 3418 case SPDK_NVME_FEAT_INTERRUPT_COALESCING: 3419 return get_features_generic(req, ctrlr->feat.interrupt_coalescing.raw); 3420 case SPDK_NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 3421 return nvmf_ctrlr_get_features_interrupt_vector_configuration(req); 3422 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3423 return get_features_generic(req, ctrlr->feat.write_atomicity.raw); 3424 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3425 return get_features_generic(req, ctrlr->feat.async_event_configuration.raw); 3426 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3427 return get_features_generic(req, ctrlr->feat.keep_alive_timer.raw); 3428 case SPDK_NVME_FEAT_HOST_IDENTIFIER: 3429 return nvmf_ctrlr_get_features_host_identifier(req); 3430 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3431 return nvmf_ctrlr_get_features_reservation_notification_mask(req); 3432 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3433 return nvmf_ctrlr_get_features_reservation_persistence(req); 3434 case SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 3435 return nvmf_ctrlr_get_features_host_behavior_support(req); 3436 default: 3437 SPDK_INFOLOG(nvmf, "Get Features command with unsupported feature ID 0x%02x\n", feature); 3438 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3439 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3440 } 3441 } 3442 3443 static int 3444 nvmf_ctrlr_set_features(struct spdk_nvmf_request *req) 3445 { 3446 uint8_t feature, save; 3447 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3448 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3449 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 3450 enum spdk_nvme_ana_state ana_state; 3451 /* 3452 * Features are not saveable by the controller as indicated by 3453 * ONCS field of the Identify Controller data. 3454 * */ 3455 save = cmd->cdw10_bits.set_features.sv; 3456 if (save) { 3457 response->status.sc = SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE; 3458 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 3459 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3460 } 3461 3462 feature = cmd->cdw10_bits.set_features.fid; 3463 3464 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 3465 /* 3466 * Features supported by Discovery controller 3467 */ 3468 switch (feature) { 3469 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3470 return nvmf_ctrlr_set_features_keep_alive_timer(req); 3471 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3472 return nvmf_ctrlr_set_features_async_event_configuration(req); 3473 default: 3474 SPDK_INFOLOG(nvmf, "Set Features command with unsupported feature ID 0x%02x\n", feature); 3475 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3476 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3477 } 3478 } 3479 /* 3480 * Process Set Features command for non-discovery controller 3481 */ 3482 ana_state = nvmf_ctrlr_get_ana_state_from_nsid(ctrlr, cmd->nsid); 3483 switch (ana_state) { 3484 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3485 case SPDK_NVME_ANA_CHANGE_STATE: 3486 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 3487 response->status.sct = SPDK_NVME_SCT_PATH; 3488 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 3489 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3490 } else { 3491 switch (feature) { 3492 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3493 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3494 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3495 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3496 response->status.sct = SPDK_NVME_SCT_PATH; 3497 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 3498 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3499 default: 3500 break; 3501 } 3502 } 3503 break; 3504 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3505 response->status.sct = SPDK_NVME_SCT_PATH; 3506 response->status.sc = SPDK_NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS; 3507 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3508 default: 3509 break; 3510 } 3511 3512 switch (feature) { 3513 case SPDK_NVME_FEAT_ARBITRATION: 3514 return nvmf_ctrlr_set_features_arbitration(req); 3515 case SPDK_NVME_FEAT_POWER_MANAGEMENT: 3516 return nvmf_ctrlr_set_features_power_management(req); 3517 case SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD: 3518 return nvmf_ctrlr_set_features_temperature_threshold(req); 3519 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3520 return nvmf_ctrlr_set_features_error_recovery(req); 3521 case SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE: 3522 return nvmf_ctrlr_set_features_volatile_write_cache(req); 3523 case SPDK_NVME_FEAT_NUMBER_OF_QUEUES: 3524 return nvmf_ctrlr_set_features_number_of_queues(req); 3525 case SPDK_NVME_FEAT_INTERRUPT_COALESCING: 3526 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 3527 response->status.sc = SPDK_NVME_SC_FEATURE_NOT_CHANGEABLE; 3528 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3529 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3530 return nvmf_ctrlr_set_features_write_atomicity(req); 3531 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3532 return nvmf_ctrlr_set_features_async_event_configuration(req); 3533 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3534 return nvmf_ctrlr_set_features_keep_alive_timer(req); 3535 case SPDK_NVME_FEAT_HOST_IDENTIFIER: 3536 return nvmf_ctrlr_set_features_host_identifier(req); 3537 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3538 return nvmf_ctrlr_set_features_reservation_notification_mask(req); 3539 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3540 return nvmf_ctrlr_set_features_reservation_persistence(req); 3541 case SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 3542 return nvmf_ctrlr_set_features_host_behavior_support(req); 3543 default: 3544 SPDK_INFOLOG(nvmf, "Set Features command with unsupported feature ID 0x%02x\n", feature); 3545 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3546 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3547 } 3548 } 3549 3550 static int 3551 nvmf_ctrlr_keep_alive(struct spdk_nvmf_request *req) 3552 { 3553 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3554 3555 SPDK_DEBUGLOG(nvmf, "Keep Alive\n"); 3556 /* 3557 * To handle keep alive just clear or reset the 3558 * ctrlr based keep alive duration counter. 3559 * When added, a separate timer based process 3560 * will monitor if the time since last recorded 3561 * keep alive has exceeded the max duration and 3562 * take appropriate action. 3563 */ 3564 ctrlr->last_keep_alive_tick = spdk_get_ticks(); 3565 3566 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3567 } 3568 3569 int 3570 nvmf_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req) 3571 { 3572 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3573 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3574 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 3575 struct spdk_nvmf_subsystem_poll_group *sgroup; 3576 int rc; 3577 3578 if (ctrlr == NULL) { 3579 SPDK_ERRLOG("Admin command sent before CONNECT\n"); 3580 response->status.sct = SPDK_NVME_SCT_GENERIC; 3581 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 3582 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3583 } 3584 3585 if (cmd->opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) { 3586 /* We do not want to treat AERs as outstanding commands, 3587 * so decrement mgmt_io_outstanding here to offset 3588 * the increment that happened prior to this call. 3589 */ 3590 sgroup = &req->qpair->group->sgroups[ctrlr->subsys->id]; 3591 assert(sgroup != NULL); 3592 sgroup->mgmt_io_outstanding--; 3593 } 3594 3595 assert(spdk_get_thread() == ctrlr->thread); 3596 3597 if (cmd->fuse != 0) { 3598 /* Fused admin commands are not supported. */ 3599 response->status.sct = SPDK_NVME_SCT_GENERIC; 3600 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3601 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3602 } 3603 3604 if (ctrlr->vcprop.cc.bits.en != 1) { 3605 SPDK_ERRLOG("Admin command sent to disabled controller\n"); 3606 response->status.sct = SPDK_NVME_SCT_GENERIC; 3607 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 3608 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3609 } 3610 3611 if (req->iovcnt && spdk_nvme_opc_get_data_transfer(cmd->opc) == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 3612 spdk_iov_memset(req->iov, req->iovcnt, 0); 3613 } 3614 3615 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 3616 /* Discovery controllers only support these admin OPS. */ 3617 switch (cmd->opc) { 3618 case SPDK_NVME_OPC_IDENTIFY: 3619 case SPDK_NVME_OPC_GET_LOG_PAGE: 3620 case SPDK_NVME_OPC_KEEP_ALIVE: 3621 case SPDK_NVME_OPC_SET_FEATURES: 3622 case SPDK_NVME_OPC_GET_FEATURES: 3623 case SPDK_NVME_OPC_ASYNC_EVENT_REQUEST: 3624 break; 3625 default: 3626 goto invalid_opcode; 3627 } 3628 } 3629 3630 /* Call a custom adm cmd handler if set. Aborts are handled in a different path (see nvmf_passthru_admin_cmd) */ 3631 if (g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].hdlr && cmd->opc != SPDK_NVME_OPC_ABORT) { 3632 rc = g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].hdlr(req); 3633 if (rc >= SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 3634 /* The handler took care of this command */ 3635 return rc; 3636 } 3637 } 3638 3639 switch (cmd->opc) { 3640 case SPDK_NVME_OPC_GET_LOG_PAGE: 3641 return nvmf_ctrlr_get_log_page(req); 3642 case SPDK_NVME_OPC_IDENTIFY: 3643 return nvmf_ctrlr_identify(req); 3644 case SPDK_NVME_OPC_ABORT: 3645 return nvmf_ctrlr_abort(req); 3646 case SPDK_NVME_OPC_GET_FEATURES: 3647 return nvmf_ctrlr_get_features(req); 3648 case SPDK_NVME_OPC_SET_FEATURES: 3649 return nvmf_ctrlr_set_features(req); 3650 case SPDK_NVME_OPC_ASYNC_EVENT_REQUEST: 3651 return nvmf_ctrlr_async_event_request(req); 3652 case SPDK_NVME_OPC_KEEP_ALIVE: 3653 return nvmf_ctrlr_keep_alive(req); 3654 3655 case SPDK_NVME_OPC_CREATE_IO_SQ: 3656 case SPDK_NVME_OPC_CREATE_IO_CQ: 3657 case SPDK_NVME_OPC_DELETE_IO_SQ: 3658 case SPDK_NVME_OPC_DELETE_IO_CQ: 3659 /* Create and Delete I/O CQ/SQ not allowed in NVMe-oF */ 3660 goto invalid_opcode; 3661 3662 default: 3663 goto invalid_opcode; 3664 } 3665 3666 invalid_opcode: 3667 SPDK_INFOLOG(nvmf, "Unsupported admin opcode 0x%x\n", cmd->opc); 3668 response->status.sct = SPDK_NVME_SCT_GENERIC; 3669 response->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3670 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3671 } 3672 3673 static int 3674 nvmf_ctrlr_process_fabrics_cmd(struct spdk_nvmf_request *req) 3675 { 3676 struct spdk_nvmf_qpair *qpair = req->qpair; 3677 struct spdk_nvmf_capsule_cmd *cap_hdr; 3678 3679 cap_hdr = &req->cmd->nvmf_cmd; 3680 3681 if (qpair->ctrlr == NULL) { 3682 /* No ctrlr established yet; the only valid command is Connect */ 3683 if (cap_hdr->fctype == SPDK_NVMF_FABRIC_COMMAND_CONNECT) { 3684 return nvmf_ctrlr_cmd_connect(req); 3685 } else { 3686 SPDK_DEBUGLOG(nvmf, "Got fctype 0x%x, expected Connect\n", 3687 cap_hdr->fctype); 3688 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3689 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 3690 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3691 } 3692 } else if (nvmf_qpair_is_admin_queue(qpair)) { 3693 /* 3694 * Controller session is established, and this is an admin queue. 3695 * Disallow Connect and allow other fabrics commands. 3696 */ 3697 switch (cap_hdr->fctype) { 3698 case SPDK_NVMF_FABRIC_COMMAND_PROPERTY_SET: 3699 return nvmf_property_set(req); 3700 case SPDK_NVMF_FABRIC_COMMAND_PROPERTY_GET: 3701 return nvmf_property_get(req); 3702 default: 3703 SPDK_DEBUGLOG(nvmf, "unknown fctype 0x%02x\n", 3704 cap_hdr->fctype); 3705 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3706 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3707 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3708 } 3709 } else { 3710 /* Controller session is established, and this is an I/O queue */ 3711 /* For now, no I/O-specific Fabrics commands are implemented (other than Connect) */ 3712 SPDK_DEBUGLOG(nvmf, "Unexpected I/O fctype 0x%x\n", cap_hdr->fctype); 3713 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3714 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3715 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3716 } 3717 } 3718 3719 static inline void 3720 nvmf_ctrlr_queue_pending_async_event(struct spdk_nvmf_ctrlr *ctrlr, 3721 union spdk_nvme_async_event_completion *event) 3722 { 3723 struct spdk_nvmf_async_event_completion *nvmf_event; 3724 3725 nvmf_event = calloc(1, sizeof(*nvmf_event)); 3726 if (!nvmf_event) { 3727 SPDK_ERRLOG("Alloc nvmf event failed, ignore the event\n"); 3728 return; 3729 } 3730 nvmf_event->event.raw = event->raw; 3731 STAILQ_INSERT_TAIL(&ctrlr->async_events, nvmf_event, link); 3732 } 3733 3734 static inline int 3735 nvmf_ctrlr_async_event_notification(struct spdk_nvmf_ctrlr *ctrlr, 3736 union spdk_nvme_async_event_completion *event) 3737 { 3738 struct spdk_nvmf_request *req; 3739 struct spdk_nvme_cpl *rsp; 3740 3741 assert(spdk_get_thread() == ctrlr->thread); 3742 3743 /* If there is no outstanding AER request, queue the event. Then 3744 * if an AER is later submitted, this event can be sent as a 3745 * response. 3746 */ 3747 if (ctrlr->nr_aer_reqs == 0) { 3748 nvmf_ctrlr_queue_pending_async_event(ctrlr, event); 3749 return 0; 3750 } 3751 3752 req = ctrlr->aer_req[--ctrlr->nr_aer_reqs]; 3753 rsp = &req->rsp->nvme_cpl; 3754 3755 rsp->cdw0 = event->raw; 3756 3757 _nvmf_request_complete(req); 3758 ctrlr->aer_req[ctrlr->nr_aer_reqs] = NULL; 3759 3760 return 0; 3761 } 3762 3763 int 3764 nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr) 3765 { 3766 union spdk_nvme_async_event_completion event = {0}; 3767 3768 /* Users may disable the event notification */ 3769 if (!ctrlr->feat.async_event_configuration.bits.ns_attr_notice) { 3770 return 0; 3771 } 3772 3773 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT)) { 3774 return 0; 3775 } 3776 3777 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 3778 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED; 3779 event.bits.log_page_identifier = SPDK_NVME_LOG_CHANGED_NS_LIST; 3780 3781 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 3782 } 3783 3784 int 3785 nvmf_ctrlr_async_event_ana_change_notice(struct spdk_nvmf_ctrlr *ctrlr) 3786 { 3787 union spdk_nvme_async_event_completion event = {0}; 3788 3789 /* Users may disable the event notification */ 3790 if (!ctrlr->feat.async_event_configuration.bits.ana_change_notice) { 3791 return 0; 3792 } 3793 3794 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT)) { 3795 return 0; 3796 } 3797 3798 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 3799 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_ANA_CHANGE; 3800 event.bits.log_page_identifier = SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS; 3801 3802 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 3803 } 3804 3805 void 3806 nvmf_ctrlr_async_event_reservation_notification(struct spdk_nvmf_ctrlr *ctrlr) 3807 { 3808 union spdk_nvme_async_event_completion event = {0}; 3809 3810 if (!ctrlr->num_avail_log_pages) { 3811 return; 3812 } 3813 3814 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT)) { 3815 return; 3816 } 3817 3818 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_IO; 3819 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL; 3820 event.bits.log_page_identifier = SPDK_NVME_LOG_RESERVATION_NOTIFICATION; 3821 3822 nvmf_ctrlr_async_event_notification(ctrlr, &event); 3823 } 3824 3825 void 3826 nvmf_ctrlr_async_event_discovery_log_change_notice(void *ctx) 3827 { 3828 union spdk_nvme_async_event_completion event = {0}; 3829 struct spdk_nvmf_ctrlr *ctrlr = ctx; 3830 3831 /* Users may disable the event notification manually or 3832 * it may not be enabled due to keep alive timeout 3833 * not being set in connect command to discovery controller. 3834 */ 3835 if (!ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice) { 3836 return; 3837 } 3838 3839 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT)) { 3840 return; 3841 } 3842 3843 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 3844 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE; 3845 event.bits.log_page_identifier = SPDK_NVME_LOG_DISCOVERY; 3846 3847 nvmf_ctrlr_async_event_notification(ctrlr, &event); 3848 } 3849 3850 int 3851 spdk_nvmf_ctrlr_async_event_error_event(struct spdk_nvmf_ctrlr *ctrlr, 3852 enum spdk_nvme_async_event_info_error info) 3853 { 3854 union spdk_nvme_async_event_completion event; 3855 3856 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ERROR_MASK_BIT)) { 3857 return 0; 3858 } 3859 3860 if (info > SPDK_NVME_ASYNC_EVENT_FW_IMAGE_LOAD) { 3861 return 0; 3862 } 3863 3864 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_ERROR; 3865 event.bits.log_page_identifier = SPDK_NVME_LOG_ERROR; 3866 event.bits.async_event_info = info; 3867 3868 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 3869 } 3870 3871 void 3872 nvmf_qpair_free_aer(struct spdk_nvmf_qpair *qpair) 3873 { 3874 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 3875 int i; 3876 3877 if (!nvmf_qpair_is_admin_queue(qpair)) { 3878 return; 3879 } 3880 3881 assert(spdk_get_thread() == ctrlr->thread); 3882 3883 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 3884 spdk_nvmf_request_free(ctrlr->aer_req[i]); 3885 ctrlr->aer_req[i] = NULL; 3886 } 3887 3888 ctrlr->nr_aer_reqs = 0; 3889 } 3890 3891 void 3892 spdk_nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr) 3893 { 3894 struct spdk_nvmf_request *req; 3895 int i; 3896 3897 assert(spdk_get_thread() == ctrlr->thread); 3898 3899 if (!ctrlr->nr_aer_reqs) { 3900 return; 3901 } 3902 3903 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 3904 req = ctrlr->aer_req[i]; 3905 3906 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3907 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 3908 _nvmf_request_complete(req); 3909 3910 ctrlr->aer_req[i] = NULL; 3911 } 3912 3913 ctrlr->nr_aer_reqs = 0; 3914 } 3915 3916 static void 3917 _nvmf_ctrlr_add_reservation_log(void *ctx) 3918 { 3919 struct spdk_nvmf_reservation_log *log = (struct spdk_nvmf_reservation_log *)ctx; 3920 struct spdk_nvmf_ctrlr *ctrlr = log->ctrlr; 3921 3922 ctrlr->log_page_count++; 3923 3924 /* Maximum number of queued log pages is 255 */ 3925 if (ctrlr->num_avail_log_pages == 0xff) { 3926 struct spdk_nvmf_reservation_log *entry; 3927 entry = TAILQ_LAST(&ctrlr->log_head, log_page_head); 3928 entry->log.log_page_count = ctrlr->log_page_count; 3929 free(log); 3930 return; 3931 } 3932 3933 log->log.log_page_count = ctrlr->log_page_count; 3934 log->log.num_avail_log_pages = ctrlr->num_avail_log_pages++; 3935 TAILQ_INSERT_TAIL(&ctrlr->log_head, log, link); 3936 3937 nvmf_ctrlr_async_event_reservation_notification(ctrlr); 3938 } 3939 3940 void 3941 nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr, 3942 struct spdk_nvmf_ns *ns, 3943 enum spdk_nvme_reservation_notification_log_page_type type) 3944 { 3945 struct spdk_nvmf_reservation_log *log; 3946 3947 switch (type) { 3948 case SPDK_NVME_RESERVATION_LOG_PAGE_EMPTY: 3949 return; 3950 case SPDK_NVME_REGISTRATION_PREEMPTED: 3951 if (ns->mask & SPDK_NVME_REGISTRATION_PREEMPTED_MASK) { 3952 return; 3953 } 3954 break; 3955 case SPDK_NVME_RESERVATION_RELEASED: 3956 if (ns->mask & SPDK_NVME_RESERVATION_RELEASED_MASK) { 3957 return; 3958 } 3959 break; 3960 case SPDK_NVME_RESERVATION_PREEMPTED: 3961 if (ns->mask & SPDK_NVME_RESERVATION_PREEMPTED_MASK) { 3962 return; 3963 } 3964 break; 3965 default: 3966 return; 3967 } 3968 3969 log = calloc(1, sizeof(*log)); 3970 if (!log) { 3971 SPDK_ERRLOG("Alloc log page failed, ignore the log\n"); 3972 return; 3973 } 3974 log->ctrlr = ctrlr; 3975 log->log.type = type; 3976 log->log.nsid = ns->nsid; 3977 3978 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_add_reservation_log, log); 3979 } 3980 3981 /* Check from subsystem poll group's namespace information data structure */ 3982 static bool 3983 nvmf_ns_info_ctrlr_is_registrant(struct spdk_nvmf_subsystem_pg_ns_info *ns_info, 3984 struct spdk_nvmf_ctrlr *ctrlr) 3985 { 3986 uint32_t i; 3987 3988 for (i = 0; i < SPDK_NVMF_MAX_NUM_REGISTRANTS; i++) { 3989 if (!spdk_uuid_compare(&ns_info->reg_hostid[i], &ctrlr->hostid)) { 3990 return true; 3991 } 3992 } 3993 3994 return false; 3995 } 3996 3997 /* 3998 * Check the NVMe command is permitted or not for current controller(Host). 3999 */ 4000 static int 4001 nvmf_ns_reservation_request_check(struct spdk_nvmf_subsystem_pg_ns_info *ns_info, 4002 struct spdk_nvmf_ctrlr *ctrlr, 4003 struct spdk_nvmf_request *req) 4004 { 4005 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 4006 enum spdk_nvme_reservation_type rtype = ns_info->rtype; 4007 uint8_t status = SPDK_NVME_SC_SUCCESS; 4008 uint8_t racqa; 4009 bool is_registrant; 4010 4011 /* No valid reservation */ 4012 if (!rtype) { 4013 return 0; 4014 } 4015 4016 is_registrant = nvmf_ns_info_ctrlr_is_registrant(ns_info, ctrlr); 4017 /* All registrants type and current ctrlr is a valid registrant */ 4018 if ((rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE_ALL_REGS || 4019 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_ALL_REGS) && is_registrant) { 4020 return 0; 4021 } else if (!spdk_uuid_compare(&ns_info->holder_id, &ctrlr->hostid)) { 4022 return 0; 4023 } 4024 4025 /* Non-holder for current controller */ 4026 switch (cmd->opc) { 4027 case SPDK_NVME_OPC_READ: 4028 case SPDK_NVME_OPC_COMPARE: 4029 if (rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) { 4030 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4031 goto exit; 4032 } 4033 if ((rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_REG_ONLY || 4034 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_ALL_REGS) && !is_registrant) { 4035 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4036 } 4037 break; 4038 case SPDK_NVME_OPC_FLUSH: 4039 case SPDK_NVME_OPC_WRITE: 4040 case SPDK_NVME_OPC_WRITE_UNCORRECTABLE: 4041 case SPDK_NVME_OPC_WRITE_ZEROES: 4042 case SPDK_NVME_OPC_DATASET_MANAGEMENT: 4043 if (rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE || 4044 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) { 4045 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4046 goto exit; 4047 } 4048 if (!is_registrant) { 4049 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4050 } 4051 break; 4052 case SPDK_NVME_OPC_RESERVATION_ACQUIRE: 4053 racqa = cmd->cdw10_bits.resv_acquire.racqa; 4054 if (racqa == SPDK_NVME_RESERVE_ACQUIRE) { 4055 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4056 goto exit; 4057 } 4058 if (!is_registrant) { 4059 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4060 } 4061 break; 4062 case SPDK_NVME_OPC_RESERVATION_RELEASE: 4063 if (!is_registrant) { 4064 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4065 } 4066 break; 4067 default: 4068 break; 4069 } 4070 4071 exit: 4072 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4073 req->rsp->nvme_cpl.status.sc = status; 4074 if (status == SPDK_NVME_SC_RESERVATION_CONFLICT) { 4075 return -EPERM; 4076 } 4077 4078 return 0; 4079 } 4080 4081 static int 4082 nvmf_ctrlr_process_io_fused_cmd(struct spdk_nvmf_request *req, struct spdk_bdev *bdev, 4083 struct spdk_bdev_desc *desc, struct spdk_io_channel *ch) 4084 { 4085 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 4086 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 4087 struct spdk_nvmf_request *first_fused_req = req->qpair->first_fused_req; 4088 int rc; 4089 4090 if (cmd->fuse == SPDK_NVME_CMD_FUSE_FIRST) { 4091 /* first fused operation (should be compare) */ 4092 if (first_fused_req != NULL) { 4093 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 4094 4095 SPDK_ERRLOG("Wrong sequence of fused operations\n"); 4096 4097 /* abort req->qpair->first_fused_request and continue with new fused command */ 4098 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4099 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 4100 _nvmf_request_complete(first_fused_req); 4101 } else if (cmd->opc != SPDK_NVME_OPC_COMPARE) { 4102 SPDK_ERRLOG("Wrong op code of fused operations\n"); 4103 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4104 rsp->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 4105 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4106 } 4107 4108 req->qpair->first_fused_req = req; 4109 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 4110 } else if (cmd->fuse == SPDK_NVME_CMD_FUSE_SECOND) { 4111 /* second fused operation (should be write) */ 4112 if (first_fused_req == NULL) { 4113 SPDK_ERRLOG("Wrong sequence of fused operations\n"); 4114 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4115 rsp->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4116 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4117 } else if (cmd->opc != SPDK_NVME_OPC_WRITE) { 4118 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 4119 4120 SPDK_ERRLOG("Wrong op code of fused operations\n"); 4121 4122 /* abort req->qpair->first_fused_request and fail current command */ 4123 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4124 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 4125 _nvmf_request_complete(first_fused_req); 4126 4127 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4128 rsp->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 4129 req->qpair->first_fused_req = NULL; 4130 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4131 } 4132 4133 /* save request of first command to generate response later */ 4134 req->first_fused_req = first_fused_req; 4135 req->qpair->first_fused_req = NULL; 4136 } else { 4137 SPDK_ERRLOG("Invalid fused command fuse field.\n"); 4138 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4139 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 4140 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4141 } 4142 4143 rc = nvmf_bdev_ctrlr_compare_and_write_cmd(bdev, desc, ch, req->first_fused_req, req); 4144 4145 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 4146 if (spdk_nvme_cpl_is_error(rsp)) { 4147 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 4148 4149 fused_response->status = rsp->status; 4150 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4151 rsp->status.sc = SPDK_NVME_SC_ABORTED_FAILED_FUSED; 4152 /* Complete first of fused commands. Second will be completed by upper layer */ 4153 _nvmf_request_complete(first_fused_req); 4154 req->first_fused_req = NULL; 4155 } 4156 } 4157 4158 return rc; 4159 } 4160 4161 bool 4162 nvmf_ctrlr_use_zcopy(struct spdk_nvmf_request *req) 4163 { 4164 struct spdk_nvmf_transport *transport = req->qpair->transport; 4165 struct spdk_nvmf_ns *ns; 4166 4167 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_NONE); 4168 4169 if (!transport->opts.zcopy) { 4170 return false; 4171 } 4172 4173 if (nvmf_qpair_is_admin_queue(req->qpair)) { 4174 /* Admin queue */ 4175 return false; 4176 } 4177 4178 if ((req->cmd->nvme_cmd.opc != SPDK_NVME_OPC_WRITE) && 4179 (req->cmd->nvme_cmd.opc != SPDK_NVME_OPC_READ)) { 4180 /* Not a READ or WRITE command */ 4181 return false; 4182 } 4183 4184 if (req->cmd->nvme_cmd.fuse != SPDK_NVME_CMD_FUSE_NONE) { 4185 /* Fused commands dont use zcopy buffers */ 4186 return false; 4187 } 4188 4189 ns = _nvmf_subsystem_get_ns(req->qpair->ctrlr->subsys, req->cmd->nvme_cmd.nsid); 4190 if (ns == NULL || ns->bdev == NULL || !ns->zcopy) { 4191 return false; 4192 } 4193 4194 req->zcopy_phase = NVMF_ZCOPY_PHASE_INIT; 4195 return true; 4196 } 4197 4198 void 4199 spdk_nvmf_request_zcopy_start(struct spdk_nvmf_request *req) 4200 { 4201 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT); 4202 4203 /* Set iovcnt to be the maximum number of iovs that the ZCOPY can use */ 4204 req->iovcnt = NVMF_REQ_MAX_BUFFERS; 4205 4206 spdk_nvmf_request_exec(req); 4207 } 4208 4209 void 4210 spdk_nvmf_request_zcopy_end(struct spdk_nvmf_request *req, bool commit) 4211 { 4212 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_EXECUTE); 4213 req->zcopy_phase = NVMF_ZCOPY_PHASE_END_PENDING; 4214 4215 nvmf_bdev_ctrlr_zcopy_end(req, commit); 4216 } 4217 4218 int 4219 nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) 4220 { 4221 uint32_t nsid; 4222 struct spdk_nvmf_ns *ns; 4223 struct spdk_bdev *bdev; 4224 struct spdk_bdev_desc *desc; 4225 struct spdk_io_channel *ch; 4226 struct spdk_nvmf_poll_group *group = req->qpair->group; 4227 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 4228 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 4229 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 4230 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4231 enum spdk_nvme_ana_state ana_state; 4232 4233 /* pre-set response details for this command */ 4234 response->status.sc = SPDK_NVME_SC_SUCCESS; 4235 nsid = cmd->nsid; 4236 4237 if (spdk_unlikely(ctrlr == NULL)) { 4238 SPDK_ERRLOG("I/O command sent before CONNECT\n"); 4239 response->status.sct = SPDK_NVME_SCT_GENERIC; 4240 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 4241 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4242 } 4243 4244 if (spdk_unlikely(ctrlr->vcprop.cc.bits.en != 1)) { 4245 SPDK_ERRLOG("I/O command sent to disabled controller\n"); 4246 response->status.sct = SPDK_NVME_SCT_GENERIC; 4247 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 4248 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4249 } 4250 4251 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, nsid); 4252 if (ns == NULL || ns->bdev == NULL) { 4253 SPDK_DEBUGLOG(nvmf, "Unsuccessful query for nsid %u\n", cmd->nsid); 4254 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4255 response->status.dnr = 1; 4256 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4257 } 4258 4259 ana_state = nvmf_ctrlr_get_ana_state(ctrlr, ns->anagrpid); 4260 if (spdk_unlikely(ana_state != SPDK_NVME_ANA_OPTIMIZED_STATE && 4261 ana_state != SPDK_NVME_ANA_NON_OPTIMIZED_STATE)) { 4262 SPDK_DEBUGLOG(nvmf, "Fail I/O command due to ANA state %d\n", 4263 ana_state); 4264 response->status.sct = SPDK_NVME_SCT_PATH; 4265 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 4266 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4267 } 4268 4269 if (spdk_likely(ctrlr->listener != NULL)) { 4270 SPDK_DTRACE_PROBE3_TICKS(nvmf_request_io_exec_path, req, 4271 ctrlr->listener->trid->traddr, 4272 ctrlr->listener->trid->trsvcid); 4273 } 4274 4275 /* scan-build falsely reporting dereference of null pointer */ 4276 assert(group != NULL && group->sgroups != NULL); 4277 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[nsid - 1]; 4278 if (nvmf_ns_reservation_request_check(ns_info, ctrlr, req)) { 4279 SPDK_DEBUGLOG(nvmf, "Reservation Conflict for nsid %u, opcode %u\n", 4280 cmd->nsid, cmd->opc); 4281 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4282 } 4283 4284 bdev = ns->bdev; 4285 desc = ns->desc; 4286 ch = ns_info->channel; 4287 4288 if (spdk_unlikely(cmd->fuse & SPDK_NVME_CMD_FUSE_MASK)) { 4289 return nvmf_ctrlr_process_io_fused_cmd(req, bdev, desc, ch); 4290 } else if (spdk_unlikely(req->qpair->first_fused_req != NULL)) { 4291 struct spdk_nvme_cpl *fused_response = &req->qpair->first_fused_req->rsp->nvme_cpl; 4292 4293 SPDK_ERRLOG("Expected second of fused commands - failing first of fused commands\n"); 4294 4295 /* abort req->qpair->first_fused_request and continue with new command */ 4296 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4297 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 4298 _nvmf_request_complete(req->qpair->first_fused_req); 4299 req->qpair->first_fused_req = NULL; 4300 } 4301 4302 if (spdk_nvmf_request_using_zcopy(req)) { 4303 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT); 4304 return nvmf_bdev_ctrlr_zcopy_start(bdev, desc, ch, req); 4305 } else { 4306 switch (cmd->opc) { 4307 case SPDK_NVME_OPC_READ: 4308 return nvmf_bdev_ctrlr_read_cmd(bdev, desc, ch, req); 4309 case SPDK_NVME_OPC_WRITE: 4310 return nvmf_bdev_ctrlr_write_cmd(bdev, desc, ch, req); 4311 case SPDK_NVME_OPC_COMPARE: 4312 return nvmf_bdev_ctrlr_compare_cmd(bdev, desc, ch, req); 4313 case SPDK_NVME_OPC_WRITE_ZEROES: 4314 return nvmf_bdev_ctrlr_write_zeroes_cmd(bdev, desc, ch, req); 4315 case SPDK_NVME_OPC_FLUSH: 4316 return nvmf_bdev_ctrlr_flush_cmd(bdev, desc, ch, req); 4317 case SPDK_NVME_OPC_DATASET_MANAGEMENT: 4318 return nvmf_bdev_ctrlr_dsm_cmd(bdev, desc, ch, req); 4319 case SPDK_NVME_OPC_RESERVATION_REGISTER: 4320 case SPDK_NVME_OPC_RESERVATION_ACQUIRE: 4321 case SPDK_NVME_OPC_RESERVATION_RELEASE: 4322 case SPDK_NVME_OPC_RESERVATION_REPORT: 4323 spdk_thread_send_msg(ctrlr->subsys->thread, nvmf_ns_reservation_request, req); 4324 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 4325 case SPDK_NVME_OPC_COPY: 4326 return nvmf_bdev_ctrlr_copy_cmd(bdev, desc, ch, req); 4327 default: 4328 return nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req); 4329 } 4330 } 4331 } 4332 4333 static void 4334 nvmf_qpair_request_cleanup(struct spdk_nvmf_qpair *qpair) 4335 { 4336 if (qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING) { 4337 assert(qpair->state_cb != NULL); 4338 4339 if (TAILQ_EMPTY(&qpair->outstanding)) { 4340 qpair->state_cb(qpair->state_cb_arg, 0); 4341 } 4342 } 4343 } 4344 4345 int 4346 spdk_nvmf_request_free(struct spdk_nvmf_request *req) 4347 { 4348 struct spdk_nvmf_qpair *qpair = req->qpair; 4349 4350 TAILQ_REMOVE(&qpair->outstanding, req, link); 4351 if (nvmf_transport_req_free(req)) { 4352 SPDK_ERRLOG("Unable to free transport level request resources.\n"); 4353 } 4354 4355 nvmf_qpair_request_cleanup(qpair); 4356 4357 return 0; 4358 } 4359 4360 static void 4361 _nvmf_request_complete(void *ctx) 4362 { 4363 struct spdk_nvmf_request *req = ctx; 4364 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 4365 struct spdk_nvmf_qpair *qpair; 4366 struct spdk_nvmf_subsystem_poll_group *sgroup = NULL; 4367 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4368 bool is_aer = false; 4369 uint32_t nsid; 4370 bool paused; 4371 uint8_t opcode; 4372 4373 rsp->sqid = 0; 4374 rsp->status.p = 0; 4375 rsp->cid = req->cmd->nvme_cmd.cid; 4376 nsid = req->cmd->nvme_cmd.nsid; 4377 opcode = req->cmd->nvmf_cmd.opcode; 4378 4379 qpair = req->qpair; 4380 if (qpair->ctrlr) { 4381 sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id]; 4382 assert(sgroup != NULL); 4383 is_aer = req->cmd->nvme_cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST; 4384 if (spdk_likely(qpair->qid != 0)) { 4385 qpair->group->stat.completed_nvme_io++; 4386 } 4387 4388 /* 4389 * Set the crd value. 4390 * If the the IO has any error, and dnr (DoNotRetry) is not 1, 4391 * and ACRE is enabled, we will set the crd to 1 to select the first CRDT. 4392 */ 4393 if (spdk_nvme_cpl_is_error(rsp) && 4394 rsp->status.dnr == 0 && 4395 qpair->ctrlr->acre_enabled) { 4396 rsp->status.crd = 1; 4397 } 4398 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) { 4399 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 4400 } 4401 4402 if (SPDK_DEBUGLOG_FLAG_ENABLED("nvmf")) { 4403 spdk_nvme_print_completion(qpair->qid, rsp); 4404 } 4405 4406 switch (req->zcopy_phase) { 4407 case NVMF_ZCOPY_PHASE_NONE: 4408 TAILQ_REMOVE(&qpair->outstanding, req, link); 4409 break; 4410 case NVMF_ZCOPY_PHASE_INIT: 4411 if (spdk_unlikely(spdk_nvme_cpl_is_error(rsp))) { 4412 req->zcopy_phase = NVMF_ZCOPY_PHASE_INIT_FAILED; 4413 TAILQ_REMOVE(&qpair->outstanding, req, link); 4414 } else { 4415 req->zcopy_phase = NVMF_ZCOPY_PHASE_EXECUTE; 4416 } 4417 break; 4418 case NVMF_ZCOPY_PHASE_EXECUTE: 4419 break; 4420 case NVMF_ZCOPY_PHASE_END_PENDING: 4421 TAILQ_REMOVE(&qpair->outstanding, req, link); 4422 req->zcopy_phase = NVMF_ZCOPY_PHASE_COMPLETE; 4423 break; 4424 default: 4425 SPDK_ERRLOG("Invalid ZCOPY phase %u\n", req->zcopy_phase); 4426 break; 4427 } 4428 4429 if (nvmf_transport_req_complete(req)) { 4430 SPDK_ERRLOG("Transport request completion error!\n"); 4431 } 4432 4433 /* AER cmd is an exception */ 4434 if (sgroup && !is_aer) { 4435 if (spdk_unlikely(opcode == SPDK_NVME_OPC_FABRIC || 4436 nvmf_qpair_is_admin_queue(qpair))) { 4437 assert(sgroup->mgmt_io_outstanding > 0); 4438 sgroup->mgmt_io_outstanding--; 4439 } else { 4440 if (req->zcopy_phase == NVMF_ZCOPY_PHASE_NONE || 4441 req->zcopy_phase == NVMF_ZCOPY_PHASE_COMPLETE || 4442 req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT_FAILED) { 4443 /* End of request */ 4444 4445 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 4446 if (spdk_likely(nsid - 1 < sgroup->num_ns)) { 4447 sgroup->ns_info[nsid - 1].io_outstanding--; 4448 } 4449 } 4450 } 4451 4452 if (spdk_unlikely(sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSING && 4453 sgroup->mgmt_io_outstanding == 0)) { 4454 paused = true; 4455 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 4456 ns_info = &sgroup->ns_info[nsid]; 4457 4458 if (ns_info->state == SPDK_NVMF_SUBSYSTEM_PAUSING && 4459 ns_info->io_outstanding > 0) { 4460 paused = false; 4461 break; 4462 } 4463 } 4464 4465 if (paused) { 4466 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 4467 sgroup->cb_fn(sgroup->cb_arg, 0); 4468 sgroup->cb_fn = NULL; 4469 sgroup->cb_arg = NULL; 4470 } 4471 } 4472 4473 } 4474 4475 nvmf_qpair_request_cleanup(qpair); 4476 } 4477 4478 int 4479 spdk_nvmf_request_complete(struct spdk_nvmf_request *req) 4480 { 4481 struct spdk_nvmf_qpair *qpair = req->qpair; 4482 4483 spdk_thread_exec_msg(qpair->group->thread, _nvmf_request_complete, req); 4484 4485 return 0; 4486 } 4487 4488 void 4489 spdk_nvmf_request_exec_fabrics(struct spdk_nvmf_request *req) 4490 { 4491 struct spdk_nvmf_qpair *qpair = req->qpair; 4492 struct spdk_nvmf_subsystem_poll_group *sgroup = NULL; 4493 enum spdk_nvmf_request_exec_status status; 4494 4495 if (qpair->ctrlr) { 4496 sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id]; 4497 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) { 4498 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 4499 } 4500 4501 assert(sgroup != NULL); 4502 sgroup->mgmt_io_outstanding++; 4503 4504 /* Place the request on the outstanding list so we can keep track of it */ 4505 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4506 4507 assert(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC); 4508 status = nvmf_ctrlr_process_fabrics_cmd(req); 4509 4510 if (status == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 4511 _nvmf_request_complete(req); 4512 } 4513 } 4514 4515 static bool 4516 nvmf_check_subsystem_active(struct spdk_nvmf_request *req) 4517 { 4518 struct spdk_nvmf_qpair *qpair = req->qpair; 4519 struct spdk_nvmf_subsystem_poll_group *sgroup = NULL; 4520 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4521 uint32_t nsid; 4522 4523 if (qpair->ctrlr) { 4524 sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id]; 4525 assert(sgroup != NULL); 4526 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) { 4527 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 4528 } 4529 4530 /* Check if the subsystem is paused (if there is a subsystem) */ 4531 if (sgroup != NULL) { 4532 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC || 4533 nvmf_qpair_is_admin_queue(qpair))) { 4534 if (sgroup->state != SPDK_NVMF_SUBSYSTEM_ACTIVE) { 4535 /* The subsystem is not currently active. Queue this request. */ 4536 TAILQ_INSERT_TAIL(&sgroup->queued, req, link); 4537 return false; 4538 } 4539 sgroup->mgmt_io_outstanding++; 4540 } else { 4541 nsid = req->cmd->nvme_cmd.nsid; 4542 4543 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 4544 if (spdk_unlikely(nsid - 1 >= sgroup->num_ns)) { 4545 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4546 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4547 req->rsp->nvme_cpl.status.dnr = 1; 4548 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4549 _nvmf_request_complete(req); 4550 return false; 4551 } 4552 4553 ns_info = &sgroup->ns_info[nsid - 1]; 4554 if (ns_info->channel == NULL) { 4555 /* This can can happen if host sends I/O to a namespace that is 4556 * in the process of being added, but before the full addition 4557 * process is complete. Report invalid namespace in that case. 4558 */ 4559 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4560 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4561 req->rsp->nvme_cpl.status.dnr = 1; 4562 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4563 ns_info->io_outstanding++; 4564 _nvmf_request_complete(req); 4565 return false; 4566 } 4567 4568 if (ns_info->state != SPDK_NVMF_SUBSYSTEM_ACTIVE) { 4569 /* The namespace is not currently active. Queue this request. */ 4570 TAILQ_INSERT_TAIL(&sgroup->queued, req, link); 4571 return false; 4572 } 4573 4574 ns_info->io_outstanding++; 4575 } 4576 4577 if (qpair->state != SPDK_NVMF_QPAIR_ACTIVE) { 4578 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4579 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 4580 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4581 _nvmf_request_complete(req); 4582 return false; 4583 } 4584 } 4585 4586 return true; 4587 } 4588 4589 void 4590 spdk_nvmf_request_exec(struct spdk_nvmf_request *req) 4591 { 4592 struct spdk_nvmf_qpair *qpair = req->qpair; 4593 struct spdk_nvmf_transport *transport = qpair->transport; 4594 enum spdk_nvmf_request_exec_status status; 4595 4596 if (!nvmf_check_subsystem_active(req)) { 4597 return; 4598 } 4599 4600 if (SPDK_DEBUGLOG_FLAG_ENABLED("nvmf")) { 4601 spdk_nvme_print_command(qpair->qid, &req->cmd->nvme_cmd); 4602 } 4603 4604 /* Place the request on the outstanding list so we can keep track of it */ 4605 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4606 4607 if (spdk_unlikely((req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC) && 4608 spdk_nvme_trtype_is_fabrics(transport->ops->type))) { 4609 status = nvmf_ctrlr_process_fabrics_cmd(req); 4610 } else if (spdk_unlikely(nvmf_qpair_is_admin_queue(qpair))) { 4611 status = nvmf_ctrlr_process_admin_cmd(req); 4612 } else { 4613 status = nvmf_ctrlr_process_io_cmd(req); 4614 } 4615 4616 if (status == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 4617 _nvmf_request_complete(req); 4618 } 4619 } 4620 4621 static bool 4622 nvmf_ctrlr_get_dif_ctx(struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvme_cmd *cmd, 4623 struct spdk_dif_ctx *dif_ctx) 4624 { 4625 struct spdk_nvmf_ns *ns; 4626 struct spdk_bdev *bdev; 4627 4628 if (ctrlr == NULL || cmd == NULL) { 4629 return false; 4630 } 4631 4632 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, cmd->nsid); 4633 if (ns == NULL || ns->bdev == NULL) { 4634 return false; 4635 } 4636 4637 bdev = ns->bdev; 4638 4639 switch (cmd->opc) { 4640 case SPDK_NVME_OPC_READ: 4641 case SPDK_NVME_OPC_WRITE: 4642 case SPDK_NVME_OPC_COMPARE: 4643 return nvmf_bdev_ctrlr_get_dif_ctx(bdev, cmd, dif_ctx); 4644 default: 4645 break; 4646 } 4647 4648 return false; 4649 } 4650 4651 bool 4652 spdk_nvmf_request_get_dif_ctx(struct spdk_nvmf_request *req, struct spdk_dif_ctx *dif_ctx) 4653 { 4654 struct spdk_nvmf_qpair *qpair = req->qpair; 4655 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 4656 4657 if (spdk_likely(ctrlr == NULL || !ctrlr->dif_insert_or_strip)) { 4658 return false; 4659 } 4660 4661 if (spdk_unlikely(qpair->state != SPDK_NVMF_QPAIR_ACTIVE)) { 4662 return false; 4663 } 4664 4665 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC)) { 4666 return false; 4667 } 4668 4669 if (spdk_unlikely(nvmf_qpair_is_admin_queue(qpair))) { 4670 return false; 4671 } 4672 4673 return nvmf_ctrlr_get_dif_ctx(ctrlr, &req->cmd->nvme_cmd, dif_ctx); 4674 } 4675 4676 void 4677 spdk_nvmf_set_custom_admin_cmd_hdlr(uint8_t opc, spdk_nvmf_custom_cmd_hdlr hdlr) 4678 { 4679 g_nvmf_custom_admin_cmd_hdlrs[opc].hdlr = hdlr; 4680 } 4681 4682 static int 4683 nvmf_passthru_admin_cmd_for_bdev_nsid(struct spdk_nvmf_request *req, uint32_t bdev_nsid) 4684 { 4685 struct spdk_bdev *bdev; 4686 struct spdk_bdev_desc *desc; 4687 struct spdk_io_channel *ch; 4688 struct spdk_nvme_cpl *response = spdk_nvmf_request_get_response(req); 4689 int rc; 4690 4691 rc = spdk_nvmf_request_get_bdev(bdev_nsid, req, &bdev, &desc, &ch); 4692 if (rc) { 4693 response->status.sct = SPDK_NVME_SCT_GENERIC; 4694 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4695 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4696 } 4697 return spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(bdev, desc, ch, req, NULL); 4698 } 4699 4700 static int 4701 nvmf_passthru_admin_cmd(struct spdk_nvmf_request *req) 4702 { 4703 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req); 4704 uint32_t bdev_nsid; 4705 4706 if (g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].nsid != 0) { 4707 bdev_nsid = g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].nsid; 4708 } else { 4709 bdev_nsid = cmd->nsid; 4710 } 4711 4712 return nvmf_passthru_admin_cmd_for_bdev_nsid(req, bdev_nsid); 4713 } 4714 4715 int 4716 nvmf_passthru_admin_cmd_for_ctrlr(struct spdk_nvmf_request *req, struct spdk_nvmf_ctrlr *ctrlr) 4717 { 4718 struct spdk_nvme_cpl *response = spdk_nvmf_request_get_response(req); 4719 struct spdk_nvmf_ns *ns; 4720 4721 ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); 4722 if (ns == NULL) { 4723 /* Is there a better sc to use here? */ 4724 response->status.sct = SPDK_NVME_SCT_GENERIC; 4725 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4726 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4727 } 4728 4729 return nvmf_passthru_admin_cmd_for_bdev_nsid(req, ns->nsid); 4730 } 4731 4732 void 4733 spdk_nvmf_set_passthru_admin_cmd(uint8_t opc, uint32_t forward_nsid) 4734 { 4735 g_nvmf_custom_admin_cmd_hdlrs[opc].hdlr = nvmf_passthru_admin_cmd; 4736 g_nvmf_custom_admin_cmd_hdlrs[opc].nsid = forward_nsid; 4737 } 4738 4739 int 4740 spdk_nvmf_request_get_bdev(uint32_t nsid, struct spdk_nvmf_request *req, 4741 struct spdk_bdev **bdev, struct spdk_bdev_desc **desc, struct spdk_io_channel **ch) 4742 { 4743 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 4744 struct spdk_nvmf_ns *ns; 4745 struct spdk_nvmf_poll_group *group = req->qpair->group; 4746 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4747 4748 *bdev = NULL; 4749 *desc = NULL; 4750 *ch = NULL; 4751 4752 ns = _nvmf_subsystem_get_ns(ctrlr->subsys, nsid); 4753 if (ns == NULL || ns->bdev == NULL) { 4754 return -EINVAL; 4755 } 4756 4757 assert(group != NULL && group->sgroups != NULL); 4758 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[nsid - 1]; 4759 *bdev = ns->bdev; 4760 *desc = ns->desc; 4761 *ch = ns_info->channel; 4762 4763 return 0; 4764 } 4765 4766 struct spdk_nvmf_ctrlr *spdk_nvmf_request_get_ctrlr(struct spdk_nvmf_request *req) 4767 { 4768 return req->qpair->ctrlr; 4769 } 4770 4771 struct spdk_nvme_cmd *spdk_nvmf_request_get_cmd(struct spdk_nvmf_request *req) 4772 { 4773 return &req->cmd->nvme_cmd; 4774 } 4775 4776 struct spdk_nvme_cpl *spdk_nvmf_request_get_response(struct spdk_nvmf_request *req) 4777 { 4778 return &req->rsp->nvme_cpl; 4779 } 4780 4781 struct spdk_nvmf_subsystem *spdk_nvmf_request_get_subsystem(struct spdk_nvmf_request *req) 4782 { 4783 return req->qpair->ctrlr->subsys; 4784 } 4785 4786 size_t 4787 spdk_nvmf_request_copy_from_buf(struct spdk_nvmf_request *req, 4788 void *buf, size_t buflen) 4789 { 4790 struct spdk_iov_xfer ix; 4791 4792 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 4793 return spdk_iov_xfer_from_buf(&ix, buf, buflen); 4794 } 4795 4796 size_t 4797 spdk_nvmf_request_copy_to_buf(struct spdk_nvmf_request *req, 4798 void *buf, size_t buflen) 4799 { 4800 struct spdk_iov_xfer ix; 4801 4802 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 4803 return spdk_iov_xfer_to_buf(&ix, buf, buflen); 4804 } 4805 4806 struct spdk_nvmf_subsystem *spdk_nvmf_ctrlr_get_subsystem(struct spdk_nvmf_ctrlr *ctrlr) 4807 { 4808 return ctrlr->subsys; 4809 } 4810 4811 uint16_t 4812 spdk_nvmf_ctrlr_get_id(struct spdk_nvmf_ctrlr *ctrlr) 4813 { 4814 return ctrlr->cntlid; 4815 } 4816 4817 struct spdk_nvmf_request *spdk_nvmf_request_get_req_to_abort(struct spdk_nvmf_request *req) 4818 { 4819 return req->req_to_abort; 4820 } 4821