1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019, 2020 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 * Copyright (c) 2024 Samsung Electronics Co., Ltd. All rights reserved. 6 */ 7 8 #include "spdk/stdinc.h" 9 10 #include "nvmf_internal.h" 11 #include "transport.h" 12 13 #include "spdk/bdev.h" 14 #include "spdk/bdev_zone.h" 15 #include "spdk/bit_array.h" 16 #include "spdk/endian.h" 17 #include "spdk/thread.h" 18 #include "spdk/nvme_spec.h" 19 #include "spdk/nvmf_cmd.h" 20 #include "spdk/string.h" 21 #include "spdk/util.h" 22 #include "spdk/version.h" 23 #include "spdk/log.h" 24 #include "spdk_internal/usdt.h" 25 26 #define MIN_KEEP_ALIVE_TIMEOUT_IN_MS 10000 27 #define NVMF_DISC_KATO_IN_MS 120000 28 #define KAS_TIME_UNIT_IN_MS 100 29 #define KAS_DEFAULT_VALUE (MIN_KEEP_ALIVE_TIMEOUT_IN_MS / KAS_TIME_UNIT_IN_MS) 30 31 #define NVMF_CC_RESET_SHN_TIMEOUT_IN_MS 10000 32 33 #define NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS (NVMF_CC_RESET_SHN_TIMEOUT_IN_MS + 5000) 34 35 #define DUPLICATE_QID_RETRY_US 1000 36 37 /* 38 * Report the SPDK version as the firmware revision. 39 * SPDK_VERSION_STRING won't fit into FR (only 8 bytes), so try to fit the most important parts. 40 */ 41 #define FW_VERSION SPDK_VERSION_MAJOR_STRING SPDK_VERSION_MINOR_STRING SPDK_VERSION_PATCH_STRING 42 43 #define ANA_TRANSITION_TIME_IN_SEC 10 44 45 #define NVMF_ABORT_COMMAND_LIMIT 3 46 47 /* 48 * Support for custom admin command handlers 49 */ 50 struct spdk_nvmf_custom_admin_cmd { 51 spdk_nvmf_custom_cmd_hdlr hdlr; 52 uint32_t nsid; /* nsid to forward */ 53 }; 54 55 static struct spdk_nvmf_custom_admin_cmd g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_MAX_OPC + 1]; 56 57 static void _nvmf_request_complete(void *ctx); 58 int nvmf_passthru_admin_cmd_for_ctrlr(struct spdk_nvmf_request *req, struct spdk_nvmf_ctrlr *ctrlr); 59 static int nvmf_passthru_admin_cmd(struct spdk_nvmf_request *req); 60 61 static inline void 62 nvmf_invalid_connect_response(struct spdk_nvmf_fabric_connect_rsp *rsp, 63 uint8_t iattr, uint16_t ipo) 64 { 65 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 66 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 67 rsp->status_code_specific.invalid.iattr = iattr; 68 rsp->status_code_specific.invalid.ipo = ipo; 69 } 70 71 #define SPDK_NVMF_INVALID_CONNECT_CMD(rsp, field) \ 72 nvmf_invalid_connect_response(rsp, 0, offsetof(struct spdk_nvmf_fabric_connect_cmd, field)) 73 #define SPDK_NVMF_INVALID_CONNECT_DATA(rsp, field) \ 74 nvmf_invalid_connect_response(rsp, 1, offsetof(struct spdk_nvmf_fabric_connect_data, field)) 75 76 77 static void 78 nvmf_ctrlr_stop_keep_alive_timer(struct spdk_nvmf_ctrlr *ctrlr) 79 { 80 if (!ctrlr) { 81 SPDK_ERRLOG("Controller is NULL\n"); 82 return; 83 } 84 85 if (ctrlr->keep_alive_poller == NULL) { 86 return; 87 } 88 89 SPDK_DEBUGLOG(nvmf, "Stop keep alive poller\n"); 90 spdk_poller_unregister(&ctrlr->keep_alive_poller); 91 } 92 93 static void 94 nvmf_ctrlr_stop_association_timer(struct spdk_nvmf_ctrlr *ctrlr) 95 { 96 if (!ctrlr) { 97 SPDK_ERRLOG("Controller is NULL\n"); 98 assert(false); 99 return; 100 } 101 102 if (ctrlr->association_timer == NULL) { 103 return; 104 } 105 106 SPDK_DEBUGLOG(nvmf, "Stop association timer\n"); 107 spdk_poller_unregister(&ctrlr->association_timer); 108 } 109 110 static void 111 nvmf_ctrlr_disconnect_qpairs_done(struct spdk_io_channel_iter *i, int status) 112 { 113 if (status == 0) { 114 SPDK_DEBUGLOG(nvmf, "ctrlr disconnect qpairs complete successfully\n"); 115 } else { 116 SPDK_ERRLOG("Fail to disconnect ctrlr qpairs\n"); 117 } 118 } 119 120 static int 121 _nvmf_ctrlr_disconnect_qpairs_on_pg(struct spdk_io_channel_iter *i, bool include_admin) 122 { 123 int rc = 0; 124 struct spdk_nvmf_ctrlr *ctrlr; 125 struct spdk_nvmf_qpair *qpair, *temp_qpair; 126 struct spdk_io_channel *ch; 127 struct spdk_nvmf_poll_group *group; 128 129 ctrlr = spdk_io_channel_iter_get_ctx(i); 130 ch = spdk_io_channel_iter_get_channel(i); 131 group = spdk_io_channel_get_ctx(ch); 132 133 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, temp_qpair) { 134 if (qpair->ctrlr == ctrlr && (include_admin || !nvmf_qpair_is_admin_queue(qpair))) { 135 rc = spdk_nvmf_qpair_disconnect(qpair); 136 if (rc) { 137 if (rc == -EINPROGRESS) { 138 rc = 0; 139 } else { 140 SPDK_ERRLOG("Qpair disconnect failed\n"); 141 return rc; 142 } 143 } 144 } 145 } 146 147 return rc; 148 } 149 150 static void 151 nvmf_ctrlr_disconnect_qpairs_on_pg(struct spdk_io_channel_iter *i) 152 { 153 spdk_for_each_channel_continue(i, _nvmf_ctrlr_disconnect_qpairs_on_pg(i, true)); 154 } 155 156 static void 157 nvmf_ctrlr_disconnect_io_qpairs_on_pg(struct spdk_io_channel_iter *i) 158 { 159 spdk_for_each_channel_continue(i, _nvmf_ctrlr_disconnect_qpairs_on_pg(i, false)); 160 } 161 162 static int 163 nvmf_ctrlr_keep_alive_poll(void *ctx) 164 { 165 uint64_t keep_alive_timeout_tick; 166 uint64_t now = spdk_get_ticks(); 167 struct spdk_nvmf_ctrlr *ctrlr = ctx; 168 169 if (ctrlr->in_destruct) { 170 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 171 return SPDK_POLLER_IDLE; 172 } 173 174 SPDK_DEBUGLOG(nvmf, "Polling ctrlr keep alive timeout\n"); 175 176 /* If the Keep alive feature is in use and the timer expires */ 177 keep_alive_timeout_tick = ctrlr->last_keep_alive_tick + 178 ctrlr->feat.keep_alive_timer.bits.kato * spdk_get_ticks_hz() / UINT64_C(1000); 179 if (now > keep_alive_timeout_tick) { 180 SPDK_NOTICELOG("Disconnecting host %s from subsystem %s due to keep alive timeout.\n", 181 ctrlr->hostnqn, ctrlr->subsys->subnqn); 182 /* set the Controller Fatal Status bit to '1' */ 183 if (ctrlr->vcprop.csts.bits.cfs == 0) { 184 nvmf_ctrlr_set_fatal_status(ctrlr); 185 186 /* 187 * disconnect qpairs, terminate Transport connection 188 * destroy ctrlr, break the host to controller association 189 * disconnect qpairs with qpair->ctrlr == ctrlr 190 */ 191 spdk_for_each_channel(ctrlr->subsys->tgt, 192 nvmf_ctrlr_disconnect_qpairs_on_pg, 193 ctrlr, 194 nvmf_ctrlr_disconnect_qpairs_done); 195 return SPDK_POLLER_BUSY; 196 } 197 } 198 199 return SPDK_POLLER_IDLE; 200 } 201 202 static void 203 nvmf_ctrlr_start_keep_alive_timer(struct spdk_nvmf_ctrlr *ctrlr) 204 { 205 if (!ctrlr) { 206 SPDK_ERRLOG("Controller is NULL\n"); 207 return; 208 } 209 210 /* if cleared to 0 then the Keep Alive Timer is disabled */ 211 if (ctrlr->feat.keep_alive_timer.bits.kato != 0) { 212 213 ctrlr->last_keep_alive_tick = spdk_get_ticks(); 214 215 SPDK_DEBUGLOG(nvmf, "Ctrlr add keep alive poller\n"); 216 ctrlr->keep_alive_poller = SPDK_POLLER_REGISTER(nvmf_ctrlr_keep_alive_poll, ctrlr, 217 ctrlr->feat.keep_alive_timer.bits.kato * 1000); 218 } 219 } 220 221 static void 222 nvmf_qpair_set_ctrlr(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_ctrlr *ctrlr) 223 { 224 if (qpair->ctrlr != NULL) { 225 /* Admin queues will call this function twice. */ 226 assert(qpair->ctrlr == ctrlr); 227 return; 228 } 229 230 qpair->ctrlr = ctrlr; 231 spdk_trace_owner_append_description(qpair->trace_id, 232 spdk_nvmf_subsystem_get_nqn(ctrlr->subsys)); 233 } 234 235 static int _retry_qid_check(void *ctx); 236 237 static void 238 nvmf_ctrlr_send_connect_rsp(void *ctx) 239 { 240 struct spdk_nvmf_request *req = ctx; 241 struct spdk_nvmf_qpair *qpair = req->qpair; 242 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 243 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 244 int rc; 245 246 /* The qpair might have been disconnected in the meantime */ 247 assert(qpair->state == SPDK_NVMF_QPAIR_CONNECTING || 248 qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 249 if (qpair->state == SPDK_NVMF_QPAIR_CONNECTING) { 250 if (nvmf_subsystem_host_auth_required(ctrlr->subsys, ctrlr->hostnqn)) { 251 rc = nvmf_qpair_auth_init(qpair); 252 if (rc != 0) { 253 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 254 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 255 spdk_nvmf_request_complete(req); 256 spdk_nvmf_qpair_disconnect(qpair); 257 return; 258 } 259 rsp->status_code_specific.success.authreq.atr = 1; 260 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_AUTHENTICATING); 261 } else { 262 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ENABLED); 263 } 264 } 265 266 SPDK_DEBUGLOG(nvmf, "connect capsule response: cntlid = 0x%04x\n", ctrlr->cntlid); 267 268 assert(spdk_get_thread() == qpair->group->thread); 269 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 270 rsp->status_code_specific.success.cntlid = ctrlr->cntlid; 271 spdk_nvmf_request_complete(req); 272 } 273 274 static void 275 nvmf_ctrlr_add_qpair(struct spdk_nvmf_qpair *qpair, 276 struct spdk_nvmf_ctrlr *ctrlr, 277 struct spdk_nvmf_request *req) 278 { 279 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 280 281 if (!ctrlr->admin_qpair) { 282 SPDK_ERRLOG("Inactive admin qpair\n"); 283 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 284 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 285 qpair->connect_req = NULL; 286 qpair->ctrlr = NULL; 287 spdk_nvmf_request_complete(req); 288 return; 289 } 290 291 assert(ctrlr->admin_qpair->group->thread == spdk_get_thread()); 292 293 if (spdk_bit_array_get(ctrlr->qpair_mask, qpair->qid)) { 294 if (qpair->connect_req != NULL) { 295 SPDK_ERRLOG("Got I/O connect with duplicate QID %u (cntlid:%u)\n", 296 qpair->qid, ctrlr->cntlid); 297 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 298 rsp->status.sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER; 299 qpair->connect_req = NULL; 300 qpair->ctrlr = NULL; 301 spdk_nvmf_request_complete(req); 302 } else { 303 SPDK_WARNLOG("Duplicate QID detected (cntlid:%u, qid:%u), re-check in %dus\n", 304 ctrlr->cntlid, qpair->qid, DUPLICATE_QID_RETRY_US); 305 qpair->connect_req = req; 306 /* Set qpair->ctrlr here so that we'll have it when the poller expires. */ 307 nvmf_qpair_set_ctrlr(qpair, ctrlr); 308 req->poller = SPDK_POLLER_REGISTER(_retry_qid_check, qpair, 309 DUPLICATE_QID_RETRY_US); 310 } 311 return; 312 } 313 314 SPDK_DTRACE_PROBE4_TICKS(nvmf_ctrlr_add_qpair, qpair, qpair->qid, ctrlr->subsys->subnqn, 315 ctrlr->hostnqn); 316 nvmf_qpair_set_ctrlr(qpair, ctrlr); 317 spdk_bit_array_set(ctrlr->qpair_mask, qpair->qid); 318 SPDK_DEBUGLOG(nvmf, "qpair_mask set, qid %u\n", qpair->qid); 319 320 spdk_thread_send_msg(qpair->group->thread, nvmf_ctrlr_send_connect_rsp, req); 321 } 322 323 static int 324 _retry_qid_check(void *ctx) 325 { 326 struct spdk_nvmf_qpair *qpair = ctx; 327 struct spdk_nvmf_request *req = qpair->connect_req; 328 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 329 330 spdk_poller_unregister(&req->poller); 331 SPDK_WARNLOG("Retrying adding qpair, qid:%d\n", qpair->qid); 332 nvmf_ctrlr_add_qpair(qpair, ctrlr, req); 333 return SPDK_POLLER_BUSY; 334 } 335 336 static void 337 _nvmf_ctrlr_add_admin_qpair(void *ctx) 338 { 339 struct spdk_nvmf_request *req = ctx; 340 struct spdk_nvmf_qpair *qpair = req->qpair; 341 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 342 343 ctrlr->admin_qpair = qpair; 344 ctrlr->association_timeout = qpair->transport->opts.association_timeout; 345 nvmf_ctrlr_start_keep_alive_timer(ctrlr); 346 nvmf_ctrlr_add_qpair(qpair, ctrlr, req); 347 } 348 349 static void 350 _nvmf_subsystem_add_ctrlr(void *ctx) 351 { 352 struct spdk_nvmf_request *req = ctx; 353 struct spdk_nvmf_qpair *qpair = req->qpair; 354 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 355 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 356 357 if (nvmf_subsystem_add_ctrlr(ctrlr->subsys, ctrlr)) { 358 SPDK_ERRLOG("Unable to add controller to subsystem\n"); 359 spdk_bit_array_free(&ctrlr->qpair_mask); 360 free(ctrlr); 361 qpair->ctrlr = NULL; 362 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 363 spdk_nvmf_request_complete(req); 364 return; 365 } 366 367 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_add_admin_qpair, req); 368 } 369 370 static void 371 nvmf_ctrlr_cdata_init(struct spdk_nvmf_transport *transport, struct spdk_nvmf_subsystem *subsystem, 372 struct spdk_nvmf_ctrlr_data *cdata) 373 { 374 cdata->aerl = SPDK_NVMF_MAX_ASYNC_EVENTS - 1; 375 cdata->kas = KAS_DEFAULT_VALUE; 376 cdata->vid = SPDK_PCI_VID_INTEL; 377 cdata->ssvid = SPDK_PCI_VID_INTEL; 378 /* INTEL OUI */ 379 cdata->ieee[0] = 0xe4; 380 cdata->ieee[1] = 0xd2; 381 cdata->ieee[2] = 0x5c; 382 cdata->oncs.compare = 1; 383 cdata->oncs.dsm = 1; 384 cdata->oncs.write_zeroes = 1; 385 cdata->oncs.reservations = 1; 386 cdata->oncs.copy = 1; 387 cdata->fuses.compare_and_write = 1; 388 cdata->sgls.supported = 1; 389 cdata->sgls.keyed_sgl = 1; 390 cdata->sgls.sgl_offset = 1; 391 cdata->nvmf_specific.ioccsz = sizeof(struct spdk_nvme_cmd) / 16; 392 cdata->nvmf_specific.ioccsz += transport->opts.in_capsule_data_size / 16; 393 cdata->nvmf_specific.iorcsz = sizeof(struct spdk_nvme_cpl) / 16; 394 cdata->nvmf_specific.icdoff = 0; /* offset starts directly after SQE */ 395 cdata->nvmf_specific.ctrattr.ctrlr_model = SPDK_NVMF_CTRLR_MODEL_DYNAMIC; 396 cdata->nvmf_specific.msdbd = 1; 397 398 if (transport->ops->cdata_init) { 399 transport->ops->cdata_init(transport, subsystem, cdata); 400 } 401 } 402 403 static bool 404 nvmf_subsystem_has_zns_iocs(struct spdk_nvmf_subsystem *subsystem) 405 { 406 struct spdk_nvmf_ns *ns; 407 uint32_t i; 408 409 for (i = 0; i < subsystem->max_nsid; i++) { 410 ns = subsystem->ns[i]; 411 if (ns && ns->bdev && spdk_bdev_is_zoned(ns->bdev)) { 412 return true; 413 } 414 } 415 return false; 416 } 417 418 static void 419 nvmf_ctrlr_init_visible_ns(struct spdk_nvmf_ctrlr *ctrlr) 420 { 421 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 422 struct spdk_nvmf_ns *ns; 423 424 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 425 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 426 if (ns->always_visible || nvmf_ns_find_host(ns, ctrlr->hostnqn) != NULL) { 427 nvmf_ctrlr_ns_set_visible(ctrlr, ns->nsid, true); 428 } 429 } 430 } 431 432 static struct spdk_nvmf_ctrlr * 433 nvmf_ctrlr_create(struct spdk_nvmf_subsystem *subsystem, 434 struct spdk_nvmf_request *req, 435 struct spdk_nvmf_fabric_connect_cmd *connect_cmd, 436 struct spdk_nvmf_fabric_connect_data *connect_data) 437 { 438 struct spdk_nvmf_ctrlr *ctrlr; 439 struct spdk_nvmf_transport *transport = req->qpair->transport; 440 struct spdk_nvme_transport_id listen_trid = {}; 441 bool subsys_has_multi_iocs = false; 442 443 ctrlr = calloc(1, sizeof(*ctrlr)); 444 if (ctrlr == NULL) { 445 SPDK_ERRLOG("Memory allocation failed\n"); 446 return NULL; 447 } 448 449 if (spdk_nvme_trtype_is_fabrics(transport->ops->type)) { 450 ctrlr->dynamic_ctrlr = true; 451 } else { 452 ctrlr->cntlid = connect_data->cntlid; 453 } 454 455 SPDK_DTRACE_PROBE3_TICKS(nvmf_ctrlr_create, ctrlr, subsystem->subnqn, 456 spdk_thread_get_id(req->qpair->group->thread)); 457 458 STAILQ_INIT(&ctrlr->async_events); 459 TAILQ_INIT(&ctrlr->log_head); 460 ctrlr->subsys = subsystem; 461 ctrlr->thread = req->qpair->group->thread; 462 ctrlr->disconnect_in_progress = false; 463 464 ctrlr->qpair_mask = spdk_bit_array_create(transport->opts.max_qpairs_per_ctrlr); 465 if (!ctrlr->qpair_mask) { 466 SPDK_ERRLOG("Failed to allocate controller qpair mask\n"); 467 goto err_qpair_mask; 468 } 469 470 nvmf_ctrlr_cdata_init(transport, subsystem, &ctrlr->cdata); 471 472 /* 473 * KAS: This field indicates the granularity of the Keep Alive Timer in 100ms units. 474 * If this field is cleared to 0h, then Keep Alive is not supported. 475 */ 476 if (ctrlr->cdata.kas) { 477 ctrlr->feat.keep_alive_timer.bits.kato = spdk_divide_round_up(connect_cmd->kato, 478 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS) * 479 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS; 480 } 481 482 ctrlr->feat.async_event_configuration.bits.ns_attr_notice = 1; 483 if (ctrlr->subsys->flags.ana_reporting) { 484 ctrlr->feat.async_event_configuration.bits.ana_change_notice = 1; 485 } 486 ctrlr->feat.volatile_write_cache.bits.wce = 1; 487 /* Coalescing Disable */ 488 ctrlr->feat.interrupt_vector_configuration.bits.cd = 1; 489 490 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 491 /* 492 * If keep-alive timeout is not set, discovery controllers use some 493 * arbitrary high value in order to cleanup stale discovery sessions 494 * 495 * From the 1.0a nvme-of spec: 496 * "The Keep Alive command is reserved for 497 * Discovery controllers. A transport may specify a 498 * fixed Discovery controller activity timeout value 499 * (e.g., 2 minutes). If no commands are received 500 * by a Discovery controller within that time 501 * period, the controller may perform the 502 * actions for Keep Alive Timer expiration". 503 * 504 * From the 1.1 nvme-of spec: 505 * "A host requests an explicit persistent connection 506 * to a Discovery controller and Asynchronous Event Notifications from 507 * the Discovery controller on that persistent connection by specifying 508 * a non-zero Keep Alive Timer value in the Connect command." 509 * 510 * In case non-zero KATO is used, we enable discovery_log_change_notice 511 * otherwise we disable it and use default discovery controller KATO. 512 * KATO is in millisecond. 513 */ 514 if (ctrlr->feat.keep_alive_timer.bits.kato == 0) { 515 ctrlr->feat.keep_alive_timer.bits.kato = NVMF_DISC_KATO_IN_MS; 516 ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice = 0; 517 } else { 518 ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice = 1; 519 } 520 } 521 522 /* Subtract 1 for admin queue, 1 for 0's based */ 523 ctrlr->feat.number_of_queues.bits.ncqr = transport->opts.max_qpairs_per_ctrlr - 1 - 524 1; 525 ctrlr->feat.number_of_queues.bits.nsqr = transport->opts.max_qpairs_per_ctrlr - 1 - 526 1; 527 528 spdk_uuid_copy(&ctrlr->hostid, (struct spdk_uuid *)connect_data->hostid); 529 memcpy(ctrlr->hostnqn, connect_data->hostnqn, SPDK_NVMF_NQN_MAX_LEN); 530 531 ctrlr->visible_ns = spdk_bit_array_create(subsystem->max_nsid); 532 if (!ctrlr->visible_ns) { 533 SPDK_ERRLOG("Failed to allocate visible namespace array\n"); 534 goto err_visible_ns; 535 } 536 nvmf_ctrlr_init_visible_ns(ctrlr); 537 538 ctrlr->vcprop.cap.raw = 0; 539 ctrlr->vcprop.cap.bits.cqr = 1; /* NVMe-oF specification required */ 540 ctrlr->vcprop.cap.bits.mqes = transport->opts.max_queue_depth - 541 1; /* max queue depth */ 542 ctrlr->vcprop.cap.bits.ams = 0; /* optional arb mechanisms */ 543 /* ready timeout - 500 msec units */ 544 ctrlr->vcprop.cap.bits.to = NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS / 500; 545 ctrlr->vcprop.cap.bits.dstrd = 0; /* fixed to 0 for NVMe-oF */ 546 subsys_has_multi_iocs = nvmf_subsystem_has_zns_iocs(subsystem); 547 if (subsys_has_multi_iocs) { 548 ctrlr->vcprop.cap.bits.css = 549 SPDK_NVME_CAP_CSS_IOCS; /* One or more I/O command sets supported */ 550 } else { 551 ctrlr->vcprop.cap.bits.css = SPDK_NVME_CAP_CSS_NVM; /* NVM command set */ 552 } 553 554 ctrlr->vcprop.cap.bits.mpsmin = 0; /* 2 ^ (12 + mpsmin) == 4k */ 555 ctrlr->vcprop.cap.bits.mpsmax = 0; /* 2 ^ (12 + mpsmax) == 4k */ 556 557 /* Version Supported: 1.3 */ 558 ctrlr->vcprop.vs.bits.mjr = 1; 559 ctrlr->vcprop.vs.bits.mnr = 3; 560 ctrlr->vcprop.vs.bits.ter = 0; 561 562 ctrlr->vcprop.cc.raw = 0; 563 ctrlr->vcprop.cc.bits.en = 0; /* Init controller disabled */ 564 if (subsys_has_multi_iocs) { 565 ctrlr->vcprop.cc.bits.css = 566 SPDK_NVME_CC_CSS_IOCS; /* All supported I/O Command Sets */ 567 } 568 569 ctrlr->vcprop.csts.raw = 0; 570 ctrlr->vcprop.csts.bits.rdy = 0; /* Init controller as not ready */ 571 572 SPDK_DEBUGLOG(nvmf, "cap 0x%" PRIx64 "\n", ctrlr->vcprop.cap.raw); 573 SPDK_DEBUGLOG(nvmf, "vs 0x%x\n", ctrlr->vcprop.vs.raw); 574 SPDK_DEBUGLOG(nvmf, "cc 0x%x\n", ctrlr->vcprop.cc.raw); 575 SPDK_DEBUGLOG(nvmf, "csts 0x%x\n", ctrlr->vcprop.csts.raw); 576 577 ctrlr->dif_insert_or_strip = transport->opts.dif_insert_or_strip; 578 579 if (ctrlr->subsys->subtype == SPDK_NVMF_SUBTYPE_NVME) { 580 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &listen_trid) != 0) { 581 SPDK_ERRLOG("Could not get listener transport ID\n"); 582 goto err_listener; 583 } 584 585 ctrlr->listener = nvmf_subsystem_find_listener(ctrlr->subsys, &listen_trid); 586 if (!ctrlr->listener) { 587 SPDK_ERRLOG("Listener was not found\n"); 588 goto err_listener; 589 } 590 } 591 592 nvmf_qpair_set_ctrlr(req->qpair, ctrlr); 593 spdk_thread_send_msg(subsystem->thread, _nvmf_subsystem_add_ctrlr, req); 594 595 return ctrlr; 596 err_listener: 597 spdk_bit_array_free(&ctrlr->visible_ns); 598 err_visible_ns: 599 spdk_bit_array_free(&ctrlr->qpair_mask); 600 err_qpair_mask: 601 free(ctrlr); 602 return NULL; 603 } 604 605 static void 606 _nvmf_ctrlr_destruct(void *ctx) 607 { 608 struct spdk_nvmf_ctrlr *ctrlr = ctx; 609 struct spdk_nvmf_reservation_log *log, *log_tmp; 610 struct spdk_nvmf_async_event_completion *event, *event_tmp; 611 612 SPDK_DTRACE_PROBE3_TICKS(nvmf_ctrlr_destruct, ctrlr, ctrlr->subsys->subnqn, 613 spdk_thread_get_id(ctrlr->thread)); 614 615 assert(spdk_get_thread() == ctrlr->thread); 616 assert(ctrlr->in_destruct); 617 618 SPDK_DEBUGLOG(nvmf, "Destroy ctrlr 0x%hx\n", ctrlr->cntlid); 619 if (ctrlr->disconnect_in_progress) { 620 SPDK_ERRLOG("freeing ctrlr with disconnect in progress\n"); 621 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_destruct, ctrlr); 622 return; 623 } 624 625 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 626 nvmf_ctrlr_stop_association_timer(ctrlr); 627 spdk_bit_array_free(&ctrlr->qpair_mask); 628 629 TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) { 630 TAILQ_REMOVE(&ctrlr->log_head, log, link); 631 free(log); 632 } 633 STAILQ_FOREACH_SAFE(event, &ctrlr->async_events, link, event_tmp) { 634 STAILQ_REMOVE(&ctrlr->async_events, event, spdk_nvmf_async_event_completion, link); 635 free(event); 636 } 637 spdk_bit_array_free(&ctrlr->visible_ns); 638 free(ctrlr); 639 } 640 641 void 642 nvmf_ctrlr_destruct(struct spdk_nvmf_ctrlr *ctrlr) 643 { 644 nvmf_subsystem_remove_ctrlr(ctrlr->subsys, ctrlr); 645 646 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_destruct, ctrlr); 647 } 648 649 static void 650 nvmf_ctrlr_add_io_qpair(void *ctx) 651 { 652 struct spdk_nvmf_request *req = ctx; 653 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 654 struct spdk_nvmf_qpair *qpair = req->qpair; 655 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 656 struct spdk_nvmf_qpair *admin_qpair = ctrlr->admin_qpair; 657 struct spdk_nvmf_poll_group *admin_qpair_group = NULL; 658 enum spdk_nvmf_qpair_state admin_qpair_state = SPDK_NVMF_QPAIR_UNINITIALIZED; 659 bool admin_qpair_active = false; 660 661 SPDK_DTRACE_PROBE4_TICKS(nvmf_ctrlr_add_io_qpair, ctrlr, req->qpair, req->qpair->qid, 662 spdk_thread_get_id(ctrlr->thread)); 663 664 /* Unit test will check qpair->ctrlr after calling spdk_nvmf_ctrlr_connect. 665 * For error case, the value should be NULL. So set it to NULL at first. 666 */ 667 qpair->ctrlr = NULL; 668 669 /* Make sure the controller is not being destroyed. */ 670 if (ctrlr->in_destruct) { 671 SPDK_ERRLOG("Got I/O connect while ctrlr was being destroyed.\n"); 672 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 673 goto end; 674 } 675 676 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 677 SPDK_ERRLOG("I/O connect not allowed on discovery controller\n"); 678 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 679 goto end; 680 } 681 682 if (!ctrlr->vcprop.cc.bits.en) { 683 SPDK_ERRLOG("Got I/O connect before ctrlr was enabled\n"); 684 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 685 goto end; 686 } 687 688 if (1u << ctrlr->vcprop.cc.bits.iosqes != sizeof(struct spdk_nvme_cmd)) { 689 SPDK_ERRLOG("Got I/O connect with invalid IOSQES %u\n", 690 ctrlr->vcprop.cc.bits.iosqes); 691 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 692 goto end; 693 } 694 695 if (1u << ctrlr->vcprop.cc.bits.iocqes != sizeof(struct spdk_nvme_cpl)) { 696 SPDK_ERRLOG("Got I/O connect with invalid IOCQES %u\n", 697 ctrlr->vcprop.cc.bits.iocqes); 698 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 699 goto end; 700 } 701 702 /* There is a chance that admin qpair was destroyed. This is an issue that was observed only with ESX initiators */ 703 if (admin_qpair) { 704 admin_qpair_active = spdk_nvmf_qpair_is_active(admin_qpair); 705 admin_qpair_group = admin_qpair->group; 706 admin_qpair_state = admin_qpair->state; 707 } 708 709 if (!admin_qpair_active || admin_qpair_group == NULL) { 710 /* There is a chance that admin qpair was destroyed or is being destroyed at this moment due to e.g. 711 * expired keep alive timer. Part of the qpair destruction process is change of qpair's 712 * state to DEACTIVATING and removing it from poll group */ 713 SPDK_ERRLOG("Inactive admin qpair (state %d, group %p)\n", admin_qpair_state, admin_qpair_group); 714 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 715 goto end; 716 } 717 718 /* check if we would exceed ctrlr connection limit */ 719 if (qpair->qid >= spdk_bit_array_capacity(ctrlr->qpair_mask)) { 720 SPDK_ERRLOG("Requested QID %u but Max QID is %u\n", 721 qpair->qid, spdk_bit_array_capacity(ctrlr->qpair_mask) - 1); 722 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 723 rsp->status.sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER; 724 goto end; 725 } 726 727 nvmf_ctrlr_add_qpair(qpair, ctrlr, req); 728 return; 729 end: 730 spdk_nvmf_request_complete(req); 731 } 732 733 static void 734 _nvmf_ctrlr_add_io_qpair(void *ctx) 735 { 736 struct spdk_nvmf_request *req = ctx; 737 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 738 struct spdk_nvmf_fabric_connect_data *data; 739 struct spdk_nvmf_ctrlr *ctrlr; 740 struct spdk_nvmf_qpair *qpair = req->qpair; 741 struct spdk_nvmf_qpair *admin_qpair; 742 struct spdk_nvmf_tgt *tgt = qpair->transport->tgt; 743 struct spdk_nvmf_subsystem *subsystem; 744 struct spdk_nvme_transport_id listen_trid = {}; 745 const struct spdk_nvmf_subsystem_listener *listener; 746 struct spdk_nvmf_poll_group *admin_qpair_group = NULL; 747 enum spdk_nvmf_qpair_state admin_qpair_state = SPDK_NVMF_QPAIR_UNINITIALIZED; 748 bool admin_qpair_active = false; 749 750 assert(req->iovcnt == 1); 751 752 data = req->iov[0].iov_base; 753 754 SPDK_DEBUGLOG(nvmf, "Connect I/O Queue for controller id 0x%x\n", data->cntlid); 755 756 subsystem = spdk_nvmf_tgt_find_subsystem(tgt, data->subnqn); 757 /* We already checked this in spdk_nvmf_ctrlr_connect */ 758 assert(subsystem != NULL); 759 760 ctrlr = nvmf_subsystem_get_ctrlr(subsystem, data->cntlid); 761 if (ctrlr == NULL) { 762 SPDK_ERRLOG("Unknown controller ID 0x%x\n", data->cntlid); 763 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, cntlid); 764 spdk_nvmf_request_complete(req); 765 return; 766 } 767 768 /* fail before passing a message to the controller thread. */ 769 if (ctrlr->in_destruct) { 770 SPDK_ERRLOG("Got I/O connect while ctrlr was being destroyed.\n"); 771 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 772 spdk_nvmf_request_complete(req); 773 return; 774 } 775 776 /* If ANA reporting is enabled, check if I/O connect is on the same listener. */ 777 if (subsystem->flags.ana_reporting) { 778 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &listen_trid) != 0) { 779 SPDK_ERRLOG("Could not get listener transport ID\n"); 780 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 781 spdk_nvmf_request_complete(req); 782 return; 783 } 784 785 listener = nvmf_subsystem_find_listener(subsystem, &listen_trid); 786 if (listener != ctrlr->listener) { 787 SPDK_ERRLOG("I/O connect is on a listener different from admin connect\n"); 788 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 789 spdk_nvmf_request_complete(req); 790 return; 791 } 792 } 793 794 admin_qpair = ctrlr->admin_qpair; 795 796 /* There is a chance that admin qpair was destroyed. This is an issue that was observed only with ESX initiators */ 797 if (admin_qpair) { 798 admin_qpair_active = spdk_nvmf_qpair_is_active(admin_qpair); 799 admin_qpair_group = admin_qpair->group; 800 admin_qpair_state = admin_qpair->state; 801 } 802 803 if (!admin_qpair_active || admin_qpair_group == NULL) { 804 /* There is a chance that admin qpair was destroyed or is being destroyed at this moment due to e.g. 805 * expired keep alive timer. Part of the qpair destruction process is change of qpair's 806 * state to DEACTIVATING and removing it from poll group */ 807 SPDK_ERRLOG("Inactive admin qpair (state %d, group %p)\n", admin_qpair_state, admin_qpair_group); 808 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 809 spdk_nvmf_request_complete(req); 810 return; 811 } 812 qpair->ctrlr = ctrlr; 813 spdk_thread_send_msg(admin_qpair_group->thread, nvmf_ctrlr_add_io_qpair, req); 814 } 815 816 static bool 817 nvmf_qpair_access_allowed(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_subsystem *subsystem, 818 const char *hostnqn) 819 { 820 struct spdk_nvme_transport_id listen_trid = {}; 821 822 if (!spdk_nvmf_subsystem_host_allowed(subsystem, hostnqn)) { 823 SPDK_ERRLOG("Subsystem '%s' does not allow host '%s'\n", subsystem->subnqn, hostnqn); 824 return false; 825 } 826 827 if (spdk_nvmf_qpair_get_listen_trid(qpair, &listen_trid)) { 828 SPDK_ERRLOG("Subsystem '%s' is unable to enforce access control due to an internal error.\n", 829 subsystem->subnqn); 830 return false; 831 } 832 833 if (!spdk_nvmf_subsystem_listener_allowed(subsystem, &listen_trid)) { 834 SPDK_ERRLOG("Subsystem '%s' does not allow host '%s' to connect at this address.\n", 835 subsystem->subnqn, hostnqn); 836 return false; 837 } 838 839 return true; 840 } 841 842 static int 843 _nvmf_ctrlr_connect(struct spdk_nvmf_request *req) 844 { 845 struct spdk_nvmf_fabric_connect_data *data = req->iov[0].iov_base; 846 struct spdk_nvmf_fabric_connect_cmd *cmd = &req->cmd->connect_cmd; 847 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 848 struct spdk_nvmf_qpair *qpair = req->qpair; 849 struct spdk_nvmf_transport *transport = qpair->transport; 850 struct spdk_nvmf_ctrlr *ctrlr; 851 struct spdk_nvmf_subsystem *subsystem; 852 853 SPDK_DEBUGLOG(nvmf, "recfmt 0x%x qid %u sqsize %u\n", 854 cmd->recfmt, cmd->qid, cmd->sqsize); 855 856 SPDK_DEBUGLOG(nvmf, "Connect data:\n"); 857 SPDK_DEBUGLOG(nvmf, " cntlid: 0x%04x\n", data->cntlid); 858 SPDK_DEBUGLOG(nvmf, " hostid: %08x-%04x-%04x-%02x%02x-%04x%08x ***\n", 859 ntohl(*(uint32_t *)&data->hostid[0]), 860 ntohs(*(uint16_t *)&data->hostid[4]), 861 ntohs(*(uint16_t *)&data->hostid[6]), 862 data->hostid[8], 863 data->hostid[9], 864 ntohs(*(uint16_t *)&data->hostid[10]), 865 ntohl(*(uint32_t *)&data->hostid[12])); 866 SPDK_DEBUGLOG(nvmf, " subnqn: \"%s\"\n", data->subnqn); 867 SPDK_DEBUGLOG(nvmf, " hostnqn: \"%s\"\n", data->hostnqn); 868 869 subsystem = spdk_nvmf_tgt_find_subsystem(transport->tgt, data->subnqn); 870 if (!subsystem) { 871 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 872 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 873 } 874 875 if (cmd->recfmt != 0) { 876 SPDK_ERRLOG("Connect command unsupported RECFMT %u\n", cmd->recfmt); 877 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 878 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INCOMPATIBLE_FORMAT; 879 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 880 } 881 882 /* 883 * SQSIZE is a 0-based value, so it must be at least 1 (minimum queue depth is 2) and 884 * strictly less than max_aq_depth (admin queues) or max_queue_depth (io queues). 885 */ 886 if (cmd->sqsize == 0) { 887 SPDK_ERRLOG("Invalid SQSIZE = 0\n"); 888 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 889 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 890 } 891 892 if (cmd->qid == 0) { 893 if (cmd->sqsize >= transport->opts.max_aq_depth) { 894 SPDK_ERRLOG("Invalid SQSIZE for admin queue %u (min 1, max %u)\n", 895 cmd->sqsize, transport->opts.max_aq_depth - 1); 896 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 897 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 898 } 899 } else if (cmd->sqsize >= transport->opts.max_queue_depth) { 900 SPDK_ERRLOG("Invalid SQSIZE %u (min 1, max %u)\n", 901 cmd->sqsize, transport->opts.max_queue_depth - 1); 902 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 903 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 904 } 905 906 qpair->sq_head_max = cmd->sqsize; 907 qpair->qid = cmd->qid; 908 qpair->connect_received = true; 909 910 pthread_mutex_lock(&qpair->group->mutex); 911 qpair->group->current_unassociated_qpairs--; 912 pthread_mutex_unlock(&qpair->group->mutex); 913 914 if (0 == qpair->qid) { 915 qpair->group->stat.admin_qpairs++; 916 qpair->group->stat.current_admin_qpairs++; 917 } else { 918 qpair->group->stat.io_qpairs++; 919 qpair->group->stat.current_io_qpairs++; 920 } 921 922 if (cmd->qid == 0) { 923 SPDK_DEBUGLOG(nvmf, "Connect Admin Queue for controller ID 0x%x\n", data->cntlid); 924 925 if (spdk_nvme_trtype_is_fabrics(transport->ops->type) && data->cntlid != 0xFFFF) { 926 /* This NVMf target only supports dynamic mode. */ 927 SPDK_ERRLOG("The NVMf target only supports dynamic mode (CNTLID = 0x%x).\n", data->cntlid); 928 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, cntlid); 929 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 930 } 931 932 /* Establish a new ctrlr */ 933 ctrlr = nvmf_ctrlr_create(subsystem, req, cmd, data); 934 if (!ctrlr) { 935 SPDK_ERRLOG("nvmf_ctrlr_create() failed\n"); 936 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 937 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 938 } else { 939 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 940 } 941 } else { 942 spdk_thread_send_msg(subsystem->thread, _nvmf_ctrlr_add_io_qpair, req); 943 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 944 } 945 } 946 947 static struct spdk_nvmf_subsystem_poll_group * 948 nvmf_subsystem_pg_from_connect_cmd(struct spdk_nvmf_request *req) 949 { 950 struct spdk_nvmf_fabric_connect_data *data; 951 struct spdk_nvmf_subsystem *subsystem; 952 struct spdk_nvmf_tgt *tgt; 953 954 assert(nvmf_request_is_fabric_connect(req)); 955 assert(req->qpair->ctrlr == NULL); 956 assert(req->iovcnt == 1); 957 958 data = req->iov[0].iov_base; 959 tgt = req->qpair->transport->tgt; 960 961 subsystem = spdk_nvmf_tgt_find_subsystem(tgt, data->subnqn); 962 if (subsystem == NULL) { 963 return NULL; 964 } 965 966 return &req->qpair->group->sgroups[subsystem->id]; 967 } 968 969 int 970 spdk_nvmf_ctrlr_connect(struct spdk_nvmf_request *req) 971 { 972 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 973 struct spdk_nvmf_subsystem_poll_group *sgroup; 974 struct spdk_nvmf_qpair *qpair = req->qpair; 975 enum spdk_nvmf_request_exec_status status; 976 977 if (req->iovcnt > 1) { 978 SPDK_ERRLOG("Connect command invalid iovcnt: %d\n", req->iovcnt); 979 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 980 status = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 981 goto out; 982 } 983 984 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 985 if (!sgroup) { 986 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 987 status = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 988 goto out; 989 } 990 991 sgroup->mgmt_io_outstanding++; 992 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 993 994 status = _nvmf_ctrlr_connect(req); 995 996 out: 997 if (status == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 998 _nvmf_request_complete(req); 999 } 1000 1001 return status; 1002 } 1003 1004 static int 1005 nvmf_ctrlr_cmd_connect(struct spdk_nvmf_request *req) 1006 { 1007 struct spdk_nvmf_fabric_connect_data *data = req->iov[0].iov_base; 1008 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 1009 struct spdk_nvmf_transport *transport = req->qpair->transport; 1010 struct spdk_nvmf_subsystem *subsystem; 1011 1012 if (req->length < sizeof(struct spdk_nvmf_fabric_connect_data)) { 1013 SPDK_ERRLOG("Connect command data length 0x%x too small\n", req->length); 1014 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1015 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1016 } 1017 1018 if (req->iovcnt > 1) { 1019 SPDK_ERRLOG("Connect command invalid iovcnt: %d\n", req->iovcnt); 1020 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1021 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1022 } 1023 1024 subsystem = spdk_nvmf_tgt_find_subsystem(transport->tgt, data->subnqn); 1025 if (!subsystem) { 1026 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 1027 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1028 } 1029 1030 if ((subsystem->state == SPDK_NVMF_SUBSYSTEM_INACTIVE) || 1031 (subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSING) || 1032 (subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSED) || 1033 (subsystem->state == SPDK_NVMF_SUBSYSTEM_DEACTIVATING)) { 1034 struct spdk_nvmf_subsystem_poll_group *sgroup; 1035 1036 /* Subsystem is not ready to handle a connect. Decrement 1037 * the mgmt_io_outstanding to avoid the subsystem waiting 1038 * for this command to complete before unpausing. Queued 1039 * requests get retried when subsystem resumes. 1040 */ 1041 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 1042 assert(sgroup != NULL); 1043 sgroup->mgmt_io_outstanding--; 1044 TAILQ_REMOVE(&req->qpair->outstanding, req, link); 1045 TAILQ_INSERT_TAIL(&sgroup->queued, req, link); 1046 SPDK_DEBUGLOG(nvmf, "Subsystem '%s' is not ready for connect, retrying...\n", subsystem->subnqn); 1047 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 1048 } 1049 1050 /* Ensure that hostnqn is null terminated */ 1051 if (!memchr(data->hostnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 1052 SPDK_ERRLOG("Connect HOSTNQN is not null terminated\n"); 1053 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, hostnqn); 1054 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1055 } 1056 1057 if (!nvmf_qpair_access_allowed(req->qpair, subsystem, data->hostnqn)) { 1058 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1059 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_HOST; 1060 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1061 } 1062 1063 return _nvmf_ctrlr_connect(req); 1064 } 1065 1066 static int 1067 nvmf_ctrlr_association_remove(void *ctx) 1068 { 1069 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1070 int rc; 1071 1072 nvmf_ctrlr_stop_association_timer(ctrlr); 1073 1074 if (ctrlr->in_destruct) { 1075 return SPDK_POLLER_IDLE; 1076 } 1077 SPDK_DEBUGLOG(nvmf, "Disconnecting host from subsystem %s due to association timeout.\n", 1078 ctrlr->subsys->subnqn); 1079 1080 if (ctrlr->admin_qpair) { 1081 rc = spdk_nvmf_qpair_disconnect(ctrlr->admin_qpair); 1082 if (rc < 0 && rc != -EINPROGRESS) { 1083 SPDK_ERRLOG("Fail to disconnect admin ctrlr qpair\n"); 1084 assert(false); 1085 } 1086 } 1087 1088 return SPDK_POLLER_BUSY; 1089 } 1090 1091 static int 1092 _nvmf_ctrlr_cc_reset_shn_done(void *ctx) 1093 { 1094 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1095 uint64_t now = spdk_get_ticks(); 1096 uint32_t count; 1097 1098 if (ctrlr->cc_timer) { 1099 spdk_poller_unregister(&ctrlr->cc_timer); 1100 } 1101 1102 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1103 SPDK_DEBUGLOG(nvmf, "ctrlr %p active queue count %u\n", ctrlr, count); 1104 1105 if (count > 1) { 1106 if (now < ctrlr->cc_timeout_tsc) { 1107 /* restart cc timer */ 1108 ctrlr->cc_timer = SPDK_POLLER_REGISTER(_nvmf_ctrlr_cc_reset_shn_done, ctrlr, 100 * 1000); 1109 return SPDK_POLLER_IDLE; 1110 } else { 1111 /* controller fatal status */ 1112 SPDK_WARNLOG("IO timeout, ctrlr %p is in fatal status\n", ctrlr); 1113 nvmf_ctrlr_set_fatal_status(ctrlr); 1114 } 1115 } 1116 1117 spdk_poller_unregister(&ctrlr->cc_timeout_timer); 1118 1119 if (ctrlr->disconnect_is_shn) { 1120 ctrlr->vcprop.csts.bits.shst = SPDK_NVME_SHST_COMPLETE; 1121 ctrlr->disconnect_is_shn = false; 1122 } else { 1123 /* Only a subset of the registers are cleared out on a reset */ 1124 ctrlr->vcprop.cc.raw = 0; 1125 ctrlr->vcprop.csts.raw = 0; 1126 } 1127 1128 /* After CC.EN transitions to 0 (due to shutdown or reset), the association 1129 * between the host and controller shall be preserved for at least 2 minutes */ 1130 if (ctrlr->association_timer) { 1131 SPDK_DEBUGLOG(nvmf, "Association timer already set\n"); 1132 nvmf_ctrlr_stop_association_timer(ctrlr); 1133 } 1134 if (ctrlr->association_timeout) { 1135 ctrlr->association_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_association_remove, ctrlr, 1136 ctrlr->association_timeout * 1000); 1137 } 1138 ctrlr->disconnect_in_progress = false; 1139 return SPDK_POLLER_BUSY; 1140 } 1141 1142 static void 1143 nvmf_ctrlr_cc_reset_shn_done(struct spdk_io_channel_iter *i, int status) 1144 { 1145 struct spdk_nvmf_ctrlr *ctrlr = spdk_io_channel_iter_get_ctx(i); 1146 1147 if (status < 0) { 1148 SPDK_ERRLOG("Fail to disconnect io ctrlr qpairs\n"); 1149 assert(false); 1150 } 1151 1152 _nvmf_ctrlr_cc_reset_shn_done((void *)ctrlr); 1153 } 1154 1155 static void 1156 nvmf_bdev_complete_reset(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1157 { 1158 SPDK_NOTICELOG("Resetting bdev done with %s\n", success ? "success" : "failure"); 1159 1160 spdk_bdev_free_io(bdev_io); 1161 } 1162 1163 1164 static int 1165 nvmf_ctrlr_cc_timeout(void *ctx) 1166 { 1167 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1168 struct spdk_nvmf_poll_group *group; 1169 struct spdk_nvmf_ns *ns; 1170 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1171 1172 spdk_poller_unregister(&ctrlr->cc_timeout_timer); 1173 SPDK_DEBUGLOG(nvmf, "Ctrlr %p reset or shutdown timeout\n", ctrlr); 1174 1175 if (!ctrlr->admin_qpair) { 1176 SPDK_NOTICELOG("Ctrlr %p admin qpair disconnected\n", ctrlr); 1177 return SPDK_POLLER_IDLE; 1178 } 1179 1180 group = ctrlr->admin_qpair->group; 1181 assert(group != NULL && group->sgroups != NULL); 1182 1183 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 1184 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 1185 if (ns->bdev == NULL) { 1186 continue; 1187 } 1188 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[ns->opts.nsid - 1]; 1189 SPDK_NOTICELOG("Ctrlr %p resetting NSID %u\n", ctrlr, ns->opts.nsid); 1190 spdk_bdev_reset(ns->desc, ns_info->channel, nvmf_bdev_complete_reset, NULL); 1191 } 1192 1193 return SPDK_POLLER_BUSY; 1194 } 1195 1196 const struct spdk_nvmf_registers * 1197 spdk_nvmf_ctrlr_get_regs(struct spdk_nvmf_ctrlr *ctrlr) 1198 { 1199 return &ctrlr->vcprop; 1200 } 1201 1202 void 1203 nvmf_ctrlr_set_fatal_status(struct spdk_nvmf_ctrlr *ctrlr) 1204 { 1205 ctrlr->vcprop.csts.bits.cfs = 1; 1206 } 1207 1208 static uint64_t 1209 nvmf_prop_get_cap(struct spdk_nvmf_ctrlr *ctrlr) 1210 { 1211 return ctrlr->vcprop.cap.raw; 1212 } 1213 1214 static uint64_t 1215 nvmf_prop_get_vs(struct spdk_nvmf_ctrlr *ctrlr) 1216 { 1217 return ctrlr->vcprop.vs.raw; 1218 } 1219 1220 static uint64_t 1221 nvmf_prop_get_cc(struct spdk_nvmf_ctrlr *ctrlr) 1222 { 1223 return ctrlr->vcprop.cc.raw; 1224 } 1225 1226 static bool 1227 nvmf_prop_set_cc(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1228 { 1229 union spdk_nvme_cc_register cc, diff; 1230 uint32_t cc_timeout_ms; 1231 1232 cc.raw = value; 1233 1234 SPDK_DEBUGLOG(nvmf, "cur CC: 0x%08x\n", ctrlr->vcprop.cc.raw); 1235 SPDK_DEBUGLOG(nvmf, "new CC: 0x%08x\n", cc.raw); 1236 1237 /* 1238 * Calculate which bits changed between the current and new CC. 1239 * Mark each bit as 0 once it is handled to determine if any unhandled bits were changed. 1240 */ 1241 diff.raw = cc.raw ^ ctrlr->vcprop.cc.raw; 1242 1243 if (diff.bits.en) { 1244 if (cc.bits.en) { 1245 SPDK_DEBUGLOG(nvmf, "Property Set CC Enable!\n"); 1246 nvmf_ctrlr_stop_association_timer(ctrlr); 1247 1248 ctrlr->vcprop.cc.bits.en = 1; 1249 ctrlr->vcprop.csts.bits.rdy = 1; 1250 } else { 1251 SPDK_DEBUGLOG(nvmf, "Property Set CC Disable!\n"); 1252 if (ctrlr->disconnect_in_progress) { 1253 SPDK_DEBUGLOG(nvmf, "Disconnect in progress\n"); 1254 return true; 1255 } 1256 1257 ctrlr->cc_timeout_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_cc_timeout, ctrlr, 1258 NVMF_CC_RESET_SHN_TIMEOUT_IN_MS * 1000); 1259 /* Make sure cc_timeout_ms is between cc_timeout_timer and Host reset/shutdown timeout */ 1260 cc_timeout_ms = (NVMF_CC_RESET_SHN_TIMEOUT_IN_MS + NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS) / 2; 1261 ctrlr->cc_timeout_tsc = spdk_get_ticks() + cc_timeout_ms * spdk_get_ticks_hz() / (uint64_t)1000; 1262 1263 ctrlr->vcprop.cc.bits.en = 0; 1264 ctrlr->disconnect_in_progress = true; 1265 ctrlr->disconnect_is_shn = false; 1266 spdk_for_each_channel(ctrlr->subsys->tgt, 1267 nvmf_ctrlr_disconnect_io_qpairs_on_pg, 1268 ctrlr, 1269 nvmf_ctrlr_cc_reset_shn_done); 1270 } 1271 diff.bits.en = 0; 1272 } 1273 1274 if (diff.bits.shn) { 1275 if (cc.bits.shn == SPDK_NVME_SHN_NORMAL || 1276 cc.bits.shn == SPDK_NVME_SHN_ABRUPT) { 1277 SPDK_DEBUGLOG(nvmf, "Property Set CC Shutdown %u%ub!\n", 1278 cc.bits.shn >> 1, cc.bits.shn & 1); 1279 if (ctrlr->disconnect_in_progress) { 1280 SPDK_DEBUGLOG(nvmf, "Disconnect in progress\n"); 1281 return true; 1282 } 1283 1284 ctrlr->cc_timeout_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_cc_timeout, ctrlr, 1285 NVMF_CC_RESET_SHN_TIMEOUT_IN_MS * 1000); 1286 /* Make sure cc_timeout_ms is between cc_timeout_timer and Host reset/shutdown timeout */ 1287 cc_timeout_ms = (NVMF_CC_RESET_SHN_TIMEOUT_IN_MS + NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS) / 2; 1288 ctrlr->cc_timeout_tsc = spdk_get_ticks() + cc_timeout_ms * spdk_get_ticks_hz() / (uint64_t)1000; 1289 1290 ctrlr->vcprop.cc.bits.shn = cc.bits.shn; 1291 ctrlr->disconnect_in_progress = true; 1292 ctrlr->disconnect_is_shn = true; 1293 spdk_for_each_channel(ctrlr->subsys->tgt, 1294 nvmf_ctrlr_disconnect_io_qpairs_on_pg, 1295 ctrlr, 1296 nvmf_ctrlr_cc_reset_shn_done); 1297 1298 /* From the time a shutdown is initiated the controller shall disable 1299 * Keep Alive timer */ 1300 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 1301 } else if (cc.bits.shn == 0) { 1302 ctrlr->vcprop.cc.bits.shn = 0; 1303 } else { 1304 SPDK_ERRLOG("Prop Set CC: Invalid SHN value %u%ub\n", 1305 cc.bits.shn >> 1, cc.bits.shn & 1); 1306 return false; 1307 } 1308 diff.bits.shn = 0; 1309 } 1310 1311 if (diff.bits.iosqes) { 1312 SPDK_DEBUGLOG(nvmf, "Prop Set IOSQES = %u (%u bytes)\n", 1313 cc.bits.iosqes, 1u << cc.bits.iosqes); 1314 ctrlr->vcprop.cc.bits.iosqes = cc.bits.iosqes; 1315 diff.bits.iosqes = 0; 1316 } 1317 1318 if (diff.bits.iocqes) { 1319 SPDK_DEBUGLOG(nvmf, "Prop Set IOCQES = %u (%u bytes)\n", 1320 cc.bits.iocqes, 1u << cc.bits.iocqes); 1321 ctrlr->vcprop.cc.bits.iocqes = cc.bits.iocqes; 1322 diff.bits.iocqes = 0; 1323 } 1324 1325 if (diff.bits.ams) { 1326 SPDK_ERRLOG("Arbitration Mechanism Selected (AMS) 0x%x not supported!\n", cc.bits.ams); 1327 return false; 1328 } 1329 1330 if (diff.bits.mps) { 1331 SPDK_ERRLOG("Memory Page Size (MPS) %u KiB not supported!\n", (1 << (2 + cc.bits.mps))); 1332 return false; 1333 } 1334 1335 if (diff.bits.css) { 1336 if (cc.bits.css > SPDK_NVME_CC_CSS_IOCS) { 1337 SPDK_ERRLOG("I/O Command Set Selected (CSS) 0x%x not supported!\n", cc.bits.css); 1338 return false; 1339 } 1340 diff.bits.css = 0; 1341 } 1342 1343 if (diff.raw != 0) { 1344 /* Print an error message, but don't fail the command in this case. 1345 * If we did want to fail in this case, we'd need to ensure we acted 1346 * on no other bits or the initiator gets confused. */ 1347 SPDK_ERRLOG("Prop Set CC toggled reserved bits 0x%x!\n", diff.raw); 1348 } 1349 1350 return true; 1351 } 1352 1353 static uint64_t 1354 nvmf_prop_get_csts(struct spdk_nvmf_ctrlr *ctrlr) 1355 { 1356 return ctrlr->vcprop.csts.raw; 1357 } 1358 1359 static uint64_t 1360 nvmf_prop_get_aqa(struct spdk_nvmf_ctrlr *ctrlr) 1361 { 1362 return ctrlr->vcprop.aqa.raw; 1363 } 1364 1365 static bool 1366 nvmf_prop_set_aqa(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1367 { 1368 union spdk_nvme_aqa_register aqa; 1369 1370 aqa.raw = value; 1371 1372 /* 1373 * We don't need to explicitly check for maximum size, as the fields are 1374 * limited to 12 bits (4096). 1375 */ 1376 if (aqa.bits.asqs < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES - 1 || 1377 aqa.bits.acqs < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES - 1 || 1378 aqa.bits.reserved1 != 0 || aqa.bits.reserved2 != 0) { 1379 return false; 1380 } 1381 1382 ctrlr->vcprop.aqa.raw = value; 1383 1384 return true; 1385 } 1386 1387 static uint64_t 1388 nvmf_prop_get_asq(struct spdk_nvmf_ctrlr *ctrlr) 1389 { 1390 return ctrlr->vcprop.asq; 1391 } 1392 1393 static bool 1394 nvmf_prop_set_asq_lower(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1395 { 1396 ctrlr->vcprop.asq = (ctrlr->vcprop.asq & (0xFFFFFFFFULL << 32ULL)) | value; 1397 1398 return true; 1399 } 1400 1401 static bool 1402 nvmf_prop_set_asq_upper(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1403 { 1404 ctrlr->vcprop.asq = (ctrlr->vcprop.asq & 0xFFFFFFFFULL) | ((uint64_t)value << 32ULL); 1405 1406 return true; 1407 } 1408 1409 static uint64_t 1410 nvmf_prop_get_acq(struct spdk_nvmf_ctrlr *ctrlr) 1411 { 1412 return ctrlr->vcprop.acq; 1413 } 1414 1415 static bool 1416 nvmf_prop_set_acq_lower(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1417 { 1418 ctrlr->vcprop.acq = (ctrlr->vcprop.acq & (0xFFFFFFFFULL << 32ULL)) | value; 1419 1420 return true; 1421 } 1422 1423 static bool 1424 nvmf_prop_set_acq_upper(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1425 { 1426 ctrlr->vcprop.acq = (ctrlr->vcprop.acq & 0xFFFFFFFFULL) | ((uint64_t)value << 32ULL); 1427 1428 return true; 1429 } 1430 1431 struct nvmf_prop { 1432 uint32_t ofst; 1433 uint8_t size; 1434 char name[11]; 1435 uint64_t (*get_cb)(struct spdk_nvmf_ctrlr *ctrlr); 1436 bool (*set_cb)(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value); 1437 bool (*set_upper_cb)(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value); 1438 }; 1439 1440 #define PROP(field, size, get_cb, set_cb, set_upper_cb) \ 1441 { \ 1442 offsetof(struct spdk_nvme_registers, field), \ 1443 size, \ 1444 #field, \ 1445 get_cb, set_cb, set_upper_cb \ 1446 } 1447 1448 static const struct nvmf_prop nvmf_props[] = { 1449 PROP(cap, 8, nvmf_prop_get_cap, NULL, NULL), 1450 PROP(vs, 4, nvmf_prop_get_vs, NULL, NULL), 1451 PROP(cc, 4, nvmf_prop_get_cc, nvmf_prop_set_cc, NULL), 1452 PROP(csts, 4, nvmf_prop_get_csts, NULL, NULL), 1453 PROP(aqa, 4, nvmf_prop_get_aqa, nvmf_prop_set_aqa, NULL), 1454 PROP(asq, 8, nvmf_prop_get_asq, nvmf_prop_set_asq_lower, nvmf_prop_set_asq_upper), 1455 PROP(acq, 8, nvmf_prop_get_acq, nvmf_prop_set_acq_lower, nvmf_prop_set_acq_upper), 1456 }; 1457 1458 static const struct nvmf_prop * 1459 find_prop(uint32_t ofst, uint8_t size) 1460 { 1461 size_t i; 1462 1463 for (i = 0; i < SPDK_COUNTOF(nvmf_props); i++) { 1464 const struct nvmf_prop *prop = &nvmf_props[i]; 1465 1466 if ((ofst >= prop->ofst) && (ofst + size <= prop->ofst + prop->size)) { 1467 return prop; 1468 } 1469 } 1470 1471 return NULL; 1472 } 1473 1474 static int 1475 nvmf_property_get(struct spdk_nvmf_request *req) 1476 { 1477 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1478 struct spdk_nvmf_fabric_prop_get_cmd *cmd = &req->cmd->prop_get_cmd; 1479 struct spdk_nvmf_fabric_prop_get_rsp *response = &req->rsp->prop_get_rsp; 1480 const struct nvmf_prop *prop; 1481 uint8_t size; 1482 1483 response->status.sc = 0; 1484 response->value.u64 = 0; 1485 1486 SPDK_DEBUGLOG(nvmf, "size %d, offset 0x%x\n", 1487 cmd->attrib.size, cmd->ofst); 1488 1489 switch (cmd->attrib.size) { 1490 case SPDK_NVMF_PROP_SIZE_4: 1491 size = 4; 1492 break; 1493 case SPDK_NVMF_PROP_SIZE_8: 1494 size = 8; 1495 break; 1496 default: 1497 SPDK_DEBUGLOG(nvmf, "Invalid size value %d\n", cmd->attrib.size); 1498 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1499 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1500 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1501 } 1502 1503 prop = find_prop(cmd->ofst, size); 1504 if (prop == NULL || prop->get_cb == NULL) { 1505 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1506 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1507 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1508 } 1509 1510 SPDK_DEBUGLOG(nvmf, "name: %s\n", prop->name); 1511 1512 response->value.u64 = prop->get_cb(ctrlr); 1513 1514 if (size != prop->size) { 1515 /* The size must be 4 and the prop->size is 8. Figure out which part of the property to read. */ 1516 assert(size == 4); 1517 assert(prop->size == 8); 1518 1519 if (cmd->ofst == prop->ofst) { 1520 /* Keep bottom 4 bytes only */ 1521 response->value.u64 &= 0xFFFFFFFF; 1522 } else { 1523 /* Keep top 4 bytes only */ 1524 response->value.u64 >>= 32; 1525 } 1526 } 1527 1528 SPDK_DEBUGLOG(nvmf, "response value: 0x%" PRIx64 "\n", response->value.u64); 1529 1530 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1531 } 1532 1533 static int 1534 nvmf_property_set(struct spdk_nvmf_request *req) 1535 { 1536 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1537 struct spdk_nvmf_fabric_prop_set_cmd *cmd = &req->cmd->prop_set_cmd; 1538 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1539 const struct nvmf_prop *prop; 1540 uint64_t value; 1541 uint8_t size; 1542 bool ret; 1543 1544 SPDK_DEBUGLOG(nvmf, "size %d, offset 0x%x, value 0x%" PRIx64 "\n", 1545 cmd->attrib.size, cmd->ofst, cmd->value.u64); 1546 1547 switch (cmd->attrib.size) { 1548 case SPDK_NVMF_PROP_SIZE_4: 1549 size = 4; 1550 break; 1551 case SPDK_NVMF_PROP_SIZE_8: 1552 size = 8; 1553 break; 1554 default: 1555 SPDK_DEBUGLOG(nvmf, "Invalid size value %d\n", cmd->attrib.size); 1556 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1557 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1558 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1559 } 1560 1561 prop = find_prop(cmd->ofst, size); 1562 if (prop == NULL || prop->set_cb == NULL) { 1563 SPDK_INFOLOG(nvmf, "Invalid offset 0x%x\n", cmd->ofst); 1564 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1565 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1566 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1567 } 1568 1569 SPDK_DEBUGLOG(nvmf, "name: %s\n", prop->name); 1570 1571 value = cmd->value.u64; 1572 1573 if (prop->size == 4) { 1574 ret = prop->set_cb(ctrlr, (uint32_t)value); 1575 } else if (size != prop->size) { 1576 /* The size must be 4 and the prop->size is 8. Figure out which part of the property to write. */ 1577 assert(size == 4); 1578 assert(prop->size == 8); 1579 1580 if (cmd->ofst == prop->ofst) { 1581 ret = prop->set_cb(ctrlr, (uint32_t)value); 1582 } else { 1583 ret = prop->set_upper_cb(ctrlr, (uint32_t)value); 1584 } 1585 } else { 1586 ret = prop->set_cb(ctrlr, (uint32_t)value); 1587 if (ret) { 1588 ret = prop->set_upper_cb(ctrlr, (uint32_t)(value >> 32)); 1589 } 1590 } 1591 1592 if (!ret) { 1593 SPDK_ERRLOG("prop set_cb failed\n"); 1594 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1595 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1596 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1597 } 1598 1599 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1600 } 1601 1602 static int 1603 nvmf_ctrlr_set_features_arbitration(struct spdk_nvmf_request *req) 1604 { 1605 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1606 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1607 1608 SPDK_DEBUGLOG(nvmf, "Set Features - Arbitration (cdw11 = 0x%0x)\n", cmd->cdw11); 1609 1610 ctrlr->feat.arbitration.raw = cmd->cdw11; 1611 ctrlr->feat.arbitration.bits.reserved = 0; 1612 1613 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1614 } 1615 1616 static int 1617 nvmf_ctrlr_set_features_power_management(struct spdk_nvmf_request *req) 1618 { 1619 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1620 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1621 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1622 1623 SPDK_DEBUGLOG(nvmf, "Set Features - Power Management (cdw11 = 0x%0x)\n", cmd->cdw11); 1624 1625 /* Only PS = 0 is allowed, since we report NPSS = 0 */ 1626 if (cmd->cdw11_bits.feat_power_management.bits.ps != 0) { 1627 SPDK_ERRLOG("Invalid power state %u\n", cmd->cdw11_bits.feat_power_management.bits.ps); 1628 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1629 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1630 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1631 } 1632 1633 ctrlr->feat.power_management.raw = cmd->cdw11; 1634 ctrlr->feat.power_management.bits.reserved = 0; 1635 1636 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1637 } 1638 1639 static bool 1640 temp_threshold_opts_valid(const union spdk_nvme_feat_temperature_threshold *opts) 1641 { 1642 /* 1643 * Valid TMPSEL values: 1644 * 0000b - 1000b: temperature sensors 1645 * 1111b: set all implemented temperature sensors 1646 */ 1647 if (opts->bits.tmpsel >= 9 && opts->bits.tmpsel != 15) { 1648 /* 1001b - 1110b: reserved */ 1649 SPDK_ERRLOG("Invalid TMPSEL %u\n", opts->bits.tmpsel); 1650 return false; 1651 } 1652 1653 /* 1654 * Valid THSEL values: 1655 * 00b: over temperature threshold 1656 * 01b: under temperature threshold 1657 */ 1658 if (opts->bits.thsel > 1) { 1659 /* 10b - 11b: reserved */ 1660 SPDK_ERRLOG("Invalid THSEL %u\n", opts->bits.thsel); 1661 return false; 1662 } 1663 1664 return true; 1665 } 1666 1667 static int 1668 nvmf_ctrlr_set_features_temperature_threshold(struct spdk_nvmf_request *req) 1669 { 1670 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1671 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1672 1673 SPDK_DEBUGLOG(nvmf, "Set Features - Temperature Threshold (cdw11 = 0x%0x)\n", cmd->cdw11); 1674 1675 if (!temp_threshold_opts_valid(&cmd->cdw11_bits.feat_temp_threshold)) { 1676 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1677 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1678 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1679 } 1680 1681 /* TODO: no sensors implemented - ignore new values */ 1682 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1683 } 1684 1685 static int 1686 nvmf_ctrlr_get_features_temperature_threshold(struct spdk_nvmf_request *req) 1687 { 1688 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1689 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1690 1691 SPDK_DEBUGLOG(nvmf, "Get Features - Temperature Threshold (cdw11 = 0x%0x)\n", cmd->cdw11); 1692 1693 if (!temp_threshold_opts_valid(&cmd->cdw11_bits.feat_temp_threshold)) { 1694 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1695 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1696 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1697 } 1698 1699 /* TODO: no sensors implemented - return 0 for all thresholds */ 1700 rsp->cdw0 = 0; 1701 1702 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1703 } 1704 1705 static int 1706 nvmf_ctrlr_get_features_interrupt_vector_configuration(struct spdk_nvmf_request *req) 1707 { 1708 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1709 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1710 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1711 union spdk_nvme_feat_interrupt_vector_configuration iv_conf = {}; 1712 1713 SPDK_DEBUGLOG(nvmf, "Get Features - Interrupt Vector Configuration (cdw11 = 0x%0x)\n", cmd->cdw11); 1714 1715 iv_conf.bits.iv = cmd->cdw11_bits.feat_interrupt_vector_configuration.bits.iv; 1716 iv_conf.bits.cd = ctrlr->feat.interrupt_vector_configuration.bits.cd; 1717 rsp->cdw0 = iv_conf.raw; 1718 1719 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1720 } 1721 1722 static int 1723 nvmf_ctrlr_set_features_error_recovery(struct spdk_nvmf_request *req) 1724 { 1725 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1726 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1727 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1728 1729 SPDK_DEBUGLOG(nvmf, "Set Features - Error Recovery (cdw11 = 0x%0x)\n", cmd->cdw11); 1730 1731 if (cmd->cdw11_bits.feat_error_recovery.bits.dulbe) { 1732 /* 1733 * Host is not allowed to set this bit, since we don't advertise it in 1734 * Identify Namespace. 1735 */ 1736 SPDK_ERRLOG("Host set unsupported DULBE bit\n"); 1737 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1738 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1739 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1740 } 1741 1742 ctrlr->feat.error_recovery.raw = cmd->cdw11; 1743 ctrlr->feat.error_recovery.bits.reserved = 0; 1744 1745 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1746 } 1747 1748 static int 1749 nvmf_ctrlr_set_features_volatile_write_cache(struct spdk_nvmf_request *req) 1750 { 1751 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1752 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1753 1754 SPDK_DEBUGLOG(nvmf, "Set Features - Volatile Write Cache (cdw11 = 0x%0x)\n", cmd->cdw11); 1755 1756 ctrlr->feat.volatile_write_cache.raw = cmd->cdw11; 1757 ctrlr->feat.volatile_write_cache.bits.reserved = 0; 1758 1759 SPDK_DEBUGLOG(nvmf, "Set Features - Volatile Write Cache %s\n", 1760 ctrlr->feat.volatile_write_cache.bits.wce ? "Enabled" : "Disabled"); 1761 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1762 } 1763 1764 static int 1765 nvmf_ctrlr_set_features_write_atomicity(struct spdk_nvmf_request *req) 1766 { 1767 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1768 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1769 1770 SPDK_DEBUGLOG(nvmf, "Set Features - Write Atomicity (cdw11 = 0x%0x)\n", cmd->cdw11); 1771 1772 ctrlr->feat.write_atomicity.raw = cmd->cdw11; 1773 ctrlr->feat.write_atomicity.bits.reserved = 0; 1774 1775 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1776 } 1777 1778 static int 1779 nvmf_ctrlr_set_features_host_identifier(struct spdk_nvmf_request *req) 1780 { 1781 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1782 1783 SPDK_ERRLOG("Set Features - Host Identifier not allowed\n"); 1784 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 1785 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1786 } 1787 1788 static int 1789 nvmf_ctrlr_get_features_host_identifier(struct spdk_nvmf_request *req) 1790 { 1791 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1792 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1793 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1794 struct spdk_iov_xfer ix; 1795 1796 SPDK_DEBUGLOG(nvmf, "Get Features - Host Identifier\n"); 1797 1798 if (!cmd->cdw11_bits.feat_host_identifier.bits.exhid) { 1799 /* NVMe over Fabrics requires EXHID=1 (128-bit/16-byte host ID) */ 1800 SPDK_ERRLOG("Get Features - Host Identifier with EXHID=0 not allowed\n"); 1801 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1802 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1803 } 1804 1805 if (req->iovcnt < 1 || req->length < sizeof(ctrlr->hostid)) { 1806 SPDK_ERRLOG("Invalid data buffer for Get Features - Host Identifier\n"); 1807 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1808 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1809 } 1810 1811 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 1812 spdk_iov_xfer_from_buf(&ix, &ctrlr->hostid, sizeof(ctrlr->hostid)); 1813 1814 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1815 } 1816 1817 static int 1818 nvmf_ctrlr_get_features_reservation_notification_mask(struct spdk_nvmf_request *req) 1819 { 1820 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1821 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1822 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1823 struct spdk_nvmf_ns *ns; 1824 1825 SPDK_DEBUGLOG(nvmf, "get Features - Reservation Notification Mask\n"); 1826 1827 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1828 SPDK_ERRLOG("get Features - Invalid Namespace ID\n"); 1829 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1830 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1831 } 1832 1833 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 1834 if (ns == NULL) { 1835 SPDK_ERRLOG("get Features - Invalid Namespace ID\n"); 1836 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1837 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1838 } 1839 rsp->cdw0 = ns->mask; 1840 1841 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1842 } 1843 1844 static int 1845 nvmf_ctrlr_set_features_reservation_notification_mask(struct spdk_nvmf_request *req) 1846 { 1847 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1848 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 1849 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1850 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1851 struct spdk_nvmf_ns *ns; 1852 1853 SPDK_DEBUGLOG(nvmf, "Set Features - Reservation Notification Mask\n"); 1854 1855 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1856 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 1857 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 1858 ns->mask = cmd->cdw11; 1859 } 1860 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1861 } 1862 1863 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 1864 if (ns == NULL) { 1865 SPDK_ERRLOG("Set Features - Invalid Namespace ID\n"); 1866 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1867 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1868 } 1869 ns->mask = cmd->cdw11; 1870 1871 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1872 } 1873 1874 static int 1875 nvmf_ctrlr_get_features_reservation_persistence(struct spdk_nvmf_request *req) 1876 { 1877 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1878 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1879 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1880 struct spdk_nvmf_ns *ns; 1881 1882 SPDK_DEBUGLOG(nvmf, "Get Features - Reservation Persistence\n"); 1883 1884 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 1885 /* NSID with SPDK_NVME_GLOBAL_NS_TAG (=0xffffffff) also included */ 1886 if (ns == NULL) { 1887 SPDK_ERRLOG("Get Features - Invalid Namespace ID\n"); 1888 response->status.sct = SPDK_NVME_SCT_GENERIC; 1889 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1890 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1891 } 1892 1893 response->cdw0 = ns->ptpl_activated; 1894 1895 response->status.sct = SPDK_NVME_SCT_GENERIC; 1896 response->status.sc = SPDK_NVME_SC_SUCCESS; 1897 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1898 } 1899 1900 static int 1901 nvmf_ctrlr_set_features_reservation_persistence(struct spdk_nvmf_request *req) 1902 { 1903 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1904 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1905 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1906 struct spdk_nvmf_ns *ns; 1907 bool ptpl; 1908 1909 SPDK_DEBUGLOG(nvmf, "Set Features - Reservation Persistence\n"); 1910 1911 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 1912 ptpl = cmd->cdw11_bits.feat_rsv_persistence.bits.ptpl; 1913 1914 if (cmd->nsid != SPDK_NVME_GLOBAL_NS_TAG && ns && nvmf_ns_is_ptpl_capable(ns)) { 1915 ns->ptpl_activated = ptpl; 1916 } else if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1917 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns; 1918 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 1919 if (nvmf_ns_is_ptpl_capable(ns)) { 1920 ns->ptpl_activated = ptpl; 1921 } 1922 } 1923 } else { 1924 SPDK_ERRLOG("Set Features - Invalid Namespace ID or Reservation Configuration\n"); 1925 response->status.sct = SPDK_NVME_SCT_GENERIC; 1926 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1927 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1928 } 1929 1930 /* TODO: Feature not changeable for now */ 1931 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1932 response->status.sc = SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE; 1933 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1934 } 1935 1936 static int 1937 nvmf_ctrlr_get_features_host_behavior_support(struct spdk_nvmf_request *req) 1938 { 1939 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1940 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1941 struct spdk_nvme_host_behavior host_behavior = {}; 1942 struct spdk_iov_xfer ix; 1943 1944 SPDK_DEBUGLOG(nvmf, "Get Features - Host Behavior Support\n"); 1945 1946 if (req->iovcnt < 1 || req->length < sizeof(struct spdk_nvme_host_behavior)) { 1947 SPDK_ERRLOG("invalid data buffer for Host Behavior Support\n"); 1948 response->status.sct = SPDK_NVME_SCT_GENERIC; 1949 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1950 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1951 } 1952 1953 host_behavior.acre = ctrlr->acre_enabled; 1954 host_behavior.lbafee = ctrlr->lbafee_enabled; 1955 1956 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 1957 spdk_iov_xfer_from_buf(&ix, &host_behavior, sizeof(host_behavior)); 1958 1959 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1960 } 1961 1962 static int 1963 nvmf_ctrlr_set_features_host_behavior_support(struct spdk_nvmf_request *req) 1964 { 1965 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1966 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1967 struct spdk_nvme_host_behavior *host_behavior; 1968 1969 SPDK_DEBUGLOG(nvmf, "Set Features - Host Behavior Support\n"); 1970 if (req->iovcnt != 1) { 1971 SPDK_ERRLOG("Host Behavior Support invalid iovcnt: %d\n", req->iovcnt); 1972 response->status.sct = SPDK_NVME_SCT_GENERIC; 1973 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1974 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1975 } 1976 if (req->iov[0].iov_len != sizeof(struct spdk_nvme_host_behavior)) { 1977 SPDK_ERRLOG("Host Behavior Support invalid iov_len: %zd\n", req->iov[0].iov_len); 1978 response->status.sct = SPDK_NVME_SCT_GENERIC; 1979 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1980 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1981 } 1982 1983 host_behavior = (struct spdk_nvme_host_behavior *)req->iov[0].iov_base; 1984 if (host_behavior->acre == 0) { 1985 ctrlr->acre_enabled = false; 1986 } else if (host_behavior->acre == 1) { 1987 ctrlr->acre_enabled = true; 1988 } else { 1989 SPDK_ERRLOG("Host Behavior Support invalid acre: 0x%02x\n", host_behavior->acre); 1990 response->status.sct = SPDK_NVME_SCT_GENERIC; 1991 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1992 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1993 } 1994 if (host_behavior->lbafee == 0) { 1995 ctrlr->lbafee_enabled = false; 1996 } else if (host_behavior->lbafee == 1) { 1997 ctrlr->lbafee_enabled = true; 1998 } else { 1999 SPDK_ERRLOG("Host Behavior Support invalid lbafee: 0x%02x\n", host_behavior->lbafee); 2000 response->status.sct = SPDK_NVME_SCT_GENERIC; 2001 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2002 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2003 } 2004 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2005 } 2006 2007 static int 2008 nvmf_ctrlr_set_features_keep_alive_timer(struct spdk_nvmf_request *req) 2009 { 2010 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2011 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2012 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 2013 2014 SPDK_DEBUGLOG(nvmf, "Set Features - Keep Alive Timer (%u ms)\n", cmd->cdw11); 2015 2016 /* 2017 * if attempts to disable keep alive by setting kato to 0h 2018 * a status value of keep alive invalid shall be returned 2019 */ 2020 if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato == 0) { 2021 rsp->status.sc = SPDK_NVME_SC_KEEP_ALIVE_INVALID; 2022 } else if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato < MIN_KEEP_ALIVE_TIMEOUT_IN_MS) { 2023 ctrlr->feat.keep_alive_timer.bits.kato = MIN_KEEP_ALIVE_TIMEOUT_IN_MS; 2024 } else { 2025 /* round up to milliseconds */ 2026 ctrlr->feat.keep_alive_timer.bits.kato = spdk_divide_round_up( 2027 cmd->cdw11_bits.feat_keep_alive_timer.bits.kato, 2028 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS) * 2029 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS; 2030 } 2031 2032 /* 2033 * if change the keep alive timeout value successfully 2034 * update the keep alive poller. 2035 */ 2036 if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato != 0) { 2037 if (ctrlr->keep_alive_poller != NULL) { 2038 spdk_poller_unregister(&ctrlr->keep_alive_poller); 2039 } 2040 ctrlr->keep_alive_poller = SPDK_POLLER_REGISTER(nvmf_ctrlr_keep_alive_poll, ctrlr, 2041 ctrlr->feat.keep_alive_timer.bits.kato * 1000); 2042 } 2043 2044 SPDK_DEBUGLOG(nvmf, "Set Features - Keep Alive Timer set to %u ms\n", 2045 ctrlr->feat.keep_alive_timer.bits.kato); 2046 2047 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2048 } 2049 2050 static int 2051 nvmf_ctrlr_set_features_number_of_queues(struct spdk_nvmf_request *req) 2052 { 2053 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2054 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2055 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 2056 uint32_t count; 2057 2058 SPDK_DEBUGLOG(nvmf, "Set Features - Number of Queues, cdw11 0x%x\n", 2059 req->cmd->nvme_cmd.cdw11); 2060 2061 if (cmd->cdw11_bits.feat_num_of_queues.bits.ncqr == UINT16_MAX || 2062 cmd->cdw11_bits.feat_num_of_queues.bits.nsqr == UINT16_MAX) { 2063 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2064 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2065 } 2066 2067 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 2068 /* verify that the controller is ready to process commands */ 2069 if (count > 1) { 2070 SPDK_DEBUGLOG(nvmf, "Queue pairs already active!\n"); 2071 rsp->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 2072 } else { 2073 /* 2074 * Ignore the value requested by the host - 2075 * always return the pre-configured value based on max_qpairs_allowed. 2076 */ 2077 rsp->cdw0 = ctrlr->feat.number_of_queues.raw; 2078 } 2079 2080 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2081 } 2082 2083 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_ctrlr) == 4928, 2084 "Please check migration fields that need to be added or not"); 2085 2086 static void 2087 nvmf_ctrlr_migr_data_copy(struct spdk_nvmf_ctrlr_migr_data *data, 2088 const struct spdk_nvmf_ctrlr_migr_data *data_src, size_t data_size) 2089 { 2090 assert(data); 2091 assert(data_src); 2092 assert(data_size); 2093 2094 memcpy(&data->regs, &data_src->regs, spdk_min(data->regs_size, data_src->regs_size)); 2095 memcpy(&data->feat, &data_src->feat, spdk_min(data->feat_size, data_src->feat_size)); 2096 2097 #define SET_FIELD(field) \ 2098 if (offsetof(struct spdk_nvmf_ctrlr_migr_data, field) + sizeof(data->field) <= data_size) { \ 2099 data->field = data_src->field; \ 2100 } \ 2101 2102 SET_FIELD(cntlid); 2103 SET_FIELD(acre); 2104 SET_FIELD(num_aer_cids); 2105 SET_FIELD(num_async_events); 2106 SET_FIELD(notice_aen_mask); 2107 #undef SET_FIELD 2108 2109 #define SET_ARRAY(arr) \ 2110 if (offsetof(struct spdk_nvmf_ctrlr_migr_data, arr) + sizeof(data->arr) <= data_size) { \ 2111 memcpy(&data->arr, &data_src->arr, sizeof(data->arr)); \ 2112 } \ 2113 2114 SET_ARRAY(async_events); 2115 SET_ARRAY(aer_cids); 2116 #undef SET_ARRAY 2117 } 2118 2119 int 2120 spdk_nvmf_ctrlr_save_migr_data(struct spdk_nvmf_ctrlr *ctrlr, 2121 struct spdk_nvmf_ctrlr_migr_data *data) 2122 { 2123 struct spdk_nvmf_async_event_completion *event, *event_tmp; 2124 uint32_t i; 2125 struct spdk_nvmf_ctrlr_migr_data data_local = { 2126 .data_size = offsetof(struct spdk_nvmf_ctrlr_migr_data, unused), 2127 .regs_size = sizeof(struct spdk_nvmf_registers), 2128 .feat_size = sizeof(struct spdk_nvmf_ctrlr_feat) 2129 }; 2130 2131 assert(data->data_size <= sizeof(data_local)); 2132 assert(spdk_get_thread() == ctrlr->thread); 2133 2134 memcpy(&data_local.regs, &ctrlr->vcprop, sizeof(struct spdk_nvmf_registers)); 2135 memcpy(&data_local.feat, &ctrlr->feat, sizeof(struct spdk_nvmf_ctrlr_feat)); 2136 2137 data_local.cntlid = ctrlr->cntlid; 2138 data_local.acre = ctrlr->acre_enabled; 2139 data_local.num_aer_cids = ctrlr->nr_aer_reqs; 2140 2141 STAILQ_FOREACH_SAFE(event, &ctrlr->async_events, link, event_tmp) { 2142 if (data_local.num_async_events + 1 > SPDK_NVMF_MIGR_MAX_PENDING_AERS) { 2143 SPDK_ERRLOG("ctrlr %p has too many pending AERs\n", ctrlr); 2144 break; 2145 } 2146 2147 data_local.async_events[data_local.num_async_events++].raw = event->event.raw; 2148 } 2149 2150 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 2151 struct spdk_nvmf_request *req = ctrlr->aer_req[i]; 2152 data_local.aer_cids[i] = req->cmd->nvme_cmd.cid; 2153 } 2154 data_local.notice_aen_mask = ctrlr->notice_aen_mask; 2155 2156 nvmf_ctrlr_migr_data_copy(data, &data_local, spdk_min(data->data_size, data_local.data_size)); 2157 return 0; 2158 } 2159 2160 int 2161 spdk_nvmf_ctrlr_restore_migr_data(struct spdk_nvmf_ctrlr *ctrlr, 2162 const struct spdk_nvmf_ctrlr_migr_data *data) 2163 { 2164 uint32_t i; 2165 struct spdk_nvmf_ctrlr_migr_data data_local = { 2166 .data_size = offsetof(struct spdk_nvmf_ctrlr_migr_data, unused), 2167 .regs_size = sizeof(struct spdk_nvmf_registers), 2168 .feat_size = sizeof(struct spdk_nvmf_ctrlr_feat) 2169 }; 2170 2171 assert(data->data_size <= sizeof(data_local)); 2172 assert(spdk_get_thread() == ctrlr->thread); 2173 2174 /* local version of data should have defaults set before copy */ 2175 nvmf_ctrlr_migr_data_copy(&data_local, data, spdk_min(data->data_size, data_local.data_size)); 2176 memcpy(&ctrlr->vcprop, &data_local.regs, sizeof(struct spdk_nvmf_registers)); 2177 memcpy(&ctrlr->feat, &data_local.feat, sizeof(struct spdk_nvmf_ctrlr_feat)); 2178 2179 ctrlr->cntlid = data_local.cntlid; 2180 ctrlr->acre_enabled = data_local.acre; 2181 2182 for (i = 0; i < data_local.num_async_events; i++) { 2183 struct spdk_nvmf_async_event_completion *event; 2184 2185 event = calloc(1, sizeof(*event)); 2186 if (!event) { 2187 return -ENOMEM; 2188 } 2189 2190 event->event.raw = data_local.async_events[i].raw; 2191 STAILQ_INSERT_TAIL(&ctrlr->async_events, event, link); 2192 } 2193 ctrlr->notice_aen_mask = data_local.notice_aen_mask; 2194 2195 return 0; 2196 } 2197 2198 static int 2199 nvmf_ctrlr_set_features_async_event_configuration(struct spdk_nvmf_request *req) 2200 { 2201 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2202 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2203 2204 SPDK_DEBUGLOG(nvmf, "Set Features - Async Event Configuration, cdw11 0x%08x\n", 2205 cmd->cdw11); 2206 ctrlr->feat.async_event_configuration.raw = cmd->cdw11; 2207 ctrlr->feat.async_event_configuration.bits.reserved1 = 0; 2208 ctrlr->feat.async_event_configuration.bits.reserved2 = 0; 2209 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2210 } 2211 2212 static int 2213 nvmf_ctrlr_async_event_request(struct spdk_nvmf_request *req) 2214 { 2215 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2216 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 2217 struct spdk_nvmf_async_event_completion *pending_event; 2218 2219 SPDK_DEBUGLOG(nvmf, "Async Event Request\n"); 2220 2221 /* Four asynchronous events are supported for now */ 2222 if (ctrlr->nr_aer_reqs >= SPDK_NVMF_MAX_ASYNC_EVENTS) { 2223 SPDK_DEBUGLOG(nvmf, "AERL exceeded\n"); 2224 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 2225 rsp->status.sc = SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED; 2226 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2227 } 2228 2229 if (!STAILQ_EMPTY(&ctrlr->async_events)) { 2230 pending_event = STAILQ_FIRST(&ctrlr->async_events); 2231 rsp->cdw0 = pending_event->event.raw; 2232 STAILQ_REMOVE(&ctrlr->async_events, pending_event, spdk_nvmf_async_event_completion, link); 2233 free(pending_event); 2234 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2235 } 2236 2237 ctrlr->aer_req[ctrlr->nr_aer_reqs++] = req; 2238 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 2239 } 2240 2241 static void 2242 nvmf_get_firmware_slot_log_page(struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length) 2243 { 2244 struct spdk_nvme_firmware_page fw_page; 2245 size_t copy_len; 2246 struct spdk_iov_xfer ix; 2247 2248 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2249 2250 memset(&fw_page, 0, sizeof(fw_page)); 2251 fw_page.afi.active_slot = 1; 2252 fw_page.afi.next_reset_slot = 0; 2253 spdk_strcpy_pad(fw_page.revision[0], FW_VERSION, sizeof(fw_page.revision[0]), ' '); 2254 2255 if (offset < sizeof(fw_page)) { 2256 copy_len = spdk_min(sizeof(fw_page) - offset, length); 2257 if (copy_len > 0) { 2258 spdk_iov_xfer_from_buf(&ix, (const char *)&fw_page + offset, copy_len); 2259 } 2260 } 2261 } 2262 2263 /* 2264 * Asynchronous Event Mask Bit 2265 */ 2266 enum spdk_nvme_async_event_mask_bit { 2267 /* Mask Namespace Change Notification */ 2268 SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT = 0, 2269 /* Mask Asymmetric Namespace Access Change Notification */ 2270 SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT = 1, 2271 /* Mask Discovery Log Change Notification */ 2272 SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT = 2, 2273 /* Mask Reservation Log Page Available Notification */ 2274 SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT = 3, 2275 /* Mask Error Event */ 2276 SPDK_NVME_ASYNC_EVENT_ERROR_MASK_BIT = 4, 2277 /* 4 - 63 Reserved */ 2278 }; 2279 2280 static inline void 2281 nvmf_ctrlr_unmask_aen(struct spdk_nvmf_ctrlr *ctrlr, 2282 enum spdk_nvme_async_event_mask_bit mask) 2283 { 2284 ctrlr->notice_aen_mask &= ~(1 << mask); 2285 } 2286 2287 static inline bool 2288 nvmf_ctrlr_mask_aen(struct spdk_nvmf_ctrlr *ctrlr, 2289 enum spdk_nvme_async_event_mask_bit mask) 2290 { 2291 if (ctrlr->notice_aen_mask & (1 << mask)) { 2292 return false; 2293 } else { 2294 ctrlr->notice_aen_mask |= (1 << mask); 2295 return true; 2296 } 2297 } 2298 2299 /* we have to use the typedef in the function declaration to appease astyle. */ 2300 typedef enum spdk_nvme_ana_state spdk_nvme_ana_state_t; 2301 2302 static inline spdk_nvme_ana_state_t 2303 nvmf_ctrlr_get_ana_state(struct spdk_nvmf_ctrlr *ctrlr, uint32_t anagrpid) 2304 { 2305 if (!ctrlr->subsys->flags.ana_reporting) { 2306 return SPDK_NVME_ANA_OPTIMIZED_STATE; 2307 } 2308 2309 if (spdk_unlikely(ctrlr->listener == NULL)) { 2310 return SPDK_NVME_ANA_INACCESSIBLE_STATE; 2311 } 2312 2313 assert(anagrpid - 1 < ctrlr->subsys->max_nsid); 2314 return ctrlr->listener->ana_state[anagrpid - 1]; 2315 } 2316 2317 static spdk_nvme_ana_state_t 2318 nvmf_ctrlr_get_ana_state_from_nsid(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid) 2319 { 2320 struct spdk_nvmf_ns *ns; 2321 2322 /* We do not have NVM subsystem specific ANA state. Hence if NSID is either 2323 * SPDK_NVMF_GLOBAL_NS_TAG, invalid, or for inactive namespace, return 2324 * the optimized state. 2325 */ 2326 ns = nvmf_ctrlr_get_ns(ctrlr, nsid); 2327 if (ns == NULL) { 2328 return SPDK_NVME_ANA_OPTIMIZED_STATE; 2329 } 2330 2331 return nvmf_ctrlr_get_ana_state(ctrlr, ns->anagrpid); 2332 } 2333 2334 static void 2335 nvmf_get_error_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iovcnt, 2336 uint64_t offset, uint32_t length, uint32_t rae) 2337 { 2338 if (!rae) { 2339 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ERROR_MASK_BIT); 2340 } 2341 2342 /* TODO: actually fill out log page data */ 2343 } 2344 2345 static void 2346 nvmf_get_ana_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iovcnt, 2347 uint64_t offset, uint32_t length, uint32_t rae, uint32_t rgo) 2348 { 2349 struct spdk_nvme_ana_page ana_hdr; 2350 struct spdk_nvme_ana_group_descriptor ana_desc; 2351 size_t copy_len, copied_len; 2352 uint32_t num_anagrp = 0, anagrpid; 2353 struct spdk_nvmf_ns *ns; 2354 struct spdk_iov_xfer ix; 2355 2356 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2357 2358 if (length == 0) { 2359 goto done; 2360 } 2361 2362 if (offset >= sizeof(ana_hdr)) { 2363 offset -= sizeof(ana_hdr); 2364 } else { 2365 for (anagrpid = 1; anagrpid <= ctrlr->subsys->max_nsid; anagrpid++) { 2366 if (ctrlr->subsys->ana_group[anagrpid - 1] > 0) { 2367 num_anagrp++; 2368 } 2369 } 2370 2371 memset(&ana_hdr, 0, sizeof(ana_hdr)); 2372 2373 ana_hdr.num_ana_group_desc = num_anagrp; 2374 /* TODO: Support Change Count. */ 2375 ana_hdr.change_count = 0; 2376 2377 copy_len = spdk_min(sizeof(ana_hdr) - offset, length); 2378 copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ana_hdr + offset, copy_len); 2379 assert(copied_len == copy_len); 2380 length -= copied_len; 2381 offset = 0; 2382 } 2383 2384 if (length == 0) { 2385 goto done; 2386 } 2387 2388 for (anagrpid = 1; anagrpid <= ctrlr->subsys->max_nsid; anagrpid++) { 2389 if (ctrlr->subsys->ana_group[anagrpid - 1] == 0) { 2390 continue; 2391 } 2392 2393 if (offset >= sizeof(ana_desc)) { 2394 offset -= sizeof(ana_desc); 2395 } else { 2396 memset(&ana_desc, 0, sizeof(ana_desc)); 2397 2398 ana_desc.ana_group_id = anagrpid; 2399 if (rgo) { 2400 ana_desc.num_of_nsid = 0; 2401 } else { 2402 ana_desc.num_of_nsid = ctrlr->subsys->ana_group[anagrpid - 1]; 2403 } 2404 ana_desc.ana_state = nvmf_ctrlr_get_ana_state(ctrlr, anagrpid); 2405 2406 copy_len = spdk_min(sizeof(ana_desc) - offset, length); 2407 copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ana_desc + offset, 2408 copy_len); 2409 assert(copied_len == copy_len); 2410 length -= copied_len; 2411 offset = 0; 2412 2413 if (length == 0) { 2414 goto done; 2415 } 2416 } 2417 2418 if (rgo) { 2419 continue; 2420 } 2421 2422 /* TODO: Revisit here about O(n^2) cost if we have subsystem with 2423 * many namespaces in the future. 2424 */ 2425 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 2426 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 2427 if (ns->anagrpid != anagrpid) { 2428 continue; 2429 } 2430 2431 if (offset >= sizeof(uint32_t)) { 2432 offset -= sizeof(uint32_t); 2433 continue; 2434 } 2435 2436 copy_len = spdk_min(sizeof(uint32_t) - offset, length); 2437 copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ns->nsid + offset, 2438 copy_len); 2439 assert(copied_len == copy_len); 2440 length -= copied_len; 2441 offset = 0; 2442 2443 if (length == 0) { 2444 goto done; 2445 } 2446 } 2447 } 2448 2449 done: 2450 if (!rae) { 2451 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT); 2452 } 2453 } 2454 2455 void 2456 nvmf_ctrlr_ns_changed(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid) 2457 { 2458 uint16_t max_changes = SPDK_COUNTOF(ctrlr->changed_ns_list.ns_list); 2459 uint16_t i; 2460 bool found = false; 2461 2462 for (i = 0; i < ctrlr->changed_ns_list_count; i++) { 2463 if (ctrlr->changed_ns_list.ns_list[i] == nsid) { 2464 /* nsid is already in the list */ 2465 found = true; 2466 break; 2467 } 2468 } 2469 2470 if (!found) { 2471 if (ctrlr->changed_ns_list_count == max_changes) { 2472 /* Out of space - set first entry to FFFFFFFFh and zero-fill the rest. */ 2473 ctrlr->changed_ns_list.ns_list[0] = 0xFFFFFFFFu; 2474 for (i = 1; i < max_changes; i++) { 2475 ctrlr->changed_ns_list.ns_list[i] = 0; 2476 } 2477 } else { 2478 ctrlr->changed_ns_list.ns_list[ctrlr->changed_ns_list_count++] = nsid; 2479 } 2480 } 2481 } 2482 2483 static void 2484 nvmf_get_changed_ns_list_log_page(struct spdk_nvmf_ctrlr *ctrlr, 2485 struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length, uint32_t rae) 2486 { 2487 size_t copy_length; 2488 struct spdk_iov_xfer ix; 2489 2490 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2491 2492 if (offset < sizeof(ctrlr->changed_ns_list)) { 2493 copy_length = spdk_min(length, sizeof(ctrlr->changed_ns_list) - offset); 2494 if (copy_length) { 2495 spdk_iov_xfer_from_buf(&ix, (char *)&ctrlr->changed_ns_list + offset, copy_length); 2496 } 2497 } 2498 2499 /* Clear log page each time it is read */ 2500 ctrlr->changed_ns_list_count = 0; 2501 memset(&ctrlr->changed_ns_list, 0, sizeof(ctrlr->changed_ns_list)); 2502 2503 if (!rae) { 2504 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT); 2505 } 2506 } 2507 2508 /* The structure can be modified if we provide support for other commands in future */ 2509 static const struct spdk_nvme_cmds_and_effect_log_page g_cmds_and_effect_log_page = { 2510 .admin_cmds_supported = { 2511 /* CSUPP, LBCC, NCC, NIC, CCC, CSE */ 2512 /* Get Log Page */ 2513 [SPDK_NVME_OPC_GET_LOG_PAGE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2514 /* Identify */ 2515 [SPDK_NVME_OPC_IDENTIFY] = {1, 0, 0, 0, 0, 0, 0, 0}, 2516 /* Abort */ 2517 [SPDK_NVME_OPC_ABORT] = {1, 0, 0, 0, 0, 0, 0, 0}, 2518 /* Set Features */ 2519 [SPDK_NVME_OPC_SET_FEATURES] = {1, 0, 0, 0, 0, 0, 0, 0}, 2520 /* Get Features */ 2521 [SPDK_NVME_OPC_GET_FEATURES] = {1, 0, 0, 0, 0, 0, 0, 0}, 2522 /* Async Event Request */ 2523 [SPDK_NVME_OPC_ASYNC_EVENT_REQUEST] = {1, 0, 0, 0, 0, 0, 0, 0}, 2524 /* Keep Alive */ 2525 [SPDK_NVME_OPC_KEEP_ALIVE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2526 }, 2527 .io_cmds_supported = { 2528 /* FLUSH */ 2529 [SPDK_NVME_OPC_FLUSH] = {1, 1, 0, 0, 0, 0, 0, 0}, 2530 /* WRITE */ 2531 [SPDK_NVME_OPC_WRITE] = {1, 1, 0, 0, 0, 0, 0, 0}, 2532 /* READ */ 2533 [SPDK_NVME_OPC_READ] = {1, 0, 0, 0, 0, 0, 0, 0}, 2534 /* WRITE ZEROES */ 2535 [SPDK_NVME_OPC_WRITE_ZEROES] = {1, 1, 0, 0, 0, 0, 0, 0}, 2536 /* DATASET MANAGEMENT */ 2537 [SPDK_NVME_OPC_DATASET_MANAGEMENT] = {1, 1, 0, 0, 0, 0, 0, 0}, 2538 /* COMPARE */ 2539 [SPDK_NVME_OPC_COMPARE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2540 /* ZONE MANAGEMENT SEND */ 2541 [SPDK_NVME_OPC_ZONE_MGMT_SEND] = {1, 1, 0, 0, 0, 0, 0, 0}, 2542 /* ZONE MANAGEMENT RECEIVE */ 2543 [SPDK_NVME_OPC_ZONE_MGMT_RECV] = {1, 0, 0, 0, 0, 0, 0, 0}, 2544 /* ZONE APPEND */ 2545 [SPDK_NVME_OPC_ZONE_APPEND] = {1, 1, 0, 0, 0, 0, 0, 0}, 2546 /* COPY */ 2547 [SPDK_NVME_OPC_COPY] = {1, 1, 0, 0, 0, 0, 0, 0}, 2548 }, 2549 }; 2550 2551 static void 2552 nvmf_get_cmds_and_effects_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iovcnt, 2553 uint64_t offset, uint32_t length) 2554 { 2555 uint32_t page_size = sizeof(struct spdk_nvme_cmds_and_effect_log_page); 2556 size_t copy_len = 0; 2557 struct spdk_nvme_cmds_and_effect_log_page cmds_and_effect_log_page = g_cmds_and_effect_log_page; 2558 struct spdk_nvme_cmds_and_effect_entry zero = {}; 2559 struct spdk_iov_xfer ix; 2560 2561 if (!ctrlr->cdata.oncs.write_zeroes || !nvmf_ctrlr_write_zeroes_supported(ctrlr)) { 2562 cmds_and_effect_log_page.io_cmds_supported[SPDK_NVME_OPC_WRITE_ZEROES] = zero; 2563 } 2564 if (!ctrlr->cdata.oncs.dsm || !nvmf_ctrlr_dsm_supported(ctrlr)) { 2565 cmds_and_effect_log_page.io_cmds_supported[SPDK_NVME_OPC_DATASET_MANAGEMENT] = zero; 2566 } 2567 if (!ctrlr->cdata.oncs.compare) { 2568 cmds_and_effect_log_page.io_cmds_supported[SPDK_NVME_OPC_COMPARE] = zero; 2569 } 2570 if (!nvmf_subsystem_has_zns_iocs(ctrlr->subsys)) { 2571 cmds_and_effect_log_page.io_cmds_supported[SPDK_NVME_OPC_ZONE_MGMT_SEND] = zero; 2572 cmds_and_effect_log_page.io_cmds_supported[SPDK_NVME_OPC_ZONE_MGMT_RECV] = zero; 2573 } 2574 if (!nvmf_subsystem_zone_append_supported(ctrlr->subsys)) { 2575 cmds_and_effect_log_page.io_cmds_supported[SPDK_NVME_OPC_ZONE_APPEND] = zero; 2576 } 2577 if (!ctrlr->cdata.oncs.copy) { 2578 cmds_and_effect_log_page.io_cmds_supported[SPDK_NVME_OPC_COPY] = zero; 2579 } 2580 2581 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2582 if (offset < page_size) { 2583 copy_len = spdk_min(page_size - offset, length); 2584 spdk_iov_xfer_from_buf(&ix, (char *)(&cmds_and_effect_log_page) + offset, copy_len); 2585 } 2586 } 2587 2588 static void 2589 nvmf_get_reservation_notification_log_page(struct spdk_nvmf_ctrlr *ctrlr, 2590 struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length, uint32_t rae) 2591 { 2592 uint32_t unit_log_len, avail_log_len, next_pos, copy_len; 2593 struct spdk_nvmf_reservation_log *log, *log_tmp; 2594 struct spdk_iov_xfer ix; 2595 2596 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2597 2598 unit_log_len = sizeof(struct spdk_nvme_reservation_notification_log); 2599 /* No available log, return zeroed log pages */ 2600 if (!ctrlr->num_avail_log_pages) { 2601 return; 2602 } 2603 2604 avail_log_len = ctrlr->num_avail_log_pages * unit_log_len; 2605 if (offset >= avail_log_len) { 2606 return; 2607 } 2608 2609 next_pos = 0; 2610 TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) { 2611 TAILQ_REMOVE(&ctrlr->log_head, log, link); 2612 ctrlr->num_avail_log_pages--; 2613 2614 next_pos += unit_log_len; 2615 if (next_pos > offset) { 2616 copy_len = spdk_min(next_pos - offset, length); 2617 spdk_iov_xfer_from_buf(&ix, &log->log, copy_len); 2618 length -= copy_len; 2619 offset += copy_len; 2620 } 2621 free(log); 2622 2623 if (length == 0) { 2624 break; 2625 } 2626 } 2627 2628 if (!rae) { 2629 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT); 2630 } 2631 return; 2632 } 2633 2634 static int 2635 nvmf_ctrlr_get_log_page(struct spdk_nvmf_request *req) 2636 { 2637 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2638 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2639 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2640 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 2641 struct spdk_nvme_transport_id cmd_source_trid; 2642 uint64_t offset, len; 2643 uint32_t rae, numdl, numdu; 2644 uint8_t lid; 2645 2646 if (req->iovcnt < 1) { 2647 SPDK_DEBUGLOG(nvmf, "get log command with no buffer\n"); 2648 response->status.sct = SPDK_NVME_SCT_GENERIC; 2649 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2650 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2651 } 2652 2653 offset = (uint64_t)cmd->cdw12 | ((uint64_t)cmd->cdw13 << 32); 2654 if (offset & 3) { 2655 SPDK_ERRLOG("Invalid log page offset 0x%" PRIx64 "\n", offset); 2656 response->status.sct = SPDK_NVME_SCT_GENERIC; 2657 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2658 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2659 } 2660 2661 rae = cmd->cdw10_bits.get_log_page.rae; 2662 numdl = cmd->cdw10_bits.get_log_page.numdl; 2663 numdu = cmd->cdw11_bits.get_log_page.numdu; 2664 len = ((numdu << 16) + numdl + (uint64_t)1) * 4; 2665 if (len > req->length) { 2666 SPDK_ERRLOG("Get log page: len (%" PRIu64 ") > buf size (%u)\n", 2667 len, req->length); 2668 response->status.sct = SPDK_NVME_SCT_GENERIC; 2669 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2670 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2671 } 2672 2673 lid = cmd->cdw10_bits.get_log_page.lid; 2674 SPDK_DEBUGLOG(nvmf, "Get log page: LID=0x%02X offset=0x%" PRIx64 " len=0x%" PRIx64 " rae=%u\n", 2675 lid, offset, len, rae); 2676 2677 if (spdk_nvmf_subsystem_is_discovery(subsystem)) { 2678 switch (lid) { 2679 case SPDK_NVME_LOG_DISCOVERY: 2680 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &cmd_source_trid)) { 2681 SPDK_ERRLOG("Failed to get LOG_DISCOVERY source trid\n"); 2682 response->status.sct = SPDK_NVME_SCT_GENERIC; 2683 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 2684 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2685 } 2686 nvmf_get_discovery_log_page(subsystem->tgt, ctrlr->hostnqn, req->iov, req->iovcnt, 2687 offset, len, &cmd_source_trid); 2688 if (!rae) { 2689 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT); 2690 } 2691 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2692 default: 2693 goto invalid_log_page; 2694 } 2695 } else { 2696 if (offset > len) { 2697 SPDK_ERRLOG("Get log page: offset (%" PRIu64 ") > len (%" PRIu64 ")\n", 2698 offset, len); 2699 response->status.sct = SPDK_NVME_SCT_GENERIC; 2700 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2701 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2702 } 2703 2704 switch (lid) { 2705 case SPDK_NVME_LOG_ERROR: 2706 nvmf_get_error_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2707 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2708 case SPDK_NVME_LOG_HEALTH_INFORMATION: 2709 /* TODO: actually fill out log page data */ 2710 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2711 case SPDK_NVME_LOG_FIRMWARE_SLOT: 2712 nvmf_get_firmware_slot_log_page(req->iov, req->iovcnt, offset, len); 2713 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2714 case SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS: 2715 if (subsystem->flags.ana_reporting) { 2716 uint32_t rgo = cmd->cdw10_bits.get_log_page.lsp & 1; 2717 nvmf_get_ana_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae, rgo); 2718 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2719 } else { 2720 goto invalid_log_page; 2721 } 2722 case SPDK_NVME_LOG_COMMAND_EFFECTS_LOG: 2723 nvmf_get_cmds_and_effects_log_page(ctrlr, req->iov, req->iovcnt, offset, len); 2724 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2725 case SPDK_NVME_LOG_CHANGED_NS_LIST: 2726 nvmf_get_changed_ns_list_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2727 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2728 case SPDK_NVME_LOG_RESERVATION_NOTIFICATION: 2729 nvmf_get_reservation_notification_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2730 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2731 default: 2732 goto invalid_log_page; 2733 } 2734 } 2735 2736 invalid_log_page: 2737 SPDK_INFOLOG(nvmf, "Unsupported Get Log Page 0x%02X\n", lid); 2738 response->status.sct = SPDK_NVME_SCT_GENERIC; 2739 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2740 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2741 } 2742 2743 static struct spdk_nvmf_ns * 2744 _nvmf_ctrlr_get_ns_safe(struct spdk_nvmf_ctrlr *ctrlr, 2745 uint32_t nsid, 2746 struct spdk_nvme_cpl *rsp) 2747 { 2748 struct spdk_nvmf_ns *ns; 2749 if (nsid == 0 || nsid > ctrlr->subsys->max_nsid) { 2750 SPDK_ERRLOG("Identify Namespace for invalid NSID %u\n", nsid); 2751 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2752 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 2753 return NULL; 2754 } 2755 2756 ns = nvmf_ctrlr_get_ns(ctrlr, nsid); 2757 if (ns == NULL || ns->bdev == NULL) { 2758 /* 2759 * Inactive namespaces should return a zero filled data structure. 2760 * The data buffer is already zeroed by nvmf_ctrlr_process_admin_cmd(), 2761 * so we can just return early here. 2762 */ 2763 SPDK_DEBUGLOG(nvmf, "Identify Namespace for inactive NSID %u\n", nsid); 2764 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2765 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2766 return NULL; 2767 } 2768 return ns; 2769 } 2770 2771 static void 2772 nvmf_ctrlr_identify_ns(struct spdk_nvmf_ctrlr *ctrlr, 2773 struct spdk_nvme_cmd *cmd, 2774 struct spdk_nvme_cpl *rsp, 2775 struct spdk_nvme_ns_data *nsdata, 2776 uint32_t nsid) 2777 { 2778 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2779 struct spdk_nvmf_ns *ns; 2780 uint32_t max_num_blocks, format_index; 2781 enum spdk_nvme_ana_state ana_state; 2782 2783 ns = _nvmf_ctrlr_get_ns_safe(ctrlr, nsid, rsp); 2784 if (ns == NULL) { 2785 return; 2786 } 2787 2788 nvmf_bdev_ctrlr_identify_ns(ns, nsdata, ctrlr->dif_insert_or_strip); 2789 2790 assert(ctrlr->admin_qpair); 2791 2792 format_index = spdk_nvme_ns_get_format_index(nsdata); 2793 2794 /* Due to bug in the Linux kernel NVMe driver we have to set noiob no larger than mdts */ 2795 max_num_blocks = ctrlr->admin_qpair->transport->opts.max_io_size / 2796 (1U << nsdata->lbaf[format_index].lbads); 2797 if (nsdata->noiob > max_num_blocks) { 2798 nsdata->noiob = max_num_blocks; 2799 } 2800 2801 /* Set NOWS equal to Controller MDTS */ 2802 if (nsdata->nsfeat.optperf) { 2803 nsdata->nows = max_num_blocks - 1; 2804 } 2805 2806 if (subsystem->flags.ana_reporting) { 2807 assert(ns->anagrpid - 1 < subsystem->max_nsid); 2808 nsdata->anagrpid = ns->anagrpid; 2809 2810 ana_state = nvmf_ctrlr_get_ana_state(ctrlr, ns->anagrpid); 2811 if (ana_state == SPDK_NVME_ANA_INACCESSIBLE_STATE || 2812 ana_state == SPDK_NVME_ANA_PERSISTENT_LOSS_STATE) { 2813 nsdata->nuse = 0; 2814 } 2815 } 2816 } 2817 2818 int 2819 spdk_nvmf_ctrlr_identify_ns(struct spdk_nvmf_ctrlr *ctrlr, 2820 struct spdk_nvme_cmd *cmd, 2821 struct spdk_nvme_cpl *rsp, 2822 struct spdk_nvme_ns_data *nsdata) 2823 { 2824 nvmf_ctrlr_identify_ns(ctrlr, cmd, rsp, nsdata, cmd->nsid); 2825 2826 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2827 } 2828 2829 static void 2830 identify_ns_passthru_cb(struct spdk_nvmf_request *req) 2831 { 2832 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req); 2833 struct spdk_nvmf_ctrlr *ctrlr = spdk_nvmf_request_get_ctrlr(req); 2834 struct spdk_nvme_cpl *rsp = spdk_nvmf_request_get_response(req); 2835 struct spdk_nvme_ns_data nvmf_nsdata = {}; 2836 struct spdk_nvme_ns_data nvme_nsdata = {}; 2837 size_t datalen; 2838 2839 /* This is the identify data from the NVMe drive */ 2840 datalen = spdk_nvmf_request_copy_to_buf(req, &nvme_nsdata, 2841 sizeof(nvme_nsdata)); 2842 nvmf_ctrlr_identify_ns(ctrlr, cmd, rsp, &nvmf_nsdata, req->orig_nsid); 2843 2844 /* Update fabric's namespace according to SSD's namespace */ 2845 if (nvme_nsdata.nsfeat.optperf) { 2846 nvmf_nsdata.nsfeat.optperf = nvme_nsdata.nsfeat.optperf; 2847 nvmf_nsdata.npwg = nvme_nsdata.npwg; 2848 nvmf_nsdata.npwa = nvme_nsdata.npwa; 2849 nvmf_nsdata.npdg = nvme_nsdata.npdg; 2850 nvmf_nsdata.npda = nvme_nsdata.npda; 2851 nvmf_nsdata.nows = nvme_nsdata.nows; 2852 } 2853 2854 if (nvme_nsdata.nsfeat.ns_atomic_write_unit) { 2855 nvmf_nsdata.nsfeat.ns_atomic_write_unit = nvme_nsdata.nsfeat.ns_atomic_write_unit; 2856 nvmf_nsdata.nawun = nvme_nsdata.nawun; 2857 nvmf_nsdata.nawupf = nvme_nsdata.nawupf; 2858 nvmf_nsdata.nacwu = nvme_nsdata.nacwu; 2859 } 2860 2861 nvmf_nsdata.nabsn = nvme_nsdata.nabsn; 2862 nvmf_nsdata.nabo = nvme_nsdata.nabo; 2863 nvmf_nsdata.nabspf = nvme_nsdata.nabspf; 2864 2865 spdk_nvmf_request_copy_from_buf(req, &nvmf_nsdata, datalen); 2866 } 2867 2868 int 2869 spdk_nvmf_ctrlr_identify_ns_ext(struct spdk_nvmf_request *req) 2870 { 2871 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req); 2872 struct spdk_nvmf_ctrlr *ctrlr = spdk_nvmf_request_get_ctrlr(req); 2873 struct spdk_nvmf_ns *ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 2874 struct spdk_nvme_cpl *rsp = spdk_nvmf_request_get_response(req); 2875 struct spdk_bdev *bdev; 2876 struct spdk_bdev_desc *desc; 2877 struct spdk_io_channel *ch; 2878 struct spdk_nvme_ns_data nsdata = {}; 2879 struct spdk_iov_xfer ix; 2880 int rc; 2881 2882 nvmf_ctrlr_identify_ns(ctrlr, cmd, rsp, &nsdata, cmd->nsid); 2883 2884 rc = spdk_nvmf_request_get_bdev(cmd->nsid, req, &bdev, &desc, &ch); 2885 if (rc) { 2886 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2887 } 2888 2889 if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_NVME_ADMIN)) { 2890 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 2891 spdk_iov_xfer_from_buf(&ix, &nsdata, sizeof(nsdata)); 2892 2893 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2894 } 2895 2896 assert(ns->passthru_nsid != 0); 2897 req->orig_nsid = ns->nsid; 2898 cmd->nsid = ns->passthru_nsid; 2899 2900 return spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(bdev, desc, ch, req, identify_ns_passthru_cb); 2901 } 2902 2903 static void 2904 nvmf_ctrlr_populate_oacs(struct spdk_nvmf_ctrlr *ctrlr, 2905 struct spdk_nvme_ctrlr_data *cdata) 2906 { 2907 cdata->oacs = ctrlr->cdata.oacs; 2908 2909 cdata->oacs.virtualization_management = 2910 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_VIRTUALIZATION_MANAGEMENT].hdlr != NULL; 2911 cdata->oacs.nvme_mi = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NVME_MI_SEND].hdlr != NULL 2912 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NVME_MI_RECEIVE].hdlr != NULL; 2913 cdata->oacs.directives = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DIRECTIVE_SEND].hdlr != NULL 2914 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DIRECTIVE_RECEIVE].hdlr != NULL; 2915 cdata->oacs.device_self_test = 2916 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DEVICE_SELF_TEST].hdlr != NULL; 2917 cdata->oacs.ns_manage = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NS_MANAGEMENT].hdlr != NULL 2918 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NS_ATTACHMENT].hdlr != NULL; 2919 cdata->oacs.firmware = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD].hdlr != 2920 NULL 2921 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FIRMWARE_COMMIT].hdlr != NULL; 2922 cdata->oacs.format = 2923 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FORMAT_NVM].hdlr != NULL; 2924 cdata->oacs.security = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_SECURITY_SEND].hdlr != NULL 2925 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_SECURITY_RECEIVE].hdlr != NULL; 2926 cdata->oacs.get_lba_status = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_GET_LBA_STATUS].hdlr != 2927 NULL; 2928 } 2929 2930 int 2931 spdk_nvmf_ctrlr_identify_ctrlr(struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvme_ctrlr_data *cdata) 2932 { 2933 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2934 struct spdk_nvmf_transport *transport; 2935 2936 /* 2937 * Common fields for discovery and NVM subsystems 2938 */ 2939 assert(ctrlr->admin_qpair); 2940 transport = ctrlr->admin_qpair->transport; 2941 spdk_strcpy_pad(cdata->fr, FW_VERSION, sizeof(cdata->fr), ' '); 2942 assert((transport->opts.max_io_size % 4096) == 0); 2943 cdata->mdts = spdk_u32log2(transport->opts.max_io_size / 4096); 2944 cdata->cntlid = ctrlr->cntlid; 2945 cdata->ver = ctrlr->vcprop.vs; 2946 cdata->aerl = ctrlr->cdata.aerl; 2947 cdata->lpa.edlp = 1; 2948 cdata->elpe = 127; 2949 cdata->maxcmd = transport->opts.max_queue_depth; 2950 cdata->sgls = ctrlr->cdata.sgls; 2951 cdata->fuses = ctrlr->cdata.fuses; 2952 cdata->acwu = 0; /* ACWU is 0-based. */ 2953 if (subsystem->flags.ana_reporting) { 2954 cdata->mnan = subsystem->max_nsid; 2955 } 2956 spdk_strcpy_pad(cdata->subnqn, subsystem->subnqn, sizeof(cdata->subnqn), '\0'); 2957 2958 SPDK_DEBUGLOG(nvmf, "ctrlr data: maxcmd 0x%x\n", cdata->maxcmd); 2959 SPDK_DEBUGLOG(nvmf, "sgls data: 0x%x\n", from_le32(&cdata->sgls)); 2960 2961 2962 if (spdk_nvmf_subsystem_is_discovery(subsystem)) { 2963 /* 2964 * NVM Discovery subsystem fields 2965 */ 2966 cdata->oaes.discovery_log_change_notices = 1; 2967 cdata->cntrltype = SPDK_NVME_CTRLR_DISCOVERY; 2968 } else { 2969 cdata->vid = ctrlr->cdata.vid; 2970 cdata->ssvid = ctrlr->cdata.ssvid; 2971 cdata->ieee[0] = ctrlr->cdata.ieee[0]; 2972 cdata->ieee[1] = ctrlr->cdata.ieee[1]; 2973 cdata->ieee[2] = ctrlr->cdata.ieee[2]; 2974 2975 /* 2976 * NVM subsystem fields (reserved for discovery subsystems) 2977 */ 2978 spdk_strcpy_pad(cdata->mn, spdk_nvmf_subsystem_get_mn(subsystem), sizeof(cdata->mn), ' '); 2979 spdk_strcpy_pad(cdata->sn, spdk_nvmf_subsystem_get_sn(subsystem), sizeof(cdata->sn), ' '); 2980 cdata->kas = ctrlr->cdata.kas; 2981 2982 cdata->rab = 6; 2983 cdata->cmic.multi_port = 1; 2984 cdata->cmic.multi_ctrlr = 1; 2985 cdata->oaes.ns_attribute_notices = 1; 2986 cdata->ctratt.bits.host_id_exhid_supported = 1; 2987 cdata->ctratt.bits.fdps = ctrlr->subsys->fdp_supported; 2988 cdata->cntrltype = SPDK_NVME_CTRLR_IO; 2989 /* We do not have any actual limitation to the number of abort commands. 2990 * We follow the recommendation by the NVMe specification. 2991 */ 2992 cdata->acl = NVMF_ABORT_COMMAND_LIMIT; 2993 cdata->frmw.slot1_ro = 1; 2994 cdata->frmw.num_slots = 1; 2995 2996 cdata->lpa.celp = 1; /* Command Effects log page supported */ 2997 2998 cdata->sqes.min = 6; 2999 cdata->sqes.max = 6; 3000 cdata->cqes.min = 4; 3001 cdata->cqes.max = 4; 3002 cdata->nn = subsystem->max_nsid; 3003 cdata->vwc.present = 1; 3004 cdata->vwc.flush_broadcast = SPDK_NVME_FLUSH_BROADCAST_NOT_SUPPORTED; 3005 3006 cdata->nvmf_specific = ctrlr->cdata.nvmf_specific; 3007 3008 cdata->oncs.compare = ctrlr->cdata.oncs.compare; 3009 cdata->oncs.dsm = ctrlr->cdata.oncs.dsm && nvmf_ctrlr_dsm_supported(ctrlr); 3010 cdata->oncs.write_zeroes = ctrlr->cdata.oncs.write_zeroes && 3011 nvmf_ctrlr_write_zeroes_supported(ctrlr); 3012 cdata->oncs.reservations = ctrlr->cdata.oncs.reservations; 3013 cdata->oncs.copy = ctrlr->cdata.oncs.copy; 3014 cdata->ocfs.copy_format0 = cdata->oncs.copy; 3015 if (subsystem->flags.ana_reporting) { 3016 /* Asymmetric Namespace Access Reporting is supported. */ 3017 cdata->cmic.ana_reporting = 1; 3018 cdata->oaes.ana_change_notices = 1; 3019 3020 cdata->anatt = ANA_TRANSITION_TIME_IN_SEC; 3021 /* ANA Change state is not used, and ANA Persistent Loss state 3022 * is not supported for now. 3023 */ 3024 cdata->anacap.ana_optimized_state = 1; 3025 cdata->anacap.ana_non_optimized_state = 1; 3026 cdata->anacap.ana_inaccessible_state = 1; 3027 /* ANAGRPID does not change while namespace is attached to controller */ 3028 cdata->anacap.no_change_anagrpid = 1; 3029 cdata->anagrpmax = subsystem->max_nsid; 3030 cdata->nanagrpid = subsystem->max_nsid; 3031 } 3032 3033 nvmf_ctrlr_populate_oacs(ctrlr, cdata); 3034 3035 assert(subsystem->tgt != NULL); 3036 cdata->crdt[0] = subsystem->tgt->crdt[0]; 3037 cdata->crdt[1] = subsystem->tgt->crdt[1]; 3038 cdata->crdt[2] = subsystem->tgt->crdt[2]; 3039 3040 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: ioccsz 0x%x\n", 3041 cdata->nvmf_specific.ioccsz); 3042 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: iorcsz 0x%x\n", 3043 cdata->nvmf_specific.iorcsz); 3044 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: icdoff 0x%x\n", 3045 cdata->nvmf_specific.icdoff); 3046 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: ctrattr 0x%x\n", 3047 *(uint8_t *)&cdata->nvmf_specific.ctrattr); 3048 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: msdbd 0x%x\n", 3049 cdata->nvmf_specific.msdbd); 3050 } 3051 3052 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3053 } 3054 3055 static int 3056 nvmf_ns_identify_iocs_zns(struct spdk_nvmf_ns *ns, 3057 struct spdk_nvme_cmd *cmd, 3058 struct spdk_nvme_cpl *rsp, 3059 struct spdk_nvme_zns_ns_data *nsdata_zns) 3060 { 3061 nsdata_zns->zoc.variable_zone_capacity = 0; 3062 nsdata_zns->zoc.zone_active_excursions = 0; 3063 nsdata_zns->ozcs.read_across_zone_boundaries = 1; 3064 /* Underflowing the zero based mar and mor bdev helper results in the correct 3065 value of FFFFFFFFh. */ 3066 nsdata_zns->mar = spdk_bdev_get_max_active_zones(ns->bdev) - 1; 3067 nsdata_zns->mor = spdk_bdev_get_max_open_zones(ns->bdev) - 1; 3068 nsdata_zns->rrl = 0; 3069 nsdata_zns->frl = 0; 3070 nsdata_zns->lbafe[0].zsze = spdk_bdev_get_zone_size(ns->bdev); 3071 3072 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3073 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3074 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3075 } 3076 3077 static int 3078 nvmf_ns_identify_iocs_nvm(struct spdk_nvmf_ns *ns, 3079 struct spdk_nvme_cpl *rsp, 3080 struct spdk_nvme_nvm_ns_data *nsdata_nvm) 3081 { 3082 nvmf_bdev_ctrlr_identify_iocs_nvm(ns, nsdata_nvm); 3083 3084 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3085 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3086 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3087 } 3088 3089 int 3090 spdk_nvmf_ns_identify_iocs_specific(struct spdk_nvmf_ctrlr *ctrlr, 3091 struct spdk_nvme_cmd *cmd, 3092 struct spdk_nvme_cpl *rsp, 3093 void *nsdata, 3094 size_t nsdata_size) 3095 { 3096 uint8_t csi = cmd->cdw11_bits.identify.csi; 3097 struct spdk_nvmf_ns *ns = _nvmf_ctrlr_get_ns_safe(ctrlr, cmd->nsid, rsp); 3098 3099 memset(nsdata, 0, nsdata_size); 3100 3101 if (ns == NULL) { 3102 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3103 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 3104 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3105 } 3106 3107 switch (csi) { 3108 case SPDK_NVME_CSI_ZNS: 3109 return nvmf_ns_identify_iocs_zns(ns, cmd, rsp, nsdata); 3110 case SPDK_NVME_CSI_NVM: 3111 if (!ctrlr->dif_insert_or_strip) { 3112 return nvmf_ns_identify_iocs_nvm(ns, rsp, nsdata); 3113 } 3114 break; 3115 default: 3116 break; 3117 } 3118 3119 SPDK_DEBUGLOG(nvmf, 3120 "Returning zero filled struct for the iocs specific ns " 3121 "identify command and CSI 0x%02x\n", 3122 csi); 3123 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3124 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3125 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3126 } 3127 3128 static int 3129 nvmf_ctrlr_identify_iocs_nvm(struct spdk_nvmf_ctrlr *ctrlr, 3130 struct spdk_nvme_cmd *cmd, 3131 struct spdk_nvme_cpl *rsp, 3132 struct spdk_nvme_nvm_ctrlr_data *cdata_nvm) 3133 { 3134 /* The unit of max_write_zeroes_size_kib is KiB. 3135 * The unit of wzsl is the minimum memory page size(2 ^ (12 + CAP.MPSMIN) bytes) 3136 * and is reported as a power of two (2^n). 3137 */ 3138 cdata_nvm->wzsl = spdk_u64log2(ctrlr->subsys->max_write_zeroes_size_kib >> 3139 (2 + ctrlr->vcprop.cap.bits.mpsmin)); 3140 3141 /* The unit of max_discard_size_kib is KiB. 3142 * The dmrsl indicates the maximum number of logical blocks for 3143 * dataset management command. 3144 */ 3145 cdata_nvm->dmrsl = ctrlr->subsys->max_discard_size_kib << 1; 3146 cdata_nvm->dmrl = 1; 3147 3148 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3149 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3150 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3151 } 3152 3153 static int 3154 nvmf_ctrlr_identify_iocs_zns(struct spdk_nvmf_ctrlr *ctrlr, 3155 struct spdk_nvme_cmd *cmd, 3156 struct spdk_nvme_cpl *rsp, 3157 struct spdk_nvme_zns_ctrlr_data *cdata_zns) 3158 { 3159 /* The unit of max_zone_append_size_kib is KiB. 3160 The unit of zasl is the minimum memory page size 3161 (2 ^ (12 + CAP.MPSMIN) KiB) 3162 and is reported as a power of two (2^n). */ 3163 cdata_zns->zasl = spdk_u64log2(ctrlr->subsys->max_zone_append_size_kib >> 3164 (12 + ctrlr->vcprop.cap.bits.mpsmin)); 3165 3166 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3167 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3168 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3169 } 3170 3171 int 3172 spdk_nvmf_ctrlr_identify_iocs_specific(struct spdk_nvmf_ctrlr *ctrlr, 3173 struct spdk_nvme_cmd *cmd, 3174 struct spdk_nvme_cpl *rsp, 3175 void *cdata, 3176 size_t cdata_size) 3177 { 3178 uint8_t csi = cmd->cdw11_bits.identify.csi; 3179 3180 memset(cdata, 0, cdata_size); 3181 3182 switch (csi) { 3183 case SPDK_NVME_CSI_NVM: 3184 return nvmf_ctrlr_identify_iocs_nvm(ctrlr, cmd, rsp, cdata); 3185 case SPDK_NVME_CSI_ZNS: 3186 return nvmf_ctrlr_identify_iocs_zns(ctrlr, cmd, rsp, cdata); 3187 default: 3188 break; 3189 } 3190 3191 SPDK_DEBUGLOG(nvmf, 3192 "Returning zero filled struct for the iocs specific ctrlr " 3193 "identify command and CSI 0x%02x\n", 3194 csi); 3195 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3196 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3197 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3198 } 3199 3200 static int 3201 nvmf_ctrlr_identify_active_ns_list(struct spdk_nvmf_ctrlr *ctrlr, 3202 struct spdk_nvme_cmd *cmd, 3203 struct spdk_nvme_cpl *rsp, 3204 struct spdk_nvme_ns_list *ns_list) 3205 { 3206 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 3207 struct spdk_nvmf_ns *ns; 3208 uint32_t count = 0; 3209 3210 if (cmd->nsid >= 0xfffffffeUL) { 3211 SPDK_ERRLOG("Identify Active Namespace List with invalid NSID %u\n", cmd->nsid); 3212 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 3213 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3214 } 3215 3216 memset(ns_list, 0, sizeof(*ns_list)); 3217 3218 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 3219 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 3220 if (ns->opts.nsid <= cmd->nsid || !nvmf_ctrlr_ns_is_visible(ctrlr, ns->opts.nsid)) { 3221 continue; 3222 } 3223 3224 ns_list->ns_list[count++] = ns->opts.nsid; 3225 if (count == SPDK_COUNTOF(ns_list->ns_list)) { 3226 break; 3227 } 3228 } 3229 3230 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3231 } 3232 3233 static void 3234 _add_ns_id_desc(void **buf_ptr, size_t *buf_remain, 3235 enum spdk_nvme_nidt type, 3236 const void *data, size_t data_size) 3237 { 3238 struct spdk_nvme_ns_id_desc *desc; 3239 size_t desc_size = sizeof(*desc) + data_size; 3240 3241 /* 3242 * These should never fail in practice, since all valid NS ID descriptors 3243 * should be defined so that they fit in the available 4096-byte buffer. 3244 */ 3245 assert(data_size > 0); 3246 assert(data_size <= UINT8_MAX); 3247 assert(desc_size < *buf_remain); 3248 if (data_size == 0 || data_size > UINT8_MAX || desc_size > *buf_remain) { 3249 return; 3250 } 3251 3252 desc = *buf_ptr; 3253 desc->nidt = type; 3254 desc->nidl = data_size; 3255 memcpy(desc->nid, data, data_size); 3256 3257 *buf_ptr += desc_size; 3258 *buf_remain -= desc_size; 3259 } 3260 3261 static int 3262 nvmf_ctrlr_identify_ns_id_descriptor_list( 3263 struct spdk_nvmf_ctrlr *ctrlr, 3264 struct spdk_nvme_cmd *cmd, 3265 struct spdk_nvme_cpl *rsp, 3266 void *id_desc_list, size_t id_desc_list_size) 3267 { 3268 struct spdk_nvmf_ns *ns; 3269 size_t buf_remain = id_desc_list_size; 3270 void *buf_ptr = id_desc_list; 3271 3272 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 3273 if (ns == NULL || ns->bdev == NULL) { 3274 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3275 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 3276 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3277 } 3278 3279 #define ADD_ID_DESC(type, data, size) \ 3280 do { \ 3281 if (!spdk_mem_all_zero(data, size)) { \ 3282 _add_ns_id_desc(&buf_ptr, &buf_remain, type, data, size); \ 3283 } \ 3284 } while (0) 3285 3286 ADD_ID_DESC(SPDK_NVME_NIDT_EUI64, ns->opts.eui64, sizeof(ns->opts.eui64)); 3287 ADD_ID_DESC(SPDK_NVME_NIDT_NGUID, ns->opts.nguid, sizeof(ns->opts.nguid)); 3288 ADD_ID_DESC(SPDK_NVME_NIDT_UUID, &ns->opts.uuid, sizeof(ns->opts.uuid)); 3289 ADD_ID_DESC(SPDK_NVME_NIDT_CSI, &ns->csi, sizeof(uint8_t)); 3290 3291 /* 3292 * The list is automatically 0-terminated, both in the temporary buffer 3293 * used by nvmf_ctrlr_identify(), and the eventual iov destination - 3294 * controller to host buffers in admin commands always get zeroed in 3295 * nvmf_ctrlr_process_admin_cmd(). 3296 */ 3297 3298 #undef ADD_ID_DESC 3299 3300 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3301 } 3302 3303 static int 3304 nvmf_ctrlr_identify_iocs(struct spdk_nvmf_ctrlr *ctrlr, 3305 struct spdk_nvme_cmd *cmd, 3306 struct spdk_nvme_cpl *rsp, 3307 void *cdata, size_t cdata_size) 3308 { 3309 struct spdk_nvme_iocs_vector *vector; 3310 struct spdk_nvmf_ns *ns; 3311 3312 if (cdata_size < sizeof(struct spdk_nvme_iocs_vector)) { 3313 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3314 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3315 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3316 } 3317 3318 /* For now we only support this command sent to the current 3319 * controller. 3320 */ 3321 if (cmd->cdw10_bits.identify.cntid != 0xFFFF && 3322 cmd->cdw10_bits.identify.cntid != ctrlr->cntlid) { 3323 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3324 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3325 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3326 } 3327 memset(cdata, 0, cdata_size); 3328 3329 vector = cdata; 3330 vector->nvm = 1; 3331 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 3332 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 3333 if (ns->bdev == NULL) { 3334 continue; 3335 } 3336 if (spdk_bdev_is_zoned(ns->bdev)) { 3337 vector->zns = 1; 3338 } 3339 } 3340 3341 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3342 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3343 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3344 } 3345 3346 static int 3347 nvmf_ctrlr_identify(struct spdk_nvmf_request *req) 3348 { 3349 uint8_t cns; 3350 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3351 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3352 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3353 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 3354 int ret = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3355 char tmpbuf[SPDK_NVME_IDENTIFY_BUFLEN] = ""; 3356 struct spdk_iov_xfer ix; 3357 3358 if (req->iovcnt < 1 || req->length < SPDK_NVME_IDENTIFY_BUFLEN) { 3359 SPDK_DEBUGLOG(nvmf, "identify command with invalid buffer\n"); 3360 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3361 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3362 return ret; 3363 } 3364 3365 cns = cmd->cdw10_bits.identify.cns; 3366 3367 if (spdk_nvmf_subsystem_is_discovery(subsystem) && 3368 cns != SPDK_NVME_IDENTIFY_CTRLR) { 3369 /* Discovery controllers only support Identify Controller */ 3370 goto invalid_cns; 3371 } 3372 3373 /* 3374 * We must use a temporary buffer: it's entirely possible the out buffer 3375 * is split across more than one IOV. 3376 */ 3377 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 3378 3379 SPDK_DEBUGLOG(nvmf, "Received identify command with CNS 0x%02x\n", cns); 3380 3381 switch (cns) { 3382 case SPDK_NVME_IDENTIFY_NS: 3383 /* Function below can be asynchronous & we always need to have the data in request's buffer 3384 * So just return here */ 3385 return spdk_nvmf_ctrlr_identify_ns_ext(req); 3386 case SPDK_NVME_IDENTIFY_CTRLR: 3387 ret = spdk_nvmf_ctrlr_identify_ctrlr(ctrlr, (void *)&tmpbuf); 3388 break; 3389 case SPDK_NVME_IDENTIFY_ACTIVE_NS_LIST: 3390 ret = nvmf_ctrlr_identify_active_ns_list(ctrlr, cmd, rsp, (void *)&tmpbuf); 3391 break; 3392 case SPDK_NVME_IDENTIFY_NS_ID_DESCRIPTOR_LIST: 3393 ret = nvmf_ctrlr_identify_ns_id_descriptor_list(ctrlr, cmd, rsp, 3394 tmpbuf, req->length); 3395 break; 3396 case SPDK_NVME_IDENTIFY_NS_IOCS: 3397 ret = spdk_nvmf_ns_identify_iocs_specific(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length); 3398 break; 3399 case SPDK_NVME_IDENTIFY_CTRLR_IOCS: 3400 ret = spdk_nvmf_ctrlr_identify_iocs_specific(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length); 3401 break; 3402 case SPDK_NVME_IDENTIFY_IOCS: 3403 ret = nvmf_ctrlr_identify_iocs(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length); 3404 break; 3405 default: 3406 goto invalid_cns; 3407 } 3408 3409 if (ret == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 3410 spdk_iov_xfer_from_buf(&ix, tmpbuf, sizeof(tmpbuf)); 3411 } 3412 3413 return ret; 3414 3415 invalid_cns: 3416 SPDK_DEBUGLOG(nvmf, "Identify command with unsupported CNS 0x%02x\n", cns); 3417 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3418 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3419 return ret; 3420 } 3421 3422 static bool 3423 nvmf_qpair_abort_aer(struct spdk_nvmf_qpair *qpair, uint16_t cid) 3424 { 3425 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 3426 struct spdk_nvmf_request *req; 3427 int i; 3428 3429 if (!nvmf_qpair_is_admin_queue(qpair)) { 3430 return false; 3431 } 3432 3433 assert(spdk_get_thread() == ctrlr->thread); 3434 3435 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 3436 if (ctrlr->aer_req[i]->cmd->nvme_cmd.cid == cid) { 3437 SPDK_DEBUGLOG(nvmf, "Aborting AER request\n"); 3438 req = ctrlr->aer_req[i]; 3439 ctrlr->aer_req[i] = NULL; 3440 ctrlr->nr_aer_reqs--; 3441 3442 /* Move the last req to the aborting position for making aer_reqs 3443 * in continuous 3444 */ 3445 if (i < ctrlr->nr_aer_reqs) { 3446 ctrlr->aer_req[i] = ctrlr->aer_req[ctrlr->nr_aer_reqs]; 3447 ctrlr->aer_req[ctrlr->nr_aer_reqs] = NULL; 3448 } 3449 3450 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3451 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 3452 _nvmf_request_complete(req); 3453 return true; 3454 } 3455 } 3456 3457 return false; 3458 } 3459 3460 void 3461 nvmf_qpair_abort_pending_zcopy_reqs(struct spdk_nvmf_qpair *qpair) 3462 { 3463 struct spdk_nvmf_request *req, *tmp; 3464 3465 TAILQ_FOREACH_SAFE(req, &qpair->outstanding, link, tmp) { 3466 if (req->zcopy_phase == NVMF_ZCOPY_PHASE_EXECUTE) { 3467 /* Zero-copy requests are kept on the outstanding queue from the moment 3468 * zcopy_start is sent until a zcopy_end callback is received. Therefore, 3469 * we can't remove them from the outstanding queue here, but need to rely on 3470 * the transport to do a zcopy_end to release their buffers and, in turn, 3471 * remove them from the queue. 3472 */ 3473 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3474 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 3475 nvmf_transport_req_free(req); 3476 } 3477 } 3478 } 3479 3480 static void 3481 nvmf_qpair_abort_request(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_request *req) 3482 { 3483 uint16_t cid = req->cmd->nvme_cmd.cdw10_bits.abort.cid; 3484 3485 if (nvmf_qpair_abort_aer(qpair, cid)) { 3486 SPDK_DEBUGLOG(nvmf, "abort ctrlr=%p sqid=%u cid=%u successful\n", 3487 qpair->ctrlr, qpair->qid, cid); 3488 req->rsp->nvme_cpl.cdw0 &= ~1U; /* Command successfully aborted */ 3489 3490 spdk_nvmf_request_complete(req); 3491 return; 3492 } 3493 3494 nvmf_transport_qpair_abort_request(qpair, req); 3495 } 3496 3497 static void 3498 nvmf_ctrlr_abort_done(struct spdk_io_channel_iter *i, int status) 3499 { 3500 struct spdk_nvmf_request *req = spdk_io_channel_iter_get_ctx(i); 3501 3502 if (status == 0) { 3503 /* There was no qpair whose ID matches SQID of the abort command. 3504 * Hence call _nvmf_request_complete() here. 3505 */ 3506 _nvmf_request_complete(req); 3507 } 3508 } 3509 3510 static void 3511 nvmf_ctrlr_abort_on_pg(struct spdk_io_channel_iter *i) 3512 { 3513 struct spdk_nvmf_request *req = spdk_io_channel_iter_get_ctx(i); 3514 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 3515 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 3516 uint16_t sqid = req->cmd->nvme_cmd.cdw10_bits.abort.sqid; 3517 struct spdk_nvmf_qpair *qpair; 3518 3519 TAILQ_FOREACH(qpair, &group->qpairs, link) { 3520 if (qpair->ctrlr == req->qpair->ctrlr && qpair->qid == sqid) { 3521 /* Found the qpair */ 3522 3523 nvmf_qpair_abort_request(qpair, req); 3524 3525 /* Return -1 for the status so the iteration across threads stops. */ 3526 spdk_for_each_channel_continue(i, -1); 3527 return; 3528 } 3529 } 3530 3531 spdk_for_each_channel_continue(i, 0); 3532 } 3533 3534 static int 3535 nvmf_ctrlr_abort(struct spdk_nvmf_request *req) 3536 { 3537 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3538 3539 rsp->cdw0 = 1U; /* Command not aborted */ 3540 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3541 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3542 3543 /* Send a message to each poll group, searching for this ctrlr, sqid, and command. */ 3544 spdk_for_each_channel(req->qpair->ctrlr->subsys->tgt, 3545 nvmf_ctrlr_abort_on_pg, 3546 req, 3547 nvmf_ctrlr_abort_done 3548 ); 3549 3550 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 3551 } 3552 3553 int 3554 nvmf_ctrlr_abort_request(struct spdk_nvmf_request *req) 3555 { 3556 struct spdk_nvmf_request *req_to_abort = req->req_to_abort; 3557 struct spdk_bdev *bdev; 3558 struct spdk_bdev_desc *desc; 3559 struct spdk_io_channel *ch; 3560 int rc; 3561 3562 assert(req_to_abort != NULL); 3563 3564 if (g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_ABORT].hdlr && 3565 nvmf_qpair_is_admin_queue(req_to_abort->qpair)) { 3566 return g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_ABORT].hdlr(req); 3567 } 3568 3569 rc = spdk_nvmf_request_get_bdev(req_to_abort->cmd->nvme_cmd.nsid, req_to_abort, 3570 &bdev, &desc, &ch); 3571 if (rc != 0) { 3572 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3573 } 3574 3575 return spdk_nvmf_bdev_ctrlr_abort_cmd(bdev, desc, ch, req, req_to_abort); 3576 } 3577 3578 static int 3579 get_features_generic(struct spdk_nvmf_request *req, uint32_t cdw0) 3580 { 3581 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3582 3583 rsp->cdw0 = cdw0; 3584 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3585 } 3586 3587 /* we have to use the typedef in the function declaration to appease astyle. */ 3588 typedef enum spdk_nvme_path_status_code spdk_nvme_path_status_code_t; 3589 3590 static spdk_nvme_path_status_code_t 3591 _nvme_ana_state_to_path_status(enum spdk_nvme_ana_state ana_state) 3592 { 3593 switch (ana_state) { 3594 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3595 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_INACCESSIBLE; 3596 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3597 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS; 3598 case SPDK_NVME_ANA_CHANGE_STATE: 3599 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_TRANSITION; 3600 default: 3601 return SPDK_NVME_SC_INTERNAL_PATH_ERROR; 3602 } 3603 } 3604 3605 static int 3606 nvmf_ctrlr_get_features(struct spdk_nvmf_request *req) 3607 { 3608 uint8_t feature; 3609 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3610 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3611 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 3612 enum spdk_nvme_ana_state ana_state; 3613 3614 feature = cmd->cdw10_bits.get_features.fid; 3615 3616 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 3617 /* 3618 * Features supported by Discovery controller 3619 */ 3620 switch (feature) { 3621 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3622 return get_features_generic(req, ctrlr->feat.keep_alive_timer.raw); 3623 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3624 return get_features_generic(req, ctrlr->feat.async_event_configuration.raw); 3625 default: 3626 SPDK_INFOLOG(nvmf, "Get Features command with unsupported feature ID 0x%02x\n", feature); 3627 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3628 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3629 } 3630 } 3631 /* 3632 * Process Get Features command for non-discovery controller 3633 */ 3634 ana_state = nvmf_ctrlr_get_ana_state_from_nsid(ctrlr, cmd->nsid); 3635 switch (ana_state) { 3636 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3637 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3638 case SPDK_NVME_ANA_CHANGE_STATE: 3639 switch (feature) { 3640 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3641 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3642 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3643 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3644 response->status.sct = SPDK_NVME_SCT_PATH; 3645 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 3646 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3647 default: 3648 break; 3649 } 3650 break; 3651 default: 3652 break; 3653 } 3654 3655 switch (feature) { 3656 case SPDK_NVME_FEAT_ARBITRATION: 3657 return get_features_generic(req, ctrlr->feat.arbitration.raw); 3658 case SPDK_NVME_FEAT_POWER_MANAGEMENT: 3659 return get_features_generic(req, ctrlr->feat.power_management.raw); 3660 case SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD: 3661 return nvmf_ctrlr_get_features_temperature_threshold(req); 3662 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3663 return get_features_generic(req, ctrlr->feat.error_recovery.raw); 3664 case SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE: 3665 return get_features_generic(req, ctrlr->feat.volatile_write_cache.raw); 3666 case SPDK_NVME_FEAT_NUMBER_OF_QUEUES: 3667 return get_features_generic(req, ctrlr->feat.number_of_queues.raw); 3668 case SPDK_NVME_FEAT_INTERRUPT_COALESCING: 3669 return get_features_generic(req, ctrlr->feat.interrupt_coalescing.raw); 3670 case SPDK_NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 3671 return nvmf_ctrlr_get_features_interrupt_vector_configuration(req); 3672 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3673 return get_features_generic(req, ctrlr->feat.write_atomicity.raw); 3674 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3675 return get_features_generic(req, ctrlr->feat.async_event_configuration.raw); 3676 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3677 return get_features_generic(req, ctrlr->feat.keep_alive_timer.raw); 3678 case SPDK_NVME_FEAT_HOST_IDENTIFIER: 3679 return nvmf_ctrlr_get_features_host_identifier(req); 3680 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3681 return nvmf_ctrlr_get_features_reservation_notification_mask(req); 3682 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3683 return nvmf_ctrlr_get_features_reservation_persistence(req); 3684 case SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 3685 return nvmf_ctrlr_get_features_host_behavior_support(req); 3686 default: 3687 SPDK_INFOLOG(nvmf, "Get Features command with unsupported feature ID 0x%02x\n", feature); 3688 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3689 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3690 } 3691 } 3692 3693 static int 3694 nvmf_ctrlr_set_features(struct spdk_nvmf_request *req) 3695 { 3696 uint8_t feature, save; 3697 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3698 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3699 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 3700 enum spdk_nvme_ana_state ana_state; 3701 /* 3702 * Features are not saveable by the controller as indicated by 3703 * ONCS field of the Identify Controller data. 3704 * */ 3705 save = cmd->cdw10_bits.set_features.sv; 3706 if (save) { 3707 response->status.sc = SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE; 3708 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 3709 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3710 } 3711 3712 feature = cmd->cdw10_bits.set_features.fid; 3713 3714 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 3715 /* 3716 * Features supported by Discovery controller 3717 */ 3718 switch (feature) { 3719 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3720 return nvmf_ctrlr_set_features_keep_alive_timer(req); 3721 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3722 return nvmf_ctrlr_set_features_async_event_configuration(req); 3723 default: 3724 SPDK_INFOLOG(nvmf, "Set Features command with unsupported feature ID 0x%02x\n", feature); 3725 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3726 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3727 } 3728 } 3729 /* 3730 * Process Set Features command for non-discovery controller 3731 */ 3732 ana_state = nvmf_ctrlr_get_ana_state_from_nsid(ctrlr, cmd->nsid); 3733 switch (ana_state) { 3734 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3735 case SPDK_NVME_ANA_CHANGE_STATE: 3736 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 3737 response->status.sct = SPDK_NVME_SCT_PATH; 3738 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 3739 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3740 } else { 3741 switch (feature) { 3742 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3743 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3744 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3745 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3746 response->status.sct = SPDK_NVME_SCT_PATH; 3747 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 3748 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3749 default: 3750 break; 3751 } 3752 } 3753 break; 3754 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3755 response->status.sct = SPDK_NVME_SCT_PATH; 3756 response->status.sc = SPDK_NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS; 3757 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3758 default: 3759 break; 3760 } 3761 3762 switch (feature) { 3763 case SPDK_NVME_FEAT_ARBITRATION: 3764 return nvmf_ctrlr_set_features_arbitration(req); 3765 case SPDK_NVME_FEAT_POWER_MANAGEMENT: 3766 return nvmf_ctrlr_set_features_power_management(req); 3767 case SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD: 3768 return nvmf_ctrlr_set_features_temperature_threshold(req); 3769 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3770 return nvmf_ctrlr_set_features_error_recovery(req); 3771 case SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE: 3772 return nvmf_ctrlr_set_features_volatile_write_cache(req); 3773 case SPDK_NVME_FEAT_NUMBER_OF_QUEUES: 3774 return nvmf_ctrlr_set_features_number_of_queues(req); 3775 case SPDK_NVME_FEAT_INTERRUPT_COALESCING: 3776 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 3777 response->status.sc = SPDK_NVME_SC_FEATURE_NOT_CHANGEABLE; 3778 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3779 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3780 return nvmf_ctrlr_set_features_write_atomicity(req); 3781 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3782 return nvmf_ctrlr_set_features_async_event_configuration(req); 3783 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3784 return nvmf_ctrlr_set_features_keep_alive_timer(req); 3785 case SPDK_NVME_FEAT_HOST_IDENTIFIER: 3786 return nvmf_ctrlr_set_features_host_identifier(req); 3787 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3788 return nvmf_ctrlr_set_features_reservation_notification_mask(req); 3789 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3790 return nvmf_ctrlr_set_features_reservation_persistence(req); 3791 case SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 3792 return nvmf_ctrlr_set_features_host_behavior_support(req); 3793 default: 3794 SPDK_INFOLOG(nvmf, "Set Features command with unsupported feature ID 0x%02x\n", feature); 3795 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3796 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3797 } 3798 } 3799 3800 static int 3801 nvmf_ctrlr_keep_alive(struct spdk_nvmf_request *req) 3802 { 3803 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3804 3805 SPDK_DEBUGLOG(nvmf, "Keep Alive\n"); 3806 /* 3807 * To handle keep alive just clear or reset the 3808 * ctrlr based keep alive duration counter. 3809 * When added, a separate timer based process 3810 * will monitor if the time since last recorded 3811 * keep alive has exceeded the max duration and 3812 * take appropriate action. 3813 */ 3814 ctrlr->last_keep_alive_tick = spdk_get_ticks(); 3815 3816 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3817 } 3818 3819 int 3820 nvmf_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req) 3821 { 3822 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3823 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3824 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 3825 struct spdk_nvmf_subsystem_poll_group *sgroup; 3826 int rc; 3827 3828 assert(ctrlr != NULL); 3829 if (cmd->opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) { 3830 /* We do not want to treat AERs as outstanding commands, 3831 * so decrement mgmt_io_outstanding here to offset 3832 * the increment that happened prior to this call. 3833 */ 3834 sgroup = &req->qpair->group->sgroups[ctrlr->subsys->id]; 3835 assert(sgroup != NULL); 3836 sgroup->mgmt_io_outstanding--; 3837 } 3838 3839 assert(spdk_get_thread() == ctrlr->thread); 3840 3841 if (cmd->fuse != 0) { 3842 /* Fused admin commands are not supported. */ 3843 response->status.sct = SPDK_NVME_SCT_GENERIC; 3844 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3845 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3846 } 3847 3848 if (ctrlr->vcprop.cc.bits.en != 1) { 3849 SPDK_ERRLOG("Admin command sent to disabled controller\n"); 3850 response->status.sct = SPDK_NVME_SCT_GENERIC; 3851 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 3852 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3853 } 3854 3855 if (req->iovcnt && spdk_nvme_opc_get_data_transfer(cmd->opc) == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 3856 spdk_iov_memset(req->iov, req->iovcnt, 0); 3857 } 3858 3859 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 3860 /* Discovery controllers only support these admin OPS. */ 3861 switch (cmd->opc) { 3862 case SPDK_NVME_OPC_IDENTIFY: 3863 case SPDK_NVME_OPC_GET_LOG_PAGE: 3864 case SPDK_NVME_OPC_KEEP_ALIVE: 3865 case SPDK_NVME_OPC_SET_FEATURES: 3866 case SPDK_NVME_OPC_GET_FEATURES: 3867 case SPDK_NVME_OPC_ASYNC_EVENT_REQUEST: 3868 break; 3869 default: 3870 goto invalid_opcode; 3871 } 3872 } 3873 3874 /* Call a custom adm cmd handler if set. Aborts are handled in a different path (see nvmf_passthru_admin_cmd) */ 3875 if (g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].hdlr && cmd->opc != SPDK_NVME_OPC_ABORT) { 3876 rc = g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].hdlr(req); 3877 if (rc >= SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 3878 /* The handler took care of this command */ 3879 return rc; 3880 } 3881 } 3882 3883 /* We only want to send passthrough admin commands to namespaces. 3884 * However, we don't want to passthrough a command with intended for all namespaces. 3885 */ 3886 if (ctrlr->subsys->passthrough && cmd->nsid && cmd->nsid != SPDK_NVME_GLOBAL_NS_TAG) { 3887 return nvmf_passthru_admin_cmd(req); 3888 } 3889 3890 switch (cmd->opc) { 3891 case SPDK_NVME_OPC_GET_LOG_PAGE: 3892 return nvmf_ctrlr_get_log_page(req); 3893 case SPDK_NVME_OPC_IDENTIFY: 3894 return nvmf_ctrlr_identify(req); 3895 case SPDK_NVME_OPC_ABORT: 3896 return nvmf_ctrlr_abort(req); 3897 case SPDK_NVME_OPC_GET_FEATURES: 3898 return nvmf_ctrlr_get_features(req); 3899 case SPDK_NVME_OPC_SET_FEATURES: 3900 return nvmf_ctrlr_set_features(req); 3901 case SPDK_NVME_OPC_ASYNC_EVENT_REQUEST: 3902 return nvmf_ctrlr_async_event_request(req); 3903 case SPDK_NVME_OPC_KEEP_ALIVE: 3904 return nvmf_ctrlr_keep_alive(req); 3905 3906 case SPDK_NVME_OPC_CREATE_IO_SQ: 3907 case SPDK_NVME_OPC_CREATE_IO_CQ: 3908 case SPDK_NVME_OPC_DELETE_IO_SQ: 3909 case SPDK_NVME_OPC_DELETE_IO_CQ: 3910 /* Create and Delete I/O CQ/SQ not allowed in NVMe-oF */ 3911 goto invalid_opcode; 3912 3913 default: 3914 goto invalid_opcode; 3915 } 3916 3917 invalid_opcode: 3918 SPDK_INFOLOG(nvmf, "Unsupported admin opcode 0x%x\n", cmd->opc); 3919 response->status.sct = SPDK_NVME_SCT_GENERIC; 3920 response->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3921 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3922 } 3923 3924 static int 3925 nvmf_ctrlr_process_fabrics_cmd(struct spdk_nvmf_request *req) 3926 { 3927 struct spdk_nvmf_qpair *qpair = req->qpair; 3928 struct spdk_nvmf_capsule_cmd *cap_hdr; 3929 3930 cap_hdr = &req->cmd->nvmf_cmd; 3931 3932 if (qpair->ctrlr == NULL) { 3933 /* No ctrlr established yet; the only valid command is Connect */ 3934 assert(cap_hdr->fctype == SPDK_NVMF_FABRIC_COMMAND_CONNECT); 3935 return nvmf_ctrlr_cmd_connect(req); 3936 } else if (nvmf_qpair_is_admin_queue(qpair)) { 3937 /* 3938 * Controller session is established, and this is an admin queue. 3939 * Disallow Connect and allow other fabrics commands. 3940 */ 3941 switch (cap_hdr->fctype) { 3942 case SPDK_NVMF_FABRIC_COMMAND_PROPERTY_SET: 3943 return nvmf_property_set(req); 3944 case SPDK_NVMF_FABRIC_COMMAND_PROPERTY_GET: 3945 return nvmf_property_get(req); 3946 case SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_SEND: 3947 case SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_RECV: 3948 return nvmf_auth_request_exec(req); 3949 default: 3950 SPDK_DEBUGLOG(nvmf, "unknown fctype 0x%02x\n", 3951 cap_hdr->fctype); 3952 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3953 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3954 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3955 } 3956 } else { 3957 /* 3958 * Controller session is established, and this is an I/O queue. 3959 * Disallow everything besides authentication commands. 3960 */ 3961 switch (cap_hdr->fctype) { 3962 case SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_SEND: 3963 case SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_RECV: 3964 return nvmf_auth_request_exec(req); 3965 default: 3966 SPDK_DEBUGLOG(nvmf, "Unexpected I/O fctype 0x%x\n", cap_hdr->fctype); 3967 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3968 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3969 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3970 } 3971 } 3972 } 3973 3974 static inline void 3975 nvmf_ctrlr_queue_pending_async_event(struct spdk_nvmf_ctrlr *ctrlr, 3976 union spdk_nvme_async_event_completion *event) 3977 { 3978 struct spdk_nvmf_async_event_completion *nvmf_event; 3979 3980 nvmf_event = calloc(1, sizeof(*nvmf_event)); 3981 if (!nvmf_event) { 3982 SPDK_ERRLOG("Alloc nvmf event failed, ignore the event\n"); 3983 return; 3984 } 3985 nvmf_event->event.raw = event->raw; 3986 STAILQ_INSERT_TAIL(&ctrlr->async_events, nvmf_event, link); 3987 } 3988 3989 static inline int 3990 nvmf_ctrlr_async_event_notification(struct spdk_nvmf_ctrlr *ctrlr, 3991 union spdk_nvme_async_event_completion *event) 3992 { 3993 struct spdk_nvmf_request *req; 3994 struct spdk_nvme_cpl *rsp; 3995 3996 assert(spdk_get_thread() == ctrlr->thread); 3997 3998 /* If there is no outstanding AER request, queue the event. Then 3999 * if an AER is later submitted, this event can be sent as a 4000 * response. 4001 */ 4002 if (ctrlr->nr_aer_reqs == 0) { 4003 nvmf_ctrlr_queue_pending_async_event(ctrlr, event); 4004 return 0; 4005 } 4006 4007 req = ctrlr->aer_req[--ctrlr->nr_aer_reqs]; 4008 rsp = &req->rsp->nvme_cpl; 4009 4010 rsp->cdw0 = event->raw; 4011 4012 _nvmf_request_complete(req); 4013 ctrlr->aer_req[ctrlr->nr_aer_reqs] = NULL; 4014 4015 return 0; 4016 } 4017 4018 int 4019 nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr) 4020 { 4021 union spdk_nvme_async_event_completion event = {0}; 4022 4023 /* Users may disable the event notification */ 4024 if (!ctrlr->feat.async_event_configuration.bits.ns_attr_notice) { 4025 return 0; 4026 } 4027 4028 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT)) { 4029 return 0; 4030 } 4031 4032 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 4033 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED; 4034 event.bits.log_page_identifier = SPDK_NVME_LOG_CHANGED_NS_LIST; 4035 4036 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 4037 } 4038 4039 int 4040 nvmf_ctrlr_async_event_ana_change_notice(struct spdk_nvmf_ctrlr *ctrlr) 4041 { 4042 union spdk_nvme_async_event_completion event = {0}; 4043 4044 /* Users may disable the event notification */ 4045 if (!ctrlr->feat.async_event_configuration.bits.ana_change_notice) { 4046 return 0; 4047 } 4048 4049 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT)) { 4050 return 0; 4051 } 4052 4053 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 4054 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_ANA_CHANGE; 4055 event.bits.log_page_identifier = SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS; 4056 4057 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 4058 } 4059 4060 void 4061 nvmf_ctrlr_async_event_reservation_notification(struct spdk_nvmf_ctrlr *ctrlr) 4062 { 4063 union spdk_nvme_async_event_completion event = {0}; 4064 4065 if (!ctrlr->num_avail_log_pages) { 4066 return; 4067 } 4068 4069 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT)) { 4070 return; 4071 } 4072 4073 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_IO; 4074 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL; 4075 event.bits.log_page_identifier = SPDK_NVME_LOG_RESERVATION_NOTIFICATION; 4076 4077 nvmf_ctrlr_async_event_notification(ctrlr, &event); 4078 } 4079 4080 void 4081 nvmf_ctrlr_async_event_discovery_log_change_notice(void *ctx) 4082 { 4083 union spdk_nvme_async_event_completion event = {0}; 4084 struct spdk_nvmf_ctrlr *ctrlr = ctx; 4085 4086 /* Users may disable the event notification manually or 4087 * it may not be enabled due to keep alive timeout 4088 * not being set in connect command to discovery controller. 4089 */ 4090 if (!ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice) { 4091 return; 4092 } 4093 4094 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT)) { 4095 return; 4096 } 4097 4098 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 4099 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE; 4100 event.bits.log_page_identifier = SPDK_NVME_LOG_DISCOVERY; 4101 4102 nvmf_ctrlr_async_event_notification(ctrlr, &event); 4103 } 4104 4105 int 4106 spdk_nvmf_ctrlr_async_event_error_event(struct spdk_nvmf_ctrlr *ctrlr, 4107 enum spdk_nvme_async_event_info_error info) 4108 { 4109 union spdk_nvme_async_event_completion event; 4110 4111 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ERROR_MASK_BIT)) { 4112 return 0; 4113 } 4114 4115 if (info > SPDK_NVME_ASYNC_EVENT_FW_IMAGE_LOAD) { 4116 return 0; 4117 } 4118 4119 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_ERROR; 4120 event.bits.log_page_identifier = SPDK_NVME_LOG_ERROR; 4121 event.bits.async_event_info = info; 4122 4123 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 4124 } 4125 4126 void 4127 nvmf_qpair_free_aer(struct spdk_nvmf_qpair *qpair) 4128 { 4129 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 4130 int i; 4131 4132 if (ctrlr == NULL || !nvmf_qpair_is_admin_queue(qpair)) { 4133 return; 4134 } 4135 4136 assert(spdk_get_thread() == ctrlr->thread); 4137 4138 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 4139 spdk_nvmf_request_free(ctrlr->aer_req[i]); 4140 ctrlr->aer_req[i] = NULL; 4141 } 4142 4143 ctrlr->nr_aer_reqs = 0; 4144 } 4145 4146 void 4147 spdk_nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr) 4148 { 4149 struct spdk_nvmf_request *req; 4150 int i; 4151 4152 assert(spdk_get_thread() == ctrlr->thread); 4153 4154 if (!ctrlr->nr_aer_reqs) { 4155 return; 4156 } 4157 4158 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 4159 req = ctrlr->aer_req[i]; 4160 4161 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4162 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 4163 _nvmf_request_complete(req); 4164 4165 ctrlr->aer_req[i] = NULL; 4166 } 4167 4168 ctrlr->nr_aer_reqs = 0; 4169 } 4170 4171 static void 4172 _nvmf_ctrlr_add_reservation_log(void *ctx) 4173 { 4174 struct spdk_nvmf_reservation_log *log = (struct spdk_nvmf_reservation_log *)ctx; 4175 struct spdk_nvmf_ctrlr *ctrlr = log->ctrlr; 4176 4177 ctrlr->log_page_count++; 4178 4179 /* Maximum number of queued log pages is 255 */ 4180 if (ctrlr->num_avail_log_pages == 0xff) { 4181 struct spdk_nvmf_reservation_log *entry; 4182 entry = TAILQ_LAST(&ctrlr->log_head, log_page_head); 4183 entry->log.log_page_count = ctrlr->log_page_count; 4184 free(log); 4185 return; 4186 } 4187 4188 log->log.log_page_count = ctrlr->log_page_count; 4189 log->log.num_avail_log_pages = ctrlr->num_avail_log_pages++; 4190 TAILQ_INSERT_TAIL(&ctrlr->log_head, log, link); 4191 4192 nvmf_ctrlr_async_event_reservation_notification(ctrlr); 4193 } 4194 4195 void 4196 nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr, 4197 struct spdk_nvmf_ns *ns, 4198 enum spdk_nvme_reservation_notification_log_page_type type) 4199 { 4200 struct spdk_nvmf_reservation_log *log; 4201 4202 switch (type) { 4203 case SPDK_NVME_RESERVATION_LOG_PAGE_EMPTY: 4204 return; 4205 case SPDK_NVME_REGISTRATION_PREEMPTED: 4206 if (ns->mask & SPDK_NVME_REGISTRATION_PREEMPTED_MASK) { 4207 return; 4208 } 4209 break; 4210 case SPDK_NVME_RESERVATION_RELEASED: 4211 if (ns->mask & SPDK_NVME_RESERVATION_RELEASED_MASK) { 4212 return; 4213 } 4214 break; 4215 case SPDK_NVME_RESERVATION_PREEMPTED: 4216 if (ns->mask & SPDK_NVME_RESERVATION_PREEMPTED_MASK) { 4217 return; 4218 } 4219 break; 4220 default: 4221 return; 4222 } 4223 4224 log = calloc(1, sizeof(*log)); 4225 if (!log) { 4226 SPDK_ERRLOG("Alloc log page failed, ignore the log\n"); 4227 return; 4228 } 4229 log->ctrlr = ctrlr; 4230 log->log.type = type; 4231 log->log.nsid = ns->nsid; 4232 4233 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_add_reservation_log, log); 4234 } 4235 4236 /* Check from subsystem poll group's namespace information data structure */ 4237 static bool 4238 nvmf_ns_info_ctrlr_is_registrant(struct spdk_nvmf_subsystem_pg_ns_info *ns_info, 4239 struct spdk_nvmf_ctrlr *ctrlr) 4240 { 4241 uint32_t i; 4242 4243 for (i = 0; i < SPDK_NVMF_MAX_NUM_REGISTRANTS; i++) { 4244 if (!spdk_uuid_compare(&ns_info->reg_hostid[i], &ctrlr->hostid)) { 4245 return true; 4246 } 4247 } 4248 4249 return false; 4250 } 4251 4252 /* 4253 * Check the NVMe command is permitted or not for current controller(Host). 4254 */ 4255 static int 4256 nvmf_ns_reservation_request_check(struct spdk_nvmf_subsystem_pg_ns_info *ns_info, 4257 struct spdk_nvmf_ctrlr *ctrlr, 4258 struct spdk_nvmf_request *req) 4259 { 4260 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 4261 enum spdk_nvme_reservation_type rtype = ns_info->rtype; 4262 uint8_t status = SPDK_NVME_SC_SUCCESS; 4263 uint8_t racqa; 4264 bool is_registrant; 4265 4266 /* No valid reservation */ 4267 if (!rtype) { 4268 return 0; 4269 } 4270 4271 is_registrant = nvmf_ns_info_ctrlr_is_registrant(ns_info, ctrlr); 4272 /* All registrants type and current ctrlr is a valid registrant */ 4273 if ((rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE_ALL_REGS || 4274 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_ALL_REGS) && is_registrant) { 4275 return 0; 4276 } else if (!spdk_uuid_compare(&ns_info->holder_id, &ctrlr->hostid)) { 4277 return 0; 4278 } 4279 4280 /* Non-holder for current controller */ 4281 switch (cmd->opc) { 4282 case SPDK_NVME_OPC_READ: 4283 case SPDK_NVME_OPC_COMPARE: 4284 if (rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) { 4285 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4286 goto exit; 4287 } 4288 if ((rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_REG_ONLY || 4289 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_ALL_REGS) && !is_registrant) { 4290 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4291 } 4292 break; 4293 case SPDK_NVME_OPC_FLUSH: 4294 case SPDK_NVME_OPC_WRITE: 4295 case SPDK_NVME_OPC_WRITE_UNCORRECTABLE: 4296 case SPDK_NVME_OPC_WRITE_ZEROES: 4297 case SPDK_NVME_OPC_DATASET_MANAGEMENT: 4298 if (rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE || 4299 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) { 4300 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4301 goto exit; 4302 } 4303 if (!is_registrant) { 4304 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4305 } 4306 break; 4307 case SPDK_NVME_OPC_RESERVATION_ACQUIRE: 4308 racqa = cmd->cdw10_bits.resv_acquire.racqa; 4309 if (racqa == SPDK_NVME_RESERVE_ACQUIRE) { 4310 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4311 goto exit; 4312 } 4313 if (!is_registrant) { 4314 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4315 } 4316 break; 4317 case SPDK_NVME_OPC_RESERVATION_RELEASE: 4318 if (!is_registrant) { 4319 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4320 } 4321 break; 4322 default: 4323 break; 4324 } 4325 4326 exit: 4327 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4328 req->rsp->nvme_cpl.status.sc = status; 4329 if (status == SPDK_NVME_SC_RESERVATION_CONFLICT) { 4330 return -EPERM; 4331 } 4332 4333 return 0; 4334 } 4335 4336 static int 4337 nvmf_ctrlr_process_io_fused_cmd(struct spdk_nvmf_request *req, struct spdk_bdev *bdev, 4338 struct spdk_bdev_desc *desc, struct spdk_io_channel *ch) 4339 { 4340 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 4341 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 4342 struct spdk_nvmf_request *first_fused_req = req->qpair->first_fused_req; 4343 int rc; 4344 4345 if (cmd->fuse == SPDK_NVME_CMD_FUSE_FIRST) { 4346 /* first fused operation (should be compare) */ 4347 if (first_fused_req != NULL) { 4348 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 4349 4350 SPDK_ERRLOG("Wrong sequence of fused operations\n"); 4351 4352 /* abort req->qpair->first_fused_request and continue with new fused command */ 4353 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4354 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 4355 _nvmf_request_complete(first_fused_req); 4356 } else if (cmd->opc != SPDK_NVME_OPC_COMPARE) { 4357 SPDK_ERRLOG("Wrong op code of fused operations\n"); 4358 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4359 rsp->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 4360 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4361 } 4362 4363 req->qpair->first_fused_req = req; 4364 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 4365 } else if (cmd->fuse == SPDK_NVME_CMD_FUSE_SECOND) { 4366 /* second fused operation (should be write) */ 4367 if (first_fused_req == NULL) { 4368 SPDK_ERRLOG("Wrong sequence of fused operations\n"); 4369 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4370 rsp->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4371 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4372 } else if (cmd->opc != SPDK_NVME_OPC_WRITE) { 4373 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 4374 4375 SPDK_ERRLOG("Wrong op code of fused operations\n"); 4376 4377 /* abort req->qpair->first_fused_request and fail current command */ 4378 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4379 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 4380 _nvmf_request_complete(first_fused_req); 4381 4382 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4383 rsp->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 4384 req->qpair->first_fused_req = NULL; 4385 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4386 } 4387 4388 /* save request of first command to generate response later */ 4389 req->first_fused_req = first_fused_req; 4390 req->first_fused = true; 4391 req->qpair->first_fused_req = NULL; 4392 } else { 4393 SPDK_ERRLOG("Invalid fused command fuse field.\n"); 4394 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4395 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 4396 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4397 } 4398 4399 rc = nvmf_bdev_ctrlr_compare_and_write_cmd(bdev, desc, ch, req->first_fused_req, req); 4400 4401 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 4402 if (spdk_nvme_cpl_is_error(rsp)) { 4403 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 4404 4405 fused_response->status = rsp->status; 4406 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4407 rsp->status.sc = SPDK_NVME_SC_ABORTED_FAILED_FUSED; 4408 /* Complete first of fused commands. Second will be completed by upper layer */ 4409 _nvmf_request_complete(first_fused_req); 4410 req->first_fused_req = NULL; 4411 req->first_fused = false; 4412 } 4413 } 4414 4415 return rc; 4416 } 4417 4418 bool 4419 nvmf_ctrlr_use_zcopy(struct spdk_nvmf_request *req) 4420 { 4421 struct spdk_nvmf_transport *transport = req->qpair->transport; 4422 struct spdk_nvmf_ns *ns; 4423 4424 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_NONE); 4425 4426 if (!transport->opts.zcopy) { 4427 return false; 4428 } 4429 4430 if (nvmf_qpair_is_admin_queue(req->qpair)) { 4431 /* Admin queue */ 4432 return false; 4433 } 4434 4435 if ((req->cmd->nvme_cmd.opc != SPDK_NVME_OPC_WRITE) && 4436 (req->cmd->nvme_cmd.opc != SPDK_NVME_OPC_READ)) { 4437 /* Not a READ or WRITE command */ 4438 return false; 4439 } 4440 4441 if (req->cmd->nvme_cmd.fuse != SPDK_NVME_CMD_FUSE_NONE) { 4442 /* Fused commands dont use zcopy buffers */ 4443 return false; 4444 } 4445 4446 ns = nvmf_ctrlr_get_ns(req->qpair->ctrlr, req->cmd->nvme_cmd.nsid); 4447 if (ns == NULL || ns->bdev == NULL || !ns->zcopy) { 4448 return false; 4449 } 4450 4451 req->zcopy_phase = NVMF_ZCOPY_PHASE_INIT; 4452 return true; 4453 } 4454 4455 void 4456 spdk_nvmf_request_zcopy_start(struct spdk_nvmf_request *req) 4457 { 4458 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT); 4459 4460 /* Set iovcnt to be the maximum number of iovs that the ZCOPY can use */ 4461 req->iovcnt = NVMF_REQ_MAX_BUFFERS; 4462 4463 spdk_nvmf_request_exec(req); 4464 } 4465 4466 void 4467 spdk_nvmf_request_zcopy_end(struct spdk_nvmf_request *req, bool commit) 4468 { 4469 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_EXECUTE); 4470 req->zcopy_phase = NVMF_ZCOPY_PHASE_END_PENDING; 4471 4472 nvmf_bdev_ctrlr_zcopy_end(req, commit); 4473 } 4474 4475 int 4476 nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) 4477 { 4478 uint32_t nsid; 4479 struct spdk_nvmf_ns *ns; 4480 struct spdk_bdev *bdev; 4481 struct spdk_bdev_desc *desc; 4482 struct spdk_io_channel *ch; 4483 struct spdk_nvmf_qpair *qpair = req->qpair; 4484 struct spdk_nvmf_poll_group *group = qpair->group; 4485 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 4486 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 4487 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 4488 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4489 enum spdk_nvme_ana_state ana_state; 4490 4491 /* pre-set response details for this command */ 4492 response->status.sc = SPDK_NVME_SC_SUCCESS; 4493 nsid = cmd->nsid; 4494 4495 assert(ctrlr != NULL); 4496 if (spdk_unlikely(ctrlr->vcprop.cc.bits.en != 1)) { 4497 SPDK_ERRLOG("I/O command sent to disabled controller\n"); 4498 response->status.sct = SPDK_NVME_SCT_GENERIC; 4499 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 4500 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4501 } 4502 4503 ns = nvmf_ctrlr_get_ns(ctrlr, nsid); 4504 if (spdk_unlikely(ns == NULL || ns->bdev == NULL)) { 4505 SPDK_DEBUGLOG(nvmf, "Unsuccessful query for nsid %u\n", cmd->nsid); 4506 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4507 response->status.dnr = 1; 4508 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4509 } 4510 4511 ana_state = nvmf_ctrlr_get_ana_state(ctrlr, ns->anagrpid); 4512 if (spdk_unlikely(ana_state != SPDK_NVME_ANA_OPTIMIZED_STATE && 4513 ana_state != SPDK_NVME_ANA_NON_OPTIMIZED_STATE)) { 4514 SPDK_DEBUGLOG(nvmf, "Fail I/O command due to ANA state %d\n", 4515 ana_state); 4516 response->status.sct = SPDK_NVME_SCT_PATH; 4517 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 4518 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4519 } 4520 4521 if (spdk_likely(ctrlr->listener != NULL)) { 4522 SPDK_DTRACE_PROBE3_TICKS(nvmf_request_io_exec_path, req, 4523 ctrlr->listener->trid->traddr, 4524 ctrlr->listener->trid->trsvcid); 4525 } 4526 4527 /* scan-build falsely reporting dereference of null pointer */ 4528 assert(group != NULL && group->sgroups != NULL); 4529 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[nsid - 1]; 4530 if (nvmf_ns_reservation_request_check(ns_info, ctrlr, req)) { 4531 SPDK_DEBUGLOG(nvmf, "Reservation Conflict for nsid %u, opcode %u\n", 4532 cmd->nsid, cmd->opc); 4533 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4534 } 4535 4536 bdev = ns->bdev; 4537 desc = ns->desc; 4538 ch = ns_info->channel; 4539 4540 if (spdk_unlikely(cmd->fuse & SPDK_NVME_CMD_FUSE_MASK)) { 4541 return nvmf_ctrlr_process_io_fused_cmd(req, bdev, desc, ch); 4542 } else if (spdk_unlikely(qpair->first_fused_req != NULL)) { 4543 struct spdk_nvme_cpl *fused_response = &qpair->first_fused_req->rsp->nvme_cpl; 4544 4545 SPDK_ERRLOG("Second fused cmd expected - failing first one (cntlid:%u, qid:%u, opcode:0x%x)\n", 4546 ctrlr->cntlid, qpair->qid, 4547 req->qpair->first_fused_req->cmd->nvmf_cmd.opcode); 4548 4549 /* abort qpair->first_fused_request and continue with new command */ 4550 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4551 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 4552 _nvmf_request_complete(qpair->first_fused_req); 4553 qpair->first_fused_req = NULL; 4554 } 4555 4556 if (ctrlr->subsys->passthrough) { 4557 assert(ns->passthru_nsid > 0); 4558 req->cmd->nvme_cmd.nsid = ns->passthru_nsid; 4559 4560 return nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req); 4561 } 4562 4563 if (spdk_nvmf_request_using_zcopy(req)) { 4564 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT); 4565 return nvmf_bdev_ctrlr_zcopy_start(bdev, desc, ch, req); 4566 } else { 4567 switch (cmd->opc) { 4568 case SPDK_NVME_OPC_READ: 4569 return nvmf_bdev_ctrlr_read_cmd(bdev, desc, ch, req); 4570 case SPDK_NVME_OPC_WRITE: 4571 return nvmf_bdev_ctrlr_write_cmd(bdev, desc, ch, req); 4572 case SPDK_NVME_OPC_FLUSH: 4573 return nvmf_bdev_ctrlr_flush_cmd(bdev, desc, ch, req); 4574 case SPDK_NVME_OPC_COMPARE: 4575 if (spdk_unlikely(!ctrlr->cdata.oncs.compare)) { 4576 goto invalid_opcode; 4577 } 4578 return nvmf_bdev_ctrlr_compare_cmd(bdev, desc, ch, req); 4579 case SPDK_NVME_OPC_WRITE_ZEROES: 4580 if (spdk_unlikely(!ctrlr->cdata.oncs.write_zeroes)) { 4581 goto invalid_opcode; 4582 } 4583 return nvmf_bdev_ctrlr_write_zeroes_cmd(bdev, desc, ch, req); 4584 case SPDK_NVME_OPC_DATASET_MANAGEMENT: 4585 if (spdk_unlikely(!ctrlr->cdata.oncs.dsm)) { 4586 goto invalid_opcode; 4587 } 4588 return nvmf_bdev_ctrlr_dsm_cmd(bdev, desc, ch, req); 4589 case SPDK_NVME_OPC_RESERVATION_REGISTER: 4590 case SPDK_NVME_OPC_RESERVATION_ACQUIRE: 4591 case SPDK_NVME_OPC_RESERVATION_RELEASE: 4592 case SPDK_NVME_OPC_RESERVATION_REPORT: 4593 if (spdk_unlikely(!ctrlr->cdata.oncs.reservations)) { 4594 goto invalid_opcode; 4595 } 4596 spdk_thread_send_msg(ctrlr->subsys->thread, nvmf_ns_reservation_request, req); 4597 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 4598 case SPDK_NVME_OPC_COPY: 4599 if (spdk_unlikely(!ctrlr->cdata.oncs.copy)) { 4600 goto invalid_opcode; 4601 } 4602 return nvmf_bdev_ctrlr_copy_cmd(bdev, desc, ch, req); 4603 default: 4604 if (spdk_unlikely(qpair->transport->opts.disable_command_passthru)) { 4605 goto invalid_opcode; 4606 } 4607 if (ns->passthru_nsid) { 4608 req->cmd->nvme_cmd.nsid = ns->passthru_nsid; 4609 } 4610 return nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req); 4611 } 4612 } 4613 invalid_opcode: 4614 SPDK_INFOLOG(nvmf, "Unsupported IO opcode 0x%x\n", cmd->opc); 4615 response->status.sct = SPDK_NVME_SCT_GENERIC; 4616 response->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 4617 response->status.dnr = 1; 4618 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4619 } 4620 4621 static void 4622 nvmf_qpair_request_cleanup(struct spdk_nvmf_qpair *qpair) 4623 { 4624 if (spdk_unlikely(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING)) { 4625 assert(qpair->state_cb != NULL); 4626 4627 if (TAILQ_EMPTY(&qpair->outstanding)) { 4628 qpair->state_cb(qpair->state_cb_arg, 0); 4629 } 4630 } 4631 } 4632 4633 int 4634 spdk_nvmf_request_free(struct spdk_nvmf_request *req) 4635 { 4636 struct spdk_nvmf_qpair *qpair = req->qpair; 4637 4638 TAILQ_REMOVE(&qpair->outstanding, req, link); 4639 if (spdk_unlikely(nvmf_transport_req_free(req))) { 4640 SPDK_ERRLOG("Unable to free transport level request resources.\n"); 4641 } 4642 4643 nvmf_qpair_request_cleanup(qpair); 4644 4645 return 0; 4646 } 4647 4648 static void 4649 _nvmf_request_complete(void *ctx) 4650 { 4651 struct spdk_nvmf_request *req = ctx; 4652 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 4653 struct spdk_nvmf_qpair *qpair; 4654 struct spdk_nvmf_subsystem_poll_group *sgroup = NULL; 4655 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4656 bool is_aer = false; 4657 uint32_t nsid; 4658 bool paused; 4659 uint8_t opcode; 4660 4661 rsp->sqid = 0; 4662 rsp->status.p = 0; 4663 rsp->cid = req->cmd->nvme_cmd.cid; 4664 nsid = req->cmd->nvme_cmd.nsid; 4665 opcode = req->cmd->nvmf_cmd.opcode; 4666 4667 qpair = req->qpair; 4668 if (spdk_likely(qpair->ctrlr)) { 4669 sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id]; 4670 assert(sgroup != NULL); 4671 is_aer = req->cmd->nvme_cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST; 4672 if (spdk_likely(qpair->qid != 0)) { 4673 qpair->group->stat.completed_nvme_io++; 4674 } 4675 4676 /* 4677 * Set the crd value. 4678 * If the the IO has any error, and dnr (DoNotRetry) is not 1, 4679 * and ACRE is enabled, we will set the crd to 1 to select the first CRDT. 4680 */ 4681 if (spdk_unlikely(spdk_nvme_cpl_is_error(rsp) && 4682 rsp->status.dnr == 0 && 4683 qpair->ctrlr->acre_enabled)) { 4684 rsp->status.crd = 1; 4685 } 4686 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) { 4687 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 4688 } 4689 4690 if (SPDK_DEBUGLOG_FLAG_ENABLED("nvmf")) { 4691 spdk_nvme_print_completion(qpair->qid, rsp); 4692 } 4693 4694 switch (req->zcopy_phase) { 4695 case NVMF_ZCOPY_PHASE_NONE: 4696 TAILQ_REMOVE(&qpair->outstanding, req, link); 4697 break; 4698 case NVMF_ZCOPY_PHASE_INIT: 4699 if (spdk_unlikely(spdk_nvme_cpl_is_error(rsp))) { 4700 req->zcopy_phase = NVMF_ZCOPY_PHASE_INIT_FAILED; 4701 TAILQ_REMOVE(&qpair->outstanding, req, link); 4702 } else { 4703 req->zcopy_phase = NVMF_ZCOPY_PHASE_EXECUTE; 4704 } 4705 break; 4706 case NVMF_ZCOPY_PHASE_EXECUTE: 4707 break; 4708 case NVMF_ZCOPY_PHASE_END_PENDING: 4709 TAILQ_REMOVE(&qpair->outstanding, req, link); 4710 req->zcopy_phase = NVMF_ZCOPY_PHASE_COMPLETE; 4711 break; 4712 default: 4713 SPDK_ERRLOG("Invalid ZCOPY phase %u\n", req->zcopy_phase); 4714 break; 4715 } 4716 4717 if (spdk_unlikely(nvmf_transport_req_complete(req))) { 4718 SPDK_ERRLOG("Transport request completion error!\n"); 4719 } 4720 4721 /* AER cmd is an exception */ 4722 if (spdk_likely(sgroup && !is_aer)) { 4723 if (spdk_unlikely(opcode == SPDK_NVME_OPC_FABRIC || 4724 nvmf_qpair_is_admin_queue(qpair))) { 4725 assert(sgroup->mgmt_io_outstanding > 0); 4726 sgroup->mgmt_io_outstanding--; 4727 } else { 4728 if (req->zcopy_phase == NVMF_ZCOPY_PHASE_NONE || 4729 req->zcopy_phase == NVMF_ZCOPY_PHASE_COMPLETE || 4730 req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT_FAILED) { 4731 /* End of request */ 4732 4733 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 4734 if (spdk_likely(nsid - 1 < sgroup->num_ns)) { 4735 sgroup->ns_info[nsid - 1].io_outstanding--; 4736 } 4737 } 4738 } 4739 4740 if (spdk_unlikely(sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSING && 4741 sgroup->mgmt_io_outstanding == 0)) { 4742 paused = true; 4743 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 4744 ns_info = &sgroup->ns_info[nsid]; 4745 4746 if (ns_info->state == SPDK_NVMF_SUBSYSTEM_PAUSING && 4747 ns_info->io_outstanding > 0) { 4748 paused = false; 4749 break; 4750 } 4751 } 4752 4753 if (paused) { 4754 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 4755 sgroup->cb_fn(sgroup->cb_arg, 0); 4756 sgroup->cb_fn = NULL; 4757 sgroup->cb_arg = NULL; 4758 } 4759 } 4760 4761 } 4762 4763 nvmf_qpair_request_cleanup(qpair); 4764 } 4765 4766 int 4767 spdk_nvmf_request_complete(struct spdk_nvmf_request *req) 4768 { 4769 struct spdk_nvmf_qpair *qpair = req->qpair; 4770 4771 spdk_thread_exec_msg(qpair->group->thread, _nvmf_request_complete, req); 4772 4773 return 0; 4774 } 4775 4776 static bool 4777 nvmf_check_subsystem_active(struct spdk_nvmf_request *req) 4778 { 4779 struct spdk_nvmf_qpair *qpair = req->qpair; 4780 struct spdk_nvmf_subsystem_poll_group *sgroup = NULL; 4781 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4782 uint32_t nsid; 4783 4784 if (spdk_likely(qpair->ctrlr)) { 4785 sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id]; 4786 assert(sgroup != NULL); 4787 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) { 4788 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 4789 } 4790 4791 /* Check if the subsystem is paused (if there is a subsystem) */ 4792 if (spdk_unlikely(sgroup == NULL)) { 4793 return true; 4794 } 4795 4796 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC || 4797 nvmf_qpair_is_admin_queue(qpair))) { 4798 if (sgroup->state != SPDK_NVMF_SUBSYSTEM_ACTIVE) { 4799 /* The subsystem is not currently active. Queue this request. */ 4800 TAILQ_INSERT_TAIL(&sgroup->queued, req, link); 4801 return false; 4802 } 4803 sgroup->mgmt_io_outstanding++; 4804 } else { 4805 nsid = req->cmd->nvme_cmd.nsid; 4806 4807 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 4808 if (spdk_unlikely(nsid - 1 >= sgroup->num_ns)) { 4809 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4810 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4811 req->rsp->nvme_cpl.status.dnr = 1; 4812 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4813 _nvmf_request_complete(req); 4814 return false; 4815 } 4816 4817 ns_info = &sgroup->ns_info[nsid - 1]; 4818 if (spdk_unlikely(ns_info->channel == NULL)) { 4819 /* This can can happen if host sends I/O to a namespace that is 4820 * in the process of being added, but before the full addition 4821 * process is complete. Report invalid namespace in that case. 4822 */ 4823 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4824 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4825 req->rsp->nvme_cpl.status.dnr = 1; 4826 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4827 ns_info->io_outstanding++; 4828 _nvmf_request_complete(req); 4829 return false; 4830 } 4831 4832 if (spdk_unlikely(ns_info->state != SPDK_NVMF_SUBSYSTEM_ACTIVE)) { 4833 /* The namespace is not currently active. Queue this request. */ 4834 TAILQ_INSERT_TAIL(&sgroup->queued, req, link); 4835 return false; 4836 } 4837 4838 ns_info->io_outstanding++; 4839 } 4840 4841 return true; 4842 } 4843 4844 static bool 4845 nvmf_check_qpair_active(struct spdk_nvmf_request *req) 4846 { 4847 struct spdk_nvmf_qpair *qpair = req->qpair; 4848 int sc, sct; 4849 4850 if (spdk_likely(qpair->state == SPDK_NVMF_QPAIR_ENABLED)) { 4851 return true; 4852 } 4853 4854 sct = SPDK_NVME_SCT_GENERIC; 4855 sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 4856 4857 switch (qpair->state) { 4858 case SPDK_NVMF_QPAIR_CONNECTING: 4859 if (req->cmd->nvmf_cmd.opcode != SPDK_NVME_OPC_FABRIC) { 4860 SPDK_ERRLOG("Received command 0x%x on qid %u before CONNECT\n", 4861 req->cmd->nvmf_cmd.opcode, qpair->qid); 4862 break; 4863 } 4864 if (req->cmd->nvmf_cmd.fctype != SPDK_NVMF_FABRIC_COMMAND_CONNECT) { 4865 SPDK_ERRLOG("Received fctype 0x%x on qid %u before CONNECT\n", 4866 req->cmd->nvmf_cmd.fctype, qpair->qid); 4867 break; 4868 } 4869 return true; 4870 case SPDK_NVMF_QPAIR_AUTHENTICATING: 4871 sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 4872 sc = SPDK_NVMF_FABRIC_SC_AUTH_REQUIRED; 4873 if (req->cmd->nvmf_cmd.opcode != SPDK_NVME_OPC_FABRIC) { 4874 SPDK_ERRLOG("Received command 0x%x on qid %u before authentication\n", 4875 req->cmd->nvmf_cmd.opcode, qpair->qid); 4876 break; 4877 } 4878 if (req->cmd->nvmf_cmd.fctype != SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_SEND && 4879 req->cmd->nvmf_cmd.fctype != SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_RECV) { 4880 SPDK_ERRLOG("Received fctype 0x%x on qid %u before authentication\n", 4881 req->cmd->nvmf_cmd.fctype, qpair->qid); 4882 break; 4883 } 4884 return true; 4885 default: 4886 SPDK_ERRLOG("Received command 0x%x on qid %u in state %d\n", 4887 req->cmd->nvmf_cmd.opcode, qpair->qid, qpair->state); 4888 break; 4889 } 4890 4891 req->rsp->nvme_cpl.status.sct = sct; 4892 req->rsp->nvme_cpl.status.sc = sc; 4893 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4894 _nvmf_request_complete(req); 4895 4896 return false; 4897 } 4898 4899 void 4900 spdk_nvmf_request_exec(struct spdk_nvmf_request *req) 4901 { 4902 struct spdk_nvmf_qpair *qpair = req->qpair; 4903 enum spdk_nvmf_request_exec_status status; 4904 4905 if (spdk_unlikely(!nvmf_check_subsystem_active(req))) { 4906 return; 4907 } 4908 if (spdk_unlikely(!nvmf_check_qpair_active(req))) { 4909 return; 4910 } 4911 4912 if (SPDK_DEBUGLOG_FLAG_ENABLED("nvmf")) { 4913 spdk_nvme_print_command(qpair->qid, &req->cmd->nvme_cmd); 4914 } 4915 4916 /* Place the request on the outstanding list so we can keep track of it */ 4917 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4918 4919 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC)) { 4920 status = nvmf_ctrlr_process_fabrics_cmd(req); 4921 } else if (spdk_unlikely(nvmf_qpair_is_admin_queue(qpair))) { 4922 status = nvmf_ctrlr_process_admin_cmd(req); 4923 } else { 4924 status = nvmf_ctrlr_process_io_cmd(req); 4925 } 4926 4927 if (status == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 4928 _nvmf_request_complete(req); 4929 } 4930 } 4931 4932 static bool 4933 nvmf_ctrlr_get_dif_ctx(struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvme_cmd *cmd, 4934 struct spdk_dif_ctx *dif_ctx) 4935 { 4936 struct spdk_nvmf_ns *ns; 4937 struct spdk_bdev *bdev; 4938 4939 if (ctrlr == NULL || cmd == NULL) { 4940 return false; 4941 } 4942 4943 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 4944 if (ns == NULL || ns->bdev == NULL) { 4945 return false; 4946 } 4947 4948 bdev = ns->bdev; 4949 4950 switch (cmd->opc) { 4951 case SPDK_NVME_OPC_READ: 4952 case SPDK_NVME_OPC_WRITE: 4953 case SPDK_NVME_OPC_COMPARE: 4954 return nvmf_bdev_ctrlr_get_dif_ctx(bdev, cmd, dif_ctx); 4955 default: 4956 break; 4957 } 4958 4959 return false; 4960 } 4961 4962 bool 4963 spdk_nvmf_request_get_dif_ctx(struct spdk_nvmf_request *req, struct spdk_dif_ctx *dif_ctx) 4964 { 4965 struct spdk_nvmf_qpair *qpair = req->qpair; 4966 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 4967 4968 if (spdk_likely(ctrlr == NULL || !ctrlr->dif_insert_or_strip)) { 4969 return false; 4970 } 4971 4972 if (spdk_unlikely(!spdk_nvmf_qpair_is_active(qpair))) { 4973 return false; 4974 } 4975 4976 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC)) { 4977 return false; 4978 } 4979 4980 if (spdk_unlikely(nvmf_qpair_is_admin_queue(qpair))) { 4981 return false; 4982 } 4983 4984 return nvmf_ctrlr_get_dif_ctx(ctrlr, &req->cmd->nvme_cmd, dif_ctx); 4985 } 4986 4987 void 4988 spdk_nvmf_set_custom_admin_cmd_hdlr(uint8_t opc, spdk_nvmf_custom_cmd_hdlr hdlr) 4989 { 4990 g_nvmf_custom_admin_cmd_hdlrs[opc].hdlr = hdlr; 4991 } 4992 4993 static int 4994 nvmf_passthru_admin_cmd_for_bdev_nsid(struct spdk_nvmf_request *req, uint32_t bdev_nsid) 4995 { 4996 struct spdk_bdev *bdev; 4997 struct spdk_bdev_desc *desc; 4998 struct spdk_io_channel *ch; 4999 struct spdk_nvmf_ns *ns; 5000 struct spdk_nvmf_ctrlr *ctrlr; 5001 struct spdk_nvme_cpl *response = spdk_nvmf_request_get_response(req); 5002 int rc; 5003 5004 rc = spdk_nvmf_request_get_bdev(bdev_nsid, req, &bdev, &desc, &ch); 5005 if (rc) { 5006 response->status.sct = SPDK_NVME_SCT_GENERIC; 5007 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 5008 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 5009 } 5010 5011 ctrlr = req->qpair->ctrlr; 5012 ns = nvmf_ctrlr_get_ns(ctrlr, bdev_nsid); 5013 5014 if (ns->passthru_nsid) { 5015 req->cmd->nvme_cmd.nsid = ns->passthru_nsid; 5016 } 5017 5018 return spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(bdev, desc, ch, req, NULL); 5019 } 5020 5021 static int 5022 nvmf_passthru_admin_cmd(struct spdk_nvmf_request *req) 5023 { 5024 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req); 5025 uint32_t bdev_nsid; 5026 5027 if (g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].nsid != 0) { 5028 bdev_nsid = g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].nsid; 5029 } else { 5030 bdev_nsid = cmd->nsid; 5031 } 5032 5033 return nvmf_passthru_admin_cmd_for_bdev_nsid(req, bdev_nsid); 5034 } 5035 5036 int 5037 nvmf_passthru_admin_cmd_for_ctrlr(struct spdk_nvmf_request *req, struct spdk_nvmf_ctrlr *ctrlr) 5038 { 5039 struct spdk_nvme_cpl *response = spdk_nvmf_request_get_response(req); 5040 struct spdk_nvmf_ns *ns; 5041 5042 ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); 5043 if (ns == NULL) { 5044 /* Is there a better sc to use here? */ 5045 response->status.sct = SPDK_NVME_SCT_GENERIC; 5046 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 5047 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 5048 } 5049 5050 return nvmf_passthru_admin_cmd_for_bdev_nsid(req, ns->nsid); 5051 } 5052 5053 void 5054 spdk_nvmf_set_passthru_admin_cmd(uint8_t opc, uint32_t forward_nsid) 5055 { 5056 g_nvmf_custom_admin_cmd_hdlrs[opc].hdlr = nvmf_passthru_admin_cmd; 5057 g_nvmf_custom_admin_cmd_hdlrs[opc].nsid = forward_nsid; 5058 } 5059 5060 int 5061 spdk_nvmf_request_get_bdev(uint32_t nsid, struct spdk_nvmf_request *req, 5062 struct spdk_bdev **bdev, struct spdk_bdev_desc **desc, struct spdk_io_channel **ch) 5063 { 5064 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 5065 struct spdk_nvmf_ns *ns; 5066 struct spdk_nvmf_poll_group *group = req->qpair->group; 5067 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 5068 5069 *bdev = NULL; 5070 *desc = NULL; 5071 *ch = NULL; 5072 5073 ns = nvmf_ctrlr_get_ns(ctrlr, nsid); 5074 if (ns == NULL || ns->bdev == NULL) { 5075 return -EINVAL; 5076 } 5077 5078 assert(group != NULL && group->sgroups != NULL); 5079 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[nsid - 1]; 5080 *bdev = ns->bdev; 5081 *desc = ns->desc; 5082 *ch = ns_info->channel; 5083 5084 return 0; 5085 } 5086 5087 struct spdk_nvmf_ctrlr *spdk_nvmf_request_get_ctrlr(struct spdk_nvmf_request *req) 5088 { 5089 return req->qpair->ctrlr; 5090 } 5091 5092 struct spdk_nvme_cmd *spdk_nvmf_request_get_cmd(struct spdk_nvmf_request *req) 5093 { 5094 return &req->cmd->nvme_cmd; 5095 } 5096 5097 struct spdk_nvme_cpl *spdk_nvmf_request_get_response(struct spdk_nvmf_request *req) 5098 { 5099 return &req->rsp->nvme_cpl; 5100 } 5101 5102 struct spdk_nvmf_subsystem *spdk_nvmf_request_get_subsystem(struct spdk_nvmf_request *req) 5103 { 5104 return req->qpair->ctrlr->subsys; 5105 } 5106 5107 size_t 5108 spdk_nvmf_request_copy_from_buf(struct spdk_nvmf_request *req, 5109 void *buf, size_t buflen) 5110 { 5111 struct spdk_iov_xfer ix; 5112 5113 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 5114 return spdk_iov_xfer_from_buf(&ix, buf, buflen); 5115 } 5116 5117 size_t 5118 spdk_nvmf_request_copy_to_buf(struct spdk_nvmf_request *req, 5119 void *buf, size_t buflen) 5120 { 5121 struct spdk_iov_xfer ix; 5122 5123 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 5124 return spdk_iov_xfer_to_buf(&ix, buf, buflen); 5125 } 5126 5127 struct spdk_nvmf_subsystem *spdk_nvmf_ctrlr_get_subsystem(struct spdk_nvmf_ctrlr *ctrlr) 5128 { 5129 return ctrlr->subsys; 5130 } 5131 5132 uint16_t 5133 spdk_nvmf_ctrlr_get_id(struct spdk_nvmf_ctrlr *ctrlr) 5134 { 5135 return ctrlr->cntlid; 5136 } 5137 5138 struct spdk_nvmf_request *spdk_nvmf_request_get_req_to_abort(struct spdk_nvmf_request *req) 5139 { 5140 return req->req_to_abort; 5141 } 5142