1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019, 2020 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "nvmf_internal.h" 10 #include "transport.h" 11 12 #include "spdk/bdev.h" 13 #include "spdk/bdev_zone.h" 14 #include "spdk/bit_array.h" 15 #include "spdk/endian.h" 16 #include "spdk/thread.h" 17 #include "spdk/nvme_spec.h" 18 #include "spdk/nvmf_cmd.h" 19 #include "spdk/string.h" 20 #include "spdk/util.h" 21 #include "spdk/version.h" 22 #include "spdk/log.h" 23 #include "spdk_internal/usdt.h" 24 25 #define MIN_KEEP_ALIVE_TIMEOUT_IN_MS 10000 26 #define NVMF_DISC_KATO_IN_MS 120000 27 #define KAS_TIME_UNIT_IN_MS 100 28 #define KAS_DEFAULT_VALUE (MIN_KEEP_ALIVE_TIMEOUT_IN_MS / KAS_TIME_UNIT_IN_MS) 29 30 #define NVMF_CC_RESET_SHN_TIMEOUT_IN_MS 10000 31 32 #define NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS (NVMF_CC_RESET_SHN_TIMEOUT_IN_MS + 5000) 33 34 #define DUPLICATE_QID_RETRY_US 1000 35 36 /* 37 * Report the SPDK version as the firmware revision. 38 * SPDK_VERSION_STRING won't fit into FR (only 8 bytes), so try to fit the most important parts. 39 */ 40 #define FW_VERSION SPDK_VERSION_MAJOR_STRING SPDK_VERSION_MINOR_STRING SPDK_VERSION_PATCH_STRING 41 42 #define ANA_TRANSITION_TIME_IN_SEC 10 43 44 #define NVMF_ABORT_COMMAND_LIMIT 3 45 46 /* 47 * Support for custom admin command handlers 48 */ 49 struct spdk_nvmf_custom_admin_cmd { 50 spdk_nvmf_custom_cmd_hdlr hdlr; 51 uint32_t nsid; /* nsid to forward */ 52 }; 53 54 static struct spdk_nvmf_custom_admin_cmd g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_MAX_OPC + 1]; 55 56 static void _nvmf_request_complete(void *ctx); 57 int nvmf_passthru_admin_cmd_for_ctrlr(struct spdk_nvmf_request *req, struct spdk_nvmf_ctrlr *ctrlr); 58 59 static inline void 60 nvmf_invalid_connect_response(struct spdk_nvmf_fabric_connect_rsp *rsp, 61 uint8_t iattr, uint16_t ipo) 62 { 63 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 64 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 65 rsp->status_code_specific.invalid.iattr = iattr; 66 rsp->status_code_specific.invalid.ipo = ipo; 67 } 68 69 #define SPDK_NVMF_INVALID_CONNECT_CMD(rsp, field) \ 70 nvmf_invalid_connect_response(rsp, 0, offsetof(struct spdk_nvmf_fabric_connect_cmd, field)) 71 #define SPDK_NVMF_INVALID_CONNECT_DATA(rsp, field) \ 72 nvmf_invalid_connect_response(rsp, 1, offsetof(struct spdk_nvmf_fabric_connect_data, field)) 73 74 75 static void 76 nvmf_ctrlr_stop_keep_alive_timer(struct spdk_nvmf_ctrlr *ctrlr) 77 { 78 if (!ctrlr) { 79 SPDK_ERRLOG("Controller is NULL\n"); 80 return; 81 } 82 83 if (ctrlr->keep_alive_poller == NULL) { 84 return; 85 } 86 87 SPDK_DEBUGLOG(nvmf, "Stop keep alive poller\n"); 88 spdk_poller_unregister(&ctrlr->keep_alive_poller); 89 } 90 91 static void 92 nvmf_ctrlr_stop_association_timer(struct spdk_nvmf_ctrlr *ctrlr) 93 { 94 if (!ctrlr) { 95 SPDK_ERRLOG("Controller is NULL\n"); 96 assert(false); 97 return; 98 } 99 100 if (ctrlr->association_timer == NULL) { 101 return; 102 } 103 104 SPDK_DEBUGLOG(nvmf, "Stop association timer\n"); 105 spdk_poller_unregister(&ctrlr->association_timer); 106 } 107 108 static void 109 nvmf_ctrlr_disconnect_qpairs_done(struct spdk_io_channel_iter *i, int status) 110 { 111 if (status == 0) { 112 SPDK_DEBUGLOG(nvmf, "ctrlr disconnect qpairs complete successfully\n"); 113 } else { 114 SPDK_ERRLOG("Fail to disconnect ctrlr qpairs\n"); 115 } 116 } 117 118 static int 119 _nvmf_ctrlr_disconnect_qpairs_on_pg(struct spdk_io_channel_iter *i, bool include_admin) 120 { 121 int rc = 0; 122 struct spdk_nvmf_ctrlr *ctrlr; 123 struct spdk_nvmf_qpair *qpair, *temp_qpair; 124 struct spdk_io_channel *ch; 125 struct spdk_nvmf_poll_group *group; 126 127 ctrlr = spdk_io_channel_iter_get_ctx(i); 128 ch = spdk_io_channel_iter_get_channel(i); 129 group = spdk_io_channel_get_ctx(ch); 130 131 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, temp_qpair) { 132 if (qpair->ctrlr == ctrlr && (include_admin || !nvmf_qpair_is_admin_queue(qpair))) { 133 rc = spdk_nvmf_qpair_disconnect(qpair); 134 if (rc) { 135 if (rc == -EINPROGRESS) { 136 rc = 0; 137 } else { 138 SPDK_ERRLOG("Qpair disconnect failed\n"); 139 return rc; 140 } 141 } 142 } 143 } 144 145 return rc; 146 } 147 148 static void 149 nvmf_ctrlr_disconnect_qpairs_on_pg(struct spdk_io_channel_iter *i) 150 { 151 spdk_for_each_channel_continue(i, _nvmf_ctrlr_disconnect_qpairs_on_pg(i, true)); 152 } 153 154 static void 155 nvmf_ctrlr_disconnect_io_qpairs_on_pg(struct spdk_io_channel_iter *i) 156 { 157 spdk_for_each_channel_continue(i, _nvmf_ctrlr_disconnect_qpairs_on_pg(i, false)); 158 } 159 160 static int 161 nvmf_ctrlr_keep_alive_poll(void *ctx) 162 { 163 uint64_t keep_alive_timeout_tick; 164 uint64_t now = spdk_get_ticks(); 165 struct spdk_nvmf_ctrlr *ctrlr = ctx; 166 167 if (ctrlr->in_destruct) { 168 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 169 return SPDK_POLLER_IDLE; 170 } 171 172 SPDK_DEBUGLOG(nvmf, "Polling ctrlr keep alive timeout\n"); 173 174 /* If the Keep alive feature is in use and the timer expires */ 175 keep_alive_timeout_tick = ctrlr->last_keep_alive_tick + 176 ctrlr->feat.keep_alive_timer.bits.kato * spdk_get_ticks_hz() / UINT64_C(1000); 177 if (now > keep_alive_timeout_tick) { 178 SPDK_NOTICELOG("Disconnecting host %s from subsystem %s due to keep alive timeout.\n", 179 ctrlr->hostnqn, ctrlr->subsys->subnqn); 180 /* set the Controller Fatal Status bit to '1' */ 181 if (ctrlr->vcprop.csts.bits.cfs == 0) { 182 nvmf_ctrlr_set_fatal_status(ctrlr); 183 184 /* 185 * disconnect qpairs, terminate Transport connection 186 * destroy ctrlr, break the host to controller association 187 * disconnect qpairs with qpair->ctrlr == ctrlr 188 */ 189 spdk_for_each_channel(ctrlr->subsys->tgt, 190 nvmf_ctrlr_disconnect_qpairs_on_pg, 191 ctrlr, 192 nvmf_ctrlr_disconnect_qpairs_done); 193 return SPDK_POLLER_BUSY; 194 } 195 } 196 197 return SPDK_POLLER_IDLE; 198 } 199 200 static void 201 nvmf_ctrlr_start_keep_alive_timer(struct spdk_nvmf_ctrlr *ctrlr) 202 { 203 if (!ctrlr) { 204 SPDK_ERRLOG("Controller is NULL\n"); 205 return; 206 } 207 208 /* if cleared to 0 then the Keep Alive Timer is disabled */ 209 if (ctrlr->feat.keep_alive_timer.bits.kato != 0) { 210 211 ctrlr->last_keep_alive_tick = spdk_get_ticks(); 212 213 SPDK_DEBUGLOG(nvmf, "Ctrlr add keep alive poller\n"); 214 ctrlr->keep_alive_poller = SPDK_POLLER_REGISTER(nvmf_ctrlr_keep_alive_poll, ctrlr, 215 ctrlr->feat.keep_alive_timer.bits.kato * 1000); 216 } 217 } 218 219 static void 220 nvmf_qpair_set_ctrlr(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_ctrlr *ctrlr) 221 { 222 if (qpair->ctrlr != NULL) { 223 /* Admin queues will call this function twice. */ 224 assert(qpair->ctrlr == ctrlr); 225 return; 226 } 227 228 qpair->ctrlr = ctrlr; 229 spdk_trace_owner_append_description(qpair->trace_id, 230 spdk_nvmf_subsystem_get_nqn(ctrlr->subsys)); 231 } 232 233 static int _retry_qid_check(void *ctx); 234 235 static void 236 nvmf_ctrlr_send_connect_rsp(void *ctx) 237 { 238 struct spdk_nvmf_request *req = ctx; 239 struct spdk_nvmf_qpair *qpair = req->qpair; 240 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 241 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 242 int rc; 243 244 /* The qpair might have been disconnected in the meantime */ 245 assert(qpair->state == SPDK_NVMF_QPAIR_CONNECTING || 246 qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 247 if (qpair->state == SPDK_NVMF_QPAIR_CONNECTING) { 248 if (nvmf_subsystem_host_auth_required(ctrlr->subsys, ctrlr->hostnqn)) { 249 rc = nvmf_qpair_auth_init(qpair); 250 if (rc != 0) { 251 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 252 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 253 spdk_nvmf_request_complete(req); 254 spdk_nvmf_qpair_disconnect(qpair); 255 return; 256 } 257 rsp->status_code_specific.success.authreq.atr = 1; 258 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_AUTHENTICATING); 259 } else { 260 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ENABLED); 261 } 262 } 263 264 SPDK_DEBUGLOG(nvmf, "connect capsule response: cntlid = 0x%04x\n", ctrlr->cntlid); 265 266 assert(spdk_get_thread() == qpair->group->thread); 267 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 268 rsp->status_code_specific.success.cntlid = ctrlr->cntlid; 269 spdk_nvmf_request_complete(req); 270 } 271 272 static void 273 nvmf_ctrlr_add_qpair(struct spdk_nvmf_qpair *qpair, 274 struct spdk_nvmf_ctrlr *ctrlr, 275 struct spdk_nvmf_request *req) 276 { 277 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 278 279 if (!ctrlr->admin_qpair) { 280 SPDK_ERRLOG("Inactive admin qpair\n"); 281 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 282 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 283 qpair->connect_req = NULL; 284 qpair->ctrlr = NULL; 285 spdk_nvmf_request_complete(req); 286 return; 287 } 288 289 assert(ctrlr->admin_qpair->group->thread == spdk_get_thread()); 290 291 if (spdk_bit_array_get(ctrlr->qpair_mask, qpair->qid)) { 292 if (qpair->connect_req != NULL) { 293 SPDK_ERRLOG("Got I/O connect with duplicate QID %u\n", qpair->qid); 294 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 295 rsp->status.sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER; 296 qpair->connect_req = NULL; 297 qpair->ctrlr = NULL; 298 spdk_nvmf_request_complete(req); 299 } else { 300 SPDK_WARNLOG("Duplicate QID (%d) detected, re-check in %dus\n", 301 qpair->qid, DUPLICATE_QID_RETRY_US); 302 qpair->connect_req = req; 303 /* Set qpair->ctrlr here so that we'll have it when the poller expires. */ 304 nvmf_qpair_set_ctrlr(qpair, ctrlr); 305 req->poller = SPDK_POLLER_REGISTER(_retry_qid_check, qpair, 306 DUPLICATE_QID_RETRY_US); 307 } 308 return; 309 } 310 311 SPDK_DTRACE_PROBE4_TICKS(nvmf_ctrlr_add_qpair, qpair, qpair->qid, ctrlr->subsys->subnqn, 312 ctrlr->hostnqn); 313 nvmf_qpair_set_ctrlr(qpair, ctrlr); 314 spdk_bit_array_set(ctrlr->qpair_mask, qpair->qid); 315 316 spdk_thread_send_msg(qpair->group->thread, nvmf_ctrlr_send_connect_rsp, req); 317 } 318 319 static int 320 _retry_qid_check(void *ctx) 321 { 322 struct spdk_nvmf_qpair *qpair = ctx; 323 struct spdk_nvmf_request *req = qpair->connect_req; 324 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 325 326 spdk_poller_unregister(&req->poller); 327 SPDK_WARNLOG("Retrying adding qpair, qid:%d\n", qpair->qid); 328 nvmf_ctrlr_add_qpair(qpair, ctrlr, req); 329 return SPDK_POLLER_BUSY; 330 } 331 332 static void 333 _nvmf_ctrlr_add_admin_qpair(void *ctx) 334 { 335 struct spdk_nvmf_request *req = ctx; 336 struct spdk_nvmf_qpair *qpair = req->qpair; 337 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 338 339 ctrlr->admin_qpair = qpair; 340 ctrlr->association_timeout = qpair->transport->opts.association_timeout; 341 nvmf_ctrlr_start_keep_alive_timer(ctrlr); 342 nvmf_ctrlr_add_qpair(qpair, ctrlr, req); 343 } 344 345 static void 346 _nvmf_subsystem_add_ctrlr(void *ctx) 347 { 348 struct spdk_nvmf_request *req = ctx; 349 struct spdk_nvmf_qpair *qpair = req->qpair; 350 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 351 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 352 353 if (nvmf_subsystem_add_ctrlr(ctrlr->subsys, ctrlr)) { 354 SPDK_ERRLOG("Unable to add controller to subsystem\n"); 355 spdk_bit_array_free(&ctrlr->qpair_mask); 356 free(ctrlr); 357 qpair->ctrlr = NULL; 358 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 359 spdk_nvmf_request_complete(req); 360 return; 361 } 362 363 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_add_admin_qpair, req); 364 } 365 366 static void 367 nvmf_ctrlr_cdata_init(struct spdk_nvmf_transport *transport, struct spdk_nvmf_subsystem *subsystem, 368 struct spdk_nvmf_ctrlr_data *cdata) 369 { 370 cdata->aerl = SPDK_NVMF_MAX_ASYNC_EVENTS - 1; 371 cdata->kas = KAS_DEFAULT_VALUE; 372 cdata->vid = SPDK_PCI_VID_INTEL; 373 cdata->ssvid = SPDK_PCI_VID_INTEL; 374 /* INTEL OUI */ 375 cdata->ieee[0] = 0xe4; 376 cdata->ieee[1] = 0xd2; 377 cdata->ieee[2] = 0x5c; 378 cdata->oncs.compare = 1; 379 cdata->oncs.reservations = 1; 380 cdata->fuses.compare_and_write = 1; 381 cdata->oncs.copy = 1; 382 cdata->sgls.supported = 1; 383 cdata->sgls.keyed_sgl = 1; 384 cdata->sgls.sgl_offset = 1; 385 cdata->nvmf_specific.ioccsz = sizeof(struct spdk_nvme_cmd) / 16; 386 cdata->nvmf_specific.ioccsz += transport->opts.in_capsule_data_size / 16; 387 cdata->nvmf_specific.iorcsz = sizeof(struct spdk_nvme_cpl) / 16; 388 cdata->nvmf_specific.icdoff = 0; /* offset starts directly after SQE */ 389 cdata->nvmf_specific.ctrattr.ctrlr_model = SPDK_NVMF_CTRLR_MODEL_DYNAMIC; 390 cdata->nvmf_specific.msdbd = 1; 391 392 if (transport->ops->cdata_init) { 393 transport->ops->cdata_init(transport, subsystem, cdata); 394 } 395 } 396 397 static bool 398 nvmf_subsys_has_multi_iocs(struct spdk_nvmf_subsystem *subsystem) 399 { 400 struct spdk_nvmf_ns *ns; 401 uint32_t i; 402 403 for (i = 0; i < subsystem->max_nsid; i++) { 404 ns = subsystem->ns[i]; 405 if (ns && ns->bdev && spdk_bdev_is_zoned(ns->bdev)) { 406 return true; 407 } 408 } 409 return false; 410 } 411 412 static void 413 nvmf_ctrlr_init_visible_ns(struct spdk_nvmf_ctrlr *ctrlr) 414 { 415 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 416 struct spdk_nvmf_ns *ns; 417 418 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 419 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 420 if (ns->always_visible || nvmf_ns_find_host(ns, ctrlr->hostnqn) != NULL) { 421 spdk_bit_array_set(ctrlr->visible_ns, ns->nsid - 1); 422 } 423 } 424 } 425 426 static struct spdk_nvmf_ctrlr * 427 nvmf_ctrlr_create(struct spdk_nvmf_subsystem *subsystem, 428 struct spdk_nvmf_request *req, 429 struct spdk_nvmf_fabric_connect_cmd *connect_cmd, 430 struct spdk_nvmf_fabric_connect_data *connect_data) 431 { 432 struct spdk_nvmf_ctrlr *ctrlr; 433 struct spdk_nvmf_transport *transport = req->qpair->transport; 434 struct spdk_nvme_transport_id listen_trid = {}; 435 bool subsys_has_multi_iocs = false; 436 437 ctrlr = calloc(1, sizeof(*ctrlr)); 438 if (ctrlr == NULL) { 439 SPDK_ERRLOG("Memory allocation failed\n"); 440 return NULL; 441 } 442 443 if (spdk_nvme_trtype_is_fabrics(transport->ops->type)) { 444 ctrlr->dynamic_ctrlr = true; 445 } else { 446 ctrlr->cntlid = connect_data->cntlid; 447 } 448 449 SPDK_DTRACE_PROBE3_TICKS(nvmf_ctrlr_create, ctrlr, subsystem->subnqn, 450 spdk_thread_get_id(req->qpair->group->thread)); 451 452 STAILQ_INIT(&ctrlr->async_events); 453 TAILQ_INIT(&ctrlr->log_head); 454 ctrlr->subsys = subsystem; 455 ctrlr->thread = req->qpair->group->thread; 456 ctrlr->disconnect_in_progress = false; 457 458 ctrlr->qpair_mask = spdk_bit_array_create(transport->opts.max_qpairs_per_ctrlr); 459 if (!ctrlr->qpair_mask) { 460 SPDK_ERRLOG("Failed to allocate controller qpair mask\n"); 461 goto err_qpair_mask; 462 } 463 464 nvmf_ctrlr_cdata_init(transport, subsystem, &ctrlr->cdata); 465 466 /* 467 * KAS: This field indicates the granularity of the Keep Alive Timer in 100ms units. 468 * If this field is cleared to 0h, then Keep Alive is not supported. 469 */ 470 if (ctrlr->cdata.kas) { 471 ctrlr->feat.keep_alive_timer.bits.kato = spdk_divide_round_up(connect_cmd->kato, 472 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS) * 473 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS; 474 } 475 476 ctrlr->feat.async_event_configuration.bits.ns_attr_notice = 1; 477 if (ctrlr->subsys->flags.ana_reporting) { 478 ctrlr->feat.async_event_configuration.bits.ana_change_notice = 1; 479 } 480 ctrlr->feat.volatile_write_cache.bits.wce = 1; 481 /* Coalescing Disable */ 482 ctrlr->feat.interrupt_vector_configuration.bits.cd = 1; 483 484 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 485 /* 486 * If keep-alive timeout is not set, discovery controllers use some 487 * arbitrary high value in order to cleanup stale discovery sessions 488 * 489 * From the 1.0a nvme-of spec: 490 * "The Keep Alive command is reserved for 491 * Discovery controllers. A transport may specify a 492 * fixed Discovery controller activity timeout value 493 * (e.g., 2 minutes). If no commands are received 494 * by a Discovery controller within that time 495 * period, the controller may perform the 496 * actions for Keep Alive Timer expiration". 497 * 498 * From the 1.1 nvme-of spec: 499 * "A host requests an explicit persistent connection 500 * to a Discovery controller and Asynchronous Event Notifications from 501 * the Discovery controller on that persistent connection by specifying 502 * a non-zero Keep Alive Timer value in the Connect command." 503 * 504 * In case non-zero KATO is used, we enable discovery_log_change_notice 505 * otherwise we disable it and use default discovery controller KATO. 506 * KATO is in millisecond. 507 */ 508 if (ctrlr->feat.keep_alive_timer.bits.kato == 0) { 509 ctrlr->feat.keep_alive_timer.bits.kato = NVMF_DISC_KATO_IN_MS; 510 ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice = 0; 511 } else { 512 ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice = 1; 513 } 514 } 515 516 /* Subtract 1 for admin queue, 1 for 0's based */ 517 ctrlr->feat.number_of_queues.bits.ncqr = transport->opts.max_qpairs_per_ctrlr - 1 - 518 1; 519 ctrlr->feat.number_of_queues.bits.nsqr = transport->opts.max_qpairs_per_ctrlr - 1 - 520 1; 521 522 spdk_uuid_copy(&ctrlr->hostid, (struct spdk_uuid *)connect_data->hostid); 523 memcpy(ctrlr->hostnqn, connect_data->hostnqn, SPDK_NVMF_NQN_MAX_LEN); 524 525 ctrlr->visible_ns = spdk_bit_array_create(subsystem->max_nsid); 526 if (!ctrlr->visible_ns) { 527 SPDK_ERRLOG("Failed to allocate visible namespace array\n"); 528 goto err_visible_ns; 529 } 530 nvmf_ctrlr_init_visible_ns(ctrlr); 531 532 ctrlr->vcprop.cap.raw = 0; 533 ctrlr->vcprop.cap.bits.cqr = 1; /* NVMe-oF specification required */ 534 ctrlr->vcprop.cap.bits.mqes = transport->opts.max_queue_depth - 535 1; /* max queue depth */ 536 ctrlr->vcprop.cap.bits.ams = 0; /* optional arb mechanisms */ 537 /* ready timeout - 500 msec units */ 538 ctrlr->vcprop.cap.bits.to = NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS / 500; 539 ctrlr->vcprop.cap.bits.dstrd = 0; /* fixed to 0 for NVMe-oF */ 540 subsys_has_multi_iocs = nvmf_subsys_has_multi_iocs(subsystem); 541 if (subsys_has_multi_iocs) { 542 ctrlr->vcprop.cap.bits.css = 543 SPDK_NVME_CAP_CSS_IOCS; /* One or more I/O command sets supported */ 544 } else { 545 ctrlr->vcprop.cap.bits.css = SPDK_NVME_CAP_CSS_NVM; /* NVM command set */ 546 } 547 548 ctrlr->vcprop.cap.bits.mpsmin = 0; /* 2 ^ (12 + mpsmin) == 4k */ 549 ctrlr->vcprop.cap.bits.mpsmax = 0; /* 2 ^ (12 + mpsmax) == 4k */ 550 551 /* Version Supported: 1.3 */ 552 ctrlr->vcprop.vs.bits.mjr = 1; 553 ctrlr->vcprop.vs.bits.mnr = 3; 554 ctrlr->vcprop.vs.bits.ter = 0; 555 556 ctrlr->vcprop.cc.raw = 0; 557 ctrlr->vcprop.cc.bits.en = 0; /* Init controller disabled */ 558 if (subsys_has_multi_iocs) { 559 ctrlr->vcprop.cc.bits.css = 560 SPDK_NVME_CC_CSS_IOCS; /* All supported I/O Command Sets */ 561 } 562 563 ctrlr->vcprop.csts.raw = 0; 564 ctrlr->vcprop.csts.bits.rdy = 0; /* Init controller as not ready */ 565 566 SPDK_DEBUGLOG(nvmf, "cap 0x%" PRIx64 "\n", ctrlr->vcprop.cap.raw); 567 SPDK_DEBUGLOG(nvmf, "vs 0x%x\n", ctrlr->vcprop.vs.raw); 568 SPDK_DEBUGLOG(nvmf, "cc 0x%x\n", ctrlr->vcprop.cc.raw); 569 SPDK_DEBUGLOG(nvmf, "csts 0x%x\n", ctrlr->vcprop.csts.raw); 570 571 ctrlr->dif_insert_or_strip = transport->opts.dif_insert_or_strip; 572 573 if (ctrlr->subsys->subtype == SPDK_NVMF_SUBTYPE_NVME) { 574 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &listen_trid) != 0) { 575 SPDK_ERRLOG("Could not get listener transport ID\n"); 576 goto err_listener; 577 } 578 579 ctrlr->listener = nvmf_subsystem_find_listener(ctrlr->subsys, &listen_trid); 580 if (!ctrlr->listener) { 581 SPDK_ERRLOG("Listener was not found\n"); 582 goto err_listener; 583 } 584 } 585 586 nvmf_qpair_set_ctrlr(req->qpair, ctrlr); 587 spdk_thread_send_msg(subsystem->thread, _nvmf_subsystem_add_ctrlr, req); 588 589 return ctrlr; 590 err_listener: 591 spdk_bit_array_free(&ctrlr->visible_ns); 592 err_visible_ns: 593 spdk_bit_array_free(&ctrlr->qpair_mask); 594 err_qpair_mask: 595 free(ctrlr); 596 return NULL; 597 } 598 599 static void 600 _nvmf_ctrlr_destruct(void *ctx) 601 { 602 struct spdk_nvmf_ctrlr *ctrlr = ctx; 603 struct spdk_nvmf_reservation_log *log, *log_tmp; 604 struct spdk_nvmf_async_event_completion *event, *event_tmp; 605 606 SPDK_DTRACE_PROBE3_TICKS(nvmf_ctrlr_destruct, ctrlr, ctrlr->subsys->subnqn, 607 spdk_thread_get_id(ctrlr->thread)); 608 609 assert(spdk_get_thread() == ctrlr->thread); 610 assert(ctrlr->in_destruct); 611 612 SPDK_DEBUGLOG(nvmf, "Destroy ctrlr 0x%hx\n", ctrlr->cntlid); 613 if (ctrlr->disconnect_in_progress) { 614 SPDK_ERRLOG("freeing ctrlr with disconnect in progress\n"); 615 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_destruct, ctrlr); 616 return; 617 } 618 619 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 620 nvmf_ctrlr_stop_association_timer(ctrlr); 621 spdk_bit_array_free(&ctrlr->qpair_mask); 622 623 TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) { 624 TAILQ_REMOVE(&ctrlr->log_head, log, link); 625 free(log); 626 } 627 STAILQ_FOREACH_SAFE(event, &ctrlr->async_events, link, event_tmp) { 628 STAILQ_REMOVE(&ctrlr->async_events, event, spdk_nvmf_async_event_completion, link); 629 free(event); 630 } 631 spdk_bit_array_free(&ctrlr->visible_ns); 632 free(ctrlr); 633 } 634 635 void 636 nvmf_ctrlr_destruct(struct spdk_nvmf_ctrlr *ctrlr) 637 { 638 nvmf_subsystem_remove_ctrlr(ctrlr->subsys, ctrlr); 639 640 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_destruct, ctrlr); 641 } 642 643 static void 644 nvmf_ctrlr_add_io_qpair(void *ctx) 645 { 646 struct spdk_nvmf_request *req = ctx; 647 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 648 struct spdk_nvmf_qpair *qpair = req->qpair; 649 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 650 struct spdk_nvmf_qpair *admin_qpair = ctrlr->admin_qpair; 651 struct spdk_nvmf_poll_group *admin_qpair_group = NULL; 652 enum spdk_nvmf_qpair_state admin_qpair_state = SPDK_NVMF_QPAIR_UNINITIALIZED; 653 bool admin_qpair_active = false; 654 655 SPDK_DTRACE_PROBE4_TICKS(nvmf_ctrlr_add_io_qpair, ctrlr, req->qpair, req->qpair->qid, 656 spdk_thread_get_id(ctrlr->thread)); 657 658 /* Unit test will check qpair->ctrlr after calling spdk_nvmf_ctrlr_connect. 659 * For error case, the value should be NULL. So set it to NULL at first. 660 */ 661 qpair->ctrlr = NULL; 662 663 /* Make sure the controller is not being destroyed. */ 664 if (ctrlr->in_destruct) { 665 SPDK_ERRLOG("Got I/O connect while ctrlr was being destroyed.\n"); 666 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 667 goto end; 668 } 669 670 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 671 SPDK_ERRLOG("I/O connect not allowed on discovery controller\n"); 672 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 673 goto end; 674 } 675 676 if (!ctrlr->vcprop.cc.bits.en) { 677 SPDK_ERRLOG("Got I/O connect before ctrlr was enabled\n"); 678 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 679 goto end; 680 } 681 682 if (1u << ctrlr->vcprop.cc.bits.iosqes != sizeof(struct spdk_nvme_cmd)) { 683 SPDK_ERRLOG("Got I/O connect with invalid IOSQES %u\n", 684 ctrlr->vcprop.cc.bits.iosqes); 685 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 686 goto end; 687 } 688 689 if (1u << ctrlr->vcprop.cc.bits.iocqes != sizeof(struct spdk_nvme_cpl)) { 690 SPDK_ERRLOG("Got I/O connect with invalid IOCQES %u\n", 691 ctrlr->vcprop.cc.bits.iocqes); 692 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 693 goto end; 694 } 695 696 /* There is a chance that admin qpair was destroyed. This is an issue that was observed only with ESX initiators */ 697 if (admin_qpair) { 698 admin_qpair_active = spdk_nvmf_qpair_is_active(admin_qpair); 699 admin_qpair_group = admin_qpair->group; 700 admin_qpair_state = admin_qpair->state; 701 } 702 703 if (!admin_qpair_active || admin_qpair_group == NULL) { 704 /* There is a chance that admin qpair was destroyed or is being destroyed at this moment due to e.g. 705 * expired keep alive timer. Part of the qpair destruction process is change of qpair's 706 * state to DEACTIVATING and removing it from poll group */ 707 SPDK_ERRLOG("Inactive admin qpair (state %d, group %p)\n", admin_qpair_state, admin_qpair_group); 708 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 709 goto end; 710 } 711 712 /* check if we would exceed ctrlr connection limit */ 713 if (qpair->qid >= spdk_bit_array_capacity(ctrlr->qpair_mask)) { 714 SPDK_ERRLOG("Requested QID %u but Max QID is %u\n", 715 qpair->qid, spdk_bit_array_capacity(ctrlr->qpair_mask) - 1); 716 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 717 rsp->status.sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER; 718 goto end; 719 } 720 721 nvmf_ctrlr_add_qpair(qpair, ctrlr, req); 722 return; 723 end: 724 spdk_nvmf_request_complete(req); 725 } 726 727 static void 728 _nvmf_ctrlr_add_io_qpair(void *ctx) 729 { 730 struct spdk_nvmf_request *req = ctx; 731 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 732 struct spdk_nvmf_fabric_connect_data *data; 733 struct spdk_nvmf_ctrlr *ctrlr; 734 struct spdk_nvmf_qpair *qpair = req->qpair; 735 struct spdk_nvmf_qpair *admin_qpair; 736 struct spdk_nvmf_tgt *tgt = qpair->transport->tgt; 737 struct spdk_nvmf_subsystem *subsystem; 738 struct spdk_nvme_transport_id listen_trid = {}; 739 const struct spdk_nvmf_subsystem_listener *listener; 740 struct spdk_nvmf_poll_group *admin_qpair_group = NULL; 741 enum spdk_nvmf_qpair_state admin_qpair_state = SPDK_NVMF_QPAIR_UNINITIALIZED; 742 bool admin_qpair_active = false; 743 744 assert(req->iovcnt == 1); 745 746 data = req->iov[0].iov_base; 747 748 SPDK_DEBUGLOG(nvmf, "Connect I/O Queue for controller id 0x%x\n", data->cntlid); 749 750 subsystem = spdk_nvmf_tgt_find_subsystem(tgt, data->subnqn); 751 /* We already checked this in spdk_nvmf_ctrlr_connect */ 752 assert(subsystem != NULL); 753 754 ctrlr = nvmf_subsystem_get_ctrlr(subsystem, data->cntlid); 755 if (ctrlr == NULL) { 756 SPDK_ERRLOG("Unknown controller ID 0x%x\n", data->cntlid); 757 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, cntlid); 758 spdk_nvmf_request_complete(req); 759 return; 760 } 761 762 /* fail before passing a message to the controller thread. */ 763 if (ctrlr->in_destruct) { 764 SPDK_ERRLOG("Got I/O connect while ctrlr was being destroyed.\n"); 765 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 766 spdk_nvmf_request_complete(req); 767 return; 768 } 769 770 /* If ANA reporting is enabled, check if I/O connect is on the same listener. */ 771 if (subsystem->flags.ana_reporting) { 772 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &listen_trid) != 0) { 773 SPDK_ERRLOG("Could not get listener transport ID\n"); 774 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 775 spdk_nvmf_request_complete(req); 776 return; 777 } 778 779 listener = nvmf_subsystem_find_listener(subsystem, &listen_trid); 780 if (listener != ctrlr->listener) { 781 SPDK_ERRLOG("I/O connect is on a listener different from admin connect\n"); 782 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 783 spdk_nvmf_request_complete(req); 784 return; 785 } 786 } 787 788 admin_qpair = ctrlr->admin_qpair; 789 790 /* There is a chance that admin qpair was destroyed. This is an issue that was observed only with ESX initiators */ 791 if (admin_qpair) { 792 admin_qpair_active = spdk_nvmf_qpair_is_active(admin_qpair); 793 admin_qpair_group = admin_qpair->group; 794 admin_qpair_state = admin_qpair->state; 795 } 796 797 if (!admin_qpair_active || admin_qpair_group == NULL) { 798 /* There is a chance that admin qpair was destroyed or is being destroyed at this moment due to e.g. 799 * expired keep alive timer. Part of the qpair destruction process is change of qpair's 800 * state to DEACTIVATING and removing it from poll group */ 801 SPDK_ERRLOG("Inactive admin qpair (state %d, group %p)\n", admin_qpair_state, admin_qpair_group); 802 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, qid); 803 spdk_nvmf_request_complete(req); 804 return; 805 } 806 qpair->ctrlr = ctrlr; 807 spdk_thread_send_msg(admin_qpair_group->thread, nvmf_ctrlr_add_io_qpair, req); 808 } 809 810 static bool 811 nvmf_qpair_access_allowed(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_subsystem *subsystem, 812 const char *hostnqn) 813 { 814 struct spdk_nvme_transport_id listen_trid = {}; 815 816 if (!spdk_nvmf_subsystem_host_allowed(subsystem, hostnqn)) { 817 SPDK_ERRLOG("Subsystem '%s' does not allow host '%s'\n", subsystem->subnqn, hostnqn); 818 return false; 819 } 820 821 if (spdk_nvmf_qpair_get_listen_trid(qpair, &listen_trid)) { 822 SPDK_ERRLOG("Subsystem '%s' is unable to enforce access control due to an internal error.\n", 823 subsystem->subnqn); 824 return false; 825 } 826 827 if (!spdk_nvmf_subsystem_listener_allowed(subsystem, &listen_trid)) { 828 SPDK_ERRLOG("Subsystem '%s' does not allow host '%s' to connect at this address.\n", 829 subsystem->subnqn, hostnqn); 830 return false; 831 } 832 833 return true; 834 } 835 836 static int 837 _nvmf_ctrlr_connect(struct spdk_nvmf_request *req) 838 { 839 struct spdk_nvmf_fabric_connect_data *data = req->iov[0].iov_base; 840 struct spdk_nvmf_fabric_connect_cmd *cmd = &req->cmd->connect_cmd; 841 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 842 struct spdk_nvmf_qpair *qpair = req->qpair; 843 struct spdk_nvmf_transport *transport = qpair->transport; 844 struct spdk_nvmf_ctrlr *ctrlr; 845 struct spdk_nvmf_subsystem *subsystem; 846 847 SPDK_DEBUGLOG(nvmf, "recfmt 0x%x qid %u sqsize %u\n", 848 cmd->recfmt, cmd->qid, cmd->sqsize); 849 850 SPDK_DEBUGLOG(nvmf, "Connect data:\n"); 851 SPDK_DEBUGLOG(nvmf, " cntlid: 0x%04x\n", data->cntlid); 852 SPDK_DEBUGLOG(nvmf, " hostid: %08x-%04x-%04x-%02x%02x-%04x%08x ***\n", 853 ntohl(*(uint32_t *)&data->hostid[0]), 854 ntohs(*(uint16_t *)&data->hostid[4]), 855 ntohs(*(uint16_t *)&data->hostid[6]), 856 data->hostid[8], 857 data->hostid[9], 858 ntohs(*(uint16_t *)&data->hostid[10]), 859 ntohl(*(uint32_t *)&data->hostid[12])); 860 SPDK_DEBUGLOG(nvmf, " subnqn: \"%s\"\n", data->subnqn); 861 SPDK_DEBUGLOG(nvmf, " hostnqn: \"%s\"\n", data->hostnqn); 862 863 subsystem = spdk_nvmf_tgt_find_subsystem(transport->tgt, data->subnqn); 864 if (!subsystem) { 865 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 866 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 867 } 868 869 if (cmd->recfmt != 0) { 870 SPDK_ERRLOG("Connect command unsupported RECFMT %u\n", cmd->recfmt); 871 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 872 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INCOMPATIBLE_FORMAT; 873 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 874 } 875 876 /* 877 * SQSIZE is a 0-based value, so it must be at least 1 (minimum queue depth is 2) and 878 * strictly less than max_aq_depth (admin queues) or max_queue_depth (io queues). 879 */ 880 if (cmd->sqsize == 0) { 881 SPDK_ERRLOG("Invalid SQSIZE = 0\n"); 882 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 883 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 884 } 885 886 if (cmd->qid == 0) { 887 if (cmd->sqsize >= transport->opts.max_aq_depth) { 888 SPDK_ERRLOG("Invalid SQSIZE for admin queue %u (min 1, max %u)\n", 889 cmd->sqsize, transport->opts.max_aq_depth - 1); 890 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 891 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 892 } 893 } else if (cmd->sqsize >= transport->opts.max_queue_depth) { 894 SPDK_ERRLOG("Invalid SQSIZE %u (min 1, max %u)\n", 895 cmd->sqsize, transport->opts.max_queue_depth - 1); 896 SPDK_NVMF_INVALID_CONNECT_CMD(rsp, sqsize); 897 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 898 } 899 900 qpair->sq_head_max = cmd->sqsize; 901 qpair->qid = cmd->qid; 902 qpair->connect_received = true; 903 904 pthread_mutex_lock(&qpair->group->mutex); 905 qpair->group->current_unassociated_qpairs--; 906 pthread_mutex_unlock(&qpair->group->mutex); 907 908 if (0 == qpair->qid) { 909 qpair->group->stat.admin_qpairs++; 910 qpair->group->stat.current_admin_qpairs++; 911 } else { 912 qpair->group->stat.io_qpairs++; 913 qpair->group->stat.current_io_qpairs++; 914 } 915 916 if (cmd->qid == 0) { 917 SPDK_DEBUGLOG(nvmf, "Connect Admin Queue for controller ID 0x%x\n", data->cntlid); 918 919 if (spdk_nvme_trtype_is_fabrics(transport->ops->type) && data->cntlid != 0xFFFF) { 920 /* This NVMf target only supports dynamic mode. */ 921 SPDK_ERRLOG("The NVMf target only supports dynamic mode (CNTLID = 0x%x).\n", data->cntlid); 922 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, cntlid); 923 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 924 } 925 926 /* Establish a new ctrlr */ 927 ctrlr = nvmf_ctrlr_create(subsystem, req, cmd, data); 928 if (!ctrlr) { 929 SPDK_ERRLOG("nvmf_ctrlr_create() failed\n"); 930 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 931 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 932 } else { 933 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 934 } 935 } else { 936 spdk_thread_send_msg(subsystem->thread, _nvmf_ctrlr_add_io_qpair, req); 937 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 938 } 939 } 940 941 static struct spdk_nvmf_subsystem_poll_group * 942 nvmf_subsystem_pg_from_connect_cmd(struct spdk_nvmf_request *req) 943 { 944 struct spdk_nvmf_fabric_connect_data *data; 945 struct spdk_nvmf_subsystem *subsystem; 946 struct spdk_nvmf_tgt *tgt; 947 948 assert(nvmf_request_is_fabric_connect(req)); 949 assert(req->qpair->ctrlr == NULL); 950 assert(req->iovcnt == 1); 951 952 data = req->iov[0].iov_base; 953 tgt = req->qpair->transport->tgt; 954 955 subsystem = spdk_nvmf_tgt_find_subsystem(tgt, data->subnqn); 956 if (subsystem == NULL) { 957 return NULL; 958 } 959 960 return &req->qpair->group->sgroups[subsystem->id]; 961 } 962 963 int 964 spdk_nvmf_ctrlr_connect(struct spdk_nvmf_request *req) 965 { 966 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 967 struct spdk_nvmf_subsystem_poll_group *sgroup; 968 struct spdk_nvmf_qpair *qpair = req->qpair; 969 enum spdk_nvmf_request_exec_status status; 970 971 if (req->iovcnt > 1) { 972 SPDK_ERRLOG("Connect command invalid iovcnt: %d\n", req->iovcnt); 973 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 974 status = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 975 goto out; 976 } 977 978 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 979 if (!sgroup) { 980 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 981 status = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 982 goto out; 983 } 984 985 sgroup->mgmt_io_outstanding++; 986 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 987 988 status = _nvmf_ctrlr_connect(req); 989 990 out: 991 if (status == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 992 _nvmf_request_complete(req); 993 } 994 995 return status; 996 } 997 998 static int 999 nvmf_ctrlr_cmd_connect(struct spdk_nvmf_request *req) 1000 { 1001 struct spdk_nvmf_fabric_connect_data *data = req->iov[0].iov_base; 1002 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp; 1003 struct spdk_nvmf_transport *transport = req->qpair->transport; 1004 struct spdk_nvmf_subsystem *subsystem; 1005 1006 if (req->length < sizeof(struct spdk_nvmf_fabric_connect_data)) { 1007 SPDK_ERRLOG("Connect command data length 0x%x too small\n", req->length); 1008 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1009 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1010 } 1011 1012 if (req->iovcnt > 1) { 1013 SPDK_ERRLOG("Connect command invalid iovcnt: %d\n", req->iovcnt); 1014 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1015 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1016 } 1017 1018 subsystem = spdk_nvmf_tgt_find_subsystem(transport->tgt, data->subnqn); 1019 if (!subsystem) { 1020 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, subnqn); 1021 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1022 } 1023 1024 if ((subsystem->state == SPDK_NVMF_SUBSYSTEM_INACTIVE) || 1025 (subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSING) || 1026 (subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSED) || 1027 (subsystem->state == SPDK_NVMF_SUBSYSTEM_DEACTIVATING)) { 1028 struct spdk_nvmf_subsystem_poll_group *sgroup; 1029 1030 /* Subsystem is not ready to handle a connect. Decrement 1031 * the mgmt_io_outstanding to avoid the subsystem waiting 1032 * for this command to complete before unpausing. Queued 1033 * requests get retried when subsystem resumes. 1034 */ 1035 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 1036 assert(sgroup != NULL); 1037 sgroup->mgmt_io_outstanding--; 1038 TAILQ_REMOVE(&req->qpair->outstanding, req, link); 1039 TAILQ_INSERT_TAIL(&sgroup->queued, req, link); 1040 SPDK_DEBUGLOG(nvmf, "Subsystem '%s' is not ready for connect, retrying...\n", subsystem->subnqn); 1041 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 1042 } 1043 1044 /* Ensure that hostnqn is null terminated */ 1045 if (!memchr(data->hostnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 1046 SPDK_ERRLOG("Connect HOSTNQN is not null terminated\n"); 1047 SPDK_NVMF_INVALID_CONNECT_DATA(rsp, hostnqn); 1048 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1049 } 1050 1051 if (!nvmf_qpair_access_allowed(req->qpair, subsystem, data->hostnqn)) { 1052 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1053 rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_HOST; 1054 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1055 } 1056 1057 return _nvmf_ctrlr_connect(req); 1058 } 1059 1060 static int 1061 nvmf_ctrlr_association_remove(void *ctx) 1062 { 1063 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1064 int rc; 1065 1066 nvmf_ctrlr_stop_association_timer(ctrlr); 1067 1068 if (ctrlr->in_destruct) { 1069 return SPDK_POLLER_IDLE; 1070 } 1071 SPDK_DEBUGLOG(nvmf, "Disconnecting host from subsystem %s due to association timeout.\n", 1072 ctrlr->subsys->subnqn); 1073 1074 if (ctrlr->admin_qpair) { 1075 rc = spdk_nvmf_qpair_disconnect(ctrlr->admin_qpair); 1076 if (rc < 0 && rc != -EINPROGRESS) { 1077 SPDK_ERRLOG("Fail to disconnect admin ctrlr qpair\n"); 1078 assert(false); 1079 } 1080 } 1081 1082 return SPDK_POLLER_BUSY; 1083 } 1084 1085 static int 1086 _nvmf_ctrlr_cc_reset_shn_done(void *ctx) 1087 { 1088 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1089 uint64_t now = spdk_get_ticks(); 1090 uint32_t count; 1091 1092 if (ctrlr->cc_timer) { 1093 spdk_poller_unregister(&ctrlr->cc_timer); 1094 } 1095 1096 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1097 SPDK_DEBUGLOG(nvmf, "ctrlr %p active queue count %u\n", ctrlr, count); 1098 1099 if (count > 1) { 1100 if (now < ctrlr->cc_timeout_tsc) { 1101 /* restart cc timer */ 1102 ctrlr->cc_timer = SPDK_POLLER_REGISTER(_nvmf_ctrlr_cc_reset_shn_done, ctrlr, 100 * 1000); 1103 return SPDK_POLLER_IDLE; 1104 } else { 1105 /* controller fatal status */ 1106 SPDK_WARNLOG("IO timeout, ctrlr %p is in fatal status\n", ctrlr); 1107 nvmf_ctrlr_set_fatal_status(ctrlr); 1108 } 1109 } 1110 1111 spdk_poller_unregister(&ctrlr->cc_timeout_timer); 1112 1113 if (ctrlr->disconnect_is_shn) { 1114 ctrlr->vcprop.csts.bits.shst = SPDK_NVME_SHST_COMPLETE; 1115 ctrlr->disconnect_is_shn = false; 1116 } else { 1117 /* Only a subset of the registers are cleared out on a reset */ 1118 ctrlr->vcprop.cc.raw = 0; 1119 ctrlr->vcprop.csts.raw = 0; 1120 } 1121 1122 /* After CC.EN transitions to 0 (due to shutdown or reset), the association 1123 * between the host and controller shall be preserved for at least 2 minutes */ 1124 if (ctrlr->association_timer) { 1125 SPDK_DEBUGLOG(nvmf, "Association timer already set\n"); 1126 nvmf_ctrlr_stop_association_timer(ctrlr); 1127 } 1128 if (ctrlr->association_timeout) { 1129 ctrlr->association_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_association_remove, ctrlr, 1130 ctrlr->association_timeout * 1000); 1131 } 1132 ctrlr->disconnect_in_progress = false; 1133 return SPDK_POLLER_BUSY; 1134 } 1135 1136 static void 1137 nvmf_ctrlr_cc_reset_shn_done(struct spdk_io_channel_iter *i, int status) 1138 { 1139 struct spdk_nvmf_ctrlr *ctrlr = spdk_io_channel_iter_get_ctx(i); 1140 1141 if (status < 0) { 1142 SPDK_ERRLOG("Fail to disconnect io ctrlr qpairs\n"); 1143 assert(false); 1144 } 1145 1146 _nvmf_ctrlr_cc_reset_shn_done((void *)ctrlr); 1147 } 1148 1149 static void 1150 nvmf_bdev_complete_reset(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1151 { 1152 SPDK_NOTICELOG("Resetting bdev done with %s\n", success ? "success" : "failure"); 1153 1154 spdk_bdev_free_io(bdev_io); 1155 } 1156 1157 1158 static int 1159 nvmf_ctrlr_cc_timeout(void *ctx) 1160 { 1161 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1162 struct spdk_nvmf_poll_group *group; 1163 struct spdk_nvmf_ns *ns; 1164 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1165 1166 spdk_poller_unregister(&ctrlr->cc_timeout_timer); 1167 SPDK_DEBUGLOG(nvmf, "Ctrlr %p reset or shutdown timeout\n", ctrlr); 1168 1169 if (!ctrlr->admin_qpair) { 1170 SPDK_NOTICELOG("Ctrlr %p admin qpair disconnected\n", ctrlr); 1171 return SPDK_POLLER_IDLE; 1172 } 1173 1174 group = ctrlr->admin_qpair->group; 1175 assert(group != NULL && group->sgroups != NULL); 1176 1177 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 1178 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 1179 if (ns->bdev == NULL) { 1180 continue; 1181 } 1182 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[ns->opts.nsid - 1]; 1183 SPDK_NOTICELOG("Ctrlr %p resetting NSID %u\n", ctrlr, ns->opts.nsid); 1184 spdk_bdev_reset(ns->desc, ns_info->channel, nvmf_bdev_complete_reset, NULL); 1185 } 1186 1187 return SPDK_POLLER_BUSY; 1188 } 1189 1190 const struct spdk_nvmf_registers * 1191 spdk_nvmf_ctrlr_get_regs(struct spdk_nvmf_ctrlr *ctrlr) 1192 { 1193 return &ctrlr->vcprop; 1194 } 1195 1196 void 1197 nvmf_ctrlr_set_fatal_status(struct spdk_nvmf_ctrlr *ctrlr) 1198 { 1199 ctrlr->vcprop.csts.bits.cfs = 1; 1200 } 1201 1202 static uint64_t 1203 nvmf_prop_get_cap(struct spdk_nvmf_ctrlr *ctrlr) 1204 { 1205 return ctrlr->vcprop.cap.raw; 1206 } 1207 1208 static uint64_t 1209 nvmf_prop_get_vs(struct spdk_nvmf_ctrlr *ctrlr) 1210 { 1211 return ctrlr->vcprop.vs.raw; 1212 } 1213 1214 static uint64_t 1215 nvmf_prop_get_cc(struct spdk_nvmf_ctrlr *ctrlr) 1216 { 1217 return ctrlr->vcprop.cc.raw; 1218 } 1219 1220 static bool 1221 nvmf_prop_set_cc(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1222 { 1223 union spdk_nvme_cc_register cc, diff; 1224 uint32_t cc_timeout_ms; 1225 1226 cc.raw = value; 1227 1228 SPDK_DEBUGLOG(nvmf, "cur CC: 0x%08x\n", ctrlr->vcprop.cc.raw); 1229 SPDK_DEBUGLOG(nvmf, "new CC: 0x%08x\n", cc.raw); 1230 1231 /* 1232 * Calculate which bits changed between the current and new CC. 1233 * Mark each bit as 0 once it is handled to determine if any unhandled bits were changed. 1234 */ 1235 diff.raw = cc.raw ^ ctrlr->vcprop.cc.raw; 1236 1237 if (diff.bits.en) { 1238 if (cc.bits.en) { 1239 SPDK_DEBUGLOG(nvmf, "Property Set CC Enable!\n"); 1240 nvmf_ctrlr_stop_association_timer(ctrlr); 1241 1242 ctrlr->vcprop.cc.bits.en = 1; 1243 ctrlr->vcprop.csts.bits.rdy = 1; 1244 } else { 1245 SPDK_DEBUGLOG(nvmf, "Property Set CC Disable!\n"); 1246 if (ctrlr->disconnect_in_progress) { 1247 SPDK_DEBUGLOG(nvmf, "Disconnect in progress\n"); 1248 return true; 1249 } 1250 1251 ctrlr->cc_timeout_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_cc_timeout, ctrlr, 1252 NVMF_CC_RESET_SHN_TIMEOUT_IN_MS * 1000); 1253 /* Make sure cc_timeout_ms is between cc_timeout_timer and Host reset/shutdown timeout */ 1254 cc_timeout_ms = (NVMF_CC_RESET_SHN_TIMEOUT_IN_MS + NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS) / 2; 1255 ctrlr->cc_timeout_tsc = spdk_get_ticks() + cc_timeout_ms * spdk_get_ticks_hz() / (uint64_t)1000; 1256 1257 ctrlr->vcprop.cc.bits.en = 0; 1258 ctrlr->disconnect_in_progress = true; 1259 ctrlr->disconnect_is_shn = false; 1260 spdk_for_each_channel(ctrlr->subsys->tgt, 1261 nvmf_ctrlr_disconnect_io_qpairs_on_pg, 1262 ctrlr, 1263 nvmf_ctrlr_cc_reset_shn_done); 1264 } 1265 diff.bits.en = 0; 1266 } 1267 1268 if (diff.bits.shn) { 1269 if (cc.bits.shn == SPDK_NVME_SHN_NORMAL || 1270 cc.bits.shn == SPDK_NVME_SHN_ABRUPT) { 1271 SPDK_DEBUGLOG(nvmf, "Property Set CC Shutdown %u%ub!\n", 1272 cc.bits.shn >> 1, cc.bits.shn & 1); 1273 if (ctrlr->disconnect_in_progress) { 1274 SPDK_DEBUGLOG(nvmf, "Disconnect in progress\n"); 1275 return true; 1276 } 1277 1278 ctrlr->cc_timeout_timer = SPDK_POLLER_REGISTER(nvmf_ctrlr_cc_timeout, ctrlr, 1279 NVMF_CC_RESET_SHN_TIMEOUT_IN_MS * 1000); 1280 /* Make sure cc_timeout_ms is between cc_timeout_timer and Host reset/shutdown timeout */ 1281 cc_timeout_ms = (NVMF_CC_RESET_SHN_TIMEOUT_IN_MS + NVMF_CTRLR_RESET_SHN_TIMEOUT_IN_MS) / 2; 1282 ctrlr->cc_timeout_tsc = spdk_get_ticks() + cc_timeout_ms * spdk_get_ticks_hz() / (uint64_t)1000; 1283 1284 ctrlr->vcprop.cc.bits.shn = cc.bits.shn; 1285 ctrlr->disconnect_in_progress = true; 1286 ctrlr->disconnect_is_shn = true; 1287 spdk_for_each_channel(ctrlr->subsys->tgt, 1288 nvmf_ctrlr_disconnect_io_qpairs_on_pg, 1289 ctrlr, 1290 nvmf_ctrlr_cc_reset_shn_done); 1291 1292 /* From the time a shutdown is initiated the controller shall disable 1293 * Keep Alive timer */ 1294 nvmf_ctrlr_stop_keep_alive_timer(ctrlr); 1295 } else if (cc.bits.shn == 0) { 1296 ctrlr->vcprop.cc.bits.shn = 0; 1297 } else { 1298 SPDK_ERRLOG("Prop Set CC: Invalid SHN value %u%ub\n", 1299 cc.bits.shn >> 1, cc.bits.shn & 1); 1300 return false; 1301 } 1302 diff.bits.shn = 0; 1303 } 1304 1305 if (diff.bits.iosqes) { 1306 SPDK_DEBUGLOG(nvmf, "Prop Set IOSQES = %u (%u bytes)\n", 1307 cc.bits.iosqes, 1u << cc.bits.iosqes); 1308 ctrlr->vcprop.cc.bits.iosqes = cc.bits.iosqes; 1309 diff.bits.iosqes = 0; 1310 } 1311 1312 if (diff.bits.iocqes) { 1313 SPDK_DEBUGLOG(nvmf, "Prop Set IOCQES = %u (%u bytes)\n", 1314 cc.bits.iocqes, 1u << cc.bits.iocqes); 1315 ctrlr->vcprop.cc.bits.iocqes = cc.bits.iocqes; 1316 diff.bits.iocqes = 0; 1317 } 1318 1319 if (diff.bits.ams) { 1320 SPDK_ERRLOG("Arbitration Mechanism Selected (AMS) 0x%x not supported!\n", cc.bits.ams); 1321 return false; 1322 } 1323 1324 if (diff.bits.mps) { 1325 SPDK_ERRLOG("Memory Page Size (MPS) %u KiB not supported!\n", (1 << (2 + cc.bits.mps))); 1326 return false; 1327 } 1328 1329 if (diff.bits.css) { 1330 if (cc.bits.css > SPDK_NVME_CC_CSS_IOCS) { 1331 SPDK_ERRLOG("I/O Command Set Selected (CSS) 0x%x not supported!\n", cc.bits.css); 1332 return false; 1333 } 1334 diff.bits.css = 0; 1335 } 1336 1337 if (diff.raw != 0) { 1338 /* Print an error message, but don't fail the command in this case. 1339 * If we did want to fail in this case, we'd need to ensure we acted 1340 * on no other bits or the initiator gets confused. */ 1341 SPDK_ERRLOG("Prop Set CC toggled reserved bits 0x%x!\n", diff.raw); 1342 } 1343 1344 return true; 1345 } 1346 1347 static uint64_t 1348 nvmf_prop_get_csts(struct spdk_nvmf_ctrlr *ctrlr) 1349 { 1350 return ctrlr->vcprop.csts.raw; 1351 } 1352 1353 static uint64_t 1354 nvmf_prop_get_aqa(struct spdk_nvmf_ctrlr *ctrlr) 1355 { 1356 return ctrlr->vcprop.aqa.raw; 1357 } 1358 1359 static bool 1360 nvmf_prop_set_aqa(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1361 { 1362 union spdk_nvme_aqa_register aqa; 1363 1364 aqa.raw = value; 1365 1366 /* 1367 * We don't need to explicitly check for maximum size, as the fields are 1368 * limited to 12 bits (4096). 1369 */ 1370 if (aqa.bits.asqs < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES - 1 || 1371 aqa.bits.acqs < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES - 1 || 1372 aqa.bits.reserved1 != 0 || aqa.bits.reserved2 != 0) { 1373 return false; 1374 } 1375 1376 ctrlr->vcprop.aqa.raw = value; 1377 1378 return true; 1379 } 1380 1381 static uint64_t 1382 nvmf_prop_get_asq(struct spdk_nvmf_ctrlr *ctrlr) 1383 { 1384 return ctrlr->vcprop.asq; 1385 } 1386 1387 static bool 1388 nvmf_prop_set_asq_lower(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1389 { 1390 ctrlr->vcprop.asq = (ctrlr->vcprop.asq & (0xFFFFFFFFULL << 32ULL)) | value; 1391 1392 return true; 1393 } 1394 1395 static bool 1396 nvmf_prop_set_asq_upper(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1397 { 1398 ctrlr->vcprop.asq = (ctrlr->vcprop.asq & 0xFFFFFFFFULL) | ((uint64_t)value << 32ULL); 1399 1400 return true; 1401 } 1402 1403 static uint64_t 1404 nvmf_prop_get_acq(struct spdk_nvmf_ctrlr *ctrlr) 1405 { 1406 return ctrlr->vcprop.acq; 1407 } 1408 1409 static bool 1410 nvmf_prop_set_acq_lower(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1411 { 1412 ctrlr->vcprop.acq = (ctrlr->vcprop.acq & (0xFFFFFFFFULL << 32ULL)) | value; 1413 1414 return true; 1415 } 1416 1417 static bool 1418 nvmf_prop_set_acq_upper(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value) 1419 { 1420 ctrlr->vcprop.acq = (ctrlr->vcprop.acq & 0xFFFFFFFFULL) | ((uint64_t)value << 32ULL); 1421 1422 return true; 1423 } 1424 1425 struct nvmf_prop { 1426 uint32_t ofst; 1427 uint8_t size; 1428 char name[11]; 1429 uint64_t (*get_cb)(struct spdk_nvmf_ctrlr *ctrlr); 1430 bool (*set_cb)(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value); 1431 bool (*set_upper_cb)(struct spdk_nvmf_ctrlr *ctrlr, uint32_t value); 1432 }; 1433 1434 #define PROP(field, size, get_cb, set_cb, set_upper_cb) \ 1435 { \ 1436 offsetof(struct spdk_nvme_registers, field), \ 1437 size, \ 1438 #field, \ 1439 get_cb, set_cb, set_upper_cb \ 1440 } 1441 1442 static const struct nvmf_prop nvmf_props[] = { 1443 PROP(cap, 8, nvmf_prop_get_cap, NULL, NULL), 1444 PROP(vs, 4, nvmf_prop_get_vs, NULL, NULL), 1445 PROP(cc, 4, nvmf_prop_get_cc, nvmf_prop_set_cc, NULL), 1446 PROP(csts, 4, nvmf_prop_get_csts, NULL, NULL), 1447 PROP(aqa, 4, nvmf_prop_get_aqa, nvmf_prop_set_aqa, NULL), 1448 PROP(asq, 8, nvmf_prop_get_asq, nvmf_prop_set_asq_lower, nvmf_prop_set_asq_upper), 1449 PROP(acq, 8, nvmf_prop_get_acq, nvmf_prop_set_acq_lower, nvmf_prop_set_acq_upper), 1450 }; 1451 1452 static const struct nvmf_prop * 1453 find_prop(uint32_t ofst, uint8_t size) 1454 { 1455 size_t i; 1456 1457 for (i = 0; i < SPDK_COUNTOF(nvmf_props); i++) { 1458 const struct nvmf_prop *prop = &nvmf_props[i]; 1459 1460 if ((ofst >= prop->ofst) && (ofst + size <= prop->ofst + prop->size)) { 1461 return prop; 1462 } 1463 } 1464 1465 return NULL; 1466 } 1467 1468 static int 1469 nvmf_property_get(struct spdk_nvmf_request *req) 1470 { 1471 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1472 struct spdk_nvmf_fabric_prop_get_cmd *cmd = &req->cmd->prop_get_cmd; 1473 struct spdk_nvmf_fabric_prop_get_rsp *response = &req->rsp->prop_get_rsp; 1474 const struct nvmf_prop *prop; 1475 uint8_t size; 1476 1477 response->status.sc = 0; 1478 response->value.u64 = 0; 1479 1480 SPDK_DEBUGLOG(nvmf, "size %d, offset 0x%x\n", 1481 cmd->attrib.size, cmd->ofst); 1482 1483 switch (cmd->attrib.size) { 1484 case SPDK_NVMF_PROP_SIZE_4: 1485 size = 4; 1486 break; 1487 case SPDK_NVMF_PROP_SIZE_8: 1488 size = 8; 1489 break; 1490 default: 1491 SPDK_DEBUGLOG(nvmf, "Invalid size value %d\n", cmd->attrib.size); 1492 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1493 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1494 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1495 } 1496 1497 prop = find_prop(cmd->ofst, size); 1498 if (prop == NULL || prop->get_cb == NULL) { 1499 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1500 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1501 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1502 } 1503 1504 SPDK_DEBUGLOG(nvmf, "name: %s\n", prop->name); 1505 1506 response->value.u64 = prop->get_cb(ctrlr); 1507 1508 if (size != prop->size) { 1509 /* The size must be 4 and the prop->size is 8. Figure out which part of the property to read. */ 1510 assert(size == 4); 1511 assert(prop->size == 8); 1512 1513 if (cmd->ofst == prop->ofst) { 1514 /* Keep bottom 4 bytes only */ 1515 response->value.u64 &= 0xFFFFFFFF; 1516 } else { 1517 /* Keep top 4 bytes only */ 1518 response->value.u64 >>= 32; 1519 } 1520 } 1521 1522 SPDK_DEBUGLOG(nvmf, "response value: 0x%" PRIx64 "\n", response->value.u64); 1523 1524 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1525 } 1526 1527 static int 1528 nvmf_property_set(struct spdk_nvmf_request *req) 1529 { 1530 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1531 struct spdk_nvmf_fabric_prop_set_cmd *cmd = &req->cmd->prop_set_cmd; 1532 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1533 const struct nvmf_prop *prop; 1534 uint64_t value; 1535 uint8_t size; 1536 bool ret; 1537 1538 SPDK_DEBUGLOG(nvmf, "size %d, offset 0x%x, value 0x%" PRIx64 "\n", 1539 cmd->attrib.size, cmd->ofst, cmd->value.u64); 1540 1541 switch (cmd->attrib.size) { 1542 case SPDK_NVMF_PROP_SIZE_4: 1543 size = 4; 1544 break; 1545 case SPDK_NVMF_PROP_SIZE_8: 1546 size = 8; 1547 break; 1548 default: 1549 SPDK_DEBUGLOG(nvmf, "Invalid size value %d\n", cmd->attrib.size); 1550 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1551 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1552 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1553 } 1554 1555 prop = find_prop(cmd->ofst, size); 1556 if (prop == NULL || prop->set_cb == NULL) { 1557 SPDK_INFOLOG(nvmf, "Invalid offset 0x%x\n", cmd->ofst); 1558 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1559 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1560 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1561 } 1562 1563 SPDK_DEBUGLOG(nvmf, "name: %s\n", prop->name); 1564 1565 value = cmd->value.u64; 1566 1567 if (prop->size == 4) { 1568 ret = prop->set_cb(ctrlr, (uint32_t)value); 1569 } else if (size != prop->size) { 1570 /* The size must be 4 and the prop->size is 8. Figure out which part of the property to write. */ 1571 assert(size == 4); 1572 assert(prop->size == 8); 1573 1574 if (cmd->ofst == prop->ofst) { 1575 ret = prop->set_cb(ctrlr, (uint32_t)value); 1576 } else { 1577 ret = prop->set_upper_cb(ctrlr, (uint32_t)value); 1578 } 1579 } else { 1580 ret = prop->set_cb(ctrlr, (uint32_t)value); 1581 if (ret) { 1582 ret = prop->set_upper_cb(ctrlr, (uint32_t)(value >> 32)); 1583 } 1584 } 1585 1586 if (!ret) { 1587 SPDK_ERRLOG("prop set_cb failed\n"); 1588 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1589 response->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM; 1590 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1591 } 1592 1593 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1594 } 1595 1596 static int 1597 nvmf_ctrlr_set_features_arbitration(struct spdk_nvmf_request *req) 1598 { 1599 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1600 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1601 1602 SPDK_DEBUGLOG(nvmf, "Set Features - Arbitration (cdw11 = 0x%0x)\n", cmd->cdw11); 1603 1604 ctrlr->feat.arbitration.raw = cmd->cdw11; 1605 ctrlr->feat.arbitration.bits.reserved = 0; 1606 1607 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1608 } 1609 1610 static int 1611 nvmf_ctrlr_set_features_power_management(struct spdk_nvmf_request *req) 1612 { 1613 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1614 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1615 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1616 1617 SPDK_DEBUGLOG(nvmf, "Set Features - Power Management (cdw11 = 0x%0x)\n", cmd->cdw11); 1618 1619 /* Only PS = 0 is allowed, since we report NPSS = 0 */ 1620 if (cmd->cdw11_bits.feat_power_management.bits.ps != 0) { 1621 SPDK_ERRLOG("Invalid power state %u\n", cmd->cdw11_bits.feat_power_management.bits.ps); 1622 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1623 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1624 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1625 } 1626 1627 ctrlr->feat.power_management.raw = cmd->cdw11; 1628 ctrlr->feat.power_management.bits.reserved = 0; 1629 1630 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1631 } 1632 1633 static bool 1634 temp_threshold_opts_valid(const union spdk_nvme_feat_temperature_threshold *opts) 1635 { 1636 /* 1637 * Valid TMPSEL values: 1638 * 0000b - 1000b: temperature sensors 1639 * 1111b: set all implemented temperature sensors 1640 */ 1641 if (opts->bits.tmpsel >= 9 && opts->bits.tmpsel != 15) { 1642 /* 1001b - 1110b: reserved */ 1643 SPDK_ERRLOG("Invalid TMPSEL %u\n", opts->bits.tmpsel); 1644 return false; 1645 } 1646 1647 /* 1648 * Valid THSEL values: 1649 * 00b: over temperature threshold 1650 * 01b: under temperature threshold 1651 */ 1652 if (opts->bits.thsel > 1) { 1653 /* 10b - 11b: reserved */ 1654 SPDK_ERRLOG("Invalid THSEL %u\n", opts->bits.thsel); 1655 return false; 1656 } 1657 1658 return true; 1659 } 1660 1661 static int 1662 nvmf_ctrlr_set_features_temperature_threshold(struct spdk_nvmf_request *req) 1663 { 1664 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1665 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1666 1667 SPDK_DEBUGLOG(nvmf, "Set Features - Temperature Threshold (cdw11 = 0x%0x)\n", cmd->cdw11); 1668 1669 if (!temp_threshold_opts_valid(&cmd->cdw11_bits.feat_temp_threshold)) { 1670 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1671 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1672 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1673 } 1674 1675 /* TODO: no sensors implemented - ignore new values */ 1676 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1677 } 1678 1679 static int 1680 nvmf_ctrlr_get_features_temperature_threshold(struct spdk_nvmf_request *req) 1681 { 1682 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1683 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1684 1685 SPDK_DEBUGLOG(nvmf, "Get Features - Temperature Threshold (cdw11 = 0x%0x)\n", cmd->cdw11); 1686 1687 if (!temp_threshold_opts_valid(&cmd->cdw11_bits.feat_temp_threshold)) { 1688 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1689 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1690 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1691 } 1692 1693 /* TODO: no sensors implemented - return 0 for all thresholds */ 1694 rsp->cdw0 = 0; 1695 1696 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1697 } 1698 1699 static int 1700 nvmf_ctrlr_get_features_interrupt_vector_configuration(struct spdk_nvmf_request *req) 1701 { 1702 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1703 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1704 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1705 union spdk_nvme_feat_interrupt_vector_configuration iv_conf = {}; 1706 1707 SPDK_DEBUGLOG(nvmf, "Get Features - Interrupt Vector Configuration (cdw11 = 0x%0x)\n", cmd->cdw11); 1708 1709 iv_conf.bits.iv = cmd->cdw11_bits.feat_interrupt_vector_configuration.bits.iv; 1710 iv_conf.bits.cd = ctrlr->feat.interrupt_vector_configuration.bits.cd; 1711 rsp->cdw0 = iv_conf.raw; 1712 1713 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1714 } 1715 1716 static int 1717 nvmf_ctrlr_set_features_error_recovery(struct spdk_nvmf_request *req) 1718 { 1719 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1720 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1721 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1722 1723 SPDK_DEBUGLOG(nvmf, "Set Features - Error Recovery (cdw11 = 0x%0x)\n", cmd->cdw11); 1724 1725 if (cmd->cdw11_bits.feat_error_recovery.bits.dulbe) { 1726 /* 1727 * Host is not allowed to set this bit, since we don't advertise it in 1728 * Identify Namespace. 1729 */ 1730 SPDK_ERRLOG("Host set unsupported DULBE bit\n"); 1731 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1732 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1733 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1734 } 1735 1736 ctrlr->feat.error_recovery.raw = cmd->cdw11; 1737 ctrlr->feat.error_recovery.bits.reserved = 0; 1738 1739 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1740 } 1741 1742 static int 1743 nvmf_ctrlr_set_features_volatile_write_cache(struct spdk_nvmf_request *req) 1744 { 1745 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1746 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1747 1748 SPDK_DEBUGLOG(nvmf, "Set Features - Volatile Write Cache (cdw11 = 0x%0x)\n", cmd->cdw11); 1749 1750 ctrlr->feat.volatile_write_cache.raw = cmd->cdw11; 1751 ctrlr->feat.volatile_write_cache.bits.reserved = 0; 1752 1753 SPDK_DEBUGLOG(nvmf, "Set Features - Volatile Write Cache %s\n", 1754 ctrlr->feat.volatile_write_cache.bits.wce ? "Enabled" : "Disabled"); 1755 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1756 } 1757 1758 static int 1759 nvmf_ctrlr_set_features_write_atomicity(struct spdk_nvmf_request *req) 1760 { 1761 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1762 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1763 1764 SPDK_DEBUGLOG(nvmf, "Set Features - Write Atomicity (cdw11 = 0x%0x)\n", cmd->cdw11); 1765 1766 ctrlr->feat.write_atomicity.raw = cmd->cdw11; 1767 ctrlr->feat.write_atomicity.bits.reserved = 0; 1768 1769 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1770 } 1771 1772 static int 1773 nvmf_ctrlr_set_features_host_identifier(struct spdk_nvmf_request *req) 1774 { 1775 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1776 1777 SPDK_ERRLOG("Set Features - Host Identifier not allowed\n"); 1778 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 1779 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1780 } 1781 1782 static int 1783 nvmf_ctrlr_get_features_host_identifier(struct spdk_nvmf_request *req) 1784 { 1785 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1786 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1787 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1788 struct spdk_iov_xfer ix; 1789 1790 SPDK_DEBUGLOG(nvmf, "Get Features - Host Identifier\n"); 1791 1792 if (!cmd->cdw11_bits.feat_host_identifier.bits.exhid) { 1793 /* NVMe over Fabrics requires EXHID=1 (128-bit/16-byte host ID) */ 1794 SPDK_ERRLOG("Get Features - Host Identifier with EXHID=0 not allowed\n"); 1795 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1796 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1797 } 1798 1799 if (req->iovcnt < 1 || req->length < sizeof(ctrlr->hostid)) { 1800 SPDK_ERRLOG("Invalid data buffer for Get Features - Host Identifier\n"); 1801 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1802 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1803 } 1804 1805 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 1806 spdk_iov_xfer_from_buf(&ix, &ctrlr->hostid, sizeof(ctrlr->hostid)); 1807 1808 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1809 } 1810 1811 static int 1812 nvmf_ctrlr_get_features_reservation_notification_mask(struct spdk_nvmf_request *req) 1813 { 1814 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1815 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1816 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1817 struct spdk_nvmf_ns *ns; 1818 1819 SPDK_DEBUGLOG(nvmf, "get Features - Reservation Notification Mask\n"); 1820 1821 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1822 SPDK_ERRLOG("get Features - Invalid Namespace ID\n"); 1823 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1824 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1825 } 1826 1827 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 1828 if (ns == NULL) { 1829 SPDK_ERRLOG("get Features - Invalid Namespace ID\n"); 1830 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1831 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1832 } 1833 rsp->cdw0 = ns->mask; 1834 1835 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1836 } 1837 1838 static int 1839 nvmf_ctrlr_set_features_reservation_notification_mask(struct spdk_nvmf_request *req) 1840 { 1841 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1842 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 1843 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1844 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1845 struct spdk_nvmf_ns *ns; 1846 1847 SPDK_DEBUGLOG(nvmf, "Set Features - Reservation Notification Mask\n"); 1848 1849 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1850 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 1851 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 1852 ns->mask = cmd->cdw11; 1853 } 1854 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1855 } 1856 1857 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 1858 if (ns == NULL) { 1859 SPDK_ERRLOG("Set Features - Invalid Namespace ID\n"); 1860 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1861 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1862 } 1863 ns->mask = cmd->cdw11; 1864 1865 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1866 } 1867 1868 static int 1869 nvmf_ctrlr_get_features_reservation_persistence(struct spdk_nvmf_request *req) 1870 { 1871 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1872 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1873 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1874 struct spdk_nvmf_ns *ns; 1875 1876 SPDK_DEBUGLOG(nvmf, "Get Features - Reservation Persistence\n"); 1877 1878 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 1879 /* NSID with SPDK_NVME_GLOBAL_NS_TAG (=0xffffffff) also included */ 1880 if (ns == NULL) { 1881 SPDK_ERRLOG("Get Features - Invalid Namespace ID\n"); 1882 response->status.sct = SPDK_NVME_SCT_GENERIC; 1883 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1884 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1885 } 1886 1887 response->cdw0 = ns->ptpl_activated; 1888 1889 response->status.sct = SPDK_NVME_SCT_GENERIC; 1890 response->status.sc = SPDK_NVME_SC_SUCCESS; 1891 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1892 } 1893 1894 static int 1895 nvmf_ctrlr_set_features_reservation_persistence(struct spdk_nvmf_request *req) 1896 { 1897 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1898 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1899 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1900 struct spdk_nvmf_ns *ns; 1901 bool ptpl; 1902 1903 SPDK_DEBUGLOG(nvmf, "Set Features - Reservation Persistence\n"); 1904 1905 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 1906 ptpl = cmd->cdw11_bits.feat_rsv_persistence.bits.ptpl; 1907 1908 if (cmd->nsid != SPDK_NVME_GLOBAL_NS_TAG && ns && nvmf_ns_is_ptpl_capable(ns)) { 1909 ns->ptpl_activated = ptpl; 1910 } else if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1911 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns; 1912 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 1913 if (nvmf_ns_is_ptpl_capable(ns)) { 1914 ns->ptpl_activated = ptpl; 1915 } 1916 } 1917 } else { 1918 SPDK_ERRLOG("Set Features - Invalid Namespace ID or Reservation Configuration\n"); 1919 response->status.sct = SPDK_NVME_SCT_GENERIC; 1920 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1921 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1922 } 1923 1924 /* TODO: Feature not changeable for now */ 1925 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 1926 response->status.sc = SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE; 1927 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1928 } 1929 1930 static int 1931 nvmf_ctrlr_get_features_host_behavior_support(struct spdk_nvmf_request *req) 1932 { 1933 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1934 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1935 struct spdk_nvme_host_behavior host_behavior = {}; 1936 struct spdk_iov_xfer ix; 1937 1938 SPDK_DEBUGLOG(nvmf, "Get Features - Host Behavior Support\n"); 1939 1940 if (req->iovcnt < 1 || req->length < sizeof(struct spdk_nvme_host_behavior)) { 1941 SPDK_ERRLOG("invalid data buffer for Host Behavior Support\n"); 1942 response->status.sct = SPDK_NVME_SCT_GENERIC; 1943 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1944 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1945 } 1946 1947 host_behavior.acre = ctrlr->acre_enabled; 1948 1949 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 1950 spdk_iov_xfer_from_buf(&ix, &host_behavior, sizeof(host_behavior)); 1951 1952 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1953 } 1954 1955 static int 1956 nvmf_ctrlr_set_features_host_behavior_support(struct spdk_nvmf_request *req) 1957 { 1958 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1959 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1960 struct spdk_nvme_host_behavior *host_behavior; 1961 1962 SPDK_DEBUGLOG(nvmf, "Set Features - Host Behavior Support\n"); 1963 if (req->iovcnt != 1) { 1964 SPDK_ERRLOG("Host Behavior Support invalid iovcnt: %d\n", req->iovcnt); 1965 response->status.sct = SPDK_NVME_SCT_GENERIC; 1966 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1967 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1968 } 1969 if (req->iov[0].iov_len != sizeof(struct spdk_nvme_host_behavior)) { 1970 SPDK_ERRLOG("Host Behavior Support invalid iov_len: %zd\n", req->iov[0].iov_len); 1971 response->status.sct = SPDK_NVME_SCT_GENERIC; 1972 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1973 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1974 } 1975 1976 host_behavior = (struct spdk_nvme_host_behavior *)req->iov[0].iov_base; 1977 if (host_behavior->acre == 0) { 1978 ctrlr->acre_enabled = false; 1979 } else if (host_behavior->acre == 1) { 1980 ctrlr->acre_enabled = true; 1981 } else { 1982 SPDK_ERRLOG("Host Behavior Support invalid acre: 0x%02x\n", host_behavior->acre); 1983 response->status.sct = SPDK_NVME_SCT_GENERIC; 1984 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 1985 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1986 } 1987 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1988 } 1989 1990 static int 1991 nvmf_ctrlr_set_features_keep_alive_timer(struct spdk_nvmf_request *req) 1992 { 1993 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 1994 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 1995 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1996 1997 SPDK_DEBUGLOG(nvmf, "Set Features - Keep Alive Timer (%u ms)\n", cmd->cdw11); 1998 1999 /* 2000 * if attempts to disable keep alive by setting kato to 0h 2001 * a status value of keep alive invalid shall be returned 2002 */ 2003 if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato == 0) { 2004 rsp->status.sc = SPDK_NVME_SC_KEEP_ALIVE_INVALID; 2005 } else if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato < MIN_KEEP_ALIVE_TIMEOUT_IN_MS) { 2006 ctrlr->feat.keep_alive_timer.bits.kato = MIN_KEEP_ALIVE_TIMEOUT_IN_MS; 2007 } else { 2008 /* round up to milliseconds */ 2009 ctrlr->feat.keep_alive_timer.bits.kato = spdk_divide_round_up( 2010 cmd->cdw11_bits.feat_keep_alive_timer.bits.kato, 2011 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS) * 2012 KAS_DEFAULT_VALUE * KAS_TIME_UNIT_IN_MS; 2013 } 2014 2015 /* 2016 * if change the keep alive timeout value successfully 2017 * update the keep alive poller. 2018 */ 2019 if (cmd->cdw11_bits.feat_keep_alive_timer.bits.kato != 0) { 2020 if (ctrlr->keep_alive_poller != NULL) { 2021 spdk_poller_unregister(&ctrlr->keep_alive_poller); 2022 } 2023 ctrlr->keep_alive_poller = SPDK_POLLER_REGISTER(nvmf_ctrlr_keep_alive_poll, ctrlr, 2024 ctrlr->feat.keep_alive_timer.bits.kato * 1000); 2025 } 2026 2027 SPDK_DEBUGLOG(nvmf, "Set Features - Keep Alive Timer set to %u ms\n", 2028 ctrlr->feat.keep_alive_timer.bits.kato); 2029 2030 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2031 } 2032 2033 static int 2034 nvmf_ctrlr_set_features_number_of_queues(struct spdk_nvmf_request *req) 2035 { 2036 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2037 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2038 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 2039 uint32_t count; 2040 2041 SPDK_DEBUGLOG(nvmf, "Set Features - Number of Queues, cdw11 0x%x\n", 2042 req->cmd->nvme_cmd.cdw11); 2043 2044 if (cmd->cdw11_bits.feat_num_of_queues.bits.ncqr == UINT16_MAX || 2045 cmd->cdw11_bits.feat_num_of_queues.bits.nsqr == UINT16_MAX) { 2046 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2047 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2048 } 2049 2050 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 2051 /* verify that the controller is ready to process commands */ 2052 if (count > 1) { 2053 SPDK_DEBUGLOG(nvmf, "Queue pairs already active!\n"); 2054 rsp->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 2055 } else { 2056 /* 2057 * Ignore the value requested by the host - 2058 * always return the pre-configured value based on max_qpairs_allowed. 2059 */ 2060 rsp->cdw0 = ctrlr->feat.number_of_queues.raw; 2061 } 2062 2063 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2064 } 2065 2066 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_ctrlr) == 4928, 2067 "Please check migration fields that need to be added or not"); 2068 2069 static void 2070 nvmf_ctrlr_migr_data_copy(struct spdk_nvmf_ctrlr_migr_data *data, 2071 const struct spdk_nvmf_ctrlr_migr_data *data_src, size_t data_size) 2072 { 2073 assert(data); 2074 assert(data_src); 2075 assert(data_size); 2076 2077 memcpy(&data->regs, &data_src->regs, spdk_min(data->regs_size, data_src->regs_size)); 2078 memcpy(&data->feat, &data_src->feat, spdk_min(data->feat_size, data_src->feat_size)); 2079 2080 #define SET_FIELD(field) \ 2081 if (offsetof(struct spdk_nvmf_ctrlr_migr_data, field) + sizeof(data->field) <= data_size) { \ 2082 data->field = data_src->field; \ 2083 } \ 2084 2085 SET_FIELD(cntlid); 2086 SET_FIELD(acre); 2087 SET_FIELD(num_aer_cids); 2088 SET_FIELD(num_async_events); 2089 SET_FIELD(notice_aen_mask); 2090 #undef SET_FIELD 2091 2092 #define SET_ARRAY(arr) \ 2093 if (offsetof(struct spdk_nvmf_ctrlr_migr_data, arr) + sizeof(data->arr) <= data_size) { \ 2094 memcpy(&data->arr, &data_src->arr, sizeof(data->arr)); \ 2095 } \ 2096 2097 SET_ARRAY(async_events); 2098 SET_ARRAY(aer_cids); 2099 #undef SET_ARRAY 2100 } 2101 2102 int 2103 spdk_nvmf_ctrlr_save_migr_data(struct spdk_nvmf_ctrlr *ctrlr, 2104 struct spdk_nvmf_ctrlr_migr_data *data) 2105 { 2106 struct spdk_nvmf_async_event_completion *event, *event_tmp; 2107 uint32_t i; 2108 struct spdk_nvmf_ctrlr_migr_data data_local = { 2109 .data_size = offsetof(struct spdk_nvmf_ctrlr_migr_data, unused), 2110 .regs_size = sizeof(struct spdk_nvmf_registers), 2111 .feat_size = sizeof(struct spdk_nvmf_ctrlr_feat) 2112 }; 2113 2114 assert(data->data_size <= sizeof(data_local)); 2115 assert(spdk_get_thread() == ctrlr->thread); 2116 2117 memcpy(&data_local.regs, &ctrlr->vcprop, sizeof(struct spdk_nvmf_registers)); 2118 memcpy(&data_local.feat, &ctrlr->feat, sizeof(struct spdk_nvmf_ctrlr_feat)); 2119 2120 data_local.cntlid = ctrlr->cntlid; 2121 data_local.acre = ctrlr->acre_enabled; 2122 data_local.num_aer_cids = ctrlr->nr_aer_reqs; 2123 2124 STAILQ_FOREACH_SAFE(event, &ctrlr->async_events, link, event_tmp) { 2125 if (data_local.num_async_events + 1 > SPDK_NVMF_MIGR_MAX_PENDING_AERS) { 2126 SPDK_ERRLOG("ctrlr %p has too many pending AERs\n", ctrlr); 2127 break; 2128 } 2129 2130 data_local.async_events[data_local.num_async_events++].raw = event->event.raw; 2131 } 2132 2133 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 2134 struct spdk_nvmf_request *req = ctrlr->aer_req[i]; 2135 data_local.aer_cids[i] = req->cmd->nvme_cmd.cid; 2136 } 2137 data_local.notice_aen_mask = ctrlr->notice_aen_mask; 2138 2139 nvmf_ctrlr_migr_data_copy(data, &data_local, spdk_min(data->data_size, data_local.data_size)); 2140 return 0; 2141 } 2142 2143 int 2144 spdk_nvmf_ctrlr_restore_migr_data(struct spdk_nvmf_ctrlr *ctrlr, 2145 const struct spdk_nvmf_ctrlr_migr_data *data) 2146 { 2147 uint32_t i; 2148 struct spdk_nvmf_ctrlr_migr_data data_local = { 2149 .data_size = offsetof(struct spdk_nvmf_ctrlr_migr_data, unused), 2150 .regs_size = sizeof(struct spdk_nvmf_registers), 2151 .feat_size = sizeof(struct spdk_nvmf_ctrlr_feat) 2152 }; 2153 2154 assert(data->data_size <= sizeof(data_local)); 2155 assert(spdk_get_thread() == ctrlr->thread); 2156 2157 /* local version of data should have defaults set before copy */ 2158 nvmf_ctrlr_migr_data_copy(&data_local, data, spdk_min(data->data_size, data_local.data_size)); 2159 memcpy(&ctrlr->vcprop, &data_local.regs, sizeof(struct spdk_nvmf_registers)); 2160 memcpy(&ctrlr->feat, &data_local.feat, sizeof(struct spdk_nvmf_ctrlr_feat)); 2161 2162 ctrlr->cntlid = data_local.cntlid; 2163 ctrlr->acre_enabled = data_local.acre; 2164 2165 for (i = 0; i < data_local.num_async_events; i++) { 2166 struct spdk_nvmf_async_event_completion *event; 2167 2168 event = calloc(1, sizeof(*event)); 2169 if (!event) { 2170 return -ENOMEM; 2171 } 2172 2173 event->event.raw = data_local.async_events[i].raw; 2174 STAILQ_INSERT_TAIL(&ctrlr->async_events, event, link); 2175 } 2176 ctrlr->notice_aen_mask = data_local.notice_aen_mask; 2177 2178 return 0; 2179 } 2180 2181 static int 2182 nvmf_ctrlr_set_features_async_event_configuration(struct spdk_nvmf_request *req) 2183 { 2184 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2185 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2186 2187 SPDK_DEBUGLOG(nvmf, "Set Features - Async Event Configuration, cdw11 0x%08x\n", 2188 cmd->cdw11); 2189 ctrlr->feat.async_event_configuration.raw = cmd->cdw11; 2190 ctrlr->feat.async_event_configuration.bits.reserved1 = 0; 2191 ctrlr->feat.async_event_configuration.bits.reserved2 = 0; 2192 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2193 } 2194 2195 static int 2196 nvmf_ctrlr_async_event_request(struct spdk_nvmf_request *req) 2197 { 2198 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2199 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 2200 struct spdk_nvmf_async_event_completion *pending_event; 2201 2202 SPDK_DEBUGLOG(nvmf, "Async Event Request\n"); 2203 2204 /* Four asynchronous events are supported for now */ 2205 if (ctrlr->nr_aer_reqs >= SPDK_NVMF_MAX_ASYNC_EVENTS) { 2206 SPDK_DEBUGLOG(nvmf, "AERL exceeded\n"); 2207 rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 2208 rsp->status.sc = SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED; 2209 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2210 } 2211 2212 if (!STAILQ_EMPTY(&ctrlr->async_events)) { 2213 pending_event = STAILQ_FIRST(&ctrlr->async_events); 2214 rsp->cdw0 = pending_event->event.raw; 2215 STAILQ_REMOVE(&ctrlr->async_events, pending_event, spdk_nvmf_async_event_completion, link); 2216 free(pending_event); 2217 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2218 } 2219 2220 ctrlr->aer_req[ctrlr->nr_aer_reqs++] = req; 2221 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 2222 } 2223 2224 static void 2225 nvmf_get_firmware_slot_log_page(struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length) 2226 { 2227 struct spdk_nvme_firmware_page fw_page; 2228 size_t copy_len; 2229 struct spdk_iov_xfer ix; 2230 2231 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2232 2233 memset(&fw_page, 0, sizeof(fw_page)); 2234 fw_page.afi.active_slot = 1; 2235 fw_page.afi.next_reset_slot = 0; 2236 spdk_strcpy_pad(fw_page.revision[0], FW_VERSION, sizeof(fw_page.revision[0]), ' '); 2237 2238 if (offset < sizeof(fw_page)) { 2239 copy_len = spdk_min(sizeof(fw_page) - offset, length); 2240 if (copy_len > 0) { 2241 spdk_iov_xfer_from_buf(&ix, (const char *)&fw_page + offset, copy_len); 2242 } 2243 } 2244 } 2245 2246 /* 2247 * Asynchronous Event Mask Bit 2248 */ 2249 enum spdk_nvme_async_event_mask_bit { 2250 /* Mask Namespace Change Notification */ 2251 SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT = 0, 2252 /* Mask Asymmetric Namespace Access Change Notification */ 2253 SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT = 1, 2254 /* Mask Discovery Log Change Notification */ 2255 SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT = 2, 2256 /* Mask Reservation Log Page Available Notification */ 2257 SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT = 3, 2258 /* Mask Error Event */ 2259 SPDK_NVME_ASYNC_EVENT_ERROR_MASK_BIT = 4, 2260 /* 4 - 63 Reserved */ 2261 }; 2262 2263 static inline void 2264 nvmf_ctrlr_unmask_aen(struct spdk_nvmf_ctrlr *ctrlr, 2265 enum spdk_nvme_async_event_mask_bit mask) 2266 { 2267 ctrlr->notice_aen_mask &= ~(1 << mask); 2268 } 2269 2270 static inline bool 2271 nvmf_ctrlr_mask_aen(struct spdk_nvmf_ctrlr *ctrlr, 2272 enum spdk_nvme_async_event_mask_bit mask) 2273 { 2274 if (ctrlr->notice_aen_mask & (1 << mask)) { 2275 return false; 2276 } else { 2277 ctrlr->notice_aen_mask |= (1 << mask); 2278 return true; 2279 } 2280 } 2281 2282 /* we have to use the typedef in the function declaration to appease astyle. */ 2283 typedef enum spdk_nvme_ana_state spdk_nvme_ana_state_t; 2284 2285 static inline spdk_nvme_ana_state_t 2286 nvmf_ctrlr_get_ana_state(struct spdk_nvmf_ctrlr *ctrlr, uint32_t anagrpid) 2287 { 2288 if (!ctrlr->subsys->flags.ana_reporting) { 2289 return SPDK_NVME_ANA_OPTIMIZED_STATE; 2290 } 2291 2292 if (spdk_unlikely(ctrlr->listener == NULL)) { 2293 return SPDK_NVME_ANA_INACCESSIBLE_STATE; 2294 } 2295 2296 assert(anagrpid - 1 < ctrlr->subsys->max_nsid); 2297 return ctrlr->listener->ana_state[anagrpid - 1]; 2298 } 2299 2300 static spdk_nvme_ana_state_t 2301 nvmf_ctrlr_get_ana_state_from_nsid(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid) 2302 { 2303 struct spdk_nvmf_ns *ns; 2304 2305 /* We do not have NVM subsystem specific ANA state. Hence if NSID is either 2306 * SPDK_NVMF_GLOBAL_NS_TAG, invalid, or for inactive namespace, return 2307 * the optimized state. 2308 */ 2309 ns = nvmf_ctrlr_get_ns(ctrlr, nsid); 2310 if (ns == NULL) { 2311 return SPDK_NVME_ANA_OPTIMIZED_STATE; 2312 } 2313 2314 return nvmf_ctrlr_get_ana_state(ctrlr, ns->anagrpid); 2315 } 2316 2317 static void 2318 nvmf_get_error_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iovcnt, 2319 uint64_t offset, uint32_t length, uint32_t rae) 2320 { 2321 if (!rae) { 2322 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ERROR_MASK_BIT); 2323 } 2324 2325 /* TODO: actually fill out log page data */ 2326 } 2327 2328 static void 2329 nvmf_get_ana_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iovcnt, 2330 uint64_t offset, uint32_t length, uint32_t rae) 2331 { 2332 struct spdk_nvme_ana_page ana_hdr; 2333 struct spdk_nvme_ana_group_descriptor ana_desc; 2334 size_t copy_len, copied_len; 2335 uint32_t num_anagrp = 0, anagrpid; 2336 struct spdk_nvmf_ns *ns; 2337 struct spdk_iov_xfer ix; 2338 2339 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2340 2341 if (length == 0) { 2342 goto done; 2343 } 2344 2345 if (offset >= sizeof(ana_hdr)) { 2346 offset -= sizeof(ana_hdr); 2347 } else { 2348 for (anagrpid = 1; anagrpid <= ctrlr->subsys->max_nsid; anagrpid++) { 2349 if (ctrlr->subsys->ana_group[anagrpid - 1] > 0) { 2350 num_anagrp++; 2351 } 2352 } 2353 2354 memset(&ana_hdr, 0, sizeof(ana_hdr)); 2355 2356 ana_hdr.num_ana_group_desc = num_anagrp; 2357 /* TODO: Support Change Count. */ 2358 ana_hdr.change_count = 0; 2359 2360 copy_len = spdk_min(sizeof(ana_hdr) - offset, length); 2361 copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ana_hdr + offset, copy_len); 2362 assert(copied_len == copy_len); 2363 length -= copied_len; 2364 offset = 0; 2365 } 2366 2367 if (length == 0) { 2368 goto done; 2369 } 2370 2371 for (anagrpid = 1; anagrpid <= ctrlr->subsys->max_nsid; anagrpid++) { 2372 if (ctrlr->subsys->ana_group[anagrpid - 1] == 0) { 2373 continue; 2374 } 2375 2376 if (offset >= sizeof(ana_desc)) { 2377 offset -= sizeof(ana_desc); 2378 } else { 2379 memset(&ana_desc, 0, sizeof(ana_desc)); 2380 2381 ana_desc.ana_group_id = anagrpid; 2382 ana_desc.num_of_nsid = ctrlr->subsys->ana_group[anagrpid - 1]; 2383 ana_desc.ana_state = nvmf_ctrlr_get_ana_state(ctrlr, anagrpid); 2384 2385 copy_len = spdk_min(sizeof(ana_desc) - offset, length); 2386 copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ana_desc + offset, 2387 copy_len); 2388 assert(copied_len == copy_len); 2389 length -= copied_len; 2390 offset = 0; 2391 2392 if (length == 0) { 2393 goto done; 2394 } 2395 } 2396 2397 /* TODO: Revisit here about O(n^2) cost if we have subsystem with 2398 * many namespaces in the future. 2399 */ 2400 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 2401 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 2402 if (ns->anagrpid != anagrpid) { 2403 continue; 2404 } 2405 2406 if (offset >= sizeof(uint32_t)) { 2407 offset -= sizeof(uint32_t); 2408 continue; 2409 } 2410 2411 copy_len = spdk_min(sizeof(uint32_t) - offset, length); 2412 copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ns->nsid + offset, 2413 copy_len); 2414 assert(copied_len == copy_len); 2415 length -= copied_len; 2416 offset = 0; 2417 2418 if (length == 0) { 2419 goto done; 2420 } 2421 } 2422 } 2423 2424 done: 2425 if (!rae) { 2426 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT); 2427 } 2428 } 2429 2430 void 2431 nvmf_ctrlr_ns_changed(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid) 2432 { 2433 uint16_t max_changes = SPDK_COUNTOF(ctrlr->changed_ns_list.ns_list); 2434 uint16_t i; 2435 bool found = false; 2436 2437 for (i = 0; i < ctrlr->changed_ns_list_count; i++) { 2438 if (ctrlr->changed_ns_list.ns_list[i] == nsid) { 2439 /* nsid is already in the list */ 2440 found = true; 2441 break; 2442 } 2443 } 2444 2445 if (!found) { 2446 if (ctrlr->changed_ns_list_count == max_changes) { 2447 /* Out of space - set first entry to FFFFFFFFh and zero-fill the rest. */ 2448 ctrlr->changed_ns_list.ns_list[0] = 0xFFFFFFFFu; 2449 for (i = 1; i < max_changes; i++) { 2450 ctrlr->changed_ns_list.ns_list[i] = 0; 2451 } 2452 } else { 2453 ctrlr->changed_ns_list.ns_list[ctrlr->changed_ns_list_count++] = nsid; 2454 } 2455 } 2456 } 2457 2458 static void 2459 nvmf_get_changed_ns_list_log_page(struct spdk_nvmf_ctrlr *ctrlr, 2460 struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length, uint32_t rae) 2461 { 2462 size_t copy_length; 2463 struct spdk_iov_xfer ix; 2464 2465 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2466 2467 if (offset < sizeof(ctrlr->changed_ns_list)) { 2468 copy_length = spdk_min(length, sizeof(ctrlr->changed_ns_list) - offset); 2469 if (copy_length) { 2470 spdk_iov_xfer_from_buf(&ix, (char *)&ctrlr->changed_ns_list + offset, copy_length); 2471 } 2472 } 2473 2474 /* Clear log page each time it is read */ 2475 ctrlr->changed_ns_list_count = 0; 2476 memset(&ctrlr->changed_ns_list, 0, sizeof(ctrlr->changed_ns_list)); 2477 2478 if (!rae) { 2479 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT); 2480 } 2481 } 2482 2483 /* The structure can be modified if we provide support for other commands in future */ 2484 static const struct spdk_nvme_cmds_and_effect_log_page g_cmds_and_effect_log_page = { 2485 .admin_cmds_supported = { 2486 /* CSUPP, LBCC, NCC, NIC, CCC, CSE */ 2487 /* Get Log Page */ 2488 [SPDK_NVME_OPC_GET_LOG_PAGE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2489 /* Identify */ 2490 [SPDK_NVME_OPC_IDENTIFY] = {1, 0, 0, 0, 0, 0, 0, 0}, 2491 /* Abort */ 2492 [SPDK_NVME_OPC_ABORT] = {1, 0, 0, 0, 0, 0, 0, 0}, 2493 /* Set Features */ 2494 [SPDK_NVME_OPC_SET_FEATURES] = {1, 0, 0, 0, 0, 0, 0, 0}, 2495 /* Get Features */ 2496 [SPDK_NVME_OPC_GET_FEATURES] = {1, 0, 0, 0, 0, 0, 0, 0}, 2497 /* Async Event Request */ 2498 [SPDK_NVME_OPC_ASYNC_EVENT_REQUEST] = {1, 0, 0, 0, 0, 0, 0, 0}, 2499 /* Keep Alive */ 2500 [SPDK_NVME_OPC_KEEP_ALIVE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2501 }, 2502 .io_cmds_supported = { 2503 /* FLUSH */ 2504 [SPDK_NVME_OPC_FLUSH] = {1, 1, 0, 0, 0, 0, 0, 0}, 2505 /* WRITE */ 2506 [SPDK_NVME_OPC_WRITE] = {1, 1, 0, 0, 0, 0, 0, 0}, 2507 /* READ */ 2508 [SPDK_NVME_OPC_READ] = {1, 0, 0, 0, 0, 0, 0, 0}, 2509 /* WRITE ZEROES */ 2510 [SPDK_NVME_OPC_WRITE_ZEROES] = {1, 1, 0, 0, 0, 0, 0, 0}, 2511 /* DATASET MANAGEMENT */ 2512 [SPDK_NVME_OPC_DATASET_MANAGEMENT] = {1, 1, 0, 0, 0, 0, 0, 0}, 2513 /* COMPARE */ 2514 [SPDK_NVME_OPC_COMPARE] = {1, 0, 0, 0, 0, 0, 0, 0}, 2515 /* ZONE MANAGEMENT SEND */ 2516 [SPDK_NVME_OPC_ZONE_MGMT_SEND] = {1, 1, 0, 0, 0, 0, 0, 0}, 2517 /* ZONE MANAGEMENT RECEIVE */ 2518 [SPDK_NVME_OPC_ZONE_MGMT_RECV] = {1, 0, 0, 0, 0, 0, 0, 0}, 2519 /* COPY */ 2520 [SPDK_NVME_OPC_COPY] = {1, 1, 0, 0, 0, 0, 0, 0}, 2521 }, 2522 }; 2523 2524 static void 2525 nvmf_get_cmds_and_effects_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iovcnt, 2526 uint64_t offset, uint32_t length) 2527 { 2528 uint32_t page_size = sizeof(struct spdk_nvme_cmds_and_effect_log_page); 2529 size_t copy_len = 0; 2530 struct spdk_nvme_cmds_and_effect_log_page cmds_and_effect_log_page = g_cmds_and_effect_log_page; 2531 struct spdk_nvme_cmds_and_effect_entry csupp_and_lbcc_effect_entry = {1, 1, 0, 0, 0, 0, 0, 0}; 2532 struct spdk_iov_xfer ix; 2533 2534 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2535 2536 if (offset < page_size) { 2537 if (ctrlr->subsys->zone_append_supported) { 2538 cmds_and_effect_log_page.io_cmds_supported[SPDK_NVME_OPC_ZONE_APPEND] = 2539 csupp_and_lbcc_effect_entry; 2540 } 2541 copy_len = spdk_min(page_size - offset, length); 2542 spdk_iov_xfer_from_buf(&ix, (char *)(&cmds_and_effect_log_page) + offset, copy_len); 2543 } 2544 } 2545 2546 static void 2547 nvmf_get_reservation_notification_log_page(struct spdk_nvmf_ctrlr *ctrlr, 2548 struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length, uint32_t rae) 2549 { 2550 uint32_t unit_log_len, avail_log_len, next_pos, copy_len; 2551 struct spdk_nvmf_reservation_log *log, *log_tmp; 2552 struct spdk_iov_xfer ix; 2553 2554 spdk_iov_xfer_init(&ix, iovs, iovcnt); 2555 2556 unit_log_len = sizeof(struct spdk_nvme_reservation_notification_log); 2557 /* No available log, return zeroed log pages */ 2558 if (!ctrlr->num_avail_log_pages) { 2559 return; 2560 } 2561 2562 avail_log_len = ctrlr->num_avail_log_pages * unit_log_len; 2563 if (offset >= avail_log_len) { 2564 return; 2565 } 2566 2567 next_pos = 0; 2568 TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) { 2569 TAILQ_REMOVE(&ctrlr->log_head, log, link); 2570 ctrlr->num_avail_log_pages--; 2571 2572 next_pos += unit_log_len; 2573 if (next_pos > offset) { 2574 copy_len = spdk_min(next_pos - offset, length); 2575 spdk_iov_xfer_from_buf(&ix, &log->log, copy_len); 2576 length -= copy_len; 2577 offset += copy_len; 2578 } 2579 free(log); 2580 2581 if (length == 0) { 2582 break; 2583 } 2584 } 2585 2586 if (!rae) { 2587 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT); 2588 } 2589 return; 2590 } 2591 2592 static int 2593 nvmf_ctrlr_get_log_page(struct spdk_nvmf_request *req) 2594 { 2595 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 2596 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2597 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 2598 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 2599 struct spdk_nvme_transport_id cmd_source_trid; 2600 uint64_t offset, len; 2601 uint32_t rae, numdl, numdu; 2602 uint8_t lid; 2603 2604 if (req->iovcnt < 1) { 2605 SPDK_DEBUGLOG(nvmf, "get log command with no buffer\n"); 2606 response->status.sct = SPDK_NVME_SCT_GENERIC; 2607 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2608 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2609 } 2610 2611 offset = (uint64_t)cmd->cdw12 | ((uint64_t)cmd->cdw13 << 32); 2612 if (offset & 3) { 2613 SPDK_ERRLOG("Invalid log page offset 0x%" PRIx64 "\n", offset); 2614 response->status.sct = SPDK_NVME_SCT_GENERIC; 2615 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2616 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2617 } 2618 2619 rae = cmd->cdw10_bits.get_log_page.rae; 2620 numdl = cmd->cdw10_bits.get_log_page.numdl; 2621 numdu = cmd->cdw11_bits.get_log_page.numdu; 2622 len = ((numdu << 16) + numdl + (uint64_t)1) * 4; 2623 if (len > req->length) { 2624 SPDK_ERRLOG("Get log page: len (%" PRIu64 ") > buf size (%u)\n", 2625 len, req->length); 2626 response->status.sct = SPDK_NVME_SCT_GENERIC; 2627 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2628 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2629 } 2630 2631 lid = cmd->cdw10_bits.get_log_page.lid; 2632 SPDK_DEBUGLOG(nvmf, "Get log page: LID=0x%02X offset=0x%" PRIx64 " len=0x%" PRIx64 " rae=%u\n", 2633 lid, offset, len, rae); 2634 2635 if (spdk_nvmf_subsystem_is_discovery(subsystem)) { 2636 switch (lid) { 2637 case SPDK_NVME_LOG_DISCOVERY: 2638 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &cmd_source_trid)) { 2639 SPDK_ERRLOG("Failed to get LOG_DISCOVERY source trid\n"); 2640 response->status.sct = SPDK_NVME_SCT_GENERIC; 2641 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 2642 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2643 } 2644 nvmf_get_discovery_log_page(subsystem->tgt, ctrlr->hostnqn, req->iov, req->iovcnt, 2645 offset, len, &cmd_source_trid); 2646 if (!rae) { 2647 nvmf_ctrlr_unmask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT); 2648 } 2649 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2650 default: 2651 goto invalid_log_page; 2652 } 2653 } else { 2654 if (offset > len) { 2655 SPDK_ERRLOG("Get log page: offset (%" PRIu64 ") > len (%" PRIu64 ")\n", 2656 offset, len); 2657 response->status.sct = SPDK_NVME_SCT_GENERIC; 2658 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2659 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2660 } 2661 2662 switch (lid) { 2663 case SPDK_NVME_LOG_ERROR: 2664 nvmf_get_error_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2665 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2666 case SPDK_NVME_LOG_HEALTH_INFORMATION: 2667 /* TODO: actually fill out log page data */ 2668 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2669 case SPDK_NVME_LOG_FIRMWARE_SLOT: 2670 nvmf_get_firmware_slot_log_page(req->iov, req->iovcnt, offset, len); 2671 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2672 case SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS: 2673 if (subsystem->flags.ana_reporting) { 2674 nvmf_get_ana_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2675 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2676 } else { 2677 goto invalid_log_page; 2678 } 2679 case SPDK_NVME_LOG_COMMAND_EFFECTS_LOG: 2680 nvmf_get_cmds_and_effects_log_page(ctrlr, req->iov, req->iovcnt, offset, len); 2681 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2682 case SPDK_NVME_LOG_CHANGED_NS_LIST: 2683 nvmf_get_changed_ns_list_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2684 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2685 case SPDK_NVME_LOG_RESERVATION_NOTIFICATION: 2686 nvmf_get_reservation_notification_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae); 2687 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2688 default: 2689 goto invalid_log_page; 2690 } 2691 } 2692 2693 invalid_log_page: 2694 SPDK_INFOLOG(nvmf, "Unsupported Get Log Page 0x%02X\n", lid); 2695 response->status.sct = SPDK_NVME_SCT_GENERIC; 2696 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 2697 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2698 } 2699 2700 static struct spdk_nvmf_ns * 2701 _nvmf_ctrlr_get_ns_safe(struct spdk_nvmf_ctrlr *ctrlr, 2702 uint32_t nsid, 2703 struct spdk_nvme_cpl *rsp) 2704 { 2705 struct spdk_nvmf_ns *ns; 2706 if (nsid == 0 || nsid > ctrlr->subsys->max_nsid) { 2707 SPDK_ERRLOG("Identify Namespace for invalid NSID %u\n", nsid); 2708 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2709 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 2710 return NULL; 2711 } 2712 2713 ns = nvmf_ctrlr_get_ns(ctrlr, nsid); 2714 if (ns == NULL || ns->bdev == NULL) { 2715 /* 2716 * Inactive namespaces should return a zero filled data structure. 2717 * The data buffer is already zeroed by nvmf_ctrlr_process_admin_cmd(), 2718 * so we can just return early here. 2719 */ 2720 SPDK_DEBUGLOG(nvmf, "Identify Namespace for inactive NSID %u\n", nsid); 2721 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2722 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2723 return NULL; 2724 } 2725 return ns; 2726 } 2727 2728 int 2729 spdk_nvmf_ctrlr_identify_ns(struct spdk_nvmf_ctrlr *ctrlr, 2730 struct spdk_nvme_cmd *cmd, 2731 struct spdk_nvme_cpl *rsp, 2732 struct spdk_nvme_ns_data *nsdata) 2733 { 2734 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2735 struct spdk_nvmf_ns *ns; 2736 uint32_t max_num_blocks, format_index; 2737 enum spdk_nvme_ana_state ana_state; 2738 2739 ns = _nvmf_ctrlr_get_ns_safe(ctrlr, cmd->nsid, rsp); 2740 if (ns == NULL) { 2741 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2742 } 2743 2744 nvmf_bdev_ctrlr_identify_ns(ns, nsdata, ctrlr->dif_insert_or_strip); 2745 2746 assert(ctrlr->admin_qpair); 2747 2748 format_index = spdk_nvme_ns_get_format_index(nsdata); 2749 2750 /* Due to bug in the Linux kernel NVMe driver we have to set noiob no larger than mdts */ 2751 max_num_blocks = ctrlr->admin_qpair->transport->opts.max_io_size / 2752 (1U << nsdata->lbaf[format_index].lbads); 2753 if (nsdata->noiob > max_num_blocks) { 2754 nsdata->noiob = max_num_blocks; 2755 } 2756 2757 /* Set NOWS equal to Controller MDTS */ 2758 if (nsdata->nsfeat.optperf) { 2759 nsdata->nows = max_num_blocks - 1; 2760 } 2761 2762 if (subsystem->flags.ana_reporting) { 2763 assert(ns->anagrpid - 1 < subsystem->max_nsid); 2764 nsdata->anagrpid = ns->anagrpid; 2765 2766 ana_state = nvmf_ctrlr_get_ana_state(ctrlr, ns->anagrpid); 2767 if (ana_state == SPDK_NVME_ANA_INACCESSIBLE_STATE || 2768 ana_state == SPDK_NVME_ANA_PERSISTENT_LOSS_STATE) { 2769 nsdata->nuse = 0; 2770 } 2771 } 2772 2773 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2774 } 2775 2776 static void 2777 nvmf_ctrlr_populate_oacs(struct spdk_nvmf_ctrlr *ctrlr, 2778 struct spdk_nvme_ctrlr_data *cdata) 2779 { 2780 cdata->oacs = ctrlr->cdata.oacs; 2781 2782 cdata->oacs.virtualization_management = 2783 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_VIRTUALIZATION_MANAGEMENT].hdlr != NULL; 2784 cdata->oacs.nvme_mi = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NVME_MI_SEND].hdlr != NULL 2785 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NVME_MI_RECEIVE].hdlr != NULL; 2786 cdata->oacs.directives = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DIRECTIVE_SEND].hdlr != NULL 2787 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DIRECTIVE_RECEIVE].hdlr != NULL; 2788 cdata->oacs.device_self_test = 2789 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_DEVICE_SELF_TEST].hdlr != NULL; 2790 cdata->oacs.ns_manage = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NS_MANAGEMENT].hdlr != NULL 2791 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_NS_ATTACHMENT].hdlr != NULL; 2792 cdata->oacs.firmware = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD].hdlr != 2793 NULL 2794 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FIRMWARE_COMMIT].hdlr != NULL; 2795 cdata->oacs.format = 2796 g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_FORMAT_NVM].hdlr != NULL; 2797 cdata->oacs.security = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_SECURITY_SEND].hdlr != NULL 2798 && g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_SECURITY_RECEIVE].hdlr != NULL; 2799 cdata->oacs.get_lba_status = g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_GET_LBA_STATUS].hdlr != 2800 NULL; 2801 } 2802 2803 int 2804 spdk_nvmf_ctrlr_identify_ctrlr(struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvme_ctrlr_data *cdata) 2805 { 2806 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 2807 struct spdk_nvmf_transport *transport; 2808 2809 /* 2810 * Common fields for discovery and NVM subsystems 2811 */ 2812 assert(ctrlr->admin_qpair); 2813 transport = ctrlr->admin_qpair->transport; 2814 spdk_strcpy_pad(cdata->fr, FW_VERSION, sizeof(cdata->fr), ' '); 2815 assert((transport->opts.max_io_size % 4096) == 0); 2816 cdata->mdts = spdk_u32log2(transport->opts.max_io_size / 4096); 2817 cdata->cntlid = ctrlr->cntlid; 2818 cdata->ver = ctrlr->vcprop.vs; 2819 cdata->aerl = ctrlr->cdata.aerl; 2820 cdata->lpa.edlp = 1; 2821 cdata->elpe = 127; 2822 cdata->maxcmd = transport->opts.max_queue_depth; 2823 cdata->sgls = ctrlr->cdata.sgls; 2824 cdata->fuses = ctrlr->cdata.fuses; 2825 cdata->acwu = 0; /* ACWU is 0-based. */ 2826 if (subsystem->flags.ana_reporting) { 2827 cdata->mnan = subsystem->max_nsid; 2828 } 2829 spdk_strcpy_pad(cdata->subnqn, subsystem->subnqn, sizeof(cdata->subnqn), '\0'); 2830 2831 SPDK_DEBUGLOG(nvmf, "ctrlr data: maxcmd 0x%x\n", cdata->maxcmd); 2832 SPDK_DEBUGLOG(nvmf, "sgls data: 0x%x\n", from_le32(&cdata->sgls)); 2833 2834 2835 if (spdk_nvmf_subsystem_is_discovery(subsystem)) { 2836 /* 2837 * NVM Discovery subsystem fields 2838 */ 2839 cdata->oaes.discovery_log_change_notices = 1; 2840 } else { 2841 cdata->vid = ctrlr->cdata.vid; 2842 cdata->ssvid = ctrlr->cdata.ssvid; 2843 cdata->ieee[0] = ctrlr->cdata.ieee[0]; 2844 cdata->ieee[1] = ctrlr->cdata.ieee[1]; 2845 cdata->ieee[2] = ctrlr->cdata.ieee[2]; 2846 2847 /* 2848 * NVM subsystem fields (reserved for discovery subsystems) 2849 */ 2850 spdk_strcpy_pad(cdata->mn, spdk_nvmf_subsystem_get_mn(subsystem), sizeof(cdata->mn), ' '); 2851 spdk_strcpy_pad(cdata->sn, spdk_nvmf_subsystem_get_sn(subsystem), sizeof(cdata->sn), ' '); 2852 cdata->kas = ctrlr->cdata.kas; 2853 2854 cdata->rab = 6; 2855 cdata->cmic.multi_port = 1; 2856 cdata->cmic.multi_ctrlr = 1; 2857 cdata->oaes.ns_attribute_notices = 1; 2858 cdata->ctratt.bits.host_id_exhid_supported = 1; 2859 cdata->ctratt.bits.fdps = ctrlr->subsys->fdp_supported; 2860 /* We do not have any actual limitation to the number of abort commands. 2861 * We follow the recommendation by the NVMe specification. 2862 */ 2863 cdata->acl = NVMF_ABORT_COMMAND_LIMIT; 2864 cdata->frmw.slot1_ro = 1; 2865 cdata->frmw.num_slots = 1; 2866 2867 cdata->lpa.celp = 1; /* Command Effects log page supported */ 2868 2869 cdata->sqes.min = 6; 2870 cdata->sqes.max = 6; 2871 cdata->cqes.min = 4; 2872 cdata->cqes.max = 4; 2873 cdata->nn = subsystem->max_nsid; 2874 cdata->vwc.present = 1; 2875 cdata->vwc.flush_broadcast = SPDK_NVME_FLUSH_BROADCAST_NOT_SUPPORTED; 2876 2877 cdata->nvmf_specific = ctrlr->cdata.nvmf_specific; 2878 2879 cdata->oncs.compare = ctrlr->cdata.oncs.compare; 2880 cdata->oncs.dsm = nvmf_ctrlr_dsm_supported(ctrlr); 2881 cdata->oncs.write_zeroes = nvmf_ctrlr_write_zeroes_supported(ctrlr); 2882 cdata->oncs.reservations = ctrlr->cdata.oncs.reservations; 2883 cdata->oncs.copy = ctrlr->cdata.oncs.copy; 2884 cdata->ocfs.copy_format0 = cdata->oncs.copy; 2885 if (subsystem->flags.ana_reporting) { 2886 /* Asymmetric Namespace Access Reporting is supported. */ 2887 cdata->cmic.ana_reporting = 1; 2888 cdata->oaes.ana_change_notices = 1; 2889 2890 cdata->anatt = ANA_TRANSITION_TIME_IN_SEC; 2891 /* ANA Change state is not used, and ANA Persistent Loss state 2892 * is not supported for now. 2893 */ 2894 cdata->anacap.ana_optimized_state = 1; 2895 cdata->anacap.ana_non_optimized_state = 1; 2896 cdata->anacap.ana_inaccessible_state = 1; 2897 /* ANAGRPID does not change while namespace is attached to controller */ 2898 cdata->anacap.no_change_anagrpid = 1; 2899 cdata->anagrpmax = subsystem->max_nsid; 2900 cdata->nanagrpid = subsystem->max_nsid; 2901 } 2902 2903 nvmf_ctrlr_populate_oacs(ctrlr, cdata); 2904 2905 assert(subsystem->tgt != NULL); 2906 cdata->crdt[0] = subsystem->tgt->crdt[0]; 2907 cdata->crdt[1] = subsystem->tgt->crdt[1]; 2908 cdata->crdt[2] = subsystem->tgt->crdt[2]; 2909 2910 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: ioccsz 0x%x\n", 2911 cdata->nvmf_specific.ioccsz); 2912 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: iorcsz 0x%x\n", 2913 cdata->nvmf_specific.iorcsz); 2914 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: icdoff 0x%x\n", 2915 cdata->nvmf_specific.icdoff); 2916 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: ctrattr 0x%x\n", 2917 *(uint8_t *)&cdata->nvmf_specific.ctrattr); 2918 SPDK_DEBUGLOG(nvmf, "ext ctrlr data: msdbd 0x%x\n", 2919 cdata->nvmf_specific.msdbd); 2920 } 2921 2922 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2923 } 2924 2925 static int 2926 nvmf_ns_identify_iocs_zns(struct spdk_nvmf_ns *ns, 2927 struct spdk_nvme_cmd *cmd, 2928 struct spdk_nvme_cpl *rsp, 2929 struct spdk_nvme_zns_ns_data *nsdata_zns) 2930 { 2931 nsdata_zns->zoc.variable_zone_capacity = 0; 2932 nsdata_zns->zoc.zone_active_excursions = 0; 2933 nsdata_zns->ozcs.read_across_zone_boundaries = 1; 2934 /* Underflowing the zero based mar and mor bdev helper results in the correct 2935 value of FFFFFFFFh. */ 2936 nsdata_zns->mar = spdk_bdev_get_max_active_zones(ns->bdev) - 1; 2937 nsdata_zns->mor = spdk_bdev_get_max_open_zones(ns->bdev) - 1; 2938 nsdata_zns->rrl = 0; 2939 nsdata_zns->frl = 0; 2940 nsdata_zns->lbafe[0].zsze = spdk_bdev_get_zone_size(ns->bdev); 2941 2942 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2943 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2944 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2945 } 2946 2947 int 2948 spdk_nvmf_ns_identify_iocs_specific(struct spdk_nvmf_ctrlr *ctrlr, 2949 struct spdk_nvme_cmd *cmd, 2950 struct spdk_nvme_cpl *rsp, 2951 void *nsdata, 2952 size_t nsdata_size) 2953 { 2954 uint8_t csi = cmd->cdw11_bits.identify.csi; 2955 struct spdk_nvmf_ns *ns = _nvmf_ctrlr_get_ns_safe(ctrlr, cmd->nsid, rsp); 2956 2957 memset(nsdata, 0, nsdata_size); 2958 2959 if (ns == NULL) { 2960 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2961 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 2962 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2963 } 2964 2965 switch (csi) { 2966 case SPDK_NVME_CSI_ZNS: 2967 return nvmf_ns_identify_iocs_zns(ns, cmd, rsp, nsdata); 2968 default: 2969 break; 2970 } 2971 2972 SPDK_DEBUGLOG(nvmf, 2973 "Returning zero filled struct for the iocs specific ns " 2974 "identify command and CSI 0x%02x\n", 2975 csi); 2976 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 2977 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 2978 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 2979 } 2980 2981 static int 2982 nvmf_ctrlr_identify_iocs_nvm(struct spdk_nvmf_ctrlr *ctrlr, 2983 struct spdk_nvme_cmd *cmd, 2984 struct spdk_nvme_cpl *rsp, 2985 struct spdk_nvme_nvm_ctrlr_data *cdata_nvm) 2986 { 2987 /* The unit of max_write_zeroes_size_kib is KiB. 2988 * The unit of wzsl is the minimum memory page size(2 ^ (12 + CAP.MPSMIN) bytes) 2989 * and is reported as a power of two (2^n). 2990 */ 2991 cdata_nvm->wzsl = spdk_u64log2(ctrlr->subsys->max_write_zeroes_size_kib >> 2992 (2 + ctrlr->vcprop.cap.bits.mpsmin)); 2993 2994 /* The unit of max_discard_size_kib is KiB. 2995 * The dmrsl indicates the maximum number of logical blocks for 2996 * dataset management command. 2997 */ 2998 cdata_nvm->dmrsl = ctrlr->subsys->max_discard_size_kib << 1; 2999 cdata_nvm->dmrl = 1; 3000 3001 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3002 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3003 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3004 } 3005 3006 static int 3007 nvmf_ctrlr_identify_iocs_zns(struct spdk_nvmf_ctrlr *ctrlr, 3008 struct spdk_nvme_cmd *cmd, 3009 struct spdk_nvme_cpl *rsp, 3010 struct spdk_nvme_zns_ctrlr_data *cdata_zns) 3011 { 3012 /* The unit of max_zone_append_size_kib is KiB. 3013 The unit of zasl is the minimum memory page size 3014 (2 ^ (12 + CAP.MPSMIN) KiB) 3015 and is reported as a power of two (2^n). */ 3016 cdata_zns->zasl = spdk_u64log2(ctrlr->subsys->max_zone_append_size_kib >> 3017 (12 + ctrlr->vcprop.cap.bits.mpsmin)); 3018 3019 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3020 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3021 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3022 } 3023 3024 int 3025 spdk_nvmf_ctrlr_identify_iocs_specific(struct spdk_nvmf_ctrlr *ctrlr, 3026 struct spdk_nvme_cmd *cmd, 3027 struct spdk_nvme_cpl *rsp, 3028 void *cdata, 3029 size_t cdata_size) 3030 { 3031 uint8_t csi = cmd->cdw11_bits.identify.csi; 3032 3033 memset(cdata, 0, cdata_size); 3034 3035 switch (csi) { 3036 case SPDK_NVME_CSI_NVM: 3037 return nvmf_ctrlr_identify_iocs_nvm(ctrlr, cmd, rsp, cdata); 3038 case SPDK_NVME_CSI_ZNS: 3039 return nvmf_ctrlr_identify_iocs_zns(ctrlr, cmd, rsp, cdata); 3040 default: 3041 break; 3042 } 3043 3044 SPDK_DEBUGLOG(nvmf, 3045 "Returning zero filled struct for the iocs specific ctrlr " 3046 "identify command and CSI 0x%02x\n", 3047 csi); 3048 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3049 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3050 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3051 } 3052 3053 static int 3054 nvmf_ctrlr_identify_active_ns_list(struct spdk_nvmf_ctrlr *ctrlr, 3055 struct spdk_nvme_cmd *cmd, 3056 struct spdk_nvme_cpl *rsp, 3057 struct spdk_nvme_ns_list *ns_list) 3058 { 3059 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 3060 struct spdk_nvmf_ns *ns; 3061 uint32_t count = 0; 3062 3063 if (cmd->nsid >= 0xfffffffeUL) { 3064 SPDK_ERRLOG("Identify Active Namespace List with invalid NSID %u\n", cmd->nsid); 3065 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 3066 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3067 } 3068 3069 memset(ns_list, 0, sizeof(*ns_list)); 3070 3071 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 3072 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 3073 if (ns->opts.nsid <= cmd->nsid || !nvmf_ctrlr_ns_is_visible(ctrlr, ns->opts.nsid)) { 3074 continue; 3075 } 3076 3077 ns_list->ns_list[count++] = ns->opts.nsid; 3078 if (count == SPDK_COUNTOF(ns_list->ns_list)) { 3079 break; 3080 } 3081 } 3082 3083 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3084 } 3085 3086 static void 3087 _add_ns_id_desc(void **buf_ptr, size_t *buf_remain, 3088 enum spdk_nvme_nidt type, 3089 const void *data, size_t data_size) 3090 { 3091 struct spdk_nvme_ns_id_desc *desc; 3092 size_t desc_size = sizeof(*desc) + data_size; 3093 3094 /* 3095 * These should never fail in practice, since all valid NS ID descriptors 3096 * should be defined so that they fit in the available 4096-byte buffer. 3097 */ 3098 assert(data_size > 0); 3099 assert(data_size <= UINT8_MAX); 3100 assert(desc_size < *buf_remain); 3101 if (data_size == 0 || data_size > UINT8_MAX || desc_size > *buf_remain) { 3102 return; 3103 } 3104 3105 desc = *buf_ptr; 3106 desc->nidt = type; 3107 desc->nidl = data_size; 3108 memcpy(desc->nid, data, data_size); 3109 3110 *buf_ptr += desc_size; 3111 *buf_remain -= desc_size; 3112 } 3113 3114 static int 3115 nvmf_ctrlr_identify_ns_id_descriptor_list( 3116 struct spdk_nvmf_ctrlr *ctrlr, 3117 struct spdk_nvme_cmd *cmd, 3118 struct spdk_nvme_cpl *rsp, 3119 void *id_desc_list, size_t id_desc_list_size) 3120 { 3121 struct spdk_nvmf_ns *ns; 3122 size_t buf_remain = id_desc_list_size; 3123 void *buf_ptr = id_desc_list; 3124 3125 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 3126 if (ns == NULL || ns->bdev == NULL) { 3127 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3128 rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 3129 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3130 } 3131 3132 #define ADD_ID_DESC(type, data, size) \ 3133 do { \ 3134 if (!spdk_mem_all_zero(data, size)) { \ 3135 _add_ns_id_desc(&buf_ptr, &buf_remain, type, data, size); \ 3136 } \ 3137 } while (0) 3138 3139 ADD_ID_DESC(SPDK_NVME_NIDT_EUI64, ns->opts.eui64, sizeof(ns->opts.eui64)); 3140 ADD_ID_DESC(SPDK_NVME_NIDT_NGUID, ns->opts.nguid, sizeof(ns->opts.nguid)); 3141 ADD_ID_DESC(SPDK_NVME_NIDT_UUID, &ns->opts.uuid, sizeof(ns->opts.uuid)); 3142 ADD_ID_DESC(SPDK_NVME_NIDT_CSI, &ns->csi, sizeof(uint8_t)); 3143 3144 /* 3145 * The list is automatically 0-terminated, both in the temporary buffer 3146 * used by nvmf_ctrlr_identify(), and the eventual iov destination - 3147 * controller to host buffers in admin commands always get zeroed in 3148 * nvmf_ctrlr_process_admin_cmd(). 3149 */ 3150 3151 #undef ADD_ID_DESC 3152 3153 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3154 } 3155 3156 static int 3157 nvmf_ctrlr_identify_iocs(struct spdk_nvmf_ctrlr *ctrlr, 3158 struct spdk_nvme_cmd *cmd, 3159 struct spdk_nvme_cpl *rsp, 3160 void *cdata, size_t cdata_size) 3161 { 3162 struct spdk_nvme_iocs_vector *vector; 3163 struct spdk_nvmf_ns *ns; 3164 3165 if (cdata_size < sizeof(struct spdk_nvme_iocs_vector)) { 3166 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3167 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3168 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3169 } 3170 3171 /* For now we only support this command sent to the current 3172 * controller. 3173 */ 3174 if (cmd->cdw10_bits.identify.cntid != 0xFFFF && 3175 cmd->cdw10_bits.identify.cntid != ctrlr->cntlid) { 3176 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3177 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3178 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3179 } 3180 memset(cdata, 0, cdata_size); 3181 3182 vector = cdata; 3183 vector->nvm = 1; 3184 for (ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); ns != NULL; 3185 ns = spdk_nvmf_subsystem_get_next_ns(ctrlr->subsys, ns)) { 3186 if (ns->bdev == NULL) { 3187 continue; 3188 } 3189 if (spdk_bdev_is_zoned(ns->bdev)) { 3190 vector->zns = 1; 3191 } 3192 } 3193 3194 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3195 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3196 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3197 } 3198 3199 static int 3200 nvmf_ctrlr_identify(struct spdk_nvmf_request *req) 3201 { 3202 uint8_t cns; 3203 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3204 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3205 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3206 struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys; 3207 int ret = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3208 char tmpbuf[SPDK_NVME_IDENTIFY_BUFLEN] = ""; 3209 struct spdk_iov_xfer ix; 3210 3211 if (req->iovcnt < 1 || req->length < SPDK_NVME_IDENTIFY_BUFLEN) { 3212 SPDK_DEBUGLOG(nvmf, "identify command with invalid buffer\n"); 3213 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3214 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3215 return ret; 3216 } 3217 3218 cns = cmd->cdw10_bits.identify.cns; 3219 3220 if (spdk_nvmf_subsystem_is_discovery(subsystem) && 3221 cns != SPDK_NVME_IDENTIFY_CTRLR) { 3222 /* Discovery controllers only support Identify Controller */ 3223 goto invalid_cns; 3224 } 3225 3226 /* 3227 * We must use a temporary buffer: it's entirely possible the out buffer 3228 * is split across more than one IOV. 3229 */ 3230 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 3231 3232 SPDK_DEBUGLOG(nvmf, "Received identify command with CNS 0x%02x\n", cns); 3233 3234 switch (cns) { 3235 case SPDK_NVME_IDENTIFY_NS: 3236 ret = spdk_nvmf_ctrlr_identify_ns(ctrlr, cmd, rsp, (void *)&tmpbuf); 3237 break; 3238 case SPDK_NVME_IDENTIFY_CTRLR: 3239 ret = spdk_nvmf_ctrlr_identify_ctrlr(ctrlr, (void *)&tmpbuf); 3240 break; 3241 case SPDK_NVME_IDENTIFY_ACTIVE_NS_LIST: 3242 ret = nvmf_ctrlr_identify_active_ns_list(ctrlr, cmd, rsp, (void *)&tmpbuf); 3243 break; 3244 case SPDK_NVME_IDENTIFY_NS_ID_DESCRIPTOR_LIST: 3245 ret = nvmf_ctrlr_identify_ns_id_descriptor_list(ctrlr, cmd, rsp, 3246 tmpbuf, req->length); 3247 break; 3248 case SPDK_NVME_IDENTIFY_NS_IOCS: 3249 ret = spdk_nvmf_ns_identify_iocs_specific(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length); 3250 break; 3251 case SPDK_NVME_IDENTIFY_CTRLR_IOCS: 3252 ret = spdk_nvmf_ctrlr_identify_iocs_specific(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length); 3253 break; 3254 case SPDK_NVME_IDENTIFY_IOCS: 3255 ret = nvmf_ctrlr_identify_iocs(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length); 3256 break; 3257 default: 3258 goto invalid_cns; 3259 } 3260 3261 if (ret == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 3262 spdk_iov_xfer_from_buf(&ix, tmpbuf, sizeof(tmpbuf)); 3263 } 3264 3265 return ret; 3266 3267 invalid_cns: 3268 SPDK_DEBUGLOG(nvmf, "Identify command with unsupported CNS 0x%02x\n", cns); 3269 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3270 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3271 return ret; 3272 } 3273 3274 static bool 3275 nvmf_qpair_abort_aer(struct spdk_nvmf_qpair *qpair, uint16_t cid) 3276 { 3277 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 3278 struct spdk_nvmf_request *req; 3279 int i; 3280 3281 if (!nvmf_qpair_is_admin_queue(qpair)) { 3282 return false; 3283 } 3284 3285 assert(spdk_get_thread() == ctrlr->thread); 3286 3287 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 3288 if (ctrlr->aer_req[i]->cmd->nvme_cmd.cid == cid) { 3289 SPDK_DEBUGLOG(nvmf, "Aborting AER request\n"); 3290 req = ctrlr->aer_req[i]; 3291 ctrlr->aer_req[i] = NULL; 3292 ctrlr->nr_aer_reqs--; 3293 3294 /* Move the last req to the aborting position for making aer_reqs 3295 * in continuous 3296 */ 3297 if (i < ctrlr->nr_aer_reqs) { 3298 ctrlr->aer_req[i] = ctrlr->aer_req[ctrlr->nr_aer_reqs]; 3299 ctrlr->aer_req[ctrlr->nr_aer_reqs] = NULL; 3300 } 3301 3302 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3303 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 3304 _nvmf_request_complete(req); 3305 return true; 3306 } 3307 } 3308 3309 return false; 3310 } 3311 3312 void 3313 nvmf_qpair_abort_pending_zcopy_reqs(struct spdk_nvmf_qpair *qpair) 3314 { 3315 struct spdk_nvmf_request *req, *tmp; 3316 3317 TAILQ_FOREACH_SAFE(req, &qpair->outstanding, link, tmp) { 3318 if (req->zcopy_phase == NVMF_ZCOPY_PHASE_EXECUTE) { 3319 /* Zero-copy requests are kept on the outstanding queue from the moment 3320 * zcopy_start is sent until a zcopy_end callback is received. Therefore, 3321 * we can't remove them from the outstanding queue here, but need to rely on 3322 * the transport to do a zcopy_end to release their buffers and, in turn, 3323 * remove them from the queue. 3324 */ 3325 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3326 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 3327 nvmf_transport_req_free(req); 3328 } 3329 } 3330 } 3331 3332 static void 3333 nvmf_qpair_abort_request(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_request *req) 3334 { 3335 uint16_t cid = req->cmd->nvme_cmd.cdw10_bits.abort.cid; 3336 3337 if (nvmf_qpair_abort_aer(qpair, cid)) { 3338 SPDK_DEBUGLOG(nvmf, "abort ctrlr=%p sqid=%u cid=%u successful\n", 3339 qpair->ctrlr, qpair->qid, cid); 3340 req->rsp->nvme_cpl.cdw0 &= ~1U; /* Command successfully aborted */ 3341 3342 spdk_nvmf_request_complete(req); 3343 return; 3344 } 3345 3346 nvmf_transport_qpair_abort_request(qpair, req); 3347 } 3348 3349 static void 3350 nvmf_ctrlr_abort_done(struct spdk_io_channel_iter *i, int status) 3351 { 3352 struct spdk_nvmf_request *req = spdk_io_channel_iter_get_ctx(i); 3353 3354 if (status == 0) { 3355 /* There was no qpair whose ID matches SQID of the abort command. 3356 * Hence call _nvmf_request_complete() here. 3357 */ 3358 _nvmf_request_complete(req); 3359 } 3360 } 3361 3362 static void 3363 nvmf_ctrlr_abort_on_pg(struct spdk_io_channel_iter *i) 3364 { 3365 struct spdk_nvmf_request *req = spdk_io_channel_iter_get_ctx(i); 3366 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 3367 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 3368 uint16_t sqid = req->cmd->nvme_cmd.cdw10_bits.abort.sqid; 3369 struct spdk_nvmf_qpair *qpair; 3370 3371 TAILQ_FOREACH(qpair, &group->qpairs, link) { 3372 if (qpair->ctrlr == req->qpair->ctrlr && qpair->qid == sqid) { 3373 /* Found the qpair */ 3374 3375 nvmf_qpair_abort_request(qpair, req); 3376 3377 /* Return -1 for the status so the iteration across threads stops. */ 3378 spdk_for_each_channel_continue(i, -1); 3379 return; 3380 } 3381 } 3382 3383 spdk_for_each_channel_continue(i, 0); 3384 } 3385 3386 static int 3387 nvmf_ctrlr_abort(struct spdk_nvmf_request *req) 3388 { 3389 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3390 3391 rsp->cdw0 = 1U; /* Command not aborted */ 3392 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 3393 rsp->status.sc = SPDK_NVME_SC_SUCCESS; 3394 3395 /* Send a message to each poll group, searching for this ctrlr, sqid, and command. */ 3396 spdk_for_each_channel(req->qpair->ctrlr->subsys->tgt, 3397 nvmf_ctrlr_abort_on_pg, 3398 req, 3399 nvmf_ctrlr_abort_done 3400 ); 3401 3402 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 3403 } 3404 3405 int 3406 nvmf_ctrlr_abort_request(struct spdk_nvmf_request *req) 3407 { 3408 struct spdk_nvmf_request *req_to_abort = req->req_to_abort; 3409 struct spdk_bdev *bdev; 3410 struct spdk_bdev_desc *desc; 3411 struct spdk_io_channel *ch; 3412 int rc; 3413 3414 assert(req_to_abort != NULL); 3415 3416 if (g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_ABORT].hdlr && 3417 nvmf_qpair_is_admin_queue(req_to_abort->qpair)) { 3418 return g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_ABORT].hdlr(req); 3419 } 3420 3421 rc = spdk_nvmf_request_get_bdev(req_to_abort->cmd->nvme_cmd.nsid, req_to_abort, 3422 &bdev, &desc, &ch); 3423 if (rc != 0) { 3424 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3425 } 3426 3427 return spdk_nvmf_bdev_ctrlr_abort_cmd(bdev, desc, ch, req, req_to_abort); 3428 } 3429 3430 static int 3431 get_features_generic(struct spdk_nvmf_request *req, uint32_t cdw0) 3432 { 3433 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 3434 3435 rsp->cdw0 = cdw0; 3436 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3437 } 3438 3439 /* we have to use the typedef in the function declaration to appease astyle. */ 3440 typedef enum spdk_nvme_path_status_code spdk_nvme_path_status_code_t; 3441 3442 static spdk_nvme_path_status_code_t 3443 _nvme_ana_state_to_path_status(enum spdk_nvme_ana_state ana_state) 3444 { 3445 switch (ana_state) { 3446 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3447 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_INACCESSIBLE; 3448 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3449 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS; 3450 case SPDK_NVME_ANA_CHANGE_STATE: 3451 return SPDK_NVME_SC_ASYMMETRIC_ACCESS_TRANSITION; 3452 default: 3453 return SPDK_NVME_SC_INTERNAL_PATH_ERROR; 3454 } 3455 } 3456 3457 static int 3458 nvmf_ctrlr_get_features(struct spdk_nvmf_request *req) 3459 { 3460 uint8_t feature; 3461 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3462 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3463 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 3464 enum spdk_nvme_ana_state ana_state; 3465 3466 feature = cmd->cdw10_bits.get_features.fid; 3467 3468 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 3469 /* 3470 * Features supported by Discovery controller 3471 */ 3472 switch (feature) { 3473 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3474 return get_features_generic(req, ctrlr->feat.keep_alive_timer.raw); 3475 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3476 return get_features_generic(req, ctrlr->feat.async_event_configuration.raw); 3477 default: 3478 SPDK_INFOLOG(nvmf, "Get Features command with unsupported feature ID 0x%02x\n", feature); 3479 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3480 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3481 } 3482 } 3483 /* 3484 * Process Get Features command for non-discovery controller 3485 */ 3486 ana_state = nvmf_ctrlr_get_ana_state_from_nsid(ctrlr, cmd->nsid); 3487 switch (ana_state) { 3488 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3489 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3490 case SPDK_NVME_ANA_CHANGE_STATE: 3491 switch (feature) { 3492 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3493 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3494 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3495 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3496 response->status.sct = SPDK_NVME_SCT_PATH; 3497 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 3498 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3499 default: 3500 break; 3501 } 3502 break; 3503 default: 3504 break; 3505 } 3506 3507 switch (feature) { 3508 case SPDK_NVME_FEAT_ARBITRATION: 3509 return get_features_generic(req, ctrlr->feat.arbitration.raw); 3510 case SPDK_NVME_FEAT_POWER_MANAGEMENT: 3511 return get_features_generic(req, ctrlr->feat.power_management.raw); 3512 case SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD: 3513 return nvmf_ctrlr_get_features_temperature_threshold(req); 3514 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3515 return get_features_generic(req, ctrlr->feat.error_recovery.raw); 3516 case SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE: 3517 return get_features_generic(req, ctrlr->feat.volatile_write_cache.raw); 3518 case SPDK_NVME_FEAT_NUMBER_OF_QUEUES: 3519 return get_features_generic(req, ctrlr->feat.number_of_queues.raw); 3520 case SPDK_NVME_FEAT_INTERRUPT_COALESCING: 3521 return get_features_generic(req, ctrlr->feat.interrupt_coalescing.raw); 3522 case SPDK_NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 3523 return nvmf_ctrlr_get_features_interrupt_vector_configuration(req); 3524 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3525 return get_features_generic(req, ctrlr->feat.write_atomicity.raw); 3526 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3527 return get_features_generic(req, ctrlr->feat.async_event_configuration.raw); 3528 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3529 return get_features_generic(req, ctrlr->feat.keep_alive_timer.raw); 3530 case SPDK_NVME_FEAT_HOST_IDENTIFIER: 3531 return nvmf_ctrlr_get_features_host_identifier(req); 3532 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3533 return nvmf_ctrlr_get_features_reservation_notification_mask(req); 3534 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3535 return nvmf_ctrlr_get_features_reservation_persistence(req); 3536 case SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 3537 return nvmf_ctrlr_get_features_host_behavior_support(req); 3538 default: 3539 SPDK_INFOLOG(nvmf, "Get Features command with unsupported feature ID 0x%02x\n", feature); 3540 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3541 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3542 } 3543 } 3544 3545 static int 3546 nvmf_ctrlr_set_features(struct spdk_nvmf_request *req) 3547 { 3548 uint8_t feature, save; 3549 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3550 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3551 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 3552 enum spdk_nvme_ana_state ana_state; 3553 /* 3554 * Features are not saveable by the controller as indicated by 3555 * ONCS field of the Identify Controller data. 3556 * */ 3557 save = cmd->cdw10_bits.set_features.sv; 3558 if (save) { 3559 response->status.sc = SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE; 3560 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 3561 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3562 } 3563 3564 feature = cmd->cdw10_bits.set_features.fid; 3565 3566 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 3567 /* 3568 * Features supported by Discovery controller 3569 */ 3570 switch (feature) { 3571 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3572 return nvmf_ctrlr_set_features_keep_alive_timer(req); 3573 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3574 return nvmf_ctrlr_set_features_async_event_configuration(req); 3575 default: 3576 SPDK_INFOLOG(nvmf, "Set Features command with unsupported feature ID 0x%02x\n", feature); 3577 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3578 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3579 } 3580 } 3581 /* 3582 * Process Set Features command for non-discovery controller 3583 */ 3584 ana_state = nvmf_ctrlr_get_ana_state_from_nsid(ctrlr, cmd->nsid); 3585 switch (ana_state) { 3586 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3587 case SPDK_NVME_ANA_CHANGE_STATE: 3588 if (cmd->nsid == SPDK_NVME_GLOBAL_NS_TAG) { 3589 response->status.sct = SPDK_NVME_SCT_PATH; 3590 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 3591 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3592 } else { 3593 switch (feature) { 3594 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3595 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3596 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3597 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3598 response->status.sct = SPDK_NVME_SCT_PATH; 3599 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 3600 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3601 default: 3602 break; 3603 } 3604 } 3605 break; 3606 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3607 response->status.sct = SPDK_NVME_SCT_PATH; 3608 response->status.sc = SPDK_NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS; 3609 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3610 default: 3611 break; 3612 } 3613 3614 switch (feature) { 3615 case SPDK_NVME_FEAT_ARBITRATION: 3616 return nvmf_ctrlr_set_features_arbitration(req); 3617 case SPDK_NVME_FEAT_POWER_MANAGEMENT: 3618 return nvmf_ctrlr_set_features_power_management(req); 3619 case SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD: 3620 return nvmf_ctrlr_set_features_temperature_threshold(req); 3621 case SPDK_NVME_FEAT_ERROR_RECOVERY: 3622 return nvmf_ctrlr_set_features_error_recovery(req); 3623 case SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE: 3624 return nvmf_ctrlr_set_features_volatile_write_cache(req); 3625 case SPDK_NVME_FEAT_NUMBER_OF_QUEUES: 3626 return nvmf_ctrlr_set_features_number_of_queues(req); 3627 case SPDK_NVME_FEAT_INTERRUPT_COALESCING: 3628 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 3629 response->status.sc = SPDK_NVME_SC_FEATURE_NOT_CHANGEABLE; 3630 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3631 case SPDK_NVME_FEAT_WRITE_ATOMICITY: 3632 return nvmf_ctrlr_set_features_write_atomicity(req); 3633 case SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 3634 return nvmf_ctrlr_set_features_async_event_configuration(req); 3635 case SPDK_NVME_FEAT_KEEP_ALIVE_TIMER: 3636 return nvmf_ctrlr_set_features_keep_alive_timer(req); 3637 case SPDK_NVME_FEAT_HOST_IDENTIFIER: 3638 return nvmf_ctrlr_set_features_host_identifier(req); 3639 case SPDK_NVME_FEAT_HOST_RESERVE_MASK: 3640 return nvmf_ctrlr_set_features_reservation_notification_mask(req); 3641 case SPDK_NVME_FEAT_HOST_RESERVE_PERSIST: 3642 return nvmf_ctrlr_set_features_reservation_persistence(req); 3643 case SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 3644 return nvmf_ctrlr_set_features_host_behavior_support(req); 3645 default: 3646 SPDK_INFOLOG(nvmf, "Set Features command with unsupported feature ID 0x%02x\n", feature); 3647 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3648 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3649 } 3650 } 3651 3652 static int 3653 nvmf_ctrlr_keep_alive(struct spdk_nvmf_request *req) 3654 { 3655 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3656 3657 SPDK_DEBUGLOG(nvmf, "Keep Alive\n"); 3658 /* 3659 * To handle keep alive just clear or reset the 3660 * ctrlr based keep alive duration counter. 3661 * When added, a separate timer based process 3662 * will monitor if the time since last recorded 3663 * keep alive has exceeded the max duration and 3664 * take appropriate action. 3665 */ 3666 ctrlr->last_keep_alive_tick = spdk_get_ticks(); 3667 3668 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3669 } 3670 3671 int 3672 nvmf_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req) 3673 { 3674 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 3675 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 3676 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 3677 struct spdk_nvmf_subsystem_poll_group *sgroup; 3678 int rc; 3679 3680 assert(ctrlr != NULL); 3681 if (cmd->opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) { 3682 /* We do not want to treat AERs as outstanding commands, 3683 * so decrement mgmt_io_outstanding here to offset 3684 * the increment that happened prior to this call. 3685 */ 3686 sgroup = &req->qpair->group->sgroups[ctrlr->subsys->id]; 3687 assert(sgroup != NULL); 3688 sgroup->mgmt_io_outstanding--; 3689 } 3690 3691 assert(spdk_get_thread() == ctrlr->thread); 3692 3693 if (cmd->fuse != 0) { 3694 /* Fused admin commands are not supported. */ 3695 response->status.sct = SPDK_NVME_SCT_GENERIC; 3696 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 3697 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3698 } 3699 3700 if (ctrlr->vcprop.cc.bits.en != 1) { 3701 SPDK_ERRLOG("Admin command sent to disabled controller\n"); 3702 response->status.sct = SPDK_NVME_SCT_GENERIC; 3703 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 3704 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3705 } 3706 3707 if (req->iovcnt && spdk_nvme_opc_get_data_transfer(cmd->opc) == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 3708 spdk_iov_memset(req->iov, req->iovcnt, 0); 3709 } 3710 3711 if (spdk_nvmf_subsystem_is_discovery(ctrlr->subsys)) { 3712 /* Discovery controllers only support these admin OPS. */ 3713 switch (cmd->opc) { 3714 case SPDK_NVME_OPC_IDENTIFY: 3715 case SPDK_NVME_OPC_GET_LOG_PAGE: 3716 case SPDK_NVME_OPC_KEEP_ALIVE: 3717 case SPDK_NVME_OPC_SET_FEATURES: 3718 case SPDK_NVME_OPC_GET_FEATURES: 3719 case SPDK_NVME_OPC_ASYNC_EVENT_REQUEST: 3720 break; 3721 default: 3722 goto invalid_opcode; 3723 } 3724 } 3725 3726 /* Call a custom adm cmd handler if set. Aborts are handled in a different path (see nvmf_passthru_admin_cmd) */ 3727 if (g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].hdlr && cmd->opc != SPDK_NVME_OPC_ABORT) { 3728 rc = g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].hdlr(req); 3729 if (rc >= SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 3730 /* The handler took care of this command */ 3731 return rc; 3732 } 3733 } 3734 3735 switch (cmd->opc) { 3736 case SPDK_NVME_OPC_GET_LOG_PAGE: 3737 return nvmf_ctrlr_get_log_page(req); 3738 case SPDK_NVME_OPC_IDENTIFY: 3739 return nvmf_ctrlr_identify(req); 3740 case SPDK_NVME_OPC_ABORT: 3741 return nvmf_ctrlr_abort(req); 3742 case SPDK_NVME_OPC_GET_FEATURES: 3743 return nvmf_ctrlr_get_features(req); 3744 case SPDK_NVME_OPC_SET_FEATURES: 3745 return nvmf_ctrlr_set_features(req); 3746 case SPDK_NVME_OPC_ASYNC_EVENT_REQUEST: 3747 return nvmf_ctrlr_async_event_request(req); 3748 case SPDK_NVME_OPC_KEEP_ALIVE: 3749 return nvmf_ctrlr_keep_alive(req); 3750 3751 case SPDK_NVME_OPC_CREATE_IO_SQ: 3752 case SPDK_NVME_OPC_CREATE_IO_CQ: 3753 case SPDK_NVME_OPC_DELETE_IO_SQ: 3754 case SPDK_NVME_OPC_DELETE_IO_CQ: 3755 /* Create and Delete I/O CQ/SQ not allowed in NVMe-oF */ 3756 goto invalid_opcode; 3757 3758 default: 3759 goto invalid_opcode; 3760 } 3761 3762 invalid_opcode: 3763 SPDK_INFOLOG(nvmf, "Unsupported admin opcode 0x%x\n", cmd->opc); 3764 response->status.sct = SPDK_NVME_SCT_GENERIC; 3765 response->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3766 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3767 } 3768 3769 static int 3770 nvmf_ctrlr_process_fabrics_cmd(struct spdk_nvmf_request *req) 3771 { 3772 struct spdk_nvmf_qpair *qpair = req->qpair; 3773 struct spdk_nvmf_capsule_cmd *cap_hdr; 3774 3775 cap_hdr = &req->cmd->nvmf_cmd; 3776 3777 if (qpair->ctrlr == NULL) { 3778 /* No ctrlr established yet; the only valid command is Connect */ 3779 assert(cap_hdr->fctype == SPDK_NVMF_FABRIC_COMMAND_CONNECT); 3780 return nvmf_ctrlr_cmd_connect(req); 3781 } else if (nvmf_qpair_is_admin_queue(qpair)) { 3782 /* 3783 * Controller session is established, and this is an admin queue. 3784 * Disallow Connect and allow other fabrics commands. 3785 */ 3786 switch (cap_hdr->fctype) { 3787 case SPDK_NVMF_FABRIC_COMMAND_PROPERTY_SET: 3788 return nvmf_property_set(req); 3789 case SPDK_NVMF_FABRIC_COMMAND_PROPERTY_GET: 3790 return nvmf_property_get(req); 3791 case SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_SEND: 3792 case SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_RECV: 3793 return nvmf_auth_request_exec(req); 3794 default: 3795 SPDK_DEBUGLOG(nvmf, "unknown fctype 0x%02x\n", 3796 cap_hdr->fctype); 3797 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3798 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3799 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3800 } 3801 } else { 3802 /* 3803 * Controller session is established, and this is an I/O queue. 3804 * Disallow everything besides authentication commands. 3805 */ 3806 switch (cap_hdr->fctype) { 3807 case SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_SEND: 3808 case SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_RECV: 3809 return nvmf_auth_request_exec(req); 3810 default: 3811 SPDK_DEBUGLOG(nvmf, "Unexpected I/O fctype 0x%x\n", cap_hdr->fctype); 3812 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 3813 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 3814 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 3815 } 3816 } 3817 } 3818 3819 static inline void 3820 nvmf_ctrlr_queue_pending_async_event(struct spdk_nvmf_ctrlr *ctrlr, 3821 union spdk_nvme_async_event_completion *event) 3822 { 3823 struct spdk_nvmf_async_event_completion *nvmf_event; 3824 3825 nvmf_event = calloc(1, sizeof(*nvmf_event)); 3826 if (!nvmf_event) { 3827 SPDK_ERRLOG("Alloc nvmf event failed, ignore the event\n"); 3828 return; 3829 } 3830 nvmf_event->event.raw = event->raw; 3831 STAILQ_INSERT_TAIL(&ctrlr->async_events, nvmf_event, link); 3832 } 3833 3834 static inline int 3835 nvmf_ctrlr_async_event_notification(struct spdk_nvmf_ctrlr *ctrlr, 3836 union spdk_nvme_async_event_completion *event) 3837 { 3838 struct spdk_nvmf_request *req; 3839 struct spdk_nvme_cpl *rsp; 3840 3841 assert(spdk_get_thread() == ctrlr->thread); 3842 3843 /* If there is no outstanding AER request, queue the event. Then 3844 * if an AER is later submitted, this event can be sent as a 3845 * response. 3846 */ 3847 if (ctrlr->nr_aer_reqs == 0) { 3848 nvmf_ctrlr_queue_pending_async_event(ctrlr, event); 3849 return 0; 3850 } 3851 3852 req = ctrlr->aer_req[--ctrlr->nr_aer_reqs]; 3853 rsp = &req->rsp->nvme_cpl; 3854 3855 rsp->cdw0 = event->raw; 3856 3857 _nvmf_request_complete(req); 3858 ctrlr->aer_req[ctrlr->nr_aer_reqs] = NULL; 3859 3860 return 0; 3861 } 3862 3863 int 3864 nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr) 3865 { 3866 union spdk_nvme_async_event_completion event = {0}; 3867 3868 /* Users may disable the event notification */ 3869 if (!ctrlr->feat.async_event_configuration.bits.ns_attr_notice) { 3870 return 0; 3871 } 3872 3873 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGE_MASK_BIT)) { 3874 return 0; 3875 } 3876 3877 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 3878 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED; 3879 event.bits.log_page_identifier = SPDK_NVME_LOG_CHANGED_NS_LIST; 3880 3881 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 3882 } 3883 3884 int 3885 nvmf_ctrlr_async_event_ana_change_notice(struct spdk_nvmf_ctrlr *ctrlr) 3886 { 3887 union spdk_nvme_async_event_completion event = {0}; 3888 3889 /* Users may disable the event notification */ 3890 if (!ctrlr->feat.async_event_configuration.bits.ana_change_notice) { 3891 return 0; 3892 } 3893 3894 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ANA_CHANGE_MASK_BIT)) { 3895 return 0; 3896 } 3897 3898 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 3899 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_ANA_CHANGE; 3900 event.bits.log_page_identifier = SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS; 3901 3902 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 3903 } 3904 3905 void 3906 nvmf_ctrlr_async_event_reservation_notification(struct spdk_nvmf_ctrlr *ctrlr) 3907 { 3908 union spdk_nvme_async_event_completion event = {0}; 3909 3910 if (!ctrlr->num_avail_log_pages) { 3911 return; 3912 } 3913 3914 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL_MASK_BIT)) { 3915 return; 3916 } 3917 3918 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_IO; 3919 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_RESERVATION_LOG_AVAIL; 3920 event.bits.log_page_identifier = SPDK_NVME_LOG_RESERVATION_NOTIFICATION; 3921 3922 nvmf_ctrlr_async_event_notification(ctrlr, &event); 3923 } 3924 3925 void 3926 nvmf_ctrlr_async_event_discovery_log_change_notice(void *ctx) 3927 { 3928 union spdk_nvme_async_event_completion event = {0}; 3929 struct spdk_nvmf_ctrlr *ctrlr = ctx; 3930 3931 /* Users may disable the event notification manually or 3932 * it may not be enabled due to keep alive timeout 3933 * not being set in connect command to discovery controller. 3934 */ 3935 if (!ctrlr->feat.async_event_configuration.bits.discovery_log_change_notice) { 3936 return; 3937 } 3938 3939 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE_MASK_BIT)) { 3940 return; 3941 } 3942 3943 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE; 3944 event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_DISCOVERY_LOG_CHANGE; 3945 event.bits.log_page_identifier = SPDK_NVME_LOG_DISCOVERY; 3946 3947 nvmf_ctrlr_async_event_notification(ctrlr, &event); 3948 } 3949 3950 int 3951 spdk_nvmf_ctrlr_async_event_error_event(struct spdk_nvmf_ctrlr *ctrlr, 3952 enum spdk_nvme_async_event_info_error info) 3953 { 3954 union spdk_nvme_async_event_completion event; 3955 3956 if (!nvmf_ctrlr_mask_aen(ctrlr, SPDK_NVME_ASYNC_EVENT_ERROR_MASK_BIT)) { 3957 return 0; 3958 } 3959 3960 if (info > SPDK_NVME_ASYNC_EVENT_FW_IMAGE_LOAD) { 3961 return 0; 3962 } 3963 3964 event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_ERROR; 3965 event.bits.log_page_identifier = SPDK_NVME_LOG_ERROR; 3966 event.bits.async_event_info = info; 3967 3968 return nvmf_ctrlr_async_event_notification(ctrlr, &event); 3969 } 3970 3971 void 3972 nvmf_qpair_free_aer(struct spdk_nvmf_qpair *qpair) 3973 { 3974 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 3975 int i; 3976 3977 if (ctrlr == NULL || !nvmf_qpair_is_admin_queue(qpair)) { 3978 return; 3979 } 3980 3981 assert(spdk_get_thread() == ctrlr->thread); 3982 3983 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 3984 spdk_nvmf_request_free(ctrlr->aer_req[i]); 3985 ctrlr->aer_req[i] = NULL; 3986 } 3987 3988 ctrlr->nr_aer_reqs = 0; 3989 } 3990 3991 void 3992 spdk_nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr) 3993 { 3994 struct spdk_nvmf_request *req; 3995 int i; 3996 3997 assert(spdk_get_thread() == ctrlr->thread); 3998 3999 if (!ctrlr->nr_aer_reqs) { 4000 return; 4001 } 4002 4003 for (i = 0; i < ctrlr->nr_aer_reqs; i++) { 4004 req = ctrlr->aer_req[i]; 4005 4006 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4007 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 4008 _nvmf_request_complete(req); 4009 4010 ctrlr->aer_req[i] = NULL; 4011 } 4012 4013 ctrlr->nr_aer_reqs = 0; 4014 } 4015 4016 static void 4017 _nvmf_ctrlr_add_reservation_log(void *ctx) 4018 { 4019 struct spdk_nvmf_reservation_log *log = (struct spdk_nvmf_reservation_log *)ctx; 4020 struct spdk_nvmf_ctrlr *ctrlr = log->ctrlr; 4021 4022 ctrlr->log_page_count++; 4023 4024 /* Maximum number of queued log pages is 255 */ 4025 if (ctrlr->num_avail_log_pages == 0xff) { 4026 struct spdk_nvmf_reservation_log *entry; 4027 entry = TAILQ_LAST(&ctrlr->log_head, log_page_head); 4028 entry->log.log_page_count = ctrlr->log_page_count; 4029 free(log); 4030 return; 4031 } 4032 4033 log->log.log_page_count = ctrlr->log_page_count; 4034 log->log.num_avail_log_pages = ctrlr->num_avail_log_pages++; 4035 TAILQ_INSERT_TAIL(&ctrlr->log_head, log, link); 4036 4037 nvmf_ctrlr_async_event_reservation_notification(ctrlr); 4038 } 4039 4040 void 4041 nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr, 4042 struct spdk_nvmf_ns *ns, 4043 enum spdk_nvme_reservation_notification_log_page_type type) 4044 { 4045 struct spdk_nvmf_reservation_log *log; 4046 4047 switch (type) { 4048 case SPDK_NVME_RESERVATION_LOG_PAGE_EMPTY: 4049 return; 4050 case SPDK_NVME_REGISTRATION_PREEMPTED: 4051 if (ns->mask & SPDK_NVME_REGISTRATION_PREEMPTED_MASK) { 4052 return; 4053 } 4054 break; 4055 case SPDK_NVME_RESERVATION_RELEASED: 4056 if (ns->mask & SPDK_NVME_RESERVATION_RELEASED_MASK) { 4057 return; 4058 } 4059 break; 4060 case SPDK_NVME_RESERVATION_PREEMPTED: 4061 if (ns->mask & SPDK_NVME_RESERVATION_PREEMPTED_MASK) { 4062 return; 4063 } 4064 break; 4065 default: 4066 return; 4067 } 4068 4069 log = calloc(1, sizeof(*log)); 4070 if (!log) { 4071 SPDK_ERRLOG("Alloc log page failed, ignore the log\n"); 4072 return; 4073 } 4074 log->ctrlr = ctrlr; 4075 log->log.type = type; 4076 log->log.nsid = ns->nsid; 4077 4078 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_add_reservation_log, log); 4079 } 4080 4081 /* Check from subsystem poll group's namespace information data structure */ 4082 static bool 4083 nvmf_ns_info_ctrlr_is_registrant(struct spdk_nvmf_subsystem_pg_ns_info *ns_info, 4084 struct spdk_nvmf_ctrlr *ctrlr) 4085 { 4086 uint32_t i; 4087 4088 for (i = 0; i < SPDK_NVMF_MAX_NUM_REGISTRANTS; i++) { 4089 if (!spdk_uuid_compare(&ns_info->reg_hostid[i], &ctrlr->hostid)) { 4090 return true; 4091 } 4092 } 4093 4094 return false; 4095 } 4096 4097 /* 4098 * Check the NVMe command is permitted or not for current controller(Host). 4099 */ 4100 static int 4101 nvmf_ns_reservation_request_check(struct spdk_nvmf_subsystem_pg_ns_info *ns_info, 4102 struct spdk_nvmf_ctrlr *ctrlr, 4103 struct spdk_nvmf_request *req) 4104 { 4105 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 4106 enum spdk_nvme_reservation_type rtype = ns_info->rtype; 4107 uint8_t status = SPDK_NVME_SC_SUCCESS; 4108 uint8_t racqa; 4109 bool is_registrant; 4110 4111 /* No valid reservation */ 4112 if (!rtype) { 4113 return 0; 4114 } 4115 4116 is_registrant = nvmf_ns_info_ctrlr_is_registrant(ns_info, ctrlr); 4117 /* All registrants type and current ctrlr is a valid registrant */ 4118 if ((rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE_ALL_REGS || 4119 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_ALL_REGS) && is_registrant) { 4120 return 0; 4121 } else if (!spdk_uuid_compare(&ns_info->holder_id, &ctrlr->hostid)) { 4122 return 0; 4123 } 4124 4125 /* Non-holder for current controller */ 4126 switch (cmd->opc) { 4127 case SPDK_NVME_OPC_READ: 4128 case SPDK_NVME_OPC_COMPARE: 4129 if (rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) { 4130 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4131 goto exit; 4132 } 4133 if ((rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_REG_ONLY || 4134 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_ALL_REGS) && !is_registrant) { 4135 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4136 } 4137 break; 4138 case SPDK_NVME_OPC_FLUSH: 4139 case SPDK_NVME_OPC_WRITE: 4140 case SPDK_NVME_OPC_WRITE_UNCORRECTABLE: 4141 case SPDK_NVME_OPC_WRITE_ZEROES: 4142 case SPDK_NVME_OPC_DATASET_MANAGEMENT: 4143 if (rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE || 4144 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) { 4145 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4146 goto exit; 4147 } 4148 if (!is_registrant) { 4149 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4150 } 4151 break; 4152 case SPDK_NVME_OPC_RESERVATION_ACQUIRE: 4153 racqa = cmd->cdw10_bits.resv_acquire.racqa; 4154 if (racqa == SPDK_NVME_RESERVE_ACQUIRE) { 4155 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4156 goto exit; 4157 } 4158 if (!is_registrant) { 4159 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4160 } 4161 break; 4162 case SPDK_NVME_OPC_RESERVATION_RELEASE: 4163 if (!is_registrant) { 4164 status = SPDK_NVME_SC_RESERVATION_CONFLICT; 4165 } 4166 break; 4167 default: 4168 break; 4169 } 4170 4171 exit: 4172 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4173 req->rsp->nvme_cpl.status.sc = status; 4174 if (status == SPDK_NVME_SC_RESERVATION_CONFLICT) { 4175 return -EPERM; 4176 } 4177 4178 return 0; 4179 } 4180 4181 static int 4182 nvmf_ctrlr_process_io_fused_cmd(struct spdk_nvmf_request *req, struct spdk_bdev *bdev, 4183 struct spdk_bdev_desc *desc, struct spdk_io_channel *ch) 4184 { 4185 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 4186 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 4187 struct spdk_nvmf_request *first_fused_req = req->qpair->first_fused_req; 4188 int rc; 4189 4190 if (cmd->fuse == SPDK_NVME_CMD_FUSE_FIRST) { 4191 /* first fused operation (should be compare) */ 4192 if (first_fused_req != NULL) { 4193 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 4194 4195 SPDK_ERRLOG("Wrong sequence of fused operations\n"); 4196 4197 /* abort req->qpair->first_fused_request and continue with new fused command */ 4198 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4199 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 4200 _nvmf_request_complete(first_fused_req); 4201 } else if (cmd->opc != SPDK_NVME_OPC_COMPARE) { 4202 SPDK_ERRLOG("Wrong op code of fused operations\n"); 4203 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4204 rsp->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 4205 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4206 } 4207 4208 req->qpair->first_fused_req = req; 4209 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 4210 } else if (cmd->fuse == SPDK_NVME_CMD_FUSE_SECOND) { 4211 /* second fused operation (should be write) */ 4212 if (first_fused_req == NULL) { 4213 SPDK_ERRLOG("Wrong sequence of fused operations\n"); 4214 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4215 rsp->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4216 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4217 } else if (cmd->opc != SPDK_NVME_OPC_WRITE) { 4218 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 4219 4220 SPDK_ERRLOG("Wrong op code of fused operations\n"); 4221 4222 /* abort req->qpair->first_fused_request and fail current command */ 4223 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4224 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 4225 _nvmf_request_complete(first_fused_req); 4226 4227 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4228 rsp->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 4229 req->qpair->first_fused_req = NULL; 4230 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4231 } 4232 4233 /* save request of first command to generate response later */ 4234 req->first_fused_req = first_fused_req; 4235 req->first_fused = true; 4236 req->qpair->first_fused_req = NULL; 4237 } else { 4238 SPDK_ERRLOG("Invalid fused command fuse field.\n"); 4239 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4240 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 4241 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4242 } 4243 4244 rc = nvmf_bdev_ctrlr_compare_and_write_cmd(bdev, desc, ch, req->first_fused_req, req); 4245 4246 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 4247 if (spdk_nvme_cpl_is_error(rsp)) { 4248 struct spdk_nvme_cpl *fused_response = &first_fused_req->rsp->nvme_cpl; 4249 4250 fused_response->status = rsp->status; 4251 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 4252 rsp->status.sc = SPDK_NVME_SC_ABORTED_FAILED_FUSED; 4253 /* Complete first of fused commands. Second will be completed by upper layer */ 4254 _nvmf_request_complete(first_fused_req); 4255 req->first_fused_req = NULL; 4256 req->first_fused = false; 4257 } 4258 } 4259 4260 return rc; 4261 } 4262 4263 bool 4264 nvmf_ctrlr_use_zcopy(struct spdk_nvmf_request *req) 4265 { 4266 struct spdk_nvmf_transport *transport = req->qpair->transport; 4267 struct spdk_nvmf_ns *ns; 4268 4269 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_NONE); 4270 4271 if (!transport->opts.zcopy) { 4272 return false; 4273 } 4274 4275 if (nvmf_qpair_is_admin_queue(req->qpair)) { 4276 /* Admin queue */ 4277 return false; 4278 } 4279 4280 if ((req->cmd->nvme_cmd.opc != SPDK_NVME_OPC_WRITE) && 4281 (req->cmd->nvme_cmd.opc != SPDK_NVME_OPC_READ)) { 4282 /* Not a READ or WRITE command */ 4283 return false; 4284 } 4285 4286 if (req->cmd->nvme_cmd.fuse != SPDK_NVME_CMD_FUSE_NONE) { 4287 /* Fused commands dont use zcopy buffers */ 4288 return false; 4289 } 4290 4291 ns = nvmf_ctrlr_get_ns(req->qpair->ctrlr, req->cmd->nvme_cmd.nsid); 4292 if (ns == NULL || ns->bdev == NULL || !ns->zcopy) { 4293 return false; 4294 } 4295 4296 req->zcopy_phase = NVMF_ZCOPY_PHASE_INIT; 4297 return true; 4298 } 4299 4300 void 4301 spdk_nvmf_request_zcopy_start(struct spdk_nvmf_request *req) 4302 { 4303 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT); 4304 4305 /* Set iovcnt to be the maximum number of iovs that the ZCOPY can use */ 4306 req->iovcnt = NVMF_REQ_MAX_BUFFERS; 4307 4308 spdk_nvmf_request_exec(req); 4309 } 4310 4311 void 4312 spdk_nvmf_request_zcopy_end(struct spdk_nvmf_request *req, bool commit) 4313 { 4314 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_EXECUTE); 4315 req->zcopy_phase = NVMF_ZCOPY_PHASE_END_PENDING; 4316 4317 nvmf_bdev_ctrlr_zcopy_end(req, commit); 4318 } 4319 4320 int 4321 nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) 4322 { 4323 uint32_t nsid; 4324 struct spdk_nvmf_ns *ns; 4325 struct spdk_bdev *bdev; 4326 struct spdk_bdev_desc *desc; 4327 struct spdk_io_channel *ch; 4328 struct spdk_nvmf_poll_group *group = req->qpair->group; 4329 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 4330 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 4331 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 4332 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4333 enum spdk_nvme_ana_state ana_state; 4334 4335 /* pre-set response details for this command */ 4336 response->status.sc = SPDK_NVME_SC_SUCCESS; 4337 nsid = cmd->nsid; 4338 4339 assert(ctrlr != NULL); 4340 if (spdk_unlikely(ctrlr->vcprop.cc.bits.en != 1)) { 4341 SPDK_ERRLOG("I/O command sent to disabled controller\n"); 4342 response->status.sct = SPDK_NVME_SCT_GENERIC; 4343 response->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 4344 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4345 } 4346 4347 ns = nvmf_ctrlr_get_ns(ctrlr, nsid); 4348 if (spdk_unlikely(ns == NULL || ns->bdev == NULL)) { 4349 SPDK_DEBUGLOG(nvmf, "Unsuccessful query for nsid %u\n", cmd->nsid); 4350 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4351 response->status.dnr = 1; 4352 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4353 } 4354 4355 ana_state = nvmf_ctrlr_get_ana_state(ctrlr, ns->anagrpid); 4356 if (spdk_unlikely(ana_state != SPDK_NVME_ANA_OPTIMIZED_STATE && 4357 ana_state != SPDK_NVME_ANA_NON_OPTIMIZED_STATE)) { 4358 SPDK_DEBUGLOG(nvmf, "Fail I/O command due to ANA state %d\n", 4359 ana_state); 4360 response->status.sct = SPDK_NVME_SCT_PATH; 4361 response->status.sc = _nvme_ana_state_to_path_status(ana_state); 4362 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4363 } 4364 4365 if (spdk_likely(ctrlr->listener != NULL)) { 4366 SPDK_DTRACE_PROBE3_TICKS(nvmf_request_io_exec_path, req, 4367 ctrlr->listener->trid->traddr, 4368 ctrlr->listener->trid->trsvcid); 4369 } 4370 4371 /* scan-build falsely reporting dereference of null pointer */ 4372 assert(group != NULL && group->sgroups != NULL); 4373 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[nsid - 1]; 4374 if (nvmf_ns_reservation_request_check(ns_info, ctrlr, req)) { 4375 SPDK_DEBUGLOG(nvmf, "Reservation Conflict for nsid %u, opcode %u\n", 4376 cmd->nsid, cmd->opc); 4377 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4378 } 4379 4380 bdev = ns->bdev; 4381 desc = ns->desc; 4382 ch = ns_info->channel; 4383 4384 if (spdk_unlikely(cmd->fuse & SPDK_NVME_CMD_FUSE_MASK)) { 4385 return nvmf_ctrlr_process_io_fused_cmd(req, bdev, desc, ch); 4386 } else if (spdk_unlikely(req->qpair->first_fused_req != NULL)) { 4387 struct spdk_nvme_cpl *fused_response = &req->qpair->first_fused_req->rsp->nvme_cpl; 4388 4389 SPDK_ERRLOG("Second fused cmd expected - failing first one (opcode:0x%x)\n", 4390 req->qpair->first_fused_req->cmd->nvmf_cmd.opcode); 4391 4392 /* abort req->qpair->first_fused_request and continue with new command */ 4393 fused_response->status.sc = SPDK_NVME_SC_ABORTED_MISSING_FUSED; 4394 fused_response->status.sct = SPDK_NVME_SCT_GENERIC; 4395 _nvmf_request_complete(req->qpair->first_fused_req); 4396 req->qpair->first_fused_req = NULL; 4397 } 4398 4399 if (spdk_nvmf_request_using_zcopy(req)) { 4400 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT); 4401 return nvmf_bdev_ctrlr_zcopy_start(bdev, desc, ch, req); 4402 } else { 4403 switch (cmd->opc) { 4404 case SPDK_NVME_OPC_READ: 4405 return nvmf_bdev_ctrlr_read_cmd(bdev, desc, ch, req); 4406 case SPDK_NVME_OPC_WRITE: 4407 return nvmf_bdev_ctrlr_write_cmd(bdev, desc, ch, req); 4408 case SPDK_NVME_OPC_COMPARE: 4409 return nvmf_bdev_ctrlr_compare_cmd(bdev, desc, ch, req); 4410 case SPDK_NVME_OPC_WRITE_ZEROES: 4411 return nvmf_bdev_ctrlr_write_zeroes_cmd(bdev, desc, ch, req); 4412 case SPDK_NVME_OPC_FLUSH: 4413 return nvmf_bdev_ctrlr_flush_cmd(bdev, desc, ch, req); 4414 case SPDK_NVME_OPC_DATASET_MANAGEMENT: 4415 return nvmf_bdev_ctrlr_dsm_cmd(bdev, desc, ch, req); 4416 case SPDK_NVME_OPC_RESERVATION_REGISTER: 4417 case SPDK_NVME_OPC_RESERVATION_ACQUIRE: 4418 case SPDK_NVME_OPC_RESERVATION_RELEASE: 4419 case SPDK_NVME_OPC_RESERVATION_REPORT: 4420 spdk_thread_send_msg(ctrlr->subsys->thread, nvmf_ns_reservation_request, req); 4421 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 4422 case SPDK_NVME_OPC_COPY: 4423 return nvmf_bdev_ctrlr_copy_cmd(bdev, desc, ch, req); 4424 default: 4425 return nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req); 4426 } 4427 } 4428 } 4429 4430 static void 4431 nvmf_qpair_request_cleanup(struct spdk_nvmf_qpair *qpair) 4432 { 4433 if (spdk_unlikely(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING)) { 4434 assert(qpair->state_cb != NULL); 4435 4436 if (TAILQ_EMPTY(&qpair->outstanding)) { 4437 qpair->state_cb(qpair->state_cb_arg, 0); 4438 } 4439 } 4440 } 4441 4442 int 4443 spdk_nvmf_request_free(struct spdk_nvmf_request *req) 4444 { 4445 struct spdk_nvmf_qpair *qpair = req->qpair; 4446 4447 TAILQ_REMOVE(&qpair->outstanding, req, link); 4448 if (spdk_unlikely(nvmf_transport_req_free(req))) { 4449 SPDK_ERRLOG("Unable to free transport level request resources.\n"); 4450 } 4451 4452 nvmf_qpair_request_cleanup(qpair); 4453 4454 return 0; 4455 } 4456 4457 static void 4458 _nvmf_request_complete(void *ctx) 4459 { 4460 struct spdk_nvmf_request *req = ctx; 4461 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 4462 struct spdk_nvmf_qpair *qpair; 4463 struct spdk_nvmf_subsystem_poll_group *sgroup = NULL; 4464 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4465 bool is_aer = false; 4466 uint32_t nsid; 4467 bool paused; 4468 uint8_t opcode; 4469 4470 rsp->sqid = 0; 4471 rsp->status.p = 0; 4472 rsp->cid = req->cmd->nvme_cmd.cid; 4473 nsid = req->cmd->nvme_cmd.nsid; 4474 opcode = req->cmd->nvmf_cmd.opcode; 4475 4476 qpair = req->qpair; 4477 if (spdk_likely(qpair->ctrlr)) { 4478 sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id]; 4479 assert(sgroup != NULL); 4480 is_aer = req->cmd->nvme_cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST; 4481 if (spdk_likely(qpair->qid != 0)) { 4482 qpair->group->stat.completed_nvme_io++; 4483 } 4484 4485 /* 4486 * Set the crd value. 4487 * If the the IO has any error, and dnr (DoNotRetry) is not 1, 4488 * and ACRE is enabled, we will set the crd to 1 to select the first CRDT. 4489 */ 4490 if (spdk_unlikely(spdk_nvme_cpl_is_error(rsp) && 4491 rsp->status.dnr == 0 && 4492 qpair->ctrlr->acre_enabled)) { 4493 rsp->status.crd = 1; 4494 } 4495 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) { 4496 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 4497 } 4498 4499 if (SPDK_DEBUGLOG_FLAG_ENABLED("nvmf")) { 4500 spdk_nvme_print_completion(qpair->qid, rsp); 4501 } 4502 4503 switch (req->zcopy_phase) { 4504 case NVMF_ZCOPY_PHASE_NONE: 4505 TAILQ_REMOVE(&qpair->outstanding, req, link); 4506 break; 4507 case NVMF_ZCOPY_PHASE_INIT: 4508 if (spdk_unlikely(spdk_nvme_cpl_is_error(rsp))) { 4509 req->zcopy_phase = NVMF_ZCOPY_PHASE_INIT_FAILED; 4510 TAILQ_REMOVE(&qpair->outstanding, req, link); 4511 } else { 4512 req->zcopy_phase = NVMF_ZCOPY_PHASE_EXECUTE; 4513 } 4514 break; 4515 case NVMF_ZCOPY_PHASE_EXECUTE: 4516 break; 4517 case NVMF_ZCOPY_PHASE_END_PENDING: 4518 TAILQ_REMOVE(&qpair->outstanding, req, link); 4519 req->zcopy_phase = NVMF_ZCOPY_PHASE_COMPLETE; 4520 break; 4521 default: 4522 SPDK_ERRLOG("Invalid ZCOPY phase %u\n", req->zcopy_phase); 4523 break; 4524 } 4525 4526 if (spdk_unlikely(nvmf_transport_req_complete(req))) { 4527 SPDK_ERRLOG("Transport request completion error!\n"); 4528 } 4529 4530 /* AER cmd is an exception */ 4531 if (spdk_likely(sgroup && !is_aer)) { 4532 if (spdk_unlikely(opcode == SPDK_NVME_OPC_FABRIC || 4533 nvmf_qpair_is_admin_queue(qpair))) { 4534 assert(sgroup->mgmt_io_outstanding > 0); 4535 sgroup->mgmt_io_outstanding--; 4536 } else { 4537 if (req->zcopy_phase == NVMF_ZCOPY_PHASE_NONE || 4538 req->zcopy_phase == NVMF_ZCOPY_PHASE_COMPLETE || 4539 req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT_FAILED) { 4540 /* End of request */ 4541 4542 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 4543 if (spdk_likely(nsid - 1 < sgroup->num_ns)) { 4544 sgroup->ns_info[nsid - 1].io_outstanding--; 4545 } 4546 } 4547 } 4548 4549 if (spdk_unlikely(sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSING && 4550 sgroup->mgmt_io_outstanding == 0)) { 4551 paused = true; 4552 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 4553 ns_info = &sgroup->ns_info[nsid]; 4554 4555 if (ns_info->state == SPDK_NVMF_SUBSYSTEM_PAUSING && 4556 ns_info->io_outstanding > 0) { 4557 paused = false; 4558 break; 4559 } 4560 } 4561 4562 if (paused) { 4563 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 4564 sgroup->cb_fn(sgroup->cb_arg, 0); 4565 sgroup->cb_fn = NULL; 4566 sgroup->cb_arg = NULL; 4567 } 4568 } 4569 4570 } 4571 4572 nvmf_qpair_request_cleanup(qpair); 4573 } 4574 4575 int 4576 spdk_nvmf_request_complete(struct spdk_nvmf_request *req) 4577 { 4578 struct spdk_nvmf_qpair *qpair = req->qpair; 4579 4580 spdk_thread_exec_msg(qpair->group->thread, _nvmf_request_complete, req); 4581 4582 return 0; 4583 } 4584 4585 SPDK_LOG_DEPRECATION_REGISTER(nvmf_request_exec_fabrics, "spdk_nvmf_request_exec_fabrics()", 4586 "v24.09", 1); 4587 void 4588 spdk_nvmf_request_exec_fabrics(struct spdk_nvmf_request *req) 4589 { 4590 SPDK_LOG_DEPRECATED(nvmf_request_exec_fabrics); 4591 4592 return spdk_nvmf_request_exec(req); 4593 } 4594 4595 static bool 4596 nvmf_check_subsystem_active(struct spdk_nvmf_request *req) 4597 { 4598 struct spdk_nvmf_qpair *qpair = req->qpair; 4599 struct spdk_nvmf_subsystem_poll_group *sgroup = NULL; 4600 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4601 uint32_t nsid; 4602 4603 if (spdk_likely(qpair->ctrlr)) { 4604 sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id]; 4605 assert(sgroup != NULL); 4606 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) { 4607 sgroup = nvmf_subsystem_pg_from_connect_cmd(req); 4608 } 4609 4610 /* Check if the subsystem is paused (if there is a subsystem) */ 4611 if (spdk_unlikely(sgroup == NULL)) { 4612 return true; 4613 } 4614 4615 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC || 4616 nvmf_qpair_is_admin_queue(qpair))) { 4617 if (sgroup->state != SPDK_NVMF_SUBSYSTEM_ACTIVE) { 4618 /* The subsystem is not currently active. Queue this request. */ 4619 TAILQ_INSERT_TAIL(&sgroup->queued, req, link); 4620 return false; 4621 } 4622 sgroup->mgmt_io_outstanding++; 4623 } else { 4624 nsid = req->cmd->nvme_cmd.nsid; 4625 4626 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 4627 if (spdk_unlikely(nsid - 1 >= sgroup->num_ns)) { 4628 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4629 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4630 req->rsp->nvme_cpl.status.dnr = 1; 4631 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4632 _nvmf_request_complete(req); 4633 return false; 4634 } 4635 4636 ns_info = &sgroup->ns_info[nsid - 1]; 4637 if (spdk_unlikely(ns_info->channel == NULL)) { 4638 /* This can can happen if host sends I/O to a namespace that is 4639 * in the process of being added, but before the full addition 4640 * process is complete. Report invalid namespace in that case. 4641 */ 4642 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 4643 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4644 req->rsp->nvme_cpl.status.dnr = 1; 4645 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4646 ns_info->io_outstanding++; 4647 _nvmf_request_complete(req); 4648 return false; 4649 } 4650 4651 if (spdk_unlikely(ns_info->state != SPDK_NVMF_SUBSYSTEM_ACTIVE)) { 4652 /* The namespace is not currently active. Queue this request. */ 4653 TAILQ_INSERT_TAIL(&sgroup->queued, req, link); 4654 return false; 4655 } 4656 4657 ns_info->io_outstanding++; 4658 } 4659 4660 return true; 4661 } 4662 4663 static bool 4664 nvmf_check_qpair_active(struct spdk_nvmf_request *req) 4665 { 4666 struct spdk_nvmf_qpair *qpair = req->qpair; 4667 int sc, sct; 4668 4669 if (spdk_likely(qpair->state == SPDK_NVMF_QPAIR_ENABLED)) { 4670 return true; 4671 } 4672 4673 sct = SPDK_NVME_SCT_GENERIC; 4674 sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR; 4675 4676 switch (qpair->state) { 4677 case SPDK_NVMF_QPAIR_CONNECTING: 4678 if (req->cmd->nvmf_cmd.opcode != SPDK_NVME_OPC_FABRIC) { 4679 SPDK_ERRLOG("Received command 0x%x on qid %u before CONNECT\n", 4680 req->cmd->nvmf_cmd.opcode, qpair->qid); 4681 break; 4682 } 4683 if (req->cmd->nvmf_cmd.fctype != SPDK_NVMF_FABRIC_COMMAND_CONNECT) { 4684 SPDK_ERRLOG("Received fctype 0x%x on qid %u before CONNECT\n", 4685 req->cmd->nvmf_cmd.fctype, qpair->qid); 4686 break; 4687 } 4688 return true; 4689 case SPDK_NVMF_QPAIR_AUTHENTICATING: 4690 sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 4691 sc = SPDK_NVMF_FABRIC_SC_AUTH_REQUIRED; 4692 if (req->cmd->nvmf_cmd.opcode != SPDK_NVME_OPC_FABRIC) { 4693 SPDK_ERRLOG("Received command 0x%x on qid %u before authentication\n", 4694 req->cmd->nvmf_cmd.opcode, qpair->qid); 4695 break; 4696 } 4697 if (req->cmd->nvmf_cmd.fctype != SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_SEND && 4698 req->cmd->nvmf_cmd.fctype != SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_RECV) { 4699 SPDK_ERRLOG("Received fctype 0x%x on qid %u before authentication\n", 4700 req->cmd->nvmf_cmd.fctype, qpair->qid); 4701 break; 4702 } 4703 return true; 4704 default: 4705 SPDK_ERRLOG("Received command 0x%x on qid %u in state %d\n", 4706 req->cmd->nvmf_cmd.opcode, qpair->qid, qpair->state); 4707 break; 4708 } 4709 4710 req->rsp->nvme_cpl.status.sct = sct; 4711 req->rsp->nvme_cpl.status.sc = sc; 4712 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4713 _nvmf_request_complete(req); 4714 4715 return false; 4716 } 4717 4718 void 4719 spdk_nvmf_request_exec(struct spdk_nvmf_request *req) 4720 { 4721 struct spdk_nvmf_qpair *qpair = req->qpair; 4722 enum spdk_nvmf_request_exec_status status; 4723 4724 if (spdk_unlikely(!nvmf_check_subsystem_active(req))) { 4725 return; 4726 } 4727 if (spdk_unlikely(!nvmf_check_qpair_active(req))) { 4728 return; 4729 } 4730 4731 if (SPDK_DEBUGLOG_FLAG_ENABLED("nvmf")) { 4732 spdk_nvme_print_command(qpair->qid, &req->cmd->nvme_cmd); 4733 } 4734 4735 /* Place the request on the outstanding list so we can keep track of it */ 4736 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link); 4737 4738 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC)) { 4739 status = nvmf_ctrlr_process_fabrics_cmd(req); 4740 } else if (spdk_unlikely(nvmf_qpair_is_admin_queue(qpair))) { 4741 status = nvmf_ctrlr_process_admin_cmd(req); 4742 } else { 4743 status = nvmf_ctrlr_process_io_cmd(req); 4744 } 4745 4746 if (status == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 4747 _nvmf_request_complete(req); 4748 } 4749 } 4750 4751 static bool 4752 nvmf_ctrlr_get_dif_ctx(struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvme_cmd *cmd, 4753 struct spdk_dif_ctx *dif_ctx) 4754 { 4755 struct spdk_nvmf_ns *ns; 4756 struct spdk_bdev *bdev; 4757 4758 if (ctrlr == NULL || cmd == NULL) { 4759 return false; 4760 } 4761 4762 ns = nvmf_ctrlr_get_ns(ctrlr, cmd->nsid); 4763 if (ns == NULL || ns->bdev == NULL) { 4764 return false; 4765 } 4766 4767 bdev = ns->bdev; 4768 4769 switch (cmd->opc) { 4770 case SPDK_NVME_OPC_READ: 4771 case SPDK_NVME_OPC_WRITE: 4772 case SPDK_NVME_OPC_COMPARE: 4773 return nvmf_bdev_ctrlr_get_dif_ctx(bdev, cmd, dif_ctx); 4774 default: 4775 break; 4776 } 4777 4778 return false; 4779 } 4780 4781 bool 4782 spdk_nvmf_request_get_dif_ctx(struct spdk_nvmf_request *req, struct spdk_dif_ctx *dif_ctx) 4783 { 4784 struct spdk_nvmf_qpair *qpair = req->qpair; 4785 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 4786 4787 if (spdk_likely(ctrlr == NULL || !ctrlr->dif_insert_or_strip)) { 4788 return false; 4789 } 4790 4791 if (spdk_unlikely(!spdk_nvmf_qpair_is_active(qpair))) { 4792 return false; 4793 } 4794 4795 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC)) { 4796 return false; 4797 } 4798 4799 if (spdk_unlikely(nvmf_qpair_is_admin_queue(qpair))) { 4800 return false; 4801 } 4802 4803 return nvmf_ctrlr_get_dif_ctx(ctrlr, &req->cmd->nvme_cmd, dif_ctx); 4804 } 4805 4806 void 4807 spdk_nvmf_set_custom_admin_cmd_hdlr(uint8_t opc, spdk_nvmf_custom_cmd_hdlr hdlr) 4808 { 4809 g_nvmf_custom_admin_cmd_hdlrs[opc].hdlr = hdlr; 4810 } 4811 4812 static int 4813 nvmf_passthru_admin_cmd_for_bdev_nsid(struct spdk_nvmf_request *req, uint32_t bdev_nsid) 4814 { 4815 struct spdk_bdev *bdev; 4816 struct spdk_bdev_desc *desc; 4817 struct spdk_io_channel *ch; 4818 struct spdk_nvme_cpl *response = spdk_nvmf_request_get_response(req); 4819 int rc; 4820 4821 rc = spdk_nvmf_request_get_bdev(bdev_nsid, req, &bdev, &desc, &ch); 4822 if (rc) { 4823 response->status.sct = SPDK_NVME_SCT_GENERIC; 4824 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4825 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4826 } 4827 return spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(bdev, desc, ch, req, NULL); 4828 } 4829 4830 static int 4831 nvmf_passthru_admin_cmd(struct spdk_nvmf_request *req) 4832 { 4833 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req); 4834 uint32_t bdev_nsid; 4835 4836 if (g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].nsid != 0) { 4837 bdev_nsid = g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].nsid; 4838 } else { 4839 bdev_nsid = cmd->nsid; 4840 } 4841 4842 return nvmf_passthru_admin_cmd_for_bdev_nsid(req, bdev_nsid); 4843 } 4844 4845 int 4846 nvmf_passthru_admin_cmd_for_ctrlr(struct spdk_nvmf_request *req, struct spdk_nvmf_ctrlr *ctrlr) 4847 { 4848 struct spdk_nvme_cpl *response = spdk_nvmf_request_get_response(req); 4849 struct spdk_nvmf_ns *ns; 4850 4851 ns = spdk_nvmf_subsystem_get_first_ns(ctrlr->subsys); 4852 if (ns == NULL) { 4853 /* Is there a better sc to use here? */ 4854 response->status.sct = SPDK_NVME_SCT_GENERIC; 4855 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 4856 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 4857 } 4858 4859 return nvmf_passthru_admin_cmd_for_bdev_nsid(req, ns->nsid); 4860 } 4861 4862 void 4863 spdk_nvmf_set_passthru_admin_cmd(uint8_t opc, uint32_t forward_nsid) 4864 { 4865 g_nvmf_custom_admin_cmd_hdlrs[opc].hdlr = nvmf_passthru_admin_cmd; 4866 g_nvmf_custom_admin_cmd_hdlrs[opc].nsid = forward_nsid; 4867 } 4868 4869 int 4870 spdk_nvmf_request_get_bdev(uint32_t nsid, struct spdk_nvmf_request *req, 4871 struct spdk_bdev **bdev, struct spdk_bdev_desc **desc, struct spdk_io_channel **ch) 4872 { 4873 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr; 4874 struct spdk_nvmf_ns *ns; 4875 struct spdk_nvmf_poll_group *group = req->qpair->group; 4876 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 4877 4878 *bdev = NULL; 4879 *desc = NULL; 4880 *ch = NULL; 4881 4882 ns = nvmf_ctrlr_get_ns(ctrlr, nsid); 4883 if (ns == NULL || ns->bdev == NULL) { 4884 return -EINVAL; 4885 } 4886 4887 assert(group != NULL && group->sgroups != NULL); 4888 ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[nsid - 1]; 4889 *bdev = ns->bdev; 4890 *desc = ns->desc; 4891 *ch = ns_info->channel; 4892 4893 return 0; 4894 } 4895 4896 struct spdk_nvmf_ctrlr *spdk_nvmf_request_get_ctrlr(struct spdk_nvmf_request *req) 4897 { 4898 return req->qpair->ctrlr; 4899 } 4900 4901 struct spdk_nvme_cmd *spdk_nvmf_request_get_cmd(struct spdk_nvmf_request *req) 4902 { 4903 return &req->cmd->nvme_cmd; 4904 } 4905 4906 struct spdk_nvme_cpl *spdk_nvmf_request_get_response(struct spdk_nvmf_request *req) 4907 { 4908 return &req->rsp->nvme_cpl; 4909 } 4910 4911 struct spdk_nvmf_subsystem *spdk_nvmf_request_get_subsystem(struct spdk_nvmf_request *req) 4912 { 4913 return req->qpair->ctrlr->subsys; 4914 } 4915 4916 size_t 4917 spdk_nvmf_request_copy_from_buf(struct spdk_nvmf_request *req, 4918 void *buf, size_t buflen) 4919 { 4920 struct spdk_iov_xfer ix; 4921 4922 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 4923 return spdk_iov_xfer_from_buf(&ix, buf, buflen); 4924 } 4925 4926 size_t 4927 spdk_nvmf_request_copy_to_buf(struct spdk_nvmf_request *req, 4928 void *buf, size_t buflen) 4929 { 4930 struct spdk_iov_xfer ix; 4931 4932 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 4933 return spdk_iov_xfer_to_buf(&ix, buf, buflen); 4934 } 4935 4936 struct spdk_nvmf_subsystem *spdk_nvmf_ctrlr_get_subsystem(struct spdk_nvmf_ctrlr *ctrlr) 4937 { 4938 return ctrlr->subsys; 4939 } 4940 4941 uint16_t 4942 spdk_nvmf_ctrlr_get_id(struct spdk_nvmf_ctrlr *ctrlr) 4943 { 4944 return ctrlr->cntlid; 4945 } 4946 4947 struct spdk_nvmf_request *spdk_nvmf_request_get_req_to_abort(struct spdk_nvmf_request *req) 4948 { 4949 return req->req_to_abort; 4950 } 4951