1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 6 * Copyright (c) 2021, 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * * Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * * Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * * Neither the name of Intel Corporation nor the names of its 19 * contributors may be used to endorse or promote products derived 20 * from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #include "spdk/stdinc.h" 36 37 #include "bdev_nvme.h" 38 39 #include "spdk/accel_engine.h" 40 #include "spdk/config.h" 41 #include "spdk/endian.h" 42 #include "spdk/bdev.h" 43 #include "spdk/json.h" 44 #include "spdk/likely.h" 45 #include "spdk/nvme.h" 46 #include "spdk/nvme_ocssd.h" 47 #include "spdk/nvme_zns.h" 48 #include "spdk/opal.h" 49 #include "spdk/thread.h" 50 #include "spdk/string.h" 51 #include "spdk/util.h" 52 53 #include "spdk/bdev_module.h" 54 #include "spdk/log.h" 55 56 #include "spdk_internal/usdt.h" 57 58 #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true 59 #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS (10000) 60 61 static int bdev_nvme_config_json(struct spdk_json_write_ctx *w); 62 63 struct nvme_bdev_io { 64 /** array of iovecs to transfer. */ 65 struct iovec *iovs; 66 67 /** Number of iovecs in iovs array. */ 68 int iovcnt; 69 70 /** Current iovec position. */ 71 int iovpos; 72 73 /** Offset in current iovec. */ 74 uint32_t iov_offset; 75 76 /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path 77 * being reset in a reset I/O. 78 */ 79 struct nvme_io_path *io_path; 80 81 /** array of iovecs to transfer. */ 82 struct iovec *fused_iovs; 83 84 /** Number of iovecs in iovs array. */ 85 int fused_iovcnt; 86 87 /** Current iovec position. */ 88 int fused_iovpos; 89 90 /** Offset in current iovec. */ 91 uint32_t fused_iov_offset; 92 93 /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */ 94 struct spdk_nvme_cpl cpl; 95 96 /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */ 97 struct spdk_nvme_ns_cmd_ext_io_opts ext_opts; 98 99 /** Originating thread */ 100 struct spdk_thread *orig_thread; 101 102 /** Keeps track if first of fused commands was submitted */ 103 bool first_fused_submitted; 104 105 /** Keeps track if first of fused commands was completed */ 106 bool first_fused_completed; 107 108 /** Temporary pointer to zone report buffer */ 109 struct spdk_nvme_zns_zone_report *zone_report_buf; 110 111 /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */ 112 uint64_t handled_zones; 113 114 /** Expiration value in ticks to retry the current I/O. */ 115 uint64_t retry_ticks; 116 117 /* How many times the current I/O was retried. */ 118 int32_t retry_count; 119 }; 120 121 struct nvme_probe_skip_entry { 122 struct spdk_nvme_transport_id trid; 123 TAILQ_ENTRY(nvme_probe_skip_entry) tailq; 124 }; 125 /* All the controllers deleted by users via RPC are skipped by hotplug monitor */ 126 static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER( 127 g_skipped_nvme_ctrlrs); 128 129 static struct spdk_bdev_nvme_opts g_opts = { 130 .action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE, 131 .timeout_us = 0, 132 .timeout_admin_us = 0, 133 .keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS, 134 .transport_retry_count = 4, 135 .arbitration_burst = 0, 136 .low_priority_weight = 0, 137 .medium_priority_weight = 0, 138 .high_priority_weight = 0, 139 .nvme_adminq_poll_period_us = 10000ULL, 140 .nvme_ioq_poll_period_us = 0, 141 .io_queue_requests = 0, 142 .delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT, 143 .bdev_retry_count = 3, 144 .transport_ack_timeout = 0, 145 .ctrlr_loss_timeout_sec = 0, 146 .reconnect_delay_sec = 0, 147 .fast_io_fail_timeout_sec = 0, 148 }; 149 150 #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL 151 #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL 152 153 static int g_hot_insert_nvme_controller_index = 0; 154 static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT; 155 static bool g_nvme_hotplug_enabled = false; 156 static struct spdk_thread *g_bdev_nvme_init_thread; 157 static struct spdk_poller *g_hotplug_poller; 158 static struct spdk_poller *g_hotplug_probe_poller; 159 static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx; 160 161 static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 162 struct nvme_async_probe_ctx *ctx); 163 static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 164 struct nvme_async_probe_ctx *ctx); 165 static int bdev_nvme_library_init(void); 166 static void bdev_nvme_library_fini(void); 167 static void bdev_nvme_submit_request(struct spdk_io_channel *ch, 168 struct spdk_bdev_io *bdev_io); 169 static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 170 void *md, uint64_t lba_count, uint64_t lba, 171 uint32_t flags, struct spdk_bdev_ext_io_opts *ext_opts); 172 static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 173 void *md, uint64_t lba_count, uint64_t lba); 174 static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 175 void *md, uint64_t lba_count, uint64_t lba, 176 uint32_t flags, struct spdk_bdev_ext_io_opts *ext_opts); 177 static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 178 void *md, uint64_t lba_count, 179 uint64_t zslba, uint32_t flags); 180 static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 181 void *md, uint64_t lba_count, uint64_t lba, 182 uint32_t flags); 183 static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, 184 struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov, 185 int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba, 186 uint32_t flags); 187 static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, 188 uint32_t num_zones, struct spdk_bdev_zone_info *info); 189 static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 190 enum spdk_bdev_zone_action action); 191 static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, 192 struct nvme_bdev_io *bio, 193 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes); 194 static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 195 void *buf, size_t nbytes); 196 static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 197 void *buf, size_t nbytes, void *md_buf, size_t md_len); 198 static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, 199 struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort); 200 static void bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio); 201 static int bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr); 202 static int bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove); 203 static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr); 204 static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr); 205 206 static int 207 nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2) 208 { 209 return ns1->id < ns2->id ? -1 : ns1->id > ns2->id; 210 } 211 212 RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp); 213 214 struct spdk_nvme_qpair * 215 bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch) 216 { 217 struct nvme_ctrlr_channel *ctrlr_ch; 218 219 assert(ctrlr_io_ch != NULL); 220 221 ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch); 222 223 return ctrlr_ch->qpair->qpair; 224 } 225 226 static int 227 bdev_nvme_get_ctx_size(void) 228 { 229 return sizeof(struct nvme_bdev_io); 230 } 231 232 static struct spdk_bdev_module nvme_if = { 233 .name = "nvme", 234 .async_fini = true, 235 .module_init = bdev_nvme_library_init, 236 .module_fini = bdev_nvme_library_fini, 237 .config_json = bdev_nvme_config_json, 238 .get_ctx_size = bdev_nvme_get_ctx_size, 239 240 }; 241 SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if) 242 243 struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs); 244 pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER; 245 bool g_bdev_nvme_module_finish; 246 247 struct nvme_bdev_ctrlr * 248 nvme_bdev_ctrlr_get_by_name(const char *name) 249 { 250 struct nvme_bdev_ctrlr *nbdev_ctrlr; 251 252 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 253 if (strcmp(name, nbdev_ctrlr->name) == 0) { 254 break; 255 } 256 } 257 258 return nbdev_ctrlr; 259 } 260 261 static struct nvme_ctrlr * 262 nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr, 263 const struct spdk_nvme_transport_id *trid) 264 { 265 struct nvme_ctrlr *nvme_ctrlr; 266 267 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 268 if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0) { 269 break; 270 } 271 } 272 273 return nvme_ctrlr; 274 } 275 276 static struct nvme_bdev * 277 nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid) 278 { 279 struct nvme_bdev *bdev; 280 281 pthread_mutex_lock(&g_bdev_nvme_mutex); 282 TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) { 283 if (bdev->nsid == nsid) { 284 break; 285 } 286 } 287 pthread_mutex_unlock(&g_bdev_nvme_mutex); 288 289 return bdev; 290 } 291 292 struct nvme_ns * 293 nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid) 294 { 295 struct nvme_ns ns; 296 297 assert(nsid > 0); 298 299 ns.id = nsid; 300 return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns); 301 } 302 303 struct nvme_ns * 304 nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr) 305 { 306 return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces); 307 } 308 309 struct nvme_ns * 310 nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns) 311 { 312 if (ns == NULL) { 313 return NULL; 314 } 315 316 return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 317 } 318 319 static struct nvme_ctrlr * 320 nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid) 321 { 322 struct nvme_bdev_ctrlr *nbdev_ctrlr; 323 struct nvme_ctrlr *nvme_ctrlr = NULL; 324 325 pthread_mutex_lock(&g_bdev_nvme_mutex); 326 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 327 nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid); 328 if (nvme_ctrlr != NULL) { 329 break; 330 } 331 } 332 pthread_mutex_unlock(&g_bdev_nvme_mutex); 333 334 return nvme_ctrlr; 335 } 336 337 struct nvme_ctrlr * 338 nvme_ctrlr_get_by_name(const char *name) 339 { 340 struct nvme_bdev_ctrlr *nbdev_ctrlr; 341 struct nvme_ctrlr *nvme_ctrlr = NULL; 342 343 if (name == NULL) { 344 return NULL; 345 } 346 347 pthread_mutex_lock(&g_bdev_nvme_mutex); 348 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 349 if (nbdev_ctrlr != NULL) { 350 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 351 } 352 pthread_mutex_unlock(&g_bdev_nvme_mutex); 353 354 return nvme_ctrlr; 355 } 356 357 void 358 nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx) 359 { 360 struct nvme_bdev_ctrlr *nbdev_ctrlr; 361 362 pthread_mutex_lock(&g_bdev_nvme_mutex); 363 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 364 fn(nbdev_ctrlr, ctx); 365 } 366 pthread_mutex_unlock(&g_bdev_nvme_mutex); 367 } 368 369 void 370 nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w) 371 { 372 const char *trtype_str; 373 const char *adrfam_str; 374 375 trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype); 376 if (trtype_str) { 377 spdk_json_write_named_string(w, "trtype", trtype_str); 378 } 379 380 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 381 if (adrfam_str) { 382 spdk_json_write_named_string(w, "adrfam", adrfam_str); 383 } 384 385 if (trid->traddr[0] != '\0') { 386 spdk_json_write_named_string(w, "traddr", trid->traddr); 387 } 388 389 if (trid->trsvcid[0] != '\0') { 390 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 391 } 392 393 if (trid->subnqn[0] != '\0') { 394 spdk_json_write_named_string(w, "subnqn", trid->subnqn); 395 } 396 } 397 398 static void 399 nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr, 400 struct nvme_ctrlr *nvme_ctrlr) 401 { 402 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name); 403 pthread_mutex_lock(&g_bdev_nvme_mutex); 404 405 TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 406 if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) { 407 pthread_mutex_unlock(&g_bdev_nvme_mutex); 408 409 return; 410 } 411 TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 412 413 pthread_mutex_unlock(&g_bdev_nvme_mutex); 414 415 assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs)); 416 417 free(nbdev_ctrlr->name); 418 free(nbdev_ctrlr); 419 } 420 421 static void 422 _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 423 { 424 struct nvme_path_id *path_id, *tmp_path; 425 struct nvme_ns *ns, *tmp_ns; 426 427 free(nvme_ctrlr->copied_ana_desc); 428 spdk_free(nvme_ctrlr->ana_log_page); 429 430 if (nvme_ctrlr->opal_dev) { 431 spdk_opal_dev_destruct(nvme_ctrlr->opal_dev); 432 nvme_ctrlr->opal_dev = NULL; 433 } 434 435 if (nvme_ctrlr->nbdev_ctrlr) { 436 nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr); 437 } 438 439 RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) { 440 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 441 free(ns); 442 } 443 444 TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) { 445 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 446 free(path_id); 447 } 448 449 pthread_mutex_destroy(&nvme_ctrlr->mutex); 450 451 free(nvme_ctrlr); 452 453 pthread_mutex_lock(&g_bdev_nvme_mutex); 454 if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 455 pthread_mutex_unlock(&g_bdev_nvme_mutex); 456 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 457 spdk_bdev_module_fini_done(); 458 return; 459 } 460 pthread_mutex_unlock(&g_bdev_nvme_mutex); 461 } 462 463 static int 464 nvme_detach_poller(void *arg) 465 { 466 struct nvme_ctrlr *nvme_ctrlr = arg; 467 int rc; 468 469 rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx); 470 if (rc != -EAGAIN) { 471 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 472 _nvme_ctrlr_delete(nvme_ctrlr); 473 } 474 475 return SPDK_POLLER_BUSY; 476 } 477 478 static void 479 nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 480 { 481 int rc; 482 483 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 484 485 /* First, unregister the adminq poller, as the driver will poll adminq if necessary */ 486 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 487 488 /* If we got here, the reset/detach poller cannot be active */ 489 assert(nvme_ctrlr->reset_detach_poller == NULL); 490 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller, 491 nvme_ctrlr, 1000); 492 if (nvme_ctrlr->reset_detach_poller == NULL) { 493 SPDK_ERRLOG("Failed to register detach poller\n"); 494 goto error; 495 } 496 497 rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx); 498 if (rc != 0) { 499 SPDK_ERRLOG("Failed to detach the NVMe controller\n"); 500 goto error; 501 } 502 503 return; 504 error: 505 /* We don't have a good way to handle errors here, so just do what we can and delete the 506 * controller without detaching the underlying NVMe device. 507 */ 508 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 509 _nvme_ctrlr_delete(nvme_ctrlr); 510 } 511 512 static void 513 nvme_ctrlr_unregister_cb(void *io_device) 514 { 515 struct nvme_ctrlr *nvme_ctrlr = io_device; 516 517 nvme_ctrlr_delete(nvme_ctrlr); 518 } 519 520 static void 521 nvme_ctrlr_unregister(void *ctx) 522 { 523 struct nvme_ctrlr *nvme_ctrlr = ctx; 524 525 spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb); 526 } 527 528 static bool 529 nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr) 530 { 531 if (!nvme_ctrlr->destruct) { 532 return false; 533 } 534 535 if (nvme_ctrlr->ref > 0) { 536 return false; 537 } 538 539 if (nvme_ctrlr->resetting) { 540 return false; 541 } 542 543 if (nvme_ctrlr->ana_log_page_updating) { 544 return false; 545 } 546 547 return true; 548 } 549 550 static void 551 nvme_ctrlr_release(struct nvme_ctrlr *nvme_ctrlr) 552 { 553 pthread_mutex_lock(&nvme_ctrlr->mutex); 554 SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref); 555 556 assert(nvme_ctrlr->ref > 0); 557 nvme_ctrlr->ref--; 558 559 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 560 pthread_mutex_unlock(&nvme_ctrlr->mutex); 561 return; 562 } 563 564 pthread_mutex_unlock(&nvme_ctrlr->mutex); 565 566 spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr); 567 } 568 569 static struct nvme_io_path * 570 _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 571 { 572 struct nvme_io_path *io_path; 573 574 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 575 if (io_path->nvme_ns == nvme_ns) { 576 break; 577 } 578 } 579 580 return io_path; 581 } 582 583 static int 584 _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 585 { 586 struct nvme_io_path *io_path; 587 struct spdk_io_channel *ch; 588 struct nvme_ctrlr_channel *ctrlr_ch; 589 struct nvme_qpair *nvme_qpair; 590 591 io_path = calloc(1, sizeof(*io_path)); 592 if (io_path == NULL) { 593 SPDK_ERRLOG("Failed to alloc io_path.\n"); 594 return -ENOMEM; 595 } 596 597 io_path->nvme_ns = nvme_ns; 598 599 ch = spdk_get_io_channel(nvme_ns->ctrlr); 600 if (ch == NULL) { 601 free(io_path); 602 SPDK_ERRLOG("Failed to alloc io_channel.\n"); 603 return -ENOMEM; 604 } 605 606 ctrlr_ch = spdk_io_channel_get_ctx(ch); 607 608 nvme_qpair = ctrlr_ch->qpair; 609 assert(nvme_qpair != NULL); 610 611 io_path->qpair = nvme_qpair; 612 TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq); 613 614 io_path->nbdev_ch = nbdev_ch; 615 STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq); 616 617 nbdev_ch->current_io_path = NULL; 618 619 return 0; 620 } 621 622 static void 623 _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path) 624 { 625 struct spdk_io_channel *ch; 626 struct nvme_qpair *nvme_qpair; 627 struct nvme_ctrlr_channel *ctrlr_ch; 628 629 nbdev_ch->current_io_path = NULL; 630 631 STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq); 632 633 nvme_qpair = io_path->qpair; 634 assert(nvme_qpair != NULL); 635 636 TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq); 637 638 ctrlr_ch = nvme_qpair->ctrlr_ch; 639 assert(ctrlr_ch != NULL); 640 641 ch = spdk_io_channel_from_ctx(ctrlr_ch); 642 spdk_put_io_channel(ch); 643 644 free(io_path); 645 } 646 647 static void 648 _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch) 649 { 650 struct nvme_io_path *io_path, *tmp_io_path; 651 652 STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) { 653 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 654 } 655 } 656 657 static int 658 bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf) 659 { 660 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 661 struct nvme_bdev *nbdev = io_device; 662 struct nvme_ns *nvme_ns; 663 int rc; 664 665 STAILQ_INIT(&nbdev_ch->io_path_list); 666 TAILQ_INIT(&nbdev_ch->retry_io_list); 667 668 pthread_mutex_lock(&nbdev->mutex); 669 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 670 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 671 if (rc != 0) { 672 pthread_mutex_unlock(&nbdev->mutex); 673 674 _bdev_nvme_delete_io_paths(nbdev_ch); 675 return rc; 676 } 677 } 678 pthread_mutex_unlock(&nbdev->mutex); 679 680 return 0; 681 } 682 683 static void 684 bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch) 685 { 686 struct spdk_bdev_io *bdev_io, *tmp_io; 687 688 TAILQ_FOREACH_SAFE(bdev_io, &nbdev_ch->retry_io_list, module_link, tmp_io) { 689 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io, module_link); 690 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_ABORTED); 691 } 692 693 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 694 } 695 696 static void 697 bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf) 698 { 699 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 700 701 bdev_nvme_abort_retry_ios(nbdev_ch); 702 _bdev_nvme_delete_io_paths(nbdev_ch); 703 } 704 705 static inline bool 706 bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type) 707 { 708 switch (io_type) { 709 case SPDK_BDEV_IO_TYPE_RESET: 710 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 711 case SPDK_BDEV_IO_TYPE_ABORT: 712 return true; 713 default: 714 break; 715 } 716 717 return false; 718 } 719 720 static inline bool 721 nvme_ns_is_accessible(struct nvme_ns *nvme_ns) 722 { 723 if (spdk_unlikely(nvme_ns->ana_state_updating)) { 724 return false; 725 } 726 727 switch (nvme_ns->ana_state) { 728 case SPDK_NVME_ANA_OPTIMIZED_STATE: 729 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 730 return true; 731 default: 732 break; 733 } 734 735 return false; 736 } 737 738 static inline bool 739 nvme_io_path_is_connected(struct nvme_io_path *io_path) 740 { 741 if (spdk_unlikely(io_path->qpair->qpair == NULL)) { 742 return false; 743 } 744 745 if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(io_path->qpair->qpair) != 746 SPDK_NVME_QPAIR_FAILURE_NONE)) { 747 return false; 748 } 749 750 if (spdk_unlikely(io_path->qpair->ctrlr_ch->reset_iter != NULL)) { 751 return false; 752 } 753 754 if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(io_path->qpair->ctrlr->ctrlr) != 755 SPDK_NVME_QPAIR_FAILURE_NONE) { 756 return false; 757 } 758 759 return true; 760 } 761 762 static inline bool 763 nvme_io_path_is_available(struct nvme_io_path *io_path) 764 { 765 if (spdk_unlikely(!nvme_io_path_is_connected(io_path))) { 766 return false; 767 } 768 769 if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) { 770 return false; 771 } 772 773 return true; 774 } 775 776 static inline bool 777 nvme_io_path_is_failed(struct nvme_io_path *io_path) 778 { 779 struct nvme_ctrlr *nvme_ctrlr; 780 781 nvme_ctrlr = io_path->qpair->ctrlr; 782 783 if (nvme_ctrlr->destruct) { 784 return true; 785 } 786 787 if (nvme_ctrlr->fast_io_fail_timedout) { 788 return true; 789 } 790 791 if (nvme_ctrlr->resetting) { 792 if (nvme_ctrlr->opts.reconnect_delay_sec != 0) { 793 return false; 794 } else { 795 return true; 796 } 797 } 798 799 if (nvme_ctrlr->reconnect_is_delayed) { 800 return false; 801 } 802 803 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 804 return true; 805 } else { 806 return false; 807 } 808 } 809 810 static bool 811 nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr) 812 { 813 if (nvme_ctrlr->destruct) { 814 return false; 815 } 816 817 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 818 return false; 819 } 820 821 if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) { 822 return false; 823 } 824 825 return true; 826 } 827 828 static inline struct nvme_io_path * 829 bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 830 { 831 struct nvme_io_path *io_path, *non_optimized = NULL; 832 833 if (spdk_likely(nbdev_ch->current_io_path != NULL)) { 834 return nbdev_ch->current_io_path; 835 } 836 837 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 838 if (spdk_unlikely(!nvme_io_path_is_connected(io_path))) { 839 /* The device is currently resetting. */ 840 continue; 841 } 842 843 if (spdk_unlikely(io_path->nvme_ns->ana_state_updating)) { 844 continue; 845 } 846 847 switch (io_path->nvme_ns->ana_state) { 848 case SPDK_NVME_ANA_OPTIMIZED_STATE: 849 nbdev_ch->current_io_path = io_path; 850 return io_path; 851 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 852 if (non_optimized == NULL) { 853 non_optimized = io_path; 854 } 855 break; 856 default: 857 break; 858 } 859 } 860 861 return non_optimized; 862 } 863 864 /* Return true if there is any io_path whose qpair is active or ctrlr is not failed, 865 * or false otherwise. 866 * 867 * If any io_path has an active qpair but find_io_path() returned NULL, its namespace 868 * is likely to be non-accessible now but may become accessible. 869 * 870 * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr 871 * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed 872 * when starting to reset it but it is set to failed when the reset failed. Hence, if 873 * a ctrlr is unfailed, it is likely that it works fine or is resetting. 874 */ 875 static bool 876 any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch) 877 { 878 struct nvme_io_path *io_path; 879 880 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 881 if (io_path->nvme_ns->ana_transition_timedout) { 882 continue; 883 } 884 885 if (nvme_io_path_is_connected(io_path) || 886 !nvme_io_path_is_failed(io_path)) { 887 return true; 888 } 889 } 890 891 return false; 892 } 893 894 static bool 895 any_ctrlr_may_become_available(struct nvme_bdev_channel *nbdev_ch) 896 { 897 struct nvme_io_path *io_path; 898 899 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 900 if (!nvme_io_path_is_failed(io_path)) { 901 return true; 902 } 903 } 904 905 return false; 906 } 907 908 static int 909 bdev_nvme_retry_ios(void *arg) 910 { 911 struct nvme_bdev_channel *nbdev_ch = arg; 912 struct spdk_io_channel *ch = spdk_io_channel_from_ctx(nbdev_ch); 913 struct spdk_bdev_io *bdev_io, *tmp_bdev_io; 914 struct nvme_bdev_io *bio; 915 uint64_t now, delay_us; 916 917 now = spdk_get_ticks(); 918 919 TAILQ_FOREACH_SAFE(bdev_io, &nbdev_ch->retry_io_list, module_link, tmp_bdev_io) { 920 bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 921 if (bio->retry_ticks > now) { 922 break; 923 } 924 925 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io, module_link); 926 927 bdev_nvme_submit_request(ch, bdev_io); 928 } 929 930 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 931 932 bdev_io = TAILQ_FIRST(&nbdev_ch->retry_io_list); 933 if (bdev_io != NULL) { 934 bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 935 936 delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz(); 937 938 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 939 delay_us); 940 } 941 942 return SPDK_POLLER_BUSY; 943 } 944 945 static void 946 bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch, 947 struct nvme_bdev_io *bio, uint64_t delay_ms) 948 { 949 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 950 struct spdk_bdev_io *tmp_bdev_io; 951 struct nvme_bdev_io *tmp_bio; 952 953 bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL; 954 955 TAILQ_FOREACH_REVERSE(tmp_bdev_io, &nbdev_ch->retry_io_list, retry_io_head, module_link) { 956 tmp_bio = (struct nvme_bdev_io *)tmp_bdev_io->driver_ctx; 957 958 if (tmp_bio->retry_ticks <= bio->retry_ticks) { 959 TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bdev_io, bdev_io, 960 module_link); 961 return; 962 } 963 } 964 965 /* No earlier I/Os were found. This I/O must be the new head. */ 966 TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bdev_io, module_link); 967 968 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 969 970 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 971 delay_ms * 1000ULL); 972 } 973 974 static inline void 975 bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio, 976 const struct spdk_nvme_cpl *cpl) 977 { 978 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 979 struct nvme_bdev_channel *nbdev_ch; 980 struct nvme_ctrlr *nvme_ctrlr; 981 const struct spdk_nvme_ctrlr_data *cdata; 982 uint64_t delay_ms; 983 984 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 985 986 if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) { 987 goto complete; 988 } 989 990 if (cpl->status.dnr != 0 || (g_opts.bdev_retry_count != -1 && 991 bio->retry_count >= g_opts.bdev_retry_count)) { 992 goto complete; 993 } 994 995 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 996 997 assert(bio->io_path != NULL); 998 nvme_ctrlr = bio->io_path->qpair->ctrlr; 999 1000 if (spdk_nvme_cpl_is_path_error(cpl) || 1001 spdk_nvme_cpl_is_aborted_sq_deletion(cpl) || 1002 !nvme_io_path_is_available(bio->io_path) || 1003 !nvme_ctrlr_is_available(nvme_ctrlr)) { 1004 nbdev_ch->current_io_path = NULL; 1005 if (spdk_nvme_cpl_is_ana_error(cpl)) { 1006 if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) { 1007 bio->io_path->nvme_ns->ana_state_updating = true; 1008 } 1009 } 1010 delay_ms = 0; 1011 } else if (spdk_nvme_cpl_is_aborted_by_request(cpl)) { 1012 goto complete; 1013 } else { 1014 bio->retry_count++; 1015 1016 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 1017 1018 if (cpl->status.crd != 0) { 1019 delay_ms = cdata->crdt[cpl->status.crd] * 100; 1020 } else { 1021 delay_ms = 0; 1022 } 1023 } 1024 1025 if (any_io_path_may_become_available(nbdev_ch)) { 1026 bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms); 1027 return; 1028 } 1029 1030 complete: 1031 bio->retry_count = 0; 1032 spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc); 1033 } 1034 1035 static inline void 1036 bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc) 1037 { 1038 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1039 struct nvme_bdev_channel *nbdev_ch; 1040 enum spdk_bdev_io_status io_status; 1041 1042 switch (rc) { 1043 case 0: 1044 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1045 break; 1046 case -ENOMEM: 1047 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1048 break; 1049 case -ENXIO: 1050 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1051 1052 nbdev_ch->current_io_path = NULL; 1053 1054 if (any_io_path_may_become_available(nbdev_ch)) { 1055 bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL); 1056 return; 1057 } 1058 1059 /* fallthrough */ 1060 default: 1061 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1062 break; 1063 } 1064 1065 bio->retry_count = 0; 1066 spdk_bdev_io_complete(bdev_io, io_status); 1067 } 1068 1069 static inline void 1070 bdev_nvme_admin_passthru_complete(struct nvme_bdev_io *bio, int rc) 1071 { 1072 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1073 struct nvme_bdev_channel *nbdev_ch; 1074 enum spdk_bdev_io_status io_status; 1075 1076 switch (rc) { 1077 case 0: 1078 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1079 break; 1080 case -ENOMEM: 1081 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1082 break; 1083 case -ENXIO: 1084 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1085 1086 if (any_ctrlr_may_become_available(nbdev_ch)) { 1087 bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL); 1088 return; 1089 } 1090 1091 /* fallthrough */ 1092 default: 1093 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1094 break; 1095 } 1096 1097 bio->retry_count = 0; 1098 spdk_bdev_io_complete(bdev_io, io_status); 1099 } 1100 1101 static void 1102 _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair) 1103 { 1104 struct nvme_io_path *io_path; 1105 1106 TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) { 1107 io_path->nbdev_ch->current_io_path = NULL; 1108 } 1109 } 1110 1111 static void 1112 bdev_nvme_clear_io_path_cache(struct spdk_io_channel_iter *i) 1113 { 1114 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 1115 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 1116 1117 assert(ctrlr_ch->qpair != NULL); 1118 1119 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 1120 1121 spdk_for_each_channel_continue(i, 0); 1122 } 1123 1124 static void 1125 bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr, 1126 spdk_channel_for_each_cpl cpl) 1127 { 1128 spdk_for_each_channel(nvme_ctrlr, 1129 bdev_nvme_clear_io_path_cache, 1130 NULL, 1131 cpl); 1132 } 1133 1134 static struct nvme_qpair * 1135 nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair) 1136 { 1137 struct nvme_qpair *nvme_qpair; 1138 1139 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1140 if (nvme_qpair->qpair == qpair) { 1141 break; 1142 } 1143 } 1144 1145 return nvme_qpair; 1146 } 1147 1148 static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair); 1149 1150 static void 1151 bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx) 1152 { 1153 struct nvme_poll_group *group = poll_group_ctx; 1154 struct nvme_qpair *nvme_qpair; 1155 struct nvme_ctrlr_channel *ctrlr_ch; 1156 1157 nvme_qpair = nvme_poll_group_get_qpair(group, qpair); 1158 if (nvme_qpair == NULL) { 1159 return; 1160 } 1161 1162 if (nvme_qpair->qpair != NULL) { 1163 spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair); 1164 nvme_qpair->qpair = NULL; 1165 } 1166 1167 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1168 1169 ctrlr_ch = nvme_qpair->ctrlr_ch; 1170 1171 if (ctrlr_ch != NULL) { 1172 if (ctrlr_ch->reset_iter != NULL) { 1173 /* If we are already in a full reset sequence, we do not have 1174 * to restart it. Just move to the next ctrlr_channel. 1175 */ 1176 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed in a reset ctrlr sequence.\n", 1177 qpair); 1178 spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 1179 ctrlr_ch->reset_iter = NULL; 1180 } else { 1181 /* qpair was disconnected unexpectedly. Reset controller for recovery. */ 1182 SPDK_NOTICELOG("qpair %p was disconnected and freed. reset controller.\n", qpair); 1183 bdev_nvme_failover(nvme_qpair->ctrlr, false); 1184 } 1185 } else { 1186 /* In this case, ctrlr_channel is already deleted. */ 1187 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed. delete nvme_qpair.\n", qpair); 1188 nvme_qpair_delete(nvme_qpair); 1189 } 1190 } 1191 1192 static void 1193 bdev_nvme_check_io_qpairs(struct nvme_poll_group *group) 1194 { 1195 struct nvme_qpair *nvme_qpair; 1196 1197 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1198 if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) { 1199 continue; 1200 } 1201 1202 if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 1203 SPDK_NVME_QPAIR_FAILURE_NONE) { 1204 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1205 } 1206 } 1207 } 1208 1209 static int 1210 bdev_nvme_poll(void *arg) 1211 { 1212 struct nvme_poll_group *group = arg; 1213 int64_t num_completions; 1214 1215 if (group->collect_spin_stat && group->start_ticks == 0) { 1216 group->start_ticks = spdk_get_ticks(); 1217 } 1218 1219 num_completions = spdk_nvme_poll_group_process_completions(group->group, 0, 1220 bdev_nvme_disconnected_qpair_cb); 1221 if (group->collect_spin_stat) { 1222 if (num_completions > 0) { 1223 if (group->end_ticks != 0) { 1224 group->spin_ticks += (group->end_ticks - group->start_ticks); 1225 group->end_ticks = 0; 1226 } 1227 group->start_ticks = 0; 1228 } else { 1229 group->end_ticks = spdk_get_ticks(); 1230 } 1231 } 1232 1233 if (spdk_unlikely(num_completions < 0)) { 1234 bdev_nvme_check_io_qpairs(group); 1235 } 1236 1237 return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 1238 } 1239 1240 static int 1241 bdev_nvme_poll_adminq(void *arg) 1242 { 1243 int32_t rc; 1244 struct nvme_ctrlr *nvme_ctrlr = arg; 1245 nvme_ctrlr_disconnected_cb disconnected_cb; 1246 1247 assert(nvme_ctrlr != NULL); 1248 1249 rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr); 1250 if (rc < 0) { 1251 disconnected_cb = nvme_ctrlr->disconnected_cb; 1252 nvme_ctrlr->disconnected_cb = NULL; 1253 1254 if (rc == -ENXIO && disconnected_cb != NULL) { 1255 disconnected_cb(nvme_ctrlr); 1256 } else { 1257 bdev_nvme_failover(nvme_ctrlr, false); 1258 } 1259 } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) != 1260 SPDK_NVME_QPAIR_FAILURE_NONE) { 1261 bdev_nvme_clear_io_path_caches(nvme_ctrlr, NULL); 1262 } 1263 1264 return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY; 1265 } 1266 1267 static void 1268 _bdev_nvme_unregister_dev_cb(void *io_device) 1269 { 1270 struct nvme_bdev *nvme_disk = io_device; 1271 1272 free(nvme_disk->disk.name); 1273 free(nvme_disk); 1274 } 1275 1276 static int 1277 bdev_nvme_destruct(void *ctx) 1278 { 1279 struct nvme_bdev *nvme_disk = ctx; 1280 struct nvme_ns *nvme_ns, *tmp_nvme_ns; 1281 1282 SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid); 1283 1284 TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) { 1285 pthread_mutex_lock(&nvme_ns->ctrlr->mutex); 1286 1287 nvme_ns->bdev = NULL; 1288 1289 assert(nvme_ns->id > 0); 1290 1291 if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) { 1292 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1293 1294 nvme_ctrlr_release(nvme_ns->ctrlr); 1295 free(nvme_ns); 1296 } else { 1297 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1298 } 1299 } 1300 1301 pthread_mutex_lock(&g_bdev_nvme_mutex); 1302 TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq); 1303 pthread_mutex_unlock(&g_bdev_nvme_mutex); 1304 1305 spdk_io_device_unregister(nvme_disk, _bdev_nvme_unregister_dev_cb); 1306 1307 return 0; 1308 } 1309 1310 static int 1311 bdev_nvme_flush(struct nvme_bdev_io *bio, uint64_t offset, uint64_t nbytes) 1312 { 1313 bdev_nvme_io_complete(bio, 0); 1314 1315 return 0; 1316 } 1317 1318 static int 1319 bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair) 1320 { 1321 struct nvme_ctrlr *nvme_ctrlr; 1322 struct spdk_nvme_io_qpair_opts opts; 1323 struct spdk_nvme_qpair *qpair; 1324 int rc; 1325 1326 nvme_ctrlr = nvme_qpair->ctrlr; 1327 1328 spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1329 opts.delay_cmd_submit = g_opts.delay_cmd_submit; 1330 opts.create_only = true; 1331 opts.async_mode = true; 1332 opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests); 1333 g_opts.io_queue_requests = opts.io_queue_requests; 1334 1335 qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1336 if (qpair == NULL) { 1337 return -1; 1338 } 1339 1340 SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name, 1341 spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread)); 1342 1343 assert(nvme_qpair->group != NULL); 1344 1345 rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair); 1346 if (rc != 0) { 1347 SPDK_ERRLOG("Unable to begin polling on NVMe Channel.\n"); 1348 goto err; 1349 } 1350 1351 rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair); 1352 if (rc != 0) { 1353 SPDK_ERRLOG("Unable to connect I/O qpair.\n"); 1354 goto err; 1355 } 1356 1357 nvme_qpair->qpair = qpair; 1358 1359 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1360 1361 return 0; 1362 1363 err: 1364 spdk_nvme_ctrlr_free_io_qpair(qpair); 1365 1366 return rc; 1367 } 1368 1369 static void 1370 bdev_nvme_complete_pending_resets(struct spdk_io_channel_iter *i) 1371 { 1372 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 1373 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 1374 enum spdk_bdev_io_status status = SPDK_BDEV_IO_STATUS_SUCCESS; 1375 struct spdk_bdev_io *bdev_io; 1376 1377 if (spdk_io_channel_iter_get_ctx(i) != NULL) { 1378 status = SPDK_BDEV_IO_STATUS_FAILED; 1379 } 1380 1381 while (!TAILQ_EMPTY(&ctrlr_ch->pending_resets)) { 1382 bdev_io = TAILQ_FIRST(&ctrlr_ch->pending_resets); 1383 TAILQ_REMOVE(&ctrlr_ch->pending_resets, bdev_io, module_link); 1384 spdk_bdev_io_complete(bdev_io, status); 1385 } 1386 1387 spdk_for_each_channel_continue(i, 0); 1388 } 1389 1390 static void 1391 bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove) 1392 { 1393 struct nvme_path_id *path_id, *next_path; 1394 int rc __attribute__((unused)); 1395 1396 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 1397 assert(path_id); 1398 assert(path_id == nvme_ctrlr->active_path_id); 1399 next_path = TAILQ_NEXT(path_id, link); 1400 1401 path_id->is_failed = true; 1402 1403 if (next_path) { 1404 assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE); 1405 1406 SPDK_NOTICELOG("Start failover from %s:%s to %s:%s\n", path_id->trid.traddr, 1407 path_id->trid.trsvcid, next_path->trid.traddr, next_path->trid.trsvcid); 1408 1409 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 1410 nvme_ctrlr->active_path_id = next_path; 1411 rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid); 1412 assert(rc == 0); 1413 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 1414 if (!remove) { 1415 /** Shuffle the old trid to the end of the list and use the new one. 1416 * Allows for round robin through multiple connections. 1417 */ 1418 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link); 1419 } else { 1420 free(path_id); 1421 } 1422 } 1423 } 1424 1425 static bool 1426 bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr) 1427 { 1428 int32_t elapsed; 1429 1430 if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 || 1431 nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) { 1432 return false; 1433 } 1434 1435 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 1436 if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) { 1437 return true; 1438 } else { 1439 return false; 1440 } 1441 } 1442 1443 static bool 1444 bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr) 1445 { 1446 uint32_t elapsed; 1447 1448 if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) { 1449 return false; 1450 } 1451 1452 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 1453 if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) { 1454 return true; 1455 } else { 1456 return false; 1457 } 1458 } 1459 1460 enum bdev_nvme_op_after_reset { 1461 OP_NONE, 1462 OP_COMPLETE_PENDING_DESTRUCT, 1463 OP_DESTRUCT, 1464 OP_DELAYED_RECONNECT, 1465 }; 1466 1467 typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset; 1468 1469 static _bdev_nvme_op_after_reset 1470 bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success) 1471 { 1472 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 1473 /* Complete pending destruct after reset completes. */ 1474 return OP_COMPLETE_PENDING_DESTRUCT; 1475 } else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) { 1476 nvme_ctrlr->reset_start_tsc = 0; 1477 return OP_NONE; 1478 } else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 1479 return OP_DESTRUCT; 1480 } else { 1481 if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) { 1482 nvme_ctrlr->fast_io_fail_timedout = true; 1483 } 1484 bdev_nvme_failover_trid(nvme_ctrlr, false); 1485 return OP_DELAYED_RECONNECT; 1486 } 1487 } 1488 1489 static int _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, bool hotplug); 1490 static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 1491 1492 static int 1493 bdev_nvme_reconnect_delay_timer_expired(void *ctx) 1494 { 1495 struct nvme_ctrlr *nvme_ctrlr = ctx; 1496 1497 pthread_mutex_lock(&nvme_ctrlr->mutex); 1498 1499 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 1500 1501 assert(nvme_ctrlr->reconnect_is_delayed == true); 1502 nvme_ctrlr->reconnect_is_delayed = false; 1503 1504 if (nvme_ctrlr->destruct) { 1505 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1506 return SPDK_POLLER_BUSY; 1507 } 1508 1509 assert(nvme_ctrlr->resetting == false); 1510 nvme_ctrlr->resetting = true; 1511 1512 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1513 1514 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 1515 1516 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 1517 return SPDK_POLLER_BUSY; 1518 } 1519 1520 static void 1521 bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr) 1522 { 1523 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 1524 1525 assert(nvme_ctrlr->reconnect_is_delayed == false); 1526 nvme_ctrlr->reconnect_is_delayed = true; 1527 1528 assert(nvme_ctrlr->reconnect_delay_timer == NULL); 1529 nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired, 1530 nvme_ctrlr, 1531 nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC); 1532 } 1533 1534 static void 1535 _bdev_nvme_reset_complete(struct spdk_io_channel_iter *i, int status) 1536 { 1537 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1538 bool success = spdk_io_channel_iter_get_ctx(i) == NULL; 1539 struct nvme_path_id *path_id; 1540 bdev_nvme_reset_cb reset_cb_fn = nvme_ctrlr->reset_cb_fn; 1541 void *reset_cb_arg = nvme_ctrlr->reset_cb_arg; 1542 enum bdev_nvme_op_after_reset op_after_reset; 1543 1544 assert(nvme_ctrlr->thread == spdk_get_thread()); 1545 1546 nvme_ctrlr->reset_cb_fn = NULL; 1547 nvme_ctrlr->reset_cb_arg = NULL; 1548 1549 if (!success) { 1550 SPDK_ERRLOG("Resetting controller failed.\n"); 1551 } else { 1552 SPDK_NOTICELOG("Resetting controller successful.\n"); 1553 } 1554 1555 pthread_mutex_lock(&nvme_ctrlr->mutex); 1556 nvme_ctrlr->resetting = false; 1557 1558 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 1559 assert(path_id != NULL); 1560 assert(path_id == nvme_ctrlr->active_path_id); 1561 1562 path_id->is_failed = !success; 1563 1564 op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success); 1565 1566 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1567 1568 if (reset_cb_fn) { 1569 reset_cb_fn(reset_cb_arg, success); 1570 } 1571 1572 switch (op_after_reset) { 1573 case OP_COMPLETE_PENDING_DESTRUCT: 1574 nvme_ctrlr_unregister(nvme_ctrlr); 1575 break; 1576 case OP_DESTRUCT: 1577 _bdev_nvme_delete(nvme_ctrlr, false); 1578 break; 1579 case OP_DELAYED_RECONNECT: 1580 /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq. 1581 * Set callback here to start reconnect delay timer after ctrlr is really disconnected. 1582 */ 1583 assert(nvme_ctrlr->disconnected_cb == NULL); 1584 nvme_ctrlr->disconnected_cb = bdev_nvme_start_reconnect_delay_timer; 1585 1586 spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr); 1587 break; 1588 default: 1589 break; 1590 } 1591 } 1592 1593 static void 1594 bdev_nvme_reset_complete(struct nvme_ctrlr *nvme_ctrlr, bool success) 1595 { 1596 /* Make sure we clear any pending resets before returning. */ 1597 spdk_for_each_channel(nvme_ctrlr, 1598 bdev_nvme_complete_pending_resets, 1599 success ? NULL : (void *)0x1, 1600 _bdev_nvme_reset_complete); 1601 } 1602 1603 static void 1604 bdev_nvme_reset_create_qpairs_failed(struct spdk_io_channel_iter *i, int status) 1605 { 1606 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1607 1608 bdev_nvme_reset_complete(nvme_ctrlr, false); 1609 } 1610 1611 static void 1612 bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i) 1613 { 1614 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 1615 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch); 1616 struct nvme_qpair *nvme_qpair; 1617 1618 nvme_qpair = ctrlr_ch->qpair; 1619 assert(nvme_qpair != NULL); 1620 1621 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1622 1623 if (nvme_qpair->qpair != NULL) { 1624 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 1625 1626 /* The current full reset sequence will move to the next 1627 * ctrlr_channel after the qpair is actually disconnected. 1628 */ 1629 assert(ctrlr_ch->reset_iter == NULL); 1630 ctrlr_ch->reset_iter = i; 1631 } else { 1632 spdk_for_each_channel_continue(i, 0); 1633 } 1634 } 1635 1636 static void 1637 bdev_nvme_reset_create_qpairs_done(struct spdk_io_channel_iter *i, int status) 1638 { 1639 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1640 1641 if (status == 0) { 1642 bdev_nvme_reset_complete(nvme_ctrlr, true); 1643 } else { 1644 /* Delete the added qpairs and quiesce ctrlr to make the states clean. */ 1645 spdk_for_each_channel(nvme_ctrlr, 1646 bdev_nvme_reset_destroy_qpair, 1647 NULL, 1648 bdev_nvme_reset_create_qpairs_failed); 1649 } 1650 } 1651 1652 static void 1653 bdev_nvme_reset_create_qpair(struct spdk_io_channel_iter *i) 1654 { 1655 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 1656 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 1657 int rc; 1658 1659 rc = bdev_nvme_create_qpair(ctrlr_ch->qpair); 1660 1661 spdk_for_each_channel_continue(i, rc); 1662 } 1663 1664 static int 1665 bdev_nvme_reconnect_ctrlr_poll(void *arg) 1666 { 1667 struct nvme_ctrlr *nvme_ctrlr = arg; 1668 int rc = -ETIMEDOUT; 1669 1670 if (!bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 1671 rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr); 1672 if (rc == -EAGAIN) { 1673 return SPDK_POLLER_BUSY; 1674 } 1675 } 1676 1677 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 1678 if (rc == 0) { 1679 /* Recreate all of the I/O queue pairs */ 1680 spdk_for_each_channel(nvme_ctrlr, 1681 bdev_nvme_reset_create_qpair, 1682 NULL, 1683 bdev_nvme_reset_create_qpairs_done); 1684 } else { 1685 bdev_nvme_reset_complete(nvme_ctrlr, false); 1686 } 1687 return SPDK_POLLER_BUSY; 1688 } 1689 1690 static void 1691 bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 1692 { 1693 spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr); 1694 1695 assert(nvme_ctrlr->reset_detach_poller == NULL); 1696 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll, 1697 nvme_ctrlr, 0); 1698 } 1699 1700 static void 1701 bdev_nvme_reset_ctrlr(struct spdk_io_channel_iter *i, int status) 1702 { 1703 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1704 int rc __attribute__((unused)); 1705 1706 assert(status == 0); 1707 1708 /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq. 1709 * Set callback here to reconnect after ctrlr is really disconnected. 1710 */ 1711 assert(nvme_ctrlr->disconnected_cb == NULL); 1712 nvme_ctrlr->disconnected_cb = bdev_nvme_reconnect_ctrlr; 1713 1714 /* Disconnect fails if ctrlr is already resetting or removed. Both cases are 1715 * not possible. Reset is controlled and the callback to hot remove is called 1716 * when ctrlr is hot removed. 1717 */ 1718 rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr); 1719 assert(rc == 0); 1720 } 1721 1722 static void 1723 _bdev_nvme_reset(void *ctx) 1724 { 1725 struct nvme_ctrlr *nvme_ctrlr = ctx; 1726 1727 assert(nvme_ctrlr->resetting == true); 1728 assert(nvme_ctrlr->thread == spdk_get_thread()); 1729 1730 spdk_nvme_ctrlr_prepare_for_reset(nvme_ctrlr->ctrlr); 1731 1732 /* First, delete all NVMe I/O queue pairs. */ 1733 spdk_for_each_channel(nvme_ctrlr, 1734 bdev_nvme_reset_destroy_qpair, 1735 NULL, 1736 bdev_nvme_reset_ctrlr); 1737 } 1738 1739 static int 1740 bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr) 1741 { 1742 pthread_mutex_lock(&nvme_ctrlr->mutex); 1743 if (nvme_ctrlr->destruct) { 1744 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1745 return -ENXIO; 1746 } 1747 1748 if (nvme_ctrlr->resetting) { 1749 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1750 SPDK_NOTICELOG("Unable to perform reset, already in progress.\n"); 1751 return -EBUSY; 1752 } 1753 1754 if (nvme_ctrlr->reconnect_is_delayed) { 1755 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1756 SPDK_NOTICELOG("Reconnect is already scheduled.\n"); 1757 return -EBUSY; 1758 } 1759 1760 nvme_ctrlr->resetting = true; 1761 1762 assert(nvme_ctrlr->reset_start_tsc == 0); 1763 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 1764 1765 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1766 1767 spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr); 1768 return 0; 1769 } 1770 1771 int 1772 bdev_nvme_reset_rpc(struct nvme_ctrlr *nvme_ctrlr, bdev_nvme_reset_cb cb_fn, void *cb_arg) 1773 { 1774 int rc; 1775 1776 rc = bdev_nvme_reset(nvme_ctrlr); 1777 if (rc == 0) { 1778 nvme_ctrlr->reset_cb_fn = cb_fn; 1779 nvme_ctrlr->reset_cb_arg = cb_arg; 1780 } 1781 return rc; 1782 } 1783 1784 static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio); 1785 1786 static void 1787 bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio) 1788 { 1789 enum spdk_bdev_io_status io_status; 1790 1791 if (bio->cpl.cdw0 == 0) { 1792 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1793 } else { 1794 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1795 } 1796 1797 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(bio), io_status); 1798 } 1799 1800 static void 1801 _bdev_nvme_reset_io_continue(void *ctx) 1802 { 1803 struct nvme_bdev_io *bio = ctx; 1804 struct nvme_io_path *prev_io_path, *next_io_path; 1805 int rc; 1806 1807 prev_io_path = bio->io_path; 1808 bio->io_path = NULL; 1809 1810 if (bio->cpl.cdw0 != 0) { 1811 goto complete; 1812 } 1813 1814 next_io_path = STAILQ_NEXT(prev_io_path, stailq); 1815 if (next_io_path == NULL) { 1816 goto complete; 1817 } 1818 1819 rc = _bdev_nvme_reset_io(next_io_path, bio); 1820 if (rc == 0) { 1821 return; 1822 } 1823 1824 bio->cpl.cdw0 = 1; 1825 1826 complete: 1827 bdev_nvme_reset_io_complete(bio); 1828 } 1829 1830 static void 1831 bdev_nvme_reset_io_continue(void *cb_arg, bool success) 1832 { 1833 struct nvme_bdev_io *bio = cb_arg; 1834 1835 bio->cpl.cdw0 = !success; 1836 1837 spdk_thread_send_msg(bio->orig_thread, _bdev_nvme_reset_io_continue, bio); 1838 } 1839 1840 static int 1841 _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio) 1842 { 1843 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 1844 struct nvme_ctrlr_channel *ctrlr_ch; 1845 struct spdk_bdev_io *bdev_io; 1846 int rc; 1847 1848 rc = bdev_nvme_reset(nvme_ctrlr); 1849 if (rc == 0) { 1850 assert(bio->io_path == NULL); 1851 bio->io_path = io_path; 1852 1853 assert(nvme_ctrlr->reset_cb_fn == NULL); 1854 assert(nvme_ctrlr->reset_cb_arg == NULL); 1855 nvme_ctrlr->reset_cb_fn = bdev_nvme_reset_io_continue; 1856 nvme_ctrlr->reset_cb_arg = bio; 1857 } else if (rc == -EBUSY) { 1858 ctrlr_ch = io_path->qpair->ctrlr_ch; 1859 assert(ctrlr_ch != NULL); 1860 /* 1861 * Reset call is queued only if it is from the app framework. This is on purpose so that 1862 * we don't interfere with the app framework reset strategy. i.e. we are deferring to the 1863 * upper level. If they are in the middle of a reset, we won't try to schedule another one. 1864 */ 1865 bdev_io = spdk_bdev_io_from_ctx(bio); 1866 TAILQ_INSERT_TAIL(&ctrlr_ch->pending_resets, bdev_io, module_link); 1867 } else { 1868 return rc; 1869 } 1870 1871 return 0; 1872 } 1873 1874 static void 1875 bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio) 1876 { 1877 struct nvme_io_path *io_path; 1878 int rc; 1879 1880 bio->cpl.cdw0 = 0; 1881 bio->orig_thread = spdk_get_thread(); 1882 1883 /* Reset only the first nvme_ctrlr in the nvme_bdev_ctrlr for now. 1884 * 1885 * TODO: Reset all nvme_ctrlrs in the nvme_bdev_ctrlr sequentially. 1886 * This will be done in the following patches. 1887 */ 1888 io_path = STAILQ_FIRST(&nbdev_ch->io_path_list); 1889 assert(io_path != NULL); 1890 1891 rc = _bdev_nvme_reset_io(io_path, bio); 1892 if (rc != 0) { 1893 bio->cpl.cdw0 = 1; 1894 bdev_nvme_reset_io_complete(bio); 1895 } 1896 } 1897 1898 static int 1899 bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove) 1900 { 1901 pthread_mutex_lock(&nvme_ctrlr->mutex); 1902 if (nvme_ctrlr->destruct) { 1903 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1904 /* Don't bother resetting if the controller is in the process of being destructed. */ 1905 return -ENXIO; 1906 } 1907 1908 if (nvme_ctrlr->resetting) { 1909 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1910 SPDK_NOTICELOG("Unable to perform reset, already in progress.\n"); 1911 return -EBUSY; 1912 } 1913 1914 bdev_nvme_failover_trid(nvme_ctrlr, remove); 1915 1916 if (nvme_ctrlr->reconnect_is_delayed) { 1917 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1918 SPDK_NOTICELOG("Reconnect is already scheduled.\n"); 1919 1920 /* We rely on the next reconnect for the failover. */ 1921 return 0; 1922 } 1923 1924 nvme_ctrlr->resetting = true; 1925 1926 assert(nvme_ctrlr->reset_start_tsc == 0); 1927 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 1928 1929 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1930 1931 spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr); 1932 return 0; 1933 } 1934 1935 static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, 1936 uint64_t num_blocks); 1937 1938 static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, 1939 uint64_t num_blocks); 1940 1941 static void 1942 bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 1943 bool success) 1944 { 1945 struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1946 struct spdk_bdev *bdev = bdev_io->bdev; 1947 int ret; 1948 1949 if (!success) { 1950 ret = -EINVAL; 1951 goto exit; 1952 } 1953 1954 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 1955 ret = -ENXIO; 1956 goto exit; 1957 } 1958 1959 ret = bdev_nvme_readv(bio, 1960 bdev_io->u.bdev.iovs, 1961 bdev_io->u.bdev.iovcnt, 1962 bdev_io->u.bdev.md_buf, 1963 bdev_io->u.bdev.num_blocks, 1964 bdev_io->u.bdev.offset_blocks, 1965 bdev->dif_check_flags, 1966 bdev_io->u.bdev.ext_opts); 1967 1968 exit: 1969 if (spdk_unlikely(ret != 0)) { 1970 bdev_nvme_io_complete(bio, ret); 1971 } 1972 } 1973 1974 static void 1975 bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 1976 { 1977 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 1978 struct spdk_bdev *bdev = bdev_io->bdev; 1979 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1980 struct nvme_bdev_io *nbdev_io_to_abort; 1981 int rc = 0; 1982 1983 nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch); 1984 if (spdk_unlikely(!nbdev_io->io_path)) { 1985 if (!bdev_nvme_io_type_is_admin(bdev_io->type)) { 1986 rc = -ENXIO; 1987 goto exit; 1988 } 1989 1990 /* Admin commands do not use the optimal I/O path. 1991 * Simply fall through even if it is not found. 1992 */ 1993 } 1994 1995 switch (bdev_io->type) { 1996 case SPDK_BDEV_IO_TYPE_READ: 1997 if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) { 1998 rc = bdev_nvme_readv(nbdev_io, 1999 bdev_io->u.bdev.iovs, 2000 bdev_io->u.bdev.iovcnt, 2001 bdev_io->u.bdev.md_buf, 2002 bdev_io->u.bdev.num_blocks, 2003 bdev_io->u.bdev.offset_blocks, 2004 bdev->dif_check_flags, 2005 bdev_io->u.bdev.ext_opts); 2006 } else { 2007 spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb, 2008 bdev_io->u.bdev.num_blocks * bdev->blocklen); 2009 rc = 0; 2010 } 2011 break; 2012 case SPDK_BDEV_IO_TYPE_WRITE: 2013 rc = bdev_nvme_writev(nbdev_io, 2014 bdev_io->u.bdev.iovs, 2015 bdev_io->u.bdev.iovcnt, 2016 bdev_io->u.bdev.md_buf, 2017 bdev_io->u.bdev.num_blocks, 2018 bdev_io->u.bdev.offset_blocks, 2019 bdev->dif_check_flags, 2020 bdev_io->u.bdev.ext_opts); 2021 break; 2022 case SPDK_BDEV_IO_TYPE_COMPARE: 2023 rc = bdev_nvme_comparev(nbdev_io, 2024 bdev_io->u.bdev.iovs, 2025 bdev_io->u.bdev.iovcnt, 2026 bdev_io->u.bdev.md_buf, 2027 bdev_io->u.bdev.num_blocks, 2028 bdev_io->u.bdev.offset_blocks, 2029 bdev->dif_check_flags); 2030 break; 2031 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 2032 rc = bdev_nvme_comparev_and_writev(nbdev_io, 2033 bdev_io->u.bdev.iovs, 2034 bdev_io->u.bdev.iovcnt, 2035 bdev_io->u.bdev.fused_iovs, 2036 bdev_io->u.bdev.fused_iovcnt, 2037 bdev_io->u.bdev.md_buf, 2038 bdev_io->u.bdev.num_blocks, 2039 bdev_io->u.bdev.offset_blocks, 2040 bdev->dif_check_flags); 2041 break; 2042 case SPDK_BDEV_IO_TYPE_UNMAP: 2043 rc = bdev_nvme_unmap(nbdev_io, 2044 bdev_io->u.bdev.offset_blocks, 2045 bdev_io->u.bdev.num_blocks); 2046 break; 2047 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 2048 rc = bdev_nvme_write_zeroes(nbdev_io, 2049 bdev_io->u.bdev.offset_blocks, 2050 bdev_io->u.bdev.num_blocks); 2051 break; 2052 case SPDK_BDEV_IO_TYPE_RESET: 2053 nbdev_io->io_path = NULL; 2054 bdev_nvme_reset_io(nbdev_ch, nbdev_io); 2055 break; 2056 case SPDK_BDEV_IO_TYPE_FLUSH: 2057 rc = bdev_nvme_flush(nbdev_io, 2058 bdev_io->u.bdev.offset_blocks, 2059 bdev_io->u.bdev.num_blocks); 2060 break; 2061 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 2062 rc = bdev_nvme_zone_appendv(nbdev_io, 2063 bdev_io->u.bdev.iovs, 2064 bdev_io->u.bdev.iovcnt, 2065 bdev_io->u.bdev.md_buf, 2066 bdev_io->u.bdev.num_blocks, 2067 bdev_io->u.bdev.offset_blocks, 2068 bdev->dif_check_flags); 2069 break; 2070 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 2071 rc = bdev_nvme_get_zone_info(nbdev_io, 2072 bdev_io->u.zone_mgmt.zone_id, 2073 bdev_io->u.zone_mgmt.num_zones, 2074 bdev_io->u.zone_mgmt.buf); 2075 break; 2076 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 2077 rc = bdev_nvme_zone_management(nbdev_io, 2078 bdev_io->u.zone_mgmt.zone_id, 2079 bdev_io->u.zone_mgmt.zone_action); 2080 break; 2081 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 2082 nbdev_io->io_path = NULL; 2083 bdev_nvme_admin_passthru(nbdev_ch, 2084 nbdev_io, 2085 &bdev_io->u.nvme_passthru.cmd, 2086 bdev_io->u.nvme_passthru.buf, 2087 bdev_io->u.nvme_passthru.nbytes); 2088 break; 2089 case SPDK_BDEV_IO_TYPE_NVME_IO: 2090 rc = bdev_nvme_io_passthru(nbdev_io, 2091 &bdev_io->u.nvme_passthru.cmd, 2092 bdev_io->u.nvme_passthru.buf, 2093 bdev_io->u.nvme_passthru.nbytes); 2094 break; 2095 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 2096 rc = bdev_nvme_io_passthru_md(nbdev_io, 2097 &bdev_io->u.nvme_passthru.cmd, 2098 bdev_io->u.nvme_passthru.buf, 2099 bdev_io->u.nvme_passthru.nbytes, 2100 bdev_io->u.nvme_passthru.md_buf, 2101 bdev_io->u.nvme_passthru.md_len); 2102 break; 2103 case SPDK_BDEV_IO_TYPE_ABORT: 2104 nbdev_io->io_path = NULL; 2105 nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx; 2106 bdev_nvme_abort(nbdev_ch, 2107 nbdev_io, 2108 nbdev_io_to_abort); 2109 break; 2110 default: 2111 rc = -EINVAL; 2112 break; 2113 } 2114 2115 exit: 2116 if (spdk_unlikely(rc != 0)) { 2117 bdev_nvme_io_complete(nbdev_io, rc); 2118 } 2119 } 2120 2121 static bool 2122 bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 2123 { 2124 struct nvme_bdev *nbdev = ctx; 2125 struct nvme_ns *nvme_ns; 2126 struct spdk_nvme_ns *ns; 2127 struct spdk_nvme_ctrlr *ctrlr; 2128 const struct spdk_nvme_ctrlr_data *cdata; 2129 2130 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 2131 assert(nvme_ns != NULL); 2132 ns = nvme_ns->ns; 2133 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 2134 2135 switch (io_type) { 2136 case SPDK_BDEV_IO_TYPE_READ: 2137 case SPDK_BDEV_IO_TYPE_WRITE: 2138 case SPDK_BDEV_IO_TYPE_RESET: 2139 case SPDK_BDEV_IO_TYPE_FLUSH: 2140 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 2141 case SPDK_BDEV_IO_TYPE_NVME_IO: 2142 case SPDK_BDEV_IO_TYPE_ABORT: 2143 return true; 2144 2145 case SPDK_BDEV_IO_TYPE_COMPARE: 2146 return spdk_nvme_ns_supports_compare(ns); 2147 2148 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 2149 return spdk_nvme_ns_get_md_size(ns) ? true : false; 2150 2151 case SPDK_BDEV_IO_TYPE_UNMAP: 2152 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2153 return cdata->oncs.dsm; 2154 2155 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 2156 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2157 return cdata->oncs.write_zeroes; 2158 2159 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 2160 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 2161 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) { 2162 return true; 2163 } 2164 return false; 2165 2166 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 2167 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 2168 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS; 2169 2170 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 2171 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS && 2172 spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED; 2173 2174 default: 2175 return false; 2176 } 2177 } 2178 2179 static int 2180 nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch) 2181 { 2182 struct nvme_qpair *nvme_qpair; 2183 struct spdk_io_channel *pg_ch; 2184 int rc; 2185 2186 nvme_qpair = calloc(1, sizeof(*nvme_qpair)); 2187 if (!nvme_qpair) { 2188 SPDK_ERRLOG("Failed to alloc nvme_qpair.\n"); 2189 return -1; 2190 } 2191 2192 TAILQ_INIT(&nvme_qpair->io_path_list); 2193 2194 nvme_qpair->ctrlr = nvme_ctrlr; 2195 nvme_qpair->ctrlr_ch = ctrlr_ch; 2196 2197 pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs); 2198 if (!pg_ch) { 2199 free(nvme_qpair); 2200 return -1; 2201 } 2202 2203 nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch); 2204 2205 #ifdef SPDK_CONFIG_VTUNE 2206 nvme_qpair->group->collect_spin_stat = true; 2207 #else 2208 nvme_qpair->group->collect_spin_stat = false; 2209 #endif 2210 2211 rc = bdev_nvme_create_qpair(nvme_qpair); 2212 if (rc != 0) { 2213 /* nvme_ctrlr can't create IO qpair if connection is down. If nvme_ctrlr is 2214 * being reset or scheduled to reconnect later, ignore this failure. 2215 * Then IO qpair will be created later when reconnect completes. 2216 * If the user submits IO requests in the meantime, they will be queued and 2217 * resubmitted later */ 2218 if (!nvme_ctrlr->resetting && !nvme_ctrlr->reconnect_is_delayed) { 2219 spdk_put_io_channel(pg_ch); 2220 free(nvme_qpair); 2221 return rc; 2222 } 2223 } 2224 2225 TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 2226 2227 ctrlr_ch->qpair = nvme_qpair; 2228 2229 pthread_mutex_lock(&nvme_qpair->ctrlr->mutex); 2230 nvme_qpair->ctrlr->ref++; 2231 pthread_mutex_unlock(&nvme_qpair->ctrlr->mutex); 2232 2233 return 0; 2234 } 2235 2236 static int 2237 bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) 2238 { 2239 struct nvme_ctrlr *nvme_ctrlr = io_device; 2240 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 2241 2242 TAILQ_INIT(&ctrlr_ch->pending_resets); 2243 2244 return nvme_qpair_create(nvme_ctrlr, ctrlr_ch); 2245 } 2246 2247 static void 2248 nvme_qpair_delete(struct nvme_qpair *nvme_qpair) 2249 { 2250 assert(nvme_qpair->group != NULL); 2251 2252 TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 2253 2254 spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group)); 2255 2256 nvme_ctrlr_release(nvme_qpair->ctrlr); 2257 2258 free(nvme_qpair); 2259 } 2260 2261 static void 2262 bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf) 2263 { 2264 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 2265 struct nvme_qpair *nvme_qpair; 2266 2267 nvme_qpair = ctrlr_ch->qpair; 2268 assert(nvme_qpair != NULL); 2269 2270 _bdev_nvme_clear_io_path_cache(nvme_qpair); 2271 2272 if (nvme_qpair->qpair != NULL) { 2273 if (ctrlr_ch->reset_iter == NULL) { 2274 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 2275 } else { 2276 /* Skip current ctrlr_channel in a full reset sequence because 2277 * it is being deleted now. The qpair is already being disconnected. 2278 * We do not have to restart disconnecting it. 2279 */ 2280 spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 2281 } 2282 2283 /* We cannot release a reference to the poll group now. 2284 * The qpair may be disconnected asynchronously later. 2285 * We need to poll it until it is actually disconnected. 2286 * Just detach the qpair from the deleting ctrlr_channel. 2287 */ 2288 nvme_qpair->ctrlr_ch = NULL; 2289 } else { 2290 assert(ctrlr_ch->reset_iter == NULL); 2291 2292 nvme_qpair_delete(nvme_qpair); 2293 } 2294 } 2295 2296 static void 2297 bdev_nvme_submit_accel_crc32c(void *ctx, uint32_t *dst, struct iovec *iov, 2298 uint32_t iov_cnt, uint32_t seed, 2299 spdk_nvme_accel_completion_cb cb_fn, void *cb_arg) 2300 { 2301 struct nvme_poll_group *group = ctx; 2302 int rc; 2303 2304 assert(group->accel_channel != NULL); 2305 assert(cb_fn != NULL); 2306 2307 rc = spdk_accel_submit_crc32cv(group->accel_channel, dst, iov, iov_cnt, seed, cb_fn, cb_arg); 2308 if (rc) { 2309 /* For the two cases, spdk_accel_submit_crc32cv does not call the user's cb_fn */ 2310 if (rc == -ENOMEM || rc == -EINVAL) { 2311 cb_fn(cb_arg, rc); 2312 } 2313 SPDK_ERRLOG("Cannot complete the accelerated crc32c operation with iov=%p\n", iov); 2314 } 2315 } 2316 2317 static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = { 2318 .table_size = sizeof(struct spdk_nvme_accel_fn_table), 2319 .submit_accel_crc32c = bdev_nvme_submit_accel_crc32c, 2320 }; 2321 2322 static int 2323 bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf) 2324 { 2325 struct nvme_poll_group *group = ctx_buf; 2326 2327 TAILQ_INIT(&group->qpair_list); 2328 2329 group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table); 2330 if (group->group == NULL) { 2331 return -1; 2332 } 2333 2334 group->accel_channel = spdk_accel_engine_get_io_channel(); 2335 if (!group->accel_channel) { 2336 spdk_nvme_poll_group_destroy(group->group); 2337 SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n", 2338 group); 2339 return -1; 2340 } 2341 2342 group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us); 2343 2344 if (group->poller == NULL) { 2345 spdk_put_io_channel(group->accel_channel); 2346 spdk_nvme_poll_group_destroy(group->group); 2347 return -1; 2348 } 2349 2350 return 0; 2351 } 2352 2353 static void 2354 bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf) 2355 { 2356 struct nvme_poll_group *group = ctx_buf; 2357 2358 assert(TAILQ_EMPTY(&group->qpair_list)); 2359 2360 if (group->accel_channel) { 2361 spdk_put_io_channel(group->accel_channel); 2362 } 2363 2364 spdk_poller_unregister(&group->poller); 2365 if (spdk_nvme_poll_group_destroy(group->group)) { 2366 SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n"); 2367 assert(false); 2368 } 2369 } 2370 2371 static struct spdk_io_channel * 2372 bdev_nvme_get_io_channel(void *ctx) 2373 { 2374 struct nvme_bdev *nvme_bdev = ctx; 2375 2376 return spdk_get_io_channel(nvme_bdev); 2377 } 2378 2379 static void * 2380 bdev_nvme_get_module_ctx(void *ctx) 2381 { 2382 struct nvme_bdev *nvme_bdev = ctx; 2383 struct nvme_ns *nvme_ns; 2384 2385 if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) { 2386 return NULL; 2387 } 2388 2389 nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list); 2390 if (!nvme_ns) { 2391 return NULL; 2392 } 2393 2394 return nvme_ns->ns; 2395 } 2396 2397 static const char * 2398 _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state) 2399 { 2400 switch (ana_state) { 2401 case SPDK_NVME_ANA_OPTIMIZED_STATE: 2402 return "optimized"; 2403 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 2404 return "non_optimized"; 2405 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 2406 return "inaccessible"; 2407 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 2408 return "persistent_loss"; 2409 case SPDK_NVME_ANA_CHANGE_STATE: 2410 return "change"; 2411 default: 2412 return NULL; 2413 } 2414 } 2415 2416 static int 2417 bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size) 2418 { 2419 struct nvme_bdev *nbdev = ctx; 2420 struct nvme_ns *nvme_ns; 2421 2422 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 2423 assert(nvme_ns != NULL); 2424 2425 return spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, domains, array_size); 2426 } 2427 2428 static const char * 2429 nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr) 2430 { 2431 if (nvme_ctrlr->destruct) { 2432 return "deleting"; 2433 } else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 2434 return "failed"; 2435 } else if (nvme_ctrlr->resetting) { 2436 return "resetting"; 2437 } else if (nvme_ctrlr->reconnect_is_delayed > 0) { 2438 return "reconnect_is_delayed"; 2439 } else { 2440 return "enabled"; 2441 } 2442 } 2443 2444 void 2445 nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr) 2446 { 2447 struct spdk_nvme_transport_id *trid; 2448 const struct spdk_nvme_ctrlr_opts *opts; 2449 const struct spdk_nvme_ctrlr_data *cdata; 2450 2451 spdk_json_write_object_begin(w); 2452 2453 spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr)); 2454 2455 #ifdef SPDK_CONFIG_NVME_CUSE 2456 size_t cuse_name_size = 128; 2457 char cuse_name[cuse_name_size]; 2458 2459 int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size); 2460 if (rc == 0) { 2461 spdk_json_write_named_string(w, "cuse_device", cuse_name); 2462 } 2463 #endif 2464 trid = &nvme_ctrlr->active_path_id->trid; 2465 spdk_json_write_named_object_begin(w, "trid"); 2466 nvme_bdev_dump_trid_json(trid, w); 2467 spdk_json_write_object_end(w); 2468 2469 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 2470 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 2471 2472 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 2473 spdk_json_write_named_object_begin(w, "host"); 2474 spdk_json_write_named_string(w, "nqn", opts->hostnqn); 2475 spdk_json_write_named_string(w, "addr", opts->src_addr); 2476 spdk_json_write_named_string(w, "svcid", opts->src_svcid); 2477 spdk_json_write_object_end(w); 2478 2479 spdk_json_write_object_end(w); 2480 } 2481 2482 static void 2483 nvme_namespace_info_json(struct spdk_json_write_ctx *w, 2484 struct nvme_ns *nvme_ns) 2485 { 2486 struct spdk_nvme_ns *ns; 2487 struct spdk_nvme_ctrlr *ctrlr; 2488 const struct spdk_nvme_ctrlr_data *cdata; 2489 const struct spdk_nvme_transport_id *trid; 2490 union spdk_nvme_vs_register vs; 2491 const struct spdk_nvme_ns_data *nsdata; 2492 char buf[128]; 2493 2494 ns = nvme_ns->ns; 2495 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 2496 2497 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2498 trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); 2499 vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr); 2500 2501 spdk_json_write_object_begin(w); 2502 2503 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 2504 spdk_json_write_named_string(w, "pci_address", trid->traddr); 2505 } 2506 2507 spdk_json_write_named_object_begin(w, "trid"); 2508 2509 nvme_bdev_dump_trid_json(trid, w); 2510 2511 spdk_json_write_object_end(w); 2512 2513 #ifdef SPDK_CONFIG_NVME_CUSE 2514 size_t cuse_name_size = 128; 2515 char cuse_name[cuse_name_size]; 2516 2517 int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns), 2518 cuse_name, &cuse_name_size); 2519 if (rc == 0) { 2520 spdk_json_write_named_string(w, "cuse_device", cuse_name); 2521 } 2522 #endif 2523 2524 spdk_json_write_named_object_begin(w, "ctrlr_data"); 2525 2526 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 2527 2528 spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid); 2529 2530 snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn); 2531 spdk_str_trim(buf); 2532 spdk_json_write_named_string(w, "model_number", buf); 2533 2534 snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn); 2535 spdk_str_trim(buf); 2536 spdk_json_write_named_string(w, "serial_number", buf); 2537 2538 snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr); 2539 spdk_str_trim(buf); 2540 spdk_json_write_named_string(w, "firmware_revision", buf); 2541 2542 if (cdata->subnqn[0] != '\0') { 2543 spdk_json_write_named_string(w, "subnqn", cdata->subnqn); 2544 } 2545 2546 spdk_json_write_named_object_begin(w, "oacs"); 2547 2548 spdk_json_write_named_uint32(w, "security", cdata->oacs.security); 2549 spdk_json_write_named_uint32(w, "format", cdata->oacs.format); 2550 spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware); 2551 spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage); 2552 2553 spdk_json_write_object_end(w); 2554 2555 spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr); 2556 spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting); 2557 2558 spdk_json_write_object_end(w); 2559 2560 spdk_json_write_named_object_begin(w, "vs"); 2561 2562 spdk_json_write_name(w, "nvme_version"); 2563 if (vs.bits.ter) { 2564 spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter); 2565 } else { 2566 spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr); 2567 } 2568 2569 spdk_json_write_object_end(w); 2570 2571 nsdata = spdk_nvme_ns_get_data(ns); 2572 2573 spdk_json_write_named_object_begin(w, "ns_data"); 2574 2575 spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns)); 2576 2577 if (cdata->cmic.ana_reporting) { 2578 spdk_json_write_named_string(w, "ana_state", 2579 _nvme_ana_state_str(nvme_ns->ana_state)); 2580 } 2581 2582 spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share); 2583 2584 spdk_json_write_object_end(w); 2585 2586 if (cdata->oacs.security) { 2587 spdk_json_write_named_object_begin(w, "security"); 2588 2589 spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal); 2590 2591 spdk_json_write_object_end(w); 2592 } 2593 2594 spdk_json_write_object_end(w); 2595 } 2596 2597 static int 2598 bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 2599 { 2600 struct nvme_bdev *nvme_bdev = ctx; 2601 struct nvme_ns *nvme_ns; 2602 2603 pthread_mutex_lock(&nvme_bdev->mutex); 2604 spdk_json_write_named_array_begin(w, "nvme"); 2605 TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) { 2606 nvme_namespace_info_json(w, nvme_ns); 2607 } 2608 spdk_json_write_array_end(w); 2609 pthread_mutex_unlock(&nvme_bdev->mutex); 2610 2611 return 0; 2612 } 2613 2614 static void 2615 bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 2616 { 2617 /* No config per bdev needed */ 2618 } 2619 2620 static uint64_t 2621 bdev_nvme_get_spin_time(struct spdk_io_channel *ch) 2622 { 2623 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 2624 struct nvme_io_path *io_path; 2625 struct nvme_poll_group *group; 2626 uint64_t spin_time = 0; 2627 2628 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 2629 group = io_path->qpair->group; 2630 2631 if (!group || !group->collect_spin_stat) { 2632 continue; 2633 } 2634 2635 if (group->end_ticks != 0) { 2636 group->spin_ticks += (group->end_ticks - group->start_ticks); 2637 group->end_ticks = 0; 2638 } 2639 2640 spin_time += group->spin_ticks; 2641 group->start_ticks = 0; 2642 group->spin_ticks = 0; 2643 } 2644 2645 return (spin_time * 1000000ULL) / spdk_get_ticks_hz(); 2646 } 2647 2648 static const struct spdk_bdev_fn_table nvmelib_fn_table = { 2649 .destruct = bdev_nvme_destruct, 2650 .submit_request = bdev_nvme_submit_request, 2651 .io_type_supported = bdev_nvme_io_type_supported, 2652 .get_io_channel = bdev_nvme_get_io_channel, 2653 .dump_info_json = bdev_nvme_dump_info_json, 2654 .write_config_json = bdev_nvme_write_config_json, 2655 .get_spin_time = bdev_nvme_get_spin_time, 2656 .get_module_ctx = bdev_nvme_get_module_ctx, 2657 .get_memory_domains = bdev_nvme_get_memory_domains, 2658 }; 2659 2660 typedef int (*bdev_nvme_parse_ana_log_page_cb)( 2661 const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg); 2662 2663 static int 2664 bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 2665 bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg) 2666 { 2667 struct spdk_nvme_ana_group_descriptor *copied_desc; 2668 uint8_t *orig_desc; 2669 uint32_t i, desc_size, copy_len; 2670 int rc = 0; 2671 2672 if (nvme_ctrlr->ana_log_page == NULL) { 2673 return -EINVAL; 2674 } 2675 2676 copied_desc = nvme_ctrlr->copied_ana_desc; 2677 2678 orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page); 2679 copy_len = nvme_ctrlr->ana_log_page_size - sizeof(struct spdk_nvme_ana_page); 2680 2681 for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) { 2682 memcpy(copied_desc, orig_desc, copy_len); 2683 2684 rc = cb_fn(copied_desc, cb_arg); 2685 if (rc != 0) { 2686 break; 2687 } 2688 2689 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) + 2690 copied_desc->num_of_nsid * sizeof(uint32_t); 2691 orig_desc += desc_size; 2692 copy_len -= desc_size; 2693 } 2694 2695 return rc; 2696 } 2697 2698 static int 2699 nvme_ns_ana_transition_timedout(void *ctx) 2700 { 2701 struct nvme_ns *nvme_ns = ctx; 2702 2703 spdk_poller_unregister(&nvme_ns->anatt_timer); 2704 nvme_ns->ana_transition_timedout = true; 2705 2706 return SPDK_POLLER_BUSY; 2707 } 2708 2709 static void 2710 _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns, 2711 const struct spdk_nvme_ana_group_descriptor *desc) 2712 { 2713 const struct spdk_nvme_ctrlr_data *cdata; 2714 2715 nvme_ns->ana_group_id = desc->ana_group_id; 2716 nvme_ns->ana_state = desc->ana_state; 2717 nvme_ns->ana_state_updating = false; 2718 2719 switch (nvme_ns->ana_state) { 2720 case SPDK_NVME_ANA_OPTIMIZED_STATE: 2721 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 2722 nvme_ns->ana_transition_timedout = false; 2723 spdk_poller_unregister(&nvme_ns->anatt_timer); 2724 break; 2725 2726 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 2727 case SPDK_NVME_ANA_CHANGE_STATE: 2728 if (nvme_ns->anatt_timer != NULL) { 2729 break; 2730 } 2731 2732 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 2733 nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout, 2734 nvme_ns, 2735 cdata->anatt * SPDK_SEC_TO_USEC); 2736 break; 2737 default: 2738 break; 2739 } 2740 } 2741 2742 static int 2743 nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg) 2744 { 2745 struct nvme_ns *nvme_ns = cb_arg; 2746 uint32_t i; 2747 2748 for (i = 0; i < desc->num_of_nsid; i++) { 2749 if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) { 2750 continue; 2751 } 2752 2753 _nvme_ns_set_ana_state(nvme_ns, desc); 2754 return 1; 2755 } 2756 2757 return 0; 2758 } 2759 2760 static int 2761 nvme_disk_create(struct spdk_bdev *disk, const char *base_name, 2762 struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns, 2763 uint32_t prchk_flags, void *ctx) 2764 { 2765 const struct spdk_uuid *uuid; 2766 const uint8_t *nguid; 2767 const struct spdk_nvme_ctrlr_data *cdata; 2768 const struct spdk_nvme_ns_data *nsdata; 2769 enum spdk_nvme_csi csi; 2770 uint32_t atomic_bs, phys_bs, bs; 2771 2772 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2773 csi = spdk_nvme_ns_get_csi(ns); 2774 2775 switch (csi) { 2776 case SPDK_NVME_CSI_NVM: 2777 disk->product_name = "NVMe disk"; 2778 break; 2779 case SPDK_NVME_CSI_ZNS: 2780 disk->product_name = "NVMe ZNS disk"; 2781 disk->zoned = true; 2782 disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 2783 disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) / 2784 spdk_nvme_ns_get_extended_sector_size(ns); 2785 disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns); 2786 disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns); 2787 break; 2788 default: 2789 SPDK_ERRLOG("unsupported CSI: %u\n", csi); 2790 return -ENOTSUP; 2791 } 2792 2793 disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns)); 2794 if (!disk->name) { 2795 return -ENOMEM; 2796 } 2797 2798 disk->write_cache = 0; 2799 if (cdata->vwc.present) { 2800 /* Enable if the Volatile Write Cache exists */ 2801 disk->write_cache = 1; 2802 } 2803 if (cdata->oncs.write_zeroes) { 2804 disk->max_write_zeroes = UINT16_MAX + 1; 2805 } 2806 disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns); 2807 disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns); 2808 disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns); 2809 2810 nguid = spdk_nvme_ns_get_nguid(ns); 2811 if (!nguid) { 2812 uuid = spdk_nvme_ns_get_uuid(ns); 2813 if (uuid) { 2814 disk->uuid = *uuid; 2815 } 2816 } else { 2817 memcpy(&disk->uuid, nguid, sizeof(disk->uuid)); 2818 } 2819 2820 nsdata = spdk_nvme_ns_get_data(ns); 2821 bs = spdk_nvme_ns_get_sector_size(ns); 2822 atomic_bs = bs; 2823 phys_bs = bs; 2824 if (nsdata->nabo == 0) { 2825 if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) { 2826 atomic_bs = bs * (1 + nsdata->nawupf); 2827 } else { 2828 atomic_bs = bs * (1 + cdata->awupf); 2829 } 2830 } 2831 if (nsdata->nsfeat.optperf) { 2832 phys_bs = bs * (1 + nsdata->npwg); 2833 } 2834 disk->phys_blocklen = spdk_min(phys_bs, atomic_bs); 2835 2836 disk->md_len = spdk_nvme_ns_get_md_size(ns); 2837 if (disk->md_len != 0) { 2838 disk->md_interleave = nsdata->flbas.extended; 2839 disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns); 2840 if (disk->dif_type != SPDK_DIF_DISABLE) { 2841 disk->dif_is_head_of_md = nsdata->dps.md_start; 2842 disk->dif_check_flags = prchk_flags; 2843 } 2844 } 2845 2846 if (!(spdk_nvme_ctrlr_get_flags(ctrlr) & 2847 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) { 2848 disk->acwu = 0; 2849 } else if (nsdata->nsfeat.ns_atomic_write_unit) { 2850 disk->acwu = nsdata->nacwu; 2851 } else { 2852 disk->acwu = cdata->acwu; 2853 } 2854 2855 disk->ctxt = ctx; 2856 disk->fn_table = &nvmelib_fn_table; 2857 disk->module = &nvme_if; 2858 2859 return 0; 2860 } 2861 2862 static int 2863 nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 2864 { 2865 struct nvme_bdev *bdev; 2866 int rc; 2867 2868 bdev = calloc(1, sizeof(*bdev)); 2869 if (!bdev) { 2870 SPDK_ERRLOG("bdev calloc() failed\n"); 2871 return -ENOMEM; 2872 } 2873 2874 rc = pthread_mutex_init(&bdev->mutex, NULL); 2875 if (rc != 0) { 2876 free(bdev); 2877 return rc; 2878 } 2879 2880 bdev->ref = 1; 2881 TAILQ_INIT(&bdev->nvme_ns_list); 2882 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 2883 bdev->opal = nvme_ctrlr->opal_dev != NULL; 2884 2885 rc = nvme_disk_create(&bdev->disk, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ctrlr, 2886 nvme_ns->ns, nvme_ctrlr->opts.prchk_flags, bdev); 2887 if (rc != 0) { 2888 SPDK_ERRLOG("Failed to create NVMe disk\n"); 2889 pthread_mutex_destroy(&bdev->mutex); 2890 free(bdev); 2891 return rc; 2892 } 2893 2894 spdk_io_device_register(bdev, 2895 bdev_nvme_create_bdev_channel_cb, 2896 bdev_nvme_destroy_bdev_channel_cb, 2897 sizeof(struct nvme_bdev_channel), 2898 bdev->disk.name); 2899 2900 rc = spdk_bdev_register(&bdev->disk); 2901 if (rc != 0) { 2902 SPDK_ERRLOG("spdk_bdev_register() failed\n"); 2903 spdk_io_device_unregister(bdev, NULL); 2904 pthread_mutex_destroy(&bdev->mutex); 2905 free(bdev->disk.name); 2906 free(bdev); 2907 return rc; 2908 } 2909 2910 nvme_ns->bdev = bdev; 2911 bdev->nsid = nvme_ns->id; 2912 2913 bdev->nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr; 2914 TAILQ_INSERT_TAIL(&nvme_ctrlr->nbdev_ctrlr->bdevs, bdev, tailq); 2915 2916 return 0; 2917 } 2918 2919 static bool 2920 bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2) 2921 { 2922 const struct spdk_nvme_ns_data *nsdata1, *nsdata2; 2923 const struct spdk_uuid *uuid1, *uuid2; 2924 2925 nsdata1 = spdk_nvme_ns_get_data(ns1); 2926 nsdata2 = spdk_nvme_ns_get_data(ns2); 2927 uuid1 = spdk_nvme_ns_get_uuid(ns1); 2928 uuid2 = spdk_nvme_ns_get_uuid(ns2); 2929 2930 return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 && 2931 nsdata1->eui64 == nsdata2->eui64 && 2932 ((uuid1 == NULL && uuid2 == NULL) || 2933 (uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) && 2934 spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2); 2935 } 2936 2937 static bool 2938 hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 2939 struct spdk_nvme_ctrlr_opts *opts) 2940 { 2941 struct nvme_probe_skip_entry *entry; 2942 2943 TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) { 2944 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 2945 return false; 2946 } 2947 } 2948 2949 opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst; 2950 opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight; 2951 opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight; 2952 opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight; 2953 opts->disable_read_ana_log_page = true; 2954 2955 SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr); 2956 2957 return true; 2958 } 2959 2960 static void 2961 nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl) 2962 { 2963 struct nvme_ctrlr *nvme_ctrlr = ctx; 2964 2965 if (spdk_nvme_cpl_is_error(cpl)) { 2966 SPDK_WARNLOG("Abort failed. Resetting controller. sc is %u, sct is %u.\n", cpl->status.sc, 2967 cpl->status.sct); 2968 bdev_nvme_reset(nvme_ctrlr); 2969 } else if (cpl->cdw0 & 0x1) { 2970 SPDK_WARNLOG("Specified command could not be aborted.\n"); 2971 bdev_nvme_reset(nvme_ctrlr); 2972 } 2973 } 2974 2975 static void 2976 timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, 2977 struct spdk_nvme_qpair *qpair, uint16_t cid) 2978 { 2979 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 2980 union spdk_nvme_csts_register csts; 2981 int rc; 2982 2983 assert(nvme_ctrlr->ctrlr == ctrlr); 2984 2985 SPDK_WARNLOG("Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n", ctrlr, qpair, cid); 2986 2987 /* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O 2988 * queue. (Note: qpair == NULL when there's an admin cmd timeout.) Otherwise we 2989 * would submit another fabrics cmd on the admin queue to read CSTS and check for its 2990 * completion recursively. 2991 */ 2992 if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) { 2993 csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr); 2994 if (csts.bits.cfs) { 2995 SPDK_ERRLOG("Controller Fatal Status, reset required\n"); 2996 bdev_nvme_reset(nvme_ctrlr); 2997 return; 2998 } 2999 } 3000 3001 switch (g_opts.action_on_timeout) { 3002 case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT: 3003 if (qpair) { 3004 /* Don't send abort to ctrlr when ctrlr is not available. */ 3005 pthread_mutex_lock(&nvme_ctrlr->mutex); 3006 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 3007 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3008 SPDK_NOTICELOG("Quit abort. Ctrlr is not available.\n"); 3009 return; 3010 } 3011 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3012 3013 rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid, 3014 nvme_abort_cpl, nvme_ctrlr); 3015 if (rc == 0) { 3016 return; 3017 } 3018 3019 SPDK_ERRLOG("Unable to send abort. Resetting, rc is %d.\n", rc); 3020 } 3021 3022 /* FALLTHROUGH */ 3023 case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET: 3024 bdev_nvme_reset(nvme_ctrlr); 3025 break; 3026 case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE: 3027 SPDK_DEBUGLOG(bdev_nvme, "No action for nvme controller timeout.\n"); 3028 break; 3029 default: 3030 SPDK_ERRLOG("An invalid timeout action value is found.\n"); 3031 break; 3032 } 3033 } 3034 3035 static void 3036 nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc) 3037 { 3038 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 3039 struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx; 3040 3041 if (rc == 0) { 3042 nvme_ns->probe_ctx = NULL; 3043 pthread_mutex_lock(&nvme_ctrlr->mutex); 3044 nvme_ctrlr->ref++; 3045 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3046 } else { 3047 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 3048 free(nvme_ns); 3049 } 3050 3051 if (ctx) { 3052 ctx->populates_in_progress--; 3053 if (ctx->populates_in_progress == 0) { 3054 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 3055 } 3056 } 3057 } 3058 3059 static void 3060 bdev_nvme_add_io_path(struct spdk_io_channel_iter *i) 3061 { 3062 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 3063 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 3064 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3065 int rc; 3066 3067 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 3068 if (rc != 0) { 3069 SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n"); 3070 } 3071 3072 spdk_for_each_channel_continue(i, rc); 3073 } 3074 3075 static void 3076 bdev_nvme_delete_io_path(struct spdk_io_channel_iter *i) 3077 { 3078 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 3079 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 3080 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3081 struct nvme_io_path *io_path; 3082 3083 io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns); 3084 if (io_path != NULL) { 3085 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 3086 } 3087 3088 spdk_for_each_channel_continue(i, 0); 3089 } 3090 3091 static void 3092 bdev_nvme_add_io_path_failed(struct spdk_io_channel_iter *i, int status) 3093 { 3094 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3095 3096 nvme_ctrlr_populate_namespace_done(nvme_ns, -1); 3097 } 3098 3099 static void 3100 bdev_nvme_add_io_path_done(struct spdk_io_channel_iter *i, int status) 3101 { 3102 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3103 struct nvme_bdev *bdev = spdk_io_channel_iter_get_io_device(i); 3104 3105 if (status == 0) { 3106 nvme_ctrlr_populate_namespace_done(nvme_ns, 0); 3107 } else { 3108 /* Delete the added io_paths and fail populating the namespace. */ 3109 spdk_for_each_channel(bdev, 3110 bdev_nvme_delete_io_path, 3111 nvme_ns, 3112 bdev_nvme_add_io_path_failed); 3113 } 3114 } 3115 3116 static int 3117 nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns) 3118 { 3119 struct nvme_ns *tmp_ns; 3120 const struct spdk_nvme_ns_data *nsdata; 3121 3122 nsdata = spdk_nvme_ns_get_data(nvme_ns->ns); 3123 if (!nsdata->nmic.can_share) { 3124 SPDK_ERRLOG("Namespace cannot be shared.\n"); 3125 return -EINVAL; 3126 } 3127 3128 pthread_mutex_lock(&bdev->mutex); 3129 3130 tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list); 3131 assert(tmp_ns != NULL); 3132 3133 if (!bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) { 3134 pthread_mutex_unlock(&bdev->mutex); 3135 SPDK_ERRLOG("Namespaces are not identical.\n"); 3136 return -EINVAL; 3137 } 3138 3139 bdev->ref++; 3140 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 3141 nvme_ns->bdev = bdev; 3142 3143 pthread_mutex_unlock(&bdev->mutex); 3144 3145 /* Add nvme_io_path to nvme_bdev_channels dynamically. */ 3146 spdk_for_each_channel(bdev, 3147 bdev_nvme_add_io_path, 3148 nvme_ns, 3149 bdev_nvme_add_io_path_done); 3150 3151 return 0; 3152 } 3153 3154 static void 3155 nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 3156 { 3157 struct spdk_nvme_ns *ns; 3158 struct nvme_bdev *bdev; 3159 int rc = 0; 3160 3161 ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id); 3162 if (!ns) { 3163 SPDK_DEBUGLOG(bdev_nvme, "Invalid NS %d\n", nvme_ns->id); 3164 rc = -EINVAL; 3165 goto done; 3166 } 3167 3168 nvme_ns->ns = ns; 3169 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 3170 3171 if (nvme_ctrlr->ana_log_page != NULL) { 3172 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns); 3173 } 3174 3175 bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id); 3176 if (bdev == NULL) { 3177 rc = nvme_bdev_create(nvme_ctrlr, nvme_ns); 3178 } else { 3179 rc = nvme_bdev_add_ns(bdev, nvme_ns); 3180 if (rc == 0) { 3181 return; 3182 } 3183 } 3184 done: 3185 nvme_ctrlr_populate_namespace_done(nvme_ns, rc); 3186 } 3187 3188 static void 3189 nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns) 3190 { 3191 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 3192 3193 assert(nvme_ctrlr != NULL); 3194 3195 pthread_mutex_lock(&nvme_ctrlr->mutex); 3196 3197 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 3198 3199 if (nvme_ns->bdev != NULL) { 3200 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3201 return; 3202 } 3203 3204 free(nvme_ns); 3205 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3206 3207 nvme_ctrlr_release(nvme_ctrlr); 3208 } 3209 3210 static void 3211 bdev_nvme_delete_io_path_done(struct spdk_io_channel_iter *i, int status) 3212 { 3213 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3214 3215 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 3216 } 3217 3218 static void 3219 nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 3220 { 3221 struct nvme_bdev *bdev; 3222 3223 spdk_poller_unregister(&nvme_ns->anatt_timer); 3224 3225 bdev = nvme_ns->bdev; 3226 if (bdev != NULL) { 3227 pthread_mutex_lock(&bdev->mutex); 3228 3229 assert(bdev->ref > 0); 3230 bdev->ref--; 3231 if (bdev->ref == 0) { 3232 pthread_mutex_unlock(&bdev->mutex); 3233 3234 spdk_bdev_unregister(&bdev->disk, NULL, NULL); 3235 } else { 3236 /* spdk_bdev_unregister() is not called until the last nvme_ns is 3237 * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list 3238 * and clear nvme_ns->bdev here. 3239 */ 3240 TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq); 3241 nvme_ns->bdev = NULL; 3242 3243 pthread_mutex_unlock(&bdev->mutex); 3244 3245 /* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that, 3246 * we call depopulate_namespace_done() to avoid use-after-free. 3247 */ 3248 spdk_for_each_channel(bdev, 3249 bdev_nvme_delete_io_path, 3250 nvme_ns, 3251 bdev_nvme_delete_io_path_done); 3252 return; 3253 } 3254 } 3255 3256 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 3257 } 3258 3259 static void 3260 nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 3261 struct nvme_async_probe_ctx *ctx) 3262 { 3263 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 3264 struct nvme_ns *nvme_ns, *next; 3265 struct spdk_nvme_ns *ns; 3266 struct nvme_bdev *bdev; 3267 uint32_t nsid; 3268 int rc; 3269 uint64_t num_sectors; 3270 3271 if (ctx) { 3272 /* Initialize this count to 1 to handle the populate functions 3273 * calling nvme_ctrlr_populate_namespace_done() immediately. 3274 */ 3275 ctx->populates_in_progress = 1; 3276 } 3277 3278 /* First loop over our existing namespaces and see if they have been 3279 * removed. */ 3280 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 3281 while (nvme_ns != NULL) { 3282 next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 3283 3284 if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 3285 /* NS is still there but attributes may have changed */ 3286 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id); 3287 num_sectors = spdk_nvme_ns_get_num_sectors(ns); 3288 bdev = nvme_ns->bdev; 3289 assert(bdev != NULL); 3290 if (bdev->disk.blockcnt != num_sectors) { 3291 SPDK_NOTICELOG("NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n", 3292 nvme_ns->id, 3293 bdev->disk.name, 3294 bdev->disk.blockcnt, 3295 num_sectors); 3296 rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors); 3297 if (rc != 0) { 3298 SPDK_ERRLOG("Could not change num blocks for nvme bdev: name %s, errno: %d.\n", 3299 bdev->disk.name, rc); 3300 } 3301 } 3302 } else { 3303 /* Namespace was removed */ 3304 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 3305 } 3306 3307 nvme_ns = next; 3308 } 3309 3310 /* Loop through all of the namespaces at the nvme level and see if any of them are new */ 3311 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 3312 while (nsid != 0) { 3313 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 3314 3315 if (nvme_ns == NULL) { 3316 /* Found a new one */ 3317 nvme_ns = calloc(1, sizeof(struct nvme_ns)); 3318 if (nvme_ns == NULL) { 3319 SPDK_ERRLOG("Failed to allocate namespace\n"); 3320 /* This just fails to attach the namespace. It may work on a future attempt. */ 3321 continue; 3322 } 3323 3324 nvme_ns->id = nsid; 3325 nvme_ns->ctrlr = nvme_ctrlr; 3326 3327 nvme_ns->bdev = NULL; 3328 3329 if (ctx) { 3330 ctx->populates_in_progress++; 3331 } 3332 nvme_ns->probe_ctx = ctx; 3333 3334 RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 3335 3336 nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns); 3337 } 3338 3339 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid); 3340 } 3341 3342 if (ctx) { 3343 /* Decrement this count now that the loop is over to account 3344 * for the one we started with. If the count is then 0, we 3345 * know any populate_namespace functions completed immediately, 3346 * so we'll kick the callback here. 3347 */ 3348 ctx->populates_in_progress--; 3349 if (ctx->populates_in_progress == 0) { 3350 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 3351 } 3352 } 3353 3354 } 3355 3356 static void 3357 nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr) 3358 { 3359 struct nvme_ns *nvme_ns, *tmp; 3360 3361 RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) { 3362 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 3363 } 3364 } 3365 3366 static int 3367 nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc, 3368 void *cb_arg) 3369 { 3370 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 3371 struct nvme_ns *nvme_ns; 3372 uint32_t i, nsid; 3373 3374 for (i = 0; i < desc->num_of_nsid; i++) { 3375 nsid = desc->nsid[i]; 3376 if (nsid == 0) { 3377 continue; 3378 } 3379 3380 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 3381 3382 assert(nvme_ns != NULL); 3383 if (nvme_ns == NULL) { 3384 /* Target told us that an inactive namespace had an ANA change */ 3385 continue; 3386 } 3387 3388 _nvme_ns_set_ana_state(nvme_ns, desc); 3389 } 3390 3391 return 0; 3392 } 3393 3394 static void 3395 _nvme_ctrlr_read_ana_log_page_done(struct spdk_io_channel_iter *i, int status) 3396 { 3397 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 3398 3399 pthread_mutex_lock(&nvme_ctrlr->mutex); 3400 3401 assert(nvme_ctrlr->ana_log_page_updating == true); 3402 nvme_ctrlr->ana_log_page_updating = false; 3403 3404 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 3405 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3406 return; 3407 } 3408 3409 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3410 3411 nvme_ctrlr_unregister(nvme_ctrlr); 3412 } 3413 3414 static void 3415 bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 3416 { 3417 struct nvme_ns *nvme_ns; 3418 3419 spdk_free(nvme_ctrlr->ana_log_page); 3420 nvme_ctrlr->ana_log_page = NULL; 3421 3422 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 3423 nvme_ns != NULL; 3424 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 3425 nvme_ns->ana_state_updating = false; 3426 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 3427 } 3428 } 3429 3430 static void 3431 nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl) 3432 { 3433 struct nvme_ctrlr *nvme_ctrlr = ctx; 3434 3435 if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) { 3436 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states, 3437 nvme_ctrlr); 3438 } else { 3439 bdev_nvme_disable_read_ana_log_page(nvme_ctrlr); 3440 } 3441 3442 bdev_nvme_clear_io_path_caches(nvme_ctrlr, _nvme_ctrlr_read_ana_log_page_done); 3443 } 3444 3445 static int 3446 nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 3447 { 3448 int rc; 3449 3450 if (nvme_ctrlr->ana_log_page == NULL) { 3451 return -EINVAL; 3452 } 3453 3454 pthread_mutex_lock(&nvme_ctrlr->mutex); 3455 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 3456 nvme_ctrlr->ana_log_page_updating) { 3457 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3458 return -EBUSY; 3459 } 3460 3461 nvme_ctrlr->ana_log_page_updating = true; 3462 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3463 3464 rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr, 3465 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 3466 SPDK_NVME_GLOBAL_NS_TAG, 3467 nvme_ctrlr->ana_log_page, 3468 nvme_ctrlr->ana_log_page_size, 0, 3469 nvme_ctrlr_read_ana_log_page_done, 3470 nvme_ctrlr); 3471 if (rc != 0) { 3472 nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL); 3473 } 3474 3475 return rc; 3476 } 3477 3478 static void 3479 aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 3480 { 3481 struct nvme_ctrlr *nvme_ctrlr = arg; 3482 union spdk_nvme_async_event_completion event; 3483 3484 if (spdk_nvme_cpl_is_error(cpl)) { 3485 SPDK_WARNLOG("AER request execute failed"); 3486 return; 3487 } 3488 3489 event.raw = cpl->cdw0; 3490 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 3491 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) { 3492 nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL); 3493 } else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 3494 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) { 3495 nvme_ctrlr_read_ana_log_page(nvme_ctrlr); 3496 } 3497 } 3498 3499 static void 3500 populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, size_t count, int rc) 3501 { 3502 if (ctx->cb_fn) { 3503 ctx->cb_fn(ctx->cb_ctx, count, rc); 3504 } 3505 3506 ctx->namespaces_populated = true; 3507 if (ctx->probe_done) { 3508 /* The probe was already completed, so we need to free the context 3509 * here. This can happen for cases like OCSSD, where we need to 3510 * send additional commands to the SSD after attach. 3511 */ 3512 free(ctx); 3513 } 3514 } 3515 3516 static void 3517 nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr, 3518 struct nvme_async_probe_ctx *ctx) 3519 { 3520 spdk_io_device_register(nvme_ctrlr, 3521 bdev_nvme_create_ctrlr_channel_cb, 3522 bdev_nvme_destroy_ctrlr_channel_cb, 3523 sizeof(struct nvme_ctrlr_channel), 3524 nvme_ctrlr->nbdev_ctrlr->name); 3525 3526 nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx); 3527 } 3528 3529 static void 3530 nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl) 3531 { 3532 struct nvme_ctrlr *nvme_ctrlr = _ctx; 3533 struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx; 3534 3535 nvme_ctrlr->probe_ctx = NULL; 3536 3537 if (spdk_nvme_cpl_is_error(cpl)) { 3538 nvme_ctrlr_delete(nvme_ctrlr); 3539 3540 if (ctx != NULL) { 3541 populate_namespaces_cb(ctx, 0, -1); 3542 } 3543 return; 3544 } 3545 3546 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 3547 } 3548 3549 static int 3550 nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 3551 struct nvme_async_probe_ctx *ctx) 3552 { 3553 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 3554 const struct spdk_nvme_ctrlr_data *cdata; 3555 uint32_t ana_log_page_size; 3556 3557 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3558 3559 ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 3560 sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->nn * 3561 sizeof(uint32_t); 3562 3563 nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL, 3564 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3565 if (nvme_ctrlr->ana_log_page == NULL) { 3566 SPDK_ERRLOG("could not allocate ANA log page buffer\n"); 3567 return -ENXIO; 3568 } 3569 3570 /* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned. 3571 * Hence copy each descriptor to a temporary area when parsing it. 3572 * 3573 * Allocate a buffer whose size is as large as ANA log page buffer because 3574 * we do not know the size of a descriptor until actually reading it. 3575 */ 3576 nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size); 3577 if (nvme_ctrlr->copied_ana_desc == NULL) { 3578 SPDK_ERRLOG("could not allocate a buffer to parse ANA descriptor\n"); 3579 return -ENOMEM; 3580 } 3581 3582 nvme_ctrlr->ana_log_page_size = ana_log_page_size; 3583 3584 nvme_ctrlr->probe_ctx = ctx; 3585 3586 return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, 3587 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 3588 SPDK_NVME_GLOBAL_NS_TAG, 3589 nvme_ctrlr->ana_log_page, 3590 nvme_ctrlr->ana_log_page_size, 0, 3591 nvme_ctrlr_init_ana_log_page_done, 3592 nvme_ctrlr); 3593 } 3594 3595 /* hostnqn and subnqn were already verified before attaching a controller. 3596 * Hence check only the multipath capability and cntlid here. 3597 */ 3598 static bool 3599 bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr) 3600 { 3601 struct nvme_ctrlr *tmp; 3602 const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata; 3603 3604 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3605 3606 if (!cdata->cmic.multi_ctrlr) { 3607 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 3608 return false; 3609 } 3610 3611 TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) { 3612 tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr); 3613 3614 if (!tmp_cdata->cmic.multi_ctrlr) { 3615 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 3616 return false; 3617 } 3618 if (cdata->cntlid == tmp_cdata->cntlid) { 3619 SPDK_ERRLOG("cntlid %u are duplicated.\n", tmp_cdata->cntlid); 3620 return false; 3621 } 3622 } 3623 3624 return true; 3625 } 3626 3627 static int 3628 nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr) 3629 { 3630 struct nvme_bdev_ctrlr *nbdev_ctrlr; 3631 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 3632 int rc = 0; 3633 3634 pthread_mutex_lock(&g_bdev_nvme_mutex); 3635 3636 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 3637 if (nbdev_ctrlr != NULL) { 3638 if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) { 3639 rc = -EINVAL; 3640 goto exit; 3641 } 3642 } else { 3643 nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr)); 3644 if (nbdev_ctrlr == NULL) { 3645 SPDK_ERRLOG("Failed to allocate nvme_bdev_ctrlr.\n"); 3646 rc = -ENOMEM; 3647 goto exit; 3648 } 3649 nbdev_ctrlr->name = strdup(name); 3650 if (nbdev_ctrlr->name == NULL) { 3651 SPDK_ERRLOG("Failed to allocate name of nvme_bdev_ctrlr.\n"); 3652 free(nbdev_ctrlr); 3653 goto exit; 3654 } 3655 TAILQ_INIT(&nbdev_ctrlr->ctrlrs); 3656 TAILQ_INIT(&nbdev_ctrlr->bdevs); 3657 TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 3658 } 3659 nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr; 3660 TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 3661 exit: 3662 pthread_mutex_unlock(&g_bdev_nvme_mutex); 3663 return rc; 3664 } 3665 3666 static int 3667 nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr, 3668 const char *name, 3669 const struct spdk_nvme_transport_id *trid, 3670 struct nvme_async_probe_ctx *ctx) 3671 { 3672 struct nvme_ctrlr *nvme_ctrlr; 3673 struct nvme_path_id *path_id; 3674 const struct spdk_nvme_ctrlr_data *cdata; 3675 int rc; 3676 3677 nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr)); 3678 if (nvme_ctrlr == NULL) { 3679 SPDK_ERRLOG("Failed to allocate device struct\n"); 3680 return -ENOMEM; 3681 } 3682 3683 rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL); 3684 if (rc != 0) { 3685 free(nvme_ctrlr); 3686 return rc; 3687 } 3688 3689 TAILQ_INIT(&nvme_ctrlr->trids); 3690 3691 RB_INIT(&nvme_ctrlr->namespaces); 3692 3693 path_id = calloc(1, sizeof(*path_id)); 3694 if (path_id == NULL) { 3695 SPDK_ERRLOG("Failed to allocate trid entry pointer\n"); 3696 rc = -ENOMEM; 3697 goto err; 3698 } 3699 3700 path_id->trid = *trid; 3701 if (ctx != NULL) { 3702 memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr)); 3703 memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid)); 3704 } 3705 nvme_ctrlr->active_path_id = path_id; 3706 TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link); 3707 3708 nvme_ctrlr->thread = spdk_get_thread(); 3709 nvme_ctrlr->ctrlr = ctrlr; 3710 nvme_ctrlr->ref = 1; 3711 3712 if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) { 3713 SPDK_ERRLOG("OCSSDs are not supported"); 3714 rc = -ENOTSUP; 3715 goto err; 3716 } 3717 3718 if (ctx != NULL) { 3719 memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts)); 3720 } else { 3721 bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts); 3722 } 3723 3724 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr, 3725 g_opts.nvme_adminq_poll_period_us); 3726 3727 if (g_opts.timeout_us > 0) { 3728 /* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */ 3729 /* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */ 3730 uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ? 3731 g_opts.timeout_us : g_opts.timeout_admin_us; 3732 spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us, 3733 adm_timeout_us, timeout_cb, nvme_ctrlr); 3734 } 3735 3736 spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr); 3737 spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr); 3738 3739 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 3740 SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) { 3741 nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr); 3742 } 3743 3744 rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr); 3745 if (rc != 0) { 3746 goto err; 3747 } 3748 3749 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3750 3751 if (cdata->cmic.ana_reporting) { 3752 rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx); 3753 if (rc == 0) { 3754 return 0; 3755 } 3756 } else { 3757 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 3758 return 0; 3759 } 3760 3761 err: 3762 nvme_ctrlr_delete(nvme_ctrlr); 3763 return rc; 3764 } 3765 3766 void 3767 bdev_nvme_get_default_ctrlr_opts(struct nvme_ctrlr_opts *opts) 3768 { 3769 opts->prchk_flags = 0; 3770 opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec; 3771 opts->reconnect_delay_sec = g_opts.reconnect_delay_sec; 3772 opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec; 3773 } 3774 3775 static void 3776 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 3777 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts) 3778 { 3779 char *name; 3780 3781 name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++); 3782 if (!name) { 3783 SPDK_ERRLOG("Failed to assign name to NVMe device\n"); 3784 return; 3785 } 3786 3787 SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name); 3788 3789 nvme_ctrlr_create(ctrlr, name, trid, NULL); 3790 3791 free(name); 3792 } 3793 3794 static void 3795 _nvme_ctrlr_destruct(void *ctx) 3796 { 3797 struct nvme_ctrlr *nvme_ctrlr = ctx; 3798 3799 nvme_ctrlr_depopulate_namespaces(nvme_ctrlr); 3800 nvme_ctrlr_release(nvme_ctrlr); 3801 } 3802 3803 static int 3804 _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 3805 { 3806 struct nvme_probe_skip_entry *entry; 3807 3808 pthread_mutex_lock(&nvme_ctrlr->mutex); 3809 3810 /* The controller's destruction was already started */ 3811 if (nvme_ctrlr->destruct) { 3812 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3813 return 0; 3814 } 3815 3816 if (!hotplug && 3817 nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 3818 entry = calloc(1, sizeof(*entry)); 3819 if (!entry) { 3820 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3821 return -ENOMEM; 3822 } 3823 entry->trid = nvme_ctrlr->active_path_id->trid; 3824 TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq); 3825 } 3826 3827 nvme_ctrlr->destruct = true; 3828 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3829 3830 _nvme_ctrlr_destruct(nvme_ctrlr); 3831 3832 return 0; 3833 } 3834 3835 static void 3836 remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr) 3837 { 3838 struct nvme_ctrlr *nvme_ctrlr = cb_ctx; 3839 3840 _bdev_nvme_delete(nvme_ctrlr, true); 3841 } 3842 3843 static int 3844 bdev_nvme_hotplug_probe(void *arg) 3845 { 3846 if (g_hotplug_probe_ctx == NULL) { 3847 spdk_poller_unregister(&g_hotplug_probe_poller); 3848 return SPDK_POLLER_IDLE; 3849 } 3850 3851 if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) { 3852 g_hotplug_probe_ctx = NULL; 3853 spdk_poller_unregister(&g_hotplug_probe_poller); 3854 } 3855 3856 return SPDK_POLLER_BUSY; 3857 } 3858 3859 static int 3860 bdev_nvme_hotplug(void *arg) 3861 { 3862 struct spdk_nvme_transport_id trid_pcie; 3863 3864 if (g_hotplug_probe_ctx) { 3865 return SPDK_POLLER_BUSY; 3866 } 3867 3868 memset(&trid_pcie, 0, sizeof(trid_pcie)); 3869 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 3870 3871 g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL, 3872 hotplug_probe_cb, attach_cb, NULL); 3873 3874 if (g_hotplug_probe_ctx) { 3875 assert(g_hotplug_probe_poller == NULL); 3876 g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000); 3877 } 3878 3879 return SPDK_POLLER_BUSY; 3880 } 3881 3882 void 3883 bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts) 3884 { 3885 *opts = g_opts; 3886 } 3887 3888 static bool bdev_nvme_check_multipath_params(int32_t ctrlr_loss_timeout_sec, 3889 uint32_t reconnect_delay_sec, 3890 uint32_t fast_io_fail_timeout_sec); 3891 3892 static int 3893 bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts) 3894 { 3895 if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) { 3896 /* Can't set timeout_admin_us without also setting timeout_us */ 3897 SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n"); 3898 return -EINVAL; 3899 } 3900 3901 if (opts->bdev_retry_count < -1) { 3902 SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n"); 3903 return -EINVAL; 3904 } 3905 3906 if (!bdev_nvme_check_multipath_params(opts->ctrlr_loss_timeout_sec, 3907 opts->reconnect_delay_sec, 3908 opts->fast_io_fail_timeout_sec)) { 3909 return -EINVAL; 3910 } 3911 3912 return 0; 3913 } 3914 3915 int 3916 bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) 3917 { 3918 int ret = bdev_nvme_validate_opts(opts); 3919 if (ret) { 3920 SPDK_WARNLOG("Failed to set nvme opts.\n"); 3921 return ret; 3922 } 3923 3924 if (g_bdev_nvme_init_thread != NULL) { 3925 if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 3926 return -EPERM; 3927 } 3928 } 3929 3930 g_opts = *opts; 3931 3932 return 0; 3933 } 3934 3935 struct set_nvme_hotplug_ctx { 3936 uint64_t period_us; 3937 bool enabled; 3938 spdk_msg_fn fn; 3939 void *fn_ctx; 3940 }; 3941 3942 static void 3943 set_nvme_hotplug_period_cb(void *_ctx) 3944 { 3945 struct set_nvme_hotplug_ctx *ctx = _ctx; 3946 3947 spdk_poller_unregister(&g_hotplug_poller); 3948 if (ctx->enabled) { 3949 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us); 3950 } 3951 3952 g_nvme_hotplug_poll_period_us = ctx->period_us; 3953 g_nvme_hotplug_enabled = ctx->enabled; 3954 if (ctx->fn) { 3955 ctx->fn(ctx->fn_ctx); 3956 } 3957 3958 free(ctx); 3959 } 3960 3961 int 3962 bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx) 3963 { 3964 struct set_nvme_hotplug_ctx *ctx; 3965 3966 if (enabled == true && !spdk_process_is_primary()) { 3967 return -EPERM; 3968 } 3969 3970 ctx = calloc(1, sizeof(*ctx)); 3971 if (ctx == NULL) { 3972 return -ENOMEM; 3973 } 3974 3975 period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us; 3976 ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX); 3977 ctx->enabled = enabled; 3978 ctx->fn = cb; 3979 ctx->fn_ctx = cb_ctx; 3980 3981 spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx); 3982 return 0; 3983 } 3984 3985 static void 3986 nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 3987 struct nvme_async_probe_ctx *ctx) 3988 { 3989 struct nvme_ns *nvme_ns; 3990 struct nvme_bdev *nvme_bdev; 3991 size_t j; 3992 3993 assert(nvme_ctrlr != NULL); 3994 3995 if (ctx->names == NULL) { 3996 populate_namespaces_cb(ctx, 0, 0); 3997 return; 3998 } 3999 4000 /* 4001 * Report the new bdevs that were created in this call. 4002 * There can be more than one bdev per NVMe controller. 4003 */ 4004 j = 0; 4005 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4006 while (nvme_ns != NULL) { 4007 nvme_bdev = nvme_ns->bdev; 4008 if (j < ctx->count) { 4009 ctx->names[j] = nvme_bdev->disk.name; 4010 j++; 4011 } else { 4012 SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n", 4013 ctx->count); 4014 populate_namespaces_cb(ctx, 0, -ERANGE); 4015 return; 4016 } 4017 4018 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 4019 } 4020 4021 populate_namespaces_cb(ctx, j, 0); 4022 } 4023 4024 static int 4025 bdev_nvme_compare_trids(struct nvme_ctrlr *nvme_ctrlr, 4026 struct spdk_nvme_ctrlr *new_ctrlr, 4027 struct spdk_nvme_transport_id *trid) 4028 { 4029 struct nvme_path_id *tmp_trid; 4030 4031 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 4032 SPDK_ERRLOG("PCIe failover is not supported.\n"); 4033 return -ENOTSUP; 4034 } 4035 4036 /* Currently we only support failover to the same transport type. */ 4037 if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) { 4038 return -EINVAL; 4039 } 4040 4041 /* Currently we only support failover to the same NQN. */ 4042 if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) { 4043 return -EINVAL; 4044 } 4045 4046 /* Skip all the other checks if we've already registered this path. */ 4047 TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) { 4048 if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) { 4049 return -EEXIST; 4050 } 4051 } 4052 4053 return 0; 4054 } 4055 4056 static int 4057 bdev_nvme_compare_namespaces(struct nvme_ctrlr *nvme_ctrlr, 4058 struct spdk_nvme_ctrlr *new_ctrlr) 4059 { 4060 struct nvme_ns *nvme_ns; 4061 struct spdk_nvme_ns *new_ns; 4062 4063 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4064 while (nvme_ns != NULL) { 4065 new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id); 4066 assert(new_ns != NULL); 4067 4068 if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) { 4069 return -EINVAL; 4070 } 4071 4072 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 4073 } 4074 4075 return 0; 4076 } 4077 4078 static int 4079 _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 4080 struct spdk_nvme_transport_id *trid) 4081 { 4082 struct nvme_path_id *new_trid, *tmp_trid; 4083 4084 new_trid = calloc(1, sizeof(*new_trid)); 4085 if (new_trid == NULL) { 4086 return -ENOMEM; 4087 } 4088 new_trid->trid = *trid; 4089 new_trid->is_failed = false; 4090 4091 TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) { 4092 if (tmp_trid->is_failed && tmp_trid != nvme_ctrlr->active_path_id) { 4093 TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link); 4094 return 0; 4095 } 4096 } 4097 4098 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link); 4099 return 0; 4100 } 4101 4102 /* This is the case that a secondary path is added to an existing 4103 * nvme_ctrlr for failover. After checking if it can access the same 4104 * namespaces as the primary path, it is disconnected until failover occurs. 4105 */ 4106 static int 4107 bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 4108 struct spdk_nvme_ctrlr *new_ctrlr, 4109 struct spdk_nvme_transport_id *trid) 4110 { 4111 int rc; 4112 4113 assert(nvme_ctrlr != NULL); 4114 4115 pthread_mutex_lock(&nvme_ctrlr->mutex); 4116 4117 rc = bdev_nvme_compare_trids(nvme_ctrlr, new_ctrlr, trid); 4118 if (rc != 0) { 4119 goto exit; 4120 } 4121 4122 rc = bdev_nvme_compare_namespaces(nvme_ctrlr, new_ctrlr); 4123 if (rc != 0) { 4124 goto exit; 4125 } 4126 4127 rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid); 4128 4129 exit: 4130 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4131 4132 spdk_nvme_detach(new_ctrlr); 4133 4134 return rc; 4135 } 4136 4137 static void 4138 connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 4139 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 4140 { 4141 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 4142 struct nvme_async_probe_ctx *ctx; 4143 int rc; 4144 4145 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 4146 ctx->ctrlr_attached = true; 4147 4148 rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx); 4149 if (rc != 0) { 4150 populate_namespaces_cb(ctx, 0, rc); 4151 } 4152 } 4153 4154 static void 4155 connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 4156 struct spdk_nvme_ctrlr *ctrlr, 4157 const struct spdk_nvme_ctrlr_opts *opts) 4158 { 4159 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 4160 struct nvme_ctrlr *nvme_ctrlr; 4161 struct nvme_async_probe_ctx *ctx; 4162 int rc; 4163 4164 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 4165 ctx->ctrlr_attached = true; 4166 4167 nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name); 4168 if (nvme_ctrlr) { 4169 rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid); 4170 } else { 4171 rc = -ENODEV; 4172 } 4173 4174 populate_namespaces_cb(ctx, 0, rc); 4175 } 4176 4177 static int 4178 bdev_nvme_async_poll(void *arg) 4179 { 4180 struct nvme_async_probe_ctx *ctx = arg; 4181 int rc; 4182 4183 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 4184 if (spdk_unlikely(rc != -EAGAIN)) { 4185 ctx->probe_done = true; 4186 spdk_poller_unregister(&ctx->poller); 4187 if (!ctx->ctrlr_attached) { 4188 /* The probe is done, but no controller was attached. 4189 * That means we had a failure, so report -EIO back to 4190 * the caller (usually the RPC). populate_namespaces_cb() 4191 * will take care of freeing the nvme_async_probe_ctx. 4192 */ 4193 populate_namespaces_cb(ctx, 0, -EIO); 4194 } else if (ctx->namespaces_populated) { 4195 /* The namespaces for the attached controller were all 4196 * populated and the response was already sent to the 4197 * caller (usually the RPC). So free the context here. 4198 */ 4199 free(ctx); 4200 } 4201 } 4202 4203 return SPDK_POLLER_BUSY; 4204 } 4205 4206 static bool 4207 bdev_nvme_check_multipath_params(int32_t ctrlr_loss_timeout_sec, 4208 uint32_t reconnect_delay_sec, 4209 uint32_t fast_io_fail_timeout_sec) 4210 { 4211 if (ctrlr_loss_timeout_sec < -1) { 4212 SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n"); 4213 return false; 4214 } else if (ctrlr_loss_timeout_sec == -1) { 4215 if (reconnect_delay_sec == 0) { 4216 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 4217 return false; 4218 } else if (fast_io_fail_timeout_sec != 0 && 4219 fast_io_fail_timeout_sec < reconnect_delay_sec) { 4220 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n"); 4221 return false; 4222 } 4223 } else if (ctrlr_loss_timeout_sec != 0) { 4224 if (reconnect_delay_sec == 0) { 4225 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 4226 return false; 4227 } else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) { 4228 SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n"); 4229 return false; 4230 } else if (fast_io_fail_timeout_sec != 0) { 4231 if (fast_io_fail_timeout_sec < reconnect_delay_sec) { 4232 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n"); 4233 return false; 4234 } else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) { 4235 SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n"); 4236 return false; 4237 } 4238 } 4239 } else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) { 4240 SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n"); 4241 return false; 4242 } 4243 4244 return true; 4245 } 4246 4247 int 4248 bdev_nvme_create(struct spdk_nvme_transport_id *trid, 4249 const char *base_name, 4250 const char **names, 4251 uint32_t count, 4252 spdk_bdev_create_nvme_fn cb_fn, 4253 void *cb_ctx, 4254 struct spdk_nvme_ctrlr_opts *drv_opts, 4255 struct nvme_ctrlr_opts *bdev_opts, 4256 bool multipath) 4257 { 4258 struct nvme_probe_skip_entry *entry, *tmp; 4259 struct nvme_async_probe_ctx *ctx; 4260 spdk_nvme_attach_cb attach_cb; 4261 4262 /* TODO expand this check to include both the host and target TRIDs. 4263 * Only if both are the same should we fail. 4264 */ 4265 if (nvme_ctrlr_get(trid) != NULL) { 4266 SPDK_ERRLOG("A controller with the provided trid (traddr: %s) already exists.\n", trid->traddr); 4267 return -EEXIST; 4268 } 4269 4270 if (bdev_opts != NULL && 4271 !bdev_nvme_check_multipath_params(bdev_opts->ctrlr_loss_timeout_sec, 4272 bdev_opts->reconnect_delay_sec, 4273 bdev_opts->fast_io_fail_timeout_sec)) { 4274 return -EINVAL; 4275 } 4276 4277 ctx = calloc(1, sizeof(*ctx)); 4278 if (!ctx) { 4279 return -ENOMEM; 4280 } 4281 ctx->base_name = base_name; 4282 ctx->names = names; 4283 ctx->count = count; 4284 ctx->cb_fn = cb_fn; 4285 ctx->cb_ctx = cb_ctx; 4286 ctx->trid = *trid; 4287 4288 if (bdev_opts) { 4289 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 4290 } else { 4291 bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts); 4292 } 4293 4294 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 4295 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) { 4296 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 4297 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 4298 free(entry); 4299 break; 4300 } 4301 } 4302 } 4303 4304 if (drv_opts) { 4305 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 4306 } else { 4307 spdk_nvme_ctrlr_get_default_ctrlr_opts(&ctx->drv_opts, sizeof(ctx->drv_opts)); 4308 } 4309 4310 ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count; 4311 ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout; 4312 ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms; 4313 ctx->drv_opts.disable_read_ana_log_page = true; 4314 4315 if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || multipath) { 4316 attach_cb = connect_attach_cb; 4317 } else { 4318 attach_cb = connect_set_failover_cb; 4319 } 4320 4321 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb); 4322 if (ctx->probe_ctx == NULL) { 4323 SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr); 4324 free(ctx); 4325 return -ENODEV; 4326 } 4327 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000); 4328 4329 return 0; 4330 } 4331 4332 int 4333 bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id) 4334 { 4335 struct nvme_bdev_ctrlr *nbdev_ctrlr; 4336 struct nvme_ctrlr *nvme_ctrlr, *tmp_nvme_ctrlr; 4337 struct nvme_path_id *p, *t; 4338 int rc = -ENXIO; 4339 4340 if (name == NULL || path_id == NULL) { 4341 return -EINVAL; 4342 } 4343 4344 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 4345 if (nbdev_ctrlr == NULL) { 4346 SPDK_ERRLOG("Failed to find NVMe bdev controller\n"); 4347 return -ENODEV; 4348 } 4349 4350 TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) { 4351 TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) { 4352 if (path_id->trid.trtype != 0) { 4353 if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) { 4354 if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) { 4355 continue; 4356 } 4357 } else { 4358 if (path_id->trid.trtype != p->trid.trtype) { 4359 continue; 4360 } 4361 } 4362 } 4363 4364 if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) { 4365 if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) { 4366 continue; 4367 } 4368 } 4369 4370 if (path_id->trid.adrfam != 0) { 4371 if (path_id->trid.adrfam != p->trid.adrfam) { 4372 continue; 4373 } 4374 } 4375 4376 if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) { 4377 if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) { 4378 continue; 4379 } 4380 } 4381 4382 if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) { 4383 if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) { 4384 continue; 4385 } 4386 } 4387 4388 if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) { 4389 if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) { 4390 continue; 4391 } 4392 } 4393 4394 if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) { 4395 if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) { 4396 continue; 4397 } 4398 } 4399 4400 /* If we made it here, then this path is a match! Now we need to remove it. */ 4401 if (p == nvme_ctrlr->active_path_id) { 4402 /* This is the active path in use right now. The active path is always the first in the list. */ 4403 4404 if (!TAILQ_NEXT(p, link)) { 4405 /* The current path is the only path. */ 4406 rc = _bdev_nvme_delete(nvme_ctrlr, false); 4407 } else { 4408 /* There is an alternative path. */ 4409 rc = bdev_nvme_failover(nvme_ctrlr, true); 4410 } 4411 } else { 4412 /* We are not using the specified path. */ 4413 TAILQ_REMOVE(&nvme_ctrlr->trids, p, link); 4414 free(p); 4415 rc = 0; 4416 } 4417 4418 if (rc < 0 && rc != -ENXIO) { 4419 return rc; 4420 } 4421 4422 4423 } 4424 } 4425 4426 /* All nvme_ctrlrs were deleted or no nvme_ctrlr which had the trid was found. */ 4427 return rc; 4428 } 4429 4430 #define DISCOVERY_INFOLOG(ctx, format, ...) \ 4431 SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 4432 4433 #define DISCOVERY_ERRLOG(ctx, format, ...) \ 4434 SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 4435 4436 struct discovery_entry_ctx { 4437 char name[128]; 4438 struct spdk_nvme_transport_id trid; 4439 struct spdk_nvme_ctrlr_opts drv_opts; 4440 struct spdk_nvmf_discovery_log_page_entry entry; 4441 TAILQ_ENTRY(discovery_entry_ctx) tailq; 4442 struct discovery_ctx *ctx; 4443 }; 4444 4445 struct discovery_ctx { 4446 char *name; 4447 spdk_bdev_nvme_start_discovery_fn start_cb_fn; 4448 spdk_bdev_nvme_stop_discovery_fn stop_cb_fn; 4449 void *cb_ctx; 4450 struct spdk_nvme_probe_ctx *probe_ctx; 4451 struct spdk_nvme_detach_ctx *detach_ctx; 4452 struct spdk_nvme_ctrlr *ctrlr; 4453 struct spdk_nvme_transport_id trid; 4454 struct discovery_entry_ctx *entry_ctx_in_use; 4455 struct spdk_poller *poller; 4456 struct spdk_nvme_ctrlr_opts drv_opts; 4457 struct nvme_ctrlr_opts bdev_opts; 4458 struct spdk_nvmf_discovery_log_page *log_page; 4459 TAILQ_ENTRY(discovery_ctx) tailq; 4460 TAILQ_HEAD(, discovery_entry_ctx) nvm_entry_ctxs; 4461 TAILQ_HEAD(, discovery_entry_ctx) discovery_entry_ctxs; 4462 int rc; 4463 bool wait_for_attach; 4464 /* Denotes if a discovery is currently in progress for this context. 4465 * That includes connecting to newly discovered subsystems. Used to 4466 * ensure we do not start a new discovery until an existing one is 4467 * complete. 4468 */ 4469 bool in_progress; 4470 4471 /* Denotes if another discovery is needed after the one in progress 4472 * completes. Set when we receive an AER completion while a discovery 4473 * is already in progress. 4474 */ 4475 bool pending; 4476 4477 /* Signal to the discovery context poller that it should stop the 4478 * discovery service, including detaching from the current discovery 4479 * controller. 4480 */ 4481 bool stop; 4482 4483 struct spdk_thread *calling_thread; 4484 uint32_t index; 4485 uint32_t attach_in_progress; 4486 char *hostnqn; 4487 }; 4488 4489 TAILQ_HEAD(discovery_ctxs, discovery_ctx); 4490 static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs); 4491 4492 static void get_discovery_log_page(struct discovery_ctx *ctx); 4493 4494 static void 4495 free_discovery_ctx(struct discovery_ctx *ctx) 4496 { 4497 free(ctx->hostnqn); 4498 free(ctx->name); 4499 free(ctx); 4500 } 4501 4502 static void 4503 discovery_complete(struct discovery_ctx *ctx) 4504 { 4505 ctx->in_progress = false; 4506 if (ctx->pending) { 4507 ctx->pending = false; 4508 get_discovery_log_page(ctx); 4509 } 4510 } 4511 4512 static void 4513 build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid, 4514 struct spdk_nvmf_discovery_log_page_entry *entry) 4515 { 4516 char *space; 4517 4518 trid->trtype = entry->trtype; 4519 trid->adrfam = entry->adrfam; 4520 memcpy(trid->traddr, entry->traddr, sizeof(trid->traddr)); 4521 memcpy(trid->trsvcid, entry->trsvcid, sizeof(trid->trsvcid)); 4522 memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn)); 4523 4524 /* We want the traddr, trsvcid and subnqn fields to be NULL-terminated. 4525 * But the log page entries typically pad them with spaces, not zeroes. 4526 * So add a NULL terminator to each of these fields at the appropriate 4527 * location. 4528 */ 4529 space = strchr(trid->traddr, ' '); 4530 if (space) { 4531 *space = 0; 4532 } 4533 space = strchr(trid->trsvcid, ' '); 4534 if (space) { 4535 *space = 0; 4536 } 4537 space = strchr(trid->subnqn, ' '); 4538 if (space) { 4539 *space = 0; 4540 } 4541 } 4542 4543 static void 4544 discovery_remove_controllers(struct discovery_ctx *ctx) 4545 { 4546 struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page; 4547 struct discovery_entry_ctx *entry_ctx, *tmp; 4548 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 4549 struct spdk_nvme_transport_id old_trid; 4550 uint64_t numrec, i; 4551 bool found; 4552 4553 numrec = from_le64(&log_page->numrec); 4554 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) { 4555 found = false; 4556 old_entry = &entry_ctx->entry; 4557 build_trid_from_log_page_entry(&old_trid, old_entry); 4558 for (i = 0; i < numrec; i++) { 4559 new_entry = &log_page->entries[i]; 4560 if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) { 4561 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n", 4562 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 4563 found = true; 4564 break; 4565 } 4566 } 4567 if (!found) { 4568 struct nvme_path_id path = {}; 4569 4570 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n", 4571 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 4572 4573 path.trid = entry_ctx->trid; 4574 bdev_nvme_delete(entry_ctx->name, &path); 4575 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 4576 free(entry_ctx); 4577 } 4578 } 4579 free(log_page); 4580 ctx->log_page = NULL; 4581 discovery_complete(ctx); 4582 } 4583 4584 static void 4585 discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc) 4586 { 4587 struct discovery_entry_ctx *entry_ctx = cb_ctx; 4588 struct discovery_ctx *ctx = entry_ctx->ctx;; 4589 4590 DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name); 4591 ctx->attach_in_progress--; 4592 if (ctx->attach_in_progress == 0) { 4593 if (ctx->start_cb_fn) { 4594 ctx->start_cb_fn(ctx->cb_ctx); 4595 ctx->start_cb_fn = NULL; 4596 ctx->cb_ctx = NULL; 4597 } 4598 discovery_remove_controllers(ctx); 4599 } 4600 } 4601 4602 static struct discovery_entry_ctx * 4603 create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid) 4604 { 4605 struct discovery_entry_ctx *new_ctx; 4606 4607 new_ctx = calloc(1, sizeof(*new_ctx)); 4608 if (new_ctx == NULL) { 4609 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 4610 return NULL; 4611 } 4612 4613 new_ctx->ctx = ctx; 4614 memcpy(&new_ctx->trid, trid, sizeof(*trid)); 4615 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 4616 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 4617 return new_ctx; 4618 } 4619 4620 static void 4621 discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl, 4622 struct spdk_nvmf_discovery_log_page *log_page) 4623 { 4624 struct discovery_ctx *ctx = cb_arg; 4625 struct discovery_entry_ctx *entry_ctx, *tmp; 4626 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 4627 uint64_t numrec, i; 4628 bool found; 4629 4630 if (rc || spdk_nvme_cpl_is_error(cpl)) { 4631 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 4632 return; 4633 } 4634 4635 ctx->log_page = log_page; 4636 assert(ctx->attach_in_progress == 0); 4637 numrec = from_le64(&log_page->numrec); 4638 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) { 4639 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 4640 free(entry_ctx); 4641 } 4642 for (i = 0; i < numrec; i++) { 4643 found = false; 4644 new_entry = &log_page->entries[i]; 4645 if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 4646 struct discovery_entry_ctx *new_ctx; 4647 struct spdk_nvme_transport_id trid; 4648 4649 build_trid_from_log_page_entry(&trid, new_entry); 4650 new_ctx = create_discovery_entry_ctx(ctx, &trid); 4651 if (new_ctx == NULL) { 4652 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 4653 break; 4654 } 4655 4656 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq); 4657 continue; 4658 } 4659 TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) { 4660 old_entry = &entry_ctx->entry; 4661 if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) { 4662 found = true; 4663 break; 4664 } 4665 } 4666 if (!found) { 4667 struct discovery_entry_ctx *subnqn_ctx, *new_ctx; 4668 4669 TAILQ_FOREACH(subnqn_ctx, &ctx->nvm_entry_ctxs, tailq) { 4670 if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn, 4671 sizeof(new_entry->subnqn))) { 4672 break; 4673 } 4674 } 4675 4676 new_ctx = calloc(1, sizeof(*new_ctx)); 4677 if (new_ctx == NULL) { 4678 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 4679 break; 4680 } 4681 4682 new_ctx->ctx = ctx; 4683 memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry)); 4684 build_trid_from_log_page_entry(&new_ctx->trid, new_entry); 4685 if (subnqn_ctx) { 4686 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name); 4687 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n", 4688 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 4689 new_ctx->name); 4690 } else { 4691 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++); 4692 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n", 4693 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 4694 new_ctx->name); 4695 } 4696 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 4697 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 4698 rc = bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0, 4699 discovery_attach_controller_done, new_ctx, 4700 &new_ctx->drv_opts, &ctx->bdev_opts, true); 4701 if (rc == 0) { 4702 TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq); 4703 ctx->attach_in_progress++; 4704 } else { 4705 DISCOVERY_ERRLOG(ctx, "bdev_nvme_create failed (%s)\n", spdk_strerror(-rc)); 4706 } 4707 } 4708 } 4709 4710 if (ctx->attach_in_progress == 0) { 4711 discovery_remove_controllers(ctx); 4712 } 4713 } 4714 4715 static void 4716 get_discovery_log_page(struct discovery_ctx *ctx) 4717 { 4718 int rc; 4719 4720 assert(ctx->in_progress == false); 4721 ctx->in_progress = true; 4722 rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx); 4723 if (rc != 0) { 4724 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 4725 } 4726 DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n"); 4727 } 4728 4729 static void 4730 discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 4731 { 4732 struct discovery_ctx *ctx = arg; 4733 uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16; 4734 4735 if (spdk_nvme_cpl_is_error(cpl)) { 4736 DISCOVERY_ERRLOG(ctx, "aer failed\n"); 4737 return; 4738 } 4739 4740 if (log_page_id != SPDK_NVME_LOG_DISCOVERY) { 4741 DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id); 4742 return; 4743 } 4744 4745 DISCOVERY_INFOLOG(ctx, "got aer\n"); 4746 if (ctx->in_progress) { 4747 ctx->pending = true; 4748 return; 4749 } 4750 4751 get_discovery_log_page(ctx); 4752 } 4753 4754 static void 4755 discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 4756 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 4757 { 4758 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 4759 struct discovery_ctx *ctx; 4760 4761 ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts); 4762 4763 DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n"); 4764 ctx->probe_ctx = NULL; 4765 ctx->ctrlr = ctrlr; 4766 spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx); 4767 } 4768 4769 static int 4770 discovery_poller(void *arg) 4771 { 4772 struct discovery_ctx *ctx = arg; 4773 struct spdk_nvme_transport_id *trid; 4774 int rc; 4775 4776 if (ctx->detach_ctx) { 4777 rc = spdk_nvme_detach_poll_async(ctx->detach_ctx); 4778 if (rc != -EAGAIN) { 4779 ctx->detach_ctx = NULL; 4780 ctx->ctrlr = NULL; 4781 } 4782 } else if (ctx->stop) { 4783 if (ctx->ctrlr != NULL) { 4784 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 4785 if (rc == 0) { 4786 return SPDK_POLLER_BUSY; 4787 } 4788 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 4789 } 4790 spdk_poller_unregister(&ctx->poller); 4791 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 4792 ctx->stop_cb_fn(ctx->cb_ctx); 4793 free_discovery_ctx(ctx); 4794 } else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) { 4795 assert(ctx->entry_ctx_in_use == NULL); 4796 ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 4797 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 4798 trid = &ctx->entry_ctx_in_use->trid; 4799 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb); 4800 if (ctx->probe_ctx) { 4801 spdk_poller_unregister(&ctx->poller); 4802 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000); 4803 } else { 4804 DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n"); 4805 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 4806 ctx->entry_ctx_in_use = NULL; 4807 } 4808 } else if (ctx->probe_ctx) { 4809 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 4810 if (rc != -EAGAIN) { 4811 DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n"); 4812 ctx->rc = rc; 4813 if (rc == 0) { 4814 get_discovery_log_page(ctx); 4815 } 4816 } 4817 } else { 4818 rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr); 4819 if (rc < 0) { 4820 spdk_poller_unregister(&ctx->poller); 4821 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 4822 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 4823 ctx->entry_ctx_in_use = NULL; 4824 4825 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 4826 if (rc != 0) { 4827 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 4828 ctx->ctrlr = NULL; 4829 } 4830 } 4831 } 4832 4833 return SPDK_POLLER_BUSY; 4834 } 4835 4836 static void 4837 start_discovery_poller(void *arg) 4838 { 4839 struct discovery_ctx *ctx = arg; 4840 4841 TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq); 4842 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 4843 } 4844 4845 int 4846 bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid, 4847 const char *base_name, 4848 struct spdk_nvme_ctrlr_opts *drv_opts, 4849 struct nvme_ctrlr_opts *bdev_opts, 4850 spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx) 4851 { 4852 struct discovery_ctx *ctx; 4853 struct discovery_entry_ctx *discovery_entry_ctx; 4854 4855 ctx = calloc(1, sizeof(*ctx)); 4856 if (ctx == NULL) { 4857 return -ENOMEM; 4858 } 4859 4860 ctx->name = strdup(base_name); 4861 if (ctx->name == NULL) { 4862 free_discovery_ctx(ctx); 4863 return -ENOMEM; 4864 } 4865 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 4866 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 4867 ctx->bdev_opts.from_discovery_service = true; 4868 ctx->calling_thread = spdk_get_thread(); 4869 if (ctx->start_cb_fn) { 4870 /* We can use this when dumping json to denote if this RPC parameter 4871 * was specified or not. 4872 */ 4873 ctx->wait_for_attach = true; 4874 } 4875 ctx->start_cb_fn = cb_fn; 4876 ctx->cb_ctx = cb_ctx; 4877 TAILQ_INIT(&ctx->nvm_entry_ctxs); 4878 TAILQ_INIT(&ctx->discovery_entry_ctxs); 4879 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 4880 memcpy(&ctx->trid, trid, sizeof(*trid)); 4881 /* Even if user did not specify hostnqn, we can still strdup("\0"); */ 4882 ctx->hostnqn = strdup(ctx->drv_opts.hostnqn); 4883 if (ctx->hostnqn == NULL) { 4884 free_discovery_ctx(ctx); 4885 return -ENOMEM; 4886 } 4887 discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid); 4888 if (discovery_entry_ctx == NULL) { 4889 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 4890 free_discovery_ctx(ctx); 4891 return -ENOMEM; 4892 } 4893 4894 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq); 4895 spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx); 4896 return 0; 4897 } 4898 4899 int 4900 bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 4901 { 4902 struct discovery_ctx *ctx; 4903 4904 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 4905 if (strcmp(name, ctx->name) == 0) { 4906 if (ctx->stop) { 4907 return -EALREADY; 4908 } 4909 ctx->stop = true; 4910 ctx->stop_cb_fn = cb_fn; 4911 ctx->cb_ctx = cb_ctx; 4912 while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) { 4913 struct discovery_entry_ctx *entry_ctx; 4914 struct nvme_path_id path = {}; 4915 4916 entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs); 4917 path.trid = entry_ctx->trid; 4918 bdev_nvme_delete(entry_ctx->name, &path); 4919 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 4920 free(entry_ctx); 4921 } 4922 while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) { 4923 struct discovery_entry_ctx *entry_ctx; 4924 4925 entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 4926 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 4927 free(entry_ctx); 4928 } 4929 return 0; 4930 } 4931 } 4932 4933 return -ENOENT; 4934 } 4935 4936 static int 4937 bdev_nvme_library_init(void) 4938 { 4939 g_bdev_nvme_init_thread = spdk_get_thread(); 4940 4941 spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb, 4942 bdev_nvme_destroy_poll_group_cb, 4943 sizeof(struct nvme_poll_group), "nvme_poll_groups"); 4944 4945 return 0; 4946 } 4947 4948 static void 4949 bdev_nvme_fini_destruct_ctrlrs(void) 4950 { 4951 struct nvme_bdev_ctrlr *nbdev_ctrlr; 4952 struct nvme_ctrlr *nvme_ctrlr; 4953 4954 pthread_mutex_lock(&g_bdev_nvme_mutex); 4955 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 4956 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 4957 pthread_mutex_lock(&nvme_ctrlr->mutex); 4958 if (nvme_ctrlr->destruct) { 4959 /* This controller's destruction was already started 4960 * before the application started shutting down 4961 */ 4962 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4963 continue; 4964 } 4965 nvme_ctrlr->destruct = true; 4966 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4967 4968 spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct, 4969 nvme_ctrlr); 4970 } 4971 } 4972 4973 g_bdev_nvme_module_finish = true; 4974 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 4975 pthread_mutex_unlock(&g_bdev_nvme_mutex); 4976 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 4977 spdk_bdev_module_fini_done(); 4978 return; 4979 } 4980 4981 pthread_mutex_unlock(&g_bdev_nvme_mutex); 4982 } 4983 4984 static void 4985 check_discovery_fini(void *arg) 4986 { 4987 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 4988 bdev_nvme_fini_destruct_ctrlrs(); 4989 } 4990 } 4991 4992 static void 4993 bdev_nvme_library_fini(void) 4994 { 4995 struct nvme_probe_skip_entry *entry, *entry_tmp; 4996 struct discovery_ctx *ctx; 4997 4998 spdk_poller_unregister(&g_hotplug_poller); 4999 free(g_hotplug_probe_ctx); 5000 g_hotplug_probe_ctx = NULL; 5001 5002 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) { 5003 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 5004 free(entry); 5005 } 5006 5007 assert(spdk_get_thread() == g_bdev_nvme_init_thread); 5008 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 5009 bdev_nvme_fini_destruct_ctrlrs(); 5010 } else { 5011 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 5012 ctx->stop = true; 5013 ctx->stop_cb_fn = check_discovery_fini; 5014 } 5015 } 5016 } 5017 5018 static void 5019 bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio) 5020 { 5021 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 5022 struct spdk_bdev *bdev = bdev_io->bdev; 5023 struct spdk_dif_ctx dif_ctx; 5024 struct spdk_dif_error err_blk = {}; 5025 int rc; 5026 5027 rc = spdk_dif_ctx_init(&dif_ctx, 5028 bdev->blocklen, bdev->md_len, bdev->md_interleave, 5029 bdev->dif_is_head_of_md, bdev->dif_type, bdev->dif_check_flags, 5030 bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0); 5031 if (rc != 0) { 5032 SPDK_ERRLOG("Initialization of DIF context failed\n"); 5033 return; 5034 } 5035 5036 if (bdev->md_interleave) { 5037 rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 5038 bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 5039 } else { 5040 struct iovec md_iov = { 5041 .iov_base = bdev_io->u.bdev.md_buf, 5042 .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len, 5043 }; 5044 5045 rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 5046 &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 5047 } 5048 5049 if (rc != 0) { 5050 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", 5051 err_blk.err_type, err_blk.err_offset); 5052 } else { 5053 SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n"); 5054 } 5055 } 5056 5057 static void 5058 bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 5059 { 5060 struct nvme_bdev_io *bio = ref; 5061 5062 if (spdk_nvme_cpl_is_success(cpl)) { 5063 /* Run PI verification for read data buffer. */ 5064 bdev_nvme_verify_pi_error(bio); 5065 } 5066 5067 /* Return original completion status */ 5068 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 5069 } 5070 5071 static void 5072 bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 5073 { 5074 struct nvme_bdev_io *bio = ref; 5075 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 5076 int ret; 5077 5078 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 5079 SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n", 5080 cpl->status.sct, cpl->status.sc); 5081 5082 /* Save completion status to use after verifying PI error. */ 5083 bio->cpl = *cpl; 5084 5085 if (spdk_likely(nvme_io_path_is_available(bio->io_path))) { 5086 /* Read without PI checking to verify PI error. */ 5087 ret = bdev_nvme_no_pi_readv(bio, 5088 bdev_io->u.bdev.iovs, 5089 bdev_io->u.bdev.iovcnt, 5090 bdev_io->u.bdev.md_buf, 5091 bdev_io->u.bdev.num_blocks, 5092 bdev_io->u.bdev.offset_blocks); 5093 if (ret == 0) { 5094 return; 5095 } 5096 } 5097 } 5098 5099 bdev_nvme_io_complete_nvme_status(bio, cpl); 5100 } 5101 5102 static void 5103 bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 5104 { 5105 struct nvme_bdev_io *bio = ref; 5106 5107 if (spdk_nvme_cpl_is_pi_error(cpl)) { 5108 SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n", 5109 cpl->status.sct, cpl->status.sc); 5110 /* Run PI verification for write data buffer if PI error is detected. */ 5111 bdev_nvme_verify_pi_error(bio); 5112 } 5113 5114 bdev_nvme_io_complete_nvme_status(bio, cpl); 5115 } 5116 5117 static void 5118 bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl) 5119 { 5120 struct nvme_bdev_io *bio = ref; 5121 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 5122 5123 /* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks. 5124 * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error(). 5125 */ 5126 bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0; 5127 5128 if (spdk_nvme_cpl_is_pi_error(cpl)) { 5129 SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n", 5130 cpl->status.sct, cpl->status.sc); 5131 /* Run PI verification for zone append data buffer if PI error is detected. */ 5132 bdev_nvme_verify_pi_error(bio); 5133 } 5134 5135 bdev_nvme_io_complete_nvme_status(bio, cpl); 5136 } 5137 5138 static void 5139 bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl) 5140 { 5141 struct nvme_bdev_io *bio = ref; 5142 5143 if (spdk_nvme_cpl_is_pi_error(cpl)) { 5144 SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n", 5145 cpl->status.sct, cpl->status.sc); 5146 /* Run PI verification for compare data buffer if PI error is detected. */ 5147 bdev_nvme_verify_pi_error(bio); 5148 } 5149 5150 bdev_nvme_io_complete_nvme_status(bio, cpl); 5151 } 5152 5153 static void 5154 bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 5155 { 5156 struct nvme_bdev_io *bio = ref; 5157 5158 /* Compare operation completion */ 5159 if (!bio->first_fused_completed) { 5160 /* Save compare result for write callback */ 5161 bio->cpl = *cpl; 5162 bio->first_fused_completed = true; 5163 return; 5164 } 5165 5166 /* Write operation completion */ 5167 if (spdk_nvme_cpl_is_error(&bio->cpl)) { 5168 /* If bio->cpl is already an error, it means the compare operation failed. In that case, 5169 * complete the IO with the compare operation's status. 5170 */ 5171 if (!spdk_nvme_cpl_is_error(cpl)) { 5172 SPDK_ERRLOG("Unexpected write success after compare failure.\n"); 5173 } 5174 5175 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 5176 } else { 5177 bdev_nvme_io_complete_nvme_status(bio, cpl); 5178 } 5179 } 5180 5181 static void 5182 bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl) 5183 { 5184 struct nvme_bdev_io *bio = ref; 5185 5186 bdev_nvme_io_complete_nvme_status(bio, cpl); 5187 } 5188 5189 static int 5190 fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc) 5191 { 5192 switch (desc->zs) { 5193 case SPDK_NVME_ZONE_STATE_EMPTY: 5194 info->state = SPDK_BDEV_ZONE_STATE_EMPTY; 5195 break; 5196 case SPDK_NVME_ZONE_STATE_IOPEN: 5197 info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN; 5198 break; 5199 case SPDK_NVME_ZONE_STATE_EOPEN: 5200 info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN; 5201 break; 5202 case SPDK_NVME_ZONE_STATE_CLOSED: 5203 info->state = SPDK_BDEV_ZONE_STATE_CLOSED; 5204 break; 5205 case SPDK_NVME_ZONE_STATE_RONLY: 5206 info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY; 5207 break; 5208 case SPDK_NVME_ZONE_STATE_FULL: 5209 info->state = SPDK_BDEV_ZONE_STATE_FULL; 5210 break; 5211 case SPDK_NVME_ZONE_STATE_OFFLINE: 5212 info->state = SPDK_BDEV_ZONE_STATE_OFFLINE; 5213 break; 5214 default: 5215 SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs); 5216 return -EIO; 5217 } 5218 5219 info->zone_id = desc->zslba; 5220 info->write_pointer = desc->wp; 5221 info->capacity = desc->zcap; 5222 5223 return 0; 5224 } 5225 5226 static void 5227 bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl) 5228 { 5229 struct nvme_bdev_io *bio = ref; 5230 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 5231 uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id; 5232 uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones; 5233 struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf; 5234 uint64_t max_zones_per_buf, i; 5235 uint32_t zone_report_bufsize; 5236 struct spdk_nvme_ns *ns; 5237 struct spdk_nvme_qpair *qpair; 5238 int ret; 5239 5240 if (spdk_nvme_cpl_is_error(cpl)) { 5241 goto out_complete_io_nvme_cpl; 5242 } 5243 5244 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 5245 ret = -ENXIO; 5246 goto out_complete_io_ret; 5247 } 5248 5249 ns = bio->io_path->nvme_ns->ns; 5250 qpair = bio->io_path->qpair->qpair; 5251 5252 zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 5253 max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) / 5254 sizeof(bio->zone_report_buf->descs[0]); 5255 5256 if (bio->zone_report_buf->nr_zones > max_zones_per_buf) { 5257 ret = -EINVAL; 5258 goto out_complete_io_ret; 5259 } 5260 5261 if (!bio->zone_report_buf->nr_zones) { 5262 ret = -EINVAL; 5263 goto out_complete_io_ret; 5264 } 5265 5266 for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) { 5267 ret = fill_zone_from_report(&info[bio->handled_zones], 5268 &bio->zone_report_buf->descs[i]); 5269 if (ret) { 5270 goto out_complete_io_ret; 5271 } 5272 bio->handled_zones++; 5273 } 5274 5275 if (bio->handled_zones < zones_to_copy) { 5276 uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 5277 uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones); 5278 5279 memset(bio->zone_report_buf, 0, zone_report_bufsize); 5280 ret = spdk_nvme_zns_report_zones(ns, qpair, 5281 bio->zone_report_buf, zone_report_bufsize, 5282 slba, SPDK_NVME_ZRA_LIST_ALL, true, 5283 bdev_nvme_get_zone_info_done, bio); 5284 if (!ret) { 5285 return; 5286 } else { 5287 goto out_complete_io_ret; 5288 } 5289 } 5290 5291 out_complete_io_nvme_cpl: 5292 free(bio->zone_report_buf); 5293 bio->zone_report_buf = NULL; 5294 bdev_nvme_io_complete_nvme_status(bio, cpl); 5295 return; 5296 5297 out_complete_io_ret: 5298 free(bio->zone_report_buf); 5299 bio->zone_report_buf = NULL; 5300 bdev_nvme_io_complete(bio, ret); 5301 } 5302 5303 static void 5304 bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl) 5305 { 5306 struct nvme_bdev_io *bio = ref; 5307 5308 bdev_nvme_io_complete_nvme_status(bio, cpl); 5309 } 5310 5311 static void 5312 bdev_nvme_admin_passthru_complete_nvme_status(void *ctx) 5313 { 5314 struct nvme_bdev_io *bio = ctx; 5315 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 5316 const struct spdk_nvme_cpl *cpl = &bio->cpl; 5317 struct nvme_bdev_channel *nbdev_ch; 5318 struct nvme_ctrlr *nvme_ctrlr; 5319 const struct spdk_nvme_ctrlr_data *cdata; 5320 uint64_t delay_ms; 5321 5322 assert(bdev_nvme_io_type_is_admin(bdev_io->type)); 5323 5324 if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) { 5325 goto complete; 5326 } 5327 5328 if (cpl->status.dnr != 0 || (g_opts.bdev_retry_count != -1 && 5329 bio->retry_count >= g_opts.bdev_retry_count)) { 5330 goto complete; 5331 } 5332 5333 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 5334 nvme_ctrlr = bio->io_path->qpair->ctrlr; 5335 5336 if (spdk_nvme_cpl_is_path_error(cpl) || 5337 spdk_nvme_cpl_is_aborted_sq_deletion(cpl) || 5338 !nvme_ctrlr_is_available(nvme_ctrlr)) { 5339 delay_ms = 0; 5340 } else if (spdk_nvme_cpl_is_aborted_by_request(cpl)) { 5341 goto complete; 5342 } else { 5343 bio->retry_count++; 5344 5345 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 5346 5347 if (cpl->status.crd != 0) { 5348 delay_ms = cdata->crdt[cpl->status.crd] * 100; 5349 } else { 5350 delay_ms = 0; 5351 } 5352 } 5353 5354 if (any_ctrlr_may_become_available(nbdev_ch)) { 5355 bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms); 5356 return; 5357 } 5358 5359 complete: 5360 bio->retry_count = 0; 5361 spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc); 5362 } 5363 5364 static void 5365 bdev_nvme_abort_complete(void *ctx) 5366 { 5367 struct nvme_bdev_io *bio = ctx; 5368 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 5369 5370 if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) { 5371 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); 5372 } else { 5373 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 5374 } 5375 } 5376 5377 static void 5378 bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl) 5379 { 5380 struct nvme_bdev_io *bio = ref; 5381 5382 bio->cpl = *cpl; 5383 spdk_thread_send_msg(bio->orig_thread, bdev_nvme_abort_complete, bio); 5384 } 5385 5386 static void 5387 bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl) 5388 { 5389 struct nvme_bdev_io *bio = ref; 5390 5391 bio->cpl = *cpl; 5392 spdk_thread_send_msg(bio->orig_thread, 5393 bdev_nvme_admin_passthru_complete_nvme_status, bio); 5394 } 5395 5396 static void 5397 bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset) 5398 { 5399 struct nvme_bdev_io *bio = ref; 5400 struct iovec *iov; 5401 5402 bio->iov_offset = sgl_offset; 5403 for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) { 5404 iov = &bio->iovs[bio->iovpos]; 5405 if (bio->iov_offset < iov->iov_len) { 5406 break; 5407 } 5408 5409 bio->iov_offset -= iov->iov_len; 5410 } 5411 } 5412 5413 static int 5414 bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length) 5415 { 5416 struct nvme_bdev_io *bio = ref; 5417 struct iovec *iov; 5418 5419 assert(bio->iovpos < bio->iovcnt); 5420 5421 iov = &bio->iovs[bio->iovpos]; 5422 5423 *address = iov->iov_base; 5424 *length = iov->iov_len; 5425 5426 if (bio->iov_offset) { 5427 assert(bio->iov_offset <= iov->iov_len); 5428 *address += bio->iov_offset; 5429 *length -= bio->iov_offset; 5430 } 5431 5432 bio->iov_offset += *length; 5433 if (bio->iov_offset == iov->iov_len) { 5434 bio->iovpos++; 5435 bio->iov_offset = 0; 5436 } 5437 5438 return 0; 5439 } 5440 5441 static void 5442 bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset) 5443 { 5444 struct nvme_bdev_io *bio = ref; 5445 struct iovec *iov; 5446 5447 bio->fused_iov_offset = sgl_offset; 5448 for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) { 5449 iov = &bio->fused_iovs[bio->fused_iovpos]; 5450 if (bio->fused_iov_offset < iov->iov_len) { 5451 break; 5452 } 5453 5454 bio->fused_iov_offset -= iov->iov_len; 5455 } 5456 } 5457 5458 static int 5459 bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length) 5460 { 5461 struct nvme_bdev_io *bio = ref; 5462 struct iovec *iov; 5463 5464 assert(bio->fused_iovpos < bio->fused_iovcnt); 5465 5466 iov = &bio->fused_iovs[bio->fused_iovpos]; 5467 5468 *address = iov->iov_base; 5469 *length = iov->iov_len; 5470 5471 if (bio->fused_iov_offset) { 5472 assert(bio->fused_iov_offset <= iov->iov_len); 5473 *address += bio->fused_iov_offset; 5474 *length -= bio->fused_iov_offset; 5475 } 5476 5477 bio->fused_iov_offset += *length; 5478 if (bio->fused_iov_offset == iov->iov_len) { 5479 bio->fused_iovpos++; 5480 bio->fused_iov_offset = 0; 5481 } 5482 5483 return 0; 5484 } 5485 5486 static int 5487 bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 5488 void *md, uint64_t lba_count, uint64_t lba) 5489 { 5490 int rc; 5491 5492 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n", 5493 lba_count, lba); 5494 5495 bio->iovs = iov; 5496 bio->iovcnt = iovcnt; 5497 bio->iovpos = 0; 5498 bio->iov_offset = 0; 5499 5500 rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns, 5501 bio->io_path->qpair->qpair, 5502 lba, lba_count, 5503 bdev_nvme_no_pi_readv_done, bio, 0, 5504 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 5505 md, 0, 0); 5506 5507 if (rc != 0 && rc != -ENOMEM) { 5508 SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc); 5509 } 5510 return rc; 5511 } 5512 5513 static int 5514 bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 5515 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 5516 struct spdk_bdev_ext_io_opts *ext_opts) 5517 { 5518 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 5519 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 5520 int rc; 5521 5522 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n", 5523 lba_count, lba); 5524 5525 bio->iovs = iov; 5526 bio->iovcnt = iovcnt; 5527 bio->iovpos = 0; 5528 bio->iov_offset = 0; 5529 5530 if (ext_opts) { 5531 bio->ext_opts.size = sizeof(struct spdk_nvme_ns_cmd_ext_io_opts); 5532 bio->ext_opts.memory_domain = ext_opts->memory_domain; 5533 bio->ext_opts.memory_domain_ctx = ext_opts->memory_domain_ctx; 5534 bio->ext_opts.io_flags = flags; 5535 bio->ext_opts.metadata = md; 5536 5537 rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count, 5538 bdev_nvme_readv_done, bio, 5539 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 5540 &bio->ext_opts); 5541 } else if (iovcnt == 1) { 5542 rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, iov[0].iov_base, md, lba, 5543 lba_count, 5544 bdev_nvme_readv_done, bio, 5545 flags, 5546 0, 0); 5547 } else { 5548 rc = spdk_nvme_ns_cmd_readv_with_md(ns, qpair, lba, lba_count, 5549 bdev_nvme_readv_done, bio, flags, 5550 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 5551 md, 0, 0); 5552 } 5553 5554 if (rc != 0 && rc != -ENOMEM) { 5555 SPDK_ERRLOG("readv failed: rc = %d\n", rc); 5556 } 5557 return rc; 5558 } 5559 5560 static int 5561 bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 5562 void *md, uint64_t lba_count, uint64_t lba, 5563 uint32_t flags, struct spdk_bdev_ext_io_opts *ext_opts) 5564 { 5565 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 5566 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 5567 int rc; 5568 5569 SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 5570 lba_count, lba); 5571 5572 bio->iovs = iov; 5573 bio->iovcnt = iovcnt; 5574 bio->iovpos = 0; 5575 bio->iov_offset = 0; 5576 5577 if (ext_opts) { 5578 bio->ext_opts.size = sizeof(struct spdk_nvme_ns_cmd_ext_io_opts); 5579 bio->ext_opts.memory_domain = ext_opts->memory_domain; 5580 bio->ext_opts.memory_domain_ctx = ext_opts->memory_domain_ctx; 5581 bio->ext_opts.io_flags = flags; 5582 bio->ext_opts.metadata = md; 5583 5584 rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count, 5585 bdev_nvme_writev_done, bio, 5586 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 5587 &bio->ext_opts); 5588 } else if (iovcnt == 1) { 5589 rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, iov[0].iov_base, md, lba, 5590 lba_count, 5591 bdev_nvme_writev_done, bio, 5592 flags, 5593 0, 0); 5594 } else { 5595 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 5596 bdev_nvme_writev_done, bio, flags, 5597 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 5598 md, 0, 0); 5599 } 5600 5601 if (rc != 0 && rc != -ENOMEM) { 5602 SPDK_ERRLOG("writev failed: rc = %d\n", rc); 5603 } 5604 return rc; 5605 } 5606 5607 static int 5608 bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 5609 void *md, uint64_t lba_count, uint64_t zslba, 5610 uint32_t flags) 5611 { 5612 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 5613 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 5614 int rc; 5615 5616 SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n", 5617 lba_count, zslba); 5618 5619 bio->iovs = iov; 5620 bio->iovcnt = iovcnt; 5621 bio->iovpos = 0; 5622 bio->iov_offset = 0; 5623 5624 if (iovcnt == 1) { 5625 rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba, 5626 lba_count, 5627 bdev_nvme_zone_appendv_done, bio, 5628 flags, 5629 0, 0); 5630 } else { 5631 rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count, 5632 bdev_nvme_zone_appendv_done, bio, flags, 5633 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 5634 md, 0, 0); 5635 } 5636 5637 if (rc != 0 && rc != -ENOMEM) { 5638 SPDK_ERRLOG("zone append failed: rc = %d\n", rc); 5639 } 5640 return rc; 5641 } 5642 5643 static int 5644 bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 5645 void *md, uint64_t lba_count, uint64_t lba, 5646 uint32_t flags) 5647 { 5648 int rc; 5649 5650 SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n", 5651 lba_count, lba); 5652 5653 bio->iovs = iov; 5654 bio->iovcnt = iovcnt; 5655 bio->iovpos = 0; 5656 bio->iov_offset = 0; 5657 5658 rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns, 5659 bio->io_path->qpair->qpair, 5660 lba, lba_count, 5661 bdev_nvme_comparev_done, bio, flags, 5662 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 5663 md, 0, 0); 5664 5665 if (rc != 0 && rc != -ENOMEM) { 5666 SPDK_ERRLOG("comparev failed: rc = %d\n", rc); 5667 } 5668 return rc; 5669 } 5670 5671 static int 5672 bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt, 5673 struct iovec *write_iov, int write_iovcnt, 5674 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags) 5675 { 5676 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 5677 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 5678 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 5679 int rc; 5680 5681 SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 5682 lba_count, lba); 5683 5684 bio->iovs = cmp_iov; 5685 bio->iovcnt = cmp_iovcnt; 5686 bio->iovpos = 0; 5687 bio->iov_offset = 0; 5688 bio->fused_iovs = write_iov; 5689 bio->fused_iovcnt = write_iovcnt; 5690 bio->fused_iovpos = 0; 5691 bio->fused_iov_offset = 0; 5692 5693 if (bdev_io->num_retries == 0) { 5694 bio->first_fused_submitted = false; 5695 bio->first_fused_completed = false; 5696 } 5697 5698 if (!bio->first_fused_submitted) { 5699 flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST; 5700 memset(&bio->cpl, 0, sizeof(bio->cpl)); 5701 5702 rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count, 5703 bdev_nvme_comparev_and_writev_done, bio, flags, 5704 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0); 5705 if (rc == 0) { 5706 bio->first_fused_submitted = true; 5707 flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST; 5708 } else { 5709 if (rc != -ENOMEM) { 5710 SPDK_ERRLOG("compare failed: rc = %d\n", rc); 5711 } 5712 return rc; 5713 } 5714 } 5715 5716 flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND; 5717 5718 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 5719 bdev_nvme_comparev_and_writev_done, bio, flags, 5720 bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0); 5721 if (rc != 0 && rc != -ENOMEM) { 5722 SPDK_ERRLOG("write failed: rc = %d\n", rc); 5723 rc = 0; 5724 } 5725 5726 return rc; 5727 } 5728 5729 static int 5730 bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 5731 { 5732 struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES]; 5733 struct spdk_nvme_dsm_range *range; 5734 uint64_t offset, remaining; 5735 uint64_t num_ranges_u64; 5736 uint16_t num_ranges; 5737 int rc; 5738 5739 num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) / 5740 SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 5741 if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) { 5742 SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks); 5743 return -EINVAL; 5744 } 5745 num_ranges = (uint16_t)num_ranges_u64; 5746 5747 offset = offset_blocks; 5748 remaining = num_blocks; 5749 range = &dsm_ranges[0]; 5750 5751 /* Fill max-size ranges until the remaining blocks fit into one range */ 5752 while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) { 5753 range->attributes.raw = 0; 5754 range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 5755 range->starting_lba = offset; 5756 5757 offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 5758 remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 5759 range++; 5760 } 5761 5762 /* Final range describes the remaining blocks */ 5763 range->attributes.raw = 0; 5764 range->length = remaining; 5765 range->starting_lba = offset; 5766 5767 rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns, 5768 bio->io_path->qpair->qpair, 5769 SPDK_NVME_DSM_ATTR_DEALLOCATE, 5770 dsm_ranges, num_ranges, 5771 bdev_nvme_queued_done, bio); 5772 5773 return rc; 5774 } 5775 5776 static int 5777 bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 5778 { 5779 if (num_blocks > UINT16_MAX + 1) { 5780 SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n"); 5781 return -EINVAL; 5782 } 5783 5784 return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns, 5785 bio->io_path->qpair->qpair, 5786 offset_blocks, num_blocks, 5787 bdev_nvme_queued_done, bio, 5788 0); 5789 } 5790 5791 static int 5792 bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones, 5793 struct spdk_bdev_zone_info *info) 5794 { 5795 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 5796 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 5797 uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 5798 uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 5799 uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns); 5800 5801 if (zone_id % zone_size != 0) { 5802 return -EINVAL; 5803 } 5804 5805 if (num_zones > total_zones || !num_zones) { 5806 return -EINVAL; 5807 } 5808 5809 assert(!bio->zone_report_buf); 5810 bio->zone_report_buf = calloc(1, zone_report_bufsize); 5811 if (!bio->zone_report_buf) { 5812 return -ENOMEM; 5813 } 5814 5815 bio->handled_zones = 0; 5816 5817 return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize, 5818 zone_id, SPDK_NVME_ZRA_LIST_ALL, true, 5819 bdev_nvme_get_zone_info_done, bio); 5820 } 5821 5822 static int 5823 bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 5824 enum spdk_bdev_zone_action action) 5825 { 5826 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 5827 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 5828 5829 switch (action) { 5830 case SPDK_BDEV_ZONE_CLOSE: 5831 return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false, 5832 bdev_nvme_zone_management_done, bio); 5833 case SPDK_BDEV_ZONE_FINISH: 5834 return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false, 5835 bdev_nvme_zone_management_done, bio); 5836 case SPDK_BDEV_ZONE_OPEN: 5837 return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false, 5838 bdev_nvme_zone_management_done, bio); 5839 case SPDK_BDEV_ZONE_RESET: 5840 return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false, 5841 bdev_nvme_zone_management_done, bio); 5842 case SPDK_BDEV_ZONE_OFFLINE: 5843 return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false, 5844 bdev_nvme_zone_management_done, bio); 5845 default: 5846 return -EINVAL; 5847 } 5848 } 5849 5850 static void 5851 bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 5852 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes) 5853 { 5854 struct nvme_io_path *io_path; 5855 struct nvme_ctrlr *nvme_ctrlr; 5856 uint32_t max_xfer_size; 5857 int rc = -ENXIO; 5858 5859 /* Choose the first ctrlr which is not failed. */ 5860 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 5861 nvme_ctrlr = io_path->qpair->ctrlr; 5862 5863 /* We should skip any unavailable nvme_ctrlr rather than checking 5864 * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO. 5865 */ 5866 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 5867 continue; 5868 } 5869 5870 max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr); 5871 5872 if (nbytes > max_xfer_size) { 5873 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 5874 rc = -EINVAL; 5875 goto err; 5876 } 5877 5878 bio->io_path = io_path; 5879 bio->orig_thread = spdk_get_thread(); 5880 5881 rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes, 5882 bdev_nvme_admin_passthru_done, bio); 5883 if (rc == 0) { 5884 return; 5885 } 5886 } 5887 5888 err: 5889 bdev_nvme_admin_passthru_complete(bio, rc); 5890 } 5891 5892 static int 5893 bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 5894 void *buf, size_t nbytes) 5895 { 5896 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 5897 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 5898 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 5899 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 5900 5901 if (nbytes > max_xfer_size) { 5902 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 5903 return -EINVAL; 5904 } 5905 5906 /* 5907 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 5908 * so fill it out automatically. 5909 */ 5910 cmd->nsid = spdk_nvme_ns_get_id(ns); 5911 5912 return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf, 5913 (uint32_t)nbytes, bdev_nvme_queued_done, bio); 5914 } 5915 5916 static int 5917 bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 5918 void *buf, size_t nbytes, void *md_buf, size_t md_len) 5919 { 5920 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 5921 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 5922 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 5923 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 5924 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 5925 5926 if (nbytes > max_xfer_size) { 5927 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 5928 return -EINVAL; 5929 } 5930 5931 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 5932 SPDK_ERRLOG("invalid meta data buffer size\n"); 5933 return -EINVAL; 5934 } 5935 5936 /* 5937 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 5938 * so fill it out automatically. 5939 */ 5940 cmd->nsid = spdk_nvme_ns_get_id(ns); 5941 5942 return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf, 5943 (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio); 5944 } 5945 5946 static void 5947 bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 5948 struct nvme_bdev_io *bio_to_abort) 5949 { 5950 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 5951 struct spdk_bdev_io *bdev_io_to_abort; 5952 struct nvme_io_path *io_path; 5953 struct nvme_ctrlr *nvme_ctrlr; 5954 int rc = 0; 5955 5956 bio->orig_thread = spdk_get_thread(); 5957 5958 /* Traverse the retry_io_list first. */ 5959 TAILQ_FOREACH(bdev_io_to_abort, &nbdev_ch->retry_io_list, module_link) { 5960 if ((struct nvme_bdev_io *)bdev_io_to_abort->driver_ctx == bio_to_abort) { 5961 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io_to_abort, module_link); 5962 spdk_bdev_io_complete(bdev_io_to_abort, SPDK_BDEV_IO_STATUS_ABORTED); 5963 5964 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); 5965 return; 5966 } 5967 } 5968 5969 /* Even admin commands, they were submitted to only nvme_ctrlrs which were 5970 * on any io_path. So traverse the io_path list for not only I/O commands 5971 * but also admin commands. 5972 */ 5973 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 5974 nvme_ctrlr = io_path->qpair->ctrlr; 5975 5976 rc = spdk_nvme_ctrlr_cmd_abort_ext(nvme_ctrlr->ctrlr, 5977 io_path->qpair->qpair, 5978 bio_to_abort, 5979 bdev_nvme_abort_done, bio); 5980 if (rc == -ENOENT) { 5981 /* If no command was found in I/O qpair, the target command may be 5982 * admin command. 5983 */ 5984 rc = spdk_nvme_ctrlr_cmd_abort_ext(nvme_ctrlr->ctrlr, 5985 NULL, 5986 bio_to_abort, 5987 bdev_nvme_abort_done, bio); 5988 } 5989 5990 if (rc != -ENOENT) { 5991 break; 5992 } 5993 } 5994 5995 if (rc != 0) { 5996 /* If no command was found or there was any error, complete the abort 5997 * request with failure. 5998 */ 5999 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 6000 } 6001 } 6002 6003 static void 6004 bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w) 6005 { 6006 const char *action; 6007 6008 if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) { 6009 action = "reset"; 6010 } else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) { 6011 action = "abort"; 6012 } else { 6013 action = "none"; 6014 } 6015 6016 spdk_json_write_object_begin(w); 6017 6018 spdk_json_write_named_string(w, "method", "bdev_nvme_set_options"); 6019 6020 spdk_json_write_named_object_begin(w, "params"); 6021 spdk_json_write_named_string(w, "action_on_timeout", action); 6022 spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us); 6023 spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us); 6024 spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms); 6025 spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count); 6026 spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst); 6027 spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight); 6028 spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight); 6029 spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight); 6030 spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us); 6031 spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us); 6032 spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests); 6033 spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit); 6034 spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count); 6035 spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout); 6036 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec); 6037 spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec); 6038 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec); 6039 spdk_json_write_object_end(w); 6040 6041 spdk_json_write_object_end(w); 6042 } 6043 6044 static void 6045 bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx) 6046 { 6047 struct spdk_nvme_transport_id trid; 6048 6049 spdk_json_write_object_begin(w); 6050 6051 spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery"); 6052 6053 spdk_json_write_named_object_begin(w, "params"); 6054 spdk_json_write_named_string(w, "name", ctx->name); 6055 spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn); 6056 6057 trid = ctx->trid; 6058 memset(trid.subnqn, 0, sizeof(trid.subnqn)); 6059 nvme_bdev_dump_trid_json(&trid, w); 6060 6061 spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach); 6062 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec); 6063 spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec); 6064 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 6065 ctx->bdev_opts.fast_io_fail_timeout_sec); 6066 spdk_json_write_object_end(w); 6067 6068 spdk_json_write_object_end(w); 6069 } 6070 6071 static void 6072 nvme_ctrlr_config_json(struct spdk_json_write_ctx *w, 6073 struct nvme_ctrlr *nvme_ctrlr) 6074 { 6075 struct spdk_nvme_transport_id *trid; 6076 6077 if (nvme_ctrlr->opts.from_discovery_service) { 6078 /* Do not emit an RPC for this - it will be implicitly 6079 * covered by a separate bdev_nvme_start_discovery RPC. 6080 */ 6081 return; 6082 } 6083 6084 trid = &nvme_ctrlr->active_path_id->trid; 6085 6086 spdk_json_write_object_begin(w); 6087 6088 spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller"); 6089 6090 spdk_json_write_named_object_begin(w, "params"); 6091 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 6092 nvme_bdev_dump_trid_json(trid, w); 6093 spdk_json_write_named_bool(w, "prchk_reftag", 6094 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0); 6095 spdk_json_write_named_bool(w, "prchk_guard", 6096 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0); 6097 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec); 6098 spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec); 6099 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 6100 nvme_ctrlr->opts.fast_io_fail_timeout_sec); 6101 6102 spdk_json_write_object_end(w); 6103 6104 spdk_json_write_object_end(w); 6105 } 6106 6107 static void 6108 bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w) 6109 { 6110 spdk_json_write_object_begin(w); 6111 spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug"); 6112 6113 spdk_json_write_named_object_begin(w, "params"); 6114 spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us); 6115 spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled); 6116 spdk_json_write_object_end(w); 6117 6118 spdk_json_write_object_end(w); 6119 } 6120 6121 static int 6122 bdev_nvme_config_json(struct spdk_json_write_ctx *w) 6123 { 6124 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6125 struct nvme_ctrlr *nvme_ctrlr; 6126 struct discovery_ctx *ctx; 6127 6128 bdev_nvme_opts_config_json(w); 6129 6130 pthread_mutex_lock(&g_bdev_nvme_mutex); 6131 6132 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 6133 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 6134 nvme_ctrlr_config_json(w, nvme_ctrlr); 6135 } 6136 } 6137 6138 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 6139 bdev_nvme_discovery_config_json(w, ctx); 6140 } 6141 6142 /* Dump as last parameter to give all NVMe bdevs chance to be constructed 6143 * before enabling hotplug poller. 6144 */ 6145 bdev_nvme_hotplug_config_json(w); 6146 6147 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6148 return 0; 6149 } 6150 6151 struct spdk_nvme_ctrlr * 6152 bdev_nvme_get_ctrlr(struct spdk_bdev *bdev) 6153 { 6154 struct nvme_bdev *nbdev; 6155 struct nvme_ns *nvme_ns; 6156 6157 if (!bdev || bdev->module != &nvme_if) { 6158 return NULL; 6159 } 6160 6161 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 6162 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 6163 assert(nvme_ns != NULL); 6164 6165 return nvme_ns->ctrlr->ctrlr; 6166 } 6167 6168 void 6169 nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path) 6170 { 6171 struct nvme_ns *nvme_ns = io_path->nvme_ns; 6172 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 6173 const struct spdk_nvme_ctrlr_data *cdata; 6174 6175 spdk_json_write_object_begin(w); 6176 6177 spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name); 6178 6179 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 6180 6181 spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid); 6182 6183 spdk_json_write_named_bool(w, "current", io_path == io_path->nbdev_ch->current_io_path); 6184 6185 spdk_json_write_named_bool(w, "connected", nvme_io_path_is_connected(io_path)); 6186 6187 spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns)); 6188 6189 spdk_json_write_object_end(w); 6190 } 6191 6192 SPDK_LOG_REGISTER_COMPONENT(bdev_nvme) 6193