1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved. 6 */ 7 8 #include "spdk/stdinc.h" 9 10 #include "bdev_nvme.h" 11 12 #include "spdk/accel.h" 13 #include "spdk/config.h" 14 #include "spdk/endian.h" 15 #include "spdk/bdev.h" 16 #include "spdk/json.h" 17 #include "spdk/likely.h" 18 #include "spdk/nvme.h" 19 #include "spdk/nvme_ocssd.h" 20 #include "spdk/nvme_zns.h" 21 #include "spdk/opal.h" 22 #include "spdk/thread.h" 23 #include "spdk/trace.h" 24 #include "spdk/string.h" 25 #include "spdk/util.h" 26 #include "spdk/uuid.h" 27 28 #include "spdk/bdev_module.h" 29 #include "spdk/log.h" 30 31 #include "spdk_internal/usdt.h" 32 #include "spdk_internal/trace_defs.h" 33 34 #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true 35 #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS (10000) 36 37 #define NSID_STR_LEN 10 38 39 static int bdev_nvme_config_json(struct spdk_json_write_ctx *w); 40 41 struct nvme_bdev_io { 42 /** array of iovecs to transfer. */ 43 struct iovec *iovs; 44 45 /** Number of iovecs in iovs array. */ 46 int iovcnt; 47 48 /** Current iovec position. */ 49 int iovpos; 50 51 /** Offset in current iovec. */ 52 uint32_t iov_offset; 53 54 /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path 55 * being reset in a reset I/O. 56 */ 57 struct nvme_io_path *io_path; 58 59 /** array of iovecs to transfer. */ 60 struct iovec *fused_iovs; 61 62 /** Number of iovecs in iovs array. */ 63 int fused_iovcnt; 64 65 /** Current iovec position. */ 66 int fused_iovpos; 67 68 /** Offset in current iovec. */ 69 uint32_t fused_iov_offset; 70 71 /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */ 72 struct spdk_nvme_cpl cpl; 73 74 /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */ 75 struct spdk_nvme_ns_cmd_ext_io_opts ext_opts; 76 77 /** Originating thread */ 78 struct spdk_thread *orig_thread; 79 80 /** Keeps track if first of fused commands was submitted */ 81 bool first_fused_submitted; 82 83 /** Keeps track if first of fused commands was completed */ 84 bool first_fused_completed; 85 86 /** Temporary pointer to zone report buffer */ 87 struct spdk_nvme_zns_zone_report *zone_report_buf; 88 89 /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */ 90 uint64_t handled_zones; 91 92 /** Expiration value in ticks to retry the current I/O. */ 93 uint64_t retry_ticks; 94 95 /* How many times the current I/O was retried. */ 96 int32_t retry_count; 97 98 /* Current tsc at submit time. */ 99 uint64_t submit_tsc; 100 }; 101 102 struct nvme_probe_skip_entry { 103 struct spdk_nvme_transport_id trid; 104 TAILQ_ENTRY(nvme_probe_skip_entry) tailq; 105 }; 106 /* All the controllers deleted by users via RPC are skipped by hotplug monitor */ 107 static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER( 108 g_skipped_nvme_ctrlrs); 109 110 static struct spdk_bdev_nvme_opts g_opts = { 111 .action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE, 112 .timeout_us = 0, 113 .timeout_admin_us = 0, 114 .keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS, 115 .transport_retry_count = 4, 116 .arbitration_burst = 0, 117 .low_priority_weight = 0, 118 .medium_priority_weight = 0, 119 .high_priority_weight = 0, 120 .nvme_adminq_poll_period_us = 10000ULL, 121 .nvme_ioq_poll_period_us = 0, 122 .io_queue_requests = 0, 123 .delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT, 124 .bdev_retry_count = 3, 125 .transport_ack_timeout = 0, 126 .ctrlr_loss_timeout_sec = 0, 127 .reconnect_delay_sec = 0, 128 .fast_io_fail_timeout_sec = 0, 129 .disable_auto_failback = false, 130 .generate_uuids = false, 131 .transport_tos = 0, 132 .nvme_error_stat = false, 133 .io_path_stat = false, 134 }; 135 136 #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL 137 #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL 138 139 static int g_hot_insert_nvme_controller_index = 0; 140 static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT; 141 static bool g_nvme_hotplug_enabled = false; 142 struct spdk_thread *g_bdev_nvme_init_thread; 143 static struct spdk_poller *g_hotplug_poller; 144 static struct spdk_poller *g_hotplug_probe_poller; 145 static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx; 146 147 static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 148 struct nvme_async_probe_ctx *ctx); 149 static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 150 struct nvme_async_probe_ctx *ctx); 151 static int bdev_nvme_library_init(void); 152 static void bdev_nvme_library_fini(void); 153 static void _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, 154 struct spdk_bdev_io *bdev_io); 155 static void bdev_nvme_submit_request(struct spdk_io_channel *ch, 156 struct spdk_bdev_io *bdev_io); 157 static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 158 void *md, uint64_t lba_count, uint64_t lba, 159 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx); 160 static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 161 void *md, uint64_t lba_count, uint64_t lba); 162 static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 163 void *md, uint64_t lba_count, uint64_t lba, 164 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx); 165 static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 166 void *md, uint64_t lba_count, 167 uint64_t zslba, uint32_t flags); 168 static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 169 void *md, uint64_t lba_count, uint64_t lba, 170 uint32_t flags); 171 static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, 172 struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov, 173 int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba, 174 uint32_t flags); 175 static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, 176 uint32_t num_zones, struct spdk_bdev_zone_info *info); 177 static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 178 enum spdk_bdev_zone_action action); 179 static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, 180 struct nvme_bdev_io *bio, 181 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes); 182 static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 183 void *buf, size_t nbytes); 184 static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 185 void *buf, size_t nbytes, void *md_buf, size_t md_len); 186 static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, 187 struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort); 188 static void bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio); 189 static int bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr); 190 static int bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove); 191 static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr); 192 static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr); 193 194 static struct nvme_ns *nvme_ns_alloc(void); 195 static void nvme_ns_free(struct nvme_ns *ns); 196 197 static int 198 nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2) 199 { 200 return ns1->id < ns2->id ? -1 : ns1->id > ns2->id; 201 } 202 203 RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp); 204 205 struct spdk_nvme_qpair * 206 bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch) 207 { 208 struct nvme_ctrlr_channel *ctrlr_ch; 209 210 assert(ctrlr_io_ch != NULL); 211 212 ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch); 213 214 return ctrlr_ch->qpair->qpair; 215 } 216 217 static int 218 bdev_nvme_get_ctx_size(void) 219 { 220 return sizeof(struct nvme_bdev_io); 221 } 222 223 static struct spdk_bdev_module nvme_if = { 224 .name = "nvme", 225 .async_fini = true, 226 .module_init = bdev_nvme_library_init, 227 .module_fini = bdev_nvme_library_fini, 228 .config_json = bdev_nvme_config_json, 229 .get_ctx_size = bdev_nvme_get_ctx_size, 230 231 }; 232 SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if) 233 234 struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs); 235 pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER; 236 bool g_bdev_nvme_module_finish; 237 238 struct nvme_bdev_ctrlr * 239 nvme_bdev_ctrlr_get_by_name(const char *name) 240 { 241 struct nvme_bdev_ctrlr *nbdev_ctrlr; 242 243 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 244 if (strcmp(name, nbdev_ctrlr->name) == 0) { 245 break; 246 } 247 } 248 249 return nbdev_ctrlr; 250 } 251 252 static struct nvme_ctrlr * 253 nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr, 254 const struct spdk_nvme_transport_id *trid) 255 { 256 struct nvme_ctrlr *nvme_ctrlr; 257 258 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 259 if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0) { 260 break; 261 } 262 } 263 264 return nvme_ctrlr; 265 } 266 267 static struct nvme_bdev * 268 nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid) 269 { 270 struct nvme_bdev *bdev; 271 272 pthread_mutex_lock(&g_bdev_nvme_mutex); 273 TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) { 274 if (bdev->nsid == nsid) { 275 break; 276 } 277 } 278 pthread_mutex_unlock(&g_bdev_nvme_mutex); 279 280 return bdev; 281 } 282 283 struct nvme_ns * 284 nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid) 285 { 286 struct nvme_ns ns; 287 288 assert(nsid > 0); 289 290 ns.id = nsid; 291 return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns); 292 } 293 294 struct nvme_ns * 295 nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr) 296 { 297 return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces); 298 } 299 300 struct nvme_ns * 301 nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns) 302 { 303 if (ns == NULL) { 304 return NULL; 305 } 306 307 return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 308 } 309 310 static struct nvme_ctrlr * 311 nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid) 312 { 313 struct nvme_bdev_ctrlr *nbdev_ctrlr; 314 struct nvme_ctrlr *nvme_ctrlr = NULL; 315 316 pthread_mutex_lock(&g_bdev_nvme_mutex); 317 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 318 nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid); 319 if (nvme_ctrlr != NULL) { 320 break; 321 } 322 } 323 pthread_mutex_unlock(&g_bdev_nvme_mutex); 324 325 return nvme_ctrlr; 326 } 327 328 struct nvme_ctrlr * 329 nvme_ctrlr_get_by_name(const char *name) 330 { 331 struct nvme_bdev_ctrlr *nbdev_ctrlr; 332 struct nvme_ctrlr *nvme_ctrlr = NULL; 333 334 if (name == NULL) { 335 return NULL; 336 } 337 338 pthread_mutex_lock(&g_bdev_nvme_mutex); 339 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 340 if (nbdev_ctrlr != NULL) { 341 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 342 } 343 pthread_mutex_unlock(&g_bdev_nvme_mutex); 344 345 return nvme_ctrlr; 346 } 347 348 void 349 nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx) 350 { 351 struct nvme_bdev_ctrlr *nbdev_ctrlr; 352 353 pthread_mutex_lock(&g_bdev_nvme_mutex); 354 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 355 fn(nbdev_ctrlr, ctx); 356 } 357 pthread_mutex_unlock(&g_bdev_nvme_mutex); 358 } 359 360 void 361 nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w) 362 { 363 const char *trtype_str; 364 const char *adrfam_str; 365 366 trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype); 367 if (trtype_str) { 368 spdk_json_write_named_string(w, "trtype", trtype_str); 369 } 370 371 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 372 if (adrfam_str) { 373 spdk_json_write_named_string(w, "adrfam", adrfam_str); 374 } 375 376 if (trid->traddr[0] != '\0') { 377 spdk_json_write_named_string(w, "traddr", trid->traddr); 378 } 379 380 if (trid->trsvcid[0] != '\0') { 381 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 382 } 383 384 if (trid->subnqn[0] != '\0') { 385 spdk_json_write_named_string(w, "subnqn", trid->subnqn); 386 } 387 } 388 389 static void 390 nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr, 391 struct nvme_ctrlr *nvme_ctrlr) 392 { 393 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name); 394 pthread_mutex_lock(&g_bdev_nvme_mutex); 395 396 TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 397 if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) { 398 pthread_mutex_unlock(&g_bdev_nvme_mutex); 399 400 return; 401 } 402 TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 403 404 pthread_mutex_unlock(&g_bdev_nvme_mutex); 405 406 assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs)); 407 408 free(nbdev_ctrlr->name); 409 free(nbdev_ctrlr); 410 } 411 412 static void 413 _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 414 { 415 struct nvme_path_id *path_id, *tmp_path; 416 struct nvme_ns *ns, *tmp_ns; 417 418 free(nvme_ctrlr->copied_ana_desc); 419 spdk_free(nvme_ctrlr->ana_log_page); 420 421 if (nvme_ctrlr->opal_dev) { 422 spdk_opal_dev_destruct(nvme_ctrlr->opal_dev); 423 nvme_ctrlr->opal_dev = NULL; 424 } 425 426 if (nvme_ctrlr->nbdev_ctrlr) { 427 nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr); 428 } 429 430 RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) { 431 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 432 nvme_ns_free(ns); 433 } 434 435 TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) { 436 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 437 free(path_id); 438 } 439 440 pthread_mutex_destroy(&nvme_ctrlr->mutex); 441 442 free(nvme_ctrlr); 443 444 pthread_mutex_lock(&g_bdev_nvme_mutex); 445 if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 446 pthread_mutex_unlock(&g_bdev_nvme_mutex); 447 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 448 spdk_bdev_module_fini_done(); 449 return; 450 } 451 pthread_mutex_unlock(&g_bdev_nvme_mutex); 452 } 453 454 static int 455 nvme_detach_poller(void *arg) 456 { 457 struct nvme_ctrlr *nvme_ctrlr = arg; 458 int rc; 459 460 rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx); 461 if (rc != -EAGAIN) { 462 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 463 _nvme_ctrlr_delete(nvme_ctrlr); 464 } 465 466 return SPDK_POLLER_BUSY; 467 } 468 469 static void 470 nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 471 { 472 int rc; 473 474 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 475 476 /* First, unregister the adminq poller, as the driver will poll adminq if necessary */ 477 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 478 479 /* If we got here, the reset/detach poller cannot be active */ 480 assert(nvme_ctrlr->reset_detach_poller == NULL); 481 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller, 482 nvme_ctrlr, 1000); 483 if (nvme_ctrlr->reset_detach_poller == NULL) { 484 SPDK_ERRLOG("Failed to register detach poller\n"); 485 goto error; 486 } 487 488 rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx); 489 if (rc != 0) { 490 SPDK_ERRLOG("Failed to detach the NVMe controller\n"); 491 goto error; 492 } 493 494 return; 495 error: 496 /* We don't have a good way to handle errors here, so just do what we can and delete the 497 * controller without detaching the underlying NVMe device. 498 */ 499 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 500 _nvme_ctrlr_delete(nvme_ctrlr); 501 } 502 503 static void 504 nvme_ctrlr_unregister_cb(void *io_device) 505 { 506 struct nvme_ctrlr *nvme_ctrlr = io_device; 507 508 nvme_ctrlr_delete(nvme_ctrlr); 509 } 510 511 static void 512 nvme_ctrlr_unregister(void *ctx) 513 { 514 struct nvme_ctrlr *nvme_ctrlr = ctx; 515 516 spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb); 517 } 518 519 static bool 520 nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr) 521 { 522 if (!nvme_ctrlr->destruct) { 523 return false; 524 } 525 526 if (nvme_ctrlr->ref > 0) { 527 return false; 528 } 529 530 if (nvme_ctrlr->resetting) { 531 return false; 532 } 533 534 if (nvme_ctrlr->ana_log_page_updating) { 535 return false; 536 } 537 538 if (nvme_ctrlr->io_path_cache_clearing) { 539 return false; 540 } 541 542 return true; 543 } 544 545 static void 546 nvme_ctrlr_release(struct nvme_ctrlr *nvme_ctrlr) 547 { 548 pthread_mutex_lock(&nvme_ctrlr->mutex); 549 SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref); 550 551 assert(nvme_ctrlr->ref > 0); 552 nvme_ctrlr->ref--; 553 554 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 555 pthread_mutex_unlock(&nvme_ctrlr->mutex); 556 return; 557 } 558 559 pthread_mutex_unlock(&nvme_ctrlr->mutex); 560 561 spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr); 562 } 563 564 static void 565 bdev_nvme_clear_current_io_path(struct nvme_bdev_channel *nbdev_ch) 566 { 567 nbdev_ch->current_io_path = NULL; 568 nbdev_ch->rr_counter = 0; 569 } 570 571 static struct nvme_io_path * 572 _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 573 { 574 struct nvme_io_path *io_path; 575 576 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 577 if (io_path->nvme_ns == nvme_ns) { 578 break; 579 } 580 } 581 582 return io_path; 583 } 584 585 static int 586 _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 587 { 588 struct nvme_io_path *io_path; 589 struct spdk_io_channel *ch; 590 struct nvme_ctrlr_channel *ctrlr_ch; 591 struct nvme_qpair *nvme_qpair; 592 593 io_path = calloc(1, sizeof(*io_path)); 594 if (io_path == NULL) { 595 SPDK_ERRLOG("Failed to alloc io_path.\n"); 596 return -ENOMEM; 597 } 598 599 if (g_opts.io_path_stat) { 600 io_path->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 601 if (io_path->stat == NULL) { 602 free(io_path); 603 SPDK_ERRLOG("Failed to alloc io_path stat.\n"); 604 return -ENOMEM; 605 } 606 spdk_bdev_reset_io_stat(io_path->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 607 } 608 609 io_path->nvme_ns = nvme_ns; 610 611 ch = spdk_get_io_channel(nvme_ns->ctrlr); 612 if (ch == NULL) { 613 free(io_path->stat); 614 free(io_path); 615 SPDK_ERRLOG("Failed to alloc io_channel.\n"); 616 return -ENOMEM; 617 } 618 619 ctrlr_ch = spdk_io_channel_get_ctx(ch); 620 621 nvme_qpair = ctrlr_ch->qpair; 622 assert(nvme_qpair != NULL); 623 624 io_path->qpair = nvme_qpair; 625 TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq); 626 627 io_path->nbdev_ch = nbdev_ch; 628 STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq); 629 630 bdev_nvme_clear_current_io_path(nbdev_ch); 631 632 return 0; 633 } 634 635 static void 636 _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path) 637 { 638 struct spdk_io_channel *ch; 639 struct nvme_qpair *nvme_qpair; 640 struct nvme_ctrlr_channel *ctrlr_ch; 641 struct nvme_bdev *nbdev; 642 643 nbdev = spdk_io_channel_get_io_device(spdk_io_channel_from_ctx(nbdev_ch)); 644 645 /* Add the statistics to nvme_ns before this path is destroyed. */ 646 pthread_mutex_lock(&nbdev->mutex); 647 if (nbdev->ref != 0 && io_path->nvme_ns->stat != NULL && io_path->stat != NULL) { 648 spdk_bdev_add_io_stat(io_path->nvme_ns->stat, io_path->stat); 649 } 650 pthread_mutex_unlock(&nbdev->mutex); 651 652 bdev_nvme_clear_current_io_path(nbdev_ch); 653 654 STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq); 655 656 nvme_qpair = io_path->qpair; 657 assert(nvme_qpair != NULL); 658 659 TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq); 660 661 ctrlr_ch = nvme_qpair->ctrlr_ch; 662 assert(ctrlr_ch != NULL); 663 664 ch = spdk_io_channel_from_ctx(ctrlr_ch); 665 spdk_put_io_channel(ch); 666 667 free(io_path->stat); 668 free(io_path); 669 } 670 671 static void 672 _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch) 673 { 674 struct nvme_io_path *io_path, *tmp_io_path; 675 676 STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) { 677 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 678 } 679 } 680 681 static int 682 bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf) 683 { 684 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 685 struct nvme_bdev *nbdev = io_device; 686 struct nvme_ns *nvme_ns; 687 int rc; 688 689 STAILQ_INIT(&nbdev_ch->io_path_list); 690 TAILQ_INIT(&nbdev_ch->retry_io_list); 691 692 pthread_mutex_lock(&nbdev->mutex); 693 694 nbdev_ch->mp_policy = nbdev->mp_policy; 695 nbdev_ch->mp_selector = nbdev->mp_selector; 696 nbdev_ch->rr_min_io = nbdev->rr_min_io; 697 698 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 699 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 700 if (rc != 0) { 701 pthread_mutex_unlock(&nbdev->mutex); 702 703 _bdev_nvme_delete_io_paths(nbdev_ch); 704 return rc; 705 } 706 } 707 pthread_mutex_unlock(&nbdev->mutex); 708 709 return 0; 710 } 711 712 /* If cpl != NULL, complete the bdev_io with nvme status based on 'cpl'. 713 * If cpl == NULL, complete the bdev_io with bdev status based on 'status'. 714 */ 715 static inline void 716 __bdev_nvme_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status, 717 const struct spdk_nvme_cpl *cpl) 718 { 719 spdk_trace_record(TRACE_BDEV_NVME_IO_DONE, 0, 0, (uintptr_t)bdev_io->driver_ctx, 720 (uintptr_t)bdev_io); 721 if (cpl) { 722 spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc); 723 } else { 724 spdk_bdev_io_complete(bdev_io, status); 725 } 726 } 727 728 static void bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch); 729 730 static void 731 bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf) 732 { 733 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 734 735 bdev_nvme_abort_retry_ios(nbdev_ch); 736 _bdev_nvme_delete_io_paths(nbdev_ch); 737 } 738 739 static inline bool 740 bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type) 741 { 742 switch (io_type) { 743 case SPDK_BDEV_IO_TYPE_RESET: 744 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 745 case SPDK_BDEV_IO_TYPE_ABORT: 746 return true; 747 default: 748 break; 749 } 750 751 return false; 752 } 753 754 static inline bool 755 nvme_ns_is_accessible(struct nvme_ns *nvme_ns) 756 { 757 if (spdk_unlikely(nvme_ns->ana_state_updating)) { 758 return false; 759 } 760 761 switch (nvme_ns->ana_state) { 762 case SPDK_NVME_ANA_OPTIMIZED_STATE: 763 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 764 return true; 765 default: 766 break; 767 } 768 769 return false; 770 } 771 772 static inline bool 773 nvme_io_path_is_connected(struct nvme_io_path *io_path) 774 { 775 if (spdk_unlikely(io_path->qpair->qpair == NULL)) { 776 return false; 777 } 778 779 if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(io_path->qpair->qpair) != 780 SPDK_NVME_QPAIR_FAILURE_NONE)) { 781 return false; 782 } 783 784 if (spdk_unlikely(io_path->qpair->ctrlr_ch->reset_iter != NULL)) { 785 return false; 786 } 787 788 if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(io_path->qpair->ctrlr->ctrlr) != 789 SPDK_NVME_QPAIR_FAILURE_NONE) { 790 return false; 791 } 792 793 return true; 794 } 795 796 static inline bool 797 nvme_io_path_is_available(struct nvme_io_path *io_path) 798 { 799 if (spdk_unlikely(!nvme_io_path_is_connected(io_path))) { 800 return false; 801 } 802 803 if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) { 804 return false; 805 } 806 807 return true; 808 } 809 810 static inline bool 811 nvme_io_path_is_failed(struct nvme_io_path *io_path) 812 { 813 struct nvme_ctrlr *nvme_ctrlr; 814 815 nvme_ctrlr = io_path->qpair->ctrlr; 816 817 if (nvme_ctrlr->destruct) { 818 return true; 819 } 820 821 if (nvme_ctrlr->fast_io_fail_timedout) { 822 return true; 823 } 824 825 if (nvme_ctrlr->resetting) { 826 if (nvme_ctrlr->opts.reconnect_delay_sec != 0) { 827 return false; 828 } else { 829 return true; 830 } 831 } 832 833 if (nvme_ctrlr->reconnect_is_delayed) { 834 return false; 835 } 836 837 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 838 return true; 839 } else { 840 return false; 841 } 842 } 843 844 static bool 845 nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr) 846 { 847 if (nvme_ctrlr->destruct) { 848 return false; 849 } 850 851 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 852 return false; 853 } 854 855 if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) { 856 return false; 857 } 858 859 return true; 860 } 861 862 /* Simulate circular linked list. */ 863 static inline struct nvme_io_path * 864 nvme_io_path_get_next(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *prev_path) 865 { 866 struct nvme_io_path *next_path; 867 868 if (prev_path != NULL) { 869 next_path = STAILQ_NEXT(prev_path, stailq); 870 if (next_path != NULL) { 871 return next_path; 872 } 873 } 874 875 return STAILQ_FIRST(&nbdev_ch->io_path_list); 876 } 877 878 static struct nvme_io_path * 879 _bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 880 { 881 struct nvme_io_path *io_path, *start, *non_optimized = NULL; 882 883 start = nvme_io_path_get_next(nbdev_ch, nbdev_ch->current_io_path); 884 885 io_path = start; 886 do { 887 if (spdk_likely(nvme_io_path_is_connected(io_path) && 888 !io_path->nvme_ns->ana_state_updating)) { 889 switch (io_path->nvme_ns->ana_state) { 890 case SPDK_NVME_ANA_OPTIMIZED_STATE: 891 nbdev_ch->current_io_path = io_path; 892 return io_path; 893 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 894 if (non_optimized == NULL) { 895 non_optimized = io_path; 896 } 897 break; 898 default: 899 break; 900 } 901 } 902 io_path = nvme_io_path_get_next(nbdev_ch, io_path); 903 } while (io_path != start); 904 905 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 906 /* We come here only if there is no optimized path. Cache even non_optimized 907 * path for load balance across multiple non_optimized paths. 908 */ 909 nbdev_ch->current_io_path = non_optimized; 910 } 911 912 return non_optimized; 913 } 914 915 static struct nvme_io_path * 916 _bdev_nvme_find_io_path_min_qd(struct nvme_bdev_channel *nbdev_ch) 917 { 918 struct nvme_io_path *io_path; 919 struct nvme_io_path *optimized = NULL, *non_optimized = NULL; 920 uint32_t opt_min_qd = UINT32_MAX, non_opt_min_qd = UINT32_MAX; 921 uint32_t num_outstanding_reqs; 922 923 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 924 if (spdk_unlikely(!nvme_io_path_is_connected(io_path))) { 925 /* The device is currently resetting. */ 926 continue; 927 } 928 929 if (spdk_unlikely(io_path->nvme_ns->ana_state_updating)) { 930 continue; 931 } 932 933 num_outstanding_reqs = spdk_nvme_qpair_get_num_outstanding_reqs(io_path->qpair->qpair); 934 switch (io_path->nvme_ns->ana_state) { 935 case SPDK_NVME_ANA_OPTIMIZED_STATE: 936 if (num_outstanding_reqs < opt_min_qd) { 937 opt_min_qd = num_outstanding_reqs; 938 optimized = io_path; 939 } 940 break; 941 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 942 if (num_outstanding_reqs < non_opt_min_qd) { 943 non_opt_min_qd = num_outstanding_reqs; 944 non_optimized = io_path; 945 } 946 break; 947 default: 948 break; 949 } 950 } 951 952 /* don't cache io path for BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH selector */ 953 if (optimized != NULL) { 954 return optimized; 955 } 956 957 return non_optimized; 958 } 959 960 static inline struct nvme_io_path * 961 bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 962 { 963 if (spdk_likely(nbdev_ch->current_io_path != NULL)) { 964 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE) { 965 return nbdev_ch->current_io_path; 966 } else if (nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 967 if (++nbdev_ch->rr_counter < nbdev_ch->rr_min_io) { 968 return nbdev_ch->current_io_path; 969 } 970 nbdev_ch->rr_counter = 0; 971 } 972 } 973 974 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE || 975 nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 976 return _bdev_nvme_find_io_path(nbdev_ch); 977 } else { 978 return _bdev_nvme_find_io_path_min_qd(nbdev_ch); 979 } 980 } 981 982 /* Return true if there is any io_path whose qpair is active or ctrlr is not failed, 983 * or false otherwise. 984 * 985 * If any io_path has an active qpair but find_io_path() returned NULL, its namespace 986 * is likely to be non-accessible now but may become accessible. 987 * 988 * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr 989 * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed 990 * when starting to reset it but it is set to failed when the reset failed. Hence, if 991 * a ctrlr is unfailed, it is likely that it works fine or is resetting. 992 */ 993 static bool 994 any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch) 995 { 996 struct nvme_io_path *io_path; 997 998 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 999 if (io_path->nvme_ns->ana_transition_timedout) { 1000 continue; 1001 } 1002 1003 if (nvme_io_path_is_connected(io_path) || 1004 !nvme_io_path_is_failed(io_path)) { 1005 return true; 1006 } 1007 } 1008 1009 return false; 1010 } 1011 1012 static void 1013 bdev_nvme_retry_io(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 1014 { 1015 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1016 struct spdk_io_channel *ch; 1017 1018 if (nbdev_io->io_path != NULL && nvme_io_path_is_available(nbdev_io->io_path)) { 1019 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 1020 } else { 1021 ch = spdk_io_channel_from_ctx(nbdev_ch); 1022 bdev_nvme_submit_request(ch, bdev_io); 1023 } 1024 } 1025 1026 static int 1027 bdev_nvme_retry_ios(void *arg) 1028 { 1029 struct nvme_bdev_channel *nbdev_ch = arg; 1030 struct spdk_bdev_io *bdev_io, *tmp_bdev_io; 1031 struct nvme_bdev_io *bio; 1032 uint64_t now, delay_us; 1033 1034 now = spdk_get_ticks(); 1035 1036 TAILQ_FOREACH_SAFE(bdev_io, &nbdev_ch->retry_io_list, module_link, tmp_bdev_io) { 1037 bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1038 if (bio->retry_ticks > now) { 1039 break; 1040 } 1041 1042 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io, module_link); 1043 1044 bdev_nvme_retry_io(nbdev_ch, bdev_io); 1045 } 1046 1047 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1048 1049 bdev_io = TAILQ_FIRST(&nbdev_ch->retry_io_list); 1050 if (bdev_io != NULL) { 1051 bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1052 1053 delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz(); 1054 1055 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1056 delay_us); 1057 } 1058 1059 return SPDK_POLLER_BUSY; 1060 } 1061 1062 static void 1063 bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch, 1064 struct nvme_bdev_io *bio, uint64_t delay_ms) 1065 { 1066 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1067 struct spdk_bdev_io *tmp_bdev_io; 1068 struct nvme_bdev_io *tmp_bio; 1069 1070 bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL; 1071 1072 TAILQ_FOREACH_REVERSE(tmp_bdev_io, &nbdev_ch->retry_io_list, retry_io_head, module_link) { 1073 tmp_bio = (struct nvme_bdev_io *)tmp_bdev_io->driver_ctx; 1074 1075 if (tmp_bio->retry_ticks <= bio->retry_ticks) { 1076 TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bdev_io, bdev_io, 1077 module_link); 1078 return; 1079 } 1080 } 1081 1082 /* No earlier I/Os were found. This I/O must be the new head. */ 1083 TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bdev_io, module_link); 1084 1085 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1086 1087 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1088 delay_ms * 1000ULL); 1089 } 1090 1091 static void 1092 bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch) 1093 { 1094 struct spdk_bdev_io *bdev_io, *tmp_io; 1095 1096 TAILQ_FOREACH_SAFE(bdev_io, &nbdev_ch->retry_io_list, module_link, tmp_io) { 1097 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io, module_link); 1098 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1099 } 1100 1101 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1102 } 1103 1104 static int 1105 bdev_nvme_abort_retry_io(struct nvme_bdev_channel *nbdev_ch, 1106 struct nvme_bdev_io *bio_to_abort) 1107 { 1108 struct spdk_bdev_io *bdev_io_to_abort; 1109 1110 TAILQ_FOREACH(bdev_io_to_abort, &nbdev_ch->retry_io_list, module_link) { 1111 if ((struct nvme_bdev_io *)bdev_io_to_abort->driver_ctx == bio_to_abort) { 1112 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io_to_abort, module_link); 1113 __bdev_nvme_io_complete(bdev_io_to_abort, SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1114 return 0; 1115 } 1116 } 1117 1118 return -ENOENT; 1119 } 1120 1121 static void 1122 bdev_nvme_update_nvme_error_stat(struct spdk_bdev_io *bdev_io, const struct spdk_nvme_cpl *cpl) 1123 { 1124 struct nvme_bdev *nbdev; 1125 uint16_t sct, sc; 1126 1127 assert(spdk_nvme_cpl_is_error(cpl)); 1128 1129 nbdev = bdev_io->bdev->ctxt; 1130 1131 if (nbdev->err_stat == NULL) { 1132 return; 1133 } 1134 1135 sct = cpl->status.sct; 1136 sc = cpl->status.sc; 1137 1138 pthread_mutex_lock(&nbdev->mutex); 1139 1140 nbdev->err_stat->status_type[sct]++; 1141 switch (sct) { 1142 case SPDK_NVME_SCT_GENERIC: 1143 case SPDK_NVME_SCT_COMMAND_SPECIFIC: 1144 case SPDK_NVME_SCT_MEDIA_ERROR: 1145 case SPDK_NVME_SCT_PATH: 1146 nbdev->err_stat->status[sct][sc]++; 1147 break; 1148 default: 1149 break; 1150 } 1151 1152 pthread_mutex_unlock(&nbdev->mutex); 1153 } 1154 1155 static inline void 1156 bdev_nvme_update_io_path_stat(struct nvme_bdev_io *bio) 1157 { 1158 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1159 uint64_t num_blocks = bdev_io->u.bdev.num_blocks; 1160 uint32_t blocklen = bdev_io->bdev->blocklen; 1161 struct spdk_bdev_io_stat *stat; 1162 uint64_t tsc_diff; 1163 1164 if (bio->io_path->stat == NULL) { 1165 return; 1166 } 1167 1168 tsc_diff = spdk_get_ticks() - bio->submit_tsc; 1169 stat = bio->io_path->stat; 1170 1171 switch (bdev_io->type) { 1172 case SPDK_BDEV_IO_TYPE_READ: 1173 stat->bytes_read += num_blocks * blocklen; 1174 stat->num_read_ops++; 1175 stat->read_latency_ticks += tsc_diff; 1176 if (stat->max_read_latency_ticks < tsc_diff) { 1177 stat->max_read_latency_ticks = tsc_diff; 1178 } 1179 if (stat->min_read_latency_ticks > tsc_diff) { 1180 stat->min_read_latency_ticks = tsc_diff; 1181 } 1182 break; 1183 case SPDK_BDEV_IO_TYPE_WRITE: 1184 stat->bytes_written += num_blocks * blocklen; 1185 stat->num_write_ops++; 1186 stat->write_latency_ticks += tsc_diff; 1187 if (stat->max_write_latency_ticks < tsc_diff) { 1188 stat->max_write_latency_ticks = tsc_diff; 1189 } 1190 if (stat->min_write_latency_ticks > tsc_diff) { 1191 stat->min_write_latency_ticks = tsc_diff; 1192 } 1193 break; 1194 case SPDK_BDEV_IO_TYPE_UNMAP: 1195 stat->bytes_unmapped += num_blocks * blocklen; 1196 stat->num_unmap_ops++; 1197 stat->unmap_latency_ticks += tsc_diff; 1198 if (stat->max_unmap_latency_ticks < tsc_diff) { 1199 stat->max_unmap_latency_ticks = tsc_diff; 1200 } 1201 if (stat->min_unmap_latency_ticks > tsc_diff) { 1202 stat->min_unmap_latency_ticks = tsc_diff; 1203 } 1204 break; 1205 case SPDK_BDEV_IO_TYPE_ZCOPY: 1206 /* Track the data in the start phase only */ 1207 if (!bdev_io->u.bdev.zcopy.start) { 1208 break; 1209 } 1210 if (bdev_io->u.bdev.zcopy.populate) { 1211 stat->bytes_read += num_blocks * blocklen; 1212 stat->num_read_ops++; 1213 stat->read_latency_ticks += tsc_diff; 1214 if (stat->max_read_latency_ticks < tsc_diff) { 1215 stat->max_read_latency_ticks = tsc_diff; 1216 } 1217 if (stat->min_read_latency_ticks > tsc_diff) { 1218 stat->min_read_latency_ticks = tsc_diff; 1219 } 1220 } else { 1221 stat->bytes_written += num_blocks * blocklen; 1222 stat->num_write_ops++; 1223 stat->write_latency_ticks += tsc_diff; 1224 if (stat->max_write_latency_ticks < tsc_diff) { 1225 stat->max_write_latency_ticks = tsc_diff; 1226 } 1227 if (stat->min_write_latency_ticks > tsc_diff) { 1228 stat->min_write_latency_ticks = tsc_diff; 1229 } 1230 } 1231 break; 1232 case SPDK_BDEV_IO_TYPE_COPY: 1233 stat->bytes_copied += num_blocks * blocklen; 1234 stat->num_copy_ops++; 1235 stat->copy_latency_ticks += tsc_diff; 1236 if (stat->max_copy_latency_ticks < tsc_diff) { 1237 stat->max_copy_latency_ticks = tsc_diff; 1238 } 1239 if (stat->min_copy_latency_ticks > tsc_diff) { 1240 stat->min_copy_latency_ticks = tsc_diff; 1241 } 1242 break; 1243 default: 1244 break; 1245 } 1246 } 1247 1248 static inline void 1249 bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio, 1250 const struct spdk_nvme_cpl *cpl) 1251 { 1252 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1253 struct nvme_bdev_channel *nbdev_ch; 1254 struct nvme_io_path *io_path; 1255 struct nvme_ctrlr *nvme_ctrlr; 1256 const struct spdk_nvme_ctrlr_data *cdata; 1257 uint64_t delay_ms; 1258 1259 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 1260 1261 if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) { 1262 bdev_nvme_update_io_path_stat(bio); 1263 goto complete; 1264 } 1265 1266 /* Update error counts before deciding if retry is needed. 1267 * Hence, error counts may be more than the number of I/O errors. 1268 */ 1269 bdev_nvme_update_nvme_error_stat(bdev_io, cpl); 1270 1271 if (cpl->status.dnr != 0 || spdk_nvme_cpl_is_aborted_by_request(cpl) || 1272 (g_opts.bdev_retry_count != -1 && bio->retry_count >= g_opts.bdev_retry_count)) { 1273 goto complete; 1274 } 1275 1276 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1277 1278 assert(bio->io_path != NULL); 1279 io_path = bio->io_path; 1280 1281 nvme_ctrlr = io_path->qpair->ctrlr; 1282 1283 if (spdk_nvme_cpl_is_path_error(cpl) || 1284 spdk_nvme_cpl_is_aborted_sq_deletion(cpl) || 1285 !nvme_io_path_is_available(io_path) || 1286 !nvme_ctrlr_is_available(nvme_ctrlr)) { 1287 bdev_nvme_clear_current_io_path(nbdev_ch); 1288 bio->io_path = NULL; 1289 if (spdk_nvme_cpl_is_ana_error(cpl)) { 1290 if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) { 1291 io_path->nvme_ns->ana_state_updating = true; 1292 } 1293 } 1294 if (!any_io_path_may_become_available(nbdev_ch)) { 1295 goto complete; 1296 } 1297 delay_ms = 0; 1298 } else { 1299 bio->retry_count++; 1300 1301 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 1302 1303 if (cpl->status.crd != 0) { 1304 delay_ms = cdata->crdt[cpl->status.crd] * 100; 1305 } else { 1306 delay_ms = 0; 1307 } 1308 } 1309 1310 bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms); 1311 return; 1312 1313 complete: 1314 bio->retry_count = 0; 1315 bio->submit_tsc = 0; 1316 __bdev_nvme_io_complete(bdev_io, 0, cpl); 1317 } 1318 1319 static inline void 1320 bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc) 1321 { 1322 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1323 struct nvme_bdev_channel *nbdev_ch; 1324 enum spdk_bdev_io_status io_status; 1325 1326 switch (rc) { 1327 case 0: 1328 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1329 break; 1330 case -ENOMEM: 1331 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1332 break; 1333 case -ENXIO: 1334 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1335 1336 bdev_nvme_clear_current_io_path(nbdev_ch); 1337 bio->io_path = NULL; 1338 1339 if (any_io_path_may_become_available(nbdev_ch)) { 1340 bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL); 1341 return; 1342 } 1343 1344 /* fallthrough */ 1345 default: 1346 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1347 break; 1348 } 1349 1350 bio->retry_count = 0; 1351 bio->submit_tsc = 0; 1352 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1353 } 1354 1355 static inline void 1356 bdev_nvme_admin_passthru_complete(struct nvme_bdev_io *bio, int rc) 1357 { 1358 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1359 enum spdk_bdev_io_status io_status; 1360 1361 switch (rc) { 1362 case 0: 1363 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1364 break; 1365 case -ENOMEM: 1366 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1367 break; 1368 case -ENXIO: 1369 /* fallthrough */ 1370 default: 1371 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1372 break; 1373 } 1374 1375 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1376 } 1377 1378 static void 1379 bdev_nvme_clear_io_path_caches_done(struct spdk_io_channel_iter *i, int status) 1380 { 1381 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1382 1383 pthread_mutex_lock(&nvme_ctrlr->mutex); 1384 1385 assert(nvme_ctrlr->io_path_cache_clearing == true); 1386 nvme_ctrlr->io_path_cache_clearing = false; 1387 1388 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 1389 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1390 return; 1391 } 1392 1393 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1394 1395 nvme_ctrlr_unregister(nvme_ctrlr); 1396 } 1397 1398 static void 1399 _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair) 1400 { 1401 struct nvme_io_path *io_path; 1402 1403 TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) { 1404 bdev_nvme_clear_current_io_path(io_path->nbdev_ch); 1405 } 1406 } 1407 1408 static void 1409 bdev_nvme_clear_io_path_cache(struct spdk_io_channel_iter *i) 1410 { 1411 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 1412 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 1413 1414 assert(ctrlr_ch->qpair != NULL); 1415 1416 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 1417 1418 spdk_for_each_channel_continue(i, 0); 1419 } 1420 1421 static void 1422 bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr) 1423 { 1424 pthread_mutex_lock(&nvme_ctrlr->mutex); 1425 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 1426 nvme_ctrlr->io_path_cache_clearing) { 1427 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1428 return; 1429 } 1430 1431 nvme_ctrlr->io_path_cache_clearing = true; 1432 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1433 1434 spdk_for_each_channel(nvme_ctrlr, 1435 bdev_nvme_clear_io_path_cache, 1436 NULL, 1437 bdev_nvme_clear_io_path_caches_done); 1438 } 1439 1440 static struct nvme_qpair * 1441 nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair) 1442 { 1443 struct nvme_qpair *nvme_qpair; 1444 1445 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1446 if (nvme_qpair->qpair == qpair) { 1447 break; 1448 } 1449 } 1450 1451 return nvme_qpair; 1452 } 1453 1454 static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair); 1455 1456 static void 1457 bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx) 1458 { 1459 struct nvme_poll_group *group = poll_group_ctx; 1460 struct nvme_qpair *nvme_qpair; 1461 struct nvme_ctrlr_channel *ctrlr_ch; 1462 1463 nvme_qpair = nvme_poll_group_get_qpair(group, qpair); 1464 if (nvme_qpair == NULL) { 1465 return; 1466 } 1467 1468 if (nvme_qpair->qpair != NULL) { 1469 spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair); 1470 nvme_qpair->qpair = NULL; 1471 } 1472 1473 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1474 1475 ctrlr_ch = nvme_qpair->ctrlr_ch; 1476 1477 if (ctrlr_ch != NULL) { 1478 if (ctrlr_ch->reset_iter != NULL) { 1479 /* If we are already in a full reset sequence, we do not have 1480 * to restart it. Just move to the next ctrlr_channel. 1481 */ 1482 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed in a reset ctrlr sequence.\n", 1483 qpair); 1484 spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 1485 ctrlr_ch->reset_iter = NULL; 1486 } else { 1487 /* qpair was disconnected unexpectedly. Reset controller for recovery. */ 1488 SPDK_NOTICELOG("qpair %p was disconnected and freed. reset controller.\n", qpair); 1489 bdev_nvme_failover(nvme_qpair->ctrlr, false); 1490 } 1491 } else { 1492 /* In this case, ctrlr_channel is already deleted. */ 1493 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed. delete nvme_qpair.\n", qpair); 1494 nvme_qpair_delete(nvme_qpair); 1495 } 1496 } 1497 1498 static void 1499 bdev_nvme_check_io_qpairs(struct nvme_poll_group *group) 1500 { 1501 struct nvme_qpair *nvme_qpair; 1502 1503 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1504 if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) { 1505 continue; 1506 } 1507 1508 if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 1509 SPDK_NVME_QPAIR_FAILURE_NONE) { 1510 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1511 } 1512 } 1513 } 1514 1515 static int 1516 bdev_nvme_poll(void *arg) 1517 { 1518 struct nvme_poll_group *group = arg; 1519 int64_t num_completions; 1520 1521 if (group->collect_spin_stat && group->start_ticks == 0) { 1522 group->start_ticks = spdk_get_ticks(); 1523 } 1524 1525 num_completions = spdk_nvme_poll_group_process_completions(group->group, 0, 1526 bdev_nvme_disconnected_qpair_cb); 1527 if (group->collect_spin_stat) { 1528 if (num_completions > 0) { 1529 if (group->end_ticks != 0) { 1530 group->spin_ticks += (group->end_ticks - group->start_ticks); 1531 group->end_ticks = 0; 1532 } 1533 group->start_ticks = 0; 1534 } else { 1535 group->end_ticks = spdk_get_ticks(); 1536 } 1537 } 1538 1539 if (spdk_unlikely(num_completions < 0)) { 1540 bdev_nvme_check_io_qpairs(group); 1541 } 1542 1543 return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 1544 } 1545 1546 static int bdev_nvme_poll_adminq(void *arg); 1547 1548 static void 1549 bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_period_us) 1550 { 1551 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 1552 1553 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, 1554 nvme_ctrlr, new_period_us); 1555 } 1556 1557 static int 1558 bdev_nvme_poll_adminq(void *arg) 1559 { 1560 int32_t rc; 1561 struct nvme_ctrlr *nvme_ctrlr = arg; 1562 nvme_ctrlr_disconnected_cb disconnected_cb; 1563 1564 assert(nvme_ctrlr != NULL); 1565 1566 rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr); 1567 if (rc < 0) { 1568 disconnected_cb = nvme_ctrlr->disconnected_cb; 1569 nvme_ctrlr->disconnected_cb = NULL; 1570 1571 if (rc == -ENXIO && disconnected_cb != NULL) { 1572 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 1573 g_opts.nvme_adminq_poll_period_us); 1574 disconnected_cb(nvme_ctrlr); 1575 } else { 1576 bdev_nvme_failover(nvme_ctrlr, false); 1577 } 1578 } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) != 1579 SPDK_NVME_QPAIR_FAILURE_NONE) { 1580 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 1581 } 1582 1583 return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY; 1584 } 1585 1586 static void 1587 _bdev_nvme_unregister_dev_cb(void *io_device) 1588 { 1589 struct nvme_bdev *nvme_disk = io_device; 1590 1591 free(nvme_disk->disk.name); 1592 free(nvme_disk->err_stat); 1593 free(nvme_disk); 1594 } 1595 1596 static int 1597 bdev_nvme_destruct(void *ctx) 1598 { 1599 struct nvme_bdev *nvme_disk = ctx; 1600 struct nvme_ns *nvme_ns, *tmp_nvme_ns; 1601 1602 SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid); 1603 1604 TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) { 1605 pthread_mutex_lock(&nvme_ns->ctrlr->mutex); 1606 1607 nvme_ns->bdev = NULL; 1608 1609 assert(nvme_ns->id > 0); 1610 1611 if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) { 1612 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1613 1614 nvme_ctrlr_release(nvme_ns->ctrlr); 1615 nvme_ns_free(nvme_ns); 1616 } else { 1617 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1618 } 1619 } 1620 1621 pthread_mutex_lock(&g_bdev_nvme_mutex); 1622 TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq); 1623 pthread_mutex_unlock(&g_bdev_nvme_mutex); 1624 1625 spdk_io_device_unregister(nvme_disk, _bdev_nvme_unregister_dev_cb); 1626 1627 return 0; 1628 } 1629 1630 static int 1631 bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair) 1632 { 1633 struct nvme_ctrlr *nvme_ctrlr; 1634 struct spdk_nvme_io_qpair_opts opts; 1635 struct spdk_nvme_qpair *qpair; 1636 int rc; 1637 1638 nvme_ctrlr = nvme_qpair->ctrlr; 1639 1640 spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1641 opts.delay_cmd_submit = g_opts.delay_cmd_submit; 1642 opts.create_only = true; 1643 opts.async_mode = true; 1644 opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests); 1645 g_opts.io_queue_requests = opts.io_queue_requests; 1646 1647 qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1648 if (qpair == NULL) { 1649 return -1; 1650 } 1651 1652 SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name, 1653 spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread)); 1654 1655 assert(nvme_qpair->group != NULL); 1656 1657 rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair); 1658 if (rc != 0) { 1659 SPDK_ERRLOG("Unable to begin polling on NVMe Channel.\n"); 1660 goto err; 1661 } 1662 1663 rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair); 1664 if (rc != 0) { 1665 SPDK_ERRLOG("Unable to connect I/O qpair.\n"); 1666 goto err; 1667 } 1668 1669 nvme_qpair->qpair = qpair; 1670 1671 if (!g_opts.disable_auto_failback) { 1672 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1673 } 1674 1675 return 0; 1676 1677 err: 1678 spdk_nvme_ctrlr_free_io_qpair(qpair); 1679 1680 return rc; 1681 } 1682 1683 static void 1684 bdev_nvme_complete_pending_resets(struct spdk_io_channel_iter *i) 1685 { 1686 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 1687 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 1688 enum spdk_bdev_io_status status = SPDK_BDEV_IO_STATUS_SUCCESS; 1689 struct spdk_bdev_io *bdev_io; 1690 1691 if (spdk_io_channel_iter_get_ctx(i) != NULL) { 1692 status = SPDK_BDEV_IO_STATUS_FAILED; 1693 } 1694 1695 while (!TAILQ_EMPTY(&ctrlr_ch->pending_resets)) { 1696 bdev_io = TAILQ_FIRST(&ctrlr_ch->pending_resets); 1697 TAILQ_REMOVE(&ctrlr_ch->pending_resets, bdev_io, module_link); 1698 __bdev_nvme_io_complete(bdev_io, status, NULL); 1699 } 1700 1701 spdk_for_each_channel_continue(i, 0); 1702 } 1703 1704 static void 1705 bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove) 1706 { 1707 struct nvme_path_id *path_id, *next_path; 1708 int rc __attribute__((unused)); 1709 1710 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 1711 assert(path_id); 1712 assert(path_id == nvme_ctrlr->active_path_id); 1713 next_path = TAILQ_NEXT(path_id, link); 1714 1715 path_id->is_failed = true; 1716 1717 if (next_path) { 1718 assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE); 1719 1720 SPDK_NOTICELOG("Start failover from %s:%s to %s:%s\n", path_id->trid.traddr, 1721 path_id->trid.trsvcid, next_path->trid.traddr, next_path->trid.trsvcid); 1722 1723 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 1724 nvme_ctrlr->active_path_id = next_path; 1725 rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid); 1726 assert(rc == 0); 1727 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 1728 if (!remove) { 1729 /** Shuffle the old trid to the end of the list and use the new one. 1730 * Allows for round robin through multiple connections. 1731 */ 1732 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link); 1733 } else { 1734 free(path_id); 1735 } 1736 } 1737 } 1738 1739 static bool 1740 bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr) 1741 { 1742 int32_t elapsed; 1743 1744 if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 || 1745 nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) { 1746 return false; 1747 } 1748 1749 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 1750 if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) { 1751 return true; 1752 } else { 1753 return false; 1754 } 1755 } 1756 1757 static bool 1758 bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr) 1759 { 1760 uint32_t elapsed; 1761 1762 if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) { 1763 return false; 1764 } 1765 1766 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 1767 if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) { 1768 return true; 1769 } else { 1770 return false; 1771 } 1772 } 1773 1774 static void bdev_nvme_reset_complete(struct nvme_ctrlr *nvme_ctrlr, bool success); 1775 1776 static void 1777 nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb cb_fn) 1778 { 1779 int rc; 1780 1781 rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr); 1782 if (rc != 0) { 1783 /* Disconnect fails if ctrlr is already resetting or removed. In this case, 1784 * fail the reset sequence immediately. 1785 */ 1786 bdev_nvme_reset_complete(nvme_ctrlr, false); 1787 return; 1788 } 1789 1790 /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq. 1791 * Set callback here to execute the specified operation after ctrlr is really disconnected. 1792 */ 1793 assert(nvme_ctrlr->disconnected_cb == NULL); 1794 nvme_ctrlr->disconnected_cb = cb_fn; 1795 1796 /* During disconnection, reduce the period to poll adminq more often. */ 1797 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0); 1798 } 1799 1800 enum bdev_nvme_op_after_reset { 1801 OP_NONE, 1802 OP_COMPLETE_PENDING_DESTRUCT, 1803 OP_DESTRUCT, 1804 OP_DELAYED_RECONNECT, 1805 }; 1806 1807 typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset; 1808 1809 static _bdev_nvme_op_after_reset 1810 bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success) 1811 { 1812 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 1813 /* Complete pending destruct after reset completes. */ 1814 return OP_COMPLETE_PENDING_DESTRUCT; 1815 } else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) { 1816 nvme_ctrlr->reset_start_tsc = 0; 1817 return OP_NONE; 1818 } else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 1819 return OP_DESTRUCT; 1820 } else { 1821 if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) { 1822 nvme_ctrlr->fast_io_fail_timedout = true; 1823 } 1824 bdev_nvme_failover_trid(nvme_ctrlr, false); 1825 return OP_DELAYED_RECONNECT; 1826 } 1827 } 1828 1829 static int bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug); 1830 static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 1831 1832 static int 1833 bdev_nvme_reconnect_delay_timer_expired(void *ctx) 1834 { 1835 struct nvme_ctrlr *nvme_ctrlr = ctx; 1836 1837 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect_delay, nvme_ctrlr->nbdev_ctrlr->name); 1838 pthread_mutex_lock(&nvme_ctrlr->mutex); 1839 1840 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 1841 1842 assert(nvme_ctrlr->reconnect_is_delayed == true); 1843 nvme_ctrlr->reconnect_is_delayed = false; 1844 1845 if (nvme_ctrlr->destruct) { 1846 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1847 return SPDK_POLLER_BUSY; 1848 } 1849 1850 assert(nvme_ctrlr->resetting == false); 1851 nvme_ctrlr->resetting = true; 1852 1853 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1854 1855 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 1856 1857 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 1858 return SPDK_POLLER_BUSY; 1859 } 1860 1861 static void 1862 bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr) 1863 { 1864 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 1865 1866 assert(nvme_ctrlr->reconnect_is_delayed == false); 1867 nvme_ctrlr->reconnect_is_delayed = true; 1868 1869 assert(nvme_ctrlr->reconnect_delay_timer == NULL); 1870 nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired, 1871 nvme_ctrlr, 1872 nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC); 1873 } 1874 1875 static void 1876 _bdev_nvme_reset_complete(struct spdk_io_channel_iter *i, int status) 1877 { 1878 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1879 bool success = spdk_io_channel_iter_get_ctx(i) == NULL; 1880 struct nvme_path_id *path_id; 1881 bdev_nvme_reset_cb reset_cb_fn = nvme_ctrlr->reset_cb_fn; 1882 void *reset_cb_arg = nvme_ctrlr->reset_cb_arg; 1883 enum bdev_nvme_op_after_reset op_after_reset; 1884 1885 assert(nvme_ctrlr->thread == spdk_get_thread()); 1886 1887 nvme_ctrlr->reset_cb_fn = NULL; 1888 nvme_ctrlr->reset_cb_arg = NULL; 1889 1890 if (!success) { 1891 SPDK_ERRLOG("Resetting controller failed.\n"); 1892 } else { 1893 SPDK_NOTICELOG("Resetting controller successful.\n"); 1894 } 1895 1896 pthread_mutex_lock(&nvme_ctrlr->mutex); 1897 nvme_ctrlr->resetting = false; 1898 1899 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 1900 assert(path_id != NULL); 1901 assert(path_id == nvme_ctrlr->active_path_id); 1902 1903 path_id->is_failed = !success; 1904 1905 op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success); 1906 1907 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1908 1909 if (reset_cb_fn) { 1910 reset_cb_fn(reset_cb_arg, success); 1911 } 1912 1913 switch (op_after_reset) { 1914 case OP_COMPLETE_PENDING_DESTRUCT: 1915 nvme_ctrlr_unregister(nvme_ctrlr); 1916 break; 1917 case OP_DESTRUCT: 1918 bdev_nvme_delete_ctrlr(nvme_ctrlr, false); 1919 break; 1920 case OP_DELAYED_RECONNECT: 1921 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_start_reconnect_delay_timer); 1922 break; 1923 default: 1924 break; 1925 } 1926 } 1927 1928 static void 1929 bdev_nvme_reset_complete(struct nvme_ctrlr *nvme_ctrlr, bool success) 1930 { 1931 /* Make sure we clear any pending resets before returning. */ 1932 spdk_for_each_channel(nvme_ctrlr, 1933 bdev_nvme_complete_pending_resets, 1934 success ? NULL : (void *)0x1, 1935 _bdev_nvme_reset_complete); 1936 } 1937 1938 static void 1939 bdev_nvme_reset_create_qpairs_failed(struct spdk_io_channel_iter *i, int status) 1940 { 1941 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1942 1943 bdev_nvme_reset_complete(nvme_ctrlr, false); 1944 } 1945 1946 static void 1947 bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i) 1948 { 1949 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 1950 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch); 1951 struct nvme_qpair *nvme_qpair; 1952 1953 nvme_qpair = ctrlr_ch->qpair; 1954 assert(nvme_qpair != NULL); 1955 1956 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1957 1958 if (nvme_qpair->qpair != NULL) { 1959 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 1960 1961 /* The current full reset sequence will move to the next 1962 * ctrlr_channel after the qpair is actually disconnected. 1963 */ 1964 assert(ctrlr_ch->reset_iter == NULL); 1965 ctrlr_ch->reset_iter = i; 1966 } else { 1967 spdk_for_each_channel_continue(i, 0); 1968 } 1969 } 1970 1971 static void 1972 bdev_nvme_reset_create_qpairs_done(struct spdk_io_channel_iter *i, int status) 1973 { 1974 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1975 1976 if (status == 0) { 1977 bdev_nvme_reset_complete(nvme_ctrlr, true); 1978 } else { 1979 /* Delete the added qpairs and quiesce ctrlr to make the states clean. */ 1980 spdk_for_each_channel(nvme_ctrlr, 1981 bdev_nvme_reset_destroy_qpair, 1982 NULL, 1983 bdev_nvme_reset_create_qpairs_failed); 1984 } 1985 } 1986 1987 static void 1988 bdev_nvme_reset_create_qpair(struct spdk_io_channel_iter *i) 1989 { 1990 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 1991 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 1992 int rc; 1993 1994 rc = bdev_nvme_create_qpair(ctrlr_ch->qpair); 1995 1996 spdk_for_each_channel_continue(i, rc); 1997 } 1998 1999 static int 2000 bdev_nvme_reconnect_ctrlr_poll(void *arg) 2001 { 2002 struct nvme_ctrlr *nvme_ctrlr = arg; 2003 int rc = -ETIMEDOUT; 2004 2005 if (!bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 2006 rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr); 2007 if (rc == -EAGAIN) { 2008 return SPDK_POLLER_BUSY; 2009 } 2010 } 2011 2012 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 2013 if (rc == 0) { 2014 /* Recreate all of the I/O queue pairs */ 2015 spdk_for_each_channel(nvme_ctrlr, 2016 bdev_nvme_reset_create_qpair, 2017 NULL, 2018 bdev_nvme_reset_create_qpairs_done); 2019 } else { 2020 bdev_nvme_reset_complete(nvme_ctrlr, false); 2021 } 2022 return SPDK_POLLER_BUSY; 2023 } 2024 2025 static void 2026 bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2027 { 2028 spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr); 2029 2030 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect, nvme_ctrlr->nbdev_ctrlr->name); 2031 assert(nvme_ctrlr->reset_detach_poller == NULL); 2032 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll, 2033 nvme_ctrlr, 0); 2034 } 2035 2036 static void 2037 bdev_nvme_reset_ctrlr(struct spdk_io_channel_iter *i, int status) 2038 { 2039 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 2040 2041 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reset, nvme_ctrlr->nbdev_ctrlr->name); 2042 assert(status == 0); 2043 2044 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2045 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2046 } else { 2047 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); 2048 } 2049 } 2050 2051 static void 2052 bdev_nvme_reset_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr) 2053 { 2054 spdk_for_each_channel(nvme_ctrlr, 2055 bdev_nvme_reset_destroy_qpair, 2056 NULL, 2057 bdev_nvme_reset_ctrlr); 2058 } 2059 2060 static void 2061 _bdev_nvme_reset(void *ctx) 2062 { 2063 struct nvme_ctrlr *nvme_ctrlr = ctx; 2064 2065 assert(nvme_ctrlr->resetting == true); 2066 assert(nvme_ctrlr->thread == spdk_get_thread()); 2067 2068 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2069 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs); 2070 } else { 2071 bdev_nvme_reset_destroy_qpairs(nvme_ctrlr); 2072 } 2073 } 2074 2075 static int 2076 bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr) 2077 { 2078 pthread_mutex_lock(&nvme_ctrlr->mutex); 2079 if (nvme_ctrlr->destruct) { 2080 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2081 return -ENXIO; 2082 } 2083 2084 if (nvme_ctrlr->resetting) { 2085 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2086 SPDK_NOTICELOG("Unable to perform reset, already in progress.\n"); 2087 return -EBUSY; 2088 } 2089 2090 if (nvme_ctrlr->reconnect_is_delayed) { 2091 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2092 SPDK_NOTICELOG("Reconnect is already scheduled.\n"); 2093 return -EBUSY; 2094 } 2095 2096 nvme_ctrlr->resetting = true; 2097 2098 assert(nvme_ctrlr->reset_start_tsc == 0); 2099 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2100 2101 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2102 2103 spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr); 2104 return 0; 2105 } 2106 2107 int 2108 bdev_nvme_reset_rpc(struct nvme_ctrlr *nvme_ctrlr, bdev_nvme_reset_cb cb_fn, void *cb_arg) 2109 { 2110 int rc; 2111 2112 rc = bdev_nvme_reset(nvme_ctrlr); 2113 if (rc == 0) { 2114 nvme_ctrlr->reset_cb_fn = cb_fn; 2115 nvme_ctrlr->reset_cb_arg = cb_arg; 2116 } 2117 return rc; 2118 } 2119 2120 static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio); 2121 2122 static void 2123 bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio) 2124 { 2125 enum spdk_bdev_io_status io_status; 2126 2127 if (bio->cpl.cdw0 == 0) { 2128 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 2129 } else { 2130 io_status = SPDK_BDEV_IO_STATUS_FAILED; 2131 } 2132 2133 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL); 2134 } 2135 2136 static void 2137 _bdev_nvme_reset_io_continue(void *ctx) 2138 { 2139 struct nvme_bdev_io *bio = ctx; 2140 struct nvme_io_path *prev_io_path, *next_io_path; 2141 int rc; 2142 2143 prev_io_path = bio->io_path; 2144 bio->io_path = NULL; 2145 2146 if (bio->cpl.cdw0 != 0) { 2147 goto complete; 2148 } 2149 2150 next_io_path = STAILQ_NEXT(prev_io_path, stailq); 2151 if (next_io_path == NULL) { 2152 goto complete; 2153 } 2154 2155 rc = _bdev_nvme_reset_io(next_io_path, bio); 2156 if (rc == 0) { 2157 return; 2158 } 2159 2160 bio->cpl.cdw0 = 1; 2161 2162 complete: 2163 bdev_nvme_reset_io_complete(bio); 2164 } 2165 2166 static void 2167 bdev_nvme_reset_io_continue(void *cb_arg, bool success) 2168 { 2169 struct nvme_bdev_io *bio = cb_arg; 2170 2171 bio->cpl.cdw0 = !success; 2172 2173 spdk_thread_send_msg(bio->orig_thread, _bdev_nvme_reset_io_continue, bio); 2174 } 2175 2176 static int 2177 _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio) 2178 { 2179 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 2180 struct nvme_ctrlr_channel *ctrlr_ch; 2181 struct spdk_bdev_io *bdev_io; 2182 int rc; 2183 2184 rc = bdev_nvme_reset(nvme_ctrlr); 2185 if (rc == 0) { 2186 assert(bio->io_path == NULL); 2187 bio->io_path = io_path; 2188 2189 assert(nvme_ctrlr->reset_cb_fn == NULL); 2190 assert(nvme_ctrlr->reset_cb_arg == NULL); 2191 nvme_ctrlr->reset_cb_fn = bdev_nvme_reset_io_continue; 2192 nvme_ctrlr->reset_cb_arg = bio; 2193 } else if (rc == -EBUSY) { 2194 ctrlr_ch = io_path->qpair->ctrlr_ch; 2195 assert(ctrlr_ch != NULL); 2196 /* 2197 * Reset call is queued only if it is from the app framework. This is on purpose so that 2198 * we don't interfere with the app framework reset strategy. i.e. we are deferring to the 2199 * upper level. If they are in the middle of a reset, we won't try to schedule another one. 2200 */ 2201 bdev_io = spdk_bdev_io_from_ctx(bio); 2202 TAILQ_INSERT_TAIL(&ctrlr_ch->pending_resets, bdev_io, module_link); 2203 rc = 0; 2204 } 2205 2206 return rc; 2207 } 2208 2209 static void 2210 bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio) 2211 { 2212 struct nvme_io_path *io_path; 2213 int rc; 2214 2215 bio->cpl.cdw0 = 0; 2216 bio->orig_thread = spdk_get_thread(); 2217 2218 /* Reset all nvme_ctrlrs of a bdev controller sequentially. */ 2219 io_path = STAILQ_FIRST(&nbdev_ch->io_path_list); 2220 assert(io_path != NULL); 2221 2222 rc = _bdev_nvme_reset_io(io_path, bio); 2223 if (rc != 0) { 2224 bio->cpl.cdw0 = 1; 2225 bdev_nvme_reset_io_complete(bio); 2226 } 2227 } 2228 2229 static int 2230 bdev_nvme_failover_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool remove) 2231 { 2232 if (nvme_ctrlr->destruct) { 2233 /* Don't bother resetting if the controller is in the process of being destructed. */ 2234 return -ENXIO; 2235 } 2236 2237 if (nvme_ctrlr->resetting) { 2238 SPDK_NOTICELOG("Unable to perform reset, already in progress.\n"); 2239 return -EBUSY; 2240 } 2241 2242 bdev_nvme_failover_trid(nvme_ctrlr, remove); 2243 2244 if (nvme_ctrlr->reconnect_is_delayed) { 2245 SPDK_NOTICELOG("Reconnect is already scheduled.\n"); 2246 2247 /* We rely on the next reconnect for the failover. */ 2248 return -EALREADY; 2249 } 2250 2251 nvme_ctrlr->resetting = true; 2252 2253 assert(nvme_ctrlr->reset_start_tsc == 0); 2254 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2255 2256 return 0; 2257 } 2258 2259 static int 2260 bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove) 2261 { 2262 int rc; 2263 2264 pthread_mutex_lock(&nvme_ctrlr->mutex); 2265 rc = bdev_nvme_failover_unsafe(nvme_ctrlr, remove); 2266 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2267 2268 if (rc == 0) { 2269 spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr); 2270 } else if (rc == -EALREADY) { 2271 rc = 0; 2272 } 2273 2274 return rc; 2275 } 2276 2277 static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, 2278 uint64_t num_blocks); 2279 2280 static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, 2281 uint64_t num_blocks); 2282 2283 static int bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, 2284 uint64_t src_offset_blocks, 2285 uint64_t num_blocks); 2286 2287 static void 2288 bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 2289 bool success) 2290 { 2291 struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 2292 struct spdk_bdev *bdev = bdev_io->bdev; 2293 int ret; 2294 2295 if (!success) { 2296 ret = -EINVAL; 2297 goto exit; 2298 } 2299 2300 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 2301 ret = -ENXIO; 2302 goto exit; 2303 } 2304 2305 ret = bdev_nvme_readv(bio, 2306 bdev_io->u.bdev.iovs, 2307 bdev_io->u.bdev.iovcnt, 2308 bdev_io->u.bdev.md_buf, 2309 bdev_io->u.bdev.num_blocks, 2310 bdev_io->u.bdev.offset_blocks, 2311 bdev->dif_check_flags, 2312 bdev_io->u.bdev.memory_domain, 2313 bdev_io->u.bdev.memory_domain_ctx); 2314 2315 exit: 2316 if (spdk_unlikely(ret != 0)) { 2317 bdev_nvme_io_complete(bio, ret); 2318 } 2319 } 2320 2321 static inline void 2322 _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 2323 { 2324 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 2325 struct spdk_bdev *bdev = bdev_io->bdev; 2326 struct nvme_bdev_io *nbdev_io_to_abort; 2327 int rc = 0; 2328 2329 switch (bdev_io->type) { 2330 case SPDK_BDEV_IO_TYPE_READ: 2331 if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) { 2332 rc = bdev_nvme_readv(nbdev_io, 2333 bdev_io->u.bdev.iovs, 2334 bdev_io->u.bdev.iovcnt, 2335 bdev_io->u.bdev.md_buf, 2336 bdev_io->u.bdev.num_blocks, 2337 bdev_io->u.bdev.offset_blocks, 2338 bdev->dif_check_flags, 2339 bdev_io->u.bdev.memory_domain, 2340 bdev_io->u.bdev.memory_domain_ctx); 2341 } else { 2342 spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb, 2343 bdev_io->u.bdev.num_blocks * bdev->blocklen); 2344 rc = 0; 2345 } 2346 break; 2347 case SPDK_BDEV_IO_TYPE_WRITE: 2348 rc = bdev_nvme_writev(nbdev_io, 2349 bdev_io->u.bdev.iovs, 2350 bdev_io->u.bdev.iovcnt, 2351 bdev_io->u.bdev.md_buf, 2352 bdev_io->u.bdev.num_blocks, 2353 bdev_io->u.bdev.offset_blocks, 2354 bdev->dif_check_flags, 2355 bdev_io->u.bdev.memory_domain, 2356 bdev_io->u.bdev.memory_domain_ctx); 2357 break; 2358 case SPDK_BDEV_IO_TYPE_COMPARE: 2359 rc = bdev_nvme_comparev(nbdev_io, 2360 bdev_io->u.bdev.iovs, 2361 bdev_io->u.bdev.iovcnt, 2362 bdev_io->u.bdev.md_buf, 2363 bdev_io->u.bdev.num_blocks, 2364 bdev_io->u.bdev.offset_blocks, 2365 bdev->dif_check_flags); 2366 break; 2367 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 2368 rc = bdev_nvme_comparev_and_writev(nbdev_io, 2369 bdev_io->u.bdev.iovs, 2370 bdev_io->u.bdev.iovcnt, 2371 bdev_io->u.bdev.fused_iovs, 2372 bdev_io->u.bdev.fused_iovcnt, 2373 bdev_io->u.bdev.md_buf, 2374 bdev_io->u.bdev.num_blocks, 2375 bdev_io->u.bdev.offset_blocks, 2376 bdev->dif_check_flags); 2377 break; 2378 case SPDK_BDEV_IO_TYPE_UNMAP: 2379 rc = bdev_nvme_unmap(nbdev_io, 2380 bdev_io->u.bdev.offset_blocks, 2381 bdev_io->u.bdev.num_blocks); 2382 break; 2383 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 2384 rc = bdev_nvme_write_zeroes(nbdev_io, 2385 bdev_io->u.bdev.offset_blocks, 2386 bdev_io->u.bdev.num_blocks); 2387 break; 2388 case SPDK_BDEV_IO_TYPE_RESET: 2389 nbdev_io->io_path = NULL; 2390 bdev_nvme_reset_io(nbdev_ch, nbdev_io); 2391 break; 2392 case SPDK_BDEV_IO_TYPE_FLUSH: 2393 bdev_nvme_io_complete(nbdev_io, 0); 2394 break; 2395 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 2396 rc = bdev_nvme_zone_appendv(nbdev_io, 2397 bdev_io->u.bdev.iovs, 2398 bdev_io->u.bdev.iovcnt, 2399 bdev_io->u.bdev.md_buf, 2400 bdev_io->u.bdev.num_blocks, 2401 bdev_io->u.bdev.offset_blocks, 2402 bdev->dif_check_flags); 2403 break; 2404 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 2405 rc = bdev_nvme_get_zone_info(nbdev_io, 2406 bdev_io->u.zone_mgmt.zone_id, 2407 bdev_io->u.zone_mgmt.num_zones, 2408 bdev_io->u.zone_mgmt.buf); 2409 break; 2410 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 2411 rc = bdev_nvme_zone_management(nbdev_io, 2412 bdev_io->u.zone_mgmt.zone_id, 2413 bdev_io->u.zone_mgmt.zone_action); 2414 break; 2415 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 2416 nbdev_io->io_path = NULL; 2417 bdev_nvme_admin_passthru(nbdev_ch, 2418 nbdev_io, 2419 &bdev_io->u.nvme_passthru.cmd, 2420 bdev_io->u.nvme_passthru.buf, 2421 bdev_io->u.nvme_passthru.nbytes); 2422 break; 2423 case SPDK_BDEV_IO_TYPE_NVME_IO: 2424 rc = bdev_nvme_io_passthru(nbdev_io, 2425 &bdev_io->u.nvme_passthru.cmd, 2426 bdev_io->u.nvme_passthru.buf, 2427 bdev_io->u.nvme_passthru.nbytes); 2428 break; 2429 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 2430 rc = bdev_nvme_io_passthru_md(nbdev_io, 2431 &bdev_io->u.nvme_passthru.cmd, 2432 bdev_io->u.nvme_passthru.buf, 2433 bdev_io->u.nvme_passthru.nbytes, 2434 bdev_io->u.nvme_passthru.md_buf, 2435 bdev_io->u.nvme_passthru.md_len); 2436 break; 2437 case SPDK_BDEV_IO_TYPE_ABORT: 2438 nbdev_io->io_path = NULL; 2439 nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx; 2440 bdev_nvme_abort(nbdev_ch, 2441 nbdev_io, 2442 nbdev_io_to_abort); 2443 break; 2444 case SPDK_BDEV_IO_TYPE_COPY: 2445 rc = bdev_nvme_copy(nbdev_io, 2446 bdev_io->u.bdev.offset_blocks, 2447 bdev_io->u.bdev.copy.src_offset_blocks, 2448 bdev_io->u.bdev.num_blocks); 2449 break; 2450 default: 2451 rc = -EINVAL; 2452 break; 2453 } 2454 2455 if (spdk_unlikely(rc != 0)) { 2456 bdev_nvme_io_complete(nbdev_io, rc); 2457 } 2458 } 2459 2460 static void 2461 bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 2462 { 2463 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 2464 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 2465 2466 if (spdk_likely(nbdev_io->submit_tsc == 0)) { 2467 nbdev_io->submit_tsc = spdk_bdev_io_get_submit_tsc(bdev_io); 2468 } else { 2469 /* There are cases where submit_tsc != 0, i.e. retry I/O. 2470 * We need to update submit_tsc here. 2471 */ 2472 nbdev_io->submit_tsc = spdk_get_ticks(); 2473 } 2474 2475 spdk_trace_record(TRACE_BDEV_NVME_IO_START, 0, 0, (uintptr_t)nbdev_io, (uintptr_t)bdev_io); 2476 nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch); 2477 if (spdk_unlikely(!nbdev_io->io_path)) { 2478 if (!bdev_nvme_io_type_is_admin(bdev_io->type)) { 2479 bdev_nvme_io_complete(nbdev_io, -ENXIO); 2480 return; 2481 } 2482 2483 /* Admin commands do not use the optimal I/O path. 2484 * Simply fall through even if it is not found. 2485 */ 2486 } 2487 2488 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 2489 } 2490 2491 static bool 2492 bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 2493 { 2494 struct nvme_bdev *nbdev = ctx; 2495 struct nvme_ns *nvme_ns; 2496 struct spdk_nvme_ns *ns; 2497 struct spdk_nvme_ctrlr *ctrlr; 2498 const struct spdk_nvme_ctrlr_data *cdata; 2499 2500 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 2501 assert(nvme_ns != NULL); 2502 ns = nvme_ns->ns; 2503 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 2504 2505 switch (io_type) { 2506 case SPDK_BDEV_IO_TYPE_READ: 2507 case SPDK_BDEV_IO_TYPE_WRITE: 2508 case SPDK_BDEV_IO_TYPE_RESET: 2509 case SPDK_BDEV_IO_TYPE_FLUSH: 2510 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 2511 case SPDK_BDEV_IO_TYPE_NVME_IO: 2512 case SPDK_BDEV_IO_TYPE_ABORT: 2513 return true; 2514 2515 case SPDK_BDEV_IO_TYPE_COMPARE: 2516 return spdk_nvme_ns_supports_compare(ns); 2517 2518 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 2519 return spdk_nvme_ns_get_md_size(ns) ? true : false; 2520 2521 case SPDK_BDEV_IO_TYPE_UNMAP: 2522 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2523 return cdata->oncs.dsm; 2524 2525 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 2526 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2527 return cdata->oncs.write_zeroes; 2528 2529 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 2530 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 2531 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) { 2532 return true; 2533 } 2534 return false; 2535 2536 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 2537 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 2538 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS; 2539 2540 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 2541 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS && 2542 spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED; 2543 2544 case SPDK_BDEV_IO_TYPE_COPY: 2545 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2546 return cdata->oncs.copy; 2547 2548 default: 2549 return false; 2550 } 2551 } 2552 2553 static int 2554 nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch) 2555 { 2556 struct nvme_qpair *nvme_qpair; 2557 struct spdk_io_channel *pg_ch; 2558 int rc; 2559 2560 nvme_qpair = calloc(1, sizeof(*nvme_qpair)); 2561 if (!nvme_qpair) { 2562 SPDK_ERRLOG("Failed to alloc nvme_qpair.\n"); 2563 return -1; 2564 } 2565 2566 TAILQ_INIT(&nvme_qpair->io_path_list); 2567 2568 nvme_qpair->ctrlr = nvme_ctrlr; 2569 nvme_qpair->ctrlr_ch = ctrlr_ch; 2570 2571 pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs); 2572 if (!pg_ch) { 2573 free(nvme_qpair); 2574 return -1; 2575 } 2576 2577 nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch); 2578 2579 #ifdef SPDK_CONFIG_VTUNE 2580 nvme_qpair->group->collect_spin_stat = true; 2581 #else 2582 nvme_qpair->group->collect_spin_stat = false; 2583 #endif 2584 2585 rc = bdev_nvme_create_qpair(nvme_qpair); 2586 if (rc != 0) { 2587 /* nvme_ctrlr can't create IO qpair if connection is down. 2588 * 2589 * If reconnect_delay_sec is non-zero, creating IO qpair is retried 2590 * after reconnect_delay_sec seconds. If bdev_retry_count is non-zero, 2591 * submitted IO will be queued until IO qpair is successfully created. 2592 * 2593 * Hence, if both are satisfied, ignore the failure. 2594 */ 2595 if (nvme_ctrlr->opts.reconnect_delay_sec == 0 || g_opts.bdev_retry_count == 0) { 2596 spdk_put_io_channel(pg_ch); 2597 free(nvme_qpair); 2598 return rc; 2599 } 2600 } 2601 2602 TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 2603 2604 ctrlr_ch->qpair = nvme_qpair; 2605 2606 pthread_mutex_lock(&nvme_qpair->ctrlr->mutex); 2607 nvme_qpair->ctrlr->ref++; 2608 pthread_mutex_unlock(&nvme_qpair->ctrlr->mutex); 2609 2610 return 0; 2611 } 2612 2613 static int 2614 bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) 2615 { 2616 struct nvme_ctrlr *nvme_ctrlr = io_device; 2617 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 2618 2619 TAILQ_INIT(&ctrlr_ch->pending_resets); 2620 2621 return nvme_qpair_create(nvme_ctrlr, ctrlr_ch); 2622 } 2623 2624 static void 2625 nvme_qpair_delete(struct nvme_qpair *nvme_qpair) 2626 { 2627 assert(nvme_qpair->group != NULL); 2628 2629 TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 2630 2631 spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group)); 2632 2633 nvme_ctrlr_release(nvme_qpair->ctrlr); 2634 2635 free(nvme_qpair); 2636 } 2637 2638 static void 2639 bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf) 2640 { 2641 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 2642 struct nvme_qpair *nvme_qpair; 2643 2644 nvme_qpair = ctrlr_ch->qpair; 2645 assert(nvme_qpair != NULL); 2646 2647 _bdev_nvme_clear_io_path_cache(nvme_qpair); 2648 2649 if (nvme_qpair->qpair != NULL) { 2650 if (ctrlr_ch->reset_iter == NULL) { 2651 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 2652 } else { 2653 /* Skip current ctrlr_channel in a full reset sequence because 2654 * it is being deleted now. The qpair is already being disconnected. 2655 * We do not have to restart disconnecting it. 2656 */ 2657 spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 2658 } 2659 2660 /* We cannot release a reference to the poll group now. 2661 * The qpair may be disconnected asynchronously later. 2662 * We need to poll it until it is actually disconnected. 2663 * Just detach the qpair from the deleting ctrlr_channel. 2664 */ 2665 nvme_qpair->ctrlr_ch = NULL; 2666 } else { 2667 assert(ctrlr_ch->reset_iter == NULL); 2668 2669 nvme_qpair_delete(nvme_qpair); 2670 } 2671 } 2672 2673 static void 2674 bdev_nvme_submit_accel_crc32c(void *ctx, uint32_t *dst, struct iovec *iov, 2675 uint32_t iov_cnt, uint32_t seed, 2676 spdk_nvme_accel_completion_cb cb_fn, void *cb_arg) 2677 { 2678 struct nvme_poll_group *group = ctx; 2679 int rc; 2680 2681 assert(group->accel_channel != NULL); 2682 assert(cb_fn != NULL); 2683 2684 rc = spdk_accel_submit_crc32cv(group->accel_channel, dst, iov, iov_cnt, seed, cb_fn, cb_arg); 2685 if (rc) { 2686 /* For the two cases, spdk_accel_submit_crc32cv does not call the user's cb_fn */ 2687 if (rc == -ENOMEM || rc == -EINVAL) { 2688 cb_fn(cb_arg, rc); 2689 } 2690 SPDK_ERRLOG("Cannot complete the accelerated crc32c operation with iov=%p\n", iov); 2691 } 2692 } 2693 2694 static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = { 2695 .table_size = sizeof(struct spdk_nvme_accel_fn_table), 2696 .submit_accel_crc32c = bdev_nvme_submit_accel_crc32c, 2697 }; 2698 2699 static int 2700 bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf) 2701 { 2702 struct nvme_poll_group *group = ctx_buf; 2703 2704 TAILQ_INIT(&group->qpair_list); 2705 2706 group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table); 2707 if (group->group == NULL) { 2708 return -1; 2709 } 2710 2711 group->accel_channel = spdk_accel_get_io_channel(); 2712 if (!group->accel_channel) { 2713 spdk_nvme_poll_group_destroy(group->group); 2714 SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n", 2715 group); 2716 return -1; 2717 } 2718 2719 group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us); 2720 2721 if (group->poller == NULL) { 2722 spdk_put_io_channel(group->accel_channel); 2723 spdk_nvme_poll_group_destroy(group->group); 2724 return -1; 2725 } 2726 2727 return 0; 2728 } 2729 2730 static void 2731 bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf) 2732 { 2733 struct nvme_poll_group *group = ctx_buf; 2734 2735 assert(TAILQ_EMPTY(&group->qpair_list)); 2736 2737 if (group->accel_channel) { 2738 spdk_put_io_channel(group->accel_channel); 2739 } 2740 2741 spdk_poller_unregister(&group->poller); 2742 if (spdk_nvme_poll_group_destroy(group->group)) { 2743 SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n"); 2744 assert(false); 2745 } 2746 } 2747 2748 static struct spdk_io_channel * 2749 bdev_nvme_get_io_channel(void *ctx) 2750 { 2751 struct nvme_bdev *nvme_bdev = ctx; 2752 2753 return spdk_get_io_channel(nvme_bdev); 2754 } 2755 2756 static void * 2757 bdev_nvme_get_module_ctx(void *ctx) 2758 { 2759 struct nvme_bdev *nvme_bdev = ctx; 2760 struct nvme_ns *nvme_ns; 2761 2762 if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) { 2763 return NULL; 2764 } 2765 2766 nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list); 2767 if (!nvme_ns) { 2768 return NULL; 2769 } 2770 2771 return nvme_ns->ns; 2772 } 2773 2774 static const char * 2775 _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state) 2776 { 2777 switch (ana_state) { 2778 case SPDK_NVME_ANA_OPTIMIZED_STATE: 2779 return "optimized"; 2780 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 2781 return "non_optimized"; 2782 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 2783 return "inaccessible"; 2784 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 2785 return "persistent_loss"; 2786 case SPDK_NVME_ANA_CHANGE_STATE: 2787 return "change"; 2788 default: 2789 return NULL; 2790 } 2791 } 2792 2793 static int 2794 bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size) 2795 { 2796 struct spdk_memory_domain **_domains = NULL; 2797 struct nvme_bdev *nbdev = ctx; 2798 struct nvme_ns *nvme_ns; 2799 int i = 0, _array_size = array_size; 2800 int rc = 0; 2801 2802 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 2803 if (domains && array_size >= i) { 2804 _domains = &domains[i]; 2805 } else { 2806 _domains = NULL; 2807 } 2808 rc = spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, _domains, _array_size); 2809 if (rc > 0) { 2810 i += rc; 2811 if (_array_size >= rc) { 2812 _array_size -= rc; 2813 } else { 2814 _array_size = 0; 2815 } 2816 } else if (rc < 0) { 2817 return rc; 2818 } 2819 } 2820 2821 return i; 2822 } 2823 2824 static const char * 2825 nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr) 2826 { 2827 if (nvme_ctrlr->destruct) { 2828 return "deleting"; 2829 } else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 2830 return "failed"; 2831 } else if (nvme_ctrlr->resetting) { 2832 return "resetting"; 2833 } else if (nvme_ctrlr->reconnect_is_delayed > 0) { 2834 return "reconnect_is_delayed"; 2835 } else { 2836 return "enabled"; 2837 } 2838 } 2839 2840 void 2841 nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr) 2842 { 2843 struct spdk_nvme_transport_id *trid; 2844 const struct spdk_nvme_ctrlr_opts *opts; 2845 const struct spdk_nvme_ctrlr_data *cdata; 2846 2847 spdk_json_write_object_begin(w); 2848 2849 spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr)); 2850 2851 #ifdef SPDK_CONFIG_NVME_CUSE 2852 size_t cuse_name_size = 128; 2853 char cuse_name[cuse_name_size]; 2854 2855 int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size); 2856 if (rc == 0) { 2857 spdk_json_write_named_string(w, "cuse_device", cuse_name); 2858 } 2859 #endif 2860 trid = &nvme_ctrlr->active_path_id->trid; 2861 spdk_json_write_named_object_begin(w, "trid"); 2862 nvme_bdev_dump_trid_json(trid, w); 2863 spdk_json_write_object_end(w); 2864 2865 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 2866 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 2867 2868 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 2869 spdk_json_write_named_object_begin(w, "host"); 2870 spdk_json_write_named_string(w, "nqn", opts->hostnqn); 2871 spdk_json_write_named_string(w, "addr", opts->src_addr); 2872 spdk_json_write_named_string(w, "svcid", opts->src_svcid); 2873 spdk_json_write_object_end(w); 2874 2875 spdk_json_write_object_end(w); 2876 } 2877 2878 static void 2879 nvme_namespace_info_json(struct spdk_json_write_ctx *w, 2880 struct nvme_ns *nvme_ns) 2881 { 2882 struct spdk_nvme_ns *ns; 2883 struct spdk_nvme_ctrlr *ctrlr; 2884 const struct spdk_nvme_ctrlr_data *cdata; 2885 const struct spdk_nvme_transport_id *trid; 2886 union spdk_nvme_vs_register vs; 2887 const struct spdk_nvme_ns_data *nsdata; 2888 char buf[128]; 2889 2890 ns = nvme_ns->ns; 2891 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 2892 2893 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2894 trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); 2895 vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr); 2896 2897 spdk_json_write_object_begin(w); 2898 2899 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 2900 spdk_json_write_named_string(w, "pci_address", trid->traddr); 2901 } 2902 2903 spdk_json_write_named_object_begin(w, "trid"); 2904 2905 nvme_bdev_dump_trid_json(trid, w); 2906 2907 spdk_json_write_object_end(w); 2908 2909 #ifdef SPDK_CONFIG_NVME_CUSE 2910 size_t cuse_name_size = 128; 2911 char cuse_name[cuse_name_size]; 2912 2913 int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns), 2914 cuse_name, &cuse_name_size); 2915 if (rc == 0) { 2916 spdk_json_write_named_string(w, "cuse_device", cuse_name); 2917 } 2918 #endif 2919 2920 spdk_json_write_named_object_begin(w, "ctrlr_data"); 2921 2922 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 2923 2924 spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid); 2925 2926 snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn); 2927 spdk_str_trim(buf); 2928 spdk_json_write_named_string(w, "model_number", buf); 2929 2930 snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn); 2931 spdk_str_trim(buf); 2932 spdk_json_write_named_string(w, "serial_number", buf); 2933 2934 snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr); 2935 spdk_str_trim(buf); 2936 spdk_json_write_named_string(w, "firmware_revision", buf); 2937 2938 if (cdata->subnqn[0] != '\0') { 2939 spdk_json_write_named_string(w, "subnqn", cdata->subnqn); 2940 } 2941 2942 spdk_json_write_named_object_begin(w, "oacs"); 2943 2944 spdk_json_write_named_uint32(w, "security", cdata->oacs.security); 2945 spdk_json_write_named_uint32(w, "format", cdata->oacs.format); 2946 spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware); 2947 spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage); 2948 2949 spdk_json_write_object_end(w); 2950 2951 spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr); 2952 spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting); 2953 2954 spdk_json_write_object_end(w); 2955 2956 spdk_json_write_named_object_begin(w, "vs"); 2957 2958 spdk_json_write_name(w, "nvme_version"); 2959 if (vs.bits.ter) { 2960 spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter); 2961 } else { 2962 spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr); 2963 } 2964 2965 spdk_json_write_object_end(w); 2966 2967 nsdata = spdk_nvme_ns_get_data(ns); 2968 2969 spdk_json_write_named_object_begin(w, "ns_data"); 2970 2971 spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns)); 2972 2973 if (cdata->cmic.ana_reporting) { 2974 spdk_json_write_named_string(w, "ana_state", 2975 _nvme_ana_state_str(nvme_ns->ana_state)); 2976 } 2977 2978 spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share); 2979 2980 spdk_json_write_object_end(w); 2981 2982 if (cdata->oacs.security) { 2983 spdk_json_write_named_object_begin(w, "security"); 2984 2985 spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal); 2986 2987 spdk_json_write_object_end(w); 2988 } 2989 2990 spdk_json_write_object_end(w); 2991 } 2992 2993 static const char * 2994 nvme_bdev_get_mp_policy_str(struct nvme_bdev *nbdev) 2995 { 2996 switch (nbdev->mp_policy) { 2997 case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE: 2998 return "active_passive"; 2999 case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE: 3000 return "active_active"; 3001 default: 3002 assert(false); 3003 return "invalid"; 3004 } 3005 } 3006 3007 static int 3008 bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 3009 { 3010 struct nvme_bdev *nvme_bdev = ctx; 3011 struct nvme_ns *nvme_ns; 3012 3013 pthread_mutex_lock(&nvme_bdev->mutex); 3014 spdk_json_write_named_array_begin(w, "nvme"); 3015 TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) { 3016 nvme_namespace_info_json(w, nvme_ns); 3017 } 3018 spdk_json_write_array_end(w); 3019 spdk_json_write_named_string(w, "mp_policy", nvme_bdev_get_mp_policy_str(nvme_bdev)); 3020 pthread_mutex_unlock(&nvme_bdev->mutex); 3021 3022 return 0; 3023 } 3024 3025 static void 3026 bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 3027 { 3028 /* No config per bdev needed */ 3029 } 3030 3031 static uint64_t 3032 bdev_nvme_get_spin_time(struct spdk_io_channel *ch) 3033 { 3034 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 3035 struct nvme_io_path *io_path; 3036 struct nvme_poll_group *group; 3037 uint64_t spin_time = 0; 3038 3039 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 3040 group = io_path->qpair->group; 3041 3042 if (!group || !group->collect_spin_stat) { 3043 continue; 3044 } 3045 3046 if (group->end_ticks != 0) { 3047 group->spin_ticks += (group->end_ticks - group->start_ticks); 3048 group->end_ticks = 0; 3049 } 3050 3051 spin_time += group->spin_ticks; 3052 group->start_ticks = 0; 3053 group->spin_ticks = 0; 3054 } 3055 3056 return (spin_time * 1000000ULL) / spdk_get_ticks_hz(); 3057 } 3058 3059 static void 3060 bdev_nvme_reset_device_stat(void *ctx) 3061 { 3062 struct nvme_bdev *nbdev = ctx; 3063 3064 if (nbdev->err_stat != NULL) { 3065 memset(nbdev->err_stat, 0, sizeof(struct nvme_error_stat)); 3066 } 3067 } 3068 3069 /* JSON string should be lowercases and underscore delimited string. */ 3070 static void 3071 bdev_nvme_format_nvme_status(char *dst, const char *src) 3072 { 3073 char tmp[256]; 3074 3075 spdk_strcpy_replace(dst, 256, src, " - ", "_"); 3076 spdk_strcpy_replace(tmp, 256, dst, "-", "_"); 3077 spdk_strcpy_replace(dst, 256, tmp, " ", "_"); 3078 spdk_strlwr(dst); 3079 } 3080 3081 static void 3082 bdev_nvme_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w) 3083 { 3084 struct nvme_bdev *nbdev = ctx; 3085 struct spdk_nvme_status status = {}; 3086 uint16_t sct, sc; 3087 char status_json[256]; 3088 const char *status_str; 3089 3090 if (nbdev->err_stat == NULL) { 3091 return; 3092 } 3093 3094 spdk_json_write_named_object_begin(w, "nvme_error"); 3095 3096 spdk_json_write_named_object_begin(w, "status_type"); 3097 for (sct = 0; sct < 8; sct++) { 3098 if (nbdev->err_stat->status_type[sct] == 0) { 3099 continue; 3100 } 3101 status.sct = sct; 3102 3103 status_str = spdk_nvme_cpl_get_status_type_string(&status); 3104 assert(status_str != NULL); 3105 bdev_nvme_format_nvme_status(status_json, status_str); 3106 3107 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status_type[sct]); 3108 } 3109 spdk_json_write_object_end(w); 3110 3111 spdk_json_write_named_object_begin(w, "status_code"); 3112 for (sct = 0; sct < 4; sct++) { 3113 status.sct = sct; 3114 for (sc = 0; sc < 256; sc++) { 3115 if (nbdev->err_stat->status[sct][sc] == 0) { 3116 continue; 3117 } 3118 status.sc = sc; 3119 3120 status_str = spdk_nvme_cpl_get_status_string(&status); 3121 assert(status_str != NULL); 3122 bdev_nvme_format_nvme_status(status_json, status_str); 3123 3124 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status[sct][sc]); 3125 } 3126 } 3127 spdk_json_write_object_end(w); 3128 3129 spdk_json_write_object_end(w); 3130 } 3131 3132 static const struct spdk_bdev_fn_table nvmelib_fn_table = { 3133 .destruct = bdev_nvme_destruct, 3134 .submit_request = bdev_nvme_submit_request, 3135 .io_type_supported = bdev_nvme_io_type_supported, 3136 .get_io_channel = bdev_nvme_get_io_channel, 3137 .dump_info_json = bdev_nvme_dump_info_json, 3138 .write_config_json = bdev_nvme_write_config_json, 3139 .get_spin_time = bdev_nvme_get_spin_time, 3140 .get_module_ctx = bdev_nvme_get_module_ctx, 3141 .get_memory_domains = bdev_nvme_get_memory_domains, 3142 .reset_device_stat = bdev_nvme_reset_device_stat, 3143 .dump_device_stat_json = bdev_nvme_dump_device_stat_json, 3144 }; 3145 3146 typedef int (*bdev_nvme_parse_ana_log_page_cb)( 3147 const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg); 3148 3149 static int 3150 bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 3151 bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg) 3152 { 3153 struct spdk_nvme_ana_group_descriptor *copied_desc; 3154 uint8_t *orig_desc; 3155 uint32_t i, desc_size, copy_len; 3156 int rc = 0; 3157 3158 if (nvme_ctrlr->ana_log_page == NULL) { 3159 return -EINVAL; 3160 } 3161 3162 copied_desc = nvme_ctrlr->copied_ana_desc; 3163 3164 orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page); 3165 copy_len = nvme_ctrlr->max_ana_log_page_size - sizeof(struct spdk_nvme_ana_page); 3166 3167 for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) { 3168 memcpy(copied_desc, orig_desc, copy_len); 3169 3170 rc = cb_fn(copied_desc, cb_arg); 3171 if (rc != 0) { 3172 break; 3173 } 3174 3175 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) + 3176 copied_desc->num_of_nsid * sizeof(uint32_t); 3177 orig_desc += desc_size; 3178 copy_len -= desc_size; 3179 } 3180 3181 return rc; 3182 } 3183 3184 static int 3185 nvme_ns_ana_transition_timedout(void *ctx) 3186 { 3187 struct nvme_ns *nvme_ns = ctx; 3188 3189 spdk_poller_unregister(&nvme_ns->anatt_timer); 3190 nvme_ns->ana_transition_timedout = true; 3191 3192 return SPDK_POLLER_BUSY; 3193 } 3194 3195 static void 3196 _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns, 3197 const struct spdk_nvme_ana_group_descriptor *desc) 3198 { 3199 const struct spdk_nvme_ctrlr_data *cdata; 3200 3201 nvme_ns->ana_group_id = desc->ana_group_id; 3202 nvme_ns->ana_state = desc->ana_state; 3203 nvme_ns->ana_state_updating = false; 3204 3205 switch (nvme_ns->ana_state) { 3206 case SPDK_NVME_ANA_OPTIMIZED_STATE: 3207 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 3208 nvme_ns->ana_transition_timedout = false; 3209 spdk_poller_unregister(&nvme_ns->anatt_timer); 3210 break; 3211 3212 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3213 case SPDK_NVME_ANA_CHANGE_STATE: 3214 if (nvme_ns->anatt_timer != NULL) { 3215 break; 3216 } 3217 3218 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 3219 nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout, 3220 nvme_ns, 3221 cdata->anatt * SPDK_SEC_TO_USEC); 3222 break; 3223 default: 3224 break; 3225 } 3226 } 3227 3228 static int 3229 nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg) 3230 { 3231 struct nvme_ns *nvme_ns = cb_arg; 3232 uint32_t i; 3233 3234 for (i = 0; i < desc->num_of_nsid; i++) { 3235 if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) { 3236 continue; 3237 } 3238 3239 _nvme_ns_set_ana_state(nvme_ns, desc); 3240 return 1; 3241 } 3242 3243 return 0; 3244 } 3245 3246 static struct spdk_uuid 3247 nvme_generate_uuid(const char *sn, uint32_t nsid) 3248 { 3249 struct spdk_uuid new_uuid, namespace_uuid; 3250 char merged_str[SPDK_NVME_CTRLR_SN_LEN + NSID_STR_LEN + 1] = {'\0'}; 3251 /* This namespace UUID was generated using uuid_generate() method. */ 3252 const char *namespace_str = {"edaed2de-24bc-4b07-b559-f47ecbe730fd"}; 3253 int size; 3254 3255 assert(strlen(sn) <= SPDK_NVME_CTRLR_SN_LEN); 3256 3257 memset(&new_uuid, 0, sizeof(new_uuid)); 3258 memset(&namespace_uuid, 0, sizeof(namespace_uuid)); 3259 3260 size = snprintf(merged_str, sizeof(merged_str), "%s%"PRIu32, sn, nsid); 3261 assert(size > 0 && (unsigned long)size < sizeof(merged_str)); 3262 3263 spdk_uuid_parse(&namespace_uuid, namespace_str); 3264 3265 spdk_uuid_generate_sha1(&new_uuid, &namespace_uuid, merged_str, size); 3266 3267 return new_uuid; 3268 } 3269 3270 static int 3271 nvme_disk_create(struct spdk_bdev *disk, const char *base_name, 3272 struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns, 3273 uint32_t prchk_flags, void *ctx) 3274 { 3275 const struct spdk_uuid *uuid; 3276 const uint8_t *nguid; 3277 const struct spdk_nvme_ctrlr_data *cdata; 3278 const struct spdk_nvme_ns_data *nsdata; 3279 const struct spdk_nvme_ctrlr_opts *opts; 3280 enum spdk_nvme_csi csi; 3281 uint32_t atomic_bs, phys_bs, bs; 3282 char sn_tmp[SPDK_NVME_CTRLR_SN_LEN + 1] = {'\0'}; 3283 3284 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3285 csi = spdk_nvme_ns_get_csi(ns); 3286 opts = spdk_nvme_ctrlr_get_opts(ctrlr); 3287 3288 switch (csi) { 3289 case SPDK_NVME_CSI_NVM: 3290 disk->product_name = "NVMe disk"; 3291 break; 3292 case SPDK_NVME_CSI_ZNS: 3293 disk->product_name = "NVMe ZNS disk"; 3294 disk->zoned = true; 3295 disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 3296 disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) / 3297 spdk_nvme_ns_get_extended_sector_size(ns); 3298 disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns); 3299 disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns); 3300 break; 3301 default: 3302 SPDK_ERRLOG("unsupported CSI: %u\n", csi); 3303 return -ENOTSUP; 3304 } 3305 3306 disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns)); 3307 if (!disk->name) { 3308 return -ENOMEM; 3309 } 3310 3311 disk->write_cache = 0; 3312 if (cdata->vwc.present) { 3313 /* Enable if the Volatile Write Cache exists */ 3314 disk->write_cache = 1; 3315 } 3316 if (cdata->oncs.write_zeroes) { 3317 disk->max_write_zeroes = UINT16_MAX + 1; 3318 } 3319 disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns); 3320 disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns); 3321 disk->max_segment_size = spdk_nvme_ctrlr_get_max_xfer_size(ctrlr); 3322 /* NVMe driver will split one request into multiple requests 3323 * based on MDTS and stripe boundary, the bdev layer will use 3324 * max_segment_size and max_num_segments to split one big IO 3325 * into multiple requests, then small request can't run out 3326 * of NVMe internal requests data structure. 3327 */ 3328 if (opts && opts->io_queue_requests) { 3329 disk->max_num_segments = opts->io_queue_requests / 2; 3330 } 3331 disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns); 3332 3333 nguid = spdk_nvme_ns_get_nguid(ns); 3334 if (!nguid) { 3335 uuid = spdk_nvme_ns_get_uuid(ns); 3336 if (uuid) { 3337 disk->uuid = *uuid; 3338 } else if (g_opts.generate_uuids) { 3339 spdk_strcpy_pad(sn_tmp, cdata->sn, SPDK_NVME_CTRLR_SN_LEN, '\0'); 3340 disk->uuid = nvme_generate_uuid(sn_tmp, spdk_nvme_ns_get_id(ns)); 3341 } 3342 } else { 3343 memcpy(&disk->uuid, nguid, sizeof(disk->uuid)); 3344 } 3345 3346 nsdata = spdk_nvme_ns_get_data(ns); 3347 bs = spdk_nvme_ns_get_sector_size(ns); 3348 atomic_bs = bs; 3349 phys_bs = bs; 3350 if (nsdata->nabo == 0) { 3351 if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) { 3352 atomic_bs = bs * (1 + nsdata->nawupf); 3353 } else { 3354 atomic_bs = bs * (1 + cdata->awupf); 3355 } 3356 } 3357 if (nsdata->nsfeat.optperf) { 3358 phys_bs = bs * (1 + nsdata->npwg); 3359 } 3360 disk->phys_blocklen = spdk_min(phys_bs, atomic_bs); 3361 3362 disk->md_len = spdk_nvme_ns_get_md_size(ns); 3363 if (disk->md_len != 0) { 3364 disk->md_interleave = nsdata->flbas.extended; 3365 disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns); 3366 if (disk->dif_type != SPDK_DIF_DISABLE) { 3367 disk->dif_is_head_of_md = nsdata->dps.md_start; 3368 disk->dif_check_flags = prchk_flags; 3369 } 3370 } 3371 3372 if (!(spdk_nvme_ctrlr_get_flags(ctrlr) & 3373 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) { 3374 disk->acwu = 0; 3375 } else if (nsdata->nsfeat.ns_atomic_write_unit) { 3376 disk->acwu = nsdata->nacwu + 1; /* 0-based */ 3377 } else { 3378 disk->acwu = cdata->acwu + 1; /* 0-based */ 3379 } 3380 3381 if (cdata->oncs.copy) { 3382 /* For now bdev interface allows only single segment copy */ 3383 disk->max_copy = nsdata->mssrl; 3384 } 3385 3386 disk->ctxt = ctx; 3387 disk->fn_table = &nvmelib_fn_table; 3388 disk->module = &nvme_if; 3389 3390 return 0; 3391 } 3392 3393 static int 3394 nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 3395 { 3396 struct nvme_bdev *bdev; 3397 int rc; 3398 3399 bdev = calloc(1, sizeof(*bdev)); 3400 if (!bdev) { 3401 SPDK_ERRLOG("bdev calloc() failed\n"); 3402 return -ENOMEM; 3403 } 3404 3405 if (g_opts.nvme_error_stat) { 3406 bdev->err_stat = calloc(1, sizeof(struct nvme_error_stat)); 3407 if (!bdev->err_stat) { 3408 SPDK_ERRLOG("err_stat calloc() failed\n"); 3409 free(bdev); 3410 return -ENOMEM; 3411 } 3412 } 3413 3414 rc = pthread_mutex_init(&bdev->mutex, NULL); 3415 if (rc != 0) { 3416 free(bdev->err_stat); 3417 free(bdev); 3418 return rc; 3419 } 3420 3421 bdev->ref = 1; 3422 bdev->mp_policy = BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE; 3423 bdev->mp_selector = BDEV_NVME_MP_SELECTOR_ROUND_ROBIN; 3424 bdev->rr_min_io = UINT32_MAX; 3425 TAILQ_INIT(&bdev->nvme_ns_list); 3426 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 3427 bdev->opal = nvme_ctrlr->opal_dev != NULL; 3428 3429 rc = nvme_disk_create(&bdev->disk, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ctrlr, 3430 nvme_ns->ns, nvme_ctrlr->opts.prchk_flags, bdev); 3431 if (rc != 0) { 3432 SPDK_ERRLOG("Failed to create NVMe disk\n"); 3433 pthread_mutex_destroy(&bdev->mutex); 3434 free(bdev->err_stat); 3435 free(bdev); 3436 return rc; 3437 } 3438 3439 spdk_io_device_register(bdev, 3440 bdev_nvme_create_bdev_channel_cb, 3441 bdev_nvme_destroy_bdev_channel_cb, 3442 sizeof(struct nvme_bdev_channel), 3443 bdev->disk.name); 3444 3445 rc = spdk_bdev_register(&bdev->disk); 3446 if (rc != 0) { 3447 SPDK_ERRLOG("spdk_bdev_register() failed\n"); 3448 spdk_io_device_unregister(bdev, NULL); 3449 pthread_mutex_destroy(&bdev->mutex); 3450 free(bdev->disk.name); 3451 free(bdev->err_stat); 3452 free(bdev); 3453 return rc; 3454 } 3455 3456 nvme_ns->bdev = bdev; 3457 bdev->nsid = nvme_ns->id; 3458 3459 bdev->nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr; 3460 TAILQ_INSERT_TAIL(&nvme_ctrlr->nbdev_ctrlr->bdevs, bdev, tailq); 3461 3462 return 0; 3463 } 3464 3465 static bool 3466 bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2) 3467 { 3468 const struct spdk_nvme_ns_data *nsdata1, *nsdata2; 3469 const struct spdk_uuid *uuid1, *uuid2; 3470 3471 nsdata1 = spdk_nvme_ns_get_data(ns1); 3472 nsdata2 = spdk_nvme_ns_get_data(ns2); 3473 uuid1 = spdk_nvme_ns_get_uuid(ns1); 3474 uuid2 = spdk_nvme_ns_get_uuid(ns2); 3475 3476 return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 && 3477 nsdata1->eui64 == nsdata2->eui64 && 3478 ((uuid1 == NULL && uuid2 == NULL) || 3479 (uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) && 3480 spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2); 3481 } 3482 3483 static bool 3484 hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 3485 struct spdk_nvme_ctrlr_opts *opts) 3486 { 3487 struct nvme_probe_skip_entry *entry; 3488 3489 TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) { 3490 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 3491 return false; 3492 } 3493 } 3494 3495 opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst; 3496 opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight; 3497 opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight; 3498 opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight; 3499 opts->disable_read_ana_log_page = true; 3500 3501 SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr); 3502 3503 return true; 3504 } 3505 3506 static void 3507 nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl) 3508 { 3509 struct nvme_ctrlr *nvme_ctrlr = ctx; 3510 3511 if (spdk_nvme_cpl_is_error(cpl)) { 3512 SPDK_WARNLOG("Abort failed. Resetting controller. sc is %u, sct is %u.\n", cpl->status.sc, 3513 cpl->status.sct); 3514 bdev_nvme_reset(nvme_ctrlr); 3515 } else if (cpl->cdw0 & 0x1) { 3516 SPDK_WARNLOG("Specified command could not be aborted.\n"); 3517 bdev_nvme_reset(nvme_ctrlr); 3518 } 3519 } 3520 3521 static void 3522 timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, 3523 struct spdk_nvme_qpair *qpair, uint16_t cid) 3524 { 3525 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 3526 union spdk_nvme_csts_register csts; 3527 int rc; 3528 3529 assert(nvme_ctrlr->ctrlr == ctrlr); 3530 3531 SPDK_WARNLOG("Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n", ctrlr, qpair, cid); 3532 3533 /* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O 3534 * queue. (Note: qpair == NULL when there's an admin cmd timeout.) Otherwise we 3535 * would submit another fabrics cmd on the admin queue to read CSTS and check for its 3536 * completion recursively. 3537 */ 3538 if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) { 3539 csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr); 3540 if (csts.bits.cfs) { 3541 SPDK_ERRLOG("Controller Fatal Status, reset required\n"); 3542 bdev_nvme_reset(nvme_ctrlr); 3543 return; 3544 } 3545 } 3546 3547 switch (g_opts.action_on_timeout) { 3548 case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT: 3549 if (qpair) { 3550 /* Don't send abort to ctrlr when ctrlr is not available. */ 3551 pthread_mutex_lock(&nvme_ctrlr->mutex); 3552 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 3553 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3554 SPDK_NOTICELOG("Quit abort. Ctrlr is not available.\n"); 3555 return; 3556 } 3557 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3558 3559 rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid, 3560 nvme_abort_cpl, nvme_ctrlr); 3561 if (rc == 0) { 3562 return; 3563 } 3564 3565 SPDK_ERRLOG("Unable to send abort. Resetting, rc is %d.\n", rc); 3566 } 3567 3568 /* FALLTHROUGH */ 3569 case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET: 3570 bdev_nvme_reset(nvme_ctrlr); 3571 break; 3572 case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE: 3573 SPDK_DEBUGLOG(bdev_nvme, "No action for nvme controller timeout.\n"); 3574 break; 3575 default: 3576 SPDK_ERRLOG("An invalid timeout action value is found.\n"); 3577 break; 3578 } 3579 } 3580 3581 static struct nvme_ns * 3582 nvme_ns_alloc(void) 3583 { 3584 struct nvme_ns *nvme_ns; 3585 3586 nvme_ns = calloc(1, sizeof(struct nvme_ns)); 3587 if (nvme_ns == NULL) { 3588 return NULL; 3589 } 3590 3591 if (g_opts.io_path_stat) { 3592 nvme_ns->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 3593 if (nvme_ns->stat == NULL) { 3594 free(nvme_ns); 3595 return NULL; 3596 } 3597 spdk_bdev_reset_io_stat(nvme_ns->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 3598 } 3599 3600 return nvme_ns; 3601 } 3602 3603 static void 3604 nvme_ns_free(struct nvme_ns *nvme_ns) 3605 { 3606 free(nvme_ns->stat); 3607 free(nvme_ns); 3608 } 3609 3610 static void 3611 nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc) 3612 { 3613 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 3614 struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx; 3615 3616 if (rc == 0) { 3617 nvme_ns->probe_ctx = NULL; 3618 pthread_mutex_lock(&nvme_ctrlr->mutex); 3619 nvme_ctrlr->ref++; 3620 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3621 } else { 3622 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 3623 nvme_ns_free(nvme_ns); 3624 } 3625 3626 if (ctx) { 3627 ctx->populates_in_progress--; 3628 if (ctx->populates_in_progress == 0) { 3629 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 3630 } 3631 } 3632 } 3633 3634 static void 3635 bdev_nvme_add_io_path(struct spdk_io_channel_iter *i) 3636 { 3637 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 3638 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 3639 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3640 int rc; 3641 3642 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 3643 if (rc != 0) { 3644 SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n"); 3645 } 3646 3647 spdk_for_each_channel_continue(i, rc); 3648 } 3649 3650 static void 3651 bdev_nvme_delete_io_path(struct spdk_io_channel_iter *i) 3652 { 3653 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 3654 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 3655 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3656 struct nvme_io_path *io_path; 3657 3658 io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns); 3659 if (io_path != NULL) { 3660 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 3661 } 3662 3663 spdk_for_each_channel_continue(i, 0); 3664 } 3665 3666 static void 3667 bdev_nvme_add_io_path_failed(struct spdk_io_channel_iter *i, int status) 3668 { 3669 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3670 3671 nvme_ctrlr_populate_namespace_done(nvme_ns, -1); 3672 } 3673 3674 static void 3675 bdev_nvme_add_io_path_done(struct spdk_io_channel_iter *i, int status) 3676 { 3677 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3678 struct nvme_bdev *bdev = spdk_io_channel_iter_get_io_device(i); 3679 3680 if (status == 0) { 3681 nvme_ctrlr_populate_namespace_done(nvme_ns, 0); 3682 } else { 3683 /* Delete the added io_paths and fail populating the namespace. */ 3684 spdk_for_each_channel(bdev, 3685 bdev_nvme_delete_io_path, 3686 nvme_ns, 3687 bdev_nvme_add_io_path_failed); 3688 } 3689 } 3690 3691 static int 3692 nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns) 3693 { 3694 struct nvme_ns *tmp_ns; 3695 const struct spdk_nvme_ns_data *nsdata; 3696 3697 nsdata = spdk_nvme_ns_get_data(nvme_ns->ns); 3698 if (!nsdata->nmic.can_share) { 3699 SPDK_ERRLOG("Namespace cannot be shared.\n"); 3700 return -EINVAL; 3701 } 3702 3703 pthread_mutex_lock(&bdev->mutex); 3704 3705 tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list); 3706 assert(tmp_ns != NULL); 3707 3708 if (!bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) { 3709 pthread_mutex_unlock(&bdev->mutex); 3710 SPDK_ERRLOG("Namespaces are not identical.\n"); 3711 return -EINVAL; 3712 } 3713 3714 bdev->ref++; 3715 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 3716 nvme_ns->bdev = bdev; 3717 3718 pthread_mutex_unlock(&bdev->mutex); 3719 3720 /* Add nvme_io_path to nvme_bdev_channels dynamically. */ 3721 spdk_for_each_channel(bdev, 3722 bdev_nvme_add_io_path, 3723 nvme_ns, 3724 bdev_nvme_add_io_path_done); 3725 3726 return 0; 3727 } 3728 3729 static void 3730 nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 3731 { 3732 struct spdk_nvme_ns *ns; 3733 struct nvme_bdev *bdev; 3734 int rc = 0; 3735 3736 ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id); 3737 if (!ns) { 3738 SPDK_DEBUGLOG(bdev_nvme, "Invalid NS %d\n", nvme_ns->id); 3739 rc = -EINVAL; 3740 goto done; 3741 } 3742 3743 nvme_ns->ns = ns; 3744 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 3745 3746 if (nvme_ctrlr->ana_log_page != NULL) { 3747 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns); 3748 } 3749 3750 bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id); 3751 if (bdev == NULL) { 3752 rc = nvme_bdev_create(nvme_ctrlr, nvme_ns); 3753 } else { 3754 rc = nvme_bdev_add_ns(bdev, nvme_ns); 3755 if (rc == 0) { 3756 return; 3757 } 3758 } 3759 done: 3760 nvme_ctrlr_populate_namespace_done(nvme_ns, rc); 3761 } 3762 3763 static void 3764 nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns) 3765 { 3766 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 3767 3768 assert(nvme_ctrlr != NULL); 3769 3770 pthread_mutex_lock(&nvme_ctrlr->mutex); 3771 3772 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 3773 3774 if (nvme_ns->bdev != NULL) { 3775 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3776 return; 3777 } 3778 3779 nvme_ns_free(nvme_ns); 3780 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3781 3782 nvme_ctrlr_release(nvme_ctrlr); 3783 } 3784 3785 static void 3786 bdev_nvme_delete_io_path_done(struct spdk_io_channel_iter *i, int status) 3787 { 3788 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3789 3790 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 3791 } 3792 3793 static void 3794 nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 3795 { 3796 struct nvme_bdev *bdev; 3797 3798 spdk_poller_unregister(&nvme_ns->anatt_timer); 3799 3800 bdev = nvme_ns->bdev; 3801 if (bdev != NULL) { 3802 pthread_mutex_lock(&bdev->mutex); 3803 3804 assert(bdev->ref > 0); 3805 bdev->ref--; 3806 if (bdev->ref == 0) { 3807 pthread_mutex_unlock(&bdev->mutex); 3808 3809 spdk_bdev_unregister(&bdev->disk, NULL, NULL); 3810 } else { 3811 /* spdk_bdev_unregister() is not called until the last nvme_ns is 3812 * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list 3813 * and clear nvme_ns->bdev here. 3814 */ 3815 TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq); 3816 nvme_ns->bdev = NULL; 3817 3818 pthread_mutex_unlock(&bdev->mutex); 3819 3820 /* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that, 3821 * we call depopulate_namespace_done() to avoid use-after-free. 3822 */ 3823 spdk_for_each_channel(bdev, 3824 bdev_nvme_delete_io_path, 3825 nvme_ns, 3826 bdev_nvme_delete_io_path_done); 3827 return; 3828 } 3829 } 3830 3831 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 3832 } 3833 3834 static void 3835 nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 3836 struct nvme_async_probe_ctx *ctx) 3837 { 3838 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 3839 struct nvme_ns *nvme_ns, *next; 3840 struct spdk_nvme_ns *ns; 3841 struct nvme_bdev *bdev; 3842 uint32_t nsid; 3843 int rc; 3844 uint64_t num_sectors; 3845 3846 if (ctx) { 3847 /* Initialize this count to 1 to handle the populate functions 3848 * calling nvme_ctrlr_populate_namespace_done() immediately. 3849 */ 3850 ctx->populates_in_progress = 1; 3851 } 3852 3853 /* First loop over our existing namespaces and see if they have been 3854 * removed. */ 3855 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 3856 while (nvme_ns != NULL) { 3857 next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 3858 3859 if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 3860 /* NS is still there but attributes may have changed */ 3861 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id); 3862 num_sectors = spdk_nvme_ns_get_num_sectors(ns); 3863 bdev = nvme_ns->bdev; 3864 assert(bdev != NULL); 3865 if (bdev->disk.blockcnt != num_sectors) { 3866 SPDK_NOTICELOG("NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n", 3867 nvme_ns->id, 3868 bdev->disk.name, 3869 bdev->disk.blockcnt, 3870 num_sectors); 3871 rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors); 3872 if (rc != 0) { 3873 SPDK_ERRLOG("Could not change num blocks for nvme bdev: name %s, errno: %d.\n", 3874 bdev->disk.name, rc); 3875 } 3876 } 3877 } else { 3878 /* Namespace was removed */ 3879 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 3880 } 3881 3882 nvme_ns = next; 3883 } 3884 3885 /* Loop through all of the namespaces at the nvme level and see if any of them are new */ 3886 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 3887 while (nsid != 0) { 3888 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 3889 3890 if (nvme_ns == NULL) { 3891 /* Found a new one */ 3892 nvme_ns = nvme_ns_alloc(); 3893 if (nvme_ns == NULL) { 3894 SPDK_ERRLOG("Failed to allocate namespace\n"); 3895 /* This just fails to attach the namespace. It may work on a future attempt. */ 3896 continue; 3897 } 3898 3899 nvme_ns->id = nsid; 3900 nvme_ns->ctrlr = nvme_ctrlr; 3901 3902 nvme_ns->bdev = NULL; 3903 3904 if (ctx) { 3905 ctx->populates_in_progress++; 3906 } 3907 nvme_ns->probe_ctx = ctx; 3908 3909 RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 3910 3911 nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns); 3912 } 3913 3914 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid); 3915 } 3916 3917 if (ctx) { 3918 /* Decrement this count now that the loop is over to account 3919 * for the one we started with. If the count is then 0, we 3920 * know any populate_namespace functions completed immediately, 3921 * so we'll kick the callback here. 3922 */ 3923 ctx->populates_in_progress--; 3924 if (ctx->populates_in_progress == 0) { 3925 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 3926 } 3927 } 3928 3929 } 3930 3931 static void 3932 nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr) 3933 { 3934 struct nvme_ns *nvme_ns, *tmp; 3935 3936 RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) { 3937 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 3938 } 3939 } 3940 3941 static uint32_t 3942 nvme_ctrlr_get_ana_log_page_size(struct nvme_ctrlr *nvme_ctrlr) 3943 { 3944 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 3945 const struct spdk_nvme_ctrlr_data *cdata; 3946 uint32_t nsid, ns_count = 0; 3947 3948 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3949 3950 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 3951 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 3952 ns_count++; 3953 } 3954 3955 return sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 3956 sizeof(struct spdk_nvme_ana_group_descriptor) + ns_count * 3957 sizeof(uint32_t); 3958 } 3959 3960 static int 3961 nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc, 3962 void *cb_arg) 3963 { 3964 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 3965 struct nvme_ns *nvme_ns; 3966 uint32_t i, nsid; 3967 3968 for (i = 0; i < desc->num_of_nsid; i++) { 3969 nsid = desc->nsid[i]; 3970 if (nsid == 0) { 3971 continue; 3972 } 3973 3974 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 3975 3976 assert(nvme_ns != NULL); 3977 if (nvme_ns == NULL) { 3978 /* Target told us that an inactive namespace had an ANA change */ 3979 continue; 3980 } 3981 3982 _nvme_ns_set_ana_state(nvme_ns, desc); 3983 } 3984 3985 return 0; 3986 } 3987 3988 static void 3989 bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 3990 { 3991 struct nvme_ns *nvme_ns; 3992 3993 spdk_free(nvme_ctrlr->ana_log_page); 3994 nvme_ctrlr->ana_log_page = NULL; 3995 3996 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 3997 nvme_ns != NULL; 3998 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 3999 nvme_ns->ana_state_updating = false; 4000 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 4001 } 4002 } 4003 4004 static void 4005 nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl) 4006 { 4007 struct nvme_ctrlr *nvme_ctrlr = ctx; 4008 4009 if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) { 4010 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states, 4011 nvme_ctrlr); 4012 } else { 4013 bdev_nvme_disable_read_ana_log_page(nvme_ctrlr); 4014 } 4015 4016 pthread_mutex_lock(&nvme_ctrlr->mutex); 4017 4018 assert(nvme_ctrlr->ana_log_page_updating == true); 4019 nvme_ctrlr->ana_log_page_updating = false; 4020 4021 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 4022 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4023 4024 nvme_ctrlr_unregister(nvme_ctrlr); 4025 } else { 4026 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4027 4028 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 4029 } 4030 } 4031 4032 static int 4033 nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 4034 { 4035 uint32_t ana_log_page_size; 4036 int rc; 4037 4038 if (nvme_ctrlr->ana_log_page == NULL) { 4039 return -EINVAL; 4040 } 4041 4042 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 4043 4044 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 4045 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 4046 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 4047 return -EINVAL; 4048 } 4049 4050 pthread_mutex_lock(&nvme_ctrlr->mutex); 4051 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 4052 nvme_ctrlr->ana_log_page_updating) { 4053 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4054 return -EBUSY; 4055 } 4056 4057 nvme_ctrlr->ana_log_page_updating = true; 4058 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4059 4060 rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr, 4061 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 4062 SPDK_NVME_GLOBAL_NS_TAG, 4063 nvme_ctrlr->ana_log_page, 4064 ana_log_page_size, 0, 4065 nvme_ctrlr_read_ana_log_page_done, 4066 nvme_ctrlr); 4067 if (rc != 0) { 4068 nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL); 4069 } 4070 4071 return rc; 4072 } 4073 4074 static void 4075 dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 4076 { 4077 } 4078 4079 struct bdev_nvme_set_preferred_path_ctx { 4080 struct spdk_bdev_desc *desc; 4081 struct nvme_ns *nvme_ns; 4082 bdev_nvme_set_preferred_path_cb cb_fn; 4083 void *cb_arg; 4084 }; 4085 4086 static void 4087 bdev_nvme_set_preferred_path_done(struct spdk_io_channel_iter *i, int status) 4088 { 4089 struct bdev_nvme_set_preferred_path_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 4090 4091 assert(ctx != NULL); 4092 assert(ctx->desc != NULL); 4093 assert(ctx->cb_fn != NULL); 4094 4095 spdk_bdev_close(ctx->desc); 4096 4097 ctx->cb_fn(ctx->cb_arg, status); 4098 4099 free(ctx); 4100 } 4101 4102 static void 4103 _bdev_nvme_set_preferred_path(struct spdk_io_channel_iter *i) 4104 { 4105 struct bdev_nvme_set_preferred_path_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 4106 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 4107 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 4108 struct nvme_io_path *io_path, *prev; 4109 4110 prev = NULL; 4111 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 4112 if (io_path->nvme_ns == ctx->nvme_ns) { 4113 break; 4114 } 4115 prev = io_path; 4116 } 4117 4118 if (io_path != NULL) { 4119 if (prev != NULL) { 4120 STAILQ_REMOVE_AFTER(&nbdev_ch->io_path_list, prev, stailq); 4121 STAILQ_INSERT_HEAD(&nbdev_ch->io_path_list, io_path, stailq); 4122 } 4123 4124 /* We can set io_path to nbdev_ch->current_io_path directly here. 4125 * However, it needs to be conditional. To simplify the code, 4126 * just clear nbdev_ch->current_io_path and let find_io_path() 4127 * fill it. 4128 * 4129 * Automatic failback may be disabled. Hence even if the io_path is 4130 * already at the head, clear nbdev_ch->current_io_path. 4131 */ 4132 bdev_nvme_clear_current_io_path(nbdev_ch); 4133 } 4134 4135 spdk_for_each_channel_continue(i, 0); 4136 } 4137 4138 static struct nvme_ns * 4139 bdev_nvme_set_preferred_ns(struct nvme_bdev *nbdev, uint16_t cntlid) 4140 { 4141 struct nvme_ns *nvme_ns, *prev; 4142 const struct spdk_nvme_ctrlr_data *cdata; 4143 4144 prev = NULL; 4145 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 4146 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 4147 4148 if (cdata->cntlid == cntlid) { 4149 break; 4150 } 4151 prev = nvme_ns; 4152 } 4153 4154 if (nvme_ns != NULL && prev != NULL) { 4155 TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq); 4156 TAILQ_INSERT_HEAD(&nbdev->nvme_ns_list, nvme_ns, tailq); 4157 } 4158 4159 return nvme_ns; 4160 } 4161 4162 /* This function supports only multipath mode. There is only a single I/O path 4163 * for each NVMe-oF controller. Hence, just move the matched I/O path to the 4164 * head of the I/O path list for each NVMe bdev channel. 4165 * 4166 * NVMe bdev channel may be acquired after completing this function. move the 4167 * matched namespace to the head of the namespace list for the NVMe bdev too. 4168 */ 4169 void 4170 bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid, 4171 bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg) 4172 { 4173 struct bdev_nvme_set_preferred_path_ctx *ctx; 4174 struct spdk_bdev *bdev; 4175 struct nvme_bdev *nbdev; 4176 int rc = 0; 4177 4178 assert(cb_fn != NULL); 4179 4180 ctx = calloc(1, sizeof(*ctx)); 4181 if (ctx == NULL) { 4182 SPDK_ERRLOG("Failed to alloc context.\n"); 4183 rc = -ENOMEM; 4184 goto err_alloc; 4185 } 4186 4187 ctx->cb_fn = cb_fn; 4188 ctx->cb_arg = cb_arg; 4189 4190 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 4191 if (rc != 0) { 4192 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 4193 goto err_open; 4194 } 4195 4196 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 4197 4198 if (bdev->module != &nvme_if) { 4199 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 4200 rc = -ENODEV; 4201 goto err_bdev; 4202 } 4203 4204 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 4205 4206 pthread_mutex_lock(&nbdev->mutex); 4207 4208 ctx->nvme_ns = bdev_nvme_set_preferred_ns(nbdev, cntlid); 4209 if (ctx->nvme_ns == NULL) { 4210 pthread_mutex_unlock(&nbdev->mutex); 4211 4212 SPDK_ERRLOG("bdev %s does not have namespace to controller %u.\n", name, cntlid); 4213 rc = -ENODEV; 4214 goto err_bdev; 4215 } 4216 4217 pthread_mutex_unlock(&nbdev->mutex); 4218 4219 spdk_for_each_channel(nbdev, 4220 _bdev_nvme_set_preferred_path, 4221 ctx, 4222 bdev_nvme_set_preferred_path_done); 4223 return; 4224 4225 err_bdev: 4226 spdk_bdev_close(ctx->desc); 4227 err_open: 4228 free(ctx); 4229 err_alloc: 4230 cb_fn(cb_arg, rc); 4231 } 4232 4233 struct bdev_nvme_set_multipath_policy_ctx { 4234 struct spdk_bdev_desc *desc; 4235 bdev_nvme_set_multipath_policy_cb cb_fn; 4236 void *cb_arg; 4237 }; 4238 4239 static void 4240 bdev_nvme_set_multipath_policy_done(struct spdk_io_channel_iter *i, int status) 4241 { 4242 struct bdev_nvme_set_multipath_policy_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 4243 4244 assert(ctx != NULL); 4245 assert(ctx->desc != NULL); 4246 assert(ctx->cb_fn != NULL); 4247 4248 spdk_bdev_close(ctx->desc); 4249 4250 ctx->cb_fn(ctx->cb_arg, status); 4251 4252 free(ctx); 4253 } 4254 4255 static void 4256 _bdev_nvme_set_multipath_policy(struct spdk_io_channel_iter *i) 4257 { 4258 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 4259 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 4260 struct nvme_bdev *nbdev = spdk_io_channel_get_io_device(_ch); 4261 4262 nbdev_ch->mp_policy = nbdev->mp_policy; 4263 nbdev_ch->mp_selector = nbdev->mp_selector; 4264 nbdev_ch->rr_min_io = nbdev->rr_min_io; 4265 bdev_nvme_clear_current_io_path(nbdev_ch); 4266 4267 spdk_for_each_channel_continue(i, 0); 4268 } 4269 4270 void 4271 bdev_nvme_set_multipath_policy(const char *name, enum bdev_nvme_multipath_policy policy, 4272 enum bdev_nvme_multipath_selector selector, uint32_t rr_min_io, 4273 bdev_nvme_set_multipath_policy_cb cb_fn, void *cb_arg) 4274 { 4275 struct bdev_nvme_set_multipath_policy_ctx *ctx; 4276 struct spdk_bdev *bdev; 4277 struct nvme_bdev *nbdev; 4278 int rc; 4279 4280 assert(cb_fn != NULL); 4281 4282 if (policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE && selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 4283 if (rr_min_io == UINT32_MAX) { 4284 rr_min_io = 1; 4285 } else if (rr_min_io == 0) { 4286 rc = -EINVAL; 4287 goto exit; 4288 } 4289 } else if (rr_min_io != UINT32_MAX) { 4290 rc = -EINVAL; 4291 goto exit; 4292 } 4293 4294 ctx = calloc(1, sizeof(*ctx)); 4295 if (ctx == NULL) { 4296 SPDK_ERRLOG("Failed to alloc context.\n"); 4297 rc = -ENOMEM; 4298 goto exit; 4299 } 4300 4301 ctx->cb_fn = cb_fn; 4302 ctx->cb_arg = cb_arg; 4303 4304 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 4305 if (rc != 0) { 4306 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 4307 rc = -ENODEV; 4308 goto err_open; 4309 } 4310 4311 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 4312 if (bdev->module != &nvme_if) { 4313 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 4314 rc = -ENODEV; 4315 goto err_module; 4316 } 4317 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 4318 4319 pthread_mutex_lock(&nbdev->mutex); 4320 nbdev->mp_policy = policy; 4321 nbdev->mp_selector = selector; 4322 nbdev->rr_min_io = rr_min_io; 4323 pthread_mutex_unlock(&nbdev->mutex); 4324 4325 spdk_for_each_channel(nbdev, 4326 _bdev_nvme_set_multipath_policy, 4327 ctx, 4328 bdev_nvme_set_multipath_policy_done); 4329 return; 4330 4331 err_module: 4332 spdk_bdev_close(ctx->desc); 4333 err_open: 4334 free(ctx); 4335 exit: 4336 cb_fn(cb_arg, rc); 4337 } 4338 4339 static void 4340 aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 4341 { 4342 struct nvme_ctrlr *nvme_ctrlr = arg; 4343 union spdk_nvme_async_event_completion event; 4344 4345 if (spdk_nvme_cpl_is_error(cpl)) { 4346 SPDK_WARNLOG("AER request execute failed\n"); 4347 return; 4348 } 4349 4350 event.raw = cpl->cdw0; 4351 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 4352 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) { 4353 nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL); 4354 } else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 4355 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) { 4356 nvme_ctrlr_read_ana_log_page(nvme_ctrlr); 4357 } 4358 } 4359 4360 static void 4361 populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, size_t count, int rc) 4362 { 4363 if (ctx->cb_fn) { 4364 ctx->cb_fn(ctx->cb_ctx, count, rc); 4365 } 4366 4367 ctx->namespaces_populated = true; 4368 if (ctx->probe_done) { 4369 /* The probe was already completed, so we need to free the context 4370 * here. This can happen for cases like OCSSD, where we need to 4371 * send additional commands to the SSD after attach. 4372 */ 4373 free(ctx); 4374 } 4375 } 4376 4377 static void 4378 nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr, 4379 struct nvme_async_probe_ctx *ctx) 4380 { 4381 spdk_io_device_register(nvme_ctrlr, 4382 bdev_nvme_create_ctrlr_channel_cb, 4383 bdev_nvme_destroy_ctrlr_channel_cb, 4384 sizeof(struct nvme_ctrlr_channel), 4385 nvme_ctrlr->nbdev_ctrlr->name); 4386 4387 nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx); 4388 } 4389 4390 static void 4391 nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl) 4392 { 4393 struct nvme_ctrlr *nvme_ctrlr = _ctx; 4394 struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx; 4395 4396 nvme_ctrlr->probe_ctx = NULL; 4397 4398 if (spdk_nvme_cpl_is_error(cpl)) { 4399 nvme_ctrlr_delete(nvme_ctrlr); 4400 4401 if (ctx != NULL) { 4402 populate_namespaces_cb(ctx, 0, -1); 4403 } 4404 return; 4405 } 4406 4407 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 4408 } 4409 4410 static int 4411 nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 4412 struct nvme_async_probe_ctx *ctx) 4413 { 4414 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4415 const struct spdk_nvme_ctrlr_data *cdata; 4416 uint32_t ana_log_page_size; 4417 4418 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4419 4420 /* Set buffer size enough to include maximum number of allowed namespaces. */ 4421 ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 4422 sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->mnan * 4423 sizeof(uint32_t); 4424 4425 nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL, 4426 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4427 if (nvme_ctrlr->ana_log_page == NULL) { 4428 SPDK_ERRLOG("could not allocate ANA log page buffer\n"); 4429 return -ENXIO; 4430 } 4431 4432 /* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned. 4433 * Hence copy each descriptor to a temporary area when parsing it. 4434 * 4435 * Allocate a buffer whose size is as large as ANA log page buffer because 4436 * we do not know the size of a descriptor until actually reading it. 4437 */ 4438 nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size); 4439 if (nvme_ctrlr->copied_ana_desc == NULL) { 4440 SPDK_ERRLOG("could not allocate a buffer to parse ANA descriptor\n"); 4441 return -ENOMEM; 4442 } 4443 4444 nvme_ctrlr->max_ana_log_page_size = ana_log_page_size; 4445 4446 nvme_ctrlr->probe_ctx = ctx; 4447 4448 /* Then, set the read size only to include the current active namespaces. */ 4449 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 4450 4451 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 4452 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 4453 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 4454 return -EINVAL; 4455 } 4456 4457 return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, 4458 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 4459 SPDK_NVME_GLOBAL_NS_TAG, 4460 nvme_ctrlr->ana_log_page, 4461 ana_log_page_size, 0, 4462 nvme_ctrlr_init_ana_log_page_done, 4463 nvme_ctrlr); 4464 } 4465 4466 /* hostnqn and subnqn were already verified before attaching a controller. 4467 * Hence check only the multipath capability and cntlid here. 4468 */ 4469 static bool 4470 bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr) 4471 { 4472 struct nvme_ctrlr *tmp; 4473 const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata; 4474 4475 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4476 4477 if (!cdata->cmic.multi_ctrlr) { 4478 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 4479 return false; 4480 } 4481 4482 TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) { 4483 tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr); 4484 4485 if (!tmp_cdata->cmic.multi_ctrlr) { 4486 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 4487 return false; 4488 } 4489 if (cdata->cntlid == tmp_cdata->cntlid) { 4490 SPDK_ERRLOG("cntlid %u are duplicated.\n", tmp_cdata->cntlid); 4491 return false; 4492 } 4493 } 4494 4495 return true; 4496 } 4497 4498 static int 4499 nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr) 4500 { 4501 struct nvme_bdev_ctrlr *nbdev_ctrlr; 4502 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4503 int rc = 0; 4504 4505 pthread_mutex_lock(&g_bdev_nvme_mutex); 4506 4507 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 4508 if (nbdev_ctrlr != NULL) { 4509 if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) { 4510 rc = -EINVAL; 4511 goto exit; 4512 } 4513 } else { 4514 nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr)); 4515 if (nbdev_ctrlr == NULL) { 4516 SPDK_ERRLOG("Failed to allocate nvme_bdev_ctrlr.\n"); 4517 rc = -ENOMEM; 4518 goto exit; 4519 } 4520 nbdev_ctrlr->name = strdup(name); 4521 if (nbdev_ctrlr->name == NULL) { 4522 SPDK_ERRLOG("Failed to allocate name of nvme_bdev_ctrlr.\n"); 4523 free(nbdev_ctrlr); 4524 goto exit; 4525 } 4526 TAILQ_INIT(&nbdev_ctrlr->ctrlrs); 4527 TAILQ_INIT(&nbdev_ctrlr->bdevs); 4528 TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 4529 } 4530 nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr; 4531 TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 4532 exit: 4533 pthread_mutex_unlock(&g_bdev_nvme_mutex); 4534 return rc; 4535 } 4536 4537 static int 4538 nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr, 4539 const char *name, 4540 const struct spdk_nvme_transport_id *trid, 4541 struct nvme_async_probe_ctx *ctx) 4542 { 4543 struct nvme_ctrlr *nvme_ctrlr; 4544 struct nvme_path_id *path_id; 4545 const struct spdk_nvme_ctrlr_data *cdata; 4546 int rc; 4547 4548 nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr)); 4549 if (nvme_ctrlr == NULL) { 4550 SPDK_ERRLOG("Failed to allocate device struct\n"); 4551 return -ENOMEM; 4552 } 4553 4554 rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL); 4555 if (rc != 0) { 4556 free(nvme_ctrlr); 4557 return rc; 4558 } 4559 4560 TAILQ_INIT(&nvme_ctrlr->trids); 4561 4562 RB_INIT(&nvme_ctrlr->namespaces); 4563 4564 path_id = calloc(1, sizeof(*path_id)); 4565 if (path_id == NULL) { 4566 SPDK_ERRLOG("Failed to allocate trid entry pointer\n"); 4567 rc = -ENOMEM; 4568 goto err; 4569 } 4570 4571 path_id->trid = *trid; 4572 if (ctx != NULL) { 4573 memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr)); 4574 memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid)); 4575 } 4576 nvme_ctrlr->active_path_id = path_id; 4577 TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link); 4578 4579 nvme_ctrlr->thread = spdk_get_thread(); 4580 nvme_ctrlr->ctrlr = ctrlr; 4581 nvme_ctrlr->ref = 1; 4582 4583 if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) { 4584 SPDK_ERRLOG("OCSSDs are not supported"); 4585 rc = -ENOTSUP; 4586 goto err; 4587 } 4588 4589 if (ctx != NULL) { 4590 memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts)); 4591 } else { 4592 bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts); 4593 } 4594 4595 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr, 4596 g_opts.nvme_adminq_poll_period_us); 4597 4598 if (g_opts.timeout_us > 0) { 4599 /* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */ 4600 /* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */ 4601 uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ? 4602 g_opts.timeout_us : g_opts.timeout_admin_us; 4603 spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us, 4604 adm_timeout_us, timeout_cb, nvme_ctrlr); 4605 } 4606 4607 spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr); 4608 spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr); 4609 4610 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 4611 SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) { 4612 nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr); 4613 } 4614 4615 rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr); 4616 if (rc != 0) { 4617 goto err; 4618 } 4619 4620 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4621 4622 if (cdata->cmic.ana_reporting) { 4623 rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx); 4624 if (rc == 0) { 4625 return 0; 4626 } 4627 } else { 4628 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 4629 return 0; 4630 } 4631 4632 err: 4633 nvme_ctrlr_delete(nvme_ctrlr); 4634 return rc; 4635 } 4636 4637 void 4638 bdev_nvme_get_default_ctrlr_opts(struct nvme_ctrlr_opts *opts) 4639 { 4640 opts->prchk_flags = 0; 4641 opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec; 4642 opts->reconnect_delay_sec = g_opts.reconnect_delay_sec; 4643 opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec; 4644 } 4645 4646 static void 4647 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 4648 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts) 4649 { 4650 char *name; 4651 4652 name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++); 4653 if (!name) { 4654 SPDK_ERRLOG("Failed to assign name to NVMe device\n"); 4655 return; 4656 } 4657 4658 if (nvme_ctrlr_create(ctrlr, name, trid, NULL) == 0) { 4659 SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name); 4660 } else { 4661 SPDK_ERRLOG("Failed to attach to %s (%s)\n", trid->traddr, name); 4662 } 4663 4664 free(name); 4665 } 4666 4667 static void 4668 _nvme_ctrlr_destruct(void *ctx) 4669 { 4670 struct nvme_ctrlr *nvme_ctrlr = ctx; 4671 4672 nvme_ctrlr_depopulate_namespaces(nvme_ctrlr); 4673 nvme_ctrlr_release(nvme_ctrlr); 4674 } 4675 4676 static int 4677 bdev_nvme_delete_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 4678 { 4679 struct nvme_probe_skip_entry *entry; 4680 4681 /* The controller's destruction was already started */ 4682 if (nvme_ctrlr->destruct) { 4683 return -EALREADY; 4684 } 4685 4686 if (!hotplug && 4687 nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 4688 entry = calloc(1, sizeof(*entry)); 4689 if (!entry) { 4690 return -ENOMEM; 4691 } 4692 entry->trid = nvme_ctrlr->active_path_id->trid; 4693 TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq); 4694 } 4695 4696 nvme_ctrlr->destruct = true; 4697 return 0; 4698 } 4699 4700 static int 4701 bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 4702 { 4703 int rc; 4704 4705 pthread_mutex_lock(&nvme_ctrlr->mutex); 4706 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, hotplug); 4707 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4708 4709 if (rc == 0) { 4710 _nvme_ctrlr_destruct(nvme_ctrlr); 4711 } else if (rc == -EALREADY) { 4712 rc = 0; 4713 } 4714 4715 return rc; 4716 } 4717 4718 static void 4719 remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr) 4720 { 4721 struct nvme_ctrlr *nvme_ctrlr = cb_ctx; 4722 4723 bdev_nvme_delete_ctrlr(nvme_ctrlr, true); 4724 } 4725 4726 static int 4727 bdev_nvme_hotplug_probe(void *arg) 4728 { 4729 if (g_hotplug_probe_ctx == NULL) { 4730 spdk_poller_unregister(&g_hotplug_probe_poller); 4731 return SPDK_POLLER_IDLE; 4732 } 4733 4734 if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) { 4735 g_hotplug_probe_ctx = NULL; 4736 spdk_poller_unregister(&g_hotplug_probe_poller); 4737 } 4738 4739 return SPDK_POLLER_BUSY; 4740 } 4741 4742 static int 4743 bdev_nvme_hotplug(void *arg) 4744 { 4745 struct spdk_nvme_transport_id trid_pcie; 4746 4747 if (g_hotplug_probe_ctx) { 4748 return SPDK_POLLER_BUSY; 4749 } 4750 4751 memset(&trid_pcie, 0, sizeof(trid_pcie)); 4752 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 4753 4754 g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL, 4755 hotplug_probe_cb, attach_cb, NULL); 4756 4757 if (g_hotplug_probe_ctx) { 4758 assert(g_hotplug_probe_poller == NULL); 4759 g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000); 4760 } 4761 4762 return SPDK_POLLER_BUSY; 4763 } 4764 4765 void 4766 bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts) 4767 { 4768 *opts = g_opts; 4769 } 4770 4771 static bool bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 4772 uint32_t reconnect_delay_sec, 4773 uint32_t fast_io_fail_timeout_sec); 4774 4775 static int 4776 bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts) 4777 { 4778 if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) { 4779 /* Can't set timeout_admin_us without also setting timeout_us */ 4780 SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n"); 4781 return -EINVAL; 4782 } 4783 4784 if (opts->bdev_retry_count < -1) { 4785 SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n"); 4786 return -EINVAL; 4787 } 4788 4789 if (!bdev_nvme_check_io_error_resiliency_params(opts->ctrlr_loss_timeout_sec, 4790 opts->reconnect_delay_sec, 4791 opts->fast_io_fail_timeout_sec)) { 4792 return -EINVAL; 4793 } 4794 4795 return 0; 4796 } 4797 4798 int 4799 bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) 4800 { 4801 int ret; 4802 4803 ret = bdev_nvme_validate_opts(opts); 4804 if (ret) { 4805 SPDK_WARNLOG("Failed to set nvme opts.\n"); 4806 return ret; 4807 } 4808 4809 if (g_bdev_nvme_init_thread != NULL) { 4810 if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 4811 return -EPERM; 4812 } 4813 } 4814 4815 if (opts->rdma_srq_size != 0) { 4816 struct spdk_nvme_transport_opts drv_opts; 4817 4818 spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts)); 4819 drv_opts.rdma_srq_size = opts->rdma_srq_size; 4820 4821 ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts)); 4822 if (ret) { 4823 SPDK_ERRLOG("Failed to set NVMe transport opts.\n"); 4824 return ret; 4825 } 4826 } 4827 4828 g_opts = *opts; 4829 4830 return 0; 4831 } 4832 4833 struct set_nvme_hotplug_ctx { 4834 uint64_t period_us; 4835 bool enabled; 4836 spdk_msg_fn fn; 4837 void *fn_ctx; 4838 }; 4839 4840 static void 4841 set_nvme_hotplug_period_cb(void *_ctx) 4842 { 4843 struct set_nvme_hotplug_ctx *ctx = _ctx; 4844 4845 spdk_poller_unregister(&g_hotplug_poller); 4846 if (ctx->enabled) { 4847 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us); 4848 } 4849 4850 g_nvme_hotplug_poll_period_us = ctx->period_us; 4851 g_nvme_hotplug_enabled = ctx->enabled; 4852 if (ctx->fn) { 4853 ctx->fn(ctx->fn_ctx); 4854 } 4855 4856 free(ctx); 4857 } 4858 4859 int 4860 bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx) 4861 { 4862 struct set_nvme_hotplug_ctx *ctx; 4863 4864 if (enabled == true && !spdk_process_is_primary()) { 4865 return -EPERM; 4866 } 4867 4868 ctx = calloc(1, sizeof(*ctx)); 4869 if (ctx == NULL) { 4870 return -ENOMEM; 4871 } 4872 4873 period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us; 4874 ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX); 4875 ctx->enabled = enabled; 4876 ctx->fn = cb; 4877 ctx->fn_ctx = cb_ctx; 4878 4879 spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx); 4880 return 0; 4881 } 4882 4883 static void 4884 nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 4885 struct nvme_async_probe_ctx *ctx) 4886 { 4887 struct nvme_ns *nvme_ns; 4888 struct nvme_bdev *nvme_bdev; 4889 size_t j; 4890 4891 assert(nvme_ctrlr != NULL); 4892 4893 if (ctx->names == NULL) { 4894 populate_namespaces_cb(ctx, 0, 0); 4895 return; 4896 } 4897 4898 /* 4899 * Report the new bdevs that were created in this call. 4900 * There can be more than one bdev per NVMe controller. 4901 */ 4902 j = 0; 4903 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4904 while (nvme_ns != NULL) { 4905 nvme_bdev = nvme_ns->bdev; 4906 if (j < ctx->count) { 4907 ctx->names[j] = nvme_bdev->disk.name; 4908 j++; 4909 } else { 4910 SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n", 4911 ctx->count); 4912 populate_namespaces_cb(ctx, 0, -ERANGE); 4913 return; 4914 } 4915 4916 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 4917 } 4918 4919 populate_namespaces_cb(ctx, j, 0); 4920 } 4921 4922 static int 4923 bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 4924 struct spdk_nvme_ctrlr *new_ctrlr, 4925 struct spdk_nvme_transport_id *trid) 4926 { 4927 struct nvme_path_id *tmp_trid; 4928 4929 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 4930 SPDK_ERRLOG("PCIe failover is not supported.\n"); 4931 return -ENOTSUP; 4932 } 4933 4934 /* Currently we only support failover to the same transport type. */ 4935 if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) { 4936 SPDK_WARNLOG("Failover from trtype: %s to a different trtype: %s is not supported currently\n", 4937 spdk_nvme_transport_id_trtype_str(nvme_ctrlr->active_path_id->trid.trtype), 4938 spdk_nvme_transport_id_trtype_str(trid->trtype)); 4939 return -EINVAL; 4940 } 4941 4942 4943 /* Currently we only support failover to the same NQN. */ 4944 if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) { 4945 SPDK_WARNLOG("Failover from subnqn: %s to a different subnqn: %s is not supported currently\n", 4946 nvme_ctrlr->active_path_id->trid.subnqn, trid->subnqn); 4947 return -EINVAL; 4948 } 4949 4950 /* Skip all the other checks if we've already registered this path. */ 4951 TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) { 4952 if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) { 4953 SPDK_WARNLOG("This path (traddr: %s subnqn: %s) is already registered\n", trid->traddr, 4954 trid->subnqn); 4955 return -EEXIST; 4956 } 4957 } 4958 4959 return 0; 4960 } 4961 4962 static int 4963 bdev_nvme_check_secondary_namespace(struct nvme_ctrlr *nvme_ctrlr, 4964 struct spdk_nvme_ctrlr *new_ctrlr) 4965 { 4966 struct nvme_ns *nvme_ns; 4967 struct spdk_nvme_ns *new_ns; 4968 4969 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4970 while (nvme_ns != NULL) { 4971 new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id); 4972 assert(new_ns != NULL); 4973 4974 if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) { 4975 return -EINVAL; 4976 } 4977 4978 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 4979 } 4980 4981 return 0; 4982 } 4983 4984 static int 4985 _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 4986 struct spdk_nvme_transport_id *trid) 4987 { 4988 struct nvme_path_id *new_trid, *tmp_trid; 4989 4990 new_trid = calloc(1, sizeof(*new_trid)); 4991 if (new_trid == NULL) { 4992 return -ENOMEM; 4993 } 4994 new_trid->trid = *trid; 4995 new_trid->is_failed = false; 4996 4997 TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) { 4998 if (tmp_trid->is_failed && tmp_trid != nvme_ctrlr->active_path_id) { 4999 TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link); 5000 return 0; 5001 } 5002 } 5003 5004 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link); 5005 return 0; 5006 } 5007 5008 /* This is the case that a secondary path is added to an existing 5009 * nvme_ctrlr for failover. After checking if it can access the same 5010 * namespaces as the primary path, it is disconnected until failover occurs. 5011 */ 5012 static int 5013 bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 5014 struct spdk_nvme_ctrlr *new_ctrlr, 5015 struct spdk_nvme_transport_id *trid) 5016 { 5017 int rc; 5018 5019 assert(nvme_ctrlr != NULL); 5020 5021 pthread_mutex_lock(&nvme_ctrlr->mutex); 5022 5023 rc = bdev_nvme_check_secondary_trid(nvme_ctrlr, new_ctrlr, trid); 5024 if (rc != 0) { 5025 goto exit; 5026 } 5027 5028 rc = bdev_nvme_check_secondary_namespace(nvme_ctrlr, new_ctrlr); 5029 if (rc != 0) { 5030 goto exit; 5031 } 5032 5033 rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid); 5034 5035 exit: 5036 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5037 5038 spdk_nvme_detach(new_ctrlr); 5039 5040 return rc; 5041 } 5042 5043 static void 5044 connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 5045 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 5046 { 5047 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 5048 struct nvme_async_probe_ctx *ctx; 5049 int rc; 5050 5051 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 5052 ctx->ctrlr_attached = true; 5053 5054 rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx); 5055 if (rc != 0) { 5056 populate_namespaces_cb(ctx, 0, rc); 5057 } 5058 } 5059 5060 static void 5061 connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 5062 struct spdk_nvme_ctrlr *ctrlr, 5063 const struct spdk_nvme_ctrlr_opts *opts) 5064 { 5065 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 5066 struct nvme_ctrlr *nvme_ctrlr; 5067 struct nvme_async_probe_ctx *ctx; 5068 int rc; 5069 5070 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 5071 ctx->ctrlr_attached = true; 5072 5073 nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name); 5074 if (nvme_ctrlr) { 5075 rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid); 5076 } else { 5077 rc = -ENODEV; 5078 } 5079 5080 populate_namespaces_cb(ctx, 0, rc); 5081 } 5082 5083 static int 5084 bdev_nvme_async_poll(void *arg) 5085 { 5086 struct nvme_async_probe_ctx *ctx = arg; 5087 int rc; 5088 5089 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 5090 if (spdk_unlikely(rc != -EAGAIN)) { 5091 ctx->probe_done = true; 5092 spdk_poller_unregister(&ctx->poller); 5093 if (!ctx->ctrlr_attached) { 5094 /* The probe is done, but no controller was attached. 5095 * That means we had a failure, so report -EIO back to 5096 * the caller (usually the RPC). populate_namespaces_cb() 5097 * will take care of freeing the nvme_async_probe_ctx. 5098 */ 5099 populate_namespaces_cb(ctx, 0, -EIO); 5100 } else if (ctx->namespaces_populated) { 5101 /* The namespaces for the attached controller were all 5102 * populated and the response was already sent to the 5103 * caller (usually the RPC). So free the context here. 5104 */ 5105 free(ctx); 5106 } 5107 } 5108 5109 return SPDK_POLLER_BUSY; 5110 } 5111 5112 static bool 5113 bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 5114 uint32_t reconnect_delay_sec, 5115 uint32_t fast_io_fail_timeout_sec) 5116 { 5117 if (ctrlr_loss_timeout_sec < -1) { 5118 SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n"); 5119 return false; 5120 } else if (ctrlr_loss_timeout_sec == -1) { 5121 if (reconnect_delay_sec == 0) { 5122 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 5123 return false; 5124 } else if (fast_io_fail_timeout_sec != 0 && 5125 fast_io_fail_timeout_sec < reconnect_delay_sec) { 5126 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n"); 5127 return false; 5128 } 5129 } else if (ctrlr_loss_timeout_sec != 0) { 5130 if (reconnect_delay_sec == 0) { 5131 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 5132 return false; 5133 } else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) { 5134 SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n"); 5135 return false; 5136 } else if (fast_io_fail_timeout_sec != 0) { 5137 if (fast_io_fail_timeout_sec < reconnect_delay_sec) { 5138 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n"); 5139 return false; 5140 } else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) { 5141 SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n"); 5142 return false; 5143 } 5144 } 5145 } else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) { 5146 SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n"); 5147 return false; 5148 } 5149 5150 return true; 5151 } 5152 5153 int 5154 bdev_nvme_create(struct spdk_nvme_transport_id *trid, 5155 const char *base_name, 5156 const char **names, 5157 uint32_t count, 5158 spdk_bdev_create_nvme_fn cb_fn, 5159 void *cb_ctx, 5160 struct spdk_nvme_ctrlr_opts *drv_opts, 5161 struct nvme_ctrlr_opts *bdev_opts, 5162 bool multipath) 5163 { 5164 struct nvme_probe_skip_entry *entry, *tmp; 5165 struct nvme_async_probe_ctx *ctx; 5166 spdk_nvme_attach_cb attach_cb; 5167 5168 /* TODO expand this check to include both the host and target TRIDs. 5169 * Only if both are the same should we fail. 5170 */ 5171 if (nvme_ctrlr_get(trid) != NULL) { 5172 SPDK_ERRLOG("A controller with the provided trid (traddr: %s) already exists.\n", trid->traddr); 5173 return -EEXIST; 5174 } 5175 5176 if (bdev_opts != NULL && 5177 !bdev_nvme_check_io_error_resiliency_params(bdev_opts->ctrlr_loss_timeout_sec, 5178 bdev_opts->reconnect_delay_sec, 5179 bdev_opts->fast_io_fail_timeout_sec)) { 5180 return -EINVAL; 5181 } 5182 5183 ctx = calloc(1, sizeof(*ctx)); 5184 if (!ctx) { 5185 return -ENOMEM; 5186 } 5187 ctx->base_name = base_name; 5188 ctx->names = names; 5189 ctx->count = count; 5190 ctx->cb_fn = cb_fn; 5191 ctx->cb_ctx = cb_ctx; 5192 ctx->trid = *trid; 5193 5194 if (bdev_opts) { 5195 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 5196 } else { 5197 bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts); 5198 } 5199 5200 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 5201 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) { 5202 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 5203 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 5204 free(entry); 5205 break; 5206 } 5207 } 5208 } 5209 5210 if (drv_opts) { 5211 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 5212 } else { 5213 spdk_nvme_ctrlr_get_default_ctrlr_opts(&ctx->drv_opts, sizeof(ctx->drv_opts)); 5214 } 5215 5216 ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count; 5217 ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout; 5218 ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms; 5219 ctx->drv_opts.disable_read_ana_log_page = true; 5220 ctx->drv_opts.transport_tos = g_opts.transport_tos; 5221 5222 if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || multipath) { 5223 attach_cb = connect_attach_cb; 5224 } else { 5225 attach_cb = connect_set_failover_cb; 5226 } 5227 5228 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb); 5229 if (ctx->probe_ctx == NULL) { 5230 SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr); 5231 free(ctx); 5232 return -ENODEV; 5233 } 5234 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000); 5235 5236 return 0; 5237 } 5238 5239 static bool 5240 nvme_path_should_delete(struct nvme_path_id *p, const struct nvme_path_id *path_id) 5241 { 5242 if (path_id->trid.trtype != 0) { 5243 if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) { 5244 if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) { 5245 return false; 5246 } 5247 } else { 5248 if (path_id->trid.trtype != p->trid.trtype) { 5249 return false; 5250 } 5251 } 5252 } 5253 5254 if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) { 5255 if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) { 5256 return false; 5257 } 5258 } 5259 5260 if (path_id->trid.adrfam != 0) { 5261 if (path_id->trid.adrfam != p->trid.adrfam) { 5262 return false; 5263 } 5264 } 5265 5266 if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) { 5267 if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) { 5268 return false; 5269 } 5270 } 5271 5272 if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) { 5273 if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) { 5274 return false; 5275 } 5276 } 5277 5278 if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) { 5279 if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) { 5280 return false; 5281 } 5282 } 5283 5284 if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) { 5285 if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) { 5286 return false; 5287 } 5288 } 5289 5290 return true; 5291 } 5292 5293 static int 5294 _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id) 5295 { 5296 struct nvme_path_id *p, *t; 5297 int rc = -ENXIO; 5298 5299 TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) { 5300 if (p == TAILQ_FIRST(&nvme_ctrlr->trids)) { 5301 break; 5302 } 5303 5304 if (!nvme_path_should_delete(p, path_id)) { 5305 continue; 5306 } 5307 5308 /* We are not using the specified path. */ 5309 TAILQ_REMOVE(&nvme_ctrlr->trids, p, link); 5310 free(p); 5311 rc = 0; 5312 } 5313 5314 if (p == NULL || !nvme_path_should_delete(p, path_id)) { 5315 return rc; 5316 } 5317 5318 /* If we made it here, then this path is a match! Now we need to remove it. */ 5319 5320 /* This is the active path in use right now. The active path is always the first in the list. */ 5321 assert(p == nvme_ctrlr->active_path_id); 5322 5323 if (!TAILQ_NEXT(p, link)) { 5324 /* The current path is the only path. */ 5325 rc = bdev_nvme_delete_ctrlr(nvme_ctrlr, false); 5326 } else { 5327 /* There is an alternative path. */ 5328 rc = bdev_nvme_failover(nvme_ctrlr, true); 5329 } 5330 5331 return rc; 5332 } 5333 5334 int 5335 bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id) 5336 { 5337 struct nvme_bdev_ctrlr *nbdev_ctrlr; 5338 struct nvme_ctrlr *nvme_ctrlr, *tmp_nvme_ctrlr; 5339 int rc = -ENXIO, _rc; 5340 5341 if (name == NULL || path_id == NULL) { 5342 return -EINVAL; 5343 } 5344 5345 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 5346 if (nbdev_ctrlr == NULL) { 5347 SPDK_ERRLOG("Failed to find NVMe bdev controller\n"); 5348 return -ENODEV; 5349 } 5350 5351 TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) { 5352 _rc = _bdev_nvme_delete(nvme_ctrlr, path_id); 5353 if (_rc < 0 && _rc != -ENXIO) { 5354 return _rc; 5355 } else if (_rc == 0) { 5356 /* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr 5357 * was deleted successfully. To remember the successful deletion, 5358 * overwrite rc only if _rc is zero. 5359 */ 5360 rc = 0; 5361 } 5362 } 5363 5364 /* All nvme_ctrlrs were deleted or no nvme_ctrlr which had the trid was found. */ 5365 return rc; 5366 } 5367 5368 #define DISCOVERY_INFOLOG(ctx, format, ...) \ 5369 SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 5370 5371 #define DISCOVERY_ERRLOG(ctx, format, ...) \ 5372 SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 5373 5374 struct discovery_entry_ctx { 5375 char name[128]; 5376 struct spdk_nvme_transport_id trid; 5377 struct spdk_nvme_ctrlr_opts drv_opts; 5378 struct spdk_nvmf_discovery_log_page_entry entry; 5379 TAILQ_ENTRY(discovery_entry_ctx) tailq; 5380 struct discovery_ctx *ctx; 5381 }; 5382 5383 struct discovery_ctx { 5384 char *name; 5385 spdk_bdev_nvme_start_discovery_fn start_cb_fn; 5386 spdk_bdev_nvme_stop_discovery_fn stop_cb_fn; 5387 void *cb_ctx; 5388 struct spdk_nvme_probe_ctx *probe_ctx; 5389 struct spdk_nvme_detach_ctx *detach_ctx; 5390 struct spdk_nvme_ctrlr *ctrlr; 5391 struct spdk_nvme_transport_id trid; 5392 struct discovery_entry_ctx *entry_ctx_in_use; 5393 struct spdk_poller *poller; 5394 struct spdk_nvme_ctrlr_opts drv_opts; 5395 struct nvme_ctrlr_opts bdev_opts; 5396 struct spdk_nvmf_discovery_log_page *log_page; 5397 TAILQ_ENTRY(discovery_ctx) tailq; 5398 TAILQ_HEAD(, discovery_entry_ctx) nvm_entry_ctxs; 5399 TAILQ_HEAD(, discovery_entry_ctx) discovery_entry_ctxs; 5400 int rc; 5401 bool wait_for_attach; 5402 uint64_t timeout_ticks; 5403 /* Denotes that the discovery service is being started. We're waiting 5404 * for the initial connection to the discovery controller to be 5405 * established and attach discovered NVM ctrlrs. 5406 */ 5407 bool initializing; 5408 /* Denotes if a discovery is currently in progress for this context. 5409 * That includes connecting to newly discovered subsystems. Used to 5410 * ensure we do not start a new discovery until an existing one is 5411 * complete. 5412 */ 5413 bool in_progress; 5414 5415 /* Denotes if another discovery is needed after the one in progress 5416 * completes. Set when we receive an AER completion while a discovery 5417 * is already in progress. 5418 */ 5419 bool pending; 5420 5421 /* Signal to the discovery context poller that it should stop the 5422 * discovery service, including detaching from the current discovery 5423 * controller. 5424 */ 5425 bool stop; 5426 5427 struct spdk_thread *calling_thread; 5428 uint32_t index; 5429 uint32_t attach_in_progress; 5430 char *hostnqn; 5431 5432 /* Denotes if the discovery service was started by the mdns discovery. 5433 */ 5434 bool from_mdns_discovery_service; 5435 }; 5436 5437 TAILQ_HEAD(discovery_ctxs, discovery_ctx); 5438 static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs); 5439 5440 static void get_discovery_log_page(struct discovery_ctx *ctx); 5441 5442 static void 5443 free_discovery_ctx(struct discovery_ctx *ctx) 5444 { 5445 free(ctx->log_page); 5446 free(ctx->hostnqn); 5447 free(ctx->name); 5448 free(ctx); 5449 } 5450 5451 static void 5452 discovery_complete(struct discovery_ctx *ctx) 5453 { 5454 ctx->initializing = false; 5455 ctx->in_progress = false; 5456 if (ctx->pending) { 5457 ctx->pending = false; 5458 get_discovery_log_page(ctx); 5459 } 5460 } 5461 5462 static void 5463 build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid, 5464 struct spdk_nvmf_discovery_log_page_entry *entry) 5465 { 5466 char *space; 5467 5468 trid->trtype = entry->trtype; 5469 trid->adrfam = entry->adrfam; 5470 memcpy(trid->traddr, entry->traddr, sizeof(entry->traddr)); 5471 memcpy(trid->trsvcid, entry->trsvcid, sizeof(entry->trsvcid)); 5472 memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn)); 5473 5474 /* We want the traddr, trsvcid and subnqn fields to be NULL-terminated. 5475 * But the log page entries typically pad them with spaces, not zeroes. 5476 * So add a NULL terminator to each of these fields at the appropriate 5477 * location. 5478 */ 5479 space = strchr(trid->traddr, ' '); 5480 if (space) { 5481 *space = 0; 5482 } 5483 space = strchr(trid->trsvcid, ' '); 5484 if (space) { 5485 *space = 0; 5486 } 5487 space = strchr(trid->subnqn, ' '); 5488 if (space) { 5489 *space = 0; 5490 } 5491 } 5492 5493 static void 5494 stop_discovery(struct discovery_ctx *ctx, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 5495 { 5496 ctx->stop = true; 5497 ctx->stop_cb_fn = cb_fn; 5498 ctx->cb_ctx = cb_ctx; 5499 5500 while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) { 5501 struct discovery_entry_ctx *entry_ctx; 5502 struct nvme_path_id path = {}; 5503 5504 entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs); 5505 path.trid = entry_ctx->trid; 5506 bdev_nvme_delete(entry_ctx->name, &path); 5507 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 5508 free(entry_ctx); 5509 } 5510 5511 while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) { 5512 struct discovery_entry_ctx *entry_ctx; 5513 5514 entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 5515 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 5516 free(entry_ctx); 5517 } 5518 5519 free(ctx->entry_ctx_in_use); 5520 ctx->entry_ctx_in_use = NULL; 5521 } 5522 5523 static void 5524 discovery_remove_controllers(struct discovery_ctx *ctx) 5525 { 5526 struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page; 5527 struct discovery_entry_ctx *entry_ctx, *tmp; 5528 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 5529 struct spdk_nvme_transport_id old_trid; 5530 uint64_t numrec, i; 5531 bool found; 5532 5533 numrec = from_le64(&log_page->numrec); 5534 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) { 5535 found = false; 5536 old_entry = &entry_ctx->entry; 5537 build_trid_from_log_page_entry(&old_trid, old_entry); 5538 for (i = 0; i < numrec; i++) { 5539 new_entry = &log_page->entries[i]; 5540 if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) { 5541 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n", 5542 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 5543 found = true; 5544 break; 5545 } 5546 } 5547 if (!found) { 5548 struct nvme_path_id path = {}; 5549 5550 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n", 5551 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 5552 5553 path.trid = entry_ctx->trid; 5554 bdev_nvme_delete(entry_ctx->name, &path); 5555 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 5556 free(entry_ctx); 5557 } 5558 } 5559 free(log_page); 5560 ctx->log_page = NULL; 5561 discovery_complete(ctx); 5562 } 5563 5564 static void 5565 complete_discovery_start(struct discovery_ctx *ctx, int status) 5566 { 5567 ctx->timeout_ticks = 0; 5568 ctx->rc = status; 5569 if (ctx->start_cb_fn) { 5570 ctx->start_cb_fn(ctx->cb_ctx, status); 5571 ctx->start_cb_fn = NULL; 5572 ctx->cb_ctx = NULL; 5573 } 5574 } 5575 5576 static void 5577 discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc) 5578 { 5579 struct discovery_entry_ctx *entry_ctx = cb_ctx; 5580 struct discovery_ctx *ctx = entry_ctx->ctx; 5581 5582 DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name); 5583 ctx->attach_in_progress--; 5584 if (ctx->attach_in_progress == 0) { 5585 complete_discovery_start(ctx, ctx->rc); 5586 if (ctx->initializing && ctx->rc != 0) { 5587 DISCOVERY_ERRLOG(ctx, "stopping discovery due to errors: %d\n", ctx->rc); 5588 stop_discovery(ctx, NULL, ctx->cb_ctx); 5589 } else { 5590 discovery_remove_controllers(ctx); 5591 } 5592 } 5593 } 5594 5595 static struct discovery_entry_ctx * 5596 create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid) 5597 { 5598 struct discovery_entry_ctx *new_ctx; 5599 5600 new_ctx = calloc(1, sizeof(*new_ctx)); 5601 if (new_ctx == NULL) { 5602 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 5603 return NULL; 5604 } 5605 5606 new_ctx->ctx = ctx; 5607 memcpy(&new_ctx->trid, trid, sizeof(*trid)); 5608 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 5609 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 5610 return new_ctx; 5611 } 5612 5613 static void 5614 discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl, 5615 struct spdk_nvmf_discovery_log_page *log_page) 5616 { 5617 struct discovery_ctx *ctx = cb_arg; 5618 struct discovery_entry_ctx *entry_ctx, *tmp; 5619 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 5620 uint64_t numrec, i; 5621 bool found; 5622 5623 if (rc || spdk_nvme_cpl_is_error(cpl)) { 5624 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 5625 return; 5626 } 5627 5628 ctx->log_page = log_page; 5629 assert(ctx->attach_in_progress == 0); 5630 numrec = from_le64(&log_page->numrec); 5631 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) { 5632 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 5633 free(entry_ctx); 5634 } 5635 for (i = 0; i < numrec; i++) { 5636 found = false; 5637 new_entry = &log_page->entries[i]; 5638 if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 5639 struct discovery_entry_ctx *new_ctx; 5640 struct spdk_nvme_transport_id trid = {}; 5641 5642 build_trid_from_log_page_entry(&trid, new_entry); 5643 new_ctx = create_discovery_entry_ctx(ctx, &trid); 5644 if (new_ctx == NULL) { 5645 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 5646 break; 5647 } 5648 5649 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq); 5650 continue; 5651 } 5652 TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) { 5653 old_entry = &entry_ctx->entry; 5654 if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) { 5655 found = true; 5656 break; 5657 } 5658 } 5659 if (!found) { 5660 struct discovery_entry_ctx *subnqn_ctx = NULL, *new_ctx; 5661 struct discovery_ctx *d_ctx; 5662 5663 TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) { 5664 TAILQ_FOREACH(subnqn_ctx, &d_ctx->nvm_entry_ctxs, tailq) { 5665 if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn, 5666 sizeof(new_entry->subnqn))) { 5667 break; 5668 } 5669 } 5670 if (subnqn_ctx) { 5671 break; 5672 } 5673 } 5674 5675 new_ctx = calloc(1, sizeof(*new_ctx)); 5676 if (new_ctx == NULL) { 5677 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 5678 break; 5679 } 5680 5681 new_ctx->ctx = ctx; 5682 memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry)); 5683 build_trid_from_log_page_entry(&new_ctx->trid, new_entry); 5684 if (subnqn_ctx) { 5685 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name); 5686 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n", 5687 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 5688 new_ctx->name); 5689 } else { 5690 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++); 5691 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n", 5692 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 5693 new_ctx->name); 5694 } 5695 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 5696 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 5697 rc = bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0, 5698 discovery_attach_controller_done, new_ctx, 5699 &new_ctx->drv_opts, &ctx->bdev_opts, true); 5700 if (rc == 0) { 5701 TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq); 5702 ctx->attach_in_progress++; 5703 } else { 5704 DISCOVERY_ERRLOG(ctx, "bdev_nvme_create failed (%s)\n", spdk_strerror(-rc)); 5705 } 5706 } 5707 } 5708 5709 if (ctx->attach_in_progress == 0) { 5710 discovery_remove_controllers(ctx); 5711 } 5712 } 5713 5714 static void 5715 get_discovery_log_page(struct discovery_ctx *ctx) 5716 { 5717 int rc; 5718 5719 assert(ctx->in_progress == false); 5720 ctx->in_progress = true; 5721 rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx); 5722 if (rc != 0) { 5723 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 5724 } 5725 DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n"); 5726 } 5727 5728 static void 5729 discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 5730 { 5731 struct discovery_ctx *ctx = arg; 5732 uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16; 5733 5734 if (spdk_nvme_cpl_is_error(cpl)) { 5735 DISCOVERY_ERRLOG(ctx, "aer failed\n"); 5736 return; 5737 } 5738 5739 if (log_page_id != SPDK_NVME_LOG_DISCOVERY) { 5740 DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id); 5741 return; 5742 } 5743 5744 DISCOVERY_INFOLOG(ctx, "got aer\n"); 5745 if (ctx->in_progress) { 5746 ctx->pending = true; 5747 return; 5748 } 5749 5750 get_discovery_log_page(ctx); 5751 } 5752 5753 static void 5754 discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 5755 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 5756 { 5757 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 5758 struct discovery_ctx *ctx; 5759 5760 ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts); 5761 5762 DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n"); 5763 ctx->probe_ctx = NULL; 5764 ctx->ctrlr = ctrlr; 5765 5766 if (ctx->rc != 0) { 5767 DISCOVERY_ERRLOG(ctx, "encountered error while attaching discovery ctrlr: %d\n", 5768 ctx->rc); 5769 return; 5770 } 5771 5772 spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx); 5773 } 5774 5775 static int 5776 discovery_poller(void *arg) 5777 { 5778 struct discovery_ctx *ctx = arg; 5779 struct spdk_nvme_transport_id *trid; 5780 int rc; 5781 5782 if (ctx->detach_ctx) { 5783 rc = spdk_nvme_detach_poll_async(ctx->detach_ctx); 5784 if (rc != -EAGAIN) { 5785 ctx->detach_ctx = NULL; 5786 ctx->ctrlr = NULL; 5787 } 5788 } else if (ctx->stop) { 5789 if (ctx->ctrlr != NULL) { 5790 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 5791 if (rc == 0) { 5792 return SPDK_POLLER_BUSY; 5793 } 5794 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 5795 } 5796 spdk_poller_unregister(&ctx->poller); 5797 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 5798 assert(ctx->start_cb_fn == NULL); 5799 if (ctx->stop_cb_fn != NULL) { 5800 ctx->stop_cb_fn(ctx->cb_ctx); 5801 } 5802 free_discovery_ctx(ctx); 5803 } else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) { 5804 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 5805 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 5806 assert(ctx->initializing); 5807 spdk_poller_unregister(&ctx->poller); 5808 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 5809 complete_discovery_start(ctx, -ETIMEDOUT); 5810 stop_discovery(ctx, NULL, NULL); 5811 free_discovery_ctx(ctx); 5812 return SPDK_POLLER_BUSY; 5813 } 5814 5815 assert(ctx->entry_ctx_in_use == NULL); 5816 ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 5817 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 5818 trid = &ctx->entry_ctx_in_use->trid; 5819 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb); 5820 if (ctx->probe_ctx) { 5821 spdk_poller_unregister(&ctx->poller); 5822 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000); 5823 } else { 5824 DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n"); 5825 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 5826 ctx->entry_ctx_in_use = NULL; 5827 } 5828 } else if (ctx->probe_ctx) { 5829 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 5830 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 5831 complete_discovery_start(ctx, -ETIMEDOUT); 5832 return SPDK_POLLER_BUSY; 5833 } 5834 5835 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 5836 if (rc != -EAGAIN) { 5837 if (ctx->rc != 0) { 5838 assert(ctx->initializing); 5839 stop_discovery(ctx, NULL, ctx->cb_ctx); 5840 } else { 5841 assert(rc == 0); 5842 DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n"); 5843 ctx->rc = rc; 5844 get_discovery_log_page(ctx); 5845 } 5846 } 5847 } else { 5848 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 5849 DISCOVERY_ERRLOG(ctx, "timed out while attaching NVM ctrlrs\n"); 5850 complete_discovery_start(ctx, -ETIMEDOUT); 5851 /* We need to wait until all NVM ctrlrs are attached before we stop the 5852 * discovery service to make sure we don't detach a ctrlr that is still 5853 * being attached. 5854 */ 5855 if (ctx->attach_in_progress == 0) { 5856 stop_discovery(ctx, NULL, ctx->cb_ctx); 5857 return SPDK_POLLER_BUSY; 5858 } 5859 } 5860 5861 rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr); 5862 if (rc < 0) { 5863 spdk_poller_unregister(&ctx->poller); 5864 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 5865 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 5866 ctx->entry_ctx_in_use = NULL; 5867 5868 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 5869 if (rc != 0) { 5870 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 5871 ctx->ctrlr = NULL; 5872 } 5873 } 5874 } 5875 5876 return SPDK_POLLER_BUSY; 5877 } 5878 5879 static void 5880 start_discovery_poller(void *arg) 5881 { 5882 struct discovery_ctx *ctx = arg; 5883 5884 TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq); 5885 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 5886 } 5887 5888 int 5889 bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid, 5890 const char *base_name, 5891 struct spdk_nvme_ctrlr_opts *drv_opts, 5892 struct nvme_ctrlr_opts *bdev_opts, 5893 uint64_t attach_timeout, 5894 bool from_mdns, 5895 spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx) 5896 { 5897 struct discovery_ctx *ctx; 5898 struct discovery_entry_ctx *discovery_entry_ctx; 5899 5900 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 5901 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 5902 if (strcmp(ctx->name, base_name) == 0) { 5903 return -EEXIST; 5904 } 5905 5906 if (ctx->entry_ctx_in_use != NULL) { 5907 if (!spdk_nvme_transport_id_compare(trid, &ctx->entry_ctx_in_use->trid)) { 5908 return -EEXIST; 5909 } 5910 } 5911 5912 TAILQ_FOREACH(discovery_entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 5913 if (!spdk_nvme_transport_id_compare(trid, &discovery_entry_ctx->trid)) { 5914 return -EEXIST; 5915 } 5916 } 5917 } 5918 5919 ctx = calloc(1, sizeof(*ctx)); 5920 if (ctx == NULL) { 5921 return -ENOMEM; 5922 } 5923 5924 ctx->name = strdup(base_name); 5925 if (ctx->name == NULL) { 5926 free_discovery_ctx(ctx); 5927 return -ENOMEM; 5928 } 5929 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 5930 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 5931 ctx->from_mdns_discovery_service = from_mdns; 5932 ctx->bdev_opts.from_discovery_service = true; 5933 ctx->calling_thread = spdk_get_thread(); 5934 ctx->start_cb_fn = cb_fn; 5935 ctx->cb_ctx = cb_ctx; 5936 ctx->initializing = true; 5937 if (ctx->start_cb_fn) { 5938 /* We can use this when dumping json to denote if this RPC parameter 5939 * was specified or not. 5940 */ 5941 ctx->wait_for_attach = true; 5942 } 5943 if (attach_timeout != 0) { 5944 ctx->timeout_ticks = spdk_get_ticks() + attach_timeout * 5945 spdk_get_ticks_hz() / 1000ull; 5946 } 5947 TAILQ_INIT(&ctx->nvm_entry_ctxs); 5948 TAILQ_INIT(&ctx->discovery_entry_ctxs); 5949 memcpy(&ctx->trid, trid, sizeof(*trid)); 5950 /* Even if user did not specify hostnqn, we can still strdup("\0"); */ 5951 ctx->hostnqn = strdup(ctx->drv_opts.hostnqn); 5952 if (ctx->hostnqn == NULL) { 5953 free_discovery_ctx(ctx); 5954 return -ENOMEM; 5955 } 5956 discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid); 5957 if (discovery_entry_ctx == NULL) { 5958 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 5959 free_discovery_ctx(ctx); 5960 return -ENOMEM; 5961 } 5962 5963 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq); 5964 spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx); 5965 return 0; 5966 } 5967 5968 int 5969 bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 5970 { 5971 struct discovery_ctx *ctx; 5972 5973 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 5974 if (strcmp(name, ctx->name) == 0) { 5975 if (ctx->stop) { 5976 return -EALREADY; 5977 } 5978 /* If we're still starting the discovery service and ->rc is non-zero, we're 5979 * going to stop it as soon as we can 5980 */ 5981 if (ctx->initializing && ctx->rc != 0) { 5982 return -EALREADY; 5983 } 5984 stop_discovery(ctx, cb_fn, cb_ctx); 5985 return 0; 5986 } 5987 } 5988 5989 return -ENOENT; 5990 } 5991 5992 static int 5993 bdev_nvme_library_init(void) 5994 { 5995 g_bdev_nvme_init_thread = spdk_get_thread(); 5996 5997 spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb, 5998 bdev_nvme_destroy_poll_group_cb, 5999 sizeof(struct nvme_poll_group), "nvme_poll_groups"); 6000 6001 return 0; 6002 } 6003 6004 static void 6005 bdev_nvme_fini_destruct_ctrlrs(void) 6006 { 6007 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6008 struct nvme_ctrlr *nvme_ctrlr; 6009 6010 pthread_mutex_lock(&g_bdev_nvme_mutex); 6011 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 6012 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 6013 pthread_mutex_lock(&nvme_ctrlr->mutex); 6014 if (nvme_ctrlr->destruct) { 6015 /* This controller's destruction was already started 6016 * before the application started shutting down 6017 */ 6018 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6019 continue; 6020 } 6021 nvme_ctrlr->destruct = true; 6022 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6023 6024 spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct, 6025 nvme_ctrlr); 6026 } 6027 } 6028 6029 g_bdev_nvme_module_finish = true; 6030 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 6031 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6032 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 6033 spdk_bdev_module_fini_done(); 6034 return; 6035 } 6036 6037 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6038 } 6039 6040 static void 6041 check_discovery_fini(void *arg) 6042 { 6043 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 6044 bdev_nvme_fini_destruct_ctrlrs(); 6045 } 6046 } 6047 6048 static void 6049 bdev_nvme_library_fini(void) 6050 { 6051 struct nvme_probe_skip_entry *entry, *entry_tmp; 6052 struct discovery_ctx *ctx; 6053 6054 spdk_poller_unregister(&g_hotplug_poller); 6055 free(g_hotplug_probe_ctx); 6056 g_hotplug_probe_ctx = NULL; 6057 6058 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) { 6059 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 6060 free(entry); 6061 } 6062 6063 assert(spdk_get_thread() == g_bdev_nvme_init_thread); 6064 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 6065 bdev_nvme_fini_destruct_ctrlrs(); 6066 } else { 6067 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 6068 stop_discovery(ctx, check_discovery_fini, NULL); 6069 } 6070 } 6071 } 6072 6073 static void 6074 bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio) 6075 { 6076 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6077 struct spdk_bdev *bdev = bdev_io->bdev; 6078 struct spdk_dif_ctx dif_ctx; 6079 struct spdk_dif_error err_blk = {}; 6080 int rc; 6081 6082 rc = spdk_dif_ctx_init(&dif_ctx, 6083 bdev->blocklen, bdev->md_len, bdev->md_interleave, 6084 bdev->dif_is_head_of_md, bdev->dif_type, bdev->dif_check_flags, 6085 bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0); 6086 if (rc != 0) { 6087 SPDK_ERRLOG("Initialization of DIF context failed\n"); 6088 return; 6089 } 6090 6091 if (bdev->md_interleave) { 6092 rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 6093 bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 6094 } else { 6095 struct iovec md_iov = { 6096 .iov_base = bdev_io->u.bdev.md_buf, 6097 .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len, 6098 }; 6099 6100 rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 6101 &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 6102 } 6103 6104 if (rc != 0) { 6105 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", 6106 err_blk.err_type, err_blk.err_offset); 6107 } else { 6108 SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n"); 6109 } 6110 } 6111 6112 static void 6113 bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 6114 { 6115 struct nvme_bdev_io *bio = ref; 6116 6117 if (spdk_nvme_cpl_is_success(cpl)) { 6118 /* Run PI verification for read data buffer. */ 6119 bdev_nvme_verify_pi_error(bio); 6120 } 6121 6122 /* Return original completion status */ 6123 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 6124 } 6125 6126 static void 6127 bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 6128 { 6129 struct nvme_bdev_io *bio = ref; 6130 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6131 int ret; 6132 6133 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 6134 SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n", 6135 cpl->status.sct, cpl->status.sc); 6136 6137 /* Save completion status to use after verifying PI error. */ 6138 bio->cpl = *cpl; 6139 6140 if (spdk_likely(nvme_io_path_is_available(bio->io_path))) { 6141 /* Read without PI checking to verify PI error. */ 6142 ret = bdev_nvme_no_pi_readv(bio, 6143 bdev_io->u.bdev.iovs, 6144 bdev_io->u.bdev.iovcnt, 6145 bdev_io->u.bdev.md_buf, 6146 bdev_io->u.bdev.num_blocks, 6147 bdev_io->u.bdev.offset_blocks); 6148 if (ret == 0) { 6149 return; 6150 } 6151 } 6152 } 6153 6154 bdev_nvme_io_complete_nvme_status(bio, cpl); 6155 } 6156 6157 static void 6158 bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 6159 { 6160 struct nvme_bdev_io *bio = ref; 6161 6162 if (spdk_nvme_cpl_is_pi_error(cpl)) { 6163 SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n", 6164 cpl->status.sct, cpl->status.sc); 6165 /* Run PI verification for write data buffer if PI error is detected. */ 6166 bdev_nvme_verify_pi_error(bio); 6167 } 6168 6169 bdev_nvme_io_complete_nvme_status(bio, cpl); 6170 } 6171 6172 static void 6173 bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl) 6174 { 6175 struct nvme_bdev_io *bio = ref; 6176 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6177 6178 /* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks. 6179 * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error(). 6180 */ 6181 bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0; 6182 6183 if (spdk_nvme_cpl_is_pi_error(cpl)) { 6184 SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n", 6185 cpl->status.sct, cpl->status.sc); 6186 /* Run PI verification for zone append data buffer if PI error is detected. */ 6187 bdev_nvme_verify_pi_error(bio); 6188 } 6189 6190 bdev_nvme_io_complete_nvme_status(bio, cpl); 6191 } 6192 6193 static void 6194 bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl) 6195 { 6196 struct nvme_bdev_io *bio = ref; 6197 6198 if (spdk_nvme_cpl_is_pi_error(cpl)) { 6199 SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n", 6200 cpl->status.sct, cpl->status.sc); 6201 /* Run PI verification for compare data buffer if PI error is detected. */ 6202 bdev_nvme_verify_pi_error(bio); 6203 } 6204 6205 bdev_nvme_io_complete_nvme_status(bio, cpl); 6206 } 6207 6208 static void 6209 bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 6210 { 6211 struct nvme_bdev_io *bio = ref; 6212 6213 /* Compare operation completion */ 6214 if (!bio->first_fused_completed) { 6215 /* Save compare result for write callback */ 6216 bio->cpl = *cpl; 6217 bio->first_fused_completed = true; 6218 return; 6219 } 6220 6221 /* Write operation completion */ 6222 if (spdk_nvme_cpl_is_error(&bio->cpl)) { 6223 /* If bio->cpl is already an error, it means the compare operation failed. In that case, 6224 * complete the IO with the compare operation's status. 6225 */ 6226 if (!spdk_nvme_cpl_is_error(cpl)) { 6227 SPDK_ERRLOG("Unexpected write success after compare failure.\n"); 6228 } 6229 6230 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 6231 } else { 6232 bdev_nvme_io_complete_nvme_status(bio, cpl); 6233 } 6234 } 6235 6236 static void 6237 bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl) 6238 { 6239 struct nvme_bdev_io *bio = ref; 6240 6241 bdev_nvme_io_complete_nvme_status(bio, cpl); 6242 } 6243 6244 static int 6245 fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc) 6246 { 6247 switch (desc->zt) { 6248 case SPDK_NVME_ZONE_TYPE_SEQWR: 6249 info->type = SPDK_BDEV_ZONE_TYPE_SEQWR; 6250 break; 6251 default: 6252 SPDK_ERRLOG("Invalid zone type: %#x in zone report\n", desc->zt); 6253 return -EIO; 6254 } 6255 6256 switch (desc->zs) { 6257 case SPDK_NVME_ZONE_STATE_EMPTY: 6258 info->state = SPDK_BDEV_ZONE_STATE_EMPTY; 6259 break; 6260 case SPDK_NVME_ZONE_STATE_IOPEN: 6261 info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN; 6262 break; 6263 case SPDK_NVME_ZONE_STATE_EOPEN: 6264 info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN; 6265 break; 6266 case SPDK_NVME_ZONE_STATE_CLOSED: 6267 info->state = SPDK_BDEV_ZONE_STATE_CLOSED; 6268 break; 6269 case SPDK_NVME_ZONE_STATE_RONLY: 6270 info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY; 6271 break; 6272 case SPDK_NVME_ZONE_STATE_FULL: 6273 info->state = SPDK_BDEV_ZONE_STATE_FULL; 6274 break; 6275 case SPDK_NVME_ZONE_STATE_OFFLINE: 6276 info->state = SPDK_BDEV_ZONE_STATE_OFFLINE; 6277 break; 6278 default: 6279 SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs); 6280 return -EIO; 6281 } 6282 6283 info->zone_id = desc->zslba; 6284 info->write_pointer = desc->wp; 6285 info->capacity = desc->zcap; 6286 6287 return 0; 6288 } 6289 6290 static void 6291 bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl) 6292 { 6293 struct nvme_bdev_io *bio = ref; 6294 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6295 uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id; 6296 uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones; 6297 struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf; 6298 uint64_t max_zones_per_buf, i; 6299 uint32_t zone_report_bufsize; 6300 struct spdk_nvme_ns *ns; 6301 struct spdk_nvme_qpair *qpair; 6302 int ret; 6303 6304 if (spdk_nvme_cpl_is_error(cpl)) { 6305 goto out_complete_io_nvme_cpl; 6306 } 6307 6308 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 6309 ret = -ENXIO; 6310 goto out_complete_io_ret; 6311 } 6312 6313 ns = bio->io_path->nvme_ns->ns; 6314 qpair = bio->io_path->qpair->qpair; 6315 6316 zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 6317 max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) / 6318 sizeof(bio->zone_report_buf->descs[0]); 6319 6320 if (bio->zone_report_buf->nr_zones > max_zones_per_buf) { 6321 ret = -EINVAL; 6322 goto out_complete_io_ret; 6323 } 6324 6325 if (!bio->zone_report_buf->nr_zones) { 6326 ret = -EINVAL; 6327 goto out_complete_io_ret; 6328 } 6329 6330 for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) { 6331 ret = fill_zone_from_report(&info[bio->handled_zones], 6332 &bio->zone_report_buf->descs[i]); 6333 if (ret) { 6334 goto out_complete_io_ret; 6335 } 6336 bio->handled_zones++; 6337 } 6338 6339 if (bio->handled_zones < zones_to_copy) { 6340 uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 6341 uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones); 6342 6343 memset(bio->zone_report_buf, 0, zone_report_bufsize); 6344 ret = spdk_nvme_zns_report_zones(ns, qpair, 6345 bio->zone_report_buf, zone_report_bufsize, 6346 slba, SPDK_NVME_ZRA_LIST_ALL, true, 6347 bdev_nvme_get_zone_info_done, bio); 6348 if (!ret) { 6349 return; 6350 } else { 6351 goto out_complete_io_ret; 6352 } 6353 } 6354 6355 out_complete_io_nvme_cpl: 6356 free(bio->zone_report_buf); 6357 bio->zone_report_buf = NULL; 6358 bdev_nvme_io_complete_nvme_status(bio, cpl); 6359 return; 6360 6361 out_complete_io_ret: 6362 free(bio->zone_report_buf); 6363 bio->zone_report_buf = NULL; 6364 bdev_nvme_io_complete(bio, ret); 6365 } 6366 6367 static void 6368 bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl) 6369 { 6370 struct nvme_bdev_io *bio = ref; 6371 6372 bdev_nvme_io_complete_nvme_status(bio, cpl); 6373 } 6374 6375 static void 6376 bdev_nvme_admin_passthru_complete_nvme_status(void *ctx) 6377 { 6378 struct nvme_bdev_io *bio = ctx; 6379 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6380 const struct spdk_nvme_cpl *cpl = &bio->cpl; 6381 6382 assert(bdev_nvme_io_type_is_admin(bdev_io->type)); 6383 6384 __bdev_nvme_io_complete(bdev_io, 0, cpl); 6385 } 6386 6387 static void 6388 bdev_nvme_abort_complete(void *ctx) 6389 { 6390 struct nvme_bdev_io *bio = ctx; 6391 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6392 6393 if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) { 6394 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL); 6395 } else { 6396 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL); 6397 } 6398 } 6399 6400 static void 6401 bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl) 6402 { 6403 struct nvme_bdev_io *bio = ref; 6404 6405 bio->cpl = *cpl; 6406 spdk_thread_send_msg(bio->orig_thread, bdev_nvme_abort_complete, bio); 6407 } 6408 6409 static void 6410 bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl) 6411 { 6412 struct nvme_bdev_io *bio = ref; 6413 6414 bio->cpl = *cpl; 6415 spdk_thread_send_msg(bio->orig_thread, 6416 bdev_nvme_admin_passthru_complete_nvme_status, bio); 6417 } 6418 6419 static void 6420 bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset) 6421 { 6422 struct nvme_bdev_io *bio = ref; 6423 struct iovec *iov; 6424 6425 bio->iov_offset = sgl_offset; 6426 for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) { 6427 iov = &bio->iovs[bio->iovpos]; 6428 if (bio->iov_offset < iov->iov_len) { 6429 break; 6430 } 6431 6432 bio->iov_offset -= iov->iov_len; 6433 } 6434 } 6435 6436 static int 6437 bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length) 6438 { 6439 struct nvme_bdev_io *bio = ref; 6440 struct iovec *iov; 6441 6442 assert(bio->iovpos < bio->iovcnt); 6443 6444 iov = &bio->iovs[bio->iovpos]; 6445 6446 *address = iov->iov_base; 6447 *length = iov->iov_len; 6448 6449 if (bio->iov_offset) { 6450 assert(bio->iov_offset <= iov->iov_len); 6451 *address += bio->iov_offset; 6452 *length -= bio->iov_offset; 6453 } 6454 6455 bio->iov_offset += *length; 6456 if (bio->iov_offset == iov->iov_len) { 6457 bio->iovpos++; 6458 bio->iov_offset = 0; 6459 } 6460 6461 return 0; 6462 } 6463 6464 static void 6465 bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset) 6466 { 6467 struct nvme_bdev_io *bio = ref; 6468 struct iovec *iov; 6469 6470 bio->fused_iov_offset = sgl_offset; 6471 for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) { 6472 iov = &bio->fused_iovs[bio->fused_iovpos]; 6473 if (bio->fused_iov_offset < iov->iov_len) { 6474 break; 6475 } 6476 6477 bio->fused_iov_offset -= iov->iov_len; 6478 } 6479 } 6480 6481 static int 6482 bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length) 6483 { 6484 struct nvme_bdev_io *bio = ref; 6485 struct iovec *iov; 6486 6487 assert(bio->fused_iovpos < bio->fused_iovcnt); 6488 6489 iov = &bio->fused_iovs[bio->fused_iovpos]; 6490 6491 *address = iov->iov_base; 6492 *length = iov->iov_len; 6493 6494 if (bio->fused_iov_offset) { 6495 assert(bio->fused_iov_offset <= iov->iov_len); 6496 *address += bio->fused_iov_offset; 6497 *length -= bio->fused_iov_offset; 6498 } 6499 6500 bio->fused_iov_offset += *length; 6501 if (bio->fused_iov_offset == iov->iov_len) { 6502 bio->fused_iovpos++; 6503 bio->fused_iov_offset = 0; 6504 } 6505 6506 return 0; 6507 } 6508 6509 static int 6510 bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 6511 void *md, uint64_t lba_count, uint64_t lba) 6512 { 6513 int rc; 6514 6515 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n", 6516 lba_count, lba); 6517 6518 bio->iovs = iov; 6519 bio->iovcnt = iovcnt; 6520 bio->iovpos = 0; 6521 bio->iov_offset = 0; 6522 6523 rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns, 6524 bio->io_path->qpair->qpair, 6525 lba, lba_count, 6526 bdev_nvme_no_pi_readv_done, bio, 0, 6527 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 6528 md, 0, 0); 6529 6530 if (rc != 0 && rc != -ENOMEM) { 6531 SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc); 6532 } 6533 return rc; 6534 } 6535 6536 static int 6537 bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 6538 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 6539 struct spdk_memory_domain *domain, void *domain_ctx) 6540 { 6541 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6542 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6543 int rc; 6544 6545 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n", 6546 lba_count, lba); 6547 6548 bio->iovs = iov; 6549 bio->iovcnt = iovcnt; 6550 bio->iovpos = 0; 6551 bio->iov_offset = 0; 6552 6553 bio->ext_opts.size = sizeof(struct spdk_nvme_ns_cmd_ext_io_opts); 6554 bio->ext_opts.memory_domain = domain; 6555 bio->ext_opts.memory_domain_ctx = domain_ctx; 6556 bio->ext_opts.io_flags = flags; 6557 bio->ext_opts.metadata = md; 6558 6559 rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count, 6560 bdev_nvme_readv_done, bio, 6561 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 6562 &bio->ext_opts); 6563 if (rc != 0 && rc != -ENOMEM) { 6564 SPDK_ERRLOG("readv failed: rc = %d\n", rc); 6565 } 6566 return rc; 6567 } 6568 6569 static int 6570 bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 6571 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 6572 struct spdk_memory_domain *domain, void *domain_ctx) 6573 { 6574 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6575 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6576 int rc; 6577 6578 SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 6579 lba_count, lba); 6580 6581 bio->iovs = iov; 6582 bio->iovcnt = iovcnt; 6583 bio->iovpos = 0; 6584 bio->iov_offset = 0; 6585 6586 bio->ext_opts.size = sizeof(struct spdk_nvme_ns_cmd_ext_io_opts); 6587 bio->ext_opts.memory_domain = domain; 6588 bio->ext_opts.memory_domain_ctx = domain_ctx; 6589 bio->ext_opts.io_flags = flags; 6590 bio->ext_opts.metadata = md; 6591 6592 rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count, 6593 bdev_nvme_writev_done, bio, 6594 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 6595 &bio->ext_opts); 6596 if (rc != 0 && rc != -ENOMEM) { 6597 SPDK_ERRLOG("writev failed: rc = %d\n", rc); 6598 } 6599 return rc; 6600 } 6601 6602 static int 6603 bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 6604 void *md, uint64_t lba_count, uint64_t zslba, 6605 uint32_t flags) 6606 { 6607 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6608 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6609 int rc; 6610 6611 SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n", 6612 lba_count, zslba); 6613 6614 bio->iovs = iov; 6615 bio->iovcnt = iovcnt; 6616 bio->iovpos = 0; 6617 bio->iov_offset = 0; 6618 6619 if (iovcnt == 1) { 6620 rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba, 6621 lba_count, 6622 bdev_nvme_zone_appendv_done, bio, 6623 flags, 6624 0, 0); 6625 } else { 6626 rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count, 6627 bdev_nvme_zone_appendv_done, bio, flags, 6628 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 6629 md, 0, 0); 6630 } 6631 6632 if (rc != 0 && rc != -ENOMEM) { 6633 SPDK_ERRLOG("zone append failed: rc = %d\n", rc); 6634 } 6635 return rc; 6636 } 6637 6638 static int 6639 bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 6640 void *md, uint64_t lba_count, uint64_t lba, 6641 uint32_t flags) 6642 { 6643 int rc; 6644 6645 SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n", 6646 lba_count, lba); 6647 6648 bio->iovs = iov; 6649 bio->iovcnt = iovcnt; 6650 bio->iovpos = 0; 6651 bio->iov_offset = 0; 6652 6653 rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns, 6654 bio->io_path->qpair->qpair, 6655 lba, lba_count, 6656 bdev_nvme_comparev_done, bio, flags, 6657 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 6658 md, 0, 0); 6659 6660 if (rc != 0 && rc != -ENOMEM) { 6661 SPDK_ERRLOG("comparev failed: rc = %d\n", rc); 6662 } 6663 return rc; 6664 } 6665 6666 static int 6667 bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt, 6668 struct iovec *write_iov, int write_iovcnt, 6669 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags) 6670 { 6671 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6672 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6673 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6674 int rc; 6675 6676 SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 6677 lba_count, lba); 6678 6679 bio->iovs = cmp_iov; 6680 bio->iovcnt = cmp_iovcnt; 6681 bio->iovpos = 0; 6682 bio->iov_offset = 0; 6683 bio->fused_iovs = write_iov; 6684 bio->fused_iovcnt = write_iovcnt; 6685 bio->fused_iovpos = 0; 6686 bio->fused_iov_offset = 0; 6687 6688 if (bdev_io->num_retries == 0) { 6689 bio->first_fused_submitted = false; 6690 bio->first_fused_completed = false; 6691 } 6692 6693 if (!bio->first_fused_submitted) { 6694 flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST; 6695 memset(&bio->cpl, 0, sizeof(bio->cpl)); 6696 6697 rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count, 6698 bdev_nvme_comparev_and_writev_done, bio, flags, 6699 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0); 6700 if (rc == 0) { 6701 bio->first_fused_submitted = true; 6702 flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST; 6703 } else { 6704 if (rc != -ENOMEM) { 6705 SPDK_ERRLOG("compare failed: rc = %d\n", rc); 6706 } 6707 return rc; 6708 } 6709 } 6710 6711 flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND; 6712 6713 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 6714 bdev_nvme_comparev_and_writev_done, bio, flags, 6715 bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0); 6716 if (rc != 0 && rc != -ENOMEM) { 6717 SPDK_ERRLOG("write failed: rc = %d\n", rc); 6718 rc = 0; 6719 } 6720 6721 return rc; 6722 } 6723 6724 static int 6725 bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 6726 { 6727 struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES]; 6728 struct spdk_nvme_dsm_range *range; 6729 uint64_t offset, remaining; 6730 uint64_t num_ranges_u64; 6731 uint16_t num_ranges; 6732 int rc; 6733 6734 num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) / 6735 SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 6736 if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) { 6737 SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks); 6738 return -EINVAL; 6739 } 6740 num_ranges = (uint16_t)num_ranges_u64; 6741 6742 offset = offset_blocks; 6743 remaining = num_blocks; 6744 range = &dsm_ranges[0]; 6745 6746 /* Fill max-size ranges until the remaining blocks fit into one range */ 6747 while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) { 6748 range->attributes.raw = 0; 6749 range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 6750 range->starting_lba = offset; 6751 6752 offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 6753 remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 6754 range++; 6755 } 6756 6757 /* Final range describes the remaining blocks */ 6758 range->attributes.raw = 0; 6759 range->length = remaining; 6760 range->starting_lba = offset; 6761 6762 rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns, 6763 bio->io_path->qpair->qpair, 6764 SPDK_NVME_DSM_ATTR_DEALLOCATE, 6765 dsm_ranges, num_ranges, 6766 bdev_nvme_queued_done, bio); 6767 6768 return rc; 6769 } 6770 6771 static int 6772 bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 6773 { 6774 if (num_blocks > UINT16_MAX + 1) { 6775 SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n"); 6776 return -EINVAL; 6777 } 6778 6779 return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns, 6780 bio->io_path->qpair->qpair, 6781 offset_blocks, num_blocks, 6782 bdev_nvme_queued_done, bio, 6783 0); 6784 } 6785 6786 static int 6787 bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones, 6788 struct spdk_bdev_zone_info *info) 6789 { 6790 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6791 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6792 uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 6793 uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 6794 uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns); 6795 6796 if (zone_id % zone_size != 0) { 6797 return -EINVAL; 6798 } 6799 6800 if (num_zones > total_zones || !num_zones) { 6801 return -EINVAL; 6802 } 6803 6804 assert(!bio->zone_report_buf); 6805 bio->zone_report_buf = calloc(1, zone_report_bufsize); 6806 if (!bio->zone_report_buf) { 6807 return -ENOMEM; 6808 } 6809 6810 bio->handled_zones = 0; 6811 6812 return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize, 6813 zone_id, SPDK_NVME_ZRA_LIST_ALL, true, 6814 bdev_nvme_get_zone_info_done, bio); 6815 } 6816 6817 static int 6818 bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 6819 enum spdk_bdev_zone_action action) 6820 { 6821 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6822 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6823 6824 switch (action) { 6825 case SPDK_BDEV_ZONE_CLOSE: 6826 return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false, 6827 bdev_nvme_zone_management_done, bio); 6828 case SPDK_BDEV_ZONE_FINISH: 6829 return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false, 6830 bdev_nvme_zone_management_done, bio); 6831 case SPDK_BDEV_ZONE_OPEN: 6832 return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false, 6833 bdev_nvme_zone_management_done, bio); 6834 case SPDK_BDEV_ZONE_RESET: 6835 return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false, 6836 bdev_nvme_zone_management_done, bio); 6837 case SPDK_BDEV_ZONE_OFFLINE: 6838 return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false, 6839 bdev_nvme_zone_management_done, bio); 6840 default: 6841 return -EINVAL; 6842 } 6843 } 6844 6845 static void 6846 bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 6847 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes) 6848 { 6849 struct nvme_io_path *io_path; 6850 struct nvme_ctrlr *nvme_ctrlr; 6851 uint32_t max_xfer_size; 6852 int rc = -ENXIO; 6853 6854 /* Choose the first ctrlr which is not failed. */ 6855 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 6856 nvme_ctrlr = io_path->qpair->ctrlr; 6857 6858 /* We should skip any unavailable nvme_ctrlr rather than checking 6859 * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO. 6860 */ 6861 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 6862 continue; 6863 } 6864 6865 max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr); 6866 6867 if (nbytes > max_xfer_size) { 6868 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 6869 rc = -EINVAL; 6870 goto err; 6871 } 6872 6873 bio->io_path = io_path; 6874 bio->orig_thread = spdk_get_thread(); 6875 6876 rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes, 6877 bdev_nvme_admin_passthru_done, bio); 6878 if (rc == 0) { 6879 return; 6880 } 6881 } 6882 6883 err: 6884 bdev_nvme_admin_passthru_complete(bio, rc); 6885 } 6886 6887 static int 6888 bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 6889 void *buf, size_t nbytes) 6890 { 6891 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6892 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6893 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 6894 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 6895 6896 if (nbytes > max_xfer_size) { 6897 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 6898 return -EINVAL; 6899 } 6900 6901 /* 6902 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 6903 * so fill it out automatically. 6904 */ 6905 cmd->nsid = spdk_nvme_ns_get_id(ns); 6906 6907 return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf, 6908 (uint32_t)nbytes, bdev_nvme_queued_done, bio); 6909 } 6910 6911 static int 6912 bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 6913 void *buf, size_t nbytes, void *md_buf, size_t md_len) 6914 { 6915 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6916 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6917 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 6918 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 6919 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 6920 6921 if (nbytes > max_xfer_size) { 6922 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 6923 return -EINVAL; 6924 } 6925 6926 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 6927 SPDK_ERRLOG("invalid meta data buffer size\n"); 6928 return -EINVAL; 6929 } 6930 6931 /* 6932 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 6933 * so fill it out automatically. 6934 */ 6935 cmd->nsid = spdk_nvme_ns_get_id(ns); 6936 6937 return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf, 6938 (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio); 6939 } 6940 6941 static void 6942 bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 6943 struct nvme_bdev_io *bio_to_abort) 6944 { 6945 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6946 struct nvme_io_path *io_path; 6947 struct nvme_ctrlr *nvme_ctrlr; 6948 int rc = 0; 6949 6950 bio->orig_thread = spdk_get_thread(); 6951 6952 rc = bdev_nvme_abort_retry_io(nbdev_ch, bio_to_abort); 6953 if (rc == 0) { 6954 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL); 6955 return; 6956 } 6957 6958 rc = 0; 6959 6960 /* Even admin commands, they were submitted to only nvme_ctrlrs which were 6961 * on any io_path. So traverse the io_path list for not only I/O commands 6962 * but also admin commands. 6963 */ 6964 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 6965 nvme_ctrlr = io_path->qpair->ctrlr; 6966 6967 rc = spdk_nvme_ctrlr_cmd_abort_ext(nvme_ctrlr->ctrlr, 6968 io_path->qpair->qpair, 6969 bio_to_abort, 6970 bdev_nvme_abort_done, bio); 6971 if (rc == -ENOENT) { 6972 /* If no command was found in I/O qpair, the target command may be 6973 * admin command. 6974 */ 6975 rc = spdk_nvme_ctrlr_cmd_abort_ext(nvme_ctrlr->ctrlr, 6976 NULL, 6977 bio_to_abort, 6978 bdev_nvme_abort_done, bio); 6979 } 6980 6981 if (rc != -ENOENT) { 6982 break; 6983 } 6984 } 6985 6986 if (rc != 0) { 6987 /* If no command was found or there was any error, complete the abort 6988 * request with failure. 6989 */ 6990 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL); 6991 } 6992 } 6993 6994 static int 6995 bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, uint64_t src_offset_blocks, 6996 uint64_t num_blocks) 6997 { 6998 struct spdk_nvme_scc_source_range range = { 6999 .slba = src_offset_blocks, 7000 .nlb = num_blocks - 1 7001 }; 7002 7003 return spdk_nvme_ns_cmd_copy(bio->io_path->nvme_ns->ns, 7004 bio->io_path->qpair->qpair, 7005 &range, 1, dst_offset_blocks, 7006 bdev_nvme_queued_done, bio); 7007 } 7008 7009 static void 7010 bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w) 7011 { 7012 const char *action; 7013 7014 if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) { 7015 action = "reset"; 7016 } else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) { 7017 action = "abort"; 7018 } else { 7019 action = "none"; 7020 } 7021 7022 spdk_json_write_object_begin(w); 7023 7024 spdk_json_write_named_string(w, "method", "bdev_nvme_set_options"); 7025 7026 spdk_json_write_named_object_begin(w, "params"); 7027 spdk_json_write_named_string(w, "action_on_timeout", action); 7028 spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us); 7029 spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us); 7030 spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms); 7031 spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count); 7032 spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst); 7033 spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight); 7034 spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight); 7035 spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight); 7036 spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us); 7037 spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us); 7038 spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests); 7039 spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit); 7040 spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count); 7041 spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout); 7042 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec); 7043 spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec); 7044 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec); 7045 spdk_json_write_named_bool(w, "generate_uuids", g_opts.generate_uuids); 7046 spdk_json_write_named_uint8(w, "transport_tos", g_opts.transport_tos); 7047 spdk_json_write_named_bool(w, "io_path_stat", g_opts.io_path_stat); 7048 spdk_json_write_object_end(w); 7049 7050 spdk_json_write_object_end(w); 7051 } 7052 7053 static void 7054 bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx) 7055 { 7056 struct spdk_nvme_transport_id trid; 7057 7058 spdk_json_write_object_begin(w); 7059 7060 spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery"); 7061 7062 spdk_json_write_named_object_begin(w, "params"); 7063 spdk_json_write_named_string(w, "name", ctx->name); 7064 spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn); 7065 7066 trid = ctx->trid; 7067 memset(trid.subnqn, 0, sizeof(trid.subnqn)); 7068 nvme_bdev_dump_trid_json(&trid, w); 7069 7070 spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach); 7071 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec); 7072 spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec); 7073 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 7074 ctx->bdev_opts.fast_io_fail_timeout_sec); 7075 spdk_json_write_object_end(w); 7076 7077 spdk_json_write_object_end(w); 7078 } 7079 7080 static void 7081 nvme_ctrlr_config_json(struct spdk_json_write_ctx *w, 7082 struct nvme_ctrlr *nvme_ctrlr) 7083 { 7084 struct spdk_nvme_transport_id *trid; 7085 7086 if (nvme_ctrlr->opts.from_discovery_service) { 7087 /* Do not emit an RPC for this - it will be implicitly 7088 * covered by a separate bdev_nvme_start_discovery or 7089 * bdev_nvme_start_mdns_discovery RPC. 7090 */ 7091 return; 7092 } 7093 7094 trid = &nvme_ctrlr->active_path_id->trid; 7095 7096 spdk_json_write_object_begin(w); 7097 7098 spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller"); 7099 7100 spdk_json_write_named_object_begin(w, "params"); 7101 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 7102 nvme_bdev_dump_trid_json(trid, w); 7103 spdk_json_write_named_bool(w, "prchk_reftag", 7104 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0); 7105 spdk_json_write_named_bool(w, "prchk_guard", 7106 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0); 7107 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec); 7108 spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec); 7109 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 7110 nvme_ctrlr->opts.fast_io_fail_timeout_sec); 7111 7112 spdk_json_write_object_end(w); 7113 7114 spdk_json_write_object_end(w); 7115 } 7116 7117 static void 7118 bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w) 7119 { 7120 spdk_json_write_object_begin(w); 7121 spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug"); 7122 7123 spdk_json_write_named_object_begin(w, "params"); 7124 spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us); 7125 spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled); 7126 spdk_json_write_object_end(w); 7127 7128 spdk_json_write_object_end(w); 7129 } 7130 7131 static int 7132 bdev_nvme_config_json(struct spdk_json_write_ctx *w) 7133 { 7134 struct nvme_bdev_ctrlr *nbdev_ctrlr; 7135 struct nvme_ctrlr *nvme_ctrlr; 7136 struct discovery_ctx *ctx; 7137 7138 bdev_nvme_opts_config_json(w); 7139 7140 pthread_mutex_lock(&g_bdev_nvme_mutex); 7141 7142 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 7143 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 7144 nvme_ctrlr_config_json(w, nvme_ctrlr); 7145 } 7146 } 7147 7148 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7149 if (!ctx->from_mdns_discovery_service) { 7150 bdev_nvme_discovery_config_json(w, ctx); 7151 } 7152 } 7153 7154 bdev_nvme_mdns_discovery_config_json(w); 7155 7156 /* Dump as last parameter to give all NVMe bdevs chance to be constructed 7157 * before enabling hotplug poller. 7158 */ 7159 bdev_nvme_hotplug_config_json(w); 7160 7161 pthread_mutex_unlock(&g_bdev_nvme_mutex); 7162 return 0; 7163 } 7164 7165 struct spdk_nvme_ctrlr * 7166 bdev_nvme_get_ctrlr(struct spdk_bdev *bdev) 7167 { 7168 struct nvme_bdev *nbdev; 7169 struct nvme_ns *nvme_ns; 7170 7171 if (!bdev || bdev->module != &nvme_if) { 7172 return NULL; 7173 } 7174 7175 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 7176 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 7177 assert(nvme_ns != NULL); 7178 7179 return nvme_ns->ctrlr->ctrlr; 7180 } 7181 7182 void 7183 nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path) 7184 { 7185 struct nvme_ns *nvme_ns = io_path->nvme_ns; 7186 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 7187 const struct spdk_nvme_ctrlr_data *cdata; 7188 const struct spdk_nvme_transport_id *trid; 7189 const char *adrfam_str; 7190 7191 spdk_json_write_object_begin(w); 7192 7193 spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name); 7194 7195 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 7196 trid = spdk_nvme_ctrlr_get_transport_id(nvme_ctrlr->ctrlr); 7197 7198 spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid); 7199 spdk_json_write_named_bool(w, "current", io_path == io_path->nbdev_ch->current_io_path); 7200 spdk_json_write_named_bool(w, "connected", nvme_io_path_is_connected(io_path)); 7201 spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns)); 7202 7203 spdk_json_write_named_object_begin(w, "transport"); 7204 spdk_json_write_named_string(w, "trtype", trid->trstring); 7205 spdk_json_write_named_string(w, "traddr", trid->traddr); 7206 if (trid->trsvcid[0] != '\0') { 7207 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 7208 } 7209 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 7210 if (adrfam_str) { 7211 spdk_json_write_named_string(w, "adrfam", adrfam_str); 7212 } 7213 spdk_json_write_object_end(w); 7214 7215 spdk_json_write_object_end(w); 7216 } 7217 7218 void 7219 bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w) 7220 { 7221 struct discovery_ctx *ctx; 7222 struct discovery_entry_ctx *entry_ctx; 7223 7224 spdk_json_write_array_begin(w); 7225 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7226 spdk_json_write_object_begin(w); 7227 spdk_json_write_named_string(w, "name", ctx->name); 7228 7229 spdk_json_write_named_object_begin(w, "trid"); 7230 nvme_bdev_dump_trid_json(&ctx->trid, w); 7231 spdk_json_write_object_end(w); 7232 7233 spdk_json_write_named_array_begin(w, "referrals"); 7234 TAILQ_FOREACH(entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 7235 spdk_json_write_object_begin(w); 7236 spdk_json_write_named_object_begin(w, "trid"); 7237 nvme_bdev_dump_trid_json(&entry_ctx->trid, w); 7238 spdk_json_write_object_end(w); 7239 spdk_json_write_object_end(w); 7240 } 7241 spdk_json_write_array_end(w); 7242 7243 spdk_json_write_object_end(w); 7244 } 7245 spdk_json_write_array_end(w); 7246 } 7247 7248 SPDK_LOG_REGISTER_COMPONENT(bdev_nvme) 7249 7250 SPDK_TRACE_REGISTER_FN(bdev_nvme_trace, "bdev_nvme", TRACE_GROUP_BDEV_NVME) 7251 { 7252 struct spdk_trace_tpoint_opts opts[] = { 7253 { 7254 "BDEV_NVME_IO_START", TRACE_BDEV_NVME_IO_START, 7255 OWNER_NONE, OBJECT_BDEV_NVME_IO, 1, 7256 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 7257 }, 7258 { 7259 "BDEV_NVME_IO_DONE", TRACE_BDEV_NVME_IO_DONE, 7260 OWNER_NONE, OBJECT_BDEV_NVME_IO, 0, 7261 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 7262 } 7263 }; 7264 7265 7266 spdk_trace_register_object(OBJECT_BDEV_NVME_IO, 'N'); 7267 spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts)); 7268 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 7269 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 7270 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 7271 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 7272 } 7273