1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved. 6 */ 7 8 #include "spdk/stdinc.h" 9 10 #include "bdev_nvme.h" 11 12 #include "spdk/accel.h" 13 #include "spdk/config.h" 14 #include "spdk/endian.h" 15 #include "spdk/bdev.h" 16 #include "spdk/json.h" 17 #include "spdk/likely.h" 18 #include "spdk/nvme.h" 19 #include "spdk/nvme_ocssd.h" 20 #include "spdk/nvme_zns.h" 21 #include "spdk/opal.h" 22 #include "spdk/thread.h" 23 #include "spdk/trace.h" 24 #include "spdk/string.h" 25 #include "spdk/util.h" 26 #include "spdk/uuid.h" 27 28 #include "spdk/bdev_module.h" 29 #include "spdk/log.h" 30 31 #include "spdk_internal/usdt.h" 32 #include "spdk_internal/trace_defs.h" 33 34 #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true 35 #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS (10000) 36 37 #define NSID_STR_LEN 10 38 39 static int bdev_nvme_config_json(struct spdk_json_write_ctx *w); 40 41 struct nvme_bdev_io { 42 /** array of iovecs to transfer. */ 43 struct iovec *iovs; 44 45 /** Number of iovecs in iovs array. */ 46 int iovcnt; 47 48 /** Current iovec position. */ 49 int iovpos; 50 51 /** Offset in current iovec. */ 52 uint32_t iov_offset; 53 54 /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path 55 * being reset in a reset I/O. 56 */ 57 struct nvme_io_path *io_path; 58 59 /** array of iovecs to transfer. */ 60 struct iovec *fused_iovs; 61 62 /** Number of iovecs in iovs array. */ 63 int fused_iovcnt; 64 65 /** Current iovec position. */ 66 int fused_iovpos; 67 68 /** Offset in current iovec. */ 69 uint32_t fused_iov_offset; 70 71 /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */ 72 struct spdk_nvme_cpl cpl; 73 74 /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */ 75 struct spdk_nvme_ns_cmd_ext_io_opts ext_opts; 76 77 /** Originating thread */ 78 struct spdk_thread *orig_thread; 79 80 /** Keeps track if first of fused commands was submitted */ 81 bool first_fused_submitted; 82 83 /** Keeps track if first of fused commands was completed */ 84 bool first_fused_completed; 85 86 /** Temporary pointer to zone report buffer */ 87 struct spdk_nvme_zns_zone_report *zone_report_buf; 88 89 /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */ 90 uint64_t handled_zones; 91 92 /** Expiration value in ticks to retry the current I/O. */ 93 uint64_t retry_ticks; 94 95 /* How many times the current I/O was retried. */ 96 int32_t retry_count; 97 98 /* Current tsc at submit time. */ 99 uint64_t submit_tsc; 100 }; 101 102 struct nvme_probe_skip_entry { 103 struct spdk_nvme_transport_id trid; 104 TAILQ_ENTRY(nvme_probe_skip_entry) tailq; 105 }; 106 /* All the controllers deleted by users via RPC are skipped by hotplug monitor */ 107 static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER( 108 g_skipped_nvme_ctrlrs); 109 110 static struct spdk_bdev_nvme_opts g_opts = { 111 .action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE, 112 .timeout_us = 0, 113 .timeout_admin_us = 0, 114 .keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS, 115 .transport_retry_count = 4, 116 .arbitration_burst = 0, 117 .low_priority_weight = 0, 118 .medium_priority_weight = 0, 119 .high_priority_weight = 0, 120 .nvme_adminq_poll_period_us = 10000ULL, 121 .nvme_ioq_poll_period_us = 0, 122 .io_queue_requests = 0, 123 .delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT, 124 .bdev_retry_count = 3, 125 .transport_ack_timeout = 0, 126 .ctrlr_loss_timeout_sec = 0, 127 .reconnect_delay_sec = 0, 128 .fast_io_fail_timeout_sec = 0, 129 .disable_auto_failback = false, 130 .generate_uuids = false, 131 .transport_tos = 0, 132 .nvme_error_stat = false, 133 .io_path_stat = false, 134 }; 135 136 #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL 137 #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL 138 139 static int g_hot_insert_nvme_controller_index = 0; 140 static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT; 141 static bool g_nvme_hotplug_enabled = false; 142 struct spdk_thread *g_bdev_nvme_init_thread; 143 static struct spdk_poller *g_hotplug_poller; 144 static struct spdk_poller *g_hotplug_probe_poller; 145 static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx; 146 147 static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 148 struct nvme_async_probe_ctx *ctx); 149 static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 150 struct nvme_async_probe_ctx *ctx); 151 static int bdev_nvme_library_init(void); 152 static void bdev_nvme_library_fini(void); 153 static void _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, 154 struct spdk_bdev_io *bdev_io); 155 static void bdev_nvme_submit_request(struct spdk_io_channel *ch, 156 struct spdk_bdev_io *bdev_io); 157 static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 158 void *md, uint64_t lba_count, uint64_t lba, 159 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx); 160 static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 161 void *md, uint64_t lba_count, uint64_t lba); 162 static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 163 void *md, uint64_t lba_count, uint64_t lba, 164 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx); 165 static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 166 void *md, uint64_t lba_count, 167 uint64_t zslba, uint32_t flags); 168 static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 169 void *md, uint64_t lba_count, uint64_t lba, 170 uint32_t flags); 171 static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, 172 struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov, 173 int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba, 174 uint32_t flags); 175 static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, 176 uint32_t num_zones, struct spdk_bdev_zone_info *info); 177 static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 178 enum spdk_bdev_zone_action action); 179 static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, 180 struct nvme_bdev_io *bio, 181 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes); 182 static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 183 void *buf, size_t nbytes); 184 static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 185 void *buf, size_t nbytes, void *md_buf, size_t md_len); 186 static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, 187 struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort); 188 static void bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio); 189 static int bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr); 190 static int bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove); 191 static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr); 192 static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr); 193 194 static struct nvme_ns *nvme_ns_alloc(void); 195 static void nvme_ns_free(struct nvme_ns *ns); 196 197 static int 198 nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2) 199 { 200 return ns1->id < ns2->id ? -1 : ns1->id > ns2->id; 201 } 202 203 RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp); 204 205 struct spdk_nvme_qpair * 206 bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch) 207 { 208 struct nvme_ctrlr_channel *ctrlr_ch; 209 210 assert(ctrlr_io_ch != NULL); 211 212 ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch); 213 214 return ctrlr_ch->qpair->qpair; 215 } 216 217 static int 218 bdev_nvme_get_ctx_size(void) 219 { 220 return sizeof(struct nvme_bdev_io); 221 } 222 223 static struct spdk_bdev_module nvme_if = { 224 .name = "nvme", 225 .async_fini = true, 226 .module_init = bdev_nvme_library_init, 227 .module_fini = bdev_nvme_library_fini, 228 .config_json = bdev_nvme_config_json, 229 .get_ctx_size = bdev_nvme_get_ctx_size, 230 231 }; 232 SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if) 233 234 struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs); 235 pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER; 236 bool g_bdev_nvme_module_finish; 237 238 struct nvme_bdev_ctrlr * 239 nvme_bdev_ctrlr_get_by_name(const char *name) 240 { 241 struct nvme_bdev_ctrlr *nbdev_ctrlr; 242 243 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 244 if (strcmp(name, nbdev_ctrlr->name) == 0) { 245 break; 246 } 247 } 248 249 return nbdev_ctrlr; 250 } 251 252 static struct nvme_ctrlr * 253 nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr, 254 const struct spdk_nvme_transport_id *trid) 255 { 256 struct nvme_ctrlr *nvme_ctrlr; 257 258 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 259 if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0) { 260 break; 261 } 262 } 263 264 return nvme_ctrlr; 265 } 266 267 static struct nvme_bdev * 268 nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid) 269 { 270 struct nvme_bdev *bdev; 271 272 pthread_mutex_lock(&g_bdev_nvme_mutex); 273 TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) { 274 if (bdev->nsid == nsid) { 275 break; 276 } 277 } 278 pthread_mutex_unlock(&g_bdev_nvme_mutex); 279 280 return bdev; 281 } 282 283 struct nvme_ns * 284 nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid) 285 { 286 struct nvme_ns ns; 287 288 assert(nsid > 0); 289 290 ns.id = nsid; 291 return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns); 292 } 293 294 struct nvme_ns * 295 nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr) 296 { 297 return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces); 298 } 299 300 struct nvme_ns * 301 nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns) 302 { 303 if (ns == NULL) { 304 return NULL; 305 } 306 307 return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 308 } 309 310 static struct nvme_ctrlr * 311 nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid) 312 { 313 struct nvme_bdev_ctrlr *nbdev_ctrlr; 314 struct nvme_ctrlr *nvme_ctrlr = NULL; 315 316 pthread_mutex_lock(&g_bdev_nvme_mutex); 317 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 318 nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid); 319 if (nvme_ctrlr != NULL) { 320 break; 321 } 322 } 323 pthread_mutex_unlock(&g_bdev_nvme_mutex); 324 325 return nvme_ctrlr; 326 } 327 328 struct nvme_ctrlr * 329 nvme_ctrlr_get_by_name(const char *name) 330 { 331 struct nvme_bdev_ctrlr *nbdev_ctrlr; 332 struct nvme_ctrlr *nvme_ctrlr = NULL; 333 334 if (name == NULL) { 335 return NULL; 336 } 337 338 pthread_mutex_lock(&g_bdev_nvme_mutex); 339 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 340 if (nbdev_ctrlr != NULL) { 341 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 342 } 343 pthread_mutex_unlock(&g_bdev_nvme_mutex); 344 345 return nvme_ctrlr; 346 } 347 348 void 349 nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx) 350 { 351 struct nvme_bdev_ctrlr *nbdev_ctrlr; 352 353 pthread_mutex_lock(&g_bdev_nvme_mutex); 354 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 355 fn(nbdev_ctrlr, ctx); 356 } 357 pthread_mutex_unlock(&g_bdev_nvme_mutex); 358 } 359 360 void 361 nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w) 362 { 363 const char *trtype_str; 364 const char *adrfam_str; 365 366 trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype); 367 if (trtype_str) { 368 spdk_json_write_named_string(w, "trtype", trtype_str); 369 } 370 371 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 372 if (adrfam_str) { 373 spdk_json_write_named_string(w, "adrfam", adrfam_str); 374 } 375 376 if (trid->traddr[0] != '\0') { 377 spdk_json_write_named_string(w, "traddr", trid->traddr); 378 } 379 380 if (trid->trsvcid[0] != '\0') { 381 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 382 } 383 384 if (trid->subnqn[0] != '\0') { 385 spdk_json_write_named_string(w, "subnqn", trid->subnqn); 386 } 387 } 388 389 static void 390 nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr, 391 struct nvme_ctrlr *nvme_ctrlr) 392 { 393 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name); 394 pthread_mutex_lock(&g_bdev_nvme_mutex); 395 396 TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 397 if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) { 398 pthread_mutex_unlock(&g_bdev_nvme_mutex); 399 400 return; 401 } 402 TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 403 404 pthread_mutex_unlock(&g_bdev_nvme_mutex); 405 406 assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs)); 407 408 free(nbdev_ctrlr->name); 409 free(nbdev_ctrlr); 410 } 411 412 static void 413 _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 414 { 415 struct nvme_path_id *path_id, *tmp_path; 416 struct nvme_ns *ns, *tmp_ns; 417 418 free(nvme_ctrlr->copied_ana_desc); 419 spdk_free(nvme_ctrlr->ana_log_page); 420 421 if (nvme_ctrlr->opal_dev) { 422 spdk_opal_dev_destruct(nvme_ctrlr->opal_dev); 423 nvme_ctrlr->opal_dev = NULL; 424 } 425 426 if (nvme_ctrlr->nbdev_ctrlr) { 427 nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr); 428 } 429 430 RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) { 431 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 432 nvme_ns_free(ns); 433 } 434 435 TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) { 436 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 437 free(path_id); 438 } 439 440 pthread_mutex_destroy(&nvme_ctrlr->mutex); 441 442 free(nvme_ctrlr); 443 444 pthread_mutex_lock(&g_bdev_nvme_mutex); 445 if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 446 pthread_mutex_unlock(&g_bdev_nvme_mutex); 447 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 448 spdk_bdev_module_fini_done(); 449 return; 450 } 451 pthread_mutex_unlock(&g_bdev_nvme_mutex); 452 } 453 454 static int 455 nvme_detach_poller(void *arg) 456 { 457 struct nvme_ctrlr *nvme_ctrlr = arg; 458 int rc; 459 460 rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx); 461 if (rc != -EAGAIN) { 462 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 463 _nvme_ctrlr_delete(nvme_ctrlr); 464 } 465 466 return SPDK_POLLER_BUSY; 467 } 468 469 static void 470 nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 471 { 472 int rc; 473 474 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 475 476 /* First, unregister the adminq poller, as the driver will poll adminq if necessary */ 477 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 478 479 /* If we got here, the reset/detach poller cannot be active */ 480 assert(nvme_ctrlr->reset_detach_poller == NULL); 481 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller, 482 nvme_ctrlr, 1000); 483 if (nvme_ctrlr->reset_detach_poller == NULL) { 484 SPDK_ERRLOG("Failed to register detach poller\n"); 485 goto error; 486 } 487 488 rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx); 489 if (rc != 0) { 490 SPDK_ERRLOG("Failed to detach the NVMe controller\n"); 491 goto error; 492 } 493 494 return; 495 error: 496 /* We don't have a good way to handle errors here, so just do what we can and delete the 497 * controller without detaching the underlying NVMe device. 498 */ 499 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 500 _nvme_ctrlr_delete(nvme_ctrlr); 501 } 502 503 static void 504 nvme_ctrlr_unregister_cb(void *io_device) 505 { 506 struct nvme_ctrlr *nvme_ctrlr = io_device; 507 508 nvme_ctrlr_delete(nvme_ctrlr); 509 } 510 511 static void 512 nvme_ctrlr_unregister(void *ctx) 513 { 514 struct nvme_ctrlr *nvme_ctrlr = ctx; 515 516 spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb); 517 } 518 519 static bool 520 nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr) 521 { 522 if (!nvme_ctrlr->destruct) { 523 return false; 524 } 525 526 if (nvme_ctrlr->ref > 0) { 527 return false; 528 } 529 530 if (nvme_ctrlr->resetting) { 531 return false; 532 } 533 534 if (nvme_ctrlr->ana_log_page_updating) { 535 return false; 536 } 537 538 if (nvme_ctrlr->io_path_cache_clearing) { 539 return false; 540 } 541 542 return true; 543 } 544 545 static void 546 nvme_ctrlr_release(struct nvme_ctrlr *nvme_ctrlr) 547 { 548 pthread_mutex_lock(&nvme_ctrlr->mutex); 549 SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref); 550 551 assert(nvme_ctrlr->ref > 0); 552 nvme_ctrlr->ref--; 553 554 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 555 pthread_mutex_unlock(&nvme_ctrlr->mutex); 556 return; 557 } 558 559 pthread_mutex_unlock(&nvme_ctrlr->mutex); 560 561 spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr); 562 } 563 564 static void 565 bdev_nvme_clear_current_io_path(struct nvme_bdev_channel *nbdev_ch) 566 { 567 nbdev_ch->current_io_path = NULL; 568 nbdev_ch->rr_counter = 0; 569 } 570 571 static struct nvme_io_path * 572 _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 573 { 574 struct nvme_io_path *io_path; 575 576 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 577 if (io_path->nvme_ns == nvme_ns) { 578 break; 579 } 580 } 581 582 return io_path; 583 } 584 585 static struct nvme_io_path * 586 nvme_io_path_alloc(void) 587 { 588 struct nvme_io_path *io_path; 589 590 io_path = calloc(1, sizeof(*io_path)); 591 if (io_path == NULL) { 592 SPDK_ERRLOG("Failed to alloc io_path.\n"); 593 return NULL; 594 } 595 596 if (g_opts.io_path_stat) { 597 io_path->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 598 if (io_path->stat == NULL) { 599 free(io_path); 600 SPDK_ERRLOG("Failed to alloc io_path stat.\n"); 601 return NULL; 602 } 603 spdk_bdev_reset_io_stat(io_path->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 604 } 605 606 return io_path; 607 } 608 609 static void 610 nvme_io_path_free(struct nvme_io_path *io_path) 611 { 612 free(io_path->stat); 613 free(io_path); 614 } 615 616 static int 617 _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 618 { 619 struct nvme_io_path *io_path; 620 struct spdk_io_channel *ch; 621 struct nvme_ctrlr_channel *ctrlr_ch; 622 struct nvme_qpair *nvme_qpair; 623 624 io_path = nvme_io_path_alloc(); 625 if (io_path == NULL) { 626 return -ENOMEM; 627 } 628 629 io_path->nvme_ns = nvme_ns; 630 631 ch = spdk_get_io_channel(nvme_ns->ctrlr); 632 if (ch == NULL) { 633 nvme_io_path_free(io_path); 634 SPDK_ERRLOG("Failed to alloc io_channel.\n"); 635 return -ENOMEM; 636 } 637 638 ctrlr_ch = spdk_io_channel_get_ctx(ch); 639 640 nvme_qpair = ctrlr_ch->qpair; 641 assert(nvme_qpair != NULL); 642 643 io_path->qpair = nvme_qpair; 644 TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq); 645 646 io_path->nbdev_ch = nbdev_ch; 647 STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq); 648 649 bdev_nvme_clear_current_io_path(nbdev_ch); 650 651 return 0; 652 } 653 654 static void 655 _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path) 656 { 657 struct spdk_io_channel *ch; 658 struct nvme_qpair *nvme_qpair; 659 struct nvme_ctrlr_channel *ctrlr_ch; 660 struct nvme_bdev *nbdev; 661 662 nbdev = spdk_io_channel_get_io_device(spdk_io_channel_from_ctx(nbdev_ch)); 663 664 /* Add the statistics to nvme_ns before this path is destroyed. */ 665 pthread_mutex_lock(&nbdev->mutex); 666 if (nbdev->ref != 0 && io_path->nvme_ns->stat != NULL && io_path->stat != NULL) { 667 spdk_bdev_add_io_stat(io_path->nvme_ns->stat, io_path->stat); 668 } 669 pthread_mutex_unlock(&nbdev->mutex); 670 671 bdev_nvme_clear_current_io_path(nbdev_ch); 672 673 STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq); 674 io_path->nbdev_ch = NULL; 675 676 nvme_qpair = io_path->qpair; 677 assert(nvme_qpair != NULL); 678 679 ctrlr_ch = nvme_qpair->ctrlr_ch; 680 assert(ctrlr_ch != NULL); 681 682 ch = spdk_io_channel_from_ctx(ctrlr_ch); 683 spdk_put_io_channel(ch); 684 685 /* After an io_path is removed, I/Os submitted to it may complete and update statistics 686 * of the io_path. To avoid heap-use-after-free error from this case, do not free the 687 * io_path here but free the io_path when the associated qpair is freed. It is ensured 688 * that all I/Os submitted to the io_path are completed when the associated qpair is freed. 689 */ 690 } 691 692 static void 693 _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch) 694 { 695 struct nvme_io_path *io_path, *tmp_io_path; 696 697 STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) { 698 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 699 } 700 } 701 702 static int 703 bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf) 704 { 705 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 706 struct nvme_bdev *nbdev = io_device; 707 struct nvme_ns *nvme_ns; 708 int rc; 709 710 STAILQ_INIT(&nbdev_ch->io_path_list); 711 TAILQ_INIT(&nbdev_ch->retry_io_list); 712 713 pthread_mutex_lock(&nbdev->mutex); 714 715 nbdev_ch->mp_policy = nbdev->mp_policy; 716 nbdev_ch->mp_selector = nbdev->mp_selector; 717 nbdev_ch->rr_min_io = nbdev->rr_min_io; 718 719 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 720 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 721 if (rc != 0) { 722 pthread_mutex_unlock(&nbdev->mutex); 723 724 _bdev_nvme_delete_io_paths(nbdev_ch); 725 return rc; 726 } 727 } 728 pthread_mutex_unlock(&nbdev->mutex); 729 730 return 0; 731 } 732 733 /* If cpl != NULL, complete the bdev_io with nvme status based on 'cpl'. 734 * If cpl == NULL, complete the bdev_io with bdev status based on 'status'. 735 */ 736 static inline void 737 __bdev_nvme_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status, 738 const struct spdk_nvme_cpl *cpl) 739 { 740 spdk_trace_record(TRACE_BDEV_NVME_IO_DONE, 0, 0, (uintptr_t)bdev_io->driver_ctx, 741 (uintptr_t)bdev_io); 742 if (cpl) { 743 spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc); 744 } else { 745 spdk_bdev_io_complete(bdev_io, status); 746 } 747 } 748 749 static void bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch); 750 751 static void 752 bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf) 753 { 754 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 755 756 bdev_nvme_abort_retry_ios(nbdev_ch); 757 _bdev_nvme_delete_io_paths(nbdev_ch); 758 } 759 760 static inline bool 761 bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type) 762 { 763 switch (io_type) { 764 case SPDK_BDEV_IO_TYPE_RESET: 765 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 766 case SPDK_BDEV_IO_TYPE_ABORT: 767 return true; 768 default: 769 break; 770 } 771 772 return false; 773 } 774 775 static inline bool 776 nvme_ns_is_accessible(struct nvme_ns *nvme_ns) 777 { 778 if (spdk_unlikely(nvme_ns->ana_state_updating)) { 779 return false; 780 } 781 782 switch (nvme_ns->ana_state) { 783 case SPDK_NVME_ANA_OPTIMIZED_STATE: 784 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 785 return true; 786 default: 787 break; 788 } 789 790 return false; 791 } 792 793 static inline bool 794 nvme_io_path_is_connected(struct nvme_io_path *io_path) 795 { 796 if (spdk_unlikely(io_path->qpair->qpair == NULL)) { 797 return false; 798 } 799 800 if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(io_path->qpair->qpair) != 801 SPDK_NVME_QPAIR_FAILURE_NONE)) { 802 return false; 803 } 804 805 if (spdk_unlikely(io_path->qpair->ctrlr_ch->reset_iter != NULL)) { 806 return false; 807 } 808 809 if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(io_path->qpair->ctrlr->ctrlr) != 810 SPDK_NVME_QPAIR_FAILURE_NONE) { 811 return false; 812 } 813 814 return true; 815 } 816 817 static inline bool 818 nvme_io_path_is_available(struct nvme_io_path *io_path) 819 { 820 if (spdk_unlikely(!nvme_io_path_is_connected(io_path))) { 821 return false; 822 } 823 824 if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) { 825 return false; 826 } 827 828 return true; 829 } 830 831 static inline bool 832 nvme_io_path_is_failed(struct nvme_io_path *io_path) 833 { 834 struct nvme_ctrlr *nvme_ctrlr; 835 836 nvme_ctrlr = io_path->qpair->ctrlr; 837 838 if (nvme_ctrlr->destruct) { 839 return true; 840 } 841 842 if (nvme_ctrlr->fast_io_fail_timedout) { 843 return true; 844 } 845 846 if (nvme_ctrlr->resetting) { 847 if (nvme_ctrlr->opts.reconnect_delay_sec != 0) { 848 return false; 849 } else { 850 return true; 851 } 852 } 853 854 if (nvme_ctrlr->reconnect_is_delayed) { 855 return false; 856 } 857 858 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 859 return true; 860 } else { 861 return false; 862 } 863 } 864 865 static bool 866 nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr) 867 { 868 if (nvme_ctrlr->destruct) { 869 return false; 870 } 871 872 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 873 return false; 874 } 875 876 if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) { 877 return false; 878 } 879 880 return true; 881 } 882 883 /* Simulate circular linked list. */ 884 static inline struct nvme_io_path * 885 nvme_io_path_get_next(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *prev_path) 886 { 887 struct nvme_io_path *next_path; 888 889 if (prev_path != NULL) { 890 next_path = STAILQ_NEXT(prev_path, stailq); 891 if (next_path != NULL) { 892 return next_path; 893 } 894 } 895 896 return STAILQ_FIRST(&nbdev_ch->io_path_list); 897 } 898 899 static struct nvme_io_path * 900 _bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 901 { 902 struct nvme_io_path *io_path, *start, *non_optimized = NULL; 903 904 start = nvme_io_path_get_next(nbdev_ch, nbdev_ch->current_io_path); 905 906 io_path = start; 907 do { 908 if (spdk_likely(nvme_io_path_is_connected(io_path) && 909 !io_path->nvme_ns->ana_state_updating)) { 910 switch (io_path->nvme_ns->ana_state) { 911 case SPDK_NVME_ANA_OPTIMIZED_STATE: 912 nbdev_ch->current_io_path = io_path; 913 return io_path; 914 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 915 if (non_optimized == NULL) { 916 non_optimized = io_path; 917 } 918 break; 919 default: 920 break; 921 } 922 } 923 io_path = nvme_io_path_get_next(nbdev_ch, io_path); 924 } while (io_path != start); 925 926 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 927 /* We come here only if there is no optimized path. Cache even non_optimized 928 * path for load balance across multiple non_optimized paths. 929 */ 930 nbdev_ch->current_io_path = non_optimized; 931 } 932 933 return non_optimized; 934 } 935 936 static struct nvme_io_path * 937 _bdev_nvme_find_io_path_min_qd(struct nvme_bdev_channel *nbdev_ch) 938 { 939 struct nvme_io_path *io_path; 940 struct nvme_io_path *optimized = NULL, *non_optimized = NULL; 941 uint32_t opt_min_qd = UINT32_MAX, non_opt_min_qd = UINT32_MAX; 942 uint32_t num_outstanding_reqs; 943 944 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 945 if (spdk_unlikely(!nvme_io_path_is_connected(io_path))) { 946 /* The device is currently resetting. */ 947 continue; 948 } 949 950 if (spdk_unlikely(io_path->nvme_ns->ana_state_updating)) { 951 continue; 952 } 953 954 num_outstanding_reqs = spdk_nvme_qpair_get_num_outstanding_reqs(io_path->qpair->qpair); 955 switch (io_path->nvme_ns->ana_state) { 956 case SPDK_NVME_ANA_OPTIMIZED_STATE: 957 if (num_outstanding_reqs < opt_min_qd) { 958 opt_min_qd = num_outstanding_reqs; 959 optimized = io_path; 960 } 961 break; 962 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 963 if (num_outstanding_reqs < non_opt_min_qd) { 964 non_opt_min_qd = num_outstanding_reqs; 965 non_optimized = io_path; 966 } 967 break; 968 default: 969 break; 970 } 971 } 972 973 /* don't cache io path for BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH selector */ 974 if (optimized != NULL) { 975 return optimized; 976 } 977 978 return non_optimized; 979 } 980 981 static inline struct nvme_io_path * 982 bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 983 { 984 if (spdk_likely(nbdev_ch->current_io_path != NULL)) { 985 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE) { 986 return nbdev_ch->current_io_path; 987 } else if (nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 988 if (++nbdev_ch->rr_counter < nbdev_ch->rr_min_io) { 989 return nbdev_ch->current_io_path; 990 } 991 nbdev_ch->rr_counter = 0; 992 } 993 } 994 995 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE || 996 nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 997 return _bdev_nvme_find_io_path(nbdev_ch); 998 } else { 999 return _bdev_nvme_find_io_path_min_qd(nbdev_ch); 1000 } 1001 } 1002 1003 /* Return true if there is any io_path whose qpair is active or ctrlr is not failed, 1004 * or false otherwise. 1005 * 1006 * If any io_path has an active qpair but find_io_path() returned NULL, its namespace 1007 * is likely to be non-accessible now but may become accessible. 1008 * 1009 * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr 1010 * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed 1011 * when starting to reset it but it is set to failed when the reset failed. Hence, if 1012 * a ctrlr is unfailed, it is likely that it works fine or is resetting. 1013 */ 1014 static bool 1015 any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch) 1016 { 1017 struct nvme_io_path *io_path; 1018 1019 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 1020 if (io_path->nvme_ns->ana_transition_timedout) { 1021 continue; 1022 } 1023 1024 if (nvme_io_path_is_connected(io_path) || 1025 !nvme_io_path_is_failed(io_path)) { 1026 return true; 1027 } 1028 } 1029 1030 return false; 1031 } 1032 1033 static void 1034 bdev_nvme_retry_io(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 1035 { 1036 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1037 struct spdk_io_channel *ch; 1038 1039 if (nbdev_io->io_path != NULL && nvme_io_path_is_available(nbdev_io->io_path)) { 1040 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 1041 } else { 1042 ch = spdk_io_channel_from_ctx(nbdev_ch); 1043 bdev_nvme_submit_request(ch, bdev_io); 1044 } 1045 } 1046 1047 static int 1048 bdev_nvme_retry_ios(void *arg) 1049 { 1050 struct nvme_bdev_channel *nbdev_ch = arg; 1051 struct spdk_bdev_io *bdev_io, *tmp_bdev_io; 1052 struct nvme_bdev_io *bio; 1053 uint64_t now, delay_us; 1054 1055 now = spdk_get_ticks(); 1056 1057 TAILQ_FOREACH_SAFE(bdev_io, &nbdev_ch->retry_io_list, module_link, tmp_bdev_io) { 1058 bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1059 if (bio->retry_ticks > now) { 1060 break; 1061 } 1062 1063 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io, module_link); 1064 1065 bdev_nvme_retry_io(nbdev_ch, bdev_io); 1066 } 1067 1068 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1069 1070 bdev_io = TAILQ_FIRST(&nbdev_ch->retry_io_list); 1071 if (bdev_io != NULL) { 1072 bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1073 1074 delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz(); 1075 1076 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1077 delay_us); 1078 } 1079 1080 return SPDK_POLLER_BUSY; 1081 } 1082 1083 static void 1084 bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch, 1085 struct nvme_bdev_io *bio, uint64_t delay_ms) 1086 { 1087 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1088 struct spdk_bdev_io *tmp_bdev_io; 1089 struct nvme_bdev_io *tmp_bio; 1090 1091 bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL; 1092 1093 TAILQ_FOREACH_REVERSE(tmp_bdev_io, &nbdev_ch->retry_io_list, retry_io_head, module_link) { 1094 tmp_bio = (struct nvme_bdev_io *)tmp_bdev_io->driver_ctx; 1095 1096 if (tmp_bio->retry_ticks <= bio->retry_ticks) { 1097 TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bdev_io, bdev_io, 1098 module_link); 1099 return; 1100 } 1101 } 1102 1103 /* No earlier I/Os were found. This I/O must be the new head. */ 1104 TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bdev_io, module_link); 1105 1106 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1107 1108 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1109 delay_ms * 1000ULL); 1110 } 1111 1112 static void 1113 bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch) 1114 { 1115 struct spdk_bdev_io *bdev_io, *tmp_io; 1116 1117 TAILQ_FOREACH_SAFE(bdev_io, &nbdev_ch->retry_io_list, module_link, tmp_io) { 1118 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io, module_link); 1119 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1120 } 1121 1122 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1123 } 1124 1125 static int 1126 bdev_nvme_abort_retry_io(struct nvme_bdev_channel *nbdev_ch, 1127 struct nvme_bdev_io *bio_to_abort) 1128 { 1129 struct spdk_bdev_io *bdev_io_to_abort; 1130 1131 TAILQ_FOREACH(bdev_io_to_abort, &nbdev_ch->retry_io_list, module_link) { 1132 if ((struct nvme_bdev_io *)bdev_io_to_abort->driver_ctx == bio_to_abort) { 1133 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io_to_abort, module_link); 1134 __bdev_nvme_io_complete(bdev_io_to_abort, SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1135 return 0; 1136 } 1137 } 1138 1139 return -ENOENT; 1140 } 1141 1142 static void 1143 bdev_nvme_update_nvme_error_stat(struct spdk_bdev_io *bdev_io, const struct spdk_nvme_cpl *cpl) 1144 { 1145 struct nvme_bdev *nbdev; 1146 uint16_t sct, sc; 1147 1148 assert(spdk_nvme_cpl_is_error(cpl)); 1149 1150 nbdev = bdev_io->bdev->ctxt; 1151 1152 if (nbdev->err_stat == NULL) { 1153 return; 1154 } 1155 1156 sct = cpl->status.sct; 1157 sc = cpl->status.sc; 1158 1159 pthread_mutex_lock(&nbdev->mutex); 1160 1161 nbdev->err_stat->status_type[sct]++; 1162 switch (sct) { 1163 case SPDK_NVME_SCT_GENERIC: 1164 case SPDK_NVME_SCT_COMMAND_SPECIFIC: 1165 case SPDK_NVME_SCT_MEDIA_ERROR: 1166 case SPDK_NVME_SCT_PATH: 1167 nbdev->err_stat->status[sct][sc]++; 1168 break; 1169 default: 1170 break; 1171 } 1172 1173 pthread_mutex_unlock(&nbdev->mutex); 1174 } 1175 1176 static inline void 1177 bdev_nvme_update_io_path_stat(struct nvme_bdev_io *bio) 1178 { 1179 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1180 uint64_t num_blocks = bdev_io->u.bdev.num_blocks; 1181 uint32_t blocklen = bdev_io->bdev->blocklen; 1182 struct spdk_bdev_io_stat *stat; 1183 uint64_t tsc_diff; 1184 1185 if (bio->io_path->stat == NULL) { 1186 return; 1187 } 1188 1189 tsc_diff = spdk_get_ticks() - bio->submit_tsc; 1190 stat = bio->io_path->stat; 1191 1192 switch (bdev_io->type) { 1193 case SPDK_BDEV_IO_TYPE_READ: 1194 stat->bytes_read += num_blocks * blocklen; 1195 stat->num_read_ops++; 1196 stat->read_latency_ticks += tsc_diff; 1197 if (stat->max_read_latency_ticks < tsc_diff) { 1198 stat->max_read_latency_ticks = tsc_diff; 1199 } 1200 if (stat->min_read_latency_ticks > tsc_diff) { 1201 stat->min_read_latency_ticks = tsc_diff; 1202 } 1203 break; 1204 case SPDK_BDEV_IO_TYPE_WRITE: 1205 stat->bytes_written += num_blocks * blocklen; 1206 stat->num_write_ops++; 1207 stat->write_latency_ticks += tsc_diff; 1208 if (stat->max_write_latency_ticks < tsc_diff) { 1209 stat->max_write_latency_ticks = tsc_diff; 1210 } 1211 if (stat->min_write_latency_ticks > tsc_diff) { 1212 stat->min_write_latency_ticks = tsc_diff; 1213 } 1214 break; 1215 case SPDK_BDEV_IO_TYPE_UNMAP: 1216 stat->bytes_unmapped += num_blocks * blocklen; 1217 stat->num_unmap_ops++; 1218 stat->unmap_latency_ticks += tsc_diff; 1219 if (stat->max_unmap_latency_ticks < tsc_diff) { 1220 stat->max_unmap_latency_ticks = tsc_diff; 1221 } 1222 if (stat->min_unmap_latency_ticks > tsc_diff) { 1223 stat->min_unmap_latency_ticks = tsc_diff; 1224 } 1225 break; 1226 case SPDK_BDEV_IO_TYPE_ZCOPY: 1227 /* Track the data in the start phase only */ 1228 if (!bdev_io->u.bdev.zcopy.start) { 1229 break; 1230 } 1231 if (bdev_io->u.bdev.zcopy.populate) { 1232 stat->bytes_read += num_blocks * blocklen; 1233 stat->num_read_ops++; 1234 stat->read_latency_ticks += tsc_diff; 1235 if (stat->max_read_latency_ticks < tsc_diff) { 1236 stat->max_read_latency_ticks = tsc_diff; 1237 } 1238 if (stat->min_read_latency_ticks > tsc_diff) { 1239 stat->min_read_latency_ticks = tsc_diff; 1240 } 1241 } else { 1242 stat->bytes_written += num_blocks * blocklen; 1243 stat->num_write_ops++; 1244 stat->write_latency_ticks += tsc_diff; 1245 if (stat->max_write_latency_ticks < tsc_diff) { 1246 stat->max_write_latency_ticks = tsc_diff; 1247 } 1248 if (stat->min_write_latency_ticks > tsc_diff) { 1249 stat->min_write_latency_ticks = tsc_diff; 1250 } 1251 } 1252 break; 1253 case SPDK_BDEV_IO_TYPE_COPY: 1254 stat->bytes_copied += num_blocks * blocklen; 1255 stat->num_copy_ops++; 1256 stat->copy_latency_ticks += tsc_diff; 1257 if (stat->max_copy_latency_ticks < tsc_diff) { 1258 stat->max_copy_latency_ticks = tsc_diff; 1259 } 1260 if (stat->min_copy_latency_ticks > tsc_diff) { 1261 stat->min_copy_latency_ticks = tsc_diff; 1262 } 1263 break; 1264 default: 1265 break; 1266 } 1267 } 1268 1269 static inline void 1270 bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio, 1271 const struct spdk_nvme_cpl *cpl) 1272 { 1273 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1274 struct nvme_bdev_channel *nbdev_ch; 1275 struct nvme_io_path *io_path; 1276 struct nvme_ctrlr *nvme_ctrlr; 1277 const struct spdk_nvme_ctrlr_data *cdata; 1278 uint64_t delay_ms; 1279 1280 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 1281 1282 if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) { 1283 bdev_nvme_update_io_path_stat(bio); 1284 goto complete; 1285 } 1286 1287 /* Update error counts before deciding if retry is needed. 1288 * Hence, error counts may be more than the number of I/O errors. 1289 */ 1290 bdev_nvme_update_nvme_error_stat(bdev_io, cpl); 1291 1292 if (cpl->status.dnr != 0 || spdk_nvme_cpl_is_aborted_by_request(cpl) || 1293 (g_opts.bdev_retry_count != -1 && bio->retry_count >= g_opts.bdev_retry_count)) { 1294 goto complete; 1295 } 1296 1297 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1298 1299 assert(bio->io_path != NULL); 1300 io_path = bio->io_path; 1301 1302 nvme_ctrlr = io_path->qpair->ctrlr; 1303 1304 if (spdk_nvme_cpl_is_path_error(cpl) || 1305 spdk_nvme_cpl_is_aborted_sq_deletion(cpl) || 1306 !nvme_io_path_is_available(io_path) || 1307 !nvme_ctrlr_is_available(nvme_ctrlr)) { 1308 bdev_nvme_clear_current_io_path(nbdev_ch); 1309 bio->io_path = NULL; 1310 if (spdk_nvme_cpl_is_ana_error(cpl)) { 1311 if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) { 1312 io_path->nvme_ns->ana_state_updating = true; 1313 } 1314 } 1315 if (!any_io_path_may_become_available(nbdev_ch)) { 1316 goto complete; 1317 } 1318 delay_ms = 0; 1319 } else { 1320 bio->retry_count++; 1321 1322 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 1323 1324 if (cpl->status.crd != 0) { 1325 delay_ms = cdata->crdt[cpl->status.crd] * 100; 1326 } else { 1327 delay_ms = 0; 1328 } 1329 } 1330 1331 bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms); 1332 return; 1333 1334 complete: 1335 bio->retry_count = 0; 1336 bio->submit_tsc = 0; 1337 __bdev_nvme_io_complete(bdev_io, 0, cpl); 1338 } 1339 1340 static inline void 1341 bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc) 1342 { 1343 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1344 struct nvme_bdev_channel *nbdev_ch; 1345 enum spdk_bdev_io_status io_status; 1346 1347 switch (rc) { 1348 case 0: 1349 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1350 break; 1351 case -ENOMEM: 1352 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1353 break; 1354 case -ENXIO: 1355 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1356 1357 bdev_nvme_clear_current_io_path(nbdev_ch); 1358 bio->io_path = NULL; 1359 1360 if (any_io_path_may_become_available(nbdev_ch)) { 1361 bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL); 1362 return; 1363 } 1364 1365 /* fallthrough */ 1366 default: 1367 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1368 break; 1369 } 1370 1371 bio->retry_count = 0; 1372 bio->submit_tsc = 0; 1373 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1374 } 1375 1376 static inline void 1377 bdev_nvme_admin_passthru_complete(struct nvme_bdev_io *bio, int rc) 1378 { 1379 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1380 enum spdk_bdev_io_status io_status; 1381 1382 switch (rc) { 1383 case 0: 1384 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1385 break; 1386 case -ENOMEM: 1387 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1388 break; 1389 case -ENXIO: 1390 /* fallthrough */ 1391 default: 1392 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1393 break; 1394 } 1395 1396 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1397 } 1398 1399 static void 1400 bdev_nvme_clear_io_path_caches_done(struct spdk_io_channel_iter *i, int status) 1401 { 1402 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1403 1404 pthread_mutex_lock(&nvme_ctrlr->mutex); 1405 1406 assert(nvme_ctrlr->io_path_cache_clearing == true); 1407 nvme_ctrlr->io_path_cache_clearing = false; 1408 1409 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 1410 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1411 return; 1412 } 1413 1414 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1415 1416 nvme_ctrlr_unregister(nvme_ctrlr); 1417 } 1418 1419 static void 1420 _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair) 1421 { 1422 struct nvme_io_path *io_path; 1423 1424 TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) { 1425 if (io_path->nbdev_ch == NULL) { 1426 continue; 1427 } 1428 bdev_nvme_clear_current_io_path(io_path->nbdev_ch); 1429 } 1430 } 1431 1432 static void 1433 bdev_nvme_clear_io_path_cache(struct spdk_io_channel_iter *i) 1434 { 1435 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 1436 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 1437 1438 assert(ctrlr_ch->qpair != NULL); 1439 1440 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 1441 1442 spdk_for_each_channel_continue(i, 0); 1443 } 1444 1445 static void 1446 bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr) 1447 { 1448 pthread_mutex_lock(&nvme_ctrlr->mutex); 1449 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 1450 nvme_ctrlr->io_path_cache_clearing) { 1451 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1452 return; 1453 } 1454 1455 nvme_ctrlr->io_path_cache_clearing = true; 1456 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1457 1458 spdk_for_each_channel(nvme_ctrlr, 1459 bdev_nvme_clear_io_path_cache, 1460 NULL, 1461 bdev_nvme_clear_io_path_caches_done); 1462 } 1463 1464 static struct nvme_qpair * 1465 nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair) 1466 { 1467 struct nvme_qpair *nvme_qpair; 1468 1469 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1470 if (nvme_qpair->qpair == qpair) { 1471 break; 1472 } 1473 } 1474 1475 return nvme_qpair; 1476 } 1477 1478 static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair); 1479 1480 static void 1481 bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx) 1482 { 1483 struct nvme_poll_group *group = poll_group_ctx; 1484 struct nvme_qpair *nvme_qpair; 1485 struct nvme_ctrlr_channel *ctrlr_ch; 1486 1487 nvme_qpair = nvme_poll_group_get_qpair(group, qpair); 1488 if (nvme_qpair == NULL) { 1489 return; 1490 } 1491 1492 if (nvme_qpair->qpair != NULL) { 1493 spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair); 1494 nvme_qpair->qpair = NULL; 1495 } 1496 1497 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1498 1499 ctrlr_ch = nvme_qpair->ctrlr_ch; 1500 1501 if (ctrlr_ch != NULL) { 1502 if (ctrlr_ch->reset_iter != NULL) { 1503 /* If we are already in a full reset sequence, we do not have 1504 * to restart it. Just move to the next ctrlr_channel. 1505 */ 1506 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed in a reset ctrlr sequence.\n", 1507 qpair); 1508 spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 1509 ctrlr_ch->reset_iter = NULL; 1510 } else { 1511 /* qpair was disconnected unexpectedly. Reset controller for recovery. */ 1512 SPDK_NOTICELOG("qpair %p was disconnected and freed. reset controller.\n", qpair); 1513 bdev_nvme_failover(nvme_qpair->ctrlr, false); 1514 } 1515 } else { 1516 /* In this case, ctrlr_channel is already deleted. */ 1517 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed. delete nvme_qpair.\n", qpair); 1518 nvme_qpair_delete(nvme_qpair); 1519 } 1520 } 1521 1522 static void 1523 bdev_nvme_check_io_qpairs(struct nvme_poll_group *group) 1524 { 1525 struct nvme_qpair *nvme_qpair; 1526 1527 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1528 if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) { 1529 continue; 1530 } 1531 1532 if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 1533 SPDK_NVME_QPAIR_FAILURE_NONE) { 1534 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1535 } 1536 } 1537 } 1538 1539 static int 1540 bdev_nvme_poll(void *arg) 1541 { 1542 struct nvme_poll_group *group = arg; 1543 int64_t num_completions; 1544 1545 if (group->collect_spin_stat && group->start_ticks == 0) { 1546 group->start_ticks = spdk_get_ticks(); 1547 } 1548 1549 num_completions = spdk_nvme_poll_group_process_completions(group->group, 0, 1550 bdev_nvme_disconnected_qpair_cb); 1551 if (group->collect_spin_stat) { 1552 if (num_completions > 0) { 1553 if (group->end_ticks != 0) { 1554 group->spin_ticks += (group->end_ticks - group->start_ticks); 1555 group->end_ticks = 0; 1556 } 1557 group->start_ticks = 0; 1558 } else { 1559 group->end_ticks = spdk_get_ticks(); 1560 } 1561 } 1562 1563 if (spdk_unlikely(num_completions < 0)) { 1564 bdev_nvme_check_io_qpairs(group); 1565 } 1566 1567 return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 1568 } 1569 1570 static int bdev_nvme_poll_adminq(void *arg); 1571 1572 static void 1573 bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_period_us) 1574 { 1575 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 1576 1577 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, 1578 nvme_ctrlr, new_period_us); 1579 } 1580 1581 static int 1582 bdev_nvme_poll_adminq(void *arg) 1583 { 1584 int32_t rc; 1585 struct nvme_ctrlr *nvme_ctrlr = arg; 1586 nvme_ctrlr_disconnected_cb disconnected_cb; 1587 1588 assert(nvme_ctrlr != NULL); 1589 1590 rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr); 1591 if (rc < 0) { 1592 disconnected_cb = nvme_ctrlr->disconnected_cb; 1593 nvme_ctrlr->disconnected_cb = NULL; 1594 1595 if (rc == -ENXIO && disconnected_cb != NULL) { 1596 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 1597 g_opts.nvme_adminq_poll_period_us); 1598 disconnected_cb(nvme_ctrlr); 1599 } else { 1600 bdev_nvme_failover(nvme_ctrlr, false); 1601 } 1602 } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) != 1603 SPDK_NVME_QPAIR_FAILURE_NONE) { 1604 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 1605 } 1606 1607 return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY; 1608 } 1609 1610 static void 1611 _bdev_nvme_unregister_dev_cb(void *io_device) 1612 { 1613 struct nvme_bdev *nvme_disk = io_device; 1614 1615 free(nvme_disk->disk.name); 1616 free(nvme_disk->err_stat); 1617 free(nvme_disk); 1618 } 1619 1620 static int 1621 bdev_nvme_destruct(void *ctx) 1622 { 1623 struct nvme_bdev *nvme_disk = ctx; 1624 struct nvme_ns *nvme_ns, *tmp_nvme_ns; 1625 1626 SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid); 1627 1628 TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) { 1629 pthread_mutex_lock(&nvme_ns->ctrlr->mutex); 1630 1631 nvme_ns->bdev = NULL; 1632 1633 assert(nvme_ns->id > 0); 1634 1635 if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) { 1636 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1637 1638 nvme_ctrlr_release(nvme_ns->ctrlr); 1639 nvme_ns_free(nvme_ns); 1640 } else { 1641 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1642 } 1643 } 1644 1645 pthread_mutex_lock(&g_bdev_nvme_mutex); 1646 TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq); 1647 pthread_mutex_unlock(&g_bdev_nvme_mutex); 1648 1649 spdk_io_device_unregister(nvme_disk, _bdev_nvme_unregister_dev_cb); 1650 1651 return 0; 1652 } 1653 1654 static int 1655 bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair) 1656 { 1657 struct nvme_ctrlr *nvme_ctrlr; 1658 struct spdk_nvme_io_qpair_opts opts; 1659 struct spdk_nvme_qpair *qpair; 1660 int rc; 1661 1662 nvme_ctrlr = nvme_qpair->ctrlr; 1663 1664 spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1665 opts.delay_cmd_submit = g_opts.delay_cmd_submit; 1666 opts.create_only = true; 1667 opts.async_mode = true; 1668 opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests); 1669 g_opts.io_queue_requests = opts.io_queue_requests; 1670 1671 qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1672 if (qpair == NULL) { 1673 return -1; 1674 } 1675 1676 SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name, 1677 spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread)); 1678 1679 assert(nvme_qpair->group != NULL); 1680 1681 rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair); 1682 if (rc != 0) { 1683 SPDK_ERRLOG("Unable to begin polling on NVMe Channel.\n"); 1684 goto err; 1685 } 1686 1687 rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair); 1688 if (rc != 0) { 1689 SPDK_ERRLOG("Unable to connect I/O qpair.\n"); 1690 goto err; 1691 } 1692 1693 nvme_qpair->qpair = qpair; 1694 1695 if (!g_opts.disable_auto_failback) { 1696 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1697 } 1698 1699 return 0; 1700 1701 err: 1702 spdk_nvme_ctrlr_free_io_qpair(qpair); 1703 1704 return rc; 1705 } 1706 1707 static void 1708 bdev_nvme_complete_pending_resets(struct spdk_io_channel_iter *i) 1709 { 1710 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 1711 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 1712 enum spdk_bdev_io_status status = SPDK_BDEV_IO_STATUS_SUCCESS; 1713 struct spdk_bdev_io *bdev_io; 1714 1715 if (spdk_io_channel_iter_get_ctx(i) != NULL) { 1716 status = SPDK_BDEV_IO_STATUS_FAILED; 1717 } 1718 1719 while (!TAILQ_EMPTY(&ctrlr_ch->pending_resets)) { 1720 bdev_io = TAILQ_FIRST(&ctrlr_ch->pending_resets); 1721 TAILQ_REMOVE(&ctrlr_ch->pending_resets, bdev_io, module_link); 1722 __bdev_nvme_io_complete(bdev_io, status, NULL); 1723 } 1724 1725 spdk_for_each_channel_continue(i, 0); 1726 } 1727 1728 static void 1729 bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove) 1730 { 1731 struct nvme_path_id *path_id, *next_path; 1732 int rc __attribute__((unused)); 1733 1734 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 1735 assert(path_id); 1736 assert(path_id == nvme_ctrlr->active_path_id); 1737 next_path = TAILQ_NEXT(path_id, link); 1738 1739 path_id->is_failed = true; 1740 1741 if (next_path == NULL) { 1742 return; 1743 } 1744 1745 assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE); 1746 1747 SPDK_NOTICELOG("Start failover from %s:%s to %s:%s\n", path_id->trid.traddr, 1748 path_id->trid.trsvcid, next_path->trid.traddr, next_path->trid.trsvcid); 1749 1750 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 1751 nvme_ctrlr->active_path_id = next_path; 1752 rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid); 1753 assert(rc == 0); 1754 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 1755 if (!remove) { 1756 /** Shuffle the old trid to the end of the list and use the new one. 1757 * Allows for round robin through multiple connections. 1758 */ 1759 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link); 1760 } else { 1761 free(path_id); 1762 } 1763 } 1764 1765 static bool 1766 bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr) 1767 { 1768 int32_t elapsed; 1769 1770 if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 || 1771 nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) { 1772 return false; 1773 } 1774 1775 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 1776 if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) { 1777 return true; 1778 } else { 1779 return false; 1780 } 1781 } 1782 1783 static bool 1784 bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr) 1785 { 1786 uint32_t elapsed; 1787 1788 if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) { 1789 return false; 1790 } 1791 1792 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 1793 if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) { 1794 return true; 1795 } else { 1796 return false; 1797 } 1798 } 1799 1800 static void bdev_nvme_reset_complete(struct nvme_ctrlr *nvme_ctrlr, bool success); 1801 1802 static void 1803 nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb cb_fn) 1804 { 1805 int rc; 1806 1807 rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr); 1808 if (rc != 0) { 1809 /* Disconnect fails if ctrlr is already resetting or removed. In this case, 1810 * fail the reset sequence immediately. 1811 */ 1812 bdev_nvme_reset_complete(nvme_ctrlr, false); 1813 return; 1814 } 1815 1816 /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq. 1817 * Set callback here to execute the specified operation after ctrlr is really disconnected. 1818 */ 1819 assert(nvme_ctrlr->disconnected_cb == NULL); 1820 nvme_ctrlr->disconnected_cb = cb_fn; 1821 1822 /* During disconnection, reduce the period to poll adminq more often. */ 1823 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0); 1824 } 1825 1826 enum bdev_nvme_op_after_reset { 1827 OP_NONE, 1828 OP_COMPLETE_PENDING_DESTRUCT, 1829 OP_DESTRUCT, 1830 OP_DELAYED_RECONNECT, 1831 }; 1832 1833 typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset; 1834 1835 static _bdev_nvme_op_after_reset 1836 bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success) 1837 { 1838 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 1839 /* Complete pending destruct after reset completes. */ 1840 return OP_COMPLETE_PENDING_DESTRUCT; 1841 } else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) { 1842 nvme_ctrlr->reset_start_tsc = 0; 1843 return OP_NONE; 1844 } else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 1845 return OP_DESTRUCT; 1846 } else { 1847 if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) { 1848 nvme_ctrlr->fast_io_fail_timedout = true; 1849 } 1850 bdev_nvme_failover_trid(nvme_ctrlr, false); 1851 return OP_DELAYED_RECONNECT; 1852 } 1853 } 1854 1855 static int bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug); 1856 static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 1857 1858 static int 1859 bdev_nvme_reconnect_delay_timer_expired(void *ctx) 1860 { 1861 struct nvme_ctrlr *nvme_ctrlr = ctx; 1862 1863 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect_delay, nvme_ctrlr->nbdev_ctrlr->name); 1864 pthread_mutex_lock(&nvme_ctrlr->mutex); 1865 1866 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 1867 1868 if (!nvme_ctrlr->reconnect_is_delayed) { 1869 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1870 return SPDK_POLLER_BUSY; 1871 } 1872 1873 nvme_ctrlr->reconnect_is_delayed = false; 1874 1875 if (nvme_ctrlr->destruct) { 1876 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1877 return SPDK_POLLER_BUSY; 1878 } 1879 1880 assert(nvme_ctrlr->resetting == false); 1881 nvme_ctrlr->resetting = true; 1882 1883 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1884 1885 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 1886 1887 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 1888 return SPDK_POLLER_BUSY; 1889 } 1890 1891 static void 1892 bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr) 1893 { 1894 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 1895 1896 assert(nvme_ctrlr->reconnect_is_delayed == false); 1897 nvme_ctrlr->reconnect_is_delayed = true; 1898 1899 assert(nvme_ctrlr->reconnect_delay_timer == NULL); 1900 nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired, 1901 nvme_ctrlr, 1902 nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC); 1903 } 1904 1905 static void 1906 _bdev_nvme_reset_complete(struct spdk_io_channel_iter *i, int status) 1907 { 1908 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1909 bool success = spdk_io_channel_iter_get_ctx(i) == NULL; 1910 struct nvme_path_id *path_id; 1911 bdev_nvme_reset_cb reset_cb_fn = nvme_ctrlr->reset_cb_fn; 1912 void *reset_cb_arg = nvme_ctrlr->reset_cb_arg; 1913 enum bdev_nvme_op_after_reset op_after_reset; 1914 1915 assert(nvme_ctrlr->thread == spdk_get_thread()); 1916 1917 nvme_ctrlr->reset_cb_fn = NULL; 1918 nvme_ctrlr->reset_cb_arg = NULL; 1919 1920 if (!success) { 1921 SPDK_ERRLOG("Resetting controller failed.\n"); 1922 } else { 1923 SPDK_NOTICELOG("Resetting controller successful.\n"); 1924 } 1925 1926 pthread_mutex_lock(&nvme_ctrlr->mutex); 1927 nvme_ctrlr->resetting = false; 1928 nvme_ctrlr->dont_retry = false; 1929 1930 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 1931 assert(path_id != NULL); 1932 assert(path_id == nvme_ctrlr->active_path_id); 1933 1934 path_id->is_failed = !success; 1935 1936 op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success); 1937 1938 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1939 1940 if (reset_cb_fn) { 1941 reset_cb_fn(reset_cb_arg, success); 1942 } 1943 1944 switch (op_after_reset) { 1945 case OP_COMPLETE_PENDING_DESTRUCT: 1946 nvme_ctrlr_unregister(nvme_ctrlr); 1947 break; 1948 case OP_DESTRUCT: 1949 bdev_nvme_delete_ctrlr(nvme_ctrlr, false); 1950 break; 1951 case OP_DELAYED_RECONNECT: 1952 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_start_reconnect_delay_timer); 1953 break; 1954 default: 1955 break; 1956 } 1957 } 1958 1959 static void 1960 bdev_nvme_reset_complete(struct nvme_ctrlr *nvme_ctrlr, bool success) 1961 { 1962 /* Make sure we clear any pending resets before returning. */ 1963 spdk_for_each_channel(nvme_ctrlr, 1964 bdev_nvme_complete_pending_resets, 1965 success ? NULL : (void *)0x1, 1966 _bdev_nvme_reset_complete); 1967 } 1968 1969 static void 1970 bdev_nvme_reset_create_qpairs_failed(struct spdk_io_channel_iter *i, int status) 1971 { 1972 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1973 1974 bdev_nvme_reset_complete(nvme_ctrlr, false); 1975 } 1976 1977 static void 1978 bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i) 1979 { 1980 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 1981 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch); 1982 struct nvme_qpair *nvme_qpair; 1983 1984 nvme_qpair = ctrlr_ch->qpair; 1985 assert(nvme_qpair != NULL); 1986 1987 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1988 1989 if (nvme_qpair->qpair != NULL) { 1990 if (nvme_qpair->ctrlr->dont_retry) { 1991 spdk_nvme_qpair_set_abort_dnr(nvme_qpair->qpair, true); 1992 } 1993 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 1994 1995 /* The current full reset sequence will move to the next 1996 * ctrlr_channel after the qpair is actually disconnected. 1997 */ 1998 assert(ctrlr_ch->reset_iter == NULL); 1999 ctrlr_ch->reset_iter = i; 2000 } else { 2001 spdk_for_each_channel_continue(i, 0); 2002 } 2003 } 2004 2005 static void 2006 bdev_nvme_reset_create_qpairs_done(struct spdk_io_channel_iter *i, int status) 2007 { 2008 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 2009 2010 if (status == 0) { 2011 bdev_nvme_reset_complete(nvme_ctrlr, true); 2012 } else { 2013 /* Delete the added qpairs and quiesce ctrlr to make the states clean. */ 2014 spdk_for_each_channel(nvme_ctrlr, 2015 bdev_nvme_reset_destroy_qpair, 2016 NULL, 2017 bdev_nvme_reset_create_qpairs_failed); 2018 } 2019 } 2020 2021 static void 2022 bdev_nvme_reset_create_qpair(struct spdk_io_channel_iter *i) 2023 { 2024 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 2025 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 2026 int rc; 2027 2028 rc = bdev_nvme_create_qpair(ctrlr_ch->qpair); 2029 2030 spdk_for_each_channel_continue(i, rc); 2031 } 2032 2033 static int 2034 bdev_nvme_reconnect_ctrlr_poll(void *arg) 2035 { 2036 struct nvme_ctrlr *nvme_ctrlr = arg; 2037 int rc = -ETIMEDOUT; 2038 2039 if (!bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 2040 rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr); 2041 if (rc == -EAGAIN) { 2042 return SPDK_POLLER_BUSY; 2043 } 2044 } 2045 2046 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 2047 if (rc == 0) { 2048 /* Recreate all of the I/O queue pairs */ 2049 spdk_for_each_channel(nvme_ctrlr, 2050 bdev_nvme_reset_create_qpair, 2051 NULL, 2052 bdev_nvme_reset_create_qpairs_done); 2053 } else { 2054 bdev_nvme_reset_complete(nvme_ctrlr, false); 2055 } 2056 return SPDK_POLLER_BUSY; 2057 } 2058 2059 static void 2060 bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2061 { 2062 spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr); 2063 2064 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect, nvme_ctrlr->nbdev_ctrlr->name); 2065 assert(nvme_ctrlr->reset_detach_poller == NULL); 2066 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll, 2067 nvme_ctrlr, 0); 2068 } 2069 2070 static void 2071 bdev_nvme_reset_ctrlr(struct spdk_io_channel_iter *i, int status) 2072 { 2073 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 2074 2075 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reset, nvme_ctrlr->nbdev_ctrlr->name); 2076 assert(status == 0); 2077 2078 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2079 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2080 } else { 2081 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); 2082 } 2083 } 2084 2085 static void 2086 bdev_nvme_reset_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr) 2087 { 2088 spdk_for_each_channel(nvme_ctrlr, 2089 bdev_nvme_reset_destroy_qpair, 2090 NULL, 2091 bdev_nvme_reset_ctrlr); 2092 } 2093 2094 static void 2095 _bdev_nvme_reconnect(void *ctx) 2096 { 2097 struct nvme_ctrlr *nvme_ctrlr = ctx; 2098 2099 assert(nvme_ctrlr->resetting == true); 2100 assert(nvme_ctrlr->thread == spdk_get_thread()); 2101 2102 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2103 2104 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 2105 2106 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2107 } 2108 2109 static void 2110 _bdev_nvme_reset(void *ctx) 2111 { 2112 struct nvme_ctrlr *nvme_ctrlr = ctx; 2113 2114 assert(nvme_ctrlr->resetting == true); 2115 assert(nvme_ctrlr->thread == spdk_get_thread()); 2116 2117 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2118 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs); 2119 } else { 2120 bdev_nvme_reset_destroy_qpairs(nvme_ctrlr); 2121 } 2122 } 2123 2124 static int 2125 bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr) 2126 { 2127 spdk_msg_fn msg_fn; 2128 2129 pthread_mutex_lock(&nvme_ctrlr->mutex); 2130 if (nvme_ctrlr->destruct) { 2131 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2132 return -ENXIO; 2133 } 2134 2135 if (nvme_ctrlr->resetting) { 2136 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2137 SPDK_NOTICELOG("Unable to perform reset, already in progress.\n"); 2138 return -EBUSY; 2139 } 2140 2141 nvme_ctrlr->resetting = true; 2142 nvme_ctrlr->dont_retry = true; 2143 2144 if (nvme_ctrlr->reconnect_is_delayed) { 2145 SPDK_DEBUGLOG(bdev_nvme, "Reconnect is already scheduled.\n"); 2146 msg_fn = _bdev_nvme_reconnect; 2147 nvme_ctrlr->reconnect_is_delayed = false; 2148 } else { 2149 msg_fn = _bdev_nvme_reset; 2150 assert(nvme_ctrlr->reset_start_tsc == 0); 2151 } 2152 2153 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2154 2155 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2156 2157 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 2158 return 0; 2159 } 2160 2161 int 2162 bdev_nvme_reset_rpc(struct nvme_ctrlr *nvme_ctrlr, bdev_nvme_reset_cb cb_fn, void *cb_arg) 2163 { 2164 int rc; 2165 2166 rc = bdev_nvme_reset(nvme_ctrlr); 2167 if (rc == 0) { 2168 nvme_ctrlr->reset_cb_fn = cb_fn; 2169 nvme_ctrlr->reset_cb_arg = cb_arg; 2170 } 2171 return rc; 2172 } 2173 2174 static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio); 2175 2176 static void 2177 _bdev_nvme_reset_io_complete(struct spdk_io_channel_iter *i, int status) 2178 { 2179 struct nvme_bdev_io *bio = spdk_io_channel_iter_get_ctx(i); 2180 enum spdk_bdev_io_status io_status; 2181 2182 if (bio->cpl.cdw0 == 0) { 2183 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 2184 } else { 2185 io_status = SPDK_BDEV_IO_STATUS_FAILED; 2186 } 2187 2188 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL); 2189 } 2190 2191 static void 2192 bdev_nvme_abort_bdev_channel(struct spdk_io_channel_iter *i) 2193 { 2194 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 2195 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 2196 2197 bdev_nvme_abort_retry_ios(nbdev_ch); 2198 2199 spdk_for_each_channel_continue(i, 0); 2200 } 2201 2202 static void 2203 bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio) 2204 { 2205 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2206 struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt; 2207 2208 /* Abort all queued I/Os for retry. */ 2209 spdk_for_each_channel(nbdev, 2210 bdev_nvme_abort_bdev_channel, 2211 bio, 2212 _bdev_nvme_reset_io_complete); 2213 } 2214 2215 static void 2216 _bdev_nvme_reset_io_continue(void *ctx) 2217 { 2218 struct nvme_bdev_io *bio = ctx; 2219 struct nvme_io_path *prev_io_path, *next_io_path; 2220 int rc; 2221 2222 prev_io_path = bio->io_path; 2223 bio->io_path = NULL; 2224 2225 if (bio->cpl.cdw0 != 0) { 2226 goto complete; 2227 } 2228 2229 next_io_path = STAILQ_NEXT(prev_io_path, stailq); 2230 if (next_io_path == NULL) { 2231 goto complete; 2232 } 2233 2234 rc = _bdev_nvme_reset_io(next_io_path, bio); 2235 if (rc == 0) { 2236 return; 2237 } 2238 2239 bio->cpl.cdw0 = 1; 2240 2241 complete: 2242 bdev_nvme_reset_io_complete(bio); 2243 } 2244 2245 static void 2246 bdev_nvme_reset_io_continue(void *cb_arg, bool success) 2247 { 2248 struct nvme_bdev_io *bio = cb_arg; 2249 2250 bio->cpl.cdw0 = !success; 2251 2252 spdk_thread_send_msg(bio->orig_thread, _bdev_nvme_reset_io_continue, bio); 2253 } 2254 2255 static int 2256 _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio) 2257 { 2258 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 2259 struct nvme_ctrlr_channel *ctrlr_ch; 2260 struct spdk_bdev_io *bdev_io; 2261 int rc; 2262 2263 rc = bdev_nvme_reset(nvme_ctrlr); 2264 if (rc == 0) { 2265 assert(bio->io_path == NULL); 2266 bio->io_path = io_path; 2267 2268 assert(nvme_ctrlr->reset_cb_fn == NULL); 2269 assert(nvme_ctrlr->reset_cb_arg == NULL); 2270 nvme_ctrlr->reset_cb_fn = bdev_nvme_reset_io_continue; 2271 nvme_ctrlr->reset_cb_arg = bio; 2272 } else if (rc == -EBUSY) { 2273 ctrlr_ch = io_path->qpair->ctrlr_ch; 2274 assert(ctrlr_ch != NULL); 2275 /* 2276 * Reset call is queued only if it is from the app framework. This is on purpose so that 2277 * we don't interfere with the app framework reset strategy. i.e. we are deferring to the 2278 * upper level. If they are in the middle of a reset, we won't try to schedule another one. 2279 */ 2280 bdev_io = spdk_bdev_io_from_ctx(bio); 2281 TAILQ_INSERT_TAIL(&ctrlr_ch->pending_resets, bdev_io, module_link); 2282 rc = 0; 2283 } 2284 2285 return rc; 2286 } 2287 2288 static void 2289 bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio) 2290 { 2291 struct nvme_io_path *io_path; 2292 int rc; 2293 2294 bio->cpl.cdw0 = 0; 2295 bio->orig_thread = spdk_get_thread(); 2296 2297 /* Reset all nvme_ctrlrs of a bdev controller sequentially. */ 2298 io_path = STAILQ_FIRST(&nbdev_ch->io_path_list); 2299 assert(io_path != NULL); 2300 2301 rc = _bdev_nvme_reset_io(io_path, bio); 2302 if (rc != 0) { 2303 bio->cpl.cdw0 = 1; 2304 bdev_nvme_reset_io_complete(bio); 2305 } 2306 } 2307 2308 static int 2309 bdev_nvme_failover_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool remove) 2310 { 2311 if (nvme_ctrlr->destruct) { 2312 /* Don't bother resetting if the controller is in the process of being destructed. */ 2313 return -ENXIO; 2314 } 2315 2316 if (nvme_ctrlr->resetting) { 2317 SPDK_NOTICELOG("Unable to perform reset, already in progress.\n"); 2318 return -EBUSY; 2319 } 2320 2321 bdev_nvme_failover_trid(nvme_ctrlr, remove); 2322 2323 if (nvme_ctrlr->reconnect_is_delayed) { 2324 SPDK_NOTICELOG("Reconnect is already scheduled.\n"); 2325 2326 /* We rely on the next reconnect for the failover. */ 2327 return -EALREADY; 2328 } 2329 2330 nvme_ctrlr->resetting = true; 2331 2332 assert(nvme_ctrlr->reset_start_tsc == 0); 2333 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2334 2335 return 0; 2336 } 2337 2338 static int 2339 bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove) 2340 { 2341 int rc; 2342 2343 pthread_mutex_lock(&nvme_ctrlr->mutex); 2344 rc = bdev_nvme_failover_unsafe(nvme_ctrlr, remove); 2345 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2346 2347 if (rc == 0) { 2348 spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr); 2349 } else if (rc == -EALREADY) { 2350 rc = 0; 2351 } 2352 2353 return rc; 2354 } 2355 2356 static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, 2357 uint64_t num_blocks); 2358 2359 static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, 2360 uint64_t num_blocks); 2361 2362 static int bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, 2363 uint64_t src_offset_blocks, 2364 uint64_t num_blocks); 2365 2366 static void 2367 bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 2368 bool success) 2369 { 2370 struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 2371 struct spdk_bdev *bdev = bdev_io->bdev; 2372 int ret; 2373 2374 if (!success) { 2375 ret = -EINVAL; 2376 goto exit; 2377 } 2378 2379 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 2380 ret = -ENXIO; 2381 goto exit; 2382 } 2383 2384 ret = bdev_nvme_readv(bio, 2385 bdev_io->u.bdev.iovs, 2386 bdev_io->u.bdev.iovcnt, 2387 bdev_io->u.bdev.md_buf, 2388 bdev_io->u.bdev.num_blocks, 2389 bdev_io->u.bdev.offset_blocks, 2390 bdev->dif_check_flags, 2391 bdev_io->u.bdev.memory_domain, 2392 bdev_io->u.bdev.memory_domain_ctx); 2393 2394 exit: 2395 if (spdk_unlikely(ret != 0)) { 2396 bdev_nvme_io_complete(bio, ret); 2397 } 2398 } 2399 2400 static inline void 2401 _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 2402 { 2403 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 2404 struct spdk_bdev *bdev = bdev_io->bdev; 2405 struct nvme_bdev_io *nbdev_io_to_abort; 2406 int rc = 0; 2407 2408 switch (bdev_io->type) { 2409 case SPDK_BDEV_IO_TYPE_READ: 2410 if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) { 2411 rc = bdev_nvme_readv(nbdev_io, 2412 bdev_io->u.bdev.iovs, 2413 bdev_io->u.bdev.iovcnt, 2414 bdev_io->u.bdev.md_buf, 2415 bdev_io->u.bdev.num_blocks, 2416 bdev_io->u.bdev.offset_blocks, 2417 bdev->dif_check_flags, 2418 bdev_io->u.bdev.memory_domain, 2419 bdev_io->u.bdev.memory_domain_ctx); 2420 } else { 2421 spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb, 2422 bdev_io->u.bdev.num_blocks * bdev->blocklen); 2423 rc = 0; 2424 } 2425 break; 2426 case SPDK_BDEV_IO_TYPE_WRITE: 2427 rc = bdev_nvme_writev(nbdev_io, 2428 bdev_io->u.bdev.iovs, 2429 bdev_io->u.bdev.iovcnt, 2430 bdev_io->u.bdev.md_buf, 2431 bdev_io->u.bdev.num_blocks, 2432 bdev_io->u.bdev.offset_blocks, 2433 bdev->dif_check_flags, 2434 bdev_io->u.bdev.memory_domain, 2435 bdev_io->u.bdev.memory_domain_ctx); 2436 break; 2437 case SPDK_BDEV_IO_TYPE_COMPARE: 2438 rc = bdev_nvme_comparev(nbdev_io, 2439 bdev_io->u.bdev.iovs, 2440 bdev_io->u.bdev.iovcnt, 2441 bdev_io->u.bdev.md_buf, 2442 bdev_io->u.bdev.num_blocks, 2443 bdev_io->u.bdev.offset_blocks, 2444 bdev->dif_check_flags); 2445 break; 2446 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 2447 rc = bdev_nvme_comparev_and_writev(nbdev_io, 2448 bdev_io->u.bdev.iovs, 2449 bdev_io->u.bdev.iovcnt, 2450 bdev_io->u.bdev.fused_iovs, 2451 bdev_io->u.bdev.fused_iovcnt, 2452 bdev_io->u.bdev.md_buf, 2453 bdev_io->u.bdev.num_blocks, 2454 bdev_io->u.bdev.offset_blocks, 2455 bdev->dif_check_flags); 2456 break; 2457 case SPDK_BDEV_IO_TYPE_UNMAP: 2458 rc = bdev_nvme_unmap(nbdev_io, 2459 bdev_io->u.bdev.offset_blocks, 2460 bdev_io->u.bdev.num_blocks); 2461 break; 2462 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 2463 rc = bdev_nvme_write_zeroes(nbdev_io, 2464 bdev_io->u.bdev.offset_blocks, 2465 bdev_io->u.bdev.num_blocks); 2466 break; 2467 case SPDK_BDEV_IO_TYPE_RESET: 2468 nbdev_io->io_path = NULL; 2469 bdev_nvme_reset_io(nbdev_ch, nbdev_io); 2470 break; 2471 case SPDK_BDEV_IO_TYPE_FLUSH: 2472 bdev_nvme_io_complete(nbdev_io, 0); 2473 break; 2474 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 2475 rc = bdev_nvme_zone_appendv(nbdev_io, 2476 bdev_io->u.bdev.iovs, 2477 bdev_io->u.bdev.iovcnt, 2478 bdev_io->u.bdev.md_buf, 2479 bdev_io->u.bdev.num_blocks, 2480 bdev_io->u.bdev.offset_blocks, 2481 bdev->dif_check_flags); 2482 break; 2483 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 2484 rc = bdev_nvme_get_zone_info(nbdev_io, 2485 bdev_io->u.zone_mgmt.zone_id, 2486 bdev_io->u.zone_mgmt.num_zones, 2487 bdev_io->u.zone_mgmt.buf); 2488 break; 2489 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 2490 rc = bdev_nvme_zone_management(nbdev_io, 2491 bdev_io->u.zone_mgmt.zone_id, 2492 bdev_io->u.zone_mgmt.zone_action); 2493 break; 2494 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 2495 nbdev_io->io_path = NULL; 2496 bdev_nvme_admin_passthru(nbdev_ch, 2497 nbdev_io, 2498 &bdev_io->u.nvme_passthru.cmd, 2499 bdev_io->u.nvme_passthru.buf, 2500 bdev_io->u.nvme_passthru.nbytes); 2501 break; 2502 case SPDK_BDEV_IO_TYPE_NVME_IO: 2503 rc = bdev_nvme_io_passthru(nbdev_io, 2504 &bdev_io->u.nvme_passthru.cmd, 2505 bdev_io->u.nvme_passthru.buf, 2506 bdev_io->u.nvme_passthru.nbytes); 2507 break; 2508 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 2509 rc = bdev_nvme_io_passthru_md(nbdev_io, 2510 &bdev_io->u.nvme_passthru.cmd, 2511 bdev_io->u.nvme_passthru.buf, 2512 bdev_io->u.nvme_passthru.nbytes, 2513 bdev_io->u.nvme_passthru.md_buf, 2514 bdev_io->u.nvme_passthru.md_len); 2515 break; 2516 case SPDK_BDEV_IO_TYPE_ABORT: 2517 nbdev_io->io_path = NULL; 2518 nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx; 2519 bdev_nvme_abort(nbdev_ch, 2520 nbdev_io, 2521 nbdev_io_to_abort); 2522 break; 2523 case SPDK_BDEV_IO_TYPE_COPY: 2524 rc = bdev_nvme_copy(nbdev_io, 2525 bdev_io->u.bdev.offset_blocks, 2526 bdev_io->u.bdev.copy.src_offset_blocks, 2527 bdev_io->u.bdev.num_blocks); 2528 break; 2529 default: 2530 rc = -EINVAL; 2531 break; 2532 } 2533 2534 if (spdk_unlikely(rc != 0)) { 2535 bdev_nvme_io_complete(nbdev_io, rc); 2536 } 2537 } 2538 2539 static void 2540 bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 2541 { 2542 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 2543 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 2544 2545 if (spdk_likely(nbdev_io->submit_tsc == 0)) { 2546 nbdev_io->submit_tsc = spdk_bdev_io_get_submit_tsc(bdev_io); 2547 } else { 2548 /* There are cases where submit_tsc != 0, i.e. retry I/O. 2549 * We need to update submit_tsc here. 2550 */ 2551 nbdev_io->submit_tsc = spdk_get_ticks(); 2552 } 2553 2554 spdk_trace_record(TRACE_BDEV_NVME_IO_START, 0, 0, (uintptr_t)nbdev_io, (uintptr_t)bdev_io); 2555 nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch); 2556 if (spdk_unlikely(!nbdev_io->io_path)) { 2557 if (!bdev_nvme_io_type_is_admin(bdev_io->type)) { 2558 bdev_nvme_io_complete(nbdev_io, -ENXIO); 2559 return; 2560 } 2561 2562 /* Admin commands do not use the optimal I/O path. 2563 * Simply fall through even if it is not found. 2564 */ 2565 } 2566 2567 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 2568 } 2569 2570 static bool 2571 bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 2572 { 2573 struct nvme_bdev *nbdev = ctx; 2574 struct nvme_ns *nvme_ns; 2575 struct spdk_nvme_ns *ns; 2576 struct spdk_nvme_ctrlr *ctrlr; 2577 const struct spdk_nvme_ctrlr_data *cdata; 2578 2579 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 2580 assert(nvme_ns != NULL); 2581 ns = nvme_ns->ns; 2582 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 2583 2584 switch (io_type) { 2585 case SPDK_BDEV_IO_TYPE_READ: 2586 case SPDK_BDEV_IO_TYPE_WRITE: 2587 case SPDK_BDEV_IO_TYPE_RESET: 2588 case SPDK_BDEV_IO_TYPE_FLUSH: 2589 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 2590 case SPDK_BDEV_IO_TYPE_NVME_IO: 2591 case SPDK_BDEV_IO_TYPE_ABORT: 2592 return true; 2593 2594 case SPDK_BDEV_IO_TYPE_COMPARE: 2595 return spdk_nvme_ns_supports_compare(ns); 2596 2597 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 2598 return spdk_nvme_ns_get_md_size(ns) ? true : false; 2599 2600 case SPDK_BDEV_IO_TYPE_UNMAP: 2601 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2602 return cdata->oncs.dsm; 2603 2604 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 2605 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2606 return cdata->oncs.write_zeroes; 2607 2608 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 2609 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 2610 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) { 2611 return true; 2612 } 2613 return false; 2614 2615 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 2616 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 2617 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS; 2618 2619 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 2620 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS && 2621 spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED; 2622 2623 case SPDK_BDEV_IO_TYPE_COPY: 2624 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2625 return cdata->oncs.copy; 2626 2627 default: 2628 return false; 2629 } 2630 } 2631 2632 static int 2633 nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch) 2634 { 2635 struct nvme_qpair *nvme_qpair; 2636 struct spdk_io_channel *pg_ch; 2637 int rc; 2638 2639 nvme_qpair = calloc(1, sizeof(*nvme_qpair)); 2640 if (!nvme_qpair) { 2641 SPDK_ERRLOG("Failed to alloc nvme_qpair.\n"); 2642 return -1; 2643 } 2644 2645 TAILQ_INIT(&nvme_qpair->io_path_list); 2646 2647 nvme_qpair->ctrlr = nvme_ctrlr; 2648 nvme_qpair->ctrlr_ch = ctrlr_ch; 2649 2650 pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs); 2651 if (!pg_ch) { 2652 free(nvme_qpair); 2653 return -1; 2654 } 2655 2656 nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch); 2657 2658 #ifdef SPDK_CONFIG_VTUNE 2659 nvme_qpair->group->collect_spin_stat = true; 2660 #else 2661 nvme_qpair->group->collect_spin_stat = false; 2662 #endif 2663 2664 rc = bdev_nvme_create_qpair(nvme_qpair); 2665 if (rc != 0) { 2666 /* nvme_ctrlr can't create IO qpair if connection is down. 2667 * 2668 * If reconnect_delay_sec is non-zero, creating IO qpair is retried 2669 * after reconnect_delay_sec seconds. If bdev_retry_count is non-zero, 2670 * submitted IO will be queued until IO qpair is successfully created. 2671 * 2672 * Hence, if both are satisfied, ignore the failure. 2673 */ 2674 if (nvme_ctrlr->opts.reconnect_delay_sec == 0 || g_opts.bdev_retry_count == 0) { 2675 spdk_put_io_channel(pg_ch); 2676 free(nvme_qpair); 2677 return rc; 2678 } 2679 } 2680 2681 TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 2682 2683 ctrlr_ch->qpair = nvme_qpair; 2684 2685 pthread_mutex_lock(&nvme_qpair->ctrlr->mutex); 2686 nvme_qpair->ctrlr->ref++; 2687 pthread_mutex_unlock(&nvme_qpair->ctrlr->mutex); 2688 2689 return 0; 2690 } 2691 2692 static int 2693 bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) 2694 { 2695 struct nvme_ctrlr *nvme_ctrlr = io_device; 2696 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 2697 2698 TAILQ_INIT(&ctrlr_ch->pending_resets); 2699 2700 return nvme_qpair_create(nvme_ctrlr, ctrlr_ch); 2701 } 2702 2703 static void 2704 nvme_qpair_delete(struct nvme_qpair *nvme_qpair) 2705 { 2706 struct nvme_io_path *io_path, *next; 2707 2708 assert(nvme_qpair->group != NULL); 2709 2710 TAILQ_FOREACH_SAFE(io_path, &nvme_qpair->io_path_list, tailq, next) { 2711 TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq); 2712 nvme_io_path_free(io_path); 2713 } 2714 2715 TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 2716 2717 spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group)); 2718 2719 nvme_ctrlr_release(nvme_qpair->ctrlr); 2720 2721 free(nvme_qpair); 2722 } 2723 2724 static void 2725 bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf) 2726 { 2727 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 2728 struct nvme_qpair *nvme_qpair; 2729 2730 nvme_qpair = ctrlr_ch->qpair; 2731 assert(nvme_qpair != NULL); 2732 2733 _bdev_nvme_clear_io_path_cache(nvme_qpair); 2734 2735 if (nvme_qpair->qpair != NULL) { 2736 if (ctrlr_ch->reset_iter == NULL) { 2737 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 2738 } else { 2739 /* Skip current ctrlr_channel in a full reset sequence because 2740 * it is being deleted now. The qpair is already being disconnected. 2741 * We do not have to restart disconnecting it. 2742 */ 2743 spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 2744 } 2745 2746 /* We cannot release a reference to the poll group now. 2747 * The qpair may be disconnected asynchronously later. 2748 * We need to poll it until it is actually disconnected. 2749 * Just detach the qpair from the deleting ctrlr_channel. 2750 */ 2751 nvme_qpair->ctrlr_ch = NULL; 2752 } else { 2753 assert(ctrlr_ch->reset_iter == NULL); 2754 2755 nvme_qpair_delete(nvme_qpair); 2756 } 2757 } 2758 2759 static void 2760 bdev_nvme_submit_accel_crc32c(void *ctx, uint32_t *dst, struct iovec *iov, 2761 uint32_t iov_cnt, uint32_t seed, 2762 spdk_nvme_accel_completion_cb cb_fn, void *cb_arg) 2763 { 2764 struct nvme_poll_group *group = ctx; 2765 int rc; 2766 2767 assert(cb_fn != NULL); 2768 2769 if (spdk_unlikely(!group->accel_channel)) { 2770 group->accel_channel = spdk_accel_get_io_channel(); 2771 if (!group->accel_channel) { 2772 cb_fn(cb_arg, -ENOMEM); 2773 SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n", 2774 group); 2775 return; 2776 } 2777 } 2778 2779 rc = spdk_accel_submit_crc32cv(group->accel_channel, dst, iov, iov_cnt, seed, cb_fn, cb_arg); 2780 if (rc) { 2781 /* For the two cases, spdk_accel_submit_crc32cv does not call the user's cb_fn */ 2782 if (rc == -ENOMEM || rc == -EINVAL) { 2783 cb_fn(cb_arg, rc); 2784 } 2785 SPDK_ERRLOG("Cannot complete the accelerated crc32c operation with iov=%p\n", iov); 2786 } 2787 } 2788 2789 static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = { 2790 .table_size = sizeof(struct spdk_nvme_accel_fn_table), 2791 .submit_accel_crc32c = bdev_nvme_submit_accel_crc32c, 2792 }; 2793 2794 static int 2795 bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf) 2796 { 2797 struct nvme_poll_group *group = ctx_buf; 2798 2799 TAILQ_INIT(&group->qpair_list); 2800 2801 group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table); 2802 if (group->group == NULL) { 2803 return -1; 2804 } 2805 2806 group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us); 2807 2808 if (group->poller == NULL) { 2809 spdk_nvme_poll_group_destroy(group->group); 2810 return -1; 2811 } 2812 2813 return 0; 2814 } 2815 2816 static void 2817 bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf) 2818 { 2819 struct nvme_poll_group *group = ctx_buf; 2820 2821 assert(TAILQ_EMPTY(&group->qpair_list)); 2822 2823 if (group->accel_channel) { 2824 spdk_put_io_channel(group->accel_channel); 2825 } 2826 2827 spdk_poller_unregister(&group->poller); 2828 if (spdk_nvme_poll_group_destroy(group->group)) { 2829 SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n"); 2830 assert(false); 2831 } 2832 } 2833 2834 static struct spdk_io_channel * 2835 bdev_nvme_get_io_channel(void *ctx) 2836 { 2837 struct nvme_bdev *nvme_bdev = ctx; 2838 2839 return spdk_get_io_channel(nvme_bdev); 2840 } 2841 2842 static void * 2843 bdev_nvme_get_module_ctx(void *ctx) 2844 { 2845 struct nvme_bdev *nvme_bdev = ctx; 2846 struct nvme_ns *nvme_ns; 2847 2848 if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) { 2849 return NULL; 2850 } 2851 2852 nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list); 2853 if (!nvme_ns) { 2854 return NULL; 2855 } 2856 2857 return nvme_ns->ns; 2858 } 2859 2860 static const char * 2861 _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state) 2862 { 2863 switch (ana_state) { 2864 case SPDK_NVME_ANA_OPTIMIZED_STATE: 2865 return "optimized"; 2866 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 2867 return "non_optimized"; 2868 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 2869 return "inaccessible"; 2870 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 2871 return "persistent_loss"; 2872 case SPDK_NVME_ANA_CHANGE_STATE: 2873 return "change"; 2874 default: 2875 return NULL; 2876 } 2877 } 2878 2879 static int 2880 bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size) 2881 { 2882 struct spdk_memory_domain **_domains = NULL; 2883 struct nvme_bdev *nbdev = ctx; 2884 struct nvme_ns *nvme_ns; 2885 int i = 0, _array_size = array_size; 2886 int rc = 0; 2887 2888 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 2889 if (domains && array_size >= i) { 2890 _domains = &domains[i]; 2891 } else { 2892 _domains = NULL; 2893 } 2894 rc = spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, _domains, _array_size); 2895 if (rc > 0) { 2896 i += rc; 2897 if (_array_size >= rc) { 2898 _array_size -= rc; 2899 } else { 2900 _array_size = 0; 2901 } 2902 } else if (rc < 0) { 2903 return rc; 2904 } 2905 } 2906 2907 return i; 2908 } 2909 2910 static const char * 2911 nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr) 2912 { 2913 if (nvme_ctrlr->destruct) { 2914 return "deleting"; 2915 } else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 2916 return "failed"; 2917 } else if (nvme_ctrlr->resetting) { 2918 return "resetting"; 2919 } else if (nvme_ctrlr->reconnect_is_delayed > 0) { 2920 return "reconnect_is_delayed"; 2921 } else { 2922 return "enabled"; 2923 } 2924 } 2925 2926 void 2927 nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr) 2928 { 2929 struct spdk_nvme_transport_id *trid; 2930 const struct spdk_nvme_ctrlr_opts *opts; 2931 const struct spdk_nvme_ctrlr_data *cdata; 2932 2933 spdk_json_write_object_begin(w); 2934 2935 spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr)); 2936 2937 #ifdef SPDK_CONFIG_NVME_CUSE 2938 size_t cuse_name_size = 128; 2939 char cuse_name[cuse_name_size]; 2940 2941 int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size); 2942 if (rc == 0) { 2943 spdk_json_write_named_string(w, "cuse_device", cuse_name); 2944 } 2945 #endif 2946 trid = &nvme_ctrlr->active_path_id->trid; 2947 spdk_json_write_named_object_begin(w, "trid"); 2948 nvme_bdev_dump_trid_json(trid, w); 2949 spdk_json_write_object_end(w); 2950 2951 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 2952 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 2953 2954 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 2955 spdk_json_write_named_object_begin(w, "host"); 2956 spdk_json_write_named_string(w, "nqn", opts->hostnqn); 2957 spdk_json_write_named_string(w, "addr", opts->src_addr); 2958 spdk_json_write_named_string(w, "svcid", opts->src_svcid); 2959 spdk_json_write_object_end(w); 2960 2961 spdk_json_write_object_end(w); 2962 } 2963 2964 static void 2965 nvme_namespace_info_json(struct spdk_json_write_ctx *w, 2966 struct nvme_ns *nvme_ns) 2967 { 2968 struct spdk_nvme_ns *ns; 2969 struct spdk_nvme_ctrlr *ctrlr; 2970 const struct spdk_nvme_ctrlr_data *cdata; 2971 const struct spdk_nvme_transport_id *trid; 2972 union spdk_nvme_vs_register vs; 2973 const struct spdk_nvme_ns_data *nsdata; 2974 char buf[128]; 2975 2976 ns = nvme_ns->ns; 2977 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 2978 2979 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 2980 trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); 2981 vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr); 2982 2983 spdk_json_write_object_begin(w); 2984 2985 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 2986 spdk_json_write_named_string(w, "pci_address", trid->traddr); 2987 } 2988 2989 spdk_json_write_named_object_begin(w, "trid"); 2990 2991 nvme_bdev_dump_trid_json(trid, w); 2992 2993 spdk_json_write_object_end(w); 2994 2995 #ifdef SPDK_CONFIG_NVME_CUSE 2996 size_t cuse_name_size = 128; 2997 char cuse_name[cuse_name_size]; 2998 2999 int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns), 3000 cuse_name, &cuse_name_size); 3001 if (rc == 0) { 3002 spdk_json_write_named_string(w, "cuse_device", cuse_name); 3003 } 3004 #endif 3005 3006 spdk_json_write_named_object_begin(w, "ctrlr_data"); 3007 3008 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 3009 3010 spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid); 3011 3012 snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn); 3013 spdk_str_trim(buf); 3014 spdk_json_write_named_string(w, "model_number", buf); 3015 3016 snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn); 3017 spdk_str_trim(buf); 3018 spdk_json_write_named_string(w, "serial_number", buf); 3019 3020 snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr); 3021 spdk_str_trim(buf); 3022 spdk_json_write_named_string(w, "firmware_revision", buf); 3023 3024 if (cdata->subnqn[0] != '\0') { 3025 spdk_json_write_named_string(w, "subnqn", cdata->subnqn); 3026 } 3027 3028 spdk_json_write_named_object_begin(w, "oacs"); 3029 3030 spdk_json_write_named_uint32(w, "security", cdata->oacs.security); 3031 spdk_json_write_named_uint32(w, "format", cdata->oacs.format); 3032 spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware); 3033 spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage); 3034 3035 spdk_json_write_object_end(w); 3036 3037 spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr); 3038 spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting); 3039 3040 spdk_json_write_object_end(w); 3041 3042 spdk_json_write_named_object_begin(w, "vs"); 3043 3044 spdk_json_write_name(w, "nvme_version"); 3045 if (vs.bits.ter) { 3046 spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter); 3047 } else { 3048 spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr); 3049 } 3050 3051 spdk_json_write_object_end(w); 3052 3053 nsdata = spdk_nvme_ns_get_data(ns); 3054 3055 spdk_json_write_named_object_begin(w, "ns_data"); 3056 3057 spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns)); 3058 3059 if (cdata->cmic.ana_reporting) { 3060 spdk_json_write_named_string(w, "ana_state", 3061 _nvme_ana_state_str(nvme_ns->ana_state)); 3062 } 3063 3064 spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share); 3065 3066 spdk_json_write_object_end(w); 3067 3068 if (cdata->oacs.security) { 3069 spdk_json_write_named_object_begin(w, "security"); 3070 3071 spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal); 3072 3073 spdk_json_write_object_end(w); 3074 } 3075 3076 spdk_json_write_object_end(w); 3077 } 3078 3079 static const char * 3080 nvme_bdev_get_mp_policy_str(struct nvme_bdev *nbdev) 3081 { 3082 switch (nbdev->mp_policy) { 3083 case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE: 3084 return "active_passive"; 3085 case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE: 3086 return "active_active"; 3087 default: 3088 assert(false); 3089 return "invalid"; 3090 } 3091 } 3092 3093 static int 3094 bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 3095 { 3096 struct nvme_bdev *nvme_bdev = ctx; 3097 struct nvme_ns *nvme_ns; 3098 3099 pthread_mutex_lock(&nvme_bdev->mutex); 3100 spdk_json_write_named_array_begin(w, "nvme"); 3101 TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) { 3102 nvme_namespace_info_json(w, nvme_ns); 3103 } 3104 spdk_json_write_array_end(w); 3105 spdk_json_write_named_string(w, "mp_policy", nvme_bdev_get_mp_policy_str(nvme_bdev)); 3106 pthread_mutex_unlock(&nvme_bdev->mutex); 3107 3108 return 0; 3109 } 3110 3111 static void 3112 bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 3113 { 3114 /* No config per bdev needed */ 3115 } 3116 3117 static uint64_t 3118 bdev_nvme_get_spin_time(struct spdk_io_channel *ch) 3119 { 3120 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 3121 struct nvme_io_path *io_path; 3122 struct nvme_poll_group *group; 3123 uint64_t spin_time = 0; 3124 3125 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 3126 group = io_path->qpair->group; 3127 3128 if (!group || !group->collect_spin_stat) { 3129 continue; 3130 } 3131 3132 if (group->end_ticks != 0) { 3133 group->spin_ticks += (group->end_ticks - group->start_ticks); 3134 group->end_ticks = 0; 3135 } 3136 3137 spin_time += group->spin_ticks; 3138 group->start_ticks = 0; 3139 group->spin_ticks = 0; 3140 } 3141 3142 return (spin_time * 1000000ULL) / spdk_get_ticks_hz(); 3143 } 3144 3145 static void 3146 bdev_nvme_reset_device_stat(void *ctx) 3147 { 3148 struct nvme_bdev *nbdev = ctx; 3149 3150 if (nbdev->err_stat != NULL) { 3151 memset(nbdev->err_stat, 0, sizeof(struct nvme_error_stat)); 3152 } 3153 } 3154 3155 /* JSON string should be lowercases and underscore delimited string. */ 3156 static void 3157 bdev_nvme_format_nvme_status(char *dst, const char *src) 3158 { 3159 char tmp[256]; 3160 3161 spdk_strcpy_replace(dst, 256, src, " - ", "_"); 3162 spdk_strcpy_replace(tmp, 256, dst, "-", "_"); 3163 spdk_strcpy_replace(dst, 256, tmp, " ", "_"); 3164 spdk_strlwr(dst); 3165 } 3166 3167 static void 3168 bdev_nvme_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w) 3169 { 3170 struct nvme_bdev *nbdev = ctx; 3171 struct spdk_nvme_status status = {}; 3172 uint16_t sct, sc; 3173 char status_json[256]; 3174 const char *status_str; 3175 3176 if (nbdev->err_stat == NULL) { 3177 return; 3178 } 3179 3180 spdk_json_write_named_object_begin(w, "nvme_error"); 3181 3182 spdk_json_write_named_object_begin(w, "status_type"); 3183 for (sct = 0; sct < 8; sct++) { 3184 if (nbdev->err_stat->status_type[sct] == 0) { 3185 continue; 3186 } 3187 status.sct = sct; 3188 3189 status_str = spdk_nvme_cpl_get_status_type_string(&status); 3190 assert(status_str != NULL); 3191 bdev_nvme_format_nvme_status(status_json, status_str); 3192 3193 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status_type[sct]); 3194 } 3195 spdk_json_write_object_end(w); 3196 3197 spdk_json_write_named_object_begin(w, "status_code"); 3198 for (sct = 0; sct < 4; sct++) { 3199 status.sct = sct; 3200 for (sc = 0; sc < 256; sc++) { 3201 if (nbdev->err_stat->status[sct][sc] == 0) { 3202 continue; 3203 } 3204 status.sc = sc; 3205 3206 status_str = spdk_nvme_cpl_get_status_string(&status); 3207 assert(status_str != NULL); 3208 bdev_nvme_format_nvme_status(status_json, status_str); 3209 3210 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status[sct][sc]); 3211 } 3212 } 3213 spdk_json_write_object_end(w); 3214 3215 spdk_json_write_object_end(w); 3216 } 3217 3218 static const struct spdk_bdev_fn_table nvmelib_fn_table = { 3219 .destruct = bdev_nvme_destruct, 3220 .submit_request = bdev_nvme_submit_request, 3221 .io_type_supported = bdev_nvme_io_type_supported, 3222 .get_io_channel = bdev_nvme_get_io_channel, 3223 .dump_info_json = bdev_nvme_dump_info_json, 3224 .write_config_json = bdev_nvme_write_config_json, 3225 .get_spin_time = bdev_nvme_get_spin_time, 3226 .get_module_ctx = bdev_nvme_get_module_ctx, 3227 .get_memory_domains = bdev_nvme_get_memory_domains, 3228 .reset_device_stat = bdev_nvme_reset_device_stat, 3229 .dump_device_stat_json = bdev_nvme_dump_device_stat_json, 3230 }; 3231 3232 typedef int (*bdev_nvme_parse_ana_log_page_cb)( 3233 const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg); 3234 3235 static int 3236 bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 3237 bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg) 3238 { 3239 struct spdk_nvme_ana_group_descriptor *copied_desc; 3240 uint8_t *orig_desc; 3241 uint32_t i, desc_size, copy_len; 3242 int rc = 0; 3243 3244 if (nvme_ctrlr->ana_log_page == NULL) { 3245 return -EINVAL; 3246 } 3247 3248 copied_desc = nvme_ctrlr->copied_ana_desc; 3249 3250 orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page); 3251 copy_len = nvme_ctrlr->max_ana_log_page_size - sizeof(struct spdk_nvme_ana_page); 3252 3253 for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) { 3254 memcpy(copied_desc, orig_desc, copy_len); 3255 3256 rc = cb_fn(copied_desc, cb_arg); 3257 if (rc != 0) { 3258 break; 3259 } 3260 3261 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) + 3262 copied_desc->num_of_nsid * sizeof(uint32_t); 3263 orig_desc += desc_size; 3264 copy_len -= desc_size; 3265 } 3266 3267 return rc; 3268 } 3269 3270 static int 3271 nvme_ns_ana_transition_timedout(void *ctx) 3272 { 3273 struct nvme_ns *nvme_ns = ctx; 3274 3275 spdk_poller_unregister(&nvme_ns->anatt_timer); 3276 nvme_ns->ana_transition_timedout = true; 3277 3278 return SPDK_POLLER_BUSY; 3279 } 3280 3281 static void 3282 _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns, 3283 const struct spdk_nvme_ana_group_descriptor *desc) 3284 { 3285 const struct spdk_nvme_ctrlr_data *cdata; 3286 3287 nvme_ns->ana_group_id = desc->ana_group_id; 3288 nvme_ns->ana_state = desc->ana_state; 3289 nvme_ns->ana_state_updating = false; 3290 3291 switch (nvme_ns->ana_state) { 3292 case SPDK_NVME_ANA_OPTIMIZED_STATE: 3293 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 3294 nvme_ns->ana_transition_timedout = false; 3295 spdk_poller_unregister(&nvme_ns->anatt_timer); 3296 break; 3297 3298 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3299 case SPDK_NVME_ANA_CHANGE_STATE: 3300 if (nvme_ns->anatt_timer != NULL) { 3301 break; 3302 } 3303 3304 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 3305 nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout, 3306 nvme_ns, 3307 cdata->anatt * SPDK_SEC_TO_USEC); 3308 break; 3309 default: 3310 break; 3311 } 3312 } 3313 3314 static int 3315 nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg) 3316 { 3317 struct nvme_ns *nvme_ns = cb_arg; 3318 uint32_t i; 3319 3320 for (i = 0; i < desc->num_of_nsid; i++) { 3321 if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) { 3322 continue; 3323 } 3324 3325 _nvme_ns_set_ana_state(nvme_ns, desc); 3326 return 1; 3327 } 3328 3329 return 0; 3330 } 3331 3332 static struct spdk_uuid 3333 nvme_generate_uuid(const char *sn, uint32_t nsid) 3334 { 3335 struct spdk_uuid new_uuid, namespace_uuid; 3336 char merged_str[SPDK_NVME_CTRLR_SN_LEN + NSID_STR_LEN + 1] = {'\0'}; 3337 /* This namespace UUID was generated using uuid_generate() method. */ 3338 const char *namespace_str = {"edaed2de-24bc-4b07-b559-f47ecbe730fd"}; 3339 int size; 3340 3341 assert(strlen(sn) <= SPDK_NVME_CTRLR_SN_LEN); 3342 3343 memset(&new_uuid, 0, sizeof(new_uuid)); 3344 memset(&namespace_uuid, 0, sizeof(namespace_uuid)); 3345 3346 size = snprintf(merged_str, sizeof(merged_str), "%s%"PRIu32, sn, nsid); 3347 assert(size > 0 && (unsigned long)size < sizeof(merged_str)); 3348 3349 spdk_uuid_parse(&namespace_uuid, namespace_str); 3350 3351 spdk_uuid_generate_sha1(&new_uuid, &namespace_uuid, merged_str, size); 3352 3353 return new_uuid; 3354 } 3355 3356 static int 3357 nvme_disk_create(struct spdk_bdev *disk, const char *base_name, 3358 struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns, 3359 uint32_t prchk_flags, void *ctx) 3360 { 3361 const struct spdk_uuid *uuid; 3362 const uint8_t *nguid; 3363 const struct spdk_nvme_ctrlr_data *cdata; 3364 const struct spdk_nvme_ns_data *nsdata; 3365 const struct spdk_nvme_ctrlr_opts *opts; 3366 enum spdk_nvme_csi csi; 3367 uint32_t atomic_bs, phys_bs, bs; 3368 char sn_tmp[SPDK_NVME_CTRLR_SN_LEN + 1] = {'\0'}; 3369 3370 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3371 csi = spdk_nvme_ns_get_csi(ns); 3372 opts = spdk_nvme_ctrlr_get_opts(ctrlr); 3373 3374 switch (csi) { 3375 case SPDK_NVME_CSI_NVM: 3376 disk->product_name = "NVMe disk"; 3377 break; 3378 case SPDK_NVME_CSI_ZNS: 3379 disk->product_name = "NVMe ZNS disk"; 3380 disk->zoned = true; 3381 disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 3382 disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) / 3383 spdk_nvme_ns_get_extended_sector_size(ns); 3384 disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns); 3385 disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns); 3386 break; 3387 default: 3388 SPDK_ERRLOG("unsupported CSI: %u\n", csi); 3389 return -ENOTSUP; 3390 } 3391 3392 disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns)); 3393 if (!disk->name) { 3394 return -ENOMEM; 3395 } 3396 3397 disk->write_cache = 0; 3398 if (cdata->vwc.present) { 3399 /* Enable if the Volatile Write Cache exists */ 3400 disk->write_cache = 1; 3401 } 3402 if (cdata->oncs.write_zeroes) { 3403 disk->max_write_zeroes = UINT16_MAX + 1; 3404 } 3405 disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns); 3406 disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns); 3407 disk->max_segment_size = spdk_nvme_ctrlr_get_max_xfer_size(ctrlr); 3408 /* NVMe driver will split one request into multiple requests 3409 * based on MDTS and stripe boundary, the bdev layer will use 3410 * max_segment_size and max_num_segments to split one big IO 3411 * into multiple requests, then small request can't run out 3412 * of NVMe internal requests data structure. 3413 */ 3414 if (opts && opts->io_queue_requests) { 3415 disk->max_num_segments = opts->io_queue_requests / 2; 3416 } 3417 disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns); 3418 3419 nguid = spdk_nvme_ns_get_nguid(ns); 3420 if (!nguid) { 3421 uuid = spdk_nvme_ns_get_uuid(ns); 3422 if (uuid) { 3423 disk->uuid = *uuid; 3424 } else if (g_opts.generate_uuids) { 3425 spdk_strcpy_pad(sn_tmp, cdata->sn, SPDK_NVME_CTRLR_SN_LEN, '\0'); 3426 disk->uuid = nvme_generate_uuid(sn_tmp, spdk_nvme_ns_get_id(ns)); 3427 } 3428 } else { 3429 memcpy(&disk->uuid, nguid, sizeof(disk->uuid)); 3430 } 3431 3432 nsdata = spdk_nvme_ns_get_data(ns); 3433 bs = spdk_nvme_ns_get_sector_size(ns); 3434 atomic_bs = bs; 3435 phys_bs = bs; 3436 if (nsdata->nabo == 0) { 3437 if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) { 3438 atomic_bs = bs * (1 + nsdata->nawupf); 3439 } else { 3440 atomic_bs = bs * (1 + cdata->awupf); 3441 } 3442 } 3443 if (nsdata->nsfeat.optperf) { 3444 phys_bs = bs * (1 + nsdata->npwg); 3445 } 3446 disk->phys_blocklen = spdk_min(phys_bs, atomic_bs); 3447 3448 disk->md_len = spdk_nvme_ns_get_md_size(ns); 3449 if (disk->md_len != 0) { 3450 disk->md_interleave = nsdata->flbas.extended; 3451 disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns); 3452 if (disk->dif_type != SPDK_DIF_DISABLE) { 3453 disk->dif_is_head_of_md = nsdata->dps.md_start; 3454 disk->dif_check_flags = prchk_flags; 3455 } 3456 } 3457 3458 if (!(spdk_nvme_ctrlr_get_flags(ctrlr) & 3459 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) { 3460 disk->acwu = 0; 3461 } else if (nsdata->nsfeat.ns_atomic_write_unit) { 3462 disk->acwu = nsdata->nacwu + 1; /* 0-based */ 3463 } else { 3464 disk->acwu = cdata->acwu + 1; /* 0-based */ 3465 } 3466 3467 if (cdata->oncs.copy) { 3468 /* For now bdev interface allows only single segment copy */ 3469 disk->max_copy = nsdata->mssrl; 3470 } 3471 3472 disk->ctxt = ctx; 3473 disk->fn_table = &nvmelib_fn_table; 3474 disk->module = &nvme_if; 3475 3476 return 0; 3477 } 3478 3479 static int 3480 nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 3481 { 3482 struct nvme_bdev *bdev; 3483 int rc; 3484 3485 bdev = calloc(1, sizeof(*bdev)); 3486 if (!bdev) { 3487 SPDK_ERRLOG("bdev calloc() failed\n"); 3488 return -ENOMEM; 3489 } 3490 3491 if (g_opts.nvme_error_stat) { 3492 bdev->err_stat = calloc(1, sizeof(struct nvme_error_stat)); 3493 if (!bdev->err_stat) { 3494 SPDK_ERRLOG("err_stat calloc() failed\n"); 3495 free(bdev); 3496 return -ENOMEM; 3497 } 3498 } 3499 3500 rc = pthread_mutex_init(&bdev->mutex, NULL); 3501 if (rc != 0) { 3502 free(bdev->err_stat); 3503 free(bdev); 3504 return rc; 3505 } 3506 3507 bdev->ref = 1; 3508 bdev->mp_policy = BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE; 3509 bdev->mp_selector = BDEV_NVME_MP_SELECTOR_ROUND_ROBIN; 3510 bdev->rr_min_io = UINT32_MAX; 3511 TAILQ_INIT(&bdev->nvme_ns_list); 3512 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 3513 bdev->opal = nvme_ctrlr->opal_dev != NULL; 3514 3515 rc = nvme_disk_create(&bdev->disk, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ctrlr, 3516 nvme_ns->ns, nvme_ctrlr->opts.prchk_flags, bdev); 3517 if (rc != 0) { 3518 SPDK_ERRLOG("Failed to create NVMe disk\n"); 3519 pthread_mutex_destroy(&bdev->mutex); 3520 free(bdev->err_stat); 3521 free(bdev); 3522 return rc; 3523 } 3524 3525 spdk_io_device_register(bdev, 3526 bdev_nvme_create_bdev_channel_cb, 3527 bdev_nvme_destroy_bdev_channel_cb, 3528 sizeof(struct nvme_bdev_channel), 3529 bdev->disk.name); 3530 3531 rc = spdk_bdev_register(&bdev->disk); 3532 if (rc != 0) { 3533 SPDK_ERRLOG("spdk_bdev_register() failed\n"); 3534 spdk_io_device_unregister(bdev, NULL); 3535 pthread_mutex_destroy(&bdev->mutex); 3536 free(bdev->disk.name); 3537 free(bdev->err_stat); 3538 free(bdev); 3539 return rc; 3540 } 3541 3542 nvme_ns->bdev = bdev; 3543 bdev->nsid = nvme_ns->id; 3544 3545 bdev->nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr; 3546 TAILQ_INSERT_TAIL(&nvme_ctrlr->nbdev_ctrlr->bdevs, bdev, tailq); 3547 3548 return 0; 3549 } 3550 3551 static bool 3552 bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2) 3553 { 3554 const struct spdk_nvme_ns_data *nsdata1, *nsdata2; 3555 const struct spdk_uuid *uuid1, *uuid2; 3556 3557 nsdata1 = spdk_nvme_ns_get_data(ns1); 3558 nsdata2 = spdk_nvme_ns_get_data(ns2); 3559 uuid1 = spdk_nvme_ns_get_uuid(ns1); 3560 uuid2 = spdk_nvme_ns_get_uuid(ns2); 3561 3562 return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 && 3563 nsdata1->eui64 == nsdata2->eui64 && 3564 ((uuid1 == NULL && uuid2 == NULL) || 3565 (uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) && 3566 spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2); 3567 } 3568 3569 static bool 3570 hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 3571 struct spdk_nvme_ctrlr_opts *opts) 3572 { 3573 struct nvme_probe_skip_entry *entry; 3574 3575 TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) { 3576 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 3577 return false; 3578 } 3579 } 3580 3581 opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst; 3582 opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight; 3583 opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight; 3584 opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight; 3585 opts->disable_read_ana_log_page = true; 3586 3587 SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr); 3588 3589 return true; 3590 } 3591 3592 static void 3593 nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl) 3594 { 3595 struct nvme_ctrlr *nvme_ctrlr = ctx; 3596 3597 if (spdk_nvme_cpl_is_error(cpl)) { 3598 SPDK_WARNLOG("Abort failed. Resetting controller. sc is %u, sct is %u.\n", cpl->status.sc, 3599 cpl->status.sct); 3600 bdev_nvme_reset(nvme_ctrlr); 3601 } else if (cpl->cdw0 & 0x1) { 3602 SPDK_WARNLOG("Specified command could not be aborted.\n"); 3603 bdev_nvme_reset(nvme_ctrlr); 3604 } 3605 } 3606 3607 static void 3608 timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, 3609 struct spdk_nvme_qpair *qpair, uint16_t cid) 3610 { 3611 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 3612 union spdk_nvme_csts_register csts; 3613 int rc; 3614 3615 assert(nvme_ctrlr->ctrlr == ctrlr); 3616 3617 SPDK_WARNLOG("Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n", ctrlr, qpair, cid); 3618 3619 /* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O 3620 * queue. (Note: qpair == NULL when there's an admin cmd timeout.) Otherwise we 3621 * would submit another fabrics cmd on the admin queue to read CSTS and check for its 3622 * completion recursively. 3623 */ 3624 if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) { 3625 csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr); 3626 if (csts.bits.cfs) { 3627 SPDK_ERRLOG("Controller Fatal Status, reset required\n"); 3628 bdev_nvme_reset(nvme_ctrlr); 3629 return; 3630 } 3631 } 3632 3633 switch (g_opts.action_on_timeout) { 3634 case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT: 3635 if (qpair) { 3636 /* Don't send abort to ctrlr when ctrlr is not available. */ 3637 pthread_mutex_lock(&nvme_ctrlr->mutex); 3638 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 3639 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3640 SPDK_NOTICELOG("Quit abort. Ctrlr is not available.\n"); 3641 return; 3642 } 3643 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3644 3645 rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid, 3646 nvme_abort_cpl, nvme_ctrlr); 3647 if (rc == 0) { 3648 return; 3649 } 3650 3651 SPDK_ERRLOG("Unable to send abort. Resetting, rc is %d.\n", rc); 3652 } 3653 3654 /* FALLTHROUGH */ 3655 case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET: 3656 bdev_nvme_reset(nvme_ctrlr); 3657 break; 3658 case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE: 3659 SPDK_DEBUGLOG(bdev_nvme, "No action for nvme controller timeout.\n"); 3660 break; 3661 default: 3662 SPDK_ERRLOG("An invalid timeout action value is found.\n"); 3663 break; 3664 } 3665 } 3666 3667 static struct nvme_ns * 3668 nvme_ns_alloc(void) 3669 { 3670 struct nvme_ns *nvme_ns; 3671 3672 nvme_ns = calloc(1, sizeof(struct nvme_ns)); 3673 if (nvme_ns == NULL) { 3674 return NULL; 3675 } 3676 3677 if (g_opts.io_path_stat) { 3678 nvme_ns->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 3679 if (nvme_ns->stat == NULL) { 3680 free(nvme_ns); 3681 return NULL; 3682 } 3683 spdk_bdev_reset_io_stat(nvme_ns->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 3684 } 3685 3686 return nvme_ns; 3687 } 3688 3689 static void 3690 nvme_ns_free(struct nvme_ns *nvme_ns) 3691 { 3692 free(nvme_ns->stat); 3693 free(nvme_ns); 3694 } 3695 3696 static void 3697 nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc) 3698 { 3699 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 3700 struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx; 3701 3702 if (rc == 0) { 3703 nvme_ns->probe_ctx = NULL; 3704 pthread_mutex_lock(&nvme_ctrlr->mutex); 3705 nvme_ctrlr->ref++; 3706 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3707 } else { 3708 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 3709 nvme_ns_free(nvme_ns); 3710 } 3711 3712 if (ctx) { 3713 ctx->populates_in_progress--; 3714 if (ctx->populates_in_progress == 0) { 3715 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 3716 } 3717 } 3718 } 3719 3720 static void 3721 bdev_nvme_add_io_path(struct spdk_io_channel_iter *i) 3722 { 3723 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 3724 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 3725 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3726 int rc; 3727 3728 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 3729 if (rc != 0) { 3730 SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n"); 3731 } 3732 3733 spdk_for_each_channel_continue(i, rc); 3734 } 3735 3736 static void 3737 bdev_nvme_delete_io_path(struct spdk_io_channel_iter *i) 3738 { 3739 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 3740 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 3741 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3742 struct nvme_io_path *io_path; 3743 3744 io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns); 3745 if (io_path != NULL) { 3746 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 3747 } 3748 3749 spdk_for_each_channel_continue(i, 0); 3750 } 3751 3752 static void 3753 bdev_nvme_add_io_path_failed(struct spdk_io_channel_iter *i, int status) 3754 { 3755 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3756 3757 nvme_ctrlr_populate_namespace_done(nvme_ns, -1); 3758 } 3759 3760 static void 3761 bdev_nvme_add_io_path_done(struct spdk_io_channel_iter *i, int status) 3762 { 3763 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3764 struct nvme_bdev *bdev = spdk_io_channel_iter_get_io_device(i); 3765 3766 if (status == 0) { 3767 nvme_ctrlr_populate_namespace_done(nvme_ns, 0); 3768 } else { 3769 /* Delete the added io_paths and fail populating the namespace. */ 3770 spdk_for_each_channel(bdev, 3771 bdev_nvme_delete_io_path, 3772 nvme_ns, 3773 bdev_nvme_add_io_path_failed); 3774 } 3775 } 3776 3777 static int 3778 nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns) 3779 { 3780 struct nvme_ns *tmp_ns; 3781 const struct spdk_nvme_ns_data *nsdata; 3782 3783 nsdata = spdk_nvme_ns_get_data(nvme_ns->ns); 3784 if (!nsdata->nmic.can_share) { 3785 SPDK_ERRLOG("Namespace cannot be shared.\n"); 3786 return -EINVAL; 3787 } 3788 3789 pthread_mutex_lock(&bdev->mutex); 3790 3791 tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list); 3792 assert(tmp_ns != NULL); 3793 3794 if (!bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) { 3795 pthread_mutex_unlock(&bdev->mutex); 3796 SPDK_ERRLOG("Namespaces are not identical.\n"); 3797 return -EINVAL; 3798 } 3799 3800 bdev->ref++; 3801 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 3802 nvme_ns->bdev = bdev; 3803 3804 pthread_mutex_unlock(&bdev->mutex); 3805 3806 /* Add nvme_io_path to nvme_bdev_channels dynamically. */ 3807 spdk_for_each_channel(bdev, 3808 bdev_nvme_add_io_path, 3809 nvme_ns, 3810 bdev_nvme_add_io_path_done); 3811 3812 return 0; 3813 } 3814 3815 static void 3816 nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 3817 { 3818 struct spdk_nvme_ns *ns; 3819 struct nvme_bdev *bdev; 3820 int rc = 0; 3821 3822 ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id); 3823 if (!ns) { 3824 SPDK_DEBUGLOG(bdev_nvme, "Invalid NS %d\n", nvme_ns->id); 3825 rc = -EINVAL; 3826 goto done; 3827 } 3828 3829 nvme_ns->ns = ns; 3830 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 3831 3832 if (nvme_ctrlr->ana_log_page != NULL) { 3833 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns); 3834 } 3835 3836 bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id); 3837 if (bdev == NULL) { 3838 rc = nvme_bdev_create(nvme_ctrlr, nvme_ns); 3839 } else { 3840 rc = nvme_bdev_add_ns(bdev, nvme_ns); 3841 if (rc == 0) { 3842 return; 3843 } 3844 } 3845 done: 3846 nvme_ctrlr_populate_namespace_done(nvme_ns, rc); 3847 } 3848 3849 static void 3850 nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns) 3851 { 3852 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 3853 3854 assert(nvme_ctrlr != NULL); 3855 3856 pthread_mutex_lock(&nvme_ctrlr->mutex); 3857 3858 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 3859 3860 if (nvme_ns->bdev != NULL) { 3861 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3862 return; 3863 } 3864 3865 nvme_ns_free(nvme_ns); 3866 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3867 3868 nvme_ctrlr_release(nvme_ctrlr); 3869 } 3870 3871 static void 3872 bdev_nvme_delete_io_path_done(struct spdk_io_channel_iter *i, int status) 3873 { 3874 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 3875 3876 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 3877 } 3878 3879 static void 3880 nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 3881 { 3882 struct nvme_bdev *bdev; 3883 3884 spdk_poller_unregister(&nvme_ns->anatt_timer); 3885 3886 bdev = nvme_ns->bdev; 3887 if (bdev != NULL) { 3888 pthread_mutex_lock(&bdev->mutex); 3889 3890 assert(bdev->ref > 0); 3891 bdev->ref--; 3892 if (bdev->ref == 0) { 3893 pthread_mutex_unlock(&bdev->mutex); 3894 3895 spdk_bdev_unregister(&bdev->disk, NULL, NULL); 3896 } else { 3897 /* spdk_bdev_unregister() is not called until the last nvme_ns is 3898 * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list 3899 * and clear nvme_ns->bdev here. 3900 */ 3901 TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq); 3902 nvme_ns->bdev = NULL; 3903 3904 pthread_mutex_unlock(&bdev->mutex); 3905 3906 /* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that, 3907 * we call depopulate_namespace_done() to avoid use-after-free. 3908 */ 3909 spdk_for_each_channel(bdev, 3910 bdev_nvme_delete_io_path, 3911 nvme_ns, 3912 bdev_nvme_delete_io_path_done); 3913 return; 3914 } 3915 } 3916 3917 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 3918 } 3919 3920 static void 3921 nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 3922 struct nvme_async_probe_ctx *ctx) 3923 { 3924 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 3925 struct nvme_ns *nvme_ns, *next; 3926 struct spdk_nvme_ns *ns; 3927 struct nvme_bdev *bdev; 3928 uint32_t nsid; 3929 int rc; 3930 uint64_t num_sectors; 3931 3932 if (ctx) { 3933 /* Initialize this count to 1 to handle the populate functions 3934 * calling nvme_ctrlr_populate_namespace_done() immediately. 3935 */ 3936 ctx->populates_in_progress = 1; 3937 } 3938 3939 /* First loop over our existing namespaces and see if they have been 3940 * removed. */ 3941 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 3942 while (nvme_ns != NULL) { 3943 next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 3944 3945 if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 3946 /* NS is still there but attributes may have changed */ 3947 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id); 3948 num_sectors = spdk_nvme_ns_get_num_sectors(ns); 3949 bdev = nvme_ns->bdev; 3950 assert(bdev != NULL); 3951 if (bdev->disk.blockcnt != num_sectors) { 3952 SPDK_NOTICELOG("NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n", 3953 nvme_ns->id, 3954 bdev->disk.name, 3955 bdev->disk.blockcnt, 3956 num_sectors); 3957 rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors); 3958 if (rc != 0) { 3959 SPDK_ERRLOG("Could not change num blocks for nvme bdev: name %s, errno: %d.\n", 3960 bdev->disk.name, rc); 3961 } 3962 } 3963 } else { 3964 /* Namespace was removed */ 3965 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 3966 } 3967 3968 nvme_ns = next; 3969 } 3970 3971 /* Loop through all of the namespaces at the nvme level and see if any of them are new */ 3972 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 3973 while (nsid != 0) { 3974 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 3975 3976 if (nvme_ns == NULL) { 3977 /* Found a new one */ 3978 nvme_ns = nvme_ns_alloc(); 3979 if (nvme_ns == NULL) { 3980 SPDK_ERRLOG("Failed to allocate namespace\n"); 3981 /* This just fails to attach the namespace. It may work on a future attempt. */ 3982 continue; 3983 } 3984 3985 nvme_ns->id = nsid; 3986 nvme_ns->ctrlr = nvme_ctrlr; 3987 3988 nvme_ns->bdev = NULL; 3989 3990 if (ctx) { 3991 ctx->populates_in_progress++; 3992 } 3993 nvme_ns->probe_ctx = ctx; 3994 3995 RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 3996 3997 nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns); 3998 } 3999 4000 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid); 4001 } 4002 4003 if (ctx) { 4004 /* Decrement this count now that the loop is over to account 4005 * for the one we started with. If the count is then 0, we 4006 * know any populate_namespace functions completed immediately, 4007 * so we'll kick the callback here. 4008 */ 4009 ctx->populates_in_progress--; 4010 if (ctx->populates_in_progress == 0) { 4011 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 4012 } 4013 } 4014 4015 } 4016 4017 static void 4018 nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr) 4019 { 4020 struct nvme_ns *nvme_ns, *tmp; 4021 4022 RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) { 4023 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 4024 } 4025 } 4026 4027 static uint32_t 4028 nvme_ctrlr_get_ana_log_page_size(struct nvme_ctrlr *nvme_ctrlr) 4029 { 4030 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4031 const struct spdk_nvme_ctrlr_data *cdata; 4032 uint32_t nsid, ns_count = 0; 4033 4034 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4035 4036 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 4037 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 4038 ns_count++; 4039 } 4040 4041 return sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 4042 sizeof(struct spdk_nvme_ana_group_descriptor) + ns_count * 4043 sizeof(uint32_t); 4044 } 4045 4046 static int 4047 nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc, 4048 void *cb_arg) 4049 { 4050 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 4051 struct nvme_ns *nvme_ns; 4052 uint32_t i, nsid; 4053 4054 for (i = 0; i < desc->num_of_nsid; i++) { 4055 nsid = desc->nsid[i]; 4056 if (nsid == 0) { 4057 continue; 4058 } 4059 4060 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 4061 4062 assert(nvme_ns != NULL); 4063 if (nvme_ns == NULL) { 4064 /* Target told us that an inactive namespace had an ANA change */ 4065 continue; 4066 } 4067 4068 _nvme_ns_set_ana_state(nvme_ns, desc); 4069 } 4070 4071 return 0; 4072 } 4073 4074 static void 4075 bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 4076 { 4077 struct nvme_ns *nvme_ns; 4078 4079 spdk_free(nvme_ctrlr->ana_log_page); 4080 nvme_ctrlr->ana_log_page = NULL; 4081 4082 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4083 nvme_ns != NULL; 4084 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 4085 nvme_ns->ana_state_updating = false; 4086 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 4087 } 4088 } 4089 4090 static void 4091 nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl) 4092 { 4093 struct nvme_ctrlr *nvme_ctrlr = ctx; 4094 4095 if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) { 4096 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states, 4097 nvme_ctrlr); 4098 } else { 4099 bdev_nvme_disable_read_ana_log_page(nvme_ctrlr); 4100 } 4101 4102 pthread_mutex_lock(&nvme_ctrlr->mutex); 4103 4104 assert(nvme_ctrlr->ana_log_page_updating == true); 4105 nvme_ctrlr->ana_log_page_updating = false; 4106 4107 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 4108 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4109 4110 nvme_ctrlr_unregister(nvme_ctrlr); 4111 } else { 4112 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4113 4114 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 4115 } 4116 } 4117 4118 static int 4119 nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 4120 { 4121 uint32_t ana_log_page_size; 4122 int rc; 4123 4124 if (nvme_ctrlr->ana_log_page == NULL) { 4125 return -EINVAL; 4126 } 4127 4128 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 4129 4130 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 4131 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 4132 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 4133 return -EINVAL; 4134 } 4135 4136 pthread_mutex_lock(&nvme_ctrlr->mutex); 4137 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 4138 nvme_ctrlr->ana_log_page_updating) { 4139 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4140 return -EBUSY; 4141 } 4142 4143 nvme_ctrlr->ana_log_page_updating = true; 4144 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4145 4146 rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr, 4147 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 4148 SPDK_NVME_GLOBAL_NS_TAG, 4149 nvme_ctrlr->ana_log_page, 4150 ana_log_page_size, 0, 4151 nvme_ctrlr_read_ana_log_page_done, 4152 nvme_ctrlr); 4153 if (rc != 0) { 4154 nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL); 4155 } 4156 4157 return rc; 4158 } 4159 4160 static void 4161 dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 4162 { 4163 } 4164 4165 struct bdev_nvme_set_preferred_path_ctx { 4166 struct spdk_bdev_desc *desc; 4167 struct nvme_ns *nvme_ns; 4168 bdev_nvme_set_preferred_path_cb cb_fn; 4169 void *cb_arg; 4170 }; 4171 4172 static void 4173 bdev_nvme_set_preferred_path_done(struct spdk_io_channel_iter *i, int status) 4174 { 4175 struct bdev_nvme_set_preferred_path_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 4176 4177 assert(ctx != NULL); 4178 assert(ctx->desc != NULL); 4179 assert(ctx->cb_fn != NULL); 4180 4181 spdk_bdev_close(ctx->desc); 4182 4183 ctx->cb_fn(ctx->cb_arg, status); 4184 4185 free(ctx); 4186 } 4187 4188 static void 4189 _bdev_nvme_set_preferred_path(struct spdk_io_channel_iter *i) 4190 { 4191 struct bdev_nvme_set_preferred_path_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 4192 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 4193 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 4194 struct nvme_io_path *io_path, *prev; 4195 4196 prev = NULL; 4197 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 4198 if (io_path->nvme_ns == ctx->nvme_ns) { 4199 break; 4200 } 4201 prev = io_path; 4202 } 4203 4204 if (io_path != NULL) { 4205 if (prev != NULL) { 4206 STAILQ_REMOVE_AFTER(&nbdev_ch->io_path_list, prev, stailq); 4207 STAILQ_INSERT_HEAD(&nbdev_ch->io_path_list, io_path, stailq); 4208 } 4209 4210 /* We can set io_path to nbdev_ch->current_io_path directly here. 4211 * However, it needs to be conditional. To simplify the code, 4212 * just clear nbdev_ch->current_io_path and let find_io_path() 4213 * fill it. 4214 * 4215 * Automatic failback may be disabled. Hence even if the io_path is 4216 * already at the head, clear nbdev_ch->current_io_path. 4217 */ 4218 bdev_nvme_clear_current_io_path(nbdev_ch); 4219 } 4220 4221 spdk_for_each_channel_continue(i, 0); 4222 } 4223 4224 static struct nvme_ns * 4225 bdev_nvme_set_preferred_ns(struct nvme_bdev *nbdev, uint16_t cntlid) 4226 { 4227 struct nvme_ns *nvme_ns, *prev; 4228 const struct spdk_nvme_ctrlr_data *cdata; 4229 4230 prev = NULL; 4231 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 4232 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 4233 4234 if (cdata->cntlid == cntlid) { 4235 break; 4236 } 4237 prev = nvme_ns; 4238 } 4239 4240 if (nvme_ns != NULL && prev != NULL) { 4241 TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq); 4242 TAILQ_INSERT_HEAD(&nbdev->nvme_ns_list, nvme_ns, tailq); 4243 } 4244 4245 return nvme_ns; 4246 } 4247 4248 /* This function supports only multipath mode. There is only a single I/O path 4249 * for each NVMe-oF controller. Hence, just move the matched I/O path to the 4250 * head of the I/O path list for each NVMe bdev channel. 4251 * 4252 * NVMe bdev channel may be acquired after completing this function. move the 4253 * matched namespace to the head of the namespace list for the NVMe bdev too. 4254 */ 4255 void 4256 bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid, 4257 bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg) 4258 { 4259 struct bdev_nvme_set_preferred_path_ctx *ctx; 4260 struct spdk_bdev *bdev; 4261 struct nvme_bdev *nbdev; 4262 int rc = 0; 4263 4264 assert(cb_fn != NULL); 4265 4266 ctx = calloc(1, sizeof(*ctx)); 4267 if (ctx == NULL) { 4268 SPDK_ERRLOG("Failed to alloc context.\n"); 4269 rc = -ENOMEM; 4270 goto err_alloc; 4271 } 4272 4273 ctx->cb_fn = cb_fn; 4274 ctx->cb_arg = cb_arg; 4275 4276 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 4277 if (rc != 0) { 4278 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 4279 goto err_open; 4280 } 4281 4282 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 4283 4284 if (bdev->module != &nvme_if) { 4285 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 4286 rc = -ENODEV; 4287 goto err_bdev; 4288 } 4289 4290 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 4291 4292 pthread_mutex_lock(&nbdev->mutex); 4293 4294 ctx->nvme_ns = bdev_nvme_set_preferred_ns(nbdev, cntlid); 4295 if (ctx->nvme_ns == NULL) { 4296 pthread_mutex_unlock(&nbdev->mutex); 4297 4298 SPDK_ERRLOG("bdev %s does not have namespace to controller %u.\n", name, cntlid); 4299 rc = -ENODEV; 4300 goto err_bdev; 4301 } 4302 4303 pthread_mutex_unlock(&nbdev->mutex); 4304 4305 spdk_for_each_channel(nbdev, 4306 _bdev_nvme_set_preferred_path, 4307 ctx, 4308 bdev_nvme_set_preferred_path_done); 4309 return; 4310 4311 err_bdev: 4312 spdk_bdev_close(ctx->desc); 4313 err_open: 4314 free(ctx); 4315 err_alloc: 4316 cb_fn(cb_arg, rc); 4317 } 4318 4319 struct bdev_nvme_set_multipath_policy_ctx { 4320 struct spdk_bdev_desc *desc; 4321 bdev_nvme_set_multipath_policy_cb cb_fn; 4322 void *cb_arg; 4323 }; 4324 4325 static void 4326 bdev_nvme_set_multipath_policy_done(struct spdk_io_channel_iter *i, int status) 4327 { 4328 struct bdev_nvme_set_multipath_policy_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 4329 4330 assert(ctx != NULL); 4331 assert(ctx->desc != NULL); 4332 assert(ctx->cb_fn != NULL); 4333 4334 spdk_bdev_close(ctx->desc); 4335 4336 ctx->cb_fn(ctx->cb_arg, status); 4337 4338 free(ctx); 4339 } 4340 4341 static void 4342 _bdev_nvme_set_multipath_policy(struct spdk_io_channel_iter *i) 4343 { 4344 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 4345 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 4346 struct nvme_bdev *nbdev = spdk_io_channel_get_io_device(_ch); 4347 4348 nbdev_ch->mp_policy = nbdev->mp_policy; 4349 nbdev_ch->mp_selector = nbdev->mp_selector; 4350 nbdev_ch->rr_min_io = nbdev->rr_min_io; 4351 bdev_nvme_clear_current_io_path(nbdev_ch); 4352 4353 spdk_for_each_channel_continue(i, 0); 4354 } 4355 4356 void 4357 bdev_nvme_set_multipath_policy(const char *name, enum bdev_nvme_multipath_policy policy, 4358 enum bdev_nvme_multipath_selector selector, uint32_t rr_min_io, 4359 bdev_nvme_set_multipath_policy_cb cb_fn, void *cb_arg) 4360 { 4361 struct bdev_nvme_set_multipath_policy_ctx *ctx; 4362 struct spdk_bdev *bdev; 4363 struct nvme_bdev *nbdev; 4364 int rc; 4365 4366 assert(cb_fn != NULL); 4367 4368 if (policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE && selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 4369 if (rr_min_io == UINT32_MAX) { 4370 rr_min_io = 1; 4371 } else if (rr_min_io == 0) { 4372 rc = -EINVAL; 4373 goto exit; 4374 } 4375 } else if (rr_min_io != UINT32_MAX) { 4376 rc = -EINVAL; 4377 goto exit; 4378 } 4379 4380 ctx = calloc(1, sizeof(*ctx)); 4381 if (ctx == NULL) { 4382 SPDK_ERRLOG("Failed to alloc context.\n"); 4383 rc = -ENOMEM; 4384 goto exit; 4385 } 4386 4387 ctx->cb_fn = cb_fn; 4388 ctx->cb_arg = cb_arg; 4389 4390 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 4391 if (rc != 0) { 4392 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 4393 rc = -ENODEV; 4394 goto err_open; 4395 } 4396 4397 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 4398 if (bdev->module != &nvme_if) { 4399 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 4400 rc = -ENODEV; 4401 goto err_module; 4402 } 4403 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 4404 4405 pthread_mutex_lock(&nbdev->mutex); 4406 nbdev->mp_policy = policy; 4407 nbdev->mp_selector = selector; 4408 nbdev->rr_min_io = rr_min_io; 4409 pthread_mutex_unlock(&nbdev->mutex); 4410 4411 spdk_for_each_channel(nbdev, 4412 _bdev_nvme_set_multipath_policy, 4413 ctx, 4414 bdev_nvme_set_multipath_policy_done); 4415 return; 4416 4417 err_module: 4418 spdk_bdev_close(ctx->desc); 4419 err_open: 4420 free(ctx); 4421 exit: 4422 cb_fn(cb_arg, rc); 4423 } 4424 4425 static void 4426 aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 4427 { 4428 struct nvme_ctrlr *nvme_ctrlr = arg; 4429 union spdk_nvme_async_event_completion event; 4430 4431 if (spdk_nvme_cpl_is_error(cpl)) { 4432 SPDK_WARNLOG("AER request execute failed\n"); 4433 return; 4434 } 4435 4436 event.raw = cpl->cdw0; 4437 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 4438 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) { 4439 nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL); 4440 } else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 4441 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) { 4442 nvme_ctrlr_read_ana_log_page(nvme_ctrlr); 4443 } 4444 } 4445 4446 static void 4447 populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, size_t count, int rc) 4448 { 4449 if (ctx->cb_fn) { 4450 ctx->cb_fn(ctx->cb_ctx, count, rc); 4451 } 4452 4453 ctx->namespaces_populated = true; 4454 if (ctx->probe_done) { 4455 /* The probe was already completed, so we need to free the context 4456 * here. This can happen for cases like OCSSD, where we need to 4457 * send additional commands to the SSD after attach. 4458 */ 4459 free(ctx); 4460 } 4461 } 4462 4463 static void 4464 nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr, 4465 struct nvme_async_probe_ctx *ctx) 4466 { 4467 spdk_io_device_register(nvme_ctrlr, 4468 bdev_nvme_create_ctrlr_channel_cb, 4469 bdev_nvme_destroy_ctrlr_channel_cb, 4470 sizeof(struct nvme_ctrlr_channel), 4471 nvme_ctrlr->nbdev_ctrlr->name); 4472 4473 nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx); 4474 } 4475 4476 static void 4477 nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl) 4478 { 4479 struct nvme_ctrlr *nvme_ctrlr = _ctx; 4480 struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx; 4481 4482 nvme_ctrlr->probe_ctx = NULL; 4483 4484 if (spdk_nvme_cpl_is_error(cpl)) { 4485 nvme_ctrlr_delete(nvme_ctrlr); 4486 4487 if (ctx != NULL) { 4488 populate_namespaces_cb(ctx, 0, -1); 4489 } 4490 return; 4491 } 4492 4493 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 4494 } 4495 4496 static int 4497 nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 4498 struct nvme_async_probe_ctx *ctx) 4499 { 4500 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4501 const struct spdk_nvme_ctrlr_data *cdata; 4502 uint32_t ana_log_page_size; 4503 4504 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4505 4506 /* Set buffer size enough to include maximum number of allowed namespaces. */ 4507 ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 4508 sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->mnan * 4509 sizeof(uint32_t); 4510 4511 nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL, 4512 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4513 if (nvme_ctrlr->ana_log_page == NULL) { 4514 SPDK_ERRLOG("could not allocate ANA log page buffer\n"); 4515 return -ENXIO; 4516 } 4517 4518 /* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned. 4519 * Hence copy each descriptor to a temporary area when parsing it. 4520 * 4521 * Allocate a buffer whose size is as large as ANA log page buffer because 4522 * we do not know the size of a descriptor until actually reading it. 4523 */ 4524 nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size); 4525 if (nvme_ctrlr->copied_ana_desc == NULL) { 4526 SPDK_ERRLOG("could not allocate a buffer to parse ANA descriptor\n"); 4527 return -ENOMEM; 4528 } 4529 4530 nvme_ctrlr->max_ana_log_page_size = ana_log_page_size; 4531 4532 nvme_ctrlr->probe_ctx = ctx; 4533 4534 /* Then, set the read size only to include the current active namespaces. */ 4535 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 4536 4537 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 4538 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 4539 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 4540 return -EINVAL; 4541 } 4542 4543 return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, 4544 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 4545 SPDK_NVME_GLOBAL_NS_TAG, 4546 nvme_ctrlr->ana_log_page, 4547 ana_log_page_size, 0, 4548 nvme_ctrlr_init_ana_log_page_done, 4549 nvme_ctrlr); 4550 } 4551 4552 /* hostnqn and subnqn were already verified before attaching a controller. 4553 * Hence check only the multipath capability and cntlid here. 4554 */ 4555 static bool 4556 bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr) 4557 { 4558 struct nvme_ctrlr *tmp; 4559 const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata; 4560 4561 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4562 4563 if (!cdata->cmic.multi_ctrlr) { 4564 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 4565 return false; 4566 } 4567 4568 TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) { 4569 tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr); 4570 4571 if (!tmp_cdata->cmic.multi_ctrlr) { 4572 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 4573 return false; 4574 } 4575 if (cdata->cntlid == tmp_cdata->cntlid) { 4576 SPDK_ERRLOG("cntlid %u are duplicated.\n", tmp_cdata->cntlid); 4577 return false; 4578 } 4579 } 4580 4581 return true; 4582 } 4583 4584 static int 4585 nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr) 4586 { 4587 struct nvme_bdev_ctrlr *nbdev_ctrlr; 4588 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4589 int rc = 0; 4590 4591 pthread_mutex_lock(&g_bdev_nvme_mutex); 4592 4593 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 4594 if (nbdev_ctrlr != NULL) { 4595 if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) { 4596 rc = -EINVAL; 4597 goto exit; 4598 } 4599 } else { 4600 nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr)); 4601 if (nbdev_ctrlr == NULL) { 4602 SPDK_ERRLOG("Failed to allocate nvme_bdev_ctrlr.\n"); 4603 rc = -ENOMEM; 4604 goto exit; 4605 } 4606 nbdev_ctrlr->name = strdup(name); 4607 if (nbdev_ctrlr->name == NULL) { 4608 SPDK_ERRLOG("Failed to allocate name of nvme_bdev_ctrlr.\n"); 4609 free(nbdev_ctrlr); 4610 goto exit; 4611 } 4612 TAILQ_INIT(&nbdev_ctrlr->ctrlrs); 4613 TAILQ_INIT(&nbdev_ctrlr->bdevs); 4614 TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 4615 } 4616 nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr; 4617 TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 4618 exit: 4619 pthread_mutex_unlock(&g_bdev_nvme_mutex); 4620 return rc; 4621 } 4622 4623 static int 4624 nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr, 4625 const char *name, 4626 const struct spdk_nvme_transport_id *trid, 4627 struct nvme_async_probe_ctx *ctx) 4628 { 4629 struct nvme_ctrlr *nvme_ctrlr; 4630 struct nvme_path_id *path_id; 4631 const struct spdk_nvme_ctrlr_data *cdata; 4632 int rc; 4633 4634 nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr)); 4635 if (nvme_ctrlr == NULL) { 4636 SPDK_ERRLOG("Failed to allocate device struct\n"); 4637 return -ENOMEM; 4638 } 4639 4640 rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL); 4641 if (rc != 0) { 4642 free(nvme_ctrlr); 4643 return rc; 4644 } 4645 4646 TAILQ_INIT(&nvme_ctrlr->trids); 4647 4648 RB_INIT(&nvme_ctrlr->namespaces); 4649 4650 path_id = calloc(1, sizeof(*path_id)); 4651 if (path_id == NULL) { 4652 SPDK_ERRLOG("Failed to allocate trid entry pointer\n"); 4653 rc = -ENOMEM; 4654 goto err; 4655 } 4656 4657 path_id->trid = *trid; 4658 if (ctx != NULL) { 4659 memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr)); 4660 memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid)); 4661 } 4662 nvme_ctrlr->active_path_id = path_id; 4663 TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link); 4664 4665 nvme_ctrlr->thread = spdk_get_thread(); 4666 nvme_ctrlr->ctrlr = ctrlr; 4667 nvme_ctrlr->ref = 1; 4668 4669 if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) { 4670 SPDK_ERRLOG("OCSSDs are not supported"); 4671 rc = -ENOTSUP; 4672 goto err; 4673 } 4674 4675 if (ctx != NULL) { 4676 memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts)); 4677 } else { 4678 bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts); 4679 } 4680 4681 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr, 4682 g_opts.nvme_adminq_poll_period_us); 4683 4684 if (g_opts.timeout_us > 0) { 4685 /* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */ 4686 /* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */ 4687 uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ? 4688 g_opts.timeout_us : g_opts.timeout_admin_us; 4689 spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us, 4690 adm_timeout_us, timeout_cb, nvme_ctrlr); 4691 } 4692 4693 spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr); 4694 spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr); 4695 4696 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 4697 SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) { 4698 nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr); 4699 } 4700 4701 rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr); 4702 if (rc != 0) { 4703 goto err; 4704 } 4705 4706 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4707 4708 if (cdata->cmic.ana_reporting) { 4709 rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx); 4710 if (rc == 0) { 4711 return 0; 4712 } 4713 } else { 4714 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 4715 return 0; 4716 } 4717 4718 err: 4719 nvme_ctrlr_delete(nvme_ctrlr); 4720 return rc; 4721 } 4722 4723 void 4724 bdev_nvme_get_default_ctrlr_opts(struct nvme_ctrlr_opts *opts) 4725 { 4726 opts->prchk_flags = 0; 4727 opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec; 4728 opts->reconnect_delay_sec = g_opts.reconnect_delay_sec; 4729 opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec; 4730 } 4731 4732 static void 4733 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 4734 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts) 4735 { 4736 char *name; 4737 4738 name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++); 4739 if (!name) { 4740 SPDK_ERRLOG("Failed to assign name to NVMe device\n"); 4741 return; 4742 } 4743 4744 if (nvme_ctrlr_create(ctrlr, name, trid, NULL) == 0) { 4745 SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name); 4746 } else { 4747 SPDK_ERRLOG("Failed to attach to %s (%s)\n", trid->traddr, name); 4748 } 4749 4750 free(name); 4751 } 4752 4753 static void 4754 _nvme_ctrlr_destruct(void *ctx) 4755 { 4756 struct nvme_ctrlr *nvme_ctrlr = ctx; 4757 4758 nvme_ctrlr_depopulate_namespaces(nvme_ctrlr); 4759 nvme_ctrlr_release(nvme_ctrlr); 4760 } 4761 4762 static int 4763 bdev_nvme_delete_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 4764 { 4765 struct nvme_probe_skip_entry *entry; 4766 4767 /* The controller's destruction was already started */ 4768 if (nvme_ctrlr->destruct) { 4769 return -EALREADY; 4770 } 4771 4772 if (!hotplug && 4773 nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 4774 entry = calloc(1, sizeof(*entry)); 4775 if (!entry) { 4776 return -ENOMEM; 4777 } 4778 entry->trid = nvme_ctrlr->active_path_id->trid; 4779 TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq); 4780 } 4781 4782 nvme_ctrlr->destruct = true; 4783 return 0; 4784 } 4785 4786 static int 4787 bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 4788 { 4789 int rc; 4790 4791 pthread_mutex_lock(&nvme_ctrlr->mutex); 4792 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, hotplug); 4793 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4794 4795 if (rc == 0) { 4796 _nvme_ctrlr_destruct(nvme_ctrlr); 4797 } else if (rc == -EALREADY) { 4798 rc = 0; 4799 } 4800 4801 return rc; 4802 } 4803 4804 static void 4805 remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr) 4806 { 4807 struct nvme_ctrlr *nvme_ctrlr = cb_ctx; 4808 4809 bdev_nvme_delete_ctrlr(nvme_ctrlr, true); 4810 } 4811 4812 static int 4813 bdev_nvme_hotplug_probe(void *arg) 4814 { 4815 if (g_hotplug_probe_ctx == NULL) { 4816 spdk_poller_unregister(&g_hotplug_probe_poller); 4817 return SPDK_POLLER_IDLE; 4818 } 4819 4820 if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) { 4821 g_hotplug_probe_ctx = NULL; 4822 spdk_poller_unregister(&g_hotplug_probe_poller); 4823 } 4824 4825 return SPDK_POLLER_BUSY; 4826 } 4827 4828 static int 4829 bdev_nvme_hotplug(void *arg) 4830 { 4831 struct spdk_nvme_transport_id trid_pcie; 4832 4833 if (g_hotplug_probe_ctx) { 4834 return SPDK_POLLER_BUSY; 4835 } 4836 4837 memset(&trid_pcie, 0, sizeof(trid_pcie)); 4838 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 4839 4840 g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL, 4841 hotplug_probe_cb, attach_cb, NULL); 4842 4843 if (g_hotplug_probe_ctx) { 4844 assert(g_hotplug_probe_poller == NULL); 4845 g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000); 4846 } 4847 4848 return SPDK_POLLER_BUSY; 4849 } 4850 4851 void 4852 bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts) 4853 { 4854 *opts = g_opts; 4855 } 4856 4857 static bool bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 4858 uint32_t reconnect_delay_sec, 4859 uint32_t fast_io_fail_timeout_sec); 4860 4861 static int 4862 bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts) 4863 { 4864 if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) { 4865 /* Can't set timeout_admin_us without also setting timeout_us */ 4866 SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n"); 4867 return -EINVAL; 4868 } 4869 4870 if (opts->bdev_retry_count < -1) { 4871 SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n"); 4872 return -EINVAL; 4873 } 4874 4875 if (!bdev_nvme_check_io_error_resiliency_params(opts->ctrlr_loss_timeout_sec, 4876 opts->reconnect_delay_sec, 4877 opts->fast_io_fail_timeout_sec)) { 4878 return -EINVAL; 4879 } 4880 4881 return 0; 4882 } 4883 4884 int 4885 bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) 4886 { 4887 int ret; 4888 4889 ret = bdev_nvme_validate_opts(opts); 4890 if (ret) { 4891 SPDK_WARNLOG("Failed to set nvme opts.\n"); 4892 return ret; 4893 } 4894 4895 if (g_bdev_nvme_init_thread != NULL) { 4896 if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 4897 return -EPERM; 4898 } 4899 } 4900 4901 if (opts->rdma_srq_size != 0) { 4902 struct spdk_nvme_transport_opts drv_opts; 4903 4904 spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts)); 4905 drv_opts.rdma_srq_size = opts->rdma_srq_size; 4906 4907 ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts)); 4908 if (ret) { 4909 SPDK_ERRLOG("Failed to set NVMe transport opts.\n"); 4910 return ret; 4911 } 4912 } 4913 4914 g_opts = *opts; 4915 4916 return 0; 4917 } 4918 4919 struct set_nvme_hotplug_ctx { 4920 uint64_t period_us; 4921 bool enabled; 4922 spdk_msg_fn fn; 4923 void *fn_ctx; 4924 }; 4925 4926 static void 4927 set_nvme_hotplug_period_cb(void *_ctx) 4928 { 4929 struct set_nvme_hotplug_ctx *ctx = _ctx; 4930 4931 spdk_poller_unregister(&g_hotplug_poller); 4932 if (ctx->enabled) { 4933 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us); 4934 } 4935 4936 g_nvme_hotplug_poll_period_us = ctx->period_us; 4937 g_nvme_hotplug_enabled = ctx->enabled; 4938 if (ctx->fn) { 4939 ctx->fn(ctx->fn_ctx); 4940 } 4941 4942 free(ctx); 4943 } 4944 4945 int 4946 bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx) 4947 { 4948 struct set_nvme_hotplug_ctx *ctx; 4949 4950 if (enabled == true && !spdk_process_is_primary()) { 4951 return -EPERM; 4952 } 4953 4954 ctx = calloc(1, sizeof(*ctx)); 4955 if (ctx == NULL) { 4956 return -ENOMEM; 4957 } 4958 4959 period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us; 4960 ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX); 4961 ctx->enabled = enabled; 4962 ctx->fn = cb; 4963 ctx->fn_ctx = cb_ctx; 4964 4965 spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx); 4966 return 0; 4967 } 4968 4969 static void 4970 nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 4971 struct nvme_async_probe_ctx *ctx) 4972 { 4973 struct nvme_ns *nvme_ns; 4974 struct nvme_bdev *nvme_bdev; 4975 size_t j; 4976 4977 assert(nvme_ctrlr != NULL); 4978 4979 if (ctx->names == NULL) { 4980 populate_namespaces_cb(ctx, 0, 0); 4981 return; 4982 } 4983 4984 /* 4985 * Report the new bdevs that were created in this call. 4986 * There can be more than one bdev per NVMe controller. 4987 */ 4988 j = 0; 4989 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4990 while (nvme_ns != NULL) { 4991 nvme_bdev = nvme_ns->bdev; 4992 if (j < ctx->count) { 4993 ctx->names[j] = nvme_bdev->disk.name; 4994 j++; 4995 } else { 4996 SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n", 4997 ctx->count); 4998 populate_namespaces_cb(ctx, 0, -ERANGE); 4999 return; 5000 } 5001 5002 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 5003 } 5004 5005 populate_namespaces_cb(ctx, j, 0); 5006 } 5007 5008 static int 5009 bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 5010 struct spdk_nvme_ctrlr *new_ctrlr, 5011 struct spdk_nvme_transport_id *trid) 5012 { 5013 struct nvme_path_id *tmp_trid; 5014 5015 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 5016 SPDK_ERRLOG("PCIe failover is not supported.\n"); 5017 return -ENOTSUP; 5018 } 5019 5020 /* Currently we only support failover to the same transport type. */ 5021 if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) { 5022 SPDK_WARNLOG("Failover from trtype: %s to a different trtype: %s is not supported currently\n", 5023 spdk_nvme_transport_id_trtype_str(nvme_ctrlr->active_path_id->trid.trtype), 5024 spdk_nvme_transport_id_trtype_str(trid->trtype)); 5025 return -EINVAL; 5026 } 5027 5028 5029 /* Currently we only support failover to the same NQN. */ 5030 if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) { 5031 SPDK_WARNLOG("Failover from subnqn: %s to a different subnqn: %s is not supported currently\n", 5032 nvme_ctrlr->active_path_id->trid.subnqn, trid->subnqn); 5033 return -EINVAL; 5034 } 5035 5036 /* Skip all the other checks if we've already registered this path. */ 5037 TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) { 5038 if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) { 5039 SPDK_WARNLOG("This path (traddr: %s subnqn: %s) is already registered\n", trid->traddr, 5040 trid->subnqn); 5041 return -EEXIST; 5042 } 5043 } 5044 5045 return 0; 5046 } 5047 5048 static int 5049 bdev_nvme_check_secondary_namespace(struct nvme_ctrlr *nvme_ctrlr, 5050 struct spdk_nvme_ctrlr *new_ctrlr) 5051 { 5052 struct nvme_ns *nvme_ns; 5053 struct spdk_nvme_ns *new_ns; 5054 5055 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 5056 while (nvme_ns != NULL) { 5057 new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id); 5058 assert(new_ns != NULL); 5059 5060 if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) { 5061 return -EINVAL; 5062 } 5063 5064 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 5065 } 5066 5067 return 0; 5068 } 5069 5070 static int 5071 _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 5072 struct spdk_nvme_transport_id *trid) 5073 { 5074 struct nvme_path_id *new_trid, *tmp_trid; 5075 5076 new_trid = calloc(1, sizeof(*new_trid)); 5077 if (new_trid == NULL) { 5078 return -ENOMEM; 5079 } 5080 new_trid->trid = *trid; 5081 new_trid->is_failed = false; 5082 5083 TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) { 5084 if (tmp_trid->is_failed && tmp_trid != nvme_ctrlr->active_path_id) { 5085 TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link); 5086 return 0; 5087 } 5088 } 5089 5090 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link); 5091 return 0; 5092 } 5093 5094 /* This is the case that a secondary path is added to an existing 5095 * nvme_ctrlr for failover. After checking if it can access the same 5096 * namespaces as the primary path, it is disconnected until failover occurs. 5097 */ 5098 static int 5099 bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 5100 struct spdk_nvme_ctrlr *new_ctrlr, 5101 struct spdk_nvme_transport_id *trid) 5102 { 5103 int rc; 5104 5105 assert(nvme_ctrlr != NULL); 5106 5107 pthread_mutex_lock(&nvme_ctrlr->mutex); 5108 5109 rc = bdev_nvme_check_secondary_trid(nvme_ctrlr, new_ctrlr, trid); 5110 if (rc != 0) { 5111 goto exit; 5112 } 5113 5114 rc = bdev_nvme_check_secondary_namespace(nvme_ctrlr, new_ctrlr); 5115 if (rc != 0) { 5116 goto exit; 5117 } 5118 5119 rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid); 5120 5121 exit: 5122 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5123 5124 spdk_nvme_detach(new_ctrlr); 5125 5126 return rc; 5127 } 5128 5129 static void 5130 connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 5131 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 5132 { 5133 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 5134 struct nvme_async_probe_ctx *ctx; 5135 int rc; 5136 5137 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 5138 ctx->ctrlr_attached = true; 5139 5140 rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx); 5141 if (rc != 0) { 5142 populate_namespaces_cb(ctx, 0, rc); 5143 } 5144 } 5145 5146 static void 5147 connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 5148 struct spdk_nvme_ctrlr *ctrlr, 5149 const struct spdk_nvme_ctrlr_opts *opts) 5150 { 5151 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 5152 struct nvme_ctrlr *nvme_ctrlr; 5153 struct nvme_async_probe_ctx *ctx; 5154 int rc; 5155 5156 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 5157 ctx->ctrlr_attached = true; 5158 5159 nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name); 5160 if (nvme_ctrlr) { 5161 rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid); 5162 } else { 5163 rc = -ENODEV; 5164 } 5165 5166 populate_namespaces_cb(ctx, 0, rc); 5167 } 5168 5169 static int 5170 bdev_nvme_async_poll(void *arg) 5171 { 5172 struct nvme_async_probe_ctx *ctx = arg; 5173 int rc; 5174 5175 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 5176 if (spdk_unlikely(rc != -EAGAIN)) { 5177 ctx->probe_done = true; 5178 spdk_poller_unregister(&ctx->poller); 5179 if (!ctx->ctrlr_attached) { 5180 /* The probe is done, but no controller was attached. 5181 * That means we had a failure, so report -EIO back to 5182 * the caller (usually the RPC). populate_namespaces_cb() 5183 * will take care of freeing the nvme_async_probe_ctx. 5184 */ 5185 populate_namespaces_cb(ctx, 0, -EIO); 5186 } else if (ctx->namespaces_populated) { 5187 /* The namespaces for the attached controller were all 5188 * populated and the response was already sent to the 5189 * caller (usually the RPC). So free the context here. 5190 */ 5191 free(ctx); 5192 } 5193 } 5194 5195 return SPDK_POLLER_BUSY; 5196 } 5197 5198 static bool 5199 bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 5200 uint32_t reconnect_delay_sec, 5201 uint32_t fast_io_fail_timeout_sec) 5202 { 5203 if (ctrlr_loss_timeout_sec < -1) { 5204 SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n"); 5205 return false; 5206 } else if (ctrlr_loss_timeout_sec == -1) { 5207 if (reconnect_delay_sec == 0) { 5208 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 5209 return false; 5210 } else if (fast_io_fail_timeout_sec != 0 && 5211 fast_io_fail_timeout_sec < reconnect_delay_sec) { 5212 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n"); 5213 return false; 5214 } 5215 } else if (ctrlr_loss_timeout_sec != 0) { 5216 if (reconnect_delay_sec == 0) { 5217 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 5218 return false; 5219 } else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) { 5220 SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n"); 5221 return false; 5222 } else if (fast_io_fail_timeout_sec != 0) { 5223 if (fast_io_fail_timeout_sec < reconnect_delay_sec) { 5224 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n"); 5225 return false; 5226 } else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) { 5227 SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n"); 5228 return false; 5229 } 5230 } 5231 } else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) { 5232 SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n"); 5233 return false; 5234 } 5235 5236 return true; 5237 } 5238 5239 int 5240 bdev_nvme_create(struct spdk_nvme_transport_id *trid, 5241 const char *base_name, 5242 const char **names, 5243 uint32_t count, 5244 spdk_bdev_create_nvme_fn cb_fn, 5245 void *cb_ctx, 5246 struct spdk_nvme_ctrlr_opts *drv_opts, 5247 struct nvme_ctrlr_opts *bdev_opts, 5248 bool multipath) 5249 { 5250 struct nvme_probe_skip_entry *entry, *tmp; 5251 struct nvme_async_probe_ctx *ctx; 5252 spdk_nvme_attach_cb attach_cb; 5253 5254 /* TODO expand this check to include both the host and target TRIDs. 5255 * Only if both are the same should we fail. 5256 */ 5257 if (nvme_ctrlr_get(trid) != NULL) { 5258 SPDK_ERRLOG("A controller with the provided trid (traddr: %s) already exists.\n", trid->traddr); 5259 return -EEXIST; 5260 } 5261 5262 if (bdev_opts != NULL && 5263 !bdev_nvme_check_io_error_resiliency_params(bdev_opts->ctrlr_loss_timeout_sec, 5264 bdev_opts->reconnect_delay_sec, 5265 bdev_opts->fast_io_fail_timeout_sec)) { 5266 return -EINVAL; 5267 } 5268 5269 ctx = calloc(1, sizeof(*ctx)); 5270 if (!ctx) { 5271 return -ENOMEM; 5272 } 5273 ctx->base_name = base_name; 5274 ctx->names = names; 5275 ctx->count = count; 5276 ctx->cb_fn = cb_fn; 5277 ctx->cb_ctx = cb_ctx; 5278 ctx->trid = *trid; 5279 5280 if (bdev_opts) { 5281 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 5282 } else { 5283 bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts); 5284 } 5285 5286 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 5287 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) { 5288 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 5289 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 5290 free(entry); 5291 break; 5292 } 5293 } 5294 } 5295 5296 if (drv_opts) { 5297 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 5298 } else { 5299 spdk_nvme_ctrlr_get_default_ctrlr_opts(&ctx->drv_opts, sizeof(ctx->drv_opts)); 5300 } 5301 5302 ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count; 5303 ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout; 5304 ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms; 5305 ctx->drv_opts.disable_read_ana_log_page = true; 5306 ctx->drv_opts.transport_tos = g_opts.transport_tos; 5307 5308 if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || multipath) { 5309 attach_cb = connect_attach_cb; 5310 } else { 5311 attach_cb = connect_set_failover_cb; 5312 } 5313 5314 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb); 5315 if (ctx->probe_ctx == NULL) { 5316 SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr); 5317 free(ctx); 5318 return -ENODEV; 5319 } 5320 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000); 5321 5322 return 0; 5323 } 5324 5325 static bool 5326 nvme_path_should_delete(struct nvme_path_id *p, const struct nvme_path_id *path_id) 5327 { 5328 if (path_id->trid.trtype != 0) { 5329 if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) { 5330 if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) { 5331 return false; 5332 } 5333 } else { 5334 if (path_id->trid.trtype != p->trid.trtype) { 5335 return false; 5336 } 5337 } 5338 } 5339 5340 if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) { 5341 if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) { 5342 return false; 5343 } 5344 } 5345 5346 if (path_id->trid.adrfam != 0) { 5347 if (path_id->trid.adrfam != p->trid.adrfam) { 5348 return false; 5349 } 5350 } 5351 5352 if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) { 5353 if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) { 5354 return false; 5355 } 5356 } 5357 5358 if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) { 5359 if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) { 5360 return false; 5361 } 5362 } 5363 5364 if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) { 5365 if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) { 5366 return false; 5367 } 5368 } 5369 5370 if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) { 5371 if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) { 5372 return false; 5373 } 5374 } 5375 5376 return true; 5377 } 5378 5379 static int 5380 _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id) 5381 { 5382 struct nvme_path_id *p, *t; 5383 spdk_msg_fn msg_fn; 5384 int rc = -ENXIO; 5385 5386 pthread_mutex_lock(&nvme_ctrlr->mutex); 5387 5388 TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) { 5389 if (p == TAILQ_FIRST(&nvme_ctrlr->trids)) { 5390 break; 5391 } 5392 5393 if (!nvme_path_should_delete(p, path_id)) { 5394 continue; 5395 } 5396 5397 /* We are not using the specified path. */ 5398 TAILQ_REMOVE(&nvme_ctrlr->trids, p, link); 5399 free(p); 5400 rc = 0; 5401 } 5402 5403 if (p == NULL || !nvme_path_should_delete(p, path_id)) { 5404 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5405 return rc; 5406 } 5407 5408 /* If we made it here, then this path is a match! Now we need to remove it. */ 5409 5410 /* This is the active path in use right now. The active path is always the first in the list. */ 5411 assert(p == nvme_ctrlr->active_path_id); 5412 5413 if (!TAILQ_NEXT(p, link)) { 5414 /* The current path is the only path. */ 5415 msg_fn = _nvme_ctrlr_destruct; 5416 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, false); 5417 } else { 5418 /* There is an alternative path. */ 5419 msg_fn = _bdev_nvme_reset; 5420 rc = bdev_nvme_failover_unsafe(nvme_ctrlr, true); 5421 } 5422 5423 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5424 5425 if (rc == 0) { 5426 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 5427 } else if (rc == -EALREADY) { 5428 rc = 0; 5429 } 5430 5431 return rc; 5432 } 5433 5434 int 5435 bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id) 5436 { 5437 struct nvme_bdev_ctrlr *nbdev_ctrlr; 5438 struct nvme_ctrlr *nvme_ctrlr, *tmp_nvme_ctrlr; 5439 int rc = -ENXIO, _rc; 5440 5441 if (name == NULL || path_id == NULL) { 5442 return -EINVAL; 5443 } 5444 5445 pthread_mutex_lock(&g_bdev_nvme_mutex); 5446 5447 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 5448 if (nbdev_ctrlr == NULL) { 5449 pthread_mutex_unlock(&g_bdev_nvme_mutex); 5450 5451 SPDK_ERRLOG("Failed to find NVMe bdev controller\n"); 5452 return -ENODEV; 5453 } 5454 5455 TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) { 5456 _rc = _bdev_nvme_delete(nvme_ctrlr, path_id); 5457 if (_rc < 0 && _rc != -ENXIO) { 5458 pthread_mutex_unlock(&g_bdev_nvme_mutex); 5459 5460 return _rc; 5461 } else if (_rc == 0) { 5462 /* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr 5463 * was deleted successfully. To remember the successful deletion, 5464 * overwrite rc only if _rc is zero. 5465 */ 5466 rc = 0; 5467 } 5468 } 5469 5470 pthread_mutex_unlock(&g_bdev_nvme_mutex); 5471 5472 /* All nvme_ctrlrs were deleted or no nvme_ctrlr which had the trid was found. */ 5473 return rc; 5474 } 5475 5476 #define DISCOVERY_INFOLOG(ctx, format, ...) \ 5477 SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 5478 5479 #define DISCOVERY_ERRLOG(ctx, format, ...) \ 5480 SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 5481 5482 struct discovery_entry_ctx { 5483 char name[128]; 5484 struct spdk_nvme_transport_id trid; 5485 struct spdk_nvme_ctrlr_opts drv_opts; 5486 struct spdk_nvmf_discovery_log_page_entry entry; 5487 TAILQ_ENTRY(discovery_entry_ctx) tailq; 5488 struct discovery_ctx *ctx; 5489 }; 5490 5491 struct discovery_ctx { 5492 char *name; 5493 spdk_bdev_nvme_start_discovery_fn start_cb_fn; 5494 spdk_bdev_nvme_stop_discovery_fn stop_cb_fn; 5495 void *cb_ctx; 5496 struct spdk_nvme_probe_ctx *probe_ctx; 5497 struct spdk_nvme_detach_ctx *detach_ctx; 5498 struct spdk_nvme_ctrlr *ctrlr; 5499 struct spdk_nvme_transport_id trid; 5500 struct discovery_entry_ctx *entry_ctx_in_use; 5501 struct spdk_poller *poller; 5502 struct spdk_nvme_ctrlr_opts drv_opts; 5503 struct nvme_ctrlr_opts bdev_opts; 5504 struct spdk_nvmf_discovery_log_page *log_page; 5505 TAILQ_ENTRY(discovery_ctx) tailq; 5506 TAILQ_HEAD(, discovery_entry_ctx) nvm_entry_ctxs; 5507 TAILQ_HEAD(, discovery_entry_ctx) discovery_entry_ctxs; 5508 int rc; 5509 bool wait_for_attach; 5510 uint64_t timeout_ticks; 5511 /* Denotes that the discovery service is being started. We're waiting 5512 * for the initial connection to the discovery controller to be 5513 * established and attach discovered NVM ctrlrs. 5514 */ 5515 bool initializing; 5516 /* Denotes if a discovery is currently in progress for this context. 5517 * That includes connecting to newly discovered subsystems. Used to 5518 * ensure we do not start a new discovery until an existing one is 5519 * complete. 5520 */ 5521 bool in_progress; 5522 5523 /* Denotes if another discovery is needed after the one in progress 5524 * completes. Set when we receive an AER completion while a discovery 5525 * is already in progress. 5526 */ 5527 bool pending; 5528 5529 /* Signal to the discovery context poller that it should stop the 5530 * discovery service, including detaching from the current discovery 5531 * controller. 5532 */ 5533 bool stop; 5534 5535 struct spdk_thread *calling_thread; 5536 uint32_t index; 5537 uint32_t attach_in_progress; 5538 char *hostnqn; 5539 5540 /* Denotes if the discovery service was started by the mdns discovery. 5541 */ 5542 bool from_mdns_discovery_service; 5543 }; 5544 5545 TAILQ_HEAD(discovery_ctxs, discovery_ctx); 5546 static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs); 5547 5548 static void get_discovery_log_page(struct discovery_ctx *ctx); 5549 5550 static void 5551 free_discovery_ctx(struct discovery_ctx *ctx) 5552 { 5553 free(ctx->log_page); 5554 free(ctx->hostnqn); 5555 free(ctx->name); 5556 free(ctx); 5557 } 5558 5559 static void 5560 discovery_complete(struct discovery_ctx *ctx) 5561 { 5562 ctx->initializing = false; 5563 ctx->in_progress = false; 5564 if (ctx->pending) { 5565 ctx->pending = false; 5566 get_discovery_log_page(ctx); 5567 } 5568 } 5569 5570 static void 5571 build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid, 5572 struct spdk_nvmf_discovery_log_page_entry *entry) 5573 { 5574 char *space; 5575 5576 trid->trtype = entry->trtype; 5577 trid->adrfam = entry->adrfam; 5578 memcpy(trid->traddr, entry->traddr, sizeof(entry->traddr)); 5579 memcpy(trid->trsvcid, entry->trsvcid, sizeof(entry->trsvcid)); 5580 memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn)); 5581 5582 /* We want the traddr, trsvcid and subnqn fields to be NULL-terminated. 5583 * But the log page entries typically pad them with spaces, not zeroes. 5584 * So add a NULL terminator to each of these fields at the appropriate 5585 * location. 5586 */ 5587 space = strchr(trid->traddr, ' '); 5588 if (space) { 5589 *space = 0; 5590 } 5591 space = strchr(trid->trsvcid, ' '); 5592 if (space) { 5593 *space = 0; 5594 } 5595 space = strchr(trid->subnqn, ' '); 5596 if (space) { 5597 *space = 0; 5598 } 5599 } 5600 5601 static void 5602 stop_discovery(struct discovery_ctx *ctx, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 5603 { 5604 ctx->stop = true; 5605 ctx->stop_cb_fn = cb_fn; 5606 ctx->cb_ctx = cb_ctx; 5607 5608 while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) { 5609 struct discovery_entry_ctx *entry_ctx; 5610 struct nvme_path_id path = {}; 5611 5612 entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs); 5613 path.trid = entry_ctx->trid; 5614 bdev_nvme_delete(entry_ctx->name, &path); 5615 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 5616 free(entry_ctx); 5617 } 5618 5619 while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) { 5620 struct discovery_entry_ctx *entry_ctx; 5621 5622 entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 5623 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 5624 free(entry_ctx); 5625 } 5626 5627 free(ctx->entry_ctx_in_use); 5628 ctx->entry_ctx_in_use = NULL; 5629 } 5630 5631 static void 5632 discovery_remove_controllers(struct discovery_ctx *ctx) 5633 { 5634 struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page; 5635 struct discovery_entry_ctx *entry_ctx, *tmp; 5636 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 5637 struct spdk_nvme_transport_id old_trid; 5638 uint64_t numrec, i; 5639 bool found; 5640 5641 numrec = from_le64(&log_page->numrec); 5642 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) { 5643 found = false; 5644 old_entry = &entry_ctx->entry; 5645 build_trid_from_log_page_entry(&old_trid, old_entry); 5646 for (i = 0; i < numrec; i++) { 5647 new_entry = &log_page->entries[i]; 5648 if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) { 5649 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n", 5650 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 5651 found = true; 5652 break; 5653 } 5654 } 5655 if (!found) { 5656 struct nvme_path_id path = {}; 5657 5658 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n", 5659 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 5660 5661 path.trid = entry_ctx->trid; 5662 bdev_nvme_delete(entry_ctx->name, &path); 5663 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 5664 free(entry_ctx); 5665 } 5666 } 5667 free(log_page); 5668 ctx->log_page = NULL; 5669 discovery_complete(ctx); 5670 } 5671 5672 static void 5673 complete_discovery_start(struct discovery_ctx *ctx, int status) 5674 { 5675 ctx->timeout_ticks = 0; 5676 ctx->rc = status; 5677 if (ctx->start_cb_fn) { 5678 ctx->start_cb_fn(ctx->cb_ctx, status); 5679 ctx->start_cb_fn = NULL; 5680 ctx->cb_ctx = NULL; 5681 } 5682 } 5683 5684 static void 5685 discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc) 5686 { 5687 struct discovery_entry_ctx *entry_ctx = cb_ctx; 5688 struct discovery_ctx *ctx = entry_ctx->ctx; 5689 5690 DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name); 5691 ctx->attach_in_progress--; 5692 if (ctx->attach_in_progress == 0) { 5693 complete_discovery_start(ctx, ctx->rc); 5694 if (ctx->initializing && ctx->rc != 0) { 5695 DISCOVERY_ERRLOG(ctx, "stopping discovery due to errors: %d\n", ctx->rc); 5696 stop_discovery(ctx, NULL, ctx->cb_ctx); 5697 } else { 5698 discovery_remove_controllers(ctx); 5699 } 5700 } 5701 } 5702 5703 static struct discovery_entry_ctx * 5704 create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid) 5705 { 5706 struct discovery_entry_ctx *new_ctx; 5707 5708 new_ctx = calloc(1, sizeof(*new_ctx)); 5709 if (new_ctx == NULL) { 5710 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 5711 return NULL; 5712 } 5713 5714 new_ctx->ctx = ctx; 5715 memcpy(&new_ctx->trid, trid, sizeof(*trid)); 5716 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 5717 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 5718 return new_ctx; 5719 } 5720 5721 static void 5722 discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl, 5723 struct spdk_nvmf_discovery_log_page *log_page) 5724 { 5725 struct discovery_ctx *ctx = cb_arg; 5726 struct discovery_entry_ctx *entry_ctx, *tmp; 5727 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 5728 uint64_t numrec, i; 5729 bool found; 5730 5731 if (rc || spdk_nvme_cpl_is_error(cpl)) { 5732 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 5733 return; 5734 } 5735 5736 ctx->log_page = log_page; 5737 assert(ctx->attach_in_progress == 0); 5738 numrec = from_le64(&log_page->numrec); 5739 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) { 5740 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 5741 free(entry_ctx); 5742 } 5743 for (i = 0; i < numrec; i++) { 5744 found = false; 5745 new_entry = &log_page->entries[i]; 5746 if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 5747 struct discovery_entry_ctx *new_ctx; 5748 struct spdk_nvme_transport_id trid = {}; 5749 5750 build_trid_from_log_page_entry(&trid, new_entry); 5751 new_ctx = create_discovery_entry_ctx(ctx, &trid); 5752 if (new_ctx == NULL) { 5753 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 5754 break; 5755 } 5756 5757 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq); 5758 continue; 5759 } 5760 TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) { 5761 old_entry = &entry_ctx->entry; 5762 if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) { 5763 found = true; 5764 break; 5765 } 5766 } 5767 if (!found) { 5768 struct discovery_entry_ctx *subnqn_ctx = NULL, *new_ctx; 5769 struct discovery_ctx *d_ctx; 5770 5771 TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) { 5772 TAILQ_FOREACH(subnqn_ctx, &d_ctx->nvm_entry_ctxs, tailq) { 5773 if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn, 5774 sizeof(new_entry->subnqn))) { 5775 break; 5776 } 5777 } 5778 if (subnqn_ctx) { 5779 break; 5780 } 5781 } 5782 5783 new_ctx = calloc(1, sizeof(*new_ctx)); 5784 if (new_ctx == NULL) { 5785 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 5786 break; 5787 } 5788 5789 new_ctx->ctx = ctx; 5790 memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry)); 5791 build_trid_from_log_page_entry(&new_ctx->trid, new_entry); 5792 if (subnqn_ctx) { 5793 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name); 5794 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n", 5795 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 5796 new_ctx->name); 5797 } else { 5798 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++); 5799 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n", 5800 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 5801 new_ctx->name); 5802 } 5803 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 5804 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 5805 rc = bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0, 5806 discovery_attach_controller_done, new_ctx, 5807 &new_ctx->drv_opts, &ctx->bdev_opts, true); 5808 if (rc == 0) { 5809 TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq); 5810 ctx->attach_in_progress++; 5811 } else { 5812 DISCOVERY_ERRLOG(ctx, "bdev_nvme_create failed (%s)\n", spdk_strerror(-rc)); 5813 } 5814 } 5815 } 5816 5817 if (ctx->attach_in_progress == 0) { 5818 discovery_remove_controllers(ctx); 5819 } 5820 } 5821 5822 static void 5823 get_discovery_log_page(struct discovery_ctx *ctx) 5824 { 5825 int rc; 5826 5827 assert(ctx->in_progress == false); 5828 ctx->in_progress = true; 5829 rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx); 5830 if (rc != 0) { 5831 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 5832 } 5833 DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n"); 5834 } 5835 5836 static void 5837 discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 5838 { 5839 struct discovery_ctx *ctx = arg; 5840 uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16; 5841 5842 if (spdk_nvme_cpl_is_error(cpl)) { 5843 DISCOVERY_ERRLOG(ctx, "aer failed\n"); 5844 return; 5845 } 5846 5847 if (log_page_id != SPDK_NVME_LOG_DISCOVERY) { 5848 DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id); 5849 return; 5850 } 5851 5852 DISCOVERY_INFOLOG(ctx, "got aer\n"); 5853 if (ctx->in_progress) { 5854 ctx->pending = true; 5855 return; 5856 } 5857 5858 get_discovery_log_page(ctx); 5859 } 5860 5861 static void 5862 discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 5863 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 5864 { 5865 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 5866 struct discovery_ctx *ctx; 5867 5868 ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts); 5869 5870 DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n"); 5871 ctx->probe_ctx = NULL; 5872 ctx->ctrlr = ctrlr; 5873 5874 if (ctx->rc != 0) { 5875 DISCOVERY_ERRLOG(ctx, "encountered error while attaching discovery ctrlr: %d\n", 5876 ctx->rc); 5877 return; 5878 } 5879 5880 spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx); 5881 } 5882 5883 static int 5884 discovery_poller(void *arg) 5885 { 5886 struct discovery_ctx *ctx = arg; 5887 struct spdk_nvme_transport_id *trid; 5888 int rc; 5889 5890 if (ctx->detach_ctx) { 5891 rc = spdk_nvme_detach_poll_async(ctx->detach_ctx); 5892 if (rc != -EAGAIN) { 5893 ctx->detach_ctx = NULL; 5894 ctx->ctrlr = NULL; 5895 } 5896 } else if (ctx->stop) { 5897 if (ctx->ctrlr != NULL) { 5898 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 5899 if (rc == 0) { 5900 return SPDK_POLLER_BUSY; 5901 } 5902 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 5903 } 5904 spdk_poller_unregister(&ctx->poller); 5905 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 5906 assert(ctx->start_cb_fn == NULL); 5907 if (ctx->stop_cb_fn != NULL) { 5908 ctx->stop_cb_fn(ctx->cb_ctx); 5909 } 5910 free_discovery_ctx(ctx); 5911 } else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) { 5912 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 5913 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 5914 assert(ctx->initializing); 5915 spdk_poller_unregister(&ctx->poller); 5916 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 5917 complete_discovery_start(ctx, -ETIMEDOUT); 5918 stop_discovery(ctx, NULL, NULL); 5919 free_discovery_ctx(ctx); 5920 return SPDK_POLLER_BUSY; 5921 } 5922 5923 assert(ctx->entry_ctx_in_use == NULL); 5924 ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 5925 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 5926 trid = &ctx->entry_ctx_in_use->trid; 5927 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb); 5928 if (ctx->probe_ctx) { 5929 spdk_poller_unregister(&ctx->poller); 5930 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000); 5931 } else { 5932 DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n"); 5933 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 5934 ctx->entry_ctx_in_use = NULL; 5935 } 5936 } else if (ctx->probe_ctx) { 5937 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 5938 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 5939 complete_discovery_start(ctx, -ETIMEDOUT); 5940 return SPDK_POLLER_BUSY; 5941 } 5942 5943 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 5944 if (rc != -EAGAIN) { 5945 if (ctx->rc != 0) { 5946 assert(ctx->initializing); 5947 stop_discovery(ctx, NULL, ctx->cb_ctx); 5948 } else { 5949 assert(rc == 0); 5950 DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n"); 5951 ctx->rc = rc; 5952 get_discovery_log_page(ctx); 5953 } 5954 } 5955 } else { 5956 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 5957 DISCOVERY_ERRLOG(ctx, "timed out while attaching NVM ctrlrs\n"); 5958 complete_discovery_start(ctx, -ETIMEDOUT); 5959 /* We need to wait until all NVM ctrlrs are attached before we stop the 5960 * discovery service to make sure we don't detach a ctrlr that is still 5961 * being attached. 5962 */ 5963 if (ctx->attach_in_progress == 0) { 5964 stop_discovery(ctx, NULL, ctx->cb_ctx); 5965 return SPDK_POLLER_BUSY; 5966 } 5967 } 5968 5969 rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr); 5970 if (rc < 0) { 5971 spdk_poller_unregister(&ctx->poller); 5972 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 5973 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 5974 ctx->entry_ctx_in_use = NULL; 5975 5976 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 5977 if (rc != 0) { 5978 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 5979 ctx->ctrlr = NULL; 5980 } 5981 } 5982 } 5983 5984 return SPDK_POLLER_BUSY; 5985 } 5986 5987 static void 5988 start_discovery_poller(void *arg) 5989 { 5990 struct discovery_ctx *ctx = arg; 5991 5992 TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq); 5993 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 5994 } 5995 5996 int 5997 bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid, 5998 const char *base_name, 5999 struct spdk_nvme_ctrlr_opts *drv_opts, 6000 struct nvme_ctrlr_opts *bdev_opts, 6001 uint64_t attach_timeout, 6002 bool from_mdns, 6003 spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx) 6004 { 6005 struct discovery_ctx *ctx; 6006 struct discovery_entry_ctx *discovery_entry_ctx; 6007 6008 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 6009 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 6010 if (strcmp(ctx->name, base_name) == 0) { 6011 return -EEXIST; 6012 } 6013 6014 if (ctx->entry_ctx_in_use != NULL) { 6015 if (!spdk_nvme_transport_id_compare(trid, &ctx->entry_ctx_in_use->trid)) { 6016 return -EEXIST; 6017 } 6018 } 6019 6020 TAILQ_FOREACH(discovery_entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 6021 if (!spdk_nvme_transport_id_compare(trid, &discovery_entry_ctx->trid)) { 6022 return -EEXIST; 6023 } 6024 } 6025 } 6026 6027 ctx = calloc(1, sizeof(*ctx)); 6028 if (ctx == NULL) { 6029 return -ENOMEM; 6030 } 6031 6032 ctx->name = strdup(base_name); 6033 if (ctx->name == NULL) { 6034 free_discovery_ctx(ctx); 6035 return -ENOMEM; 6036 } 6037 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 6038 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 6039 ctx->from_mdns_discovery_service = from_mdns; 6040 ctx->bdev_opts.from_discovery_service = true; 6041 ctx->calling_thread = spdk_get_thread(); 6042 ctx->start_cb_fn = cb_fn; 6043 ctx->cb_ctx = cb_ctx; 6044 ctx->initializing = true; 6045 if (ctx->start_cb_fn) { 6046 /* We can use this when dumping json to denote if this RPC parameter 6047 * was specified or not. 6048 */ 6049 ctx->wait_for_attach = true; 6050 } 6051 if (attach_timeout != 0) { 6052 ctx->timeout_ticks = spdk_get_ticks() + attach_timeout * 6053 spdk_get_ticks_hz() / 1000ull; 6054 } 6055 TAILQ_INIT(&ctx->nvm_entry_ctxs); 6056 TAILQ_INIT(&ctx->discovery_entry_ctxs); 6057 memcpy(&ctx->trid, trid, sizeof(*trid)); 6058 /* Even if user did not specify hostnqn, we can still strdup("\0"); */ 6059 ctx->hostnqn = strdup(ctx->drv_opts.hostnqn); 6060 if (ctx->hostnqn == NULL) { 6061 free_discovery_ctx(ctx); 6062 return -ENOMEM; 6063 } 6064 discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid); 6065 if (discovery_entry_ctx == NULL) { 6066 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 6067 free_discovery_ctx(ctx); 6068 return -ENOMEM; 6069 } 6070 6071 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq); 6072 spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx); 6073 return 0; 6074 } 6075 6076 int 6077 bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 6078 { 6079 struct discovery_ctx *ctx; 6080 6081 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 6082 if (strcmp(name, ctx->name) == 0) { 6083 if (ctx->stop) { 6084 return -EALREADY; 6085 } 6086 /* If we're still starting the discovery service and ->rc is non-zero, we're 6087 * going to stop it as soon as we can 6088 */ 6089 if (ctx->initializing && ctx->rc != 0) { 6090 return -EALREADY; 6091 } 6092 stop_discovery(ctx, cb_fn, cb_ctx); 6093 return 0; 6094 } 6095 } 6096 6097 return -ENOENT; 6098 } 6099 6100 static int 6101 bdev_nvme_library_init(void) 6102 { 6103 g_bdev_nvme_init_thread = spdk_get_thread(); 6104 6105 spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb, 6106 bdev_nvme_destroy_poll_group_cb, 6107 sizeof(struct nvme_poll_group), "nvme_poll_groups"); 6108 6109 return 0; 6110 } 6111 6112 static void 6113 bdev_nvme_fini_destruct_ctrlrs(void) 6114 { 6115 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6116 struct nvme_ctrlr *nvme_ctrlr; 6117 6118 pthread_mutex_lock(&g_bdev_nvme_mutex); 6119 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 6120 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 6121 pthread_mutex_lock(&nvme_ctrlr->mutex); 6122 if (nvme_ctrlr->destruct) { 6123 /* This controller's destruction was already started 6124 * before the application started shutting down 6125 */ 6126 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6127 continue; 6128 } 6129 nvme_ctrlr->destruct = true; 6130 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6131 6132 spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct, 6133 nvme_ctrlr); 6134 } 6135 } 6136 6137 g_bdev_nvme_module_finish = true; 6138 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 6139 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6140 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 6141 spdk_bdev_module_fini_done(); 6142 return; 6143 } 6144 6145 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6146 } 6147 6148 static void 6149 check_discovery_fini(void *arg) 6150 { 6151 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 6152 bdev_nvme_fini_destruct_ctrlrs(); 6153 } 6154 } 6155 6156 static void 6157 bdev_nvme_library_fini(void) 6158 { 6159 struct nvme_probe_skip_entry *entry, *entry_tmp; 6160 struct discovery_ctx *ctx; 6161 6162 spdk_poller_unregister(&g_hotplug_poller); 6163 free(g_hotplug_probe_ctx); 6164 g_hotplug_probe_ctx = NULL; 6165 6166 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) { 6167 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 6168 free(entry); 6169 } 6170 6171 assert(spdk_get_thread() == g_bdev_nvme_init_thread); 6172 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 6173 bdev_nvme_fini_destruct_ctrlrs(); 6174 } else { 6175 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 6176 stop_discovery(ctx, check_discovery_fini, NULL); 6177 } 6178 } 6179 } 6180 6181 static void 6182 bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio) 6183 { 6184 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6185 struct spdk_bdev *bdev = bdev_io->bdev; 6186 struct spdk_dif_ctx dif_ctx; 6187 struct spdk_dif_error err_blk = {}; 6188 int rc; 6189 6190 rc = spdk_dif_ctx_init(&dif_ctx, 6191 bdev->blocklen, bdev->md_len, bdev->md_interleave, 6192 bdev->dif_is_head_of_md, bdev->dif_type, bdev->dif_check_flags, 6193 bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0); 6194 if (rc != 0) { 6195 SPDK_ERRLOG("Initialization of DIF context failed\n"); 6196 return; 6197 } 6198 6199 if (bdev->md_interleave) { 6200 rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 6201 bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 6202 } else { 6203 struct iovec md_iov = { 6204 .iov_base = bdev_io->u.bdev.md_buf, 6205 .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len, 6206 }; 6207 6208 rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 6209 &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 6210 } 6211 6212 if (rc != 0) { 6213 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", 6214 err_blk.err_type, err_blk.err_offset); 6215 } else { 6216 SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n"); 6217 } 6218 } 6219 6220 static void 6221 bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 6222 { 6223 struct nvme_bdev_io *bio = ref; 6224 6225 if (spdk_nvme_cpl_is_success(cpl)) { 6226 /* Run PI verification for read data buffer. */ 6227 bdev_nvme_verify_pi_error(bio); 6228 } 6229 6230 /* Return original completion status */ 6231 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 6232 } 6233 6234 static void 6235 bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 6236 { 6237 struct nvme_bdev_io *bio = ref; 6238 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6239 int ret; 6240 6241 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 6242 SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n", 6243 cpl->status.sct, cpl->status.sc); 6244 6245 /* Save completion status to use after verifying PI error. */ 6246 bio->cpl = *cpl; 6247 6248 if (spdk_likely(nvme_io_path_is_available(bio->io_path))) { 6249 /* Read without PI checking to verify PI error. */ 6250 ret = bdev_nvme_no_pi_readv(bio, 6251 bdev_io->u.bdev.iovs, 6252 bdev_io->u.bdev.iovcnt, 6253 bdev_io->u.bdev.md_buf, 6254 bdev_io->u.bdev.num_blocks, 6255 bdev_io->u.bdev.offset_blocks); 6256 if (ret == 0) { 6257 return; 6258 } 6259 } 6260 } 6261 6262 bdev_nvme_io_complete_nvme_status(bio, cpl); 6263 } 6264 6265 static void 6266 bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 6267 { 6268 struct nvme_bdev_io *bio = ref; 6269 6270 if (spdk_nvme_cpl_is_pi_error(cpl)) { 6271 SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n", 6272 cpl->status.sct, cpl->status.sc); 6273 /* Run PI verification for write data buffer if PI error is detected. */ 6274 bdev_nvme_verify_pi_error(bio); 6275 } 6276 6277 bdev_nvme_io_complete_nvme_status(bio, cpl); 6278 } 6279 6280 static void 6281 bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl) 6282 { 6283 struct nvme_bdev_io *bio = ref; 6284 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6285 6286 /* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks. 6287 * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error(). 6288 */ 6289 bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0; 6290 6291 if (spdk_nvme_cpl_is_pi_error(cpl)) { 6292 SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n", 6293 cpl->status.sct, cpl->status.sc); 6294 /* Run PI verification for zone append data buffer if PI error is detected. */ 6295 bdev_nvme_verify_pi_error(bio); 6296 } 6297 6298 bdev_nvme_io_complete_nvme_status(bio, cpl); 6299 } 6300 6301 static void 6302 bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl) 6303 { 6304 struct nvme_bdev_io *bio = ref; 6305 6306 if (spdk_nvme_cpl_is_pi_error(cpl)) { 6307 SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n", 6308 cpl->status.sct, cpl->status.sc); 6309 /* Run PI verification for compare data buffer if PI error is detected. */ 6310 bdev_nvme_verify_pi_error(bio); 6311 } 6312 6313 bdev_nvme_io_complete_nvme_status(bio, cpl); 6314 } 6315 6316 static void 6317 bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 6318 { 6319 struct nvme_bdev_io *bio = ref; 6320 6321 /* Compare operation completion */ 6322 if (!bio->first_fused_completed) { 6323 /* Save compare result for write callback */ 6324 bio->cpl = *cpl; 6325 bio->first_fused_completed = true; 6326 return; 6327 } 6328 6329 /* Write operation completion */ 6330 if (spdk_nvme_cpl_is_error(&bio->cpl)) { 6331 /* If bio->cpl is already an error, it means the compare operation failed. In that case, 6332 * complete the IO with the compare operation's status. 6333 */ 6334 if (!spdk_nvme_cpl_is_error(cpl)) { 6335 SPDK_ERRLOG("Unexpected write success after compare failure.\n"); 6336 } 6337 6338 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 6339 } else { 6340 bdev_nvme_io_complete_nvme_status(bio, cpl); 6341 } 6342 } 6343 6344 static void 6345 bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl) 6346 { 6347 struct nvme_bdev_io *bio = ref; 6348 6349 bdev_nvme_io_complete_nvme_status(bio, cpl); 6350 } 6351 6352 static int 6353 fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc) 6354 { 6355 switch (desc->zt) { 6356 case SPDK_NVME_ZONE_TYPE_SEQWR: 6357 info->type = SPDK_BDEV_ZONE_TYPE_SEQWR; 6358 break; 6359 default: 6360 SPDK_ERRLOG("Invalid zone type: %#x in zone report\n", desc->zt); 6361 return -EIO; 6362 } 6363 6364 switch (desc->zs) { 6365 case SPDK_NVME_ZONE_STATE_EMPTY: 6366 info->state = SPDK_BDEV_ZONE_STATE_EMPTY; 6367 break; 6368 case SPDK_NVME_ZONE_STATE_IOPEN: 6369 info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN; 6370 break; 6371 case SPDK_NVME_ZONE_STATE_EOPEN: 6372 info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN; 6373 break; 6374 case SPDK_NVME_ZONE_STATE_CLOSED: 6375 info->state = SPDK_BDEV_ZONE_STATE_CLOSED; 6376 break; 6377 case SPDK_NVME_ZONE_STATE_RONLY: 6378 info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY; 6379 break; 6380 case SPDK_NVME_ZONE_STATE_FULL: 6381 info->state = SPDK_BDEV_ZONE_STATE_FULL; 6382 break; 6383 case SPDK_NVME_ZONE_STATE_OFFLINE: 6384 info->state = SPDK_BDEV_ZONE_STATE_OFFLINE; 6385 break; 6386 default: 6387 SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs); 6388 return -EIO; 6389 } 6390 6391 info->zone_id = desc->zslba; 6392 info->write_pointer = desc->wp; 6393 info->capacity = desc->zcap; 6394 6395 return 0; 6396 } 6397 6398 static void 6399 bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl) 6400 { 6401 struct nvme_bdev_io *bio = ref; 6402 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6403 uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id; 6404 uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones; 6405 struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf; 6406 uint64_t max_zones_per_buf, i; 6407 uint32_t zone_report_bufsize; 6408 struct spdk_nvme_ns *ns; 6409 struct spdk_nvme_qpair *qpair; 6410 int ret; 6411 6412 if (spdk_nvme_cpl_is_error(cpl)) { 6413 goto out_complete_io_nvme_cpl; 6414 } 6415 6416 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 6417 ret = -ENXIO; 6418 goto out_complete_io_ret; 6419 } 6420 6421 ns = bio->io_path->nvme_ns->ns; 6422 qpair = bio->io_path->qpair->qpair; 6423 6424 zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 6425 max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) / 6426 sizeof(bio->zone_report_buf->descs[0]); 6427 6428 if (bio->zone_report_buf->nr_zones > max_zones_per_buf) { 6429 ret = -EINVAL; 6430 goto out_complete_io_ret; 6431 } 6432 6433 if (!bio->zone_report_buf->nr_zones) { 6434 ret = -EINVAL; 6435 goto out_complete_io_ret; 6436 } 6437 6438 for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) { 6439 ret = fill_zone_from_report(&info[bio->handled_zones], 6440 &bio->zone_report_buf->descs[i]); 6441 if (ret) { 6442 goto out_complete_io_ret; 6443 } 6444 bio->handled_zones++; 6445 } 6446 6447 if (bio->handled_zones < zones_to_copy) { 6448 uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 6449 uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones); 6450 6451 memset(bio->zone_report_buf, 0, zone_report_bufsize); 6452 ret = spdk_nvme_zns_report_zones(ns, qpair, 6453 bio->zone_report_buf, zone_report_bufsize, 6454 slba, SPDK_NVME_ZRA_LIST_ALL, true, 6455 bdev_nvme_get_zone_info_done, bio); 6456 if (!ret) { 6457 return; 6458 } else { 6459 goto out_complete_io_ret; 6460 } 6461 } 6462 6463 out_complete_io_nvme_cpl: 6464 free(bio->zone_report_buf); 6465 bio->zone_report_buf = NULL; 6466 bdev_nvme_io_complete_nvme_status(bio, cpl); 6467 return; 6468 6469 out_complete_io_ret: 6470 free(bio->zone_report_buf); 6471 bio->zone_report_buf = NULL; 6472 bdev_nvme_io_complete(bio, ret); 6473 } 6474 6475 static void 6476 bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl) 6477 { 6478 struct nvme_bdev_io *bio = ref; 6479 6480 bdev_nvme_io_complete_nvme_status(bio, cpl); 6481 } 6482 6483 static void 6484 bdev_nvme_admin_passthru_complete_nvme_status(void *ctx) 6485 { 6486 struct nvme_bdev_io *bio = ctx; 6487 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6488 const struct spdk_nvme_cpl *cpl = &bio->cpl; 6489 6490 assert(bdev_nvme_io_type_is_admin(bdev_io->type)); 6491 6492 __bdev_nvme_io_complete(bdev_io, 0, cpl); 6493 } 6494 6495 static void 6496 bdev_nvme_abort_complete(void *ctx) 6497 { 6498 struct nvme_bdev_io *bio = ctx; 6499 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6500 6501 if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) { 6502 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL); 6503 } else { 6504 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL); 6505 } 6506 } 6507 6508 static void 6509 bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl) 6510 { 6511 struct nvme_bdev_io *bio = ref; 6512 6513 bio->cpl = *cpl; 6514 spdk_thread_send_msg(bio->orig_thread, bdev_nvme_abort_complete, bio); 6515 } 6516 6517 static void 6518 bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl) 6519 { 6520 struct nvme_bdev_io *bio = ref; 6521 6522 bio->cpl = *cpl; 6523 spdk_thread_send_msg(bio->orig_thread, 6524 bdev_nvme_admin_passthru_complete_nvme_status, bio); 6525 } 6526 6527 static void 6528 bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset) 6529 { 6530 struct nvme_bdev_io *bio = ref; 6531 struct iovec *iov; 6532 6533 bio->iov_offset = sgl_offset; 6534 for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) { 6535 iov = &bio->iovs[bio->iovpos]; 6536 if (bio->iov_offset < iov->iov_len) { 6537 break; 6538 } 6539 6540 bio->iov_offset -= iov->iov_len; 6541 } 6542 } 6543 6544 static int 6545 bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length) 6546 { 6547 struct nvme_bdev_io *bio = ref; 6548 struct iovec *iov; 6549 6550 assert(bio->iovpos < bio->iovcnt); 6551 6552 iov = &bio->iovs[bio->iovpos]; 6553 6554 *address = iov->iov_base; 6555 *length = iov->iov_len; 6556 6557 if (bio->iov_offset) { 6558 assert(bio->iov_offset <= iov->iov_len); 6559 *address += bio->iov_offset; 6560 *length -= bio->iov_offset; 6561 } 6562 6563 bio->iov_offset += *length; 6564 if (bio->iov_offset == iov->iov_len) { 6565 bio->iovpos++; 6566 bio->iov_offset = 0; 6567 } 6568 6569 return 0; 6570 } 6571 6572 static void 6573 bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset) 6574 { 6575 struct nvme_bdev_io *bio = ref; 6576 struct iovec *iov; 6577 6578 bio->fused_iov_offset = sgl_offset; 6579 for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) { 6580 iov = &bio->fused_iovs[bio->fused_iovpos]; 6581 if (bio->fused_iov_offset < iov->iov_len) { 6582 break; 6583 } 6584 6585 bio->fused_iov_offset -= iov->iov_len; 6586 } 6587 } 6588 6589 static int 6590 bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length) 6591 { 6592 struct nvme_bdev_io *bio = ref; 6593 struct iovec *iov; 6594 6595 assert(bio->fused_iovpos < bio->fused_iovcnt); 6596 6597 iov = &bio->fused_iovs[bio->fused_iovpos]; 6598 6599 *address = iov->iov_base; 6600 *length = iov->iov_len; 6601 6602 if (bio->fused_iov_offset) { 6603 assert(bio->fused_iov_offset <= iov->iov_len); 6604 *address += bio->fused_iov_offset; 6605 *length -= bio->fused_iov_offset; 6606 } 6607 6608 bio->fused_iov_offset += *length; 6609 if (bio->fused_iov_offset == iov->iov_len) { 6610 bio->fused_iovpos++; 6611 bio->fused_iov_offset = 0; 6612 } 6613 6614 return 0; 6615 } 6616 6617 static int 6618 bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 6619 void *md, uint64_t lba_count, uint64_t lba) 6620 { 6621 int rc; 6622 6623 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n", 6624 lba_count, lba); 6625 6626 bio->iovs = iov; 6627 bio->iovcnt = iovcnt; 6628 bio->iovpos = 0; 6629 bio->iov_offset = 0; 6630 6631 rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns, 6632 bio->io_path->qpair->qpair, 6633 lba, lba_count, 6634 bdev_nvme_no_pi_readv_done, bio, 0, 6635 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 6636 md, 0, 0); 6637 6638 if (rc != 0 && rc != -ENOMEM) { 6639 SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc); 6640 } 6641 return rc; 6642 } 6643 6644 static int 6645 bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 6646 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 6647 struct spdk_memory_domain *domain, void *domain_ctx) 6648 { 6649 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6650 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6651 int rc; 6652 6653 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n", 6654 lba_count, lba); 6655 6656 bio->iovs = iov; 6657 bio->iovcnt = iovcnt; 6658 bio->iovpos = 0; 6659 bio->iov_offset = 0; 6660 6661 if (domain != NULL) { 6662 bio->ext_opts.size = sizeof(struct spdk_nvme_ns_cmd_ext_io_opts); 6663 bio->ext_opts.memory_domain = domain; 6664 bio->ext_opts.memory_domain_ctx = domain_ctx; 6665 bio->ext_opts.io_flags = flags; 6666 bio->ext_opts.metadata = md; 6667 6668 rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count, 6669 bdev_nvme_readv_done, bio, 6670 bdev_nvme_queued_reset_sgl, 6671 bdev_nvme_queued_next_sge, 6672 &bio->ext_opts); 6673 } else if (iovcnt == 1) { 6674 rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, iov[0].iov_base, 6675 md, lba, lba_count, bdev_nvme_readv_done, 6676 bio, flags, 0, 0); 6677 } else { 6678 rc = spdk_nvme_ns_cmd_readv_with_md(ns, qpair, lba, lba_count, 6679 bdev_nvme_readv_done, bio, flags, 6680 bdev_nvme_queued_reset_sgl, 6681 bdev_nvme_queued_next_sge, md, 0, 0); 6682 } 6683 6684 if (rc != 0 && rc != -ENOMEM) { 6685 SPDK_ERRLOG("readv failed: rc = %d\n", rc); 6686 } 6687 return rc; 6688 } 6689 6690 static int 6691 bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 6692 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 6693 struct spdk_memory_domain *domain, void *domain_ctx) 6694 { 6695 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6696 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6697 int rc; 6698 6699 SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 6700 lba_count, lba); 6701 6702 bio->iovs = iov; 6703 bio->iovcnt = iovcnt; 6704 bio->iovpos = 0; 6705 bio->iov_offset = 0; 6706 6707 if (domain != NULL) { 6708 bio->ext_opts.size = sizeof(struct spdk_nvme_ns_cmd_ext_io_opts); 6709 bio->ext_opts.memory_domain = domain; 6710 bio->ext_opts.memory_domain_ctx = domain_ctx; 6711 bio->ext_opts.io_flags = flags; 6712 bio->ext_opts.metadata = md; 6713 6714 rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count, 6715 bdev_nvme_writev_done, bio, 6716 bdev_nvme_queued_reset_sgl, 6717 bdev_nvme_queued_next_sge, 6718 &bio->ext_opts); 6719 } else if (iovcnt == 1) { 6720 rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, iov[0].iov_base, 6721 md, lba, lba_count, bdev_nvme_writev_done, 6722 bio, flags, 0, 0); 6723 } else { 6724 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 6725 bdev_nvme_writev_done, bio, flags, 6726 bdev_nvme_queued_reset_sgl, 6727 bdev_nvme_queued_next_sge, md, 0, 0); 6728 } 6729 6730 if (rc != 0 && rc != -ENOMEM) { 6731 SPDK_ERRLOG("writev failed: rc = %d\n", rc); 6732 } 6733 return rc; 6734 } 6735 6736 static int 6737 bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 6738 void *md, uint64_t lba_count, uint64_t zslba, 6739 uint32_t flags) 6740 { 6741 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6742 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6743 int rc; 6744 6745 SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n", 6746 lba_count, zslba); 6747 6748 bio->iovs = iov; 6749 bio->iovcnt = iovcnt; 6750 bio->iovpos = 0; 6751 bio->iov_offset = 0; 6752 6753 if (iovcnt == 1) { 6754 rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba, 6755 lba_count, 6756 bdev_nvme_zone_appendv_done, bio, 6757 flags, 6758 0, 0); 6759 } else { 6760 rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count, 6761 bdev_nvme_zone_appendv_done, bio, flags, 6762 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 6763 md, 0, 0); 6764 } 6765 6766 if (rc != 0 && rc != -ENOMEM) { 6767 SPDK_ERRLOG("zone append failed: rc = %d\n", rc); 6768 } 6769 return rc; 6770 } 6771 6772 static int 6773 bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 6774 void *md, uint64_t lba_count, uint64_t lba, 6775 uint32_t flags) 6776 { 6777 int rc; 6778 6779 SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n", 6780 lba_count, lba); 6781 6782 bio->iovs = iov; 6783 bio->iovcnt = iovcnt; 6784 bio->iovpos = 0; 6785 bio->iov_offset = 0; 6786 6787 rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns, 6788 bio->io_path->qpair->qpair, 6789 lba, lba_count, 6790 bdev_nvme_comparev_done, bio, flags, 6791 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 6792 md, 0, 0); 6793 6794 if (rc != 0 && rc != -ENOMEM) { 6795 SPDK_ERRLOG("comparev failed: rc = %d\n", rc); 6796 } 6797 return rc; 6798 } 6799 6800 static int 6801 bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt, 6802 struct iovec *write_iov, int write_iovcnt, 6803 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags) 6804 { 6805 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6806 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6807 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 6808 int rc; 6809 6810 SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 6811 lba_count, lba); 6812 6813 bio->iovs = cmp_iov; 6814 bio->iovcnt = cmp_iovcnt; 6815 bio->iovpos = 0; 6816 bio->iov_offset = 0; 6817 bio->fused_iovs = write_iov; 6818 bio->fused_iovcnt = write_iovcnt; 6819 bio->fused_iovpos = 0; 6820 bio->fused_iov_offset = 0; 6821 6822 if (bdev_io->num_retries == 0) { 6823 bio->first_fused_submitted = false; 6824 bio->first_fused_completed = false; 6825 } 6826 6827 if (!bio->first_fused_submitted) { 6828 flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST; 6829 memset(&bio->cpl, 0, sizeof(bio->cpl)); 6830 6831 rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count, 6832 bdev_nvme_comparev_and_writev_done, bio, flags, 6833 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0); 6834 if (rc == 0) { 6835 bio->first_fused_submitted = true; 6836 flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST; 6837 } else { 6838 if (rc != -ENOMEM) { 6839 SPDK_ERRLOG("compare failed: rc = %d\n", rc); 6840 } 6841 return rc; 6842 } 6843 } 6844 6845 flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND; 6846 6847 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 6848 bdev_nvme_comparev_and_writev_done, bio, flags, 6849 bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0); 6850 if (rc != 0 && rc != -ENOMEM) { 6851 SPDK_ERRLOG("write failed: rc = %d\n", rc); 6852 rc = 0; 6853 } 6854 6855 return rc; 6856 } 6857 6858 static int 6859 bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 6860 { 6861 struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES]; 6862 struct spdk_nvme_dsm_range *range; 6863 uint64_t offset, remaining; 6864 uint64_t num_ranges_u64; 6865 uint16_t num_ranges; 6866 int rc; 6867 6868 num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) / 6869 SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 6870 if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) { 6871 SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks); 6872 return -EINVAL; 6873 } 6874 num_ranges = (uint16_t)num_ranges_u64; 6875 6876 offset = offset_blocks; 6877 remaining = num_blocks; 6878 range = &dsm_ranges[0]; 6879 6880 /* Fill max-size ranges until the remaining blocks fit into one range */ 6881 while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) { 6882 range->attributes.raw = 0; 6883 range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 6884 range->starting_lba = offset; 6885 6886 offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 6887 remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 6888 range++; 6889 } 6890 6891 /* Final range describes the remaining blocks */ 6892 range->attributes.raw = 0; 6893 range->length = remaining; 6894 range->starting_lba = offset; 6895 6896 rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns, 6897 bio->io_path->qpair->qpair, 6898 SPDK_NVME_DSM_ATTR_DEALLOCATE, 6899 dsm_ranges, num_ranges, 6900 bdev_nvme_queued_done, bio); 6901 6902 return rc; 6903 } 6904 6905 static int 6906 bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 6907 { 6908 if (num_blocks > UINT16_MAX + 1) { 6909 SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n"); 6910 return -EINVAL; 6911 } 6912 6913 return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns, 6914 bio->io_path->qpair->qpair, 6915 offset_blocks, num_blocks, 6916 bdev_nvme_queued_done, bio, 6917 0); 6918 } 6919 6920 static int 6921 bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones, 6922 struct spdk_bdev_zone_info *info) 6923 { 6924 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6925 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6926 uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 6927 uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 6928 uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns); 6929 6930 if (zone_id % zone_size != 0) { 6931 return -EINVAL; 6932 } 6933 6934 if (num_zones > total_zones || !num_zones) { 6935 return -EINVAL; 6936 } 6937 6938 assert(!bio->zone_report_buf); 6939 bio->zone_report_buf = calloc(1, zone_report_bufsize); 6940 if (!bio->zone_report_buf) { 6941 return -ENOMEM; 6942 } 6943 6944 bio->handled_zones = 0; 6945 6946 return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize, 6947 zone_id, SPDK_NVME_ZRA_LIST_ALL, true, 6948 bdev_nvme_get_zone_info_done, bio); 6949 } 6950 6951 static int 6952 bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 6953 enum spdk_bdev_zone_action action) 6954 { 6955 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 6956 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 6957 6958 switch (action) { 6959 case SPDK_BDEV_ZONE_CLOSE: 6960 return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false, 6961 bdev_nvme_zone_management_done, bio); 6962 case SPDK_BDEV_ZONE_FINISH: 6963 return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false, 6964 bdev_nvme_zone_management_done, bio); 6965 case SPDK_BDEV_ZONE_OPEN: 6966 return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false, 6967 bdev_nvme_zone_management_done, bio); 6968 case SPDK_BDEV_ZONE_RESET: 6969 return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false, 6970 bdev_nvme_zone_management_done, bio); 6971 case SPDK_BDEV_ZONE_OFFLINE: 6972 return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false, 6973 bdev_nvme_zone_management_done, bio); 6974 default: 6975 return -EINVAL; 6976 } 6977 } 6978 6979 static void 6980 bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 6981 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes) 6982 { 6983 struct nvme_io_path *io_path; 6984 struct nvme_ctrlr *nvme_ctrlr; 6985 uint32_t max_xfer_size; 6986 int rc = -ENXIO; 6987 6988 /* Choose the first ctrlr which is not failed. */ 6989 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 6990 nvme_ctrlr = io_path->qpair->ctrlr; 6991 6992 /* We should skip any unavailable nvme_ctrlr rather than checking 6993 * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO. 6994 */ 6995 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 6996 continue; 6997 } 6998 6999 max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr); 7000 7001 if (nbytes > max_xfer_size) { 7002 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 7003 rc = -EINVAL; 7004 goto err; 7005 } 7006 7007 bio->io_path = io_path; 7008 bio->orig_thread = spdk_get_thread(); 7009 7010 rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes, 7011 bdev_nvme_admin_passthru_done, bio); 7012 if (rc == 0) { 7013 return; 7014 } 7015 } 7016 7017 err: 7018 bdev_nvme_admin_passthru_complete(bio, rc); 7019 } 7020 7021 static int 7022 bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 7023 void *buf, size_t nbytes) 7024 { 7025 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7026 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7027 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 7028 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 7029 7030 if (nbytes > max_xfer_size) { 7031 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 7032 return -EINVAL; 7033 } 7034 7035 /* 7036 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 7037 * so fill it out automatically. 7038 */ 7039 cmd->nsid = spdk_nvme_ns_get_id(ns); 7040 7041 return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf, 7042 (uint32_t)nbytes, bdev_nvme_queued_done, bio); 7043 } 7044 7045 static int 7046 bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 7047 void *buf, size_t nbytes, void *md_buf, size_t md_len) 7048 { 7049 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7050 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7051 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 7052 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 7053 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 7054 7055 if (nbytes > max_xfer_size) { 7056 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 7057 return -EINVAL; 7058 } 7059 7060 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 7061 SPDK_ERRLOG("invalid meta data buffer size\n"); 7062 return -EINVAL; 7063 } 7064 7065 /* 7066 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 7067 * so fill it out automatically. 7068 */ 7069 cmd->nsid = spdk_nvme_ns_get_id(ns); 7070 7071 return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf, 7072 (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio); 7073 } 7074 7075 static void 7076 bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 7077 struct nvme_bdev_io *bio_to_abort) 7078 { 7079 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7080 struct nvme_io_path *io_path; 7081 struct nvme_ctrlr *nvme_ctrlr; 7082 int rc = 0; 7083 7084 bio->orig_thread = spdk_get_thread(); 7085 7086 rc = bdev_nvme_abort_retry_io(nbdev_ch, bio_to_abort); 7087 if (rc == 0) { 7088 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL); 7089 return; 7090 } 7091 7092 rc = 0; 7093 7094 /* Even admin commands, they were submitted to only nvme_ctrlrs which were 7095 * on any io_path. So traverse the io_path list for not only I/O commands 7096 * but also admin commands. 7097 */ 7098 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 7099 nvme_ctrlr = io_path->qpair->ctrlr; 7100 7101 rc = spdk_nvme_ctrlr_cmd_abort_ext(nvme_ctrlr->ctrlr, 7102 io_path->qpair->qpair, 7103 bio_to_abort, 7104 bdev_nvme_abort_done, bio); 7105 if (rc == -ENOENT) { 7106 /* If no command was found in I/O qpair, the target command may be 7107 * admin command. 7108 */ 7109 rc = spdk_nvme_ctrlr_cmd_abort_ext(nvme_ctrlr->ctrlr, 7110 NULL, 7111 bio_to_abort, 7112 bdev_nvme_abort_done, bio); 7113 } 7114 7115 if (rc != -ENOENT) { 7116 break; 7117 } 7118 } 7119 7120 if (rc != 0) { 7121 /* If no command was found or there was any error, complete the abort 7122 * request with failure. 7123 */ 7124 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL); 7125 } 7126 } 7127 7128 static int 7129 bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, uint64_t src_offset_blocks, 7130 uint64_t num_blocks) 7131 { 7132 struct spdk_nvme_scc_source_range range = { 7133 .slba = src_offset_blocks, 7134 .nlb = num_blocks - 1 7135 }; 7136 7137 return spdk_nvme_ns_cmd_copy(bio->io_path->nvme_ns->ns, 7138 bio->io_path->qpair->qpair, 7139 &range, 1, dst_offset_blocks, 7140 bdev_nvme_queued_done, bio); 7141 } 7142 7143 static void 7144 bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w) 7145 { 7146 const char *action; 7147 7148 if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) { 7149 action = "reset"; 7150 } else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) { 7151 action = "abort"; 7152 } else { 7153 action = "none"; 7154 } 7155 7156 spdk_json_write_object_begin(w); 7157 7158 spdk_json_write_named_string(w, "method", "bdev_nvme_set_options"); 7159 7160 spdk_json_write_named_object_begin(w, "params"); 7161 spdk_json_write_named_string(w, "action_on_timeout", action); 7162 spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us); 7163 spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us); 7164 spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms); 7165 spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count); 7166 spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst); 7167 spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight); 7168 spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight); 7169 spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight); 7170 spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us); 7171 spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us); 7172 spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests); 7173 spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit); 7174 spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count); 7175 spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout); 7176 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec); 7177 spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec); 7178 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec); 7179 spdk_json_write_named_bool(w, "generate_uuids", g_opts.generate_uuids); 7180 spdk_json_write_named_uint8(w, "transport_tos", g_opts.transport_tos); 7181 spdk_json_write_named_bool(w, "io_path_stat", g_opts.io_path_stat); 7182 spdk_json_write_object_end(w); 7183 7184 spdk_json_write_object_end(w); 7185 } 7186 7187 static void 7188 bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx) 7189 { 7190 struct spdk_nvme_transport_id trid; 7191 7192 spdk_json_write_object_begin(w); 7193 7194 spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery"); 7195 7196 spdk_json_write_named_object_begin(w, "params"); 7197 spdk_json_write_named_string(w, "name", ctx->name); 7198 spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn); 7199 7200 trid = ctx->trid; 7201 memset(trid.subnqn, 0, sizeof(trid.subnqn)); 7202 nvme_bdev_dump_trid_json(&trid, w); 7203 7204 spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach); 7205 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec); 7206 spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec); 7207 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 7208 ctx->bdev_opts.fast_io_fail_timeout_sec); 7209 spdk_json_write_object_end(w); 7210 7211 spdk_json_write_object_end(w); 7212 } 7213 7214 static void 7215 nvme_ctrlr_config_json(struct spdk_json_write_ctx *w, 7216 struct nvme_ctrlr *nvme_ctrlr) 7217 { 7218 struct spdk_nvme_transport_id *trid; 7219 const struct spdk_nvme_ctrlr_opts *opts; 7220 7221 if (nvme_ctrlr->opts.from_discovery_service) { 7222 /* Do not emit an RPC for this - it will be implicitly 7223 * covered by a separate bdev_nvme_start_discovery or 7224 * bdev_nvme_start_mdns_discovery RPC. 7225 */ 7226 return; 7227 } 7228 7229 trid = &nvme_ctrlr->active_path_id->trid; 7230 7231 spdk_json_write_object_begin(w); 7232 7233 spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller"); 7234 7235 spdk_json_write_named_object_begin(w, "params"); 7236 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 7237 nvme_bdev_dump_trid_json(trid, w); 7238 spdk_json_write_named_bool(w, "prchk_reftag", 7239 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0); 7240 spdk_json_write_named_bool(w, "prchk_guard", 7241 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0); 7242 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec); 7243 spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec); 7244 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 7245 nvme_ctrlr->opts.fast_io_fail_timeout_sec); 7246 7247 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 7248 spdk_json_write_named_bool(w, "hdgst", opts->header_digest); 7249 spdk_json_write_named_bool(w, "ddgst", opts->data_digest); 7250 7251 spdk_json_write_object_end(w); 7252 7253 spdk_json_write_object_end(w); 7254 } 7255 7256 static void 7257 bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w) 7258 { 7259 spdk_json_write_object_begin(w); 7260 spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug"); 7261 7262 spdk_json_write_named_object_begin(w, "params"); 7263 spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us); 7264 spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled); 7265 spdk_json_write_object_end(w); 7266 7267 spdk_json_write_object_end(w); 7268 } 7269 7270 static int 7271 bdev_nvme_config_json(struct spdk_json_write_ctx *w) 7272 { 7273 struct nvme_bdev_ctrlr *nbdev_ctrlr; 7274 struct nvme_ctrlr *nvme_ctrlr; 7275 struct discovery_ctx *ctx; 7276 7277 bdev_nvme_opts_config_json(w); 7278 7279 pthread_mutex_lock(&g_bdev_nvme_mutex); 7280 7281 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 7282 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 7283 nvme_ctrlr_config_json(w, nvme_ctrlr); 7284 } 7285 } 7286 7287 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7288 if (!ctx->from_mdns_discovery_service) { 7289 bdev_nvme_discovery_config_json(w, ctx); 7290 } 7291 } 7292 7293 bdev_nvme_mdns_discovery_config_json(w); 7294 7295 /* Dump as last parameter to give all NVMe bdevs chance to be constructed 7296 * before enabling hotplug poller. 7297 */ 7298 bdev_nvme_hotplug_config_json(w); 7299 7300 pthread_mutex_unlock(&g_bdev_nvme_mutex); 7301 return 0; 7302 } 7303 7304 struct spdk_nvme_ctrlr * 7305 bdev_nvme_get_ctrlr(struct spdk_bdev *bdev) 7306 { 7307 struct nvme_bdev *nbdev; 7308 struct nvme_ns *nvme_ns; 7309 7310 if (!bdev || bdev->module != &nvme_if) { 7311 return NULL; 7312 } 7313 7314 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 7315 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 7316 assert(nvme_ns != NULL); 7317 7318 return nvme_ns->ctrlr->ctrlr; 7319 } 7320 7321 void 7322 nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path) 7323 { 7324 struct nvme_ns *nvme_ns = io_path->nvme_ns; 7325 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 7326 const struct spdk_nvme_ctrlr_data *cdata; 7327 const struct spdk_nvme_transport_id *trid; 7328 const char *adrfam_str; 7329 7330 spdk_json_write_object_begin(w); 7331 7332 spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name); 7333 7334 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 7335 trid = spdk_nvme_ctrlr_get_transport_id(nvme_ctrlr->ctrlr); 7336 7337 spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid); 7338 spdk_json_write_named_bool(w, "current", io_path->nbdev_ch != NULL && 7339 io_path == io_path->nbdev_ch->current_io_path); 7340 spdk_json_write_named_bool(w, "connected", nvme_io_path_is_connected(io_path)); 7341 spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns)); 7342 7343 spdk_json_write_named_object_begin(w, "transport"); 7344 spdk_json_write_named_string(w, "trtype", trid->trstring); 7345 spdk_json_write_named_string(w, "traddr", trid->traddr); 7346 if (trid->trsvcid[0] != '\0') { 7347 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 7348 } 7349 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 7350 if (adrfam_str) { 7351 spdk_json_write_named_string(w, "adrfam", adrfam_str); 7352 } 7353 spdk_json_write_object_end(w); 7354 7355 spdk_json_write_object_end(w); 7356 } 7357 7358 void 7359 bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w) 7360 { 7361 struct discovery_ctx *ctx; 7362 struct discovery_entry_ctx *entry_ctx; 7363 7364 spdk_json_write_array_begin(w); 7365 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7366 spdk_json_write_object_begin(w); 7367 spdk_json_write_named_string(w, "name", ctx->name); 7368 7369 spdk_json_write_named_object_begin(w, "trid"); 7370 nvme_bdev_dump_trid_json(&ctx->trid, w); 7371 spdk_json_write_object_end(w); 7372 7373 spdk_json_write_named_array_begin(w, "referrals"); 7374 TAILQ_FOREACH(entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 7375 spdk_json_write_object_begin(w); 7376 spdk_json_write_named_object_begin(w, "trid"); 7377 nvme_bdev_dump_trid_json(&entry_ctx->trid, w); 7378 spdk_json_write_object_end(w); 7379 spdk_json_write_object_end(w); 7380 } 7381 spdk_json_write_array_end(w); 7382 7383 spdk_json_write_object_end(w); 7384 } 7385 spdk_json_write_array_end(w); 7386 } 7387 7388 SPDK_LOG_REGISTER_COMPONENT(bdev_nvme) 7389 7390 SPDK_TRACE_REGISTER_FN(bdev_nvme_trace, "bdev_nvme", TRACE_GROUP_BDEV_NVME) 7391 { 7392 struct spdk_trace_tpoint_opts opts[] = { 7393 { 7394 "BDEV_NVME_IO_START", TRACE_BDEV_NVME_IO_START, 7395 OWNER_NONE, OBJECT_BDEV_NVME_IO, 1, 7396 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 7397 }, 7398 { 7399 "BDEV_NVME_IO_DONE", TRACE_BDEV_NVME_IO_DONE, 7400 OWNER_NONE, OBJECT_BDEV_NVME_IO, 0, 7401 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 7402 } 7403 }; 7404 7405 7406 spdk_trace_register_object(OBJECT_BDEV_NVME_IO, 'N'); 7407 spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts)); 7408 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 7409 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 7410 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 7411 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 7412 } 7413