1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved. 6 */ 7 8 #include "spdk/stdinc.h" 9 10 #include "bdev_nvme.h" 11 12 #include "spdk/accel.h" 13 #include "spdk/config.h" 14 #include "spdk/endian.h" 15 #include "spdk/bdev.h" 16 #include "spdk/json.h" 17 #include "spdk/keyring.h" 18 #include "spdk/likely.h" 19 #include "spdk/nvme.h" 20 #include "spdk/nvme_ocssd.h" 21 #include "spdk/nvme_zns.h" 22 #include "spdk/opal.h" 23 #include "spdk/thread.h" 24 #include "spdk/trace.h" 25 #include "spdk/string.h" 26 #include "spdk/util.h" 27 #include "spdk/uuid.h" 28 29 #include "spdk/bdev_module.h" 30 #include "spdk/log.h" 31 32 #include "spdk_internal/usdt.h" 33 #include "spdk_internal/trace_defs.h" 34 35 #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true 36 #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS (10000) 37 38 #define NSID_STR_LEN 10 39 40 #define SPDK_CONTROLLER_NAME_MAX 512 41 42 static int bdev_nvme_config_json(struct spdk_json_write_ctx *w); 43 44 struct nvme_bdev_io { 45 /** array of iovecs to transfer. */ 46 struct iovec *iovs; 47 48 /** Number of iovecs in iovs array. */ 49 int iovcnt; 50 51 /** Current iovec position. */ 52 int iovpos; 53 54 /** Offset in current iovec. */ 55 uint32_t iov_offset; 56 57 /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path 58 * being reset in a reset I/O. 59 */ 60 struct nvme_io_path *io_path; 61 62 /** array of iovecs to transfer. */ 63 struct iovec *fused_iovs; 64 65 /** Number of iovecs in iovs array. */ 66 int fused_iovcnt; 67 68 /** Current iovec position. */ 69 int fused_iovpos; 70 71 /** Offset in current iovec. */ 72 uint32_t fused_iov_offset; 73 74 /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */ 75 struct spdk_nvme_cpl cpl; 76 77 /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */ 78 struct spdk_nvme_ns_cmd_ext_io_opts ext_opts; 79 80 /** Keeps track if first of fused commands was submitted */ 81 bool first_fused_submitted; 82 83 /** Keeps track if first of fused commands was completed */ 84 bool first_fused_completed; 85 86 /** Temporary pointer to zone report buffer */ 87 struct spdk_nvme_zns_zone_report *zone_report_buf; 88 89 /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */ 90 uint64_t handled_zones; 91 92 /** Expiration value in ticks to retry the current I/O. */ 93 uint64_t retry_ticks; 94 95 /* How many times the current I/O was retried. */ 96 int32_t retry_count; 97 98 /* Current tsc at submit time. */ 99 uint64_t submit_tsc; 100 101 /* Used to put nvme_bdev_io into the list */ 102 TAILQ_ENTRY(nvme_bdev_io) retry_link; 103 }; 104 105 struct nvme_probe_skip_entry { 106 struct spdk_nvme_transport_id trid; 107 TAILQ_ENTRY(nvme_probe_skip_entry) tailq; 108 }; 109 /* All the controllers deleted by users via RPC are skipped by hotplug monitor */ 110 static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER( 111 g_skipped_nvme_ctrlrs); 112 113 #define BDEV_NVME_DEFAULT_DIGESTS (SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA256) | \ 114 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA384) | \ 115 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA512)) 116 117 #define BDEV_NVME_DEFAULT_DHGROUPS (SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_NULL) | \ 118 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_2048) | \ 119 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_3072) | \ 120 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_4096) | \ 121 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_6144) | \ 122 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_8192)) 123 124 static struct spdk_bdev_nvme_opts g_opts = { 125 .action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE, 126 .timeout_us = 0, 127 .timeout_admin_us = 0, 128 .keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS, 129 .transport_retry_count = 4, 130 .arbitration_burst = 0, 131 .low_priority_weight = 0, 132 .medium_priority_weight = 0, 133 .high_priority_weight = 0, 134 .nvme_adminq_poll_period_us = 10000ULL, 135 .nvme_ioq_poll_period_us = 0, 136 .io_queue_requests = 0, 137 .delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT, 138 .bdev_retry_count = 3, 139 .transport_ack_timeout = 0, 140 .ctrlr_loss_timeout_sec = 0, 141 .reconnect_delay_sec = 0, 142 .fast_io_fail_timeout_sec = 0, 143 .disable_auto_failback = false, 144 .generate_uuids = false, 145 .transport_tos = 0, 146 .nvme_error_stat = false, 147 .io_path_stat = false, 148 .allow_accel_sequence = false, 149 .dhchap_digests = BDEV_NVME_DEFAULT_DIGESTS, 150 .dhchap_dhgroups = BDEV_NVME_DEFAULT_DHGROUPS, 151 }; 152 153 #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL 154 #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL 155 156 static int g_hot_insert_nvme_controller_index = 0; 157 static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT; 158 static bool g_nvme_hotplug_enabled = false; 159 struct spdk_thread *g_bdev_nvme_init_thread; 160 static struct spdk_poller *g_hotplug_poller; 161 static struct spdk_poller *g_hotplug_probe_poller; 162 static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx; 163 164 static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 165 struct nvme_async_probe_ctx *ctx); 166 static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 167 struct nvme_async_probe_ctx *ctx); 168 static int bdev_nvme_library_init(void); 169 static void bdev_nvme_library_fini(void); 170 static void _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, 171 struct spdk_bdev_io *bdev_io); 172 static void bdev_nvme_submit_request(struct spdk_io_channel *ch, 173 struct spdk_bdev_io *bdev_io); 174 static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 175 void *md, uint64_t lba_count, uint64_t lba, 176 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx, 177 struct spdk_accel_sequence *seq); 178 static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 179 void *md, uint64_t lba_count, uint64_t lba); 180 static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 181 void *md, uint64_t lba_count, uint64_t lba, 182 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx, 183 struct spdk_accel_sequence *seq, 184 union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13); 185 static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 186 void *md, uint64_t lba_count, 187 uint64_t zslba, uint32_t flags); 188 static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 189 void *md, uint64_t lba_count, uint64_t lba, 190 uint32_t flags); 191 static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, 192 struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov, 193 int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba, 194 uint32_t flags); 195 static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, 196 uint32_t num_zones, struct spdk_bdev_zone_info *info); 197 static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 198 enum spdk_bdev_zone_action action); 199 static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, 200 struct nvme_bdev_io *bio, 201 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes); 202 static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 203 void *buf, size_t nbytes); 204 static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 205 void *buf, size_t nbytes, void *md_buf, size_t md_len); 206 static int bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 207 struct iovec *iov, int iovcnt, size_t nbytes, 208 void *md_buf, size_t md_len); 209 static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, 210 struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort); 211 static void bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio); 212 static int bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 213 static int bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 214 static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr); 215 static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr); 216 217 static struct nvme_ns *nvme_ns_alloc(void); 218 static void nvme_ns_free(struct nvme_ns *ns); 219 220 static int 221 nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2) 222 { 223 return ns1->id < ns2->id ? -1 : ns1->id > ns2->id; 224 } 225 226 RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp); 227 228 struct spdk_nvme_qpair * 229 bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch) 230 { 231 struct nvme_ctrlr_channel *ctrlr_ch; 232 233 assert(ctrlr_io_ch != NULL); 234 235 ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch); 236 237 return ctrlr_ch->qpair->qpair; 238 } 239 240 static int 241 bdev_nvme_get_ctx_size(void) 242 { 243 return sizeof(struct nvme_bdev_io); 244 } 245 246 static struct spdk_bdev_module nvme_if = { 247 .name = "nvme", 248 .async_fini = true, 249 .module_init = bdev_nvme_library_init, 250 .module_fini = bdev_nvme_library_fini, 251 .config_json = bdev_nvme_config_json, 252 .get_ctx_size = bdev_nvme_get_ctx_size, 253 254 }; 255 SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if) 256 257 struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs); 258 pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER; 259 bool g_bdev_nvme_module_finish; 260 261 struct nvme_bdev_ctrlr * 262 nvme_bdev_ctrlr_get_by_name(const char *name) 263 { 264 struct nvme_bdev_ctrlr *nbdev_ctrlr; 265 266 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 267 if (strcmp(name, nbdev_ctrlr->name) == 0) { 268 break; 269 } 270 } 271 272 return nbdev_ctrlr; 273 } 274 275 static struct nvme_ctrlr * 276 nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr, 277 const struct spdk_nvme_transport_id *trid, const char *hostnqn) 278 { 279 const struct spdk_nvme_ctrlr_opts *opts; 280 struct nvme_ctrlr *nvme_ctrlr; 281 282 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 283 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 284 if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0 && 285 strcmp(hostnqn, opts->hostnqn) == 0) { 286 break; 287 } 288 } 289 290 return nvme_ctrlr; 291 } 292 293 struct nvme_ctrlr * 294 nvme_bdev_ctrlr_get_ctrlr_by_id(struct nvme_bdev_ctrlr *nbdev_ctrlr, 295 uint16_t cntlid) 296 { 297 struct nvme_ctrlr *nvme_ctrlr; 298 const struct spdk_nvme_ctrlr_data *cdata; 299 300 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 301 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 302 if (cdata->cntlid == cntlid) { 303 break; 304 } 305 } 306 307 return nvme_ctrlr; 308 } 309 310 static struct nvme_bdev * 311 nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid) 312 { 313 struct nvme_bdev *bdev; 314 315 pthread_mutex_lock(&g_bdev_nvme_mutex); 316 TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) { 317 if (bdev->nsid == nsid) { 318 break; 319 } 320 } 321 pthread_mutex_unlock(&g_bdev_nvme_mutex); 322 323 return bdev; 324 } 325 326 struct nvme_ns * 327 nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid) 328 { 329 struct nvme_ns ns; 330 331 assert(nsid > 0); 332 333 ns.id = nsid; 334 return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns); 335 } 336 337 struct nvme_ns * 338 nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr) 339 { 340 return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces); 341 } 342 343 struct nvme_ns * 344 nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns) 345 { 346 if (ns == NULL) { 347 return NULL; 348 } 349 350 return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 351 } 352 353 static struct nvme_ctrlr * 354 nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid, const char *hostnqn) 355 { 356 struct nvme_bdev_ctrlr *nbdev_ctrlr; 357 struct nvme_ctrlr *nvme_ctrlr = NULL; 358 359 pthread_mutex_lock(&g_bdev_nvme_mutex); 360 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 361 nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid, hostnqn); 362 if (nvme_ctrlr != NULL) { 363 break; 364 } 365 } 366 pthread_mutex_unlock(&g_bdev_nvme_mutex); 367 368 return nvme_ctrlr; 369 } 370 371 struct nvme_ctrlr * 372 nvme_ctrlr_get_by_name(const char *name) 373 { 374 struct nvme_bdev_ctrlr *nbdev_ctrlr; 375 struct nvme_ctrlr *nvme_ctrlr = NULL; 376 377 if (name == NULL) { 378 return NULL; 379 } 380 381 pthread_mutex_lock(&g_bdev_nvme_mutex); 382 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 383 if (nbdev_ctrlr != NULL) { 384 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 385 } 386 pthread_mutex_unlock(&g_bdev_nvme_mutex); 387 388 return nvme_ctrlr; 389 } 390 391 void 392 nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx) 393 { 394 struct nvme_bdev_ctrlr *nbdev_ctrlr; 395 396 pthread_mutex_lock(&g_bdev_nvme_mutex); 397 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 398 fn(nbdev_ctrlr, ctx); 399 } 400 pthread_mutex_unlock(&g_bdev_nvme_mutex); 401 } 402 403 void 404 nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w) 405 { 406 const char *trtype_str; 407 const char *adrfam_str; 408 409 trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype); 410 if (trtype_str) { 411 spdk_json_write_named_string(w, "trtype", trtype_str); 412 } 413 414 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 415 if (adrfam_str) { 416 spdk_json_write_named_string(w, "adrfam", adrfam_str); 417 } 418 419 if (trid->traddr[0] != '\0') { 420 spdk_json_write_named_string(w, "traddr", trid->traddr); 421 } 422 423 if (trid->trsvcid[0] != '\0') { 424 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 425 } 426 427 if (trid->subnqn[0] != '\0') { 428 spdk_json_write_named_string(w, "subnqn", trid->subnqn); 429 } 430 } 431 432 static void 433 nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr, 434 struct nvme_ctrlr *nvme_ctrlr) 435 { 436 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name); 437 pthread_mutex_lock(&g_bdev_nvme_mutex); 438 439 TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 440 if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) { 441 pthread_mutex_unlock(&g_bdev_nvme_mutex); 442 443 return; 444 } 445 TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 446 447 pthread_mutex_unlock(&g_bdev_nvme_mutex); 448 449 assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs)); 450 451 free(nbdev_ctrlr->name); 452 free(nbdev_ctrlr); 453 } 454 455 static void 456 _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 457 { 458 struct nvme_path_id *path_id, *tmp_path; 459 struct nvme_ns *ns, *tmp_ns; 460 461 free(nvme_ctrlr->copied_ana_desc); 462 spdk_free(nvme_ctrlr->ana_log_page); 463 464 if (nvme_ctrlr->opal_dev) { 465 spdk_opal_dev_destruct(nvme_ctrlr->opal_dev); 466 nvme_ctrlr->opal_dev = NULL; 467 } 468 469 if (nvme_ctrlr->nbdev_ctrlr) { 470 nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr); 471 } 472 473 RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) { 474 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 475 nvme_ns_free(ns); 476 } 477 478 TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) { 479 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 480 free(path_id); 481 } 482 483 pthread_mutex_destroy(&nvme_ctrlr->mutex); 484 spdk_keyring_put_key(nvme_ctrlr->psk); 485 spdk_keyring_put_key(nvme_ctrlr->dhchap_key); 486 spdk_keyring_put_key(nvme_ctrlr->dhchap_ctrlr_key); 487 free(nvme_ctrlr); 488 489 pthread_mutex_lock(&g_bdev_nvme_mutex); 490 if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 491 pthread_mutex_unlock(&g_bdev_nvme_mutex); 492 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 493 spdk_bdev_module_fini_done(); 494 return; 495 } 496 pthread_mutex_unlock(&g_bdev_nvme_mutex); 497 } 498 499 static int 500 nvme_detach_poller(void *arg) 501 { 502 struct nvme_ctrlr *nvme_ctrlr = arg; 503 int rc; 504 505 rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx); 506 if (rc != -EAGAIN) { 507 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 508 _nvme_ctrlr_delete(nvme_ctrlr); 509 } 510 511 return SPDK_POLLER_BUSY; 512 } 513 514 static void 515 nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 516 { 517 int rc; 518 519 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 520 521 /* First, unregister the adminq poller, as the driver will poll adminq if necessary */ 522 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 523 524 /* If we got here, the reset/detach poller cannot be active */ 525 assert(nvme_ctrlr->reset_detach_poller == NULL); 526 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller, 527 nvme_ctrlr, 1000); 528 if (nvme_ctrlr->reset_detach_poller == NULL) { 529 SPDK_ERRLOG("Failed to register detach poller\n"); 530 goto error; 531 } 532 533 rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx); 534 if (rc != 0) { 535 SPDK_ERRLOG("Failed to detach the NVMe controller\n"); 536 goto error; 537 } 538 539 return; 540 error: 541 /* We don't have a good way to handle errors here, so just do what we can and delete the 542 * controller without detaching the underlying NVMe device. 543 */ 544 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 545 _nvme_ctrlr_delete(nvme_ctrlr); 546 } 547 548 static void 549 nvme_ctrlr_unregister_cb(void *io_device) 550 { 551 struct nvme_ctrlr *nvme_ctrlr = io_device; 552 553 nvme_ctrlr_delete(nvme_ctrlr); 554 } 555 556 static void 557 nvme_ctrlr_unregister(void *ctx) 558 { 559 struct nvme_ctrlr *nvme_ctrlr = ctx; 560 561 spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb); 562 } 563 564 static bool 565 nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr) 566 { 567 if (!nvme_ctrlr->destruct) { 568 return false; 569 } 570 571 if (nvme_ctrlr->ref > 0) { 572 return false; 573 } 574 575 if (nvme_ctrlr->resetting) { 576 return false; 577 } 578 579 if (nvme_ctrlr->ana_log_page_updating) { 580 return false; 581 } 582 583 if (nvme_ctrlr->io_path_cache_clearing) { 584 return false; 585 } 586 587 return true; 588 } 589 590 static void 591 nvme_ctrlr_release(struct nvme_ctrlr *nvme_ctrlr) 592 { 593 pthread_mutex_lock(&nvme_ctrlr->mutex); 594 SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref); 595 596 assert(nvme_ctrlr->ref > 0); 597 nvme_ctrlr->ref--; 598 599 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 600 pthread_mutex_unlock(&nvme_ctrlr->mutex); 601 return; 602 } 603 604 pthread_mutex_unlock(&nvme_ctrlr->mutex); 605 606 spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr); 607 } 608 609 static void 610 bdev_nvme_clear_current_io_path(struct nvme_bdev_channel *nbdev_ch) 611 { 612 nbdev_ch->current_io_path = NULL; 613 nbdev_ch->rr_counter = 0; 614 } 615 616 static struct nvme_io_path * 617 _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 618 { 619 struct nvme_io_path *io_path; 620 621 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 622 if (io_path->nvme_ns == nvme_ns) { 623 break; 624 } 625 } 626 627 return io_path; 628 } 629 630 static struct nvme_io_path * 631 nvme_io_path_alloc(void) 632 { 633 struct nvme_io_path *io_path; 634 635 io_path = calloc(1, sizeof(*io_path)); 636 if (io_path == NULL) { 637 SPDK_ERRLOG("Failed to alloc io_path.\n"); 638 return NULL; 639 } 640 641 if (g_opts.io_path_stat) { 642 io_path->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 643 if (io_path->stat == NULL) { 644 free(io_path); 645 SPDK_ERRLOG("Failed to alloc io_path stat.\n"); 646 return NULL; 647 } 648 spdk_bdev_reset_io_stat(io_path->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 649 } 650 651 return io_path; 652 } 653 654 static void 655 nvme_io_path_free(struct nvme_io_path *io_path) 656 { 657 free(io_path->stat); 658 free(io_path); 659 } 660 661 static int 662 _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 663 { 664 struct nvme_io_path *io_path; 665 struct spdk_io_channel *ch; 666 struct nvme_ctrlr_channel *ctrlr_ch; 667 struct nvme_qpair *nvme_qpair; 668 669 io_path = nvme_io_path_alloc(); 670 if (io_path == NULL) { 671 return -ENOMEM; 672 } 673 674 io_path->nvme_ns = nvme_ns; 675 676 ch = spdk_get_io_channel(nvme_ns->ctrlr); 677 if (ch == NULL) { 678 nvme_io_path_free(io_path); 679 SPDK_ERRLOG("Failed to alloc io_channel.\n"); 680 return -ENOMEM; 681 } 682 683 ctrlr_ch = spdk_io_channel_get_ctx(ch); 684 685 nvme_qpair = ctrlr_ch->qpair; 686 assert(nvme_qpair != NULL); 687 688 io_path->qpair = nvme_qpair; 689 TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq); 690 691 io_path->nbdev_ch = nbdev_ch; 692 STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq); 693 694 bdev_nvme_clear_current_io_path(nbdev_ch); 695 696 return 0; 697 } 698 699 static void 700 bdev_nvme_clear_retry_io_path(struct nvme_bdev_channel *nbdev_ch, 701 struct nvme_io_path *io_path) 702 { 703 struct nvme_bdev_io *bio; 704 705 TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) { 706 if (bio->io_path == io_path) { 707 bio->io_path = NULL; 708 } 709 } 710 } 711 712 static void 713 _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path) 714 { 715 struct spdk_io_channel *ch; 716 struct nvme_qpair *nvme_qpair; 717 struct nvme_ctrlr_channel *ctrlr_ch; 718 struct nvme_bdev *nbdev; 719 720 nbdev = spdk_io_channel_get_io_device(spdk_io_channel_from_ctx(nbdev_ch)); 721 722 /* Add the statistics to nvme_ns before this path is destroyed. */ 723 pthread_mutex_lock(&nbdev->mutex); 724 if (nbdev->ref != 0 && io_path->nvme_ns->stat != NULL && io_path->stat != NULL) { 725 spdk_bdev_add_io_stat(io_path->nvme_ns->stat, io_path->stat); 726 } 727 pthread_mutex_unlock(&nbdev->mutex); 728 729 bdev_nvme_clear_current_io_path(nbdev_ch); 730 bdev_nvme_clear_retry_io_path(nbdev_ch, io_path); 731 732 STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq); 733 io_path->nbdev_ch = NULL; 734 735 nvme_qpair = io_path->qpair; 736 assert(nvme_qpair != NULL); 737 738 ctrlr_ch = nvme_qpair->ctrlr_ch; 739 assert(ctrlr_ch != NULL); 740 741 ch = spdk_io_channel_from_ctx(ctrlr_ch); 742 spdk_put_io_channel(ch); 743 744 /* After an io_path is removed, I/Os submitted to it may complete and update statistics 745 * of the io_path. To avoid heap-use-after-free error from this case, do not free the 746 * io_path here but free the io_path when the associated qpair is freed. It is ensured 747 * that all I/Os submitted to the io_path are completed when the associated qpair is freed. 748 */ 749 } 750 751 static void 752 _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch) 753 { 754 struct nvme_io_path *io_path, *tmp_io_path; 755 756 STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) { 757 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 758 } 759 } 760 761 static int 762 bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf) 763 { 764 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 765 struct nvme_bdev *nbdev = io_device; 766 struct nvme_ns *nvme_ns; 767 int rc; 768 769 STAILQ_INIT(&nbdev_ch->io_path_list); 770 TAILQ_INIT(&nbdev_ch->retry_io_list); 771 772 pthread_mutex_lock(&nbdev->mutex); 773 774 nbdev_ch->mp_policy = nbdev->mp_policy; 775 nbdev_ch->mp_selector = nbdev->mp_selector; 776 nbdev_ch->rr_min_io = nbdev->rr_min_io; 777 778 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 779 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 780 if (rc != 0) { 781 pthread_mutex_unlock(&nbdev->mutex); 782 783 _bdev_nvme_delete_io_paths(nbdev_ch); 784 return rc; 785 } 786 } 787 pthread_mutex_unlock(&nbdev->mutex); 788 789 return 0; 790 } 791 792 /* If cpl != NULL, complete the bdev_io with nvme status based on 'cpl'. 793 * If cpl == NULL, complete the bdev_io with bdev status based on 'status'. 794 */ 795 static inline void 796 __bdev_nvme_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status, 797 const struct spdk_nvme_cpl *cpl) 798 { 799 spdk_trace_record(TRACE_BDEV_NVME_IO_DONE, 0, 0, (uintptr_t)bdev_io->driver_ctx, 800 (uintptr_t)bdev_io); 801 if (cpl) { 802 spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc); 803 } else { 804 spdk_bdev_io_complete(bdev_io, status); 805 } 806 } 807 808 static void bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch); 809 810 static void 811 bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf) 812 { 813 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 814 815 bdev_nvme_abort_retry_ios(nbdev_ch); 816 _bdev_nvme_delete_io_paths(nbdev_ch); 817 } 818 819 static inline bool 820 bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type) 821 { 822 switch (io_type) { 823 case SPDK_BDEV_IO_TYPE_RESET: 824 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 825 case SPDK_BDEV_IO_TYPE_ABORT: 826 return true; 827 default: 828 break; 829 } 830 831 return false; 832 } 833 834 static inline bool 835 nvme_ns_is_active(struct nvme_ns *nvme_ns) 836 { 837 if (spdk_unlikely(nvme_ns->ana_state_updating)) { 838 return false; 839 } 840 841 if (spdk_unlikely(nvme_ns->ns == NULL)) { 842 return false; 843 } 844 845 return true; 846 } 847 848 static inline bool 849 nvme_ns_is_accessible(struct nvme_ns *nvme_ns) 850 { 851 if (spdk_unlikely(!nvme_ns_is_active(nvme_ns))) { 852 return false; 853 } 854 855 switch (nvme_ns->ana_state) { 856 case SPDK_NVME_ANA_OPTIMIZED_STATE: 857 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 858 return true; 859 default: 860 break; 861 } 862 863 return false; 864 } 865 866 static inline bool 867 nvme_qpair_is_connected(struct nvme_qpair *nvme_qpair) 868 { 869 if (spdk_unlikely(nvme_qpair->qpair == NULL)) { 870 return false; 871 } 872 873 if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 874 SPDK_NVME_QPAIR_FAILURE_NONE)) { 875 return false; 876 } 877 878 if (spdk_unlikely(nvme_qpair->ctrlr_ch->reset_iter != NULL)) { 879 return false; 880 } 881 882 return true; 883 } 884 885 static inline bool 886 nvme_io_path_is_available(struct nvme_io_path *io_path) 887 { 888 if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) { 889 return false; 890 } 891 892 if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) { 893 return false; 894 } 895 896 return true; 897 } 898 899 static inline bool 900 nvme_ctrlr_is_failed(struct nvme_ctrlr *nvme_ctrlr) 901 { 902 if (nvme_ctrlr->destruct) { 903 return true; 904 } 905 906 if (nvme_ctrlr->fast_io_fail_timedout) { 907 return true; 908 } 909 910 if (nvme_ctrlr->resetting) { 911 if (nvme_ctrlr->opts.reconnect_delay_sec != 0) { 912 return false; 913 } else { 914 return true; 915 } 916 } 917 918 if (nvme_ctrlr->reconnect_is_delayed) { 919 return false; 920 } 921 922 if (nvme_ctrlr->disabled) { 923 return true; 924 } 925 926 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 927 return true; 928 } else { 929 return false; 930 } 931 } 932 933 static bool 934 nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr) 935 { 936 if (nvme_ctrlr->destruct) { 937 return false; 938 } 939 940 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 941 return false; 942 } 943 944 if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) { 945 return false; 946 } 947 948 if (nvme_ctrlr->disabled) { 949 return false; 950 } 951 952 return true; 953 } 954 955 /* Simulate circular linked list. */ 956 static inline struct nvme_io_path * 957 nvme_io_path_get_next(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *prev_path) 958 { 959 struct nvme_io_path *next_path; 960 961 if (prev_path != NULL) { 962 next_path = STAILQ_NEXT(prev_path, stailq); 963 if (next_path != NULL) { 964 return next_path; 965 } 966 } 967 968 return STAILQ_FIRST(&nbdev_ch->io_path_list); 969 } 970 971 static struct nvme_io_path * 972 _bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 973 { 974 struct nvme_io_path *io_path, *start, *non_optimized = NULL; 975 976 start = nvme_io_path_get_next(nbdev_ch, nbdev_ch->current_io_path); 977 978 io_path = start; 979 do { 980 if (spdk_likely(nvme_io_path_is_available(io_path))) { 981 switch (io_path->nvme_ns->ana_state) { 982 case SPDK_NVME_ANA_OPTIMIZED_STATE: 983 nbdev_ch->current_io_path = io_path; 984 return io_path; 985 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 986 if (non_optimized == NULL) { 987 non_optimized = io_path; 988 } 989 break; 990 default: 991 assert(false); 992 break; 993 } 994 } 995 io_path = nvme_io_path_get_next(nbdev_ch, io_path); 996 } while (io_path != start); 997 998 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 999 /* We come here only if there is no optimized path. Cache even non_optimized 1000 * path for load balance across multiple non_optimized paths. 1001 */ 1002 nbdev_ch->current_io_path = non_optimized; 1003 } 1004 1005 return non_optimized; 1006 } 1007 1008 static struct nvme_io_path * 1009 _bdev_nvme_find_io_path_min_qd(struct nvme_bdev_channel *nbdev_ch) 1010 { 1011 struct nvme_io_path *io_path; 1012 struct nvme_io_path *optimized = NULL, *non_optimized = NULL; 1013 uint32_t opt_min_qd = UINT32_MAX, non_opt_min_qd = UINT32_MAX; 1014 uint32_t num_outstanding_reqs; 1015 1016 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 1017 if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) { 1018 /* The device is currently resetting. */ 1019 continue; 1020 } 1021 1022 if (spdk_unlikely(!nvme_ns_is_active(io_path->nvme_ns))) { 1023 continue; 1024 } 1025 1026 num_outstanding_reqs = spdk_nvme_qpair_get_num_outstanding_reqs(io_path->qpair->qpair); 1027 switch (io_path->nvme_ns->ana_state) { 1028 case SPDK_NVME_ANA_OPTIMIZED_STATE: 1029 if (num_outstanding_reqs < opt_min_qd) { 1030 opt_min_qd = num_outstanding_reqs; 1031 optimized = io_path; 1032 } 1033 break; 1034 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 1035 if (num_outstanding_reqs < non_opt_min_qd) { 1036 non_opt_min_qd = num_outstanding_reqs; 1037 non_optimized = io_path; 1038 } 1039 break; 1040 default: 1041 break; 1042 } 1043 } 1044 1045 /* don't cache io path for BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH selector */ 1046 if (optimized != NULL) { 1047 return optimized; 1048 } 1049 1050 return non_optimized; 1051 } 1052 1053 static inline struct nvme_io_path * 1054 bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 1055 { 1056 if (spdk_likely(nbdev_ch->current_io_path != NULL)) { 1057 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE) { 1058 return nbdev_ch->current_io_path; 1059 } else if (nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 1060 if (++nbdev_ch->rr_counter < nbdev_ch->rr_min_io) { 1061 return nbdev_ch->current_io_path; 1062 } 1063 nbdev_ch->rr_counter = 0; 1064 } 1065 } 1066 1067 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE || 1068 nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 1069 return _bdev_nvme_find_io_path(nbdev_ch); 1070 } else { 1071 return _bdev_nvme_find_io_path_min_qd(nbdev_ch); 1072 } 1073 } 1074 1075 /* Return true if there is any io_path whose qpair is active or ctrlr is not failed, 1076 * or false otherwise. 1077 * 1078 * If any io_path has an active qpair but find_io_path() returned NULL, its namespace 1079 * is likely to be non-accessible now but may become accessible. 1080 * 1081 * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr 1082 * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed 1083 * when starting to reset it but it is set to failed when the reset failed. Hence, if 1084 * a ctrlr is unfailed, it is likely that it works fine or is resetting. 1085 */ 1086 static bool 1087 any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch) 1088 { 1089 struct nvme_io_path *io_path; 1090 1091 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 1092 if (io_path->nvme_ns->ana_transition_timedout) { 1093 continue; 1094 } 1095 1096 if (nvme_qpair_is_connected(io_path->qpair) || 1097 !nvme_ctrlr_is_failed(io_path->qpair->ctrlr)) { 1098 return true; 1099 } 1100 } 1101 1102 return false; 1103 } 1104 1105 static void 1106 bdev_nvme_retry_io(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 1107 { 1108 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1109 struct spdk_io_channel *ch; 1110 1111 if (nbdev_io->io_path != NULL && nvme_io_path_is_available(nbdev_io->io_path)) { 1112 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 1113 } else { 1114 ch = spdk_io_channel_from_ctx(nbdev_ch); 1115 bdev_nvme_submit_request(ch, bdev_io); 1116 } 1117 } 1118 1119 static int 1120 bdev_nvme_retry_ios(void *arg) 1121 { 1122 struct nvme_bdev_channel *nbdev_ch = arg; 1123 struct nvme_bdev_io *bio, *tmp_bio; 1124 uint64_t now, delay_us; 1125 1126 now = spdk_get_ticks(); 1127 1128 TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) { 1129 if (bio->retry_ticks > now) { 1130 break; 1131 } 1132 1133 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1134 1135 bdev_nvme_retry_io(nbdev_ch, spdk_bdev_io_from_ctx(bio)); 1136 } 1137 1138 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1139 1140 bio = TAILQ_FIRST(&nbdev_ch->retry_io_list); 1141 if (bio != NULL) { 1142 delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz(); 1143 1144 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1145 delay_us); 1146 } 1147 1148 return SPDK_POLLER_BUSY; 1149 } 1150 1151 static void 1152 bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch, 1153 struct nvme_bdev_io *bio, uint64_t delay_ms) 1154 { 1155 struct nvme_bdev_io *tmp_bio; 1156 1157 bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL; 1158 1159 TAILQ_FOREACH_REVERSE(tmp_bio, &nbdev_ch->retry_io_list, retry_io_head, retry_link) { 1160 if (tmp_bio->retry_ticks <= bio->retry_ticks) { 1161 TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bio, bio, 1162 retry_link); 1163 return; 1164 } 1165 } 1166 1167 /* No earlier I/Os were found. This I/O must be the new head. */ 1168 TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bio, retry_link); 1169 1170 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1171 1172 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1173 delay_ms * 1000ULL); 1174 } 1175 1176 static void 1177 bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch) 1178 { 1179 struct nvme_bdev_io *bio, *tmp_bio; 1180 1181 TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) { 1182 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1183 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1184 } 1185 1186 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1187 } 1188 1189 static int 1190 bdev_nvme_abort_retry_io(struct nvme_bdev_channel *nbdev_ch, 1191 struct nvme_bdev_io *bio_to_abort) 1192 { 1193 struct nvme_bdev_io *bio; 1194 1195 TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) { 1196 if (bio == bio_to_abort) { 1197 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1198 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1199 return 0; 1200 } 1201 } 1202 1203 return -ENOENT; 1204 } 1205 1206 static void 1207 bdev_nvme_update_nvme_error_stat(struct spdk_bdev_io *bdev_io, const struct spdk_nvme_cpl *cpl) 1208 { 1209 struct nvme_bdev *nbdev; 1210 uint16_t sct, sc; 1211 1212 assert(spdk_nvme_cpl_is_error(cpl)); 1213 1214 nbdev = bdev_io->bdev->ctxt; 1215 1216 if (nbdev->err_stat == NULL) { 1217 return; 1218 } 1219 1220 sct = cpl->status.sct; 1221 sc = cpl->status.sc; 1222 1223 pthread_mutex_lock(&nbdev->mutex); 1224 1225 nbdev->err_stat->status_type[sct]++; 1226 switch (sct) { 1227 case SPDK_NVME_SCT_GENERIC: 1228 case SPDK_NVME_SCT_COMMAND_SPECIFIC: 1229 case SPDK_NVME_SCT_MEDIA_ERROR: 1230 case SPDK_NVME_SCT_PATH: 1231 nbdev->err_stat->status[sct][sc]++; 1232 break; 1233 default: 1234 break; 1235 } 1236 1237 pthread_mutex_unlock(&nbdev->mutex); 1238 } 1239 1240 static inline void 1241 bdev_nvme_update_io_path_stat(struct nvme_bdev_io *bio) 1242 { 1243 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1244 uint64_t num_blocks = bdev_io->u.bdev.num_blocks; 1245 uint32_t blocklen = bdev_io->bdev->blocklen; 1246 struct spdk_bdev_io_stat *stat; 1247 uint64_t tsc_diff; 1248 1249 if (bio->io_path->stat == NULL) { 1250 return; 1251 } 1252 1253 tsc_diff = spdk_get_ticks() - bio->submit_tsc; 1254 stat = bio->io_path->stat; 1255 1256 switch (bdev_io->type) { 1257 case SPDK_BDEV_IO_TYPE_READ: 1258 stat->bytes_read += num_blocks * blocklen; 1259 stat->num_read_ops++; 1260 stat->read_latency_ticks += tsc_diff; 1261 if (stat->max_read_latency_ticks < tsc_diff) { 1262 stat->max_read_latency_ticks = tsc_diff; 1263 } 1264 if (stat->min_read_latency_ticks > tsc_diff) { 1265 stat->min_read_latency_ticks = tsc_diff; 1266 } 1267 break; 1268 case SPDK_BDEV_IO_TYPE_WRITE: 1269 stat->bytes_written += num_blocks * blocklen; 1270 stat->num_write_ops++; 1271 stat->write_latency_ticks += tsc_diff; 1272 if (stat->max_write_latency_ticks < tsc_diff) { 1273 stat->max_write_latency_ticks = tsc_diff; 1274 } 1275 if (stat->min_write_latency_ticks > tsc_diff) { 1276 stat->min_write_latency_ticks = tsc_diff; 1277 } 1278 break; 1279 case SPDK_BDEV_IO_TYPE_UNMAP: 1280 stat->bytes_unmapped += num_blocks * blocklen; 1281 stat->num_unmap_ops++; 1282 stat->unmap_latency_ticks += tsc_diff; 1283 if (stat->max_unmap_latency_ticks < tsc_diff) { 1284 stat->max_unmap_latency_ticks = tsc_diff; 1285 } 1286 if (stat->min_unmap_latency_ticks > tsc_diff) { 1287 stat->min_unmap_latency_ticks = tsc_diff; 1288 } 1289 break; 1290 case SPDK_BDEV_IO_TYPE_ZCOPY: 1291 /* Track the data in the start phase only */ 1292 if (!bdev_io->u.bdev.zcopy.start) { 1293 break; 1294 } 1295 if (bdev_io->u.bdev.zcopy.populate) { 1296 stat->bytes_read += num_blocks * blocklen; 1297 stat->num_read_ops++; 1298 stat->read_latency_ticks += tsc_diff; 1299 if (stat->max_read_latency_ticks < tsc_diff) { 1300 stat->max_read_latency_ticks = tsc_diff; 1301 } 1302 if (stat->min_read_latency_ticks > tsc_diff) { 1303 stat->min_read_latency_ticks = tsc_diff; 1304 } 1305 } else { 1306 stat->bytes_written += num_blocks * blocklen; 1307 stat->num_write_ops++; 1308 stat->write_latency_ticks += tsc_diff; 1309 if (stat->max_write_latency_ticks < tsc_diff) { 1310 stat->max_write_latency_ticks = tsc_diff; 1311 } 1312 if (stat->min_write_latency_ticks > tsc_diff) { 1313 stat->min_write_latency_ticks = tsc_diff; 1314 } 1315 } 1316 break; 1317 case SPDK_BDEV_IO_TYPE_COPY: 1318 stat->bytes_copied += num_blocks * blocklen; 1319 stat->num_copy_ops++; 1320 stat->copy_latency_ticks += tsc_diff; 1321 if (stat->max_copy_latency_ticks < tsc_diff) { 1322 stat->max_copy_latency_ticks = tsc_diff; 1323 } 1324 if (stat->min_copy_latency_ticks > tsc_diff) { 1325 stat->min_copy_latency_ticks = tsc_diff; 1326 } 1327 break; 1328 default: 1329 break; 1330 } 1331 } 1332 1333 static bool 1334 bdev_nvme_check_retry_io(struct nvme_bdev_io *bio, 1335 const struct spdk_nvme_cpl *cpl, 1336 struct nvme_bdev_channel *nbdev_ch, 1337 uint64_t *_delay_ms) 1338 { 1339 struct nvme_io_path *io_path = bio->io_path; 1340 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 1341 const struct spdk_nvme_ctrlr_data *cdata; 1342 1343 if (spdk_nvme_cpl_is_path_error(cpl) || 1344 spdk_nvme_cpl_is_aborted_sq_deletion(cpl) || 1345 !nvme_io_path_is_available(io_path) || 1346 !nvme_ctrlr_is_available(nvme_ctrlr)) { 1347 bdev_nvme_clear_current_io_path(nbdev_ch); 1348 bio->io_path = NULL; 1349 if (spdk_nvme_cpl_is_ana_error(cpl)) { 1350 if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) { 1351 io_path->nvme_ns->ana_state_updating = true; 1352 } 1353 } 1354 if (!any_io_path_may_become_available(nbdev_ch)) { 1355 return false; 1356 } 1357 *_delay_ms = 0; 1358 } else { 1359 bio->retry_count++; 1360 1361 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 1362 1363 if (cpl->status.crd != 0) { 1364 *_delay_ms = cdata->crdt[cpl->status.crd] * 100; 1365 } else { 1366 *_delay_ms = 0; 1367 } 1368 } 1369 1370 return true; 1371 } 1372 1373 static inline void 1374 bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio, 1375 const struct spdk_nvme_cpl *cpl) 1376 { 1377 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1378 struct nvme_bdev_channel *nbdev_ch; 1379 uint64_t delay_ms; 1380 1381 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 1382 1383 if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) { 1384 bdev_nvme_update_io_path_stat(bio); 1385 goto complete; 1386 } 1387 1388 /* Update error counts before deciding if retry is needed. 1389 * Hence, error counts may be more than the number of I/O errors. 1390 */ 1391 bdev_nvme_update_nvme_error_stat(bdev_io, cpl); 1392 1393 if (cpl->status.dnr != 0 || spdk_nvme_cpl_is_aborted_by_request(cpl) || 1394 (g_opts.bdev_retry_count != -1 && bio->retry_count >= g_opts.bdev_retry_count)) { 1395 goto complete; 1396 } 1397 1398 /* At this point we don't know whether the sequence was successfully executed or not, so we 1399 * cannot retry the IO */ 1400 if (bdev_io->u.bdev.accel_sequence != NULL) { 1401 goto complete; 1402 } 1403 1404 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1405 1406 if (bdev_nvme_check_retry_io(bio, cpl, nbdev_ch, &delay_ms)) { 1407 bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms); 1408 return; 1409 } 1410 1411 complete: 1412 bio->retry_count = 0; 1413 bio->submit_tsc = 0; 1414 bdev_io->u.bdev.accel_sequence = NULL; 1415 __bdev_nvme_io_complete(bdev_io, 0, cpl); 1416 } 1417 1418 static inline void 1419 bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc) 1420 { 1421 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1422 struct nvme_bdev_channel *nbdev_ch; 1423 enum spdk_bdev_io_status io_status; 1424 1425 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 1426 1427 switch (rc) { 1428 case 0: 1429 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1430 break; 1431 case -ENOMEM: 1432 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1433 break; 1434 case -ENXIO: 1435 if (g_opts.bdev_retry_count == -1 || bio->retry_count < g_opts.bdev_retry_count) { 1436 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1437 1438 bdev_nvme_clear_current_io_path(nbdev_ch); 1439 bio->io_path = NULL; 1440 1441 if (any_io_path_may_become_available(nbdev_ch)) { 1442 bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL); 1443 return; 1444 } 1445 } 1446 1447 /* fallthrough */ 1448 default: 1449 spdk_accel_sequence_abort(bdev_io->u.bdev.accel_sequence); 1450 bdev_io->u.bdev.accel_sequence = NULL; 1451 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1452 break; 1453 } 1454 1455 bio->retry_count = 0; 1456 bio->submit_tsc = 0; 1457 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1458 } 1459 1460 static inline void 1461 bdev_nvme_admin_complete(struct nvme_bdev_io *bio, int rc) 1462 { 1463 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1464 enum spdk_bdev_io_status io_status; 1465 1466 switch (rc) { 1467 case 0: 1468 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1469 break; 1470 case -ENOMEM: 1471 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1472 break; 1473 case -ENXIO: 1474 /* fallthrough */ 1475 default: 1476 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1477 break; 1478 } 1479 1480 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1481 } 1482 1483 static void 1484 bdev_nvme_clear_io_path_caches_done(struct spdk_io_channel_iter *i, int status) 1485 { 1486 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 1487 1488 pthread_mutex_lock(&nvme_ctrlr->mutex); 1489 1490 assert(nvme_ctrlr->io_path_cache_clearing == true); 1491 nvme_ctrlr->io_path_cache_clearing = false; 1492 1493 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 1494 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1495 return; 1496 } 1497 1498 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1499 1500 nvme_ctrlr_unregister(nvme_ctrlr); 1501 } 1502 1503 static void 1504 _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair) 1505 { 1506 struct nvme_io_path *io_path; 1507 1508 TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) { 1509 if (io_path->nbdev_ch == NULL) { 1510 continue; 1511 } 1512 bdev_nvme_clear_current_io_path(io_path->nbdev_ch); 1513 } 1514 } 1515 1516 static void 1517 bdev_nvme_clear_io_path_cache(struct spdk_io_channel_iter *i) 1518 { 1519 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 1520 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 1521 1522 assert(ctrlr_ch->qpair != NULL); 1523 1524 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 1525 1526 spdk_for_each_channel_continue(i, 0); 1527 } 1528 1529 static void 1530 bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr) 1531 { 1532 pthread_mutex_lock(&nvme_ctrlr->mutex); 1533 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 1534 nvme_ctrlr->io_path_cache_clearing) { 1535 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1536 return; 1537 } 1538 1539 nvme_ctrlr->io_path_cache_clearing = true; 1540 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1541 1542 spdk_for_each_channel(nvme_ctrlr, 1543 bdev_nvme_clear_io_path_cache, 1544 NULL, 1545 bdev_nvme_clear_io_path_caches_done); 1546 } 1547 1548 static struct nvme_qpair * 1549 nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair) 1550 { 1551 struct nvme_qpair *nvme_qpair; 1552 1553 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1554 if (nvme_qpair->qpair == qpair) { 1555 break; 1556 } 1557 } 1558 1559 return nvme_qpair; 1560 } 1561 1562 static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair); 1563 1564 static void 1565 bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx) 1566 { 1567 struct nvme_poll_group *group = poll_group_ctx; 1568 struct nvme_qpair *nvme_qpair; 1569 struct nvme_ctrlr_channel *ctrlr_ch; 1570 int status; 1571 1572 nvme_qpair = nvme_poll_group_get_qpair(group, qpair); 1573 if (nvme_qpair == NULL) { 1574 return; 1575 } 1576 1577 if (nvme_qpair->qpair != NULL) { 1578 spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair); 1579 nvme_qpair->qpair = NULL; 1580 } 1581 1582 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1583 1584 ctrlr_ch = nvme_qpair->ctrlr_ch; 1585 1586 if (ctrlr_ch != NULL) { 1587 if (ctrlr_ch->reset_iter != NULL) { 1588 /* We are in a full reset sequence. */ 1589 if (ctrlr_ch->connect_poller != NULL) { 1590 /* qpair was failed to connect. Abort the reset sequence. */ 1591 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was failed to connect. abort the reset ctrlr sequence.\n", 1592 qpair); 1593 spdk_poller_unregister(&ctrlr_ch->connect_poller); 1594 status = -1; 1595 } else { 1596 /* qpair was completed to disconnect. Just move to the next ctrlr_channel. */ 1597 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed in a reset ctrlr sequence.\n", 1598 qpair); 1599 status = 0; 1600 } 1601 spdk_for_each_channel_continue(ctrlr_ch->reset_iter, status); 1602 ctrlr_ch->reset_iter = NULL; 1603 } else { 1604 /* qpair was disconnected unexpectedly. Reset controller for recovery. */ 1605 SPDK_NOTICELOG("qpair %p was disconnected and freed. reset controller.\n", qpair); 1606 bdev_nvme_failover_ctrlr(nvme_qpair->ctrlr); 1607 } 1608 } else { 1609 /* In this case, ctrlr_channel is already deleted. */ 1610 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed. delete nvme_qpair.\n", qpair); 1611 nvme_qpair_delete(nvme_qpair); 1612 } 1613 } 1614 1615 static void 1616 bdev_nvme_check_io_qpairs(struct nvme_poll_group *group) 1617 { 1618 struct nvme_qpair *nvme_qpair; 1619 1620 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1621 if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) { 1622 continue; 1623 } 1624 1625 if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 1626 SPDK_NVME_QPAIR_FAILURE_NONE) { 1627 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1628 } 1629 } 1630 } 1631 1632 static int 1633 bdev_nvme_poll(void *arg) 1634 { 1635 struct nvme_poll_group *group = arg; 1636 int64_t num_completions; 1637 1638 if (group->collect_spin_stat && group->start_ticks == 0) { 1639 group->start_ticks = spdk_get_ticks(); 1640 } 1641 1642 num_completions = spdk_nvme_poll_group_process_completions(group->group, 0, 1643 bdev_nvme_disconnected_qpair_cb); 1644 if (group->collect_spin_stat) { 1645 if (num_completions > 0) { 1646 if (group->end_ticks != 0) { 1647 group->spin_ticks += (group->end_ticks - group->start_ticks); 1648 group->end_ticks = 0; 1649 } 1650 group->start_ticks = 0; 1651 } else { 1652 group->end_ticks = spdk_get_ticks(); 1653 } 1654 } 1655 1656 if (spdk_unlikely(num_completions < 0)) { 1657 bdev_nvme_check_io_qpairs(group); 1658 } 1659 1660 return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 1661 } 1662 1663 static int bdev_nvme_poll_adminq(void *arg); 1664 1665 static void 1666 bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_period_us) 1667 { 1668 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 1669 1670 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, 1671 nvme_ctrlr, new_period_us); 1672 } 1673 1674 static int 1675 bdev_nvme_poll_adminq(void *arg) 1676 { 1677 int32_t rc; 1678 struct nvme_ctrlr *nvme_ctrlr = arg; 1679 nvme_ctrlr_disconnected_cb disconnected_cb; 1680 1681 assert(nvme_ctrlr != NULL); 1682 1683 rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr); 1684 if (rc < 0) { 1685 disconnected_cb = nvme_ctrlr->disconnected_cb; 1686 nvme_ctrlr->disconnected_cb = NULL; 1687 1688 if (disconnected_cb != NULL) { 1689 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 1690 g_opts.nvme_adminq_poll_period_us); 1691 disconnected_cb(nvme_ctrlr); 1692 } else { 1693 bdev_nvme_failover_ctrlr(nvme_ctrlr); 1694 } 1695 } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) != 1696 SPDK_NVME_QPAIR_FAILURE_NONE) { 1697 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 1698 } 1699 1700 return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY; 1701 } 1702 1703 static void 1704 nvme_bdev_free(void *io_device) 1705 { 1706 struct nvme_bdev *nvme_disk = io_device; 1707 1708 pthread_mutex_destroy(&nvme_disk->mutex); 1709 free(nvme_disk->disk.name); 1710 free(nvme_disk->err_stat); 1711 free(nvme_disk); 1712 } 1713 1714 static int 1715 bdev_nvme_destruct(void *ctx) 1716 { 1717 struct nvme_bdev *nvme_disk = ctx; 1718 struct nvme_ns *nvme_ns, *tmp_nvme_ns; 1719 1720 SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid); 1721 1722 TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) { 1723 pthread_mutex_lock(&nvme_ns->ctrlr->mutex); 1724 1725 nvme_ns->bdev = NULL; 1726 1727 assert(nvme_ns->id > 0); 1728 1729 if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) { 1730 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1731 1732 nvme_ctrlr_release(nvme_ns->ctrlr); 1733 nvme_ns_free(nvme_ns); 1734 } else { 1735 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1736 } 1737 } 1738 1739 pthread_mutex_lock(&g_bdev_nvme_mutex); 1740 TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq); 1741 pthread_mutex_unlock(&g_bdev_nvme_mutex); 1742 1743 spdk_io_device_unregister(nvme_disk, nvme_bdev_free); 1744 1745 return 0; 1746 } 1747 1748 static int 1749 bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair) 1750 { 1751 struct nvme_ctrlr *nvme_ctrlr; 1752 struct spdk_nvme_io_qpair_opts opts; 1753 struct spdk_nvme_qpair *qpair; 1754 int rc; 1755 1756 nvme_ctrlr = nvme_qpair->ctrlr; 1757 1758 spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1759 opts.delay_cmd_submit = g_opts.delay_cmd_submit; 1760 opts.create_only = true; 1761 opts.async_mode = true; 1762 opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests); 1763 g_opts.io_queue_requests = opts.io_queue_requests; 1764 1765 qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1766 if (qpair == NULL) { 1767 return -1; 1768 } 1769 1770 SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name, 1771 spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread)); 1772 1773 assert(nvme_qpair->group != NULL); 1774 1775 rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair); 1776 if (rc != 0) { 1777 SPDK_ERRLOG("Unable to begin polling on NVMe Channel.\n"); 1778 goto err; 1779 } 1780 1781 rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair); 1782 if (rc != 0) { 1783 SPDK_ERRLOG("Unable to connect I/O qpair.\n"); 1784 goto err; 1785 } 1786 1787 nvme_qpair->qpair = qpair; 1788 1789 if (!g_opts.disable_auto_failback) { 1790 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1791 } 1792 1793 return 0; 1794 1795 err: 1796 spdk_nvme_ctrlr_free_io_qpair(qpair); 1797 1798 return rc; 1799 } 1800 1801 static void bdev_nvme_reset_io_continue(void *cb_arg, int rc); 1802 1803 static void 1804 bdev_nvme_complete_pending_resets(struct spdk_io_channel_iter *i) 1805 { 1806 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 1807 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 1808 int rc = 0; 1809 struct nvme_bdev_io *bio; 1810 1811 if (spdk_io_channel_iter_get_ctx(i) != NULL) { 1812 rc = -1; 1813 } 1814 1815 while (!TAILQ_EMPTY(&ctrlr_ch->pending_resets)) { 1816 bio = TAILQ_FIRST(&ctrlr_ch->pending_resets); 1817 TAILQ_REMOVE(&ctrlr_ch->pending_resets, bio, retry_link); 1818 1819 bdev_nvme_reset_io_continue(bio, rc); 1820 } 1821 1822 spdk_for_each_channel_continue(i, 0); 1823 } 1824 1825 /* This function marks the current trid as failed by storing the current ticks 1826 * and then sets the next trid to the active trid within a controller if exists. 1827 * 1828 * The purpose of the boolean return value is to request the caller to disconnect 1829 * the current trid now to try connecting the next trid. 1830 */ 1831 static bool 1832 bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove, bool start) 1833 { 1834 struct nvme_path_id *path_id, *next_path; 1835 int rc __attribute__((unused)); 1836 1837 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 1838 assert(path_id); 1839 assert(path_id == nvme_ctrlr->active_path_id); 1840 next_path = TAILQ_NEXT(path_id, link); 1841 1842 /* Update the last failed time. It means the trid is failed if its last 1843 * failed time is non-zero. 1844 */ 1845 path_id->last_failed_tsc = spdk_get_ticks(); 1846 1847 if (next_path == NULL) { 1848 /* There is no alternate trid within a controller. */ 1849 return false; 1850 } 1851 1852 if (!start && nvme_ctrlr->opts.reconnect_delay_sec == 0) { 1853 /* Connect is not retried in a controller reset sequence. Connecting 1854 * the next trid will be done by the next bdev_nvme_failover_ctrlr() call. 1855 */ 1856 return false; 1857 } 1858 1859 assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE); 1860 1861 SPDK_NOTICELOG("Start failover from %s:%s to %s:%s\n", path_id->trid.traddr, 1862 path_id->trid.trsvcid, next_path->trid.traddr, next_path->trid.trsvcid); 1863 1864 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 1865 nvme_ctrlr->active_path_id = next_path; 1866 rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid); 1867 assert(rc == 0); 1868 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 1869 if (!remove) { 1870 /** Shuffle the old trid to the end of the list and use the new one. 1871 * Allows for round robin through multiple connections. 1872 */ 1873 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link); 1874 } else { 1875 free(path_id); 1876 } 1877 1878 if (start || next_path->last_failed_tsc == 0) { 1879 /* bdev_nvme_failover_ctrlr() is just called or the next trid is not failed 1880 * or used yet. Try the next trid now. 1881 */ 1882 return true; 1883 } 1884 1885 if (spdk_get_ticks() > next_path->last_failed_tsc + spdk_get_ticks_hz() * 1886 nvme_ctrlr->opts.reconnect_delay_sec) { 1887 /* Enough backoff passed since the next trid failed. Try the next trid now. */ 1888 return true; 1889 } 1890 1891 /* The next trid will be tried after reconnect_delay_sec seconds. */ 1892 return false; 1893 } 1894 1895 static bool 1896 bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr) 1897 { 1898 int32_t elapsed; 1899 1900 if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 || 1901 nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) { 1902 return false; 1903 } 1904 1905 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 1906 if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) { 1907 return true; 1908 } else { 1909 return false; 1910 } 1911 } 1912 1913 static bool 1914 bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr) 1915 { 1916 uint32_t elapsed; 1917 1918 if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) { 1919 return false; 1920 } 1921 1922 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 1923 if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) { 1924 return true; 1925 } else { 1926 return false; 1927 } 1928 } 1929 1930 static void bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success); 1931 1932 static void 1933 nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb cb_fn) 1934 { 1935 int rc; 1936 1937 rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr); 1938 if (rc != 0) { 1939 /* Disconnect fails if ctrlr is already resetting or removed. In this case, 1940 * fail the reset sequence immediately. 1941 */ 1942 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 1943 return; 1944 } 1945 1946 /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq. 1947 * Set callback here to execute the specified operation after ctrlr is really disconnected. 1948 */ 1949 assert(nvme_ctrlr->disconnected_cb == NULL); 1950 nvme_ctrlr->disconnected_cb = cb_fn; 1951 1952 /* During disconnection, reduce the period to poll adminq more often. */ 1953 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0); 1954 } 1955 1956 enum bdev_nvme_op_after_reset { 1957 OP_NONE, 1958 OP_COMPLETE_PENDING_DESTRUCT, 1959 OP_DESTRUCT, 1960 OP_DELAYED_RECONNECT, 1961 OP_FAILOVER, 1962 }; 1963 1964 typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset; 1965 1966 static _bdev_nvme_op_after_reset 1967 bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success) 1968 { 1969 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 1970 /* Complete pending destruct after reset completes. */ 1971 return OP_COMPLETE_PENDING_DESTRUCT; 1972 } else if (nvme_ctrlr->pending_failover) { 1973 nvme_ctrlr->pending_failover = false; 1974 nvme_ctrlr->reset_start_tsc = 0; 1975 return OP_FAILOVER; 1976 } else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) { 1977 nvme_ctrlr->reset_start_tsc = 0; 1978 return OP_NONE; 1979 } else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 1980 return OP_DESTRUCT; 1981 } else { 1982 if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) { 1983 nvme_ctrlr->fast_io_fail_timedout = true; 1984 } 1985 return OP_DELAYED_RECONNECT; 1986 } 1987 } 1988 1989 static int bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug); 1990 static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 1991 1992 static int 1993 bdev_nvme_reconnect_delay_timer_expired(void *ctx) 1994 { 1995 struct nvme_ctrlr *nvme_ctrlr = ctx; 1996 1997 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect_delay, nvme_ctrlr->nbdev_ctrlr->name); 1998 pthread_mutex_lock(&nvme_ctrlr->mutex); 1999 2000 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2001 2002 if (!nvme_ctrlr->reconnect_is_delayed) { 2003 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2004 return SPDK_POLLER_BUSY; 2005 } 2006 2007 nvme_ctrlr->reconnect_is_delayed = false; 2008 2009 if (nvme_ctrlr->destruct) { 2010 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2011 return SPDK_POLLER_BUSY; 2012 } 2013 2014 assert(nvme_ctrlr->resetting == false); 2015 nvme_ctrlr->resetting = true; 2016 2017 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2018 2019 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 2020 2021 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2022 return SPDK_POLLER_BUSY; 2023 } 2024 2025 static void 2026 bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr) 2027 { 2028 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 2029 2030 assert(nvme_ctrlr->reconnect_is_delayed == false); 2031 nvme_ctrlr->reconnect_is_delayed = true; 2032 2033 assert(nvme_ctrlr->reconnect_delay_timer == NULL); 2034 nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired, 2035 nvme_ctrlr, 2036 nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC); 2037 } 2038 2039 static void remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr); 2040 2041 static void 2042 _bdev_nvme_reset_ctrlr_complete(struct spdk_io_channel_iter *i, int status) 2043 { 2044 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 2045 bool success = spdk_io_channel_iter_get_ctx(i) == NULL; 2046 bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn; 2047 void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg; 2048 enum bdev_nvme_op_after_reset op_after_reset; 2049 2050 assert(nvme_ctrlr->thread == spdk_get_thread()); 2051 2052 nvme_ctrlr->ctrlr_op_cb_fn = NULL; 2053 nvme_ctrlr->ctrlr_op_cb_arg = NULL; 2054 2055 if (!success) { 2056 SPDK_ERRLOG("Resetting controller failed.\n"); 2057 } else { 2058 SPDK_NOTICELOG("Resetting controller successful.\n"); 2059 } 2060 2061 pthread_mutex_lock(&nvme_ctrlr->mutex); 2062 nvme_ctrlr->resetting = false; 2063 nvme_ctrlr->dont_retry = false; 2064 nvme_ctrlr->in_failover = false; 2065 2066 op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success); 2067 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2068 2069 /* Delay callbacks when the next operation is a failover. */ 2070 if (ctrlr_op_cb_fn && op_after_reset != OP_FAILOVER) { 2071 ctrlr_op_cb_fn(ctrlr_op_cb_arg, success ? 0 : -1); 2072 } 2073 2074 switch (op_after_reset) { 2075 case OP_COMPLETE_PENDING_DESTRUCT: 2076 nvme_ctrlr_unregister(nvme_ctrlr); 2077 break; 2078 case OP_DESTRUCT: 2079 bdev_nvme_delete_ctrlr(nvme_ctrlr, false); 2080 remove_discovery_entry(nvme_ctrlr); 2081 break; 2082 case OP_DELAYED_RECONNECT: 2083 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_start_reconnect_delay_timer); 2084 break; 2085 case OP_FAILOVER: 2086 nvme_ctrlr->ctrlr_op_cb_fn = ctrlr_op_cb_fn; 2087 nvme_ctrlr->ctrlr_op_cb_arg = ctrlr_op_cb_arg; 2088 bdev_nvme_failover_ctrlr(nvme_ctrlr); 2089 break; 2090 default: 2091 break; 2092 } 2093 } 2094 2095 static void 2096 bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success) 2097 { 2098 pthread_mutex_lock(&nvme_ctrlr->mutex); 2099 if (!success) { 2100 /* Connecting the active trid failed. Set the next alternate trid to the 2101 * active trid if it exists. 2102 */ 2103 if (bdev_nvme_failover_trid(nvme_ctrlr, false, false)) { 2104 /* The next alternate trid exists and is ready to try. Try it now. */ 2105 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2106 2107 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); 2108 return; 2109 } 2110 2111 /* We came here if there is no alternate trid or if the next trid exists but 2112 * is not ready to try. We will try the active trid after reconnect_delay_sec 2113 * seconds if it is non-zero or at the next reset call otherwise. 2114 */ 2115 } else { 2116 /* Connecting the active trid succeeded. Clear the last failed time because it 2117 * means the trid is failed if its last failed time is non-zero. 2118 */ 2119 nvme_ctrlr->active_path_id->last_failed_tsc = 0; 2120 } 2121 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2122 2123 /* Make sure we clear any pending resets before returning. */ 2124 spdk_for_each_channel(nvme_ctrlr, 2125 bdev_nvme_complete_pending_resets, 2126 success ? NULL : (void *)0x1, 2127 _bdev_nvme_reset_ctrlr_complete); 2128 } 2129 2130 static void 2131 bdev_nvme_reset_create_qpairs_failed(struct spdk_io_channel_iter *i, int status) 2132 { 2133 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 2134 2135 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2136 } 2137 2138 static void 2139 bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i) 2140 { 2141 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 2142 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch); 2143 struct nvme_qpair *nvme_qpair; 2144 2145 nvme_qpair = ctrlr_ch->qpair; 2146 assert(nvme_qpair != NULL); 2147 2148 _bdev_nvme_clear_io_path_cache(nvme_qpair); 2149 2150 if (nvme_qpair->qpair != NULL) { 2151 if (nvme_qpair->ctrlr->dont_retry) { 2152 spdk_nvme_qpair_set_abort_dnr(nvme_qpair->qpair, true); 2153 } 2154 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 2155 2156 /* The current full reset sequence will move to the next 2157 * ctrlr_channel after the qpair is actually disconnected. 2158 */ 2159 assert(ctrlr_ch->reset_iter == NULL); 2160 ctrlr_ch->reset_iter = i; 2161 } else { 2162 spdk_for_each_channel_continue(i, 0); 2163 } 2164 } 2165 2166 static void 2167 bdev_nvme_reset_create_qpairs_done(struct spdk_io_channel_iter *i, int status) 2168 { 2169 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 2170 2171 if (status == 0) { 2172 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, true); 2173 } else { 2174 /* Delete the added qpairs and quiesce ctrlr to make the states clean. */ 2175 spdk_for_each_channel(nvme_ctrlr, 2176 bdev_nvme_reset_destroy_qpair, 2177 NULL, 2178 bdev_nvme_reset_create_qpairs_failed); 2179 } 2180 } 2181 2182 static int 2183 bdev_nvme_reset_check_qpair_connected(void *ctx) 2184 { 2185 struct nvme_ctrlr_channel *ctrlr_ch = ctx; 2186 2187 if (ctrlr_ch->reset_iter == NULL) { 2188 /* qpair was already failed to connect and the reset sequence is being aborted. */ 2189 assert(ctrlr_ch->connect_poller == NULL); 2190 assert(ctrlr_ch->qpair->qpair == NULL); 2191 return SPDK_POLLER_BUSY; 2192 } 2193 2194 assert(ctrlr_ch->qpair->qpair != NULL); 2195 2196 if (!spdk_nvme_qpair_is_connected(ctrlr_ch->qpair->qpair)) { 2197 return SPDK_POLLER_BUSY; 2198 } 2199 2200 spdk_poller_unregister(&ctrlr_ch->connect_poller); 2201 2202 /* qpair was completed to connect. Move to the next ctrlr_channel */ 2203 spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 2204 ctrlr_ch->reset_iter = NULL; 2205 2206 if (!g_opts.disable_auto_failback) { 2207 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 2208 } 2209 2210 return SPDK_POLLER_BUSY; 2211 } 2212 2213 static void 2214 bdev_nvme_reset_create_qpair(struct spdk_io_channel_iter *i) 2215 { 2216 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 2217 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch); 2218 int rc; 2219 2220 rc = bdev_nvme_create_qpair(ctrlr_ch->qpair); 2221 if (rc == 0) { 2222 ctrlr_ch->connect_poller = SPDK_POLLER_REGISTER(bdev_nvme_reset_check_qpair_connected, 2223 ctrlr_ch, 0); 2224 2225 /* The current full reset sequence will move to the next 2226 * ctrlr_channel after the qpair is actually connected. 2227 */ 2228 assert(ctrlr_ch->reset_iter == NULL); 2229 ctrlr_ch->reset_iter = i; 2230 } else { 2231 spdk_for_each_channel_continue(i, rc); 2232 } 2233 } 2234 2235 static void 2236 nvme_ctrlr_check_namespaces(struct nvme_ctrlr *nvme_ctrlr) 2237 { 2238 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 2239 struct nvme_ns *nvme_ns; 2240 2241 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 2242 nvme_ns != NULL; 2243 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 2244 if (!spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 2245 SPDK_DEBUGLOG(bdev_nvme, "NSID %u was removed during reset.\n", nvme_ns->id); 2246 /* NS can be added again. Just nullify nvme_ns->ns. */ 2247 nvme_ns->ns = NULL; 2248 } 2249 } 2250 } 2251 2252 2253 static int 2254 bdev_nvme_reconnect_ctrlr_poll(void *arg) 2255 { 2256 struct nvme_ctrlr *nvme_ctrlr = arg; 2257 int rc = -ETIMEDOUT; 2258 2259 if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 2260 /* Mark the ctrlr as failed. The next call to 2261 * spdk_nvme_ctrlr_reconnect_poll_async() will then 2262 * do the necessary cleanup and return failure. 2263 */ 2264 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 2265 } 2266 2267 rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr); 2268 if (rc == -EAGAIN) { 2269 return SPDK_POLLER_BUSY; 2270 } 2271 2272 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 2273 if (rc == 0) { 2274 nvme_ctrlr_check_namespaces(nvme_ctrlr); 2275 2276 /* Recreate all of the I/O queue pairs */ 2277 spdk_for_each_channel(nvme_ctrlr, 2278 bdev_nvme_reset_create_qpair, 2279 NULL, 2280 bdev_nvme_reset_create_qpairs_done); 2281 } else { 2282 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2283 } 2284 return SPDK_POLLER_BUSY; 2285 } 2286 2287 static void 2288 bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2289 { 2290 spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr); 2291 2292 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect, nvme_ctrlr->nbdev_ctrlr->name); 2293 assert(nvme_ctrlr->reset_detach_poller == NULL); 2294 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll, 2295 nvme_ctrlr, 0); 2296 } 2297 2298 static void 2299 bdev_nvme_reset_destroy_qpair_done(struct spdk_io_channel_iter *i, int status) 2300 { 2301 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 2302 2303 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reset, nvme_ctrlr->nbdev_ctrlr->name); 2304 assert(status == 0); 2305 2306 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2307 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2308 } else { 2309 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); 2310 } 2311 } 2312 2313 static void 2314 bdev_nvme_reset_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr) 2315 { 2316 spdk_for_each_channel(nvme_ctrlr, 2317 bdev_nvme_reset_destroy_qpair, 2318 NULL, 2319 bdev_nvme_reset_destroy_qpair_done); 2320 } 2321 2322 static void 2323 bdev_nvme_reconnect_ctrlr_now(void *ctx) 2324 { 2325 struct nvme_ctrlr *nvme_ctrlr = ctx; 2326 2327 assert(nvme_ctrlr->resetting == true); 2328 assert(nvme_ctrlr->thread == spdk_get_thread()); 2329 2330 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2331 2332 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 2333 2334 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2335 } 2336 2337 static void 2338 _bdev_nvme_reset_ctrlr(void *ctx) 2339 { 2340 struct nvme_ctrlr *nvme_ctrlr = ctx; 2341 2342 assert(nvme_ctrlr->resetting == true); 2343 assert(nvme_ctrlr->thread == spdk_get_thread()); 2344 2345 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2346 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs); 2347 } else { 2348 bdev_nvme_reset_destroy_qpairs(nvme_ctrlr); 2349 } 2350 } 2351 2352 static int 2353 bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2354 { 2355 spdk_msg_fn msg_fn; 2356 2357 pthread_mutex_lock(&nvme_ctrlr->mutex); 2358 if (nvme_ctrlr->destruct) { 2359 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2360 return -ENXIO; 2361 } 2362 2363 if (nvme_ctrlr->resetting) { 2364 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2365 SPDK_NOTICELOG("Unable to perform reset, already in progress.\n"); 2366 return -EBUSY; 2367 } 2368 2369 if (nvme_ctrlr->disabled) { 2370 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2371 SPDK_NOTICELOG("Unable to perform reset. Controller is disabled.\n"); 2372 return -EALREADY; 2373 } 2374 2375 nvme_ctrlr->resetting = true; 2376 nvme_ctrlr->dont_retry = true; 2377 2378 if (nvme_ctrlr->reconnect_is_delayed) { 2379 SPDK_DEBUGLOG(bdev_nvme, "Reconnect is already scheduled.\n"); 2380 msg_fn = bdev_nvme_reconnect_ctrlr_now; 2381 nvme_ctrlr->reconnect_is_delayed = false; 2382 } else { 2383 msg_fn = _bdev_nvme_reset_ctrlr; 2384 assert(nvme_ctrlr->reset_start_tsc == 0); 2385 } 2386 2387 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2388 2389 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2390 2391 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 2392 return 0; 2393 } 2394 2395 static int 2396 bdev_nvme_enable_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2397 { 2398 pthread_mutex_lock(&nvme_ctrlr->mutex); 2399 if (nvme_ctrlr->destruct) { 2400 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2401 return -ENXIO; 2402 } 2403 2404 if (nvme_ctrlr->resetting) { 2405 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2406 return -EBUSY; 2407 } 2408 2409 if (!nvme_ctrlr->disabled) { 2410 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2411 return -EALREADY; 2412 } 2413 2414 nvme_ctrlr->disabled = false; 2415 nvme_ctrlr->resetting = true; 2416 2417 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2418 2419 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2420 2421 spdk_thread_send_msg(nvme_ctrlr->thread, bdev_nvme_reconnect_ctrlr_now, nvme_ctrlr); 2422 return 0; 2423 } 2424 2425 static void 2426 _bdev_nvme_disable_ctrlr_complete(struct spdk_io_channel_iter *i, int status) 2427 { 2428 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 2429 bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn; 2430 void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg; 2431 enum bdev_nvme_op_after_reset op_after_disable; 2432 2433 assert(nvme_ctrlr->thread == spdk_get_thread()); 2434 2435 nvme_ctrlr->ctrlr_op_cb_fn = NULL; 2436 nvme_ctrlr->ctrlr_op_cb_arg = NULL; 2437 2438 pthread_mutex_lock(&nvme_ctrlr->mutex); 2439 2440 nvme_ctrlr->resetting = false; 2441 nvme_ctrlr->dont_retry = false; 2442 2443 op_after_disable = bdev_nvme_check_op_after_reset(nvme_ctrlr, true); 2444 2445 nvme_ctrlr->disabled = true; 2446 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 2447 2448 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2449 2450 if (ctrlr_op_cb_fn) { 2451 ctrlr_op_cb_fn(ctrlr_op_cb_arg, 0); 2452 } 2453 2454 switch (op_after_disable) { 2455 case OP_COMPLETE_PENDING_DESTRUCT: 2456 nvme_ctrlr_unregister(nvme_ctrlr); 2457 break; 2458 default: 2459 break; 2460 } 2461 2462 } 2463 2464 static void 2465 bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr) 2466 { 2467 /* Make sure we clear any pending resets before returning. */ 2468 spdk_for_each_channel(nvme_ctrlr, 2469 bdev_nvme_complete_pending_resets, 2470 NULL, 2471 _bdev_nvme_disable_ctrlr_complete); 2472 } 2473 2474 static void 2475 bdev_nvme_disable_destroy_qpairs_done(struct spdk_io_channel_iter *i, int status) 2476 { 2477 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 2478 2479 assert(status == 0); 2480 2481 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2482 bdev_nvme_disable_ctrlr_complete(nvme_ctrlr); 2483 } else { 2484 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_ctrlr_complete); 2485 } 2486 } 2487 2488 static void 2489 bdev_nvme_disable_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr) 2490 { 2491 spdk_for_each_channel(nvme_ctrlr, 2492 bdev_nvme_reset_destroy_qpair, 2493 NULL, 2494 bdev_nvme_disable_destroy_qpairs_done); 2495 } 2496 2497 static void 2498 _bdev_nvme_cancel_reconnect_and_disable_ctrlr(void *ctx) 2499 { 2500 struct nvme_ctrlr *nvme_ctrlr = ctx; 2501 2502 assert(nvme_ctrlr->resetting == true); 2503 assert(nvme_ctrlr->thread == spdk_get_thread()); 2504 2505 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2506 2507 bdev_nvme_disable_ctrlr_complete(nvme_ctrlr); 2508 } 2509 2510 static void 2511 _bdev_nvme_disconnect_and_disable_ctrlr(void *ctx) 2512 { 2513 struct nvme_ctrlr *nvme_ctrlr = ctx; 2514 2515 assert(nvme_ctrlr->resetting == true); 2516 assert(nvme_ctrlr->thread == spdk_get_thread()); 2517 2518 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2519 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_destroy_qpairs); 2520 } else { 2521 bdev_nvme_disable_destroy_qpairs(nvme_ctrlr); 2522 } 2523 } 2524 2525 static int 2526 bdev_nvme_disable_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2527 { 2528 spdk_msg_fn msg_fn; 2529 2530 pthread_mutex_lock(&nvme_ctrlr->mutex); 2531 if (nvme_ctrlr->destruct) { 2532 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2533 return -ENXIO; 2534 } 2535 2536 if (nvme_ctrlr->resetting) { 2537 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2538 return -EBUSY; 2539 } 2540 2541 if (nvme_ctrlr->disabled) { 2542 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2543 return -EALREADY; 2544 } 2545 2546 nvme_ctrlr->resetting = true; 2547 nvme_ctrlr->dont_retry = true; 2548 2549 if (nvme_ctrlr->reconnect_is_delayed) { 2550 msg_fn = _bdev_nvme_cancel_reconnect_and_disable_ctrlr; 2551 nvme_ctrlr->reconnect_is_delayed = false; 2552 } else { 2553 msg_fn = _bdev_nvme_disconnect_and_disable_ctrlr; 2554 } 2555 2556 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2557 2558 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2559 2560 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 2561 return 0; 2562 } 2563 2564 static int 2565 nvme_ctrlr_op(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op, 2566 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2567 { 2568 int rc; 2569 2570 switch (op) { 2571 case NVME_CTRLR_OP_RESET: 2572 rc = bdev_nvme_reset_ctrlr(nvme_ctrlr); 2573 break; 2574 case NVME_CTRLR_OP_ENABLE: 2575 rc = bdev_nvme_enable_ctrlr(nvme_ctrlr); 2576 break; 2577 case NVME_CTRLR_OP_DISABLE: 2578 rc = bdev_nvme_disable_ctrlr(nvme_ctrlr); 2579 break; 2580 default: 2581 rc = -EINVAL; 2582 break; 2583 } 2584 2585 if (rc == 0) { 2586 assert(nvme_ctrlr->ctrlr_op_cb_fn == NULL); 2587 assert(nvme_ctrlr->ctrlr_op_cb_arg == NULL); 2588 nvme_ctrlr->ctrlr_op_cb_fn = cb_fn; 2589 nvme_ctrlr->ctrlr_op_cb_arg = cb_arg; 2590 } 2591 return rc; 2592 } 2593 2594 struct nvme_ctrlr_op_rpc_ctx { 2595 struct nvme_ctrlr *nvme_ctrlr; 2596 struct spdk_thread *orig_thread; 2597 enum nvme_ctrlr_op op; 2598 int rc; 2599 bdev_nvme_ctrlr_op_cb cb_fn; 2600 void *cb_arg; 2601 }; 2602 2603 static void 2604 _nvme_ctrlr_op_rpc_complete(void *_ctx) 2605 { 2606 struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx; 2607 2608 assert(ctx != NULL); 2609 assert(ctx->cb_fn != NULL); 2610 2611 ctx->cb_fn(ctx->cb_arg, ctx->rc); 2612 2613 free(ctx); 2614 } 2615 2616 static void 2617 nvme_ctrlr_op_rpc_complete(void *cb_arg, int rc) 2618 { 2619 struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg; 2620 2621 ctx->rc = rc; 2622 2623 spdk_thread_send_msg(ctx->orig_thread, _nvme_ctrlr_op_rpc_complete, ctx); 2624 } 2625 2626 void 2627 nvme_ctrlr_op_rpc(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op, 2628 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2629 { 2630 struct nvme_ctrlr_op_rpc_ctx *ctx; 2631 int rc; 2632 2633 assert(cb_fn != NULL); 2634 2635 ctx = calloc(1, sizeof(*ctx)); 2636 if (ctx == NULL) { 2637 SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n"); 2638 cb_fn(cb_arg, -ENOMEM); 2639 return; 2640 } 2641 2642 ctx->orig_thread = spdk_get_thread(); 2643 ctx->cb_fn = cb_fn; 2644 ctx->cb_arg = cb_arg; 2645 2646 rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_ctrlr_op_rpc_complete, ctx); 2647 if (rc == 0) { 2648 return; 2649 } else if (rc == -EALREADY) { 2650 rc = 0; 2651 } 2652 2653 nvme_ctrlr_op_rpc_complete(ctx, rc); 2654 } 2655 2656 static void nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc); 2657 2658 static void 2659 _nvme_bdev_ctrlr_op_rpc_continue(void *_ctx) 2660 { 2661 struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx; 2662 struct nvme_ctrlr *prev_nvme_ctrlr, *next_nvme_ctrlr; 2663 int rc; 2664 2665 prev_nvme_ctrlr = ctx->nvme_ctrlr; 2666 ctx->nvme_ctrlr = NULL; 2667 2668 if (ctx->rc != 0) { 2669 goto complete; 2670 } 2671 2672 next_nvme_ctrlr = TAILQ_NEXT(prev_nvme_ctrlr, tailq); 2673 if (next_nvme_ctrlr == NULL) { 2674 goto complete; 2675 } 2676 2677 rc = nvme_ctrlr_op(next_nvme_ctrlr, ctx->op, nvme_bdev_ctrlr_op_rpc_continue, ctx); 2678 if (rc == 0) { 2679 ctx->nvme_ctrlr = next_nvme_ctrlr; 2680 return; 2681 } else if (rc == -EALREADY) { 2682 ctx->nvme_ctrlr = next_nvme_ctrlr; 2683 rc = 0; 2684 } 2685 2686 ctx->rc = rc; 2687 2688 complete: 2689 ctx->cb_fn(ctx->cb_arg, ctx->rc); 2690 free(ctx); 2691 } 2692 2693 static void 2694 nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc) 2695 { 2696 struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg; 2697 2698 ctx->rc = rc; 2699 2700 spdk_thread_send_msg(ctx->orig_thread, _nvme_bdev_ctrlr_op_rpc_continue, ctx); 2701 } 2702 2703 void 2704 nvme_bdev_ctrlr_op_rpc(struct nvme_bdev_ctrlr *nbdev_ctrlr, enum nvme_ctrlr_op op, 2705 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2706 { 2707 struct nvme_ctrlr_op_rpc_ctx *ctx; 2708 struct nvme_ctrlr *nvme_ctrlr; 2709 int rc; 2710 2711 assert(cb_fn != NULL); 2712 2713 ctx = calloc(1, sizeof(*ctx)); 2714 if (ctx == NULL) { 2715 SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n"); 2716 cb_fn(cb_arg, -ENOMEM); 2717 return; 2718 } 2719 2720 ctx->orig_thread = spdk_get_thread(); 2721 ctx->op = op; 2722 ctx->cb_fn = cb_fn; 2723 ctx->cb_arg = cb_arg; 2724 2725 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 2726 assert(nvme_ctrlr != NULL); 2727 2728 rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_bdev_ctrlr_op_rpc_continue, ctx); 2729 if (rc == 0) { 2730 ctx->nvme_ctrlr = nvme_ctrlr; 2731 return; 2732 } else if (rc == -EALREADY) { 2733 ctx->nvme_ctrlr = nvme_ctrlr; 2734 rc = 0; 2735 } 2736 2737 nvme_bdev_ctrlr_op_rpc_continue(ctx, rc); 2738 } 2739 2740 static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio); 2741 2742 static void 2743 _bdev_nvme_reset_io_complete(struct spdk_io_channel_iter *i, int status) 2744 { 2745 struct nvme_bdev_io *bio = spdk_io_channel_iter_get_ctx(i); 2746 enum spdk_bdev_io_status io_status; 2747 2748 if (bio->cpl.cdw0 == 0) { 2749 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 2750 } else { 2751 io_status = SPDK_BDEV_IO_STATUS_FAILED; 2752 } 2753 2754 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL); 2755 } 2756 2757 static void 2758 bdev_nvme_abort_bdev_channel(struct spdk_io_channel_iter *i) 2759 { 2760 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 2761 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 2762 2763 bdev_nvme_abort_retry_ios(nbdev_ch); 2764 2765 spdk_for_each_channel_continue(i, 0); 2766 } 2767 2768 static void 2769 bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio) 2770 { 2771 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2772 struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt; 2773 2774 /* Abort all queued I/Os for retry. */ 2775 spdk_for_each_channel(nbdev, 2776 bdev_nvme_abort_bdev_channel, 2777 bio, 2778 _bdev_nvme_reset_io_complete); 2779 } 2780 2781 static void 2782 _bdev_nvme_reset_io_continue(void *ctx) 2783 { 2784 struct nvme_bdev_io *bio = ctx; 2785 struct nvme_io_path *prev_io_path, *next_io_path; 2786 int rc; 2787 2788 prev_io_path = bio->io_path; 2789 bio->io_path = NULL; 2790 2791 if (bio->cpl.cdw0 != 0) { 2792 goto complete; 2793 } 2794 2795 next_io_path = STAILQ_NEXT(prev_io_path, stailq); 2796 if (next_io_path == NULL) { 2797 goto complete; 2798 } 2799 2800 rc = _bdev_nvme_reset_io(next_io_path, bio); 2801 if (rc == 0) { 2802 return; 2803 } 2804 2805 bio->cpl.cdw0 = 1; 2806 2807 complete: 2808 bdev_nvme_reset_io_complete(bio); 2809 } 2810 2811 static void 2812 bdev_nvme_reset_io_continue(void *cb_arg, int rc) 2813 { 2814 struct nvme_bdev_io *bio = cb_arg; 2815 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2816 2817 bio->cpl.cdw0 = (rc == 0) ? 0 : 1; 2818 2819 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), _bdev_nvme_reset_io_continue, bio); 2820 } 2821 2822 static int 2823 _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio) 2824 { 2825 struct nvme_ctrlr_channel *ctrlr_ch; 2826 int rc; 2827 2828 rc = nvme_ctrlr_op(io_path->qpair->ctrlr, NVME_CTRLR_OP_RESET, 2829 bdev_nvme_reset_io_continue, bio); 2830 if (rc != 0 && rc != -EBUSY) { 2831 return rc; 2832 } 2833 2834 assert(bio->io_path == NULL); 2835 bio->io_path = io_path; 2836 2837 if (rc == -EBUSY) { 2838 ctrlr_ch = io_path->qpair->ctrlr_ch; 2839 assert(ctrlr_ch != NULL); 2840 /* 2841 * Reset call is queued only if it is from the app framework. This is on purpose so that 2842 * we don't interfere with the app framework reset strategy. i.e. we are deferring to the 2843 * upper level. If they are in the middle of a reset, we won't try to schedule another one. 2844 */ 2845 TAILQ_INSERT_TAIL(&ctrlr_ch->pending_resets, bio, retry_link); 2846 } 2847 2848 return 0; 2849 } 2850 2851 static void 2852 bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio) 2853 { 2854 struct nvme_io_path *io_path; 2855 int rc; 2856 2857 bio->cpl.cdw0 = 0; 2858 2859 /* Reset all nvme_ctrlrs of a bdev controller sequentially. */ 2860 io_path = STAILQ_FIRST(&nbdev_ch->io_path_list); 2861 assert(io_path != NULL); 2862 2863 rc = _bdev_nvme_reset_io(io_path, bio); 2864 if (rc != 0) { 2865 /* If the current nvme_ctrlr is disabled, skip it and move to the next nvme_ctrlr. */ 2866 rc = (rc == -EALREADY) ? 0 : rc; 2867 2868 bdev_nvme_reset_io_continue(bio, rc); 2869 } 2870 } 2871 2872 static int 2873 bdev_nvme_failover_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool remove) 2874 { 2875 if (nvme_ctrlr->destruct) { 2876 /* Don't bother resetting if the controller is in the process of being destructed. */ 2877 return -ENXIO; 2878 } 2879 2880 if (nvme_ctrlr->resetting) { 2881 if (!nvme_ctrlr->in_failover) { 2882 SPDK_NOTICELOG("Reset is already in progress. Defer failover until reset completes.\n"); 2883 2884 /* Defer failover until reset completes. */ 2885 nvme_ctrlr->pending_failover = true; 2886 return -EINPROGRESS; 2887 } else { 2888 SPDK_NOTICELOG("Unable to perform failover, already in progress.\n"); 2889 return -EBUSY; 2890 } 2891 } 2892 2893 bdev_nvme_failover_trid(nvme_ctrlr, remove, true); 2894 2895 if (nvme_ctrlr->reconnect_is_delayed) { 2896 SPDK_NOTICELOG("Reconnect is already scheduled.\n"); 2897 2898 /* We rely on the next reconnect for the failover. */ 2899 return -EALREADY; 2900 } 2901 2902 if (nvme_ctrlr->disabled) { 2903 SPDK_NOTICELOG("Controller is disabled.\n"); 2904 2905 /* We rely on the enablement for the failover. */ 2906 return -EALREADY; 2907 } 2908 2909 nvme_ctrlr->resetting = true; 2910 nvme_ctrlr->in_failover = true; 2911 2912 assert(nvme_ctrlr->reset_start_tsc == 0); 2913 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2914 2915 return 0; 2916 } 2917 2918 static int 2919 bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2920 { 2921 int rc; 2922 2923 pthread_mutex_lock(&nvme_ctrlr->mutex); 2924 rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, false); 2925 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2926 2927 if (rc == 0) { 2928 spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset_ctrlr, nvme_ctrlr); 2929 } else if (rc == -EALREADY) { 2930 rc = 0; 2931 } 2932 2933 return rc; 2934 } 2935 2936 static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, 2937 uint64_t num_blocks); 2938 2939 static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, 2940 uint64_t num_blocks); 2941 2942 static int bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, 2943 uint64_t src_offset_blocks, 2944 uint64_t num_blocks); 2945 2946 static void 2947 bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 2948 bool success) 2949 { 2950 struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 2951 int ret; 2952 2953 if (!success) { 2954 ret = -EINVAL; 2955 goto exit; 2956 } 2957 2958 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 2959 ret = -ENXIO; 2960 goto exit; 2961 } 2962 2963 ret = bdev_nvme_readv(bio, 2964 bdev_io->u.bdev.iovs, 2965 bdev_io->u.bdev.iovcnt, 2966 bdev_io->u.bdev.md_buf, 2967 bdev_io->u.bdev.num_blocks, 2968 bdev_io->u.bdev.offset_blocks, 2969 bdev_io->u.bdev.dif_check_flags, 2970 bdev_io->u.bdev.memory_domain, 2971 bdev_io->u.bdev.memory_domain_ctx, 2972 bdev_io->u.bdev.accel_sequence); 2973 2974 exit: 2975 if (spdk_unlikely(ret != 0)) { 2976 bdev_nvme_io_complete(bio, ret); 2977 } 2978 } 2979 2980 static inline void 2981 _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 2982 { 2983 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 2984 struct spdk_bdev *bdev = bdev_io->bdev; 2985 struct nvme_bdev_io *nbdev_io_to_abort; 2986 int rc = 0; 2987 2988 switch (bdev_io->type) { 2989 case SPDK_BDEV_IO_TYPE_READ: 2990 if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) { 2991 2992 rc = bdev_nvme_readv(nbdev_io, 2993 bdev_io->u.bdev.iovs, 2994 bdev_io->u.bdev.iovcnt, 2995 bdev_io->u.bdev.md_buf, 2996 bdev_io->u.bdev.num_blocks, 2997 bdev_io->u.bdev.offset_blocks, 2998 bdev_io->u.bdev.dif_check_flags, 2999 bdev_io->u.bdev.memory_domain, 3000 bdev_io->u.bdev.memory_domain_ctx, 3001 bdev_io->u.bdev.accel_sequence); 3002 } else { 3003 spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb, 3004 bdev_io->u.bdev.num_blocks * bdev->blocklen); 3005 rc = 0; 3006 } 3007 break; 3008 case SPDK_BDEV_IO_TYPE_WRITE: 3009 rc = bdev_nvme_writev(nbdev_io, 3010 bdev_io->u.bdev.iovs, 3011 bdev_io->u.bdev.iovcnt, 3012 bdev_io->u.bdev.md_buf, 3013 bdev_io->u.bdev.num_blocks, 3014 bdev_io->u.bdev.offset_blocks, 3015 bdev_io->u.bdev.dif_check_flags, 3016 bdev_io->u.bdev.memory_domain, 3017 bdev_io->u.bdev.memory_domain_ctx, 3018 bdev_io->u.bdev.accel_sequence, 3019 bdev_io->u.bdev.nvme_cdw12, 3020 bdev_io->u.bdev.nvme_cdw13); 3021 break; 3022 case SPDK_BDEV_IO_TYPE_COMPARE: 3023 rc = bdev_nvme_comparev(nbdev_io, 3024 bdev_io->u.bdev.iovs, 3025 bdev_io->u.bdev.iovcnt, 3026 bdev_io->u.bdev.md_buf, 3027 bdev_io->u.bdev.num_blocks, 3028 bdev_io->u.bdev.offset_blocks, 3029 bdev_io->u.bdev.dif_check_flags); 3030 break; 3031 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 3032 rc = bdev_nvme_comparev_and_writev(nbdev_io, 3033 bdev_io->u.bdev.iovs, 3034 bdev_io->u.bdev.iovcnt, 3035 bdev_io->u.bdev.fused_iovs, 3036 bdev_io->u.bdev.fused_iovcnt, 3037 bdev_io->u.bdev.md_buf, 3038 bdev_io->u.bdev.num_blocks, 3039 bdev_io->u.bdev.offset_blocks, 3040 bdev_io->u.bdev.dif_check_flags); 3041 break; 3042 case SPDK_BDEV_IO_TYPE_UNMAP: 3043 rc = bdev_nvme_unmap(nbdev_io, 3044 bdev_io->u.bdev.offset_blocks, 3045 bdev_io->u.bdev.num_blocks); 3046 break; 3047 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 3048 rc = bdev_nvme_write_zeroes(nbdev_io, 3049 bdev_io->u.bdev.offset_blocks, 3050 bdev_io->u.bdev.num_blocks); 3051 break; 3052 case SPDK_BDEV_IO_TYPE_RESET: 3053 nbdev_io->io_path = NULL; 3054 bdev_nvme_reset_io(nbdev_ch, nbdev_io); 3055 return; 3056 3057 case SPDK_BDEV_IO_TYPE_FLUSH: 3058 bdev_nvme_io_complete(nbdev_io, 0); 3059 return; 3060 3061 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 3062 rc = bdev_nvme_zone_appendv(nbdev_io, 3063 bdev_io->u.bdev.iovs, 3064 bdev_io->u.bdev.iovcnt, 3065 bdev_io->u.bdev.md_buf, 3066 bdev_io->u.bdev.num_blocks, 3067 bdev_io->u.bdev.offset_blocks, 3068 bdev_io->u.bdev.dif_check_flags); 3069 break; 3070 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 3071 rc = bdev_nvme_get_zone_info(nbdev_io, 3072 bdev_io->u.zone_mgmt.zone_id, 3073 bdev_io->u.zone_mgmt.num_zones, 3074 bdev_io->u.zone_mgmt.buf); 3075 break; 3076 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 3077 rc = bdev_nvme_zone_management(nbdev_io, 3078 bdev_io->u.zone_mgmt.zone_id, 3079 bdev_io->u.zone_mgmt.zone_action); 3080 break; 3081 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3082 nbdev_io->io_path = NULL; 3083 bdev_nvme_admin_passthru(nbdev_ch, 3084 nbdev_io, 3085 &bdev_io->u.nvme_passthru.cmd, 3086 bdev_io->u.nvme_passthru.buf, 3087 bdev_io->u.nvme_passthru.nbytes); 3088 return; 3089 3090 case SPDK_BDEV_IO_TYPE_NVME_IO: 3091 rc = bdev_nvme_io_passthru(nbdev_io, 3092 &bdev_io->u.nvme_passthru.cmd, 3093 bdev_io->u.nvme_passthru.buf, 3094 bdev_io->u.nvme_passthru.nbytes); 3095 break; 3096 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3097 rc = bdev_nvme_io_passthru_md(nbdev_io, 3098 &bdev_io->u.nvme_passthru.cmd, 3099 bdev_io->u.nvme_passthru.buf, 3100 bdev_io->u.nvme_passthru.nbytes, 3101 bdev_io->u.nvme_passthru.md_buf, 3102 bdev_io->u.nvme_passthru.md_len); 3103 break; 3104 case SPDK_BDEV_IO_TYPE_NVME_IOV_MD: 3105 rc = bdev_nvme_iov_passthru_md(nbdev_io, 3106 &bdev_io->u.nvme_passthru.cmd, 3107 bdev_io->u.nvme_passthru.iovs, 3108 bdev_io->u.nvme_passthru.iovcnt, 3109 bdev_io->u.nvme_passthru.nbytes, 3110 bdev_io->u.nvme_passthru.md_buf, 3111 bdev_io->u.nvme_passthru.md_len); 3112 break; 3113 case SPDK_BDEV_IO_TYPE_ABORT: 3114 nbdev_io->io_path = NULL; 3115 nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx; 3116 bdev_nvme_abort(nbdev_ch, 3117 nbdev_io, 3118 nbdev_io_to_abort); 3119 return; 3120 3121 case SPDK_BDEV_IO_TYPE_COPY: 3122 rc = bdev_nvme_copy(nbdev_io, 3123 bdev_io->u.bdev.offset_blocks, 3124 bdev_io->u.bdev.copy.src_offset_blocks, 3125 bdev_io->u.bdev.num_blocks); 3126 break; 3127 default: 3128 rc = -EINVAL; 3129 break; 3130 } 3131 3132 if (spdk_unlikely(rc != 0)) { 3133 bdev_nvme_io_complete(nbdev_io, rc); 3134 } 3135 } 3136 3137 static void 3138 bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 3139 { 3140 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 3141 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 3142 3143 if (spdk_likely(nbdev_io->submit_tsc == 0)) { 3144 nbdev_io->submit_tsc = spdk_bdev_io_get_submit_tsc(bdev_io); 3145 } else { 3146 /* There are cases where submit_tsc != 0, i.e. retry I/O. 3147 * We need to update submit_tsc here. 3148 */ 3149 nbdev_io->submit_tsc = spdk_get_ticks(); 3150 } 3151 3152 spdk_trace_record(TRACE_BDEV_NVME_IO_START, 0, 0, (uintptr_t)nbdev_io, (uintptr_t)bdev_io); 3153 nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch); 3154 if (spdk_unlikely(!nbdev_io->io_path)) { 3155 if (!bdev_nvme_io_type_is_admin(bdev_io->type)) { 3156 bdev_nvme_io_complete(nbdev_io, -ENXIO); 3157 return; 3158 } 3159 3160 /* Admin commands do not use the optimal I/O path. 3161 * Simply fall through even if it is not found. 3162 */ 3163 } 3164 3165 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 3166 } 3167 3168 static bool 3169 bdev_nvme_is_supported_csi(enum spdk_nvme_csi csi) 3170 { 3171 switch (csi) { 3172 case SPDK_NVME_CSI_NVM: 3173 return true; 3174 case SPDK_NVME_CSI_ZNS: 3175 return true; 3176 default: 3177 return false; 3178 } 3179 } 3180 3181 static bool 3182 bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 3183 { 3184 struct nvme_bdev *nbdev = ctx; 3185 struct nvme_ns *nvme_ns; 3186 struct spdk_nvme_ns *ns; 3187 struct spdk_nvme_ctrlr *ctrlr; 3188 const struct spdk_nvme_ctrlr_data *cdata; 3189 3190 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 3191 assert(nvme_ns != NULL); 3192 ns = nvme_ns->ns; 3193 if (ns == NULL) { 3194 return false; 3195 } 3196 3197 if (!bdev_nvme_is_supported_csi(spdk_nvme_ns_get_csi(ns))) { 3198 switch (io_type) { 3199 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3200 case SPDK_BDEV_IO_TYPE_NVME_IO: 3201 return true; 3202 3203 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3204 return spdk_nvme_ns_get_md_size(ns) ? true : false; 3205 3206 default: 3207 return false; 3208 } 3209 } 3210 3211 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 3212 3213 switch (io_type) { 3214 case SPDK_BDEV_IO_TYPE_READ: 3215 case SPDK_BDEV_IO_TYPE_WRITE: 3216 case SPDK_BDEV_IO_TYPE_RESET: 3217 case SPDK_BDEV_IO_TYPE_FLUSH: 3218 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3219 case SPDK_BDEV_IO_TYPE_NVME_IO: 3220 case SPDK_BDEV_IO_TYPE_ABORT: 3221 return true; 3222 3223 case SPDK_BDEV_IO_TYPE_COMPARE: 3224 return spdk_nvme_ns_supports_compare(ns); 3225 3226 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3227 return spdk_nvme_ns_get_md_size(ns) ? true : false; 3228 3229 case SPDK_BDEV_IO_TYPE_UNMAP: 3230 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3231 return cdata->oncs.dsm; 3232 3233 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 3234 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3235 return cdata->oncs.write_zeroes; 3236 3237 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 3238 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 3239 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) { 3240 return true; 3241 } 3242 return false; 3243 3244 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 3245 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 3246 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS; 3247 3248 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 3249 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS && 3250 spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED; 3251 3252 case SPDK_BDEV_IO_TYPE_COPY: 3253 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3254 return cdata->oncs.copy; 3255 3256 default: 3257 return false; 3258 } 3259 } 3260 3261 static int 3262 nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch) 3263 { 3264 struct nvme_qpair *nvme_qpair; 3265 struct spdk_io_channel *pg_ch; 3266 int rc; 3267 3268 nvme_qpair = calloc(1, sizeof(*nvme_qpair)); 3269 if (!nvme_qpair) { 3270 SPDK_ERRLOG("Failed to alloc nvme_qpair.\n"); 3271 return -1; 3272 } 3273 3274 TAILQ_INIT(&nvme_qpair->io_path_list); 3275 3276 nvme_qpair->ctrlr = nvme_ctrlr; 3277 nvme_qpair->ctrlr_ch = ctrlr_ch; 3278 3279 pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs); 3280 if (!pg_ch) { 3281 free(nvme_qpair); 3282 return -1; 3283 } 3284 3285 nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch); 3286 3287 #ifdef SPDK_CONFIG_VTUNE 3288 nvme_qpair->group->collect_spin_stat = true; 3289 #else 3290 nvme_qpair->group->collect_spin_stat = false; 3291 #endif 3292 3293 if (!nvme_ctrlr->disabled) { 3294 /* If a nvme_ctrlr is disabled, don't try to create qpair for it. Qpair will 3295 * be created when it's enabled. 3296 */ 3297 rc = bdev_nvme_create_qpair(nvme_qpair); 3298 if (rc != 0) { 3299 /* nvme_ctrlr can't create IO qpair if connection is down. 3300 * If reconnect_delay_sec is non-zero, creating IO qpair is retried 3301 * after reconnect_delay_sec seconds. If bdev_retry_count is non-zero, 3302 * submitted IO will be queued until IO qpair is successfully created. 3303 * 3304 * Hence, if both are satisfied, ignore the failure. 3305 */ 3306 if (nvme_ctrlr->opts.reconnect_delay_sec == 0 || g_opts.bdev_retry_count == 0) { 3307 spdk_put_io_channel(pg_ch); 3308 free(nvme_qpair); 3309 return rc; 3310 } 3311 } 3312 } 3313 3314 TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 3315 3316 ctrlr_ch->qpair = nvme_qpair; 3317 3318 pthread_mutex_lock(&nvme_qpair->ctrlr->mutex); 3319 nvme_qpair->ctrlr->ref++; 3320 pthread_mutex_unlock(&nvme_qpair->ctrlr->mutex); 3321 3322 return 0; 3323 } 3324 3325 static int 3326 bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) 3327 { 3328 struct nvme_ctrlr *nvme_ctrlr = io_device; 3329 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 3330 3331 TAILQ_INIT(&ctrlr_ch->pending_resets); 3332 3333 return nvme_qpair_create(nvme_ctrlr, ctrlr_ch); 3334 } 3335 3336 static void 3337 nvme_qpair_delete(struct nvme_qpair *nvme_qpair) 3338 { 3339 struct nvme_io_path *io_path, *next; 3340 3341 assert(nvme_qpair->group != NULL); 3342 3343 TAILQ_FOREACH_SAFE(io_path, &nvme_qpair->io_path_list, tailq, next) { 3344 TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq); 3345 nvme_io_path_free(io_path); 3346 } 3347 3348 TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 3349 3350 spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group)); 3351 3352 nvme_ctrlr_release(nvme_qpair->ctrlr); 3353 3354 free(nvme_qpair); 3355 } 3356 3357 static void 3358 bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf) 3359 { 3360 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 3361 struct nvme_qpair *nvme_qpair; 3362 3363 nvme_qpair = ctrlr_ch->qpair; 3364 assert(nvme_qpair != NULL); 3365 3366 _bdev_nvme_clear_io_path_cache(nvme_qpair); 3367 3368 if (nvme_qpair->qpair != NULL) { 3369 if (ctrlr_ch->reset_iter == NULL) { 3370 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 3371 } else { 3372 /* Skip current ctrlr_channel in a full reset sequence because 3373 * it is being deleted now. The qpair is already being disconnected. 3374 * We do not have to restart disconnecting it. 3375 */ 3376 spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 3377 } 3378 3379 /* We cannot release a reference to the poll group now. 3380 * The qpair may be disconnected asynchronously later. 3381 * We need to poll it until it is actually disconnected. 3382 * Just detach the qpair from the deleting ctrlr_channel. 3383 */ 3384 nvme_qpair->ctrlr_ch = NULL; 3385 } else { 3386 assert(ctrlr_ch->reset_iter == NULL); 3387 3388 nvme_qpair_delete(nvme_qpair); 3389 } 3390 } 3391 3392 static inline struct spdk_io_channel * 3393 bdev_nvme_get_accel_channel(struct nvme_poll_group *group) 3394 { 3395 if (spdk_unlikely(!group->accel_channel)) { 3396 group->accel_channel = spdk_accel_get_io_channel(); 3397 if (!group->accel_channel) { 3398 SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n", 3399 group); 3400 return NULL; 3401 } 3402 } 3403 3404 return group->accel_channel; 3405 } 3406 3407 static void 3408 bdev_nvme_submit_accel_crc32c(void *ctx, uint32_t *dst, struct iovec *iov, 3409 uint32_t iov_cnt, uint32_t seed, 3410 spdk_nvme_accel_completion_cb cb_fn, void *cb_arg) 3411 { 3412 struct spdk_io_channel *accel_ch; 3413 struct nvme_poll_group *group = ctx; 3414 int rc; 3415 3416 assert(cb_fn != NULL); 3417 3418 accel_ch = bdev_nvme_get_accel_channel(group); 3419 if (spdk_unlikely(accel_ch == NULL)) { 3420 cb_fn(cb_arg, -ENOMEM); 3421 return; 3422 } 3423 3424 rc = spdk_accel_submit_crc32cv(accel_ch, dst, iov, iov_cnt, seed, cb_fn, cb_arg); 3425 if (rc) { 3426 /* For the two cases, spdk_accel_submit_crc32cv does not call the user's cb_fn */ 3427 if (rc == -ENOMEM || rc == -EINVAL) { 3428 cb_fn(cb_arg, rc); 3429 } 3430 SPDK_ERRLOG("Cannot complete the accelerated crc32c operation with iov=%p\n", iov); 3431 } 3432 } 3433 3434 static void 3435 bdev_nvme_finish_sequence(void *seq, spdk_nvme_accel_completion_cb cb_fn, void *cb_arg) 3436 { 3437 spdk_accel_sequence_finish(seq, cb_fn, cb_arg); 3438 } 3439 3440 static void 3441 bdev_nvme_abort_sequence(void *seq) 3442 { 3443 spdk_accel_sequence_abort(seq); 3444 } 3445 3446 static void 3447 bdev_nvme_reverse_sequence(void *seq) 3448 { 3449 spdk_accel_sequence_reverse(seq); 3450 } 3451 3452 static int 3453 bdev_nvme_append_crc32c(void *ctx, void **seq, uint32_t *dst, struct iovec *iovs, uint32_t iovcnt, 3454 struct spdk_memory_domain *domain, void *domain_ctx, uint32_t seed, 3455 spdk_nvme_accel_step_cb cb_fn, void *cb_arg) 3456 { 3457 struct spdk_io_channel *ch; 3458 struct nvme_poll_group *group = ctx; 3459 3460 ch = bdev_nvme_get_accel_channel(group); 3461 if (spdk_unlikely(ch == NULL)) { 3462 return -ENOMEM; 3463 } 3464 3465 return spdk_accel_append_crc32c((struct spdk_accel_sequence **)seq, ch, dst, iovs, iovcnt, 3466 domain, domain_ctx, seed, cb_fn, cb_arg); 3467 } 3468 3469 static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = { 3470 .table_size = sizeof(struct spdk_nvme_accel_fn_table), 3471 .submit_accel_crc32c = bdev_nvme_submit_accel_crc32c, 3472 .append_crc32c = bdev_nvme_append_crc32c, 3473 .finish_sequence = bdev_nvme_finish_sequence, 3474 .reverse_sequence = bdev_nvme_reverse_sequence, 3475 .abort_sequence = bdev_nvme_abort_sequence, 3476 }; 3477 3478 static int 3479 bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf) 3480 { 3481 struct nvme_poll_group *group = ctx_buf; 3482 3483 TAILQ_INIT(&group->qpair_list); 3484 3485 group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table); 3486 if (group->group == NULL) { 3487 return -1; 3488 } 3489 3490 group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us); 3491 3492 if (group->poller == NULL) { 3493 spdk_nvme_poll_group_destroy(group->group); 3494 return -1; 3495 } 3496 3497 return 0; 3498 } 3499 3500 static void 3501 bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf) 3502 { 3503 struct nvme_poll_group *group = ctx_buf; 3504 3505 assert(TAILQ_EMPTY(&group->qpair_list)); 3506 3507 if (group->accel_channel) { 3508 spdk_put_io_channel(group->accel_channel); 3509 } 3510 3511 spdk_poller_unregister(&group->poller); 3512 if (spdk_nvme_poll_group_destroy(group->group)) { 3513 SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n"); 3514 assert(false); 3515 } 3516 } 3517 3518 static struct spdk_io_channel * 3519 bdev_nvme_get_io_channel(void *ctx) 3520 { 3521 struct nvme_bdev *nvme_bdev = ctx; 3522 3523 return spdk_get_io_channel(nvme_bdev); 3524 } 3525 3526 static void * 3527 bdev_nvme_get_module_ctx(void *ctx) 3528 { 3529 struct nvme_bdev *nvme_bdev = ctx; 3530 struct nvme_ns *nvme_ns; 3531 3532 if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) { 3533 return NULL; 3534 } 3535 3536 nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list); 3537 if (!nvme_ns) { 3538 return NULL; 3539 } 3540 3541 return nvme_ns->ns; 3542 } 3543 3544 static const char * 3545 _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state) 3546 { 3547 switch (ana_state) { 3548 case SPDK_NVME_ANA_OPTIMIZED_STATE: 3549 return "optimized"; 3550 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 3551 return "non_optimized"; 3552 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3553 return "inaccessible"; 3554 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3555 return "persistent_loss"; 3556 case SPDK_NVME_ANA_CHANGE_STATE: 3557 return "change"; 3558 default: 3559 return NULL; 3560 } 3561 } 3562 3563 static int 3564 bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size) 3565 { 3566 struct spdk_memory_domain **_domains = NULL; 3567 struct nvme_bdev *nbdev = ctx; 3568 struct nvme_ns *nvme_ns; 3569 int i = 0, _array_size = array_size; 3570 int rc = 0; 3571 3572 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 3573 if (domains && array_size >= i) { 3574 _domains = &domains[i]; 3575 } else { 3576 _domains = NULL; 3577 } 3578 rc = spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, _domains, _array_size); 3579 if (rc > 0) { 3580 i += rc; 3581 if (_array_size >= rc) { 3582 _array_size -= rc; 3583 } else { 3584 _array_size = 0; 3585 } 3586 } else if (rc < 0) { 3587 return rc; 3588 } 3589 } 3590 3591 return i; 3592 } 3593 3594 static const char * 3595 nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr) 3596 { 3597 if (nvme_ctrlr->destruct) { 3598 return "deleting"; 3599 } else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 3600 return "failed"; 3601 } else if (nvme_ctrlr->resetting) { 3602 return "resetting"; 3603 } else if (nvme_ctrlr->reconnect_is_delayed > 0) { 3604 return "reconnect_is_delayed"; 3605 } else if (nvme_ctrlr->disabled) { 3606 return "disabled"; 3607 } else { 3608 return "enabled"; 3609 } 3610 } 3611 3612 void 3613 nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr) 3614 { 3615 struct spdk_nvme_transport_id *trid; 3616 const struct spdk_nvme_ctrlr_opts *opts; 3617 const struct spdk_nvme_ctrlr_data *cdata; 3618 struct nvme_path_id *path_id; 3619 3620 spdk_json_write_object_begin(w); 3621 3622 spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr)); 3623 3624 #ifdef SPDK_CONFIG_NVME_CUSE 3625 size_t cuse_name_size = 128; 3626 char cuse_name[cuse_name_size]; 3627 3628 int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size); 3629 if (rc == 0) { 3630 spdk_json_write_named_string(w, "cuse_device", cuse_name); 3631 } 3632 #endif 3633 trid = &nvme_ctrlr->active_path_id->trid; 3634 spdk_json_write_named_object_begin(w, "trid"); 3635 nvme_bdev_dump_trid_json(trid, w); 3636 spdk_json_write_object_end(w); 3637 3638 path_id = TAILQ_NEXT(nvme_ctrlr->active_path_id, link); 3639 if (path_id != NULL) { 3640 spdk_json_write_named_array_begin(w, "alternate_trids"); 3641 do { 3642 trid = &path_id->trid; 3643 spdk_json_write_object_begin(w); 3644 nvme_bdev_dump_trid_json(trid, w); 3645 spdk_json_write_object_end(w); 3646 3647 path_id = TAILQ_NEXT(path_id, link); 3648 } while (path_id != NULL); 3649 spdk_json_write_array_end(w); 3650 } 3651 3652 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 3653 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 3654 3655 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 3656 spdk_json_write_named_object_begin(w, "host"); 3657 spdk_json_write_named_string(w, "nqn", opts->hostnqn); 3658 spdk_json_write_named_string(w, "addr", opts->src_addr); 3659 spdk_json_write_named_string(w, "svcid", opts->src_svcid); 3660 spdk_json_write_object_end(w); 3661 3662 spdk_json_write_object_end(w); 3663 } 3664 3665 static void 3666 nvme_namespace_info_json(struct spdk_json_write_ctx *w, 3667 struct nvme_ns *nvme_ns) 3668 { 3669 struct spdk_nvme_ns *ns; 3670 struct spdk_nvme_ctrlr *ctrlr; 3671 const struct spdk_nvme_ctrlr_data *cdata; 3672 const struct spdk_nvme_transport_id *trid; 3673 union spdk_nvme_vs_register vs; 3674 const struct spdk_nvme_ns_data *nsdata; 3675 char buf[128]; 3676 3677 ns = nvme_ns->ns; 3678 if (ns == NULL) { 3679 return; 3680 } 3681 3682 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 3683 3684 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3685 trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); 3686 vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr); 3687 3688 spdk_json_write_object_begin(w); 3689 3690 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 3691 spdk_json_write_named_string(w, "pci_address", trid->traddr); 3692 } 3693 3694 spdk_json_write_named_object_begin(w, "trid"); 3695 3696 nvme_bdev_dump_trid_json(trid, w); 3697 3698 spdk_json_write_object_end(w); 3699 3700 #ifdef SPDK_CONFIG_NVME_CUSE 3701 size_t cuse_name_size = 128; 3702 char cuse_name[cuse_name_size]; 3703 3704 int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns), 3705 cuse_name, &cuse_name_size); 3706 if (rc == 0) { 3707 spdk_json_write_named_string(w, "cuse_device", cuse_name); 3708 } 3709 #endif 3710 3711 spdk_json_write_named_object_begin(w, "ctrlr_data"); 3712 3713 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 3714 3715 spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid); 3716 3717 snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn); 3718 spdk_str_trim(buf); 3719 spdk_json_write_named_string(w, "model_number", buf); 3720 3721 snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn); 3722 spdk_str_trim(buf); 3723 spdk_json_write_named_string(w, "serial_number", buf); 3724 3725 snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr); 3726 spdk_str_trim(buf); 3727 spdk_json_write_named_string(w, "firmware_revision", buf); 3728 3729 if (cdata->subnqn[0] != '\0') { 3730 spdk_json_write_named_string(w, "subnqn", cdata->subnqn); 3731 } 3732 3733 spdk_json_write_named_object_begin(w, "oacs"); 3734 3735 spdk_json_write_named_uint32(w, "security", cdata->oacs.security); 3736 spdk_json_write_named_uint32(w, "format", cdata->oacs.format); 3737 spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware); 3738 spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage); 3739 3740 spdk_json_write_object_end(w); 3741 3742 spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr); 3743 spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting); 3744 3745 spdk_json_write_object_end(w); 3746 3747 spdk_json_write_named_object_begin(w, "vs"); 3748 3749 spdk_json_write_name(w, "nvme_version"); 3750 if (vs.bits.ter) { 3751 spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter); 3752 } else { 3753 spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr); 3754 } 3755 3756 spdk_json_write_object_end(w); 3757 3758 nsdata = spdk_nvme_ns_get_data(ns); 3759 3760 spdk_json_write_named_object_begin(w, "ns_data"); 3761 3762 spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns)); 3763 3764 if (cdata->cmic.ana_reporting) { 3765 spdk_json_write_named_string(w, "ana_state", 3766 _nvme_ana_state_str(nvme_ns->ana_state)); 3767 } 3768 3769 spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share); 3770 3771 spdk_json_write_object_end(w); 3772 3773 if (cdata->oacs.security) { 3774 spdk_json_write_named_object_begin(w, "security"); 3775 3776 spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal); 3777 3778 spdk_json_write_object_end(w); 3779 } 3780 3781 spdk_json_write_object_end(w); 3782 } 3783 3784 static const char * 3785 nvme_bdev_get_mp_policy_str(struct nvme_bdev *nbdev) 3786 { 3787 switch (nbdev->mp_policy) { 3788 case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE: 3789 return "active_passive"; 3790 case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE: 3791 return "active_active"; 3792 default: 3793 assert(false); 3794 return "invalid"; 3795 } 3796 } 3797 3798 static const char * 3799 nvme_bdev_get_mp_selector_str(struct nvme_bdev *nbdev) 3800 { 3801 switch (nbdev->mp_selector) { 3802 case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN: 3803 return "round_robin"; 3804 case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH: 3805 return "queue_depth"; 3806 default: 3807 assert(false); 3808 return "invalid"; 3809 } 3810 } 3811 3812 static int 3813 bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 3814 { 3815 struct nvme_bdev *nvme_bdev = ctx; 3816 struct nvme_ns *nvme_ns; 3817 3818 pthread_mutex_lock(&nvme_bdev->mutex); 3819 spdk_json_write_named_array_begin(w, "nvme"); 3820 TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) { 3821 nvme_namespace_info_json(w, nvme_ns); 3822 } 3823 spdk_json_write_array_end(w); 3824 spdk_json_write_named_string(w, "mp_policy", nvme_bdev_get_mp_policy_str(nvme_bdev)); 3825 if (nvme_bdev->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 3826 spdk_json_write_named_string(w, "selector", nvme_bdev_get_mp_selector_str(nvme_bdev)); 3827 if (nvme_bdev->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 3828 spdk_json_write_named_uint32(w, "rr_min_io", nvme_bdev->rr_min_io); 3829 } 3830 } 3831 pthread_mutex_unlock(&nvme_bdev->mutex); 3832 3833 return 0; 3834 } 3835 3836 static void 3837 bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 3838 { 3839 /* No config per bdev needed */ 3840 } 3841 3842 static uint64_t 3843 bdev_nvme_get_spin_time(struct spdk_io_channel *ch) 3844 { 3845 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 3846 struct nvme_io_path *io_path; 3847 struct nvme_poll_group *group; 3848 uint64_t spin_time = 0; 3849 3850 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 3851 group = io_path->qpair->group; 3852 3853 if (!group || !group->collect_spin_stat) { 3854 continue; 3855 } 3856 3857 if (group->end_ticks != 0) { 3858 group->spin_ticks += (group->end_ticks - group->start_ticks); 3859 group->end_ticks = 0; 3860 } 3861 3862 spin_time += group->spin_ticks; 3863 group->start_ticks = 0; 3864 group->spin_ticks = 0; 3865 } 3866 3867 return (spin_time * 1000000ULL) / spdk_get_ticks_hz(); 3868 } 3869 3870 static void 3871 bdev_nvme_reset_device_stat(void *ctx) 3872 { 3873 struct nvme_bdev *nbdev = ctx; 3874 3875 if (nbdev->err_stat != NULL) { 3876 memset(nbdev->err_stat, 0, sizeof(struct nvme_error_stat)); 3877 } 3878 } 3879 3880 /* JSON string should be lowercases and underscore delimited string. */ 3881 static void 3882 bdev_nvme_format_nvme_status(char *dst, const char *src) 3883 { 3884 char tmp[256]; 3885 3886 spdk_strcpy_replace(dst, 256, src, " - ", "_"); 3887 spdk_strcpy_replace(tmp, 256, dst, "-", "_"); 3888 spdk_strcpy_replace(dst, 256, tmp, " ", "_"); 3889 spdk_strlwr(dst); 3890 } 3891 3892 static void 3893 bdev_nvme_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w) 3894 { 3895 struct nvme_bdev *nbdev = ctx; 3896 struct spdk_nvme_status status = {}; 3897 uint16_t sct, sc; 3898 char status_json[256]; 3899 const char *status_str; 3900 3901 if (nbdev->err_stat == NULL) { 3902 return; 3903 } 3904 3905 spdk_json_write_named_object_begin(w, "nvme_error"); 3906 3907 spdk_json_write_named_object_begin(w, "status_type"); 3908 for (sct = 0; sct < 8; sct++) { 3909 if (nbdev->err_stat->status_type[sct] == 0) { 3910 continue; 3911 } 3912 status.sct = sct; 3913 3914 status_str = spdk_nvme_cpl_get_status_type_string(&status); 3915 assert(status_str != NULL); 3916 bdev_nvme_format_nvme_status(status_json, status_str); 3917 3918 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status_type[sct]); 3919 } 3920 spdk_json_write_object_end(w); 3921 3922 spdk_json_write_named_object_begin(w, "status_code"); 3923 for (sct = 0; sct < 4; sct++) { 3924 status.sct = sct; 3925 for (sc = 0; sc < 256; sc++) { 3926 if (nbdev->err_stat->status[sct][sc] == 0) { 3927 continue; 3928 } 3929 status.sc = sc; 3930 3931 status_str = spdk_nvme_cpl_get_status_string(&status); 3932 assert(status_str != NULL); 3933 bdev_nvme_format_nvme_status(status_json, status_str); 3934 3935 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status[sct][sc]); 3936 } 3937 } 3938 spdk_json_write_object_end(w); 3939 3940 spdk_json_write_object_end(w); 3941 } 3942 3943 static bool 3944 bdev_nvme_accel_sequence_supported(void *ctx, enum spdk_bdev_io_type type) 3945 { 3946 struct nvme_bdev *nbdev = ctx; 3947 struct spdk_nvme_ctrlr *ctrlr; 3948 3949 if (!g_opts.allow_accel_sequence) { 3950 return false; 3951 } 3952 3953 switch (type) { 3954 case SPDK_BDEV_IO_TYPE_WRITE: 3955 case SPDK_BDEV_IO_TYPE_READ: 3956 break; 3957 default: 3958 return false; 3959 } 3960 3961 ctrlr = bdev_nvme_get_ctrlr(&nbdev->disk); 3962 assert(ctrlr != NULL); 3963 3964 return spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ACCEL_SEQUENCE_SUPPORTED; 3965 } 3966 3967 static const struct spdk_bdev_fn_table nvmelib_fn_table = { 3968 .destruct = bdev_nvme_destruct, 3969 .submit_request = bdev_nvme_submit_request, 3970 .io_type_supported = bdev_nvme_io_type_supported, 3971 .get_io_channel = bdev_nvme_get_io_channel, 3972 .dump_info_json = bdev_nvme_dump_info_json, 3973 .write_config_json = bdev_nvme_write_config_json, 3974 .get_spin_time = bdev_nvme_get_spin_time, 3975 .get_module_ctx = bdev_nvme_get_module_ctx, 3976 .get_memory_domains = bdev_nvme_get_memory_domains, 3977 .accel_sequence_supported = bdev_nvme_accel_sequence_supported, 3978 .reset_device_stat = bdev_nvme_reset_device_stat, 3979 .dump_device_stat_json = bdev_nvme_dump_device_stat_json, 3980 }; 3981 3982 typedef int (*bdev_nvme_parse_ana_log_page_cb)( 3983 const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg); 3984 3985 static int 3986 bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 3987 bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg) 3988 { 3989 struct spdk_nvme_ana_group_descriptor *copied_desc; 3990 uint8_t *orig_desc; 3991 uint32_t i, desc_size, copy_len; 3992 int rc = 0; 3993 3994 if (nvme_ctrlr->ana_log_page == NULL) { 3995 return -EINVAL; 3996 } 3997 3998 copied_desc = nvme_ctrlr->copied_ana_desc; 3999 4000 orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page); 4001 copy_len = nvme_ctrlr->max_ana_log_page_size - sizeof(struct spdk_nvme_ana_page); 4002 4003 for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) { 4004 memcpy(copied_desc, orig_desc, copy_len); 4005 4006 rc = cb_fn(copied_desc, cb_arg); 4007 if (rc != 0) { 4008 break; 4009 } 4010 4011 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) + 4012 copied_desc->num_of_nsid * sizeof(uint32_t); 4013 orig_desc += desc_size; 4014 copy_len -= desc_size; 4015 } 4016 4017 return rc; 4018 } 4019 4020 static int 4021 nvme_ns_ana_transition_timedout(void *ctx) 4022 { 4023 struct nvme_ns *nvme_ns = ctx; 4024 4025 spdk_poller_unregister(&nvme_ns->anatt_timer); 4026 nvme_ns->ana_transition_timedout = true; 4027 4028 return SPDK_POLLER_BUSY; 4029 } 4030 4031 static void 4032 _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns, 4033 const struct spdk_nvme_ana_group_descriptor *desc) 4034 { 4035 const struct spdk_nvme_ctrlr_data *cdata; 4036 4037 nvme_ns->ana_group_id = desc->ana_group_id; 4038 nvme_ns->ana_state = desc->ana_state; 4039 nvme_ns->ana_state_updating = false; 4040 4041 switch (nvme_ns->ana_state) { 4042 case SPDK_NVME_ANA_OPTIMIZED_STATE: 4043 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 4044 nvme_ns->ana_transition_timedout = false; 4045 spdk_poller_unregister(&nvme_ns->anatt_timer); 4046 break; 4047 4048 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 4049 case SPDK_NVME_ANA_CHANGE_STATE: 4050 if (nvme_ns->anatt_timer != NULL) { 4051 break; 4052 } 4053 4054 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 4055 nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout, 4056 nvme_ns, 4057 cdata->anatt * SPDK_SEC_TO_USEC); 4058 break; 4059 default: 4060 break; 4061 } 4062 } 4063 4064 static int 4065 nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg) 4066 { 4067 struct nvme_ns *nvme_ns = cb_arg; 4068 uint32_t i; 4069 4070 assert(nvme_ns->ns != NULL); 4071 4072 for (i = 0; i < desc->num_of_nsid; i++) { 4073 if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) { 4074 continue; 4075 } 4076 4077 _nvme_ns_set_ana_state(nvme_ns, desc); 4078 return 1; 4079 } 4080 4081 return 0; 4082 } 4083 4084 static int 4085 nvme_generate_uuid(const char *sn, uint32_t nsid, struct spdk_uuid *uuid) 4086 { 4087 int rc = 0; 4088 struct spdk_uuid new_uuid, namespace_uuid; 4089 char merged_str[SPDK_NVME_CTRLR_SN_LEN + NSID_STR_LEN + 1] = {'\0'}; 4090 /* This namespace UUID was generated using uuid_generate() method. */ 4091 const char *namespace_str = {"edaed2de-24bc-4b07-b559-f47ecbe730fd"}; 4092 int size; 4093 4094 assert(strlen(sn) <= SPDK_NVME_CTRLR_SN_LEN); 4095 4096 spdk_uuid_set_null(&new_uuid); 4097 spdk_uuid_set_null(&namespace_uuid); 4098 4099 size = snprintf(merged_str, sizeof(merged_str), "%s%"PRIu32, sn, nsid); 4100 if (size <= 0 || (unsigned long)size >= sizeof(merged_str)) { 4101 return -EINVAL; 4102 } 4103 4104 spdk_uuid_parse(&namespace_uuid, namespace_str); 4105 4106 rc = spdk_uuid_generate_sha1(&new_uuid, &namespace_uuid, merged_str, size); 4107 if (rc == 0) { 4108 memcpy(uuid, &new_uuid, sizeof(struct spdk_uuid)); 4109 } 4110 4111 return rc; 4112 } 4113 4114 static int 4115 nvme_disk_create(struct spdk_bdev *disk, const char *base_name, 4116 struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns, 4117 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, void *ctx) 4118 { 4119 const struct spdk_uuid *uuid; 4120 const uint8_t *nguid; 4121 const struct spdk_nvme_ctrlr_data *cdata; 4122 const struct spdk_nvme_ns_data *nsdata; 4123 const struct spdk_nvme_ctrlr_opts *opts; 4124 enum spdk_nvme_csi csi; 4125 uint32_t atomic_bs, phys_bs, bs; 4126 char sn_tmp[SPDK_NVME_CTRLR_SN_LEN + 1] = {'\0'}; 4127 int rc; 4128 4129 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4130 csi = spdk_nvme_ns_get_csi(ns); 4131 opts = spdk_nvme_ctrlr_get_opts(ctrlr); 4132 4133 switch (csi) { 4134 case SPDK_NVME_CSI_NVM: 4135 disk->product_name = "NVMe disk"; 4136 break; 4137 case SPDK_NVME_CSI_ZNS: 4138 disk->product_name = "NVMe ZNS disk"; 4139 disk->zoned = true; 4140 disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 4141 disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) / 4142 spdk_nvme_ns_get_extended_sector_size(ns); 4143 disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns); 4144 disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns); 4145 break; 4146 default: 4147 if (bdev_opts->allow_unrecognized_csi) { 4148 disk->product_name = "NVMe Passthrough disk"; 4149 break; 4150 } 4151 SPDK_ERRLOG("unsupported CSI: %u\n", csi); 4152 return -ENOTSUP; 4153 } 4154 4155 nguid = spdk_nvme_ns_get_nguid(ns); 4156 if (!nguid) { 4157 uuid = spdk_nvme_ns_get_uuid(ns); 4158 if (uuid) { 4159 disk->uuid = *uuid; 4160 } else if (g_opts.generate_uuids) { 4161 spdk_strcpy_pad(sn_tmp, cdata->sn, SPDK_NVME_CTRLR_SN_LEN, '\0'); 4162 rc = nvme_generate_uuid(sn_tmp, spdk_nvme_ns_get_id(ns), &disk->uuid); 4163 if (rc < 0) { 4164 SPDK_ERRLOG("UUID generation failed (%s)\n", spdk_strerror(-rc)); 4165 return rc; 4166 } 4167 } 4168 } else { 4169 memcpy(&disk->uuid, nguid, sizeof(disk->uuid)); 4170 } 4171 4172 disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns)); 4173 if (!disk->name) { 4174 return -ENOMEM; 4175 } 4176 4177 disk->write_cache = 0; 4178 if (cdata->vwc.present) { 4179 /* Enable if the Volatile Write Cache exists */ 4180 disk->write_cache = 1; 4181 } 4182 if (cdata->oncs.write_zeroes) { 4183 disk->max_write_zeroes = UINT16_MAX + 1; 4184 } 4185 disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns); 4186 disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns); 4187 disk->max_segment_size = spdk_nvme_ctrlr_get_max_xfer_size(ctrlr); 4188 disk->ctratt.raw = cdata->ctratt.raw; 4189 /* NVMe driver will split one request into multiple requests 4190 * based on MDTS and stripe boundary, the bdev layer will use 4191 * max_segment_size and max_num_segments to split one big IO 4192 * into multiple requests, then small request can't run out 4193 * of NVMe internal requests data structure. 4194 */ 4195 if (opts && opts->io_queue_requests) { 4196 disk->max_num_segments = opts->io_queue_requests / 2; 4197 } 4198 if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_SGL_SUPPORTED) { 4199 /* The nvme driver will try to split I/O that have too many 4200 * SGEs, but it doesn't work if that last SGE doesn't end on 4201 * an aggregate total that is block aligned. The bdev layer has 4202 * a more robust splitting framework, so use that instead for 4203 * this case. (See issue #3269.) 4204 */ 4205 uint16_t max_sges = spdk_nvme_ctrlr_get_max_sges(ctrlr); 4206 4207 if (disk->max_num_segments == 0) { 4208 disk->max_num_segments = max_sges; 4209 } else { 4210 disk->max_num_segments = spdk_min(disk->max_num_segments, max_sges); 4211 } 4212 } 4213 disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns); 4214 4215 nsdata = spdk_nvme_ns_get_data(ns); 4216 bs = spdk_nvme_ns_get_sector_size(ns); 4217 atomic_bs = bs; 4218 phys_bs = bs; 4219 if (nsdata->nabo == 0) { 4220 if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) { 4221 atomic_bs = bs * (1 + nsdata->nawupf); 4222 } else { 4223 atomic_bs = bs * (1 + cdata->awupf); 4224 } 4225 } 4226 if (nsdata->nsfeat.optperf) { 4227 phys_bs = bs * (1 + nsdata->npwg); 4228 } 4229 disk->phys_blocklen = spdk_min(phys_bs, atomic_bs); 4230 4231 disk->md_len = spdk_nvme_ns_get_md_size(ns); 4232 if (disk->md_len != 0) { 4233 disk->md_interleave = nsdata->flbas.extended; 4234 disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns); 4235 if (disk->dif_type != SPDK_DIF_DISABLE) { 4236 disk->dif_is_head_of_md = nsdata->dps.md_start; 4237 disk->dif_check_flags = bdev_opts->prchk_flags; 4238 disk->dif_pi_format = (enum spdk_dif_pi_format)spdk_nvme_ns_get_pi_format(ns); 4239 } 4240 } 4241 4242 if (!(spdk_nvme_ctrlr_get_flags(ctrlr) & 4243 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) { 4244 disk->acwu = 0; 4245 } else if (nsdata->nsfeat.ns_atomic_write_unit) { 4246 disk->acwu = nsdata->nacwu + 1; /* 0-based */ 4247 } else { 4248 disk->acwu = cdata->acwu + 1; /* 0-based */ 4249 } 4250 4251 if (cdata->oncs.copy) { 4252 /* For now bdev interface allows only single segment copy */ 4253 disk->max_copy = nsdata->mssrl; 4254 } 4255 4256 disk->ctxt = ctx; 4257 disk->fn_table = &nvmelib_fn_table; 4258 disk->module = &nvme_if; 4259 4260 return 0; 4261 } 4262 4263 static struct nvme_bdev * 4264 nvme_bdev_alloc(void) 4265 { 4266 struct nvme_bdev *bdev; 4267 int rc; 4268 4269 bdev = calloc(1, sizeof(*bdev)); 4270 if (!bdev) { 4271 SPDK_ERRLOG("bdev calloc() failed\n"); 4272 return NULL; 4273 } 4274 4275 if (g_opts.nvme_error_stat) { 4276 bdev->err_stat = calloc(1, sizeof(struct nvme_error_stat)); 4277 if (!bdev->err_stat) { 4278 SPDK_ERRLOG("err_stat calloc() failed\n"); 4279 free(bdev); 4280 return NULL; 4281 } 4282 } 4283 4284 rc = pthread_mutex_init(&bdev->mutex, NULL); 4285 if (rc != 0) { 4286 free(bdev->err_stat); 4287 free(bdev); 4288 return NULL; 4289 } 4290 4291 bdev->ref = 1; 4292 bdev->mp_policy = BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE; 4293 bdev->mp_selector = BDEV_NVME_MP_SELECTOR_ROUND_ROBIN; 4294 bdev->rr_min_io = UINT32_MAX; 4295 TAILQ_INIT(&bdev->nvme_ns_list); 4296 4297 return bdev; 4298 } 4299 4300 static int 4301 nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4302 { 4303 struct nvme_bdev *bdev; 4304 struct nvme_bdev_ctrlr *nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr; 4305 int rc; 4306 4307 bdev = nvme_bdev_alloc(); 4308 if (bdev == NULL) { 4309 SPDK_ERRLOG("Failed to allocate NVMe bdev\n"); 4310 return -ENOMEM; 4311 } 4312 4313 bdev->opal = nvme_ctrlr->opal_dev != NULL; 4314 4315 rc = nvme_disk_create(&bdev->disk, nbdev_ctrlr->name, nvme_ctrlr->ctrlr, 4316 nvme_ns->ns, &nvme_ctrlr->opts, bdev); 4317 if (rc != 0) { 4318 SPDK_ERRLOG("Failed to create NVMe disk\n"); 4319 nvme_bdev_free(bdev); 4320 return rc; 4321 } 4322 4323 spdk_io_device_register(bdev, 4324 bdev_nvme_create_bdev_channel_cb, 4325 bdev_nvme_destroy_bdev_channel_cb, 4326 sizeof(struct nvme_bdev_channel), 4327 bdev->disk.name); 4328 4329 nvme_ns->bdev = bdev; 4330 bdev->nsid = nvme_ns->id; 4331 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 4332 4333 bdev->nbdev_ctrlr = nbdev_ctrlr; 4334 TAILQ_INSERT_TAIL(&nbdev_ctrlr->bdevs, bdev, tailq); 4335 4336 rc = spdk_bdev_register(&bdev->disk); 4337 if (rc != 0) { 4338 SPDK_ERRLOG("spdk_bdev_register() failed\n"); 4339 spdk_io_device_unregister(bdev, NULL); 4340 nvme_ns->bdev = NULL; 4341 TAILQ_REMOVE(&nbdev_ctrlr->bdevs, bdev, tailq); 4342 nvme_bdev_free(bdev); 4343 return rc; 4344 } 4345 4346 return 0; 4347 } 4348 4349 static bool 4350 bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2) 4351 { 4352 const struct spdk_nvme_ns_data *nsdata1, *nsdata2; 4353 const struct spdk_uuid *uuid1, *uuid2; 4354 4355 nsdata1 = spdk_nvme_ns_get_data(ns1); 4356 nsdata2 = spdk_nvme_ns_get_data(ns2); 4357 uuid1 = spdk_nvme_ns_get_uuid(ns1); 4358 uuid2 = spdk_nvme_ns_get_uuid(ns2); 4359 4360 return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 && 4361 nsdata1->eui64 == nsdata2->eui64 && 4362 ((uuid1 == NULL && uuid2 == NULL) || 4363 (uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) && 4364 spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2); 4365 } 4366 4367 static bool 4368 hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 4369 struct spdk_nvme_ctrlr_opts *opts) 4370 { 4371 struct nvme_probe_skip_entry *entry; 4372 4373 TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) { 4374 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 4375 return false; 4376 } 4377 } 4378 4379 opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst; 4380 opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight; 4381 opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight; 4382 opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight; 4383 opts->disable_read_ana_log_page = true; 4384 4385 SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr); 4386 4387 return true; 4388 } 4389 4390 static void 4391 nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl) 4392 { 4393 struct nvme_ctrlr *nvme_ctrlr = ctx; 4394 4395 if (spdk_nvme_cpl_is_error(cpl)) { 4396 SPDK_WARNLOG("Abort failed. Resetting controller. sc is %u, sct is %u.\n", cpl->status.sc, 4397 cpl->status.sct); 4398 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4399 } else if (cpl->cdw0 & 0x1) { 4400 SPDK_WARNLOG("Specified command could not be aborted.\n"); 4401 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4402 } 4403 } 4404 4405 static void 4406 timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, 4407 struct spdk_nvme_qpair *qpair, uint16_t cid) 4408 { 4409 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 4410 union spdk_nvme_csts_register csts; 4411 int rc; 4412 4413 assert(nvme_ctrlr->ctrlr == ctrlr); 4414 4415 SPDK_WARNLOG("Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n", ctrlr, qpair, cid); 4416 4417 /* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O 4418 * queue. (Note: qpair == NULL when there's an admin cmd timeout.) Otherwise we 4419 * would submit another fabrics cmd on the admin queue to read CSTS and check for its 4420 * completion recursively. 4421 */ 4422 if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) { 4423 csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr); 4424 if (csts.bits.cfs) { 4425 SPDK_ERRLOG("Controller Fatal Status, reset required\n"); 4426 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4427 return; 4428 } 4429 } 4430 4431 switch (g_opts.action_on_timeout) { 4432 case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT: 4433 if (qpair) { 4434 /* Don't send abort to ctrlr when ctrlr is not available. */ 4435 pthread_mutex_lock(&nvme_ctrlr->mutex); 4436 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 4437 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4438 SPDK_NOTICELOG("Quit abort. Ctrlr is not available.\n"); 4439 return; 4440 } 4441 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4442 4443 rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid, 4444 nvme_abort_cpl, nvme_ctrlr); 4445 if (rc == 0) { 4446 return; 4447 } 4448 4449 SPDK_ERRLOG("Unable to send abort. Resetting, rc is %d.\n", rc); 4450 } 4451 4452 /* FALLTHROUGH */ 4453 case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET: 4454 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4455 break; 4456 case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE: 4457 SPDK_DEBUGLOG(bdev_nvme, "No action for nvme controller timeout.\n"); 4458 break; 4459 default: 4460 SPDK_ERRLOG("An invalid timeout action value is found.\n"); 4461 break; 4462 } 4463 } 4464 4465 static struct nvme_ns * 4466 nvme_ns_alloc(void) 4467 { 4468 struct nvme_ns *nvme_ns; 4469 4470 nvme_ns = calloc(1, sizeof(struct nvme_ns)); 4471 if (nvme_ns == NULL) { 4472 return NULL; 4473 } 4474 4475 if (g_opts.io_path_stat) { 4476 nvme_ns->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 4477 if (nvme_ns->stat == NULL) { 4478 free(nvme_ns); 4479 return NULL; 4480 } 4481 spdk_bdev_reset_io_stat(nvme_ns->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 4482 } 4483 4484 return nvme_ns; 4485 } 4486 4487 static void 4488 nvme_ns_free(struct nvme_ns *nvme_ns) 4489 { 4490 free(nvme_ns->stat); 4491 free(nvme_ns); 4492 } 4493 4494 static void 4495 nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc) 4496 { 4497 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 4498 struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx; 4499 4500 if (rc == 0) { 4501 nvme_ns->probe_ctx = NULL; 4502 pthread_mutex_lock(&nvme_ctrlr->mutex); 4503 nvme_ctrlr->ref++; 4504 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4505 } else { 4506 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4507 nvme_ns_free(nvme_ns); 4508 } 4509 4510 if (ctx) { 4511 ctx->populates_in_progress--; 4512 if (ctx->populates_in_progress == 0) { 4513 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 4514 } 4515 } 4516 } 4517 4518 static void 4519 bdev_nvme_add_io_path(struct spdk_io_channel_iter *i) 4520 { 4521 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 4522 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 4523 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 4524 int rc; 4525 4526 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 4527 if (rc != 0) { 4528 SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n"); 4529 } 4530 4531 spdk_for_each_channel_continue(i, rc); 4532 } 4533 4534 static void 4535 bdev_nvme_delete_io_path(struct spdk_io_channel_iter *i) 4536 { 4537 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 4538 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 4539 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 4540 struct nvme_io_path *io_path; 4541 4542 io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns); 4543 if (io_path != NULL) { 4544 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 4545 } 4546 4547 spdk_for_each_channel_continue(i, 0); 4548 } 4549 4550 static void 4551 bdev_nvme_add_io_path_failed(struct spdk_io_channel_iter *i, int status) 4552 { 4553 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 4554 4555 nvme_ctrlr_populate_namespace_done(nvme_ns, -1); 4556 } 4557 4558 static void 4559 bdev_nvme_add_io_path_done(struct spdk_io_channel_iter *i, int status) 4560 { 4561 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 4562 struct nvme_bdev *bdev = spdk_io_channel_iter_get_io_device(i); 4563 4564 if (status == 0) { 4565 nvme_ctrlr_populate_namespace_done(nvme_ns, 0); 4566 } else { 4567 /* Delete the added io_paths and fail populating the namespace. */ 4568 spdk_for_each_channel(bdev, 4569 bdev_nvme_delete_io_path, 4570 nvme_ns, 4571 bdev_nvme_add_io_path_failed); 4572 } 4573 } 4574 4575 static int 4576 nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns) 4577 { 4578 struct nvme_ns *tmp_ns; 4579 const struct spdk_nvme_ns_data *nsdata; 4580 4581 nsdata = spdk_nvme_ns_get_data(nvme_ns->ns); 4582 if (!nsdata->nmic.can_share) { 4583 SPDK_ERRLOG("Namespace cannot be shared.\n"); 4584 return -EINVAL; 4585 } 4586 4587 pthread_mutex_lock(&bdev->mutex); 4588 4589 tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list); 4590 assert(tmp_ns != NULL); 4591 4592 if (tmp_ns->ns != NULL && !bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) { 4593 pthread_mutex_unlock(&bdev->mutex); 4594 SPDK_ERRLOG("Namespaces are not identical.\n"); 4595 return -EINVAL; 4596 } 4597 4598 bdev->ref++; 4599 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 4600 nvme_ns->bdev = bdev; 4601 4602 pthread_mutex_unlock(&bdev->mutex); 4603 4604 /* Add nvme_io_path to nvme_bdev_channels dynamically. */ 4605 spdk_for_each_channel(bdev, 4606 bdev_nvme_add_io_path, 4607 nvme_ns, 4608 bdev_nvme_add_io_path_done); 4609 4610 return 0; 4611 } 4612 4613 static void 4614 nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4615 { 4616 struct spdk_nvme_ns *ns; 4617 struct nvme_bdev *bdev; 4618 int rc = 0; 4619 4620 ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id); 4621 if (!ns) { 4622 SPDK_DEBUGLOG(bdev_nvme, "Invalid NS %d\n", nvme_ns->id); 4623 rc = -EINVAL; 4624 goto done; 4625 } 4626 4627 nvme_ns->ns = ns; 4628 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 4629 4630 if (nvme_ctrlr->ana_log_page != NULL) { 4631 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns); 4632 } 4633 4634 bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id); 4635 if (bdev == NULL) { 4636 rc = nvme_bdev_create(nvme_ctrlr, nvme_ns); 4637 } else { 4638 rc = nvme_bdev_add_ns(bdev, nvme_ns); 4639 if (rc == 0) { 4640 return; 4641 } 4642 } 4643 done: 4644 nvme_ctrlr_populate_namespace_done(nvme_ns, rc); 4645 } 4646 4647 static void 4648 nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns) 4649 { 4650 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 4651 4652 assert(nvme_ctrlr != NULL); 4653 4654 pthread_mutex_lock(&nvme_ctrlr->mutex); 4655 4656 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4657 4658 if (nvme_ns->bdev != NULL) { 4659 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4660 return; 4661 } 4662 4663 nvme_ns_free(nvme_ns); 4664 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4665 4666 nvme_ctrlr_release(nvme_ctrlr); 4667 } 4668 4669 static void 4670 bdev_nvme_delete_io_path_done(struct spdk_io_channel_iter *i, int status) 4671 { 4672 struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i); 4673 4674 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 4675 } 4676 4677 static void 4678 nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4679 { 4680 struct nvme_bdev *bdev; 4681 4682 spdk_poller_unregister(&nvme_ns->anatt_timer); 4683 4684 bdev = nvme_ns->bdev; 4685 if (bdev != NULL) { 4686 pthread_mutex_lock(&bdev->mutex); 4687 4688 assert(bdev->ref > 0); 4689 bdev->ref--; 4690 if (bdev->ref == 0) { 4691 pthread_mutex_unlock(&bdev->mutex); 4692 4693 spdk_bdev_unregister(&bdev->disk, NULL, NULL); 4694 } else { 4695 /* spdk_bdev_unregister() is not called until the last nvme_ns is 4696 * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list 4697 * and clear nvme_ns->bdev here. 4698 */ 4699 TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq); 4700 nvme_ns->bdev = NULL; 4701 4702 pthread_mutex_unlock(&bdev->mutex); 4703 4704 /* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that, 4705 * we call depopulate_namespace_done() to avoid use-after-free. 4706 */ 4707 spdk_for_each_channel(bdev, 4708 bdev_nvme_delete_io_path, 4709 nvme_ns, 4710 bdev_nvme_delete_io_path_done); 4711 return; 4712 } 4713 } 4714 4715 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 4716 } 4717 4718 static void 4719 nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 4720 struct nvme_async_probe_ctx *ctx) 4721 { 4722 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4723 struct nvme_ns *nvme_ns, *next; 4724 struct spdk_nvme_ns *ns; 4725 struct nvme_bdev *bdev; 4726 uint32_t nsid; 4727 int rc; 4728 uint64_t num_sectors; 4729 4730 if (ctx) { 4731 /* Initialize this count to 1 to handle the populate functions 4732 * calling nvme_ctrlr_populate_namespace_done() immediately. 4733 */ 4734 ctx->populates_in_progress = 1; 4735 } 4736 4737 /* First loop over our existing namespaces and see if they have been 4738 * removed. */ 4739 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4740 while (nvme_ns != NULL) { 4741 next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 4742 4743 if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 4744 /* NS is still there or added again. Its attributes may have changed. */ 4745 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id); 4746 if (nvme_ns->ns != ns) { 4747 assert(nvme_ns->ns == NULL); 4748 nvme_ns->ns = ns; 4749 SPDK_DEBUGLOG(bdev_nvme, "NSID %u was added\n", nvme_ns->id); 4750 } 4751 4752 num_sectors = spdk_nvme_ns_get_num_sectors(ns); 4753 bdev = nvme_ns->bdev; 4754 assert(bdev != NULL); 4755 if (bdev->disk.blockcnt != num_sectors) { 4756 SPDK_NOTICELOG("NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n", 4757 nvme_ns->id, 4758 bdev->disk.name, 4759 bdev->disk.blockcnt, 4760 num_sectors); 4761 rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors); 4762 if (rc != 0) { 4763 SPDK_ERRLOG("Could not change num blocks for nvme bdev: name %s, errno: %d.\n", 4764 bdev->disk.name, rc); 4765 } 4766 } 4767 } else { 4768 /* Namespace was removed */ 4769 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 4770 } 4771 4772 nvme_ns = next; 4773 } 4774 4775 /* Loop through all of the namespaces at the nvme level and see if any of them are new */ 4776 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 4777 while (nsid != 0) { 4778 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 4779 4780 if (nvme_ns == NULL) { 4781 /* Found a new one */ 4782 nvme_ns = nvme_ns_alloc(); 4783 if (nvme_ns == NULL) { 4784 SPDK_ERRLOG("Failed to allocate namespace\n"); 4785 /* This just fails to attach the namespace. It may work on a future attempt. */ 4786 continue; 4787 } 4788 4789 nvme_ns->id = nsid; 4790 nvme_ns->ctrlr = nvme_ctrlr; 4791 4792 nvme_ns->bdev = NULL; 4793 4794 if (ctx) { 4795 ctx->populates_in_progress++; 4796 } 4797 nvme_ns->probe_ctx = ctx; 4798 4799 RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4800 4801 nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns); 4802 } 4803 4804 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid); 4805 } 4806 4807 if (ctx) { 4808 /* Decrement this count now that the loop is over to account 4809 * for the one we started with. If the count is then 0, we 4810 * know any populate_namespace functions completed immediately, 4811 * so we'll kick the callback here. 4812 */ 4813 ctx->populates_in_progress--; 4814 if (ctx->populates_in_progress == 0) { 4815 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 4816 } 4817 } 4818 4819 } 4820 4821 static void 4822 nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr) 4823 { 4824 struct nvme_ns *nvme_ns, *tmp; 4825 4826 RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) { 4827 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 4828 } 4829 } 4830 4831 static uint32_t 4832 nvme_ctrlr_get_ana_log_page_size(struct nvme_ctrlr *nvme_ctrlr) 4833 { 4834 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4835 const struct spdk_nvme_ctrlr_data *cdata; 4836 uint32_t nsid, ns_count = 0; 4837 4838 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4839 4840 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 4841 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 4842 ns_count++; 4843 } 4844 4845 return sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 4846 sizeof(struct spdk_nvme_ana_group_descriptor) + ns_count * 4847 sizeof(uint32_t); 4848 } 4849 4850 static int 4851 nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc, 4852 void *cb_arg) 4853 { 4854 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 4855 struct nvme_ns *nvme_ns; 4856 uint32_t i, nsid; 4857 4858 for (i = 0; i < desc->num_of_nsid; i++) { 4859 nsid = desc->nsid[i]; 4860 if (nsid == 0) { 4861 continue; 4862 } 4863 4864 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 4865 4866 if (nvme_ns == NULL) { 4867 /* Target told us that an inactive namespace had an ANA change */ 4868 continue; 4869 } 4870 4871 _nvme_ns_set_ana_state(nvme_ns, desc); 4872 } 4873 4874 return 0; 4875 } 4876 4877 static void 4878 bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 4879 { 4880 struct nvme_ns *nvme_ns; 4881 4882 spdk_free(nvme_ctrlr->ana_log_page); 4883 nvme_ctrlr->ana_log_page = NULL; 4884 4885 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4886 nvme_ns != NULL; 4887 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 4888 nvme_ns->ana_state_updating = false; 4889 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 4890 } 4891 } 4892 4893 static void 4894 nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl) 4895 { 4896 struct nvme_ctrlr *nvme_ctrlr = ctx; 4897 4898 if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) { 4899 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states, 4900 nvme_ctrlr); 4901 } else { 4902 bdev_nvme_disable_read_ana_log_page(nvme_ctrlr); 4903 } 4904 4905 pthread_mutex_lock(&nvme_ctrlr->mutex); 4906 4907 assert(nvme_ctrlr->ana_log_page_updating == true); 4908 nvme_ctrlr->ana_log_page_updating = false; 4909 4910 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 4911 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4912 4913 nvme_ctrlr_unregister(nvme_ctrlr); 4914 } else { 4915 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4916 4917 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 4918 } 4919 } 4920 4921 static int 4922 nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 4923 { 4924 uint32_t ana_log_page_size; 4925 int rc; 4926 4927 if (nvme_ctrlr->ana_log_page == NULL) { 4928 return -EINVAL; 4929 } 4930 4931 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 4932 4933 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 4934 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 4935 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 4936 return -EINVAL; 4937 } 4938 4939 pthread_mutex_lock(&nvme_ctrlr->mutex); 4940 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 4941 nvme_ctrlr->ana_log_page_updating) { 4942 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4943 return -EBUSY; 4944 } 4945 4946 nvme_ctrlr->ana_log_page_updating = true; 4947 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4948 4949 rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr, 4950 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 4951 SPDK_NVME_GLOBAL_NS_TAG, 4952 nvme_ctrlr->ana_log_page, 4953 ana_log_page_size, 0, 4954 nvme_ctrlr_read_ana_log_page_done, 4955 nvme_ctrlr); 4956 if (rc != 0) { 4957 nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL); 4958 } 4959 4960 return rc; 4961 } 4962 4963 static void 4964 dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 4965 { 4966 } 4967 4968 struct bdev_nvme_set_preferred_path_ctx { 4969 struct spdk_bdev_desc *desc; 4970 struct nvme_ns *nvme_ns; 4971 bdev_nvme_set_preferred_path_cb cb_fn; 4972 void *cb_arg; 4973 }; 4974 4975 static void 4976 bdev_nvme_set_preferred_path_done(struct spdk_io_channel_iter *i, int status) 4977 { 4978 struct bdev_nvme_set_preferred_path_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 4979 4980 assert(ctx != NULL); 4981 assert(ctx->desc != NULL); 4982 assert(ctx->cb_fn != NULL); 4983 4984 spdk_bdev_close(ctx->desc); 4985 4986 ctx->cb_fn(ctx->cb_arg, status); 4987 4988 free(ctx); 4989 } 4990 4991 static void 4992 _bdev_nvme_set_preferred_path(struct spdk_io_channel_iter *i) 4993 { 4994 struct bdev_nvme_set_preferred_path_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 4995 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 4996 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 4997 struct nvme_io_path *io_path, *prev; 4998 4999 prev = NULL; 5000 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 5001 if (io_path->nvme_ns == ctx->nvme_ns) { 5002 break; 5003 } 5004 prev = io_path; 5005 } 5006 5007 if (io_path != NULL) { 5008 if (prev != NULL) { 5009 STAILQ_REMOVE_AFTER(&nbdev_ch->io_path_list, prev, stailq); 5010 STAILQ_INSERT_HEAD(&nbdev_ch->io_path_list, io_path, stailq); 5011 } 5012 5013 /* We can set io_path to nbdev_ch->current_io_path directly here. 5014 * However, it needs to be conditional. To simplify the code, 5015 * just clear nbdev_ch->current_io_path and let find_io_path() 5016 * fill it. 5017 * 5018 * Automatic failback may be disabled. Hence even if the io_path is 5019 * already at the head, clear nbdev_ch->current_io_path. 5020 */ 5021 bdev_nvme_clear_current_io_path(nbdev_ch); 5022 } 5023 5024 spdk_for_each_channel_continue(i, 0); 5025 } 5026 5027 static struct nvme_ns * 5028 bdev_nvme_set_preferred_ns(struct nvme_bdev *nbdev, uint16_t cntlid) 5029 { 5030 struct nvme_ns *nvme_ns, *prev; 5031 const struct spdk_nvme_ctrlr_data *cdata; 5032 5033 prev = NULL; 5034 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 5035 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 5036 5037 if (cdata->cntlid == cntlid) { 5038 break; 5039 } 5040 prev = nvme_ns; 5041 } 5042 5043 if (nvme_ns != NULL && prev != NULL) { 5044 TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq); 5045 TAILQ_INSERT_HEAD(&nbdev->nvme_ns_list, nvme_ns, tailq); 5046 } 5047 5048 return nvme_ns; 5049 } 5050 5051 /* This function supports only multipath mode. There is only a single I/O path 5052 * for each NVMe-oF controller. Hence, just move the matched I/O path to the 5053 * head of the I/O path list for each NVMe bdev channel. 5054 * 5055 * NVMe bdev channel may be acquired after completing this function. move the 5056 * matched namespace to the head of the namespace list for the NVMe bdev too. 5057 */ 5058 void 5059 bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid, 5060 bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg) 5061 { 5062 struct bdev_nvme_set_preferred_path_ctx *ctx; 5063 struct spdk_bdev *bdev; 5064 struct nvme_bdev *nbdev; 5065 int rc = 0; 5066 5067 assert(cb_fn != NULL); 5068 5069 ctx = calloc(1, sizeof(*ctx)); 5070 if (ctx == NULL) { 5071 SPDK_ERRLOG("Failed to alloc context.\n"); 5072 rc = -ENOMEM; 5073 goto err_alloc; 5074 } 5075 5076 ctx->cb_fn = cb_fn; 5077 ctx->cb_arg = cb_arg; 5078 5079 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 5080 if (rc != 0) { 5081 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 5082 goto err_open; 5083 } 5084 5085 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 5086 5087 if (bdev->module != &nvme_if) { 5088 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 5089 rc = -ENODEV; 5090 goto err_bdev; 5091 } 5092 5093 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 5094 5095 pthread_mutex_lock(&nbdev->mutex); 5096 5097 ctx->nvme_ns = bdev_nvme_set_preferred_ns(nbdev, cntlid); 5098 if (ctx->nvme_ns == NULL) { 5099 pthread_mutex_unlock(&nbdev->mutex); 5100 5101 SPDK_ERRLOG("bdev %s does not have namespace to controller %u.\n", name, cntlid); 5102 rc = -ENODEV; 5103 goto err_bdev; 5104 } 5105 5106 pthread_mutex_unlock(&nbdev->mutex); 5107 5108 spdk_for_each_channel(nbdev, 5109 _bdev_nvme_set_preferred_path, 5110 ctx, 5111 bdev_nvme_set_preferred_path_done); 5112 return; 5113 5114 err_bdev: 5115 spdk_bdev_close(ctx->desc); 5116 err_open: 5117 free(ctx); 5118 err_alloc: 5119 cb_fn(cb_arg, rc); 5120 } 5121 5122 struct bdev_nvme_set_multipath_policy_ctx { 5123 struct spdk_bdev_desc *desc; 5124 spdk_bdev_nvme_set_multipath_policy_cb cb_fn; 5125 void *cb_arg; 5126 }; 5127 5128 static void 5129 bdev_nvme_set_multipath_policy_done(struct spdk_io_channel_iter *i, int status) 5130 { 5131 struct bdev_nvme_set_multipath_policy_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 5132 5133 assert(ctx != NULL); 5134 assert(ctx->desc != NULL); 5135 assert(ctx->cb_fn != NULL); 5136 5137 spdk_bdev_close(ctx->desc); 5138 5139 ctx->cb_fn(ctx->cb_arg, status); 5140 5141 free(ctx); 5142 } 5143 5144 static void 5145 _bdev_nvme_set_multipath_policy(struct spdk_io_channel_iter *i) 5146 { 5147 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 5148 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch); 5149 struct nvme_bdev *nbdev = spdk_io_channel_get_io_device(_ch); 5150 5151 nbdev_ch->mp_policy = nbdev->mp_policy; 5152 nbdev_ch->mp_selector = nbdev->mp_selector; 5153 nbdev_ch->rr_min_io = nbdev->rr_min_io; 5154 bdev_nvme_clear_current_io_path(nbdev_ch); 5155 5156 spdk_for_each_channel_continue(i, 0); 5157 } 5158 5159 void 5160 spdk_bdev_nvme_set_multipath_policy(const char *name, enum spdk_bdev_nvme_multipath_policy policy, 5161 enum spdk_bdev_nvme_multipath_selector selector, uint32_t rr_min_io, 5162 spdk_bdev_nvme_set_multipath_policy_cb cb_fn, void *cb_arg) 5163 { 5164 struct bdev_nvme_set_multipath_policy_ctx *ctx; 5165 struct spdk_bdev *bdev; 5166 struct nvme_bdev *nbdev; 5167 int rc; 5168 5169 assert(cb_fn != NULL); 5170 5171 switch (policy) { 5172 case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE: 5173 break; 5174 case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE: 5175 switch (selector) { 5176 case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN: 5177 if (rr_min_io == UINT32_MAX) { 5178 rr_min_io = 1; 5179 } else if (rr_min_io == 0) { 5180 rc = -EINVAL; 5181 goto exit; 5182 } 5183 break; 5184 case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH: 5185 break; 5186 default: 5187 rc = -EINVAL; 5188 goto exit; 5189 } 5190 break; 5191 default: 5192 rc = -EINVAL; 5193 goto exit; 5194 } 5195 5196 ctx = calloc(1, sizeof(*ctx)); 5197 if (ctx == NULL) { 5198 SPDK_ERRLOG("Failed to alloc context.\n"); 5199 rc = -ENOMEM; 5200 goto exit; 5201 } 5202 5203 ctx->cb_fn = cb_fn; 5204 ctx->cb_arg = cb_arg; 5205 5206 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 5207 if (rc != 0) { 5208 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 5209 rc = -ENODEV; 5210 goto err_open; 5211 } 5212 5213 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 5214 if (bdev->module != &nvme_if) { 5215 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 5216 rc = -ENODEV; 5217 goto err_module; 5218 } 5219 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 5220 5221 pthread_mutex_lock(&nbdev->mutex); 5222 nbdev->mp_policy = policy; 5223 nbdev->mp_selector = selector; 5224 nbdev->rr_min_io = rr_min_io; 5225 pthread_mutex_unlock(&nbdev->mutex); 5226 5227 spdk_for_each_channel(nbdev, 5228 _bdev_nvme_set_multipath_policy, 5229 ctx, 5230 bdev_nvme_set_multipath_policy_done); 5231 return; 5232 5233 err_module: 5234 spdk_bdev_close(ctx->desc); 5235 err_open: 5236 free(ctx); 5237 exit: 5238 cb_fn(cb_arg, rc); 5239 } 5240 5241 static void 5242 aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 5243 { 5244 struct nvme_ctrlr *nvme_ctrlr = arg; 5245 union spdk_nvme_async_event_completion event; 5246 5247 if (spdk_nvme_cpl_is_error(cpl)) { 5248 SPDK_WARNLOG("AER request execute failed\n"); 5249 return; 5250 } 5251 5252 event.raw = cpl->cdw0; 5253 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 5254 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) { 5255 nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL); 5256 } else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 5257 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) { 5258 nvme_ctrlr_read_ana_log_page(nvme_ctrlr); 5259 } 5260 } 5261 5262 static void 5263 free_nvme_async_probe_ctx(struct nvme_async_probe_ctx *ctx) 5264 { 5265 spdk_keyring_put_key(ctx->drv_opts.tls_psk); 5266 spdk_keyring_put_key(ctx->drv_opts.dhchap_key); 5267 spdk_keyring_put_key(ctx->drv_opts.dhchap_ctrlr_key); 5268 free(ctx); 5269 } 5270 5271 static void 5272 populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, int rc) 5273 { 5274 if (ctx->cb_fn) { 5275 ctx->cb_fn(ctx->cb_ctx, ctx->reported_bdevs, rc); 5276 } 5277 5278 ctx->namespaces_populated = true; 5279 if (ctx->probe_done) { 5280 /* The probe was already completed, so we need to free the context 5281 * here. This can happen for cases like OCSSD, where we need to 5282 * send additional commands to the SSD after attach. 5283 */ 5284 free_nvme_async_probe_ctx(ctx); 5285 } 5286 } 5287 5288 static int 5289 bdev_nvme_remove_poller(void *ctx) 5290 { 5291 struct spdk_nvme_transport_id trid_pcie; 5292 5293 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 5294 spdk_poller_unregister(&g_hotplug_poller); 5295 return SPDK_POLLER_IDLE; 5296 } 5297 5298 memset(&trid_pcie, 0, sizeof(trid_pcie)); 5299 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 5300 5301 if (spdk_nvme_scan_attached(&trid_pcie)) { 5302 SPDK_ERRLOG_RATELIMIT("spdk_nvme_scan_attached() failed\n"); 5303 } 5304 5305 return SPDK_POLLER_BUSY; 5306 } 5307 5308 static void 5309 nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr, 5310 struct nvme_async_probe_ctx *ctx) 5311 { 5312 spdk_io_device_register(nvme_ctrlr, 5313 bdev_nvme_create_ctrlr_channel_cb, 5314 bdev_nvme_destroy_ctrlr_channel_cb, 5315 sizeof(struct nvme_ctrlr_channel), 5316 nvme_ctrlr->nbdev_ctrlr->name); 5317 5318 nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx); 5319 5320 if (g_hotplug_poller == NULL) { 5321 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL, 5322 NVME_HOTPLUG_POLL_PERIOD_DEFAULT); 5323 } 5324 } 5325 5326 static void 5327 nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl) 5328 { 5329 struct nvme_ctrlr *nvme_ctrlr = _ctx; 5330 struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx; 5331 5332 nvme_ctrlr->probe_ctx = NULL; 5333 5334 if (spdk_nvme_cpl_is_error(cpl)) { 5335 nvme_ctrlr_delete(nvme_ctrlr); 5336 5337 if (ctx != NULL) { 5338 ctx->reported_bdevs = 0; 5339 populate_namespaces_cb(ctx, -1); 5340 } 5341 return; 5342 } 5343 5344 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 5345 } 5346 5347 static int 5348 nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 5349 struct nvme_async_probe_ctx *ctx) 5350 { 5351 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 5352 const struct spdk_nvme_ctrlr_data *cdata; 5353 uint32_t ana_log_page_size; 5354 5355 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5356 5357 /* Set buffer size enough to include maximum number of allowed namespaces. */ 5358 ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 5359 sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->mnan * 5360 sizeof(uint32_t); 5361 5362 nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL, 5363 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5364 if (nvme_ctrlr->ana_log_page == NULL) { 5365 SPDK_ERRLOG("could not allocate ANA log page buffer\n"); 5366 return -ENXIO; 5367 } 5368 5369 /* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned. 5370 * Hence copy each descriptor to a temporary area when parsing it. 5371 * 5372 * Allocate a buffer whose size is as large as ANA log page buffer because 5373 * we do not know the size of a descriptor until actually reading it. 5374 */ 5375 nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size); 5376 if (nvme_ctrlr->copied_ana_desc == NULL) { 5377 SPDK_ERRLOG("could not allocate a buffer to parse ANA descriptor\n"); 5378 return -ENOMEM; 5379 } 5380 5381 nvme_ctrlr->max_ana_log_page_size = ana_log_page_size; 5382 5383 nvme_ctrlr->probe_ctx = ctx; 5384 5385 /* Then, set the read size only to include the current active namespaces. */ 5386 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 5387 5388 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 5389 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 5390 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 5391 return -EINVAL; 5392 } 5393 5394 return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, 5395 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 5396 SPDK_NVME_GLOBAL_NS_TAG, 5397 nvme_ctrlr->ana_log_page, 5398 ana_log_page_size, 0, 5399 nvme_ctrlr_init_ana_log_page_done, 5400 nvme_ctrlr); 5401 } 5402 5403 /* hostnqn and subnqn were already verified before attaching a controller. 5404 * Hence check only the multipath capability and cntlid here. 5405 */ 5406 static bool 5407 bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr) 5408 { 5409 struct nvme_ctrlr *tmp; 5410 const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata; 5411 5412 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5413 5414 if (!cdata->cmic.multi_ctrlr) { 5415 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 5416 return false; 5417 } 5418 5419 TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) { 5420 tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr); 5421 5422 if (!tmp_cdata->cmic.multi_ctrlr) { 5423 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 5424 return false; 5425 } 5426 if (cdata->cntlid == tmp_cdata->cntlid) { 5427 SPDK_ERRLOG("cntlid %u are duplicated.\n", tmp_cdata->cntlid); 5428 return false; 5429 } 5430 } 5431 5432 return true; 5433 } 5434 5435 static int 5436 nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr) 5437 { 5438 struct nvme_bdev_ctrlr *nbdev_ctrlr; 5439 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 5440 int rc = 0; 5441 5442 pthread_mutex_lock(&g_bdev_nvme_mutex); 5443 5444 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 5445 if (nbdev_ctrlr != NULL) { 5446 if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) { 5447 rc = -EINVAL; 5448 goto exit; 5449 } 5450 } else { 5451 nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr)); 5452 if (nbdev_ctrlr == NULL) { 5453 SPDK_ERRLOG("Failed to allocate nvme_bdev_ctrlr.\n"); 5454 rc = -ENOMEM; 5455 goto exit; 5456 } 5457 nbdev_ctrlr->name = strdup(name); 5458 if (nbdev_ctrlr->name == NULL) { 5459 SPDK_ERRLOG("Failed to allocate name of nvme_bdev_ctrlr.\n"); 5460 free(nbdev_ctrlr); 5461 goto exit; 5462 } 5463 TAILQ_INIT(&nbdev_ctrlr->ctrlrs); 5464 TAILQ_INIT(&nbdev_ctrlr->bdevs); 5465 TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 5466 } 5467 nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr; 5468 TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 5469 exit: 5470 pthread_mutex_unlock(&g_bdev_nvme_mutex); 5471 return rc; 5472 } 5473 5474 static int 5475 nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr, 5476 const char *name, 5477 const struct spdk_nvme_transport_id *trid, 5478 struct nvme_async_probe_ctx *ctx) 5479 { 5480 struct nvme_ctrlr *nvme_ctrlr; 5481 struct nvme_path_id *path_id; 5482 const struct spdk_nvme_ctrlr_data *cdata; 5483 int rc; 5484 5485 nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr)); 5486 if (nvme_ctrlr == NULL) { 5487 SPDK_ERRLOG("Failed to allocate device struct\n"); 5488 return -ENOMEM; 5489 } 5490 5491 rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL); 5492 if (rc != 0) { 5493 free(nvme_ctrlr); 5494 return rc; 5495 } 5496 5497 TAILQ_INIT(&nvme_ctrlr->trids); 5498 RB_INIT(&nvme_ctrlr->namespaces); 5499 5500 /* Get another reference to the key, so the first one can be released from probe_ctx */ 5501 if (ctx != NULL) { 5502 if (ctx->drv_opts.tls_psk != NULL) { 5503 nvme_ctrlr->psk = spdk_keyring_get_key( 5504 spdk_key_get_name(ctx->drv_opts.tls_psk)); 5505 if (nvme_ctrlr->psk == NULL) { 5506 /* Could only happen if the key was removed in the meantime */ 5507 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5508 spdk_key_get_name(ctx->drv_opts.tls_psk)); 5509 rc = -ENOKEY; 5510 goto err; 5511 } 5512 } 5513 5514 if (ctx->drv_opts.dhchap_key != NULL) { 5515 nvme_ctrlr->dhchap_key = spdk_keyring_get_key( 5516 spdk_key_get_name(ctx->drv_opts.dhchap_key)); 5517 if (nvme_ctrlr->dhchap_key == NULL) { 5518 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5519 spdk_key_get_name(ctx->drv_opts.dhchap_key)); 5520 rc = -ENOKEY; 5521 goto err; 5522 } 5523 } 5524 5525 if (ctx->drv_opts.dhchap_ctrlr_key != NULL) { 5526 nvme_ctrlr->dhchap_ctrlr_key = 5527 spdk_keyring_get_key( 5528 spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key)); 5529 if (nvme_ctrlr->dhchap_ctrlr_key == NULL) { 5530 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5531 spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key)); 5532 rc = -ENOKEY; 5533 goto err; 5534 } 5535 } 5536 } 5537 5538 path_id = calloc(1, sizeof(*path_id)); 5539 if (path_id == NULL) { 5540 SPDK_ERRLOG("Failed to allocate trid entry pointer\n"); 5541 rc = -ENOMEM; 5542 goto err; 5543 } 5544 5545 path_id->trid = *trid; 5546 if (ctx != NULL) { 5547 memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr)); 5548 memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid)); 5549 } 5550 nvme_ctrlr->active_path_id = path_id; 5551 TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link); 5552 5553 nvme_ctrlr->thread = spdk_get_thread(); 5554 nvme_ctrlr->ctrlr = ctrlr; 5555 nvme_ctrlr->ref = 1; 5556 5557 if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) { 5558 SPDK_ERRLOG("OCSSDs are not supported"); 5559 rc = -ENOTSUP; 5560 goto err; 5561 } 5562 5563 if (ctx != NULL) { 5564 memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts)); 5565 } else { 5566 spdk_bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts); 5567 } 5568 5569 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr, 5570 g_opts.nvme_adminq_poll_period_us); 5571 5572 if (g_opts.timeout_us > 0) { 5573 /* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */ 5574 /* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */ 5575 uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ? 5576 g_opts.timeout_us : g_opts.timeout_admin_us; 5577 spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us, 5578 adm_timeout_us, timeout_cb, nvme_ctrlr); 5579 } 5580 5581 spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr); 5582 spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr); 5583 5584 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 5585 SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) { 5586 nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr); 5587 } 5588 5589 rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr); 5590 if (rc != 0) { 5591 goto err; 5592 } 5593 5594 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5595 5596 if (cdata->cmic.ana_reporting) { 5597 rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx); 5598 if (rc == 0) { 5599 return 0; 5600 } 5601 } else { 5602 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 5603 return 0; 5604 } 5605 5606 err: 5607 nvme_ctrlr_delete(nvme_ctrlr); 5608 return rc; 5609 } 5610 5611 void 5612 spdk_bdev_nvme_get_default_ctrlr_opts(struct spdk_bdev_nvme_ctrlr_opts *opts) 5613 { 5614 opts->prchk_flags = 0; 5615 opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec; 5616 opts->reconnect_delay_sec = g_opts.reconnect_delay_sec; 5617 opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec; 5618 } 5619 5620 static void 5621 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 5622 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts) 5623 { 5624 char *name; 5625 5626 name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++); 5627 if (!name) { 5628 SPDK_ERRLOG("Failed to assign name to NVMe device\n"); 5629 return; 5630 } 5631 5632 if (nvme_ctrlr_create(ctrlr, name, trid, NULL) == 0) { 5633 SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name); 5634 } else { 5635 SPDK_ERRLOG("Failed to attach to %s (%s)\n", trid->traddr, name); 5636 } 5637 5638 free(name); 5639 } 5640 5641 static void 5642 _nvme_ctrlr_destruct(void *ctx) 5643 { 5644 struct nvme_ctrlr *nvme_ctrlr = ctx; 5645 5646 nvme_ctrlr_depopulate_namespaces(nvme_ctrlr); 5647 nvme_ctrlr_release(nvme_ctrlr); 5648 } 5649 5650 static int 5651 bdev_nvme_delete_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 5652 { 5653 struct nvme_probe_skip_entry *entry; 5654 5655 /* The controller's destruction was already started */ 5656 if (nvme_ctrlr->destruct) { 5657 return -EALREADY; 5658 } 5659 5660 if (!hotplug && 5661 nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 5662 entry = calloc(1, sizeof(*entry)); 5663 if (!entry) { 5664 return -ENOMEM; 5665 } 5666 entry->trid = nvme_ctrlr->active_path_id->trid; 5667 TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq); 5668 } 5669 5670 nvme_ctrlr->destruct = true; 5671 return 0; 5672 } 5673 5674 static int 5675 bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 5676 { 5677 int rc; 5678 5679 pthread_mutex_lock(&nvme_ctrlr->mutex); 5680 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, hotplug); 5681 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5682 5683 if (rc == 0) { 5684 _nvme_ctrlr_destruct(nvme_ctrlr); 5685 } else if (rc == -EALREADY) { 5686 rc = 0; 5687 } 5688 5689 return rc; 5690 } 5691 5692 static void 5693 remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr) 5694 { 5695 struct nvme_ctrlr *nvme_ctrlr = cb_ctx; 5696 5697 bdev_nvme_delete_ctrlr(nvme_ctrlr, true); 5698 } 5699 5700 static int 5701 bdev_nvme_hotplug_probe(void *arg) 5702 { 5703 if (g_hotplug_probe_ctx == NULL) { 5704 spdk_poller_unregister(&g_hotplug_probe_poller); 5705 return SPDK_POLLER_IDLE; 5706 } 5707 5708 if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) { 5709 g_hotplug_probe_ctx = NULL; 5710 spdk_poller_unregister(&g_hotplug_probe_poller); 5711 } 5712 5713 return SPDK_POLLER_BUSY; 5714 } 5715 5716 static int 5717 bdev_nvme_hotplug(void *arg) 5718 { 5719 struct spdk_nvme_transport_id trid_pcie; 5720 5721 if (g_hotplug_probe_ctx) { 5722 return SPDK_POLLER_BUSY; 5723 } 5724 5725 memset(&trid_pcie, 0, sizeof(trid_pcie)); 5726 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 5727 5728 g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL, 5729 hotplug_probe_cb, attach_cb, NULL); 5730 5731 if (g_hotplug_probe_ctx) { 5732 assert(g_hotplug_probe_poller == NULL); 5733 g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000); 5734 } 5735 5736 return SPDK_POLLER_BUSY; 5737 } 5738 5739 void 5740 bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts) 5741 { 5742 *opts = g_opts; 5743 } 5744 5745 static bool bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 5746 uint32_t reconnect_delay_sec, 5747 uint32_t fast_io_fail_timeout_sec); 5748 5749 static int 5750 bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts) 5751 { 5752 if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) { 5753 /* Can't set timeout_admin_us without also setting timeout_us */ 5754 SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n"); 5755 return -EINVAL; 5756 } 5757 5758 if (opts->bdev_retry_count < -1) { 5759 SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n"); 5760 return -EINVAL; 5761 } 5762 5763 if (!bdev_nvme_check_io_error_resiliency_params(opts->ctrlr_loss_timeout_sec, 5764 opts->reconnect_delay_sec, 5765 opts->fast_io_fail_timeout_sec)) { 5766 return -EINVAL; 5767 } 5768 5769 return 0; 5770 } 5771 5772 int 5773 bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) 5774 { 5775 int ret; 5776 5777 ret = bdev_nvme_validate_opts(opts); 5778 if (ret) { 5779 SPDK_WARNLOG("Failed to set nvme opts.\n"); 5780 return ret; 5781 } 5782 5783 if (g_bdev_nvme_init_thread != NULL) { 5784 if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 5785 return -EPERM; 5786 } 5787 } 5788 5789 if (opts->rdma_srq_size != 0 || 5790 opts->rdma_max_cq_size != 0 || 5791 opts->rdma_cm_event_timeout_ms != 0) { 5792 struct spdk_nvme_transport_opts drv_opts; 5793 5794 spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts)); 5795 if (opts->rdma_srq_size != 0) { 5796 drv_opts.rdma_srq_size = opts->rdma_srq_size; 5797 } 5798 if (opts->rdma_max_cq_size != 0) { 5799 drv_opts.rdma_max_cq_size = opts->rdma_max_cq_size; 5800 } 5801 if (opts->rdma_cm_event_timeout_ms != 0) { 5802 drv_opts.rdma_cm_event_timeout_ms = opts->rdma_cm_event_timeout_ms; 5803 } 5804 5805 ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts)); 5806 if (ret) { 5807 SPDK_ERRLOG("Failed to set NVMe transport opts.\n"); 5808 return ret; 5809 } 5810 } 5811 5812 g_opts = *opts; 5813 5814 return 0; 5815 } 5816 5817 struct set_nvme_hotplug_ctx { 5818 uint64_t period_us; 5819 bool enabled; 5820 spdk_msg_fn fn; 5821 void *fn_ctx; 5822 }; 5823 5824 static void 5825 set_nvme_hotplug_period_cb(void *_ctx) 5826 { 5827 struct set_nvme_hotplug_ctx *ctx = _ctx; 5828 5829 spdk_poller_unregister(&g_hotplug_poller); 5830 if (ctx->enabled) { 5831 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us); 5832 } else { 5833 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL, 5834 NVME_HOTPLUG_POLL_PERIOD_DEFAULT); 5835 } 5836 5837 g_nvme_hotplug_poll_period_us = ctx->period_us; 5838 g_nvme_hotplug_enabled = ctx->enabled; 5839 if (ctx->fn) { 5840 ctx->fn(ctx->fn_ctx); 5841 } 5842 5843 free(ctx); 5844 } 5845 5846 int 5847 bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx) 5848 { 5849 struct set_nvme_hotplug_ctx *ctx; 5850 5851 if (enabled == true && !spdk_process_is_primary()) { 5852 return -EPERM; 5853 } 5854 5855 ctx = calloc(1, sizeof(*ctx)); 5856 if (ctx == NULL) { 5857 return -ENOMEM; 5858 } 5859 5860 period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us; 5861 ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX); 5862 ctx->enabled = enabled; 5863 ctx->fn = cb; 5864 ctx->fn_ctx = cb_ctx; 5865 5866 spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx); 5867 return 0; 5868 } 5869 5870 static void 5871 nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 5872 struct nvme_async_probe_ctx *ctx) 5873 { 5874 struct nvme_ns *nvme_ns; 5875 struct nvme_bdev *nvme_bdev; 5876 size_t j; 5877 5878 assert(nvme_ctrlr != NULL); 5879 5880 if (ctx->names == NULL) { 5881 ctx->reported_bdevs = 0; 5882 populate_namespaces_cb(ctx, 0); 5883 return; 5884 } 5885 5886 /* 5887 * Report the new bdevs that were created in this call. 5888 * There can be more than one bdev per NVMe controller. 5889 */ 5890 j = 0; 5891 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 5892 while (nvme_ns != NULL) { 5893 nvme_bdev = nvme_ns->bdev; 5894 if (j < ctx->max_bdevs) { 5895 ctx->names[j] = nvme_bdev->disk.name; 5896 j++; 5897 } else { 5898 SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n", 5899 ctx->max_bdevs); 5900 ctx->reported_bdevs = 0; 5901 populate_namespaces_cb(ctx, -ERANGE); 5902 return; 5903 } 5904 5905 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 5906 } 5907 5908 ctx->reported_bdevs = j; 5909 populate_namespaces_cb(ctx, 0); 5910 } 5911 5912 static int 5913 bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 5914 struct spdk_nvme_ctrlr *new_ctrlr, 5915 struct spdk_nvme_transport_id *trid) 5916 { 5917 struct nvme_path_id *tmp_trid; 5918 5919 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 5920 SPDK_ERRLOG("PCIe failover is not supported.\n"); 5921 return -ENOTSUP; 5922 } 5923 5924 /* Currently we only support failover to the same transport type. */ 5925 if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) { 5926 SPDK_WARNLOG("Failover from trtype: %s to a different trtype: %s is not supported currently\n", 5927 spdk_nvme_transport_id_trtype_str(nvme_ctrlr->active_path_id->trid.trtype), 5928 spdk_nvme_transport_id_trtype_str(trid->trtype)); 5929 return -EINVAL; 5930 } 5931 5932 5933 /* Currently we only support failover to the same NQN. */ 5934 if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) { 5935 SPDK_WARNLOG("Failover from subnqn: %s to a different subnqn: %s is not supported currently\n", 5936 nvme_ctrlr->active_path_id->trid.subnqn, trid->subnqn); 5937 return -EINVAL; 5938 } 5939 5940 /* Skip all the other checks if we've already registered this path. */ 5941 TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) { 5942 if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) { 5943 SPDK_WARNLOG("This path (traddr: %s subnqn: %s) is already registered\n", trid->traddr, 5944 trid->subnqn); 5945 return -EALREADY; 5946 } 5947 } 5948 5949 return 0; 5950 } 5951 5952 static int 5953 bdev_nvme_check_secondary_namespace(struct nvme_ctrlr *nvme_ctrlr, 5954 struct spdk_nvme_ctrlr *new_ctrlr) 5955 { 5956 struct nvme_ns *nvme_ns; 5957 struct spdk_nvme_ns *new_ns; 5958 5959 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 5960 while (nvme_ns != NULL) { 5961 new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id); 5962 assert(new_ns != NULL); 5963 5964 if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) { 5965 return -EINVAL; 5966 } 5967 5968 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 5969 } 5970 5971 return 0; 5972 } 5973 5974 static int 5975 _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 5976 struct spdk_nvme_transport_id *trid) 5977 { 5978 struct nvme_path_id *active_id, *new_trid, *tmp_trid; 5979 5980 new_trid = calloc(1, sizeof(*new_trid)); 5981 if (new_trid == NULL) { 5982 return -ENOMEM; 5983 } 5984 new_trid->trid = *trid; 5985 5986 active_id = nvme_ctrlr->active_path_id; 5987 assert(active_id != NULL); 5988 assert(active_id == TAILQ_FIRST(&nvme_ctrlr->trids)); 5989 5990 /* Skip the active trid not to replace it until it is failed. */ 5991 tmp_trid = TAILQ_NEXT(active_id, link); 5992 if (tmp_trid == NULL) { 5993 goto add_tail; 5994 } 5995 5996 /* It means the trid is faled if its last failed time is non-zero. 5997 * Insert the new alternate trid before any failed trid. 5998 */ 5999 TAILQ_FOREACH_FROM(tmp_trid, &nvme_ctrlr->trids, link) { 6000 if (tmp_trid->last_failed_tsc != 0) { 6001 TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link); 6002 return 0; 6003 } 6004 } 6005 6006 add_tail: 6007 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link); 6008 return 0; 6009 } 6010 6011 /* This is the case that a secondary path is added to an existing 6012 * nvme_ctrlr for failover. After checking if it can access the same 6013 * namespaces as the primary path, it is disconnected until failover occurs. 6014 */ 6015 static int 6016 bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 6017 struct spdk_nvme_ctrlr *new_ctrlr, 6018 struct spdk_nvme_transport_id *trid) 6019 { 6020 int rc; 6021 6022 assert(nvme_ctrlr != NULL); 6023 6024 pthread_mutex_lock(&nvme_ctrlr->mutex); 6025 6026 rc = bdev_nvme_check_secondary_trid(nvme_ctrlr, new_ctrlr, trid); 6027 if (rc != 0) { 6028 goto exit; 6029 } 6030 6031 rc = bdev_nvme_check_secondary_namespace(nvme_ctrlr, new_ctrlr); 6032 if (rc != 0) { 6033 goto exit; 6034 } 6035 6036 rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid); 6037 6038 exit: 6039 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6040 6041 spdk_nvme_detach(new_ctrlr); 6042 6043 return rc; 6044 } 6045 6046 static void 6047 connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 6048 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 6049 { 6050 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 6051 struct nvme_async_probe_ctx *ctx; 6052 int rc; 6053 6054 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 6055 ctx->ctrlr_attached = true; 6056 6057 rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx); 6058 if (rc != 0) { 6059 ctx->reported_bdevs = 0; 6060 populate_namespaces_cb(ctx, rc); 6061 } 6062 } 6063 6064 static void 6065 connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 6066 struct spdk_nvme_ctrlr *ctrlr, 6067 const struct spdk_nvme_ctrlr_opts *opts) 6068 { 6069 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 6070 struct nvme_ctrlr *nvme_ctrlr; 6071 struct nvme_async_probe_ctx *ctx; 6072 int rc; 6073 6074 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 6075 ctx->ctrlr_attached = true; 6076 6077 nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name); 6078 if (nvme_ctrlr) { 6079 rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid); 6080 } else { 6081 rc = -ENODEV; 6082 } 6083 6084 ctx->reported_bdevs = 0; 6085 populate_namespaces_cb(ctx, rc); 6086 } 6087 6088 static int 6089 bdev_nvme_async_poll(void *arg) 6090 { 6091 struct nvme_async_probe_ctx *ctx = arg; 6092 int rc; 6093 6094 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 6095 if (spdk_unlikely(rc != -EAGAIN)) { 6096 ctx->probe_done = true; 6097 spdk_poller_unregister(&ctx->poller); 6098 if (!ctx->ctrlr_attached) { 6099 /* The probe is done, but no controller was attached. 6100 * That means we had a failure, so report -EIO back to 6101 * the caller (usually the RPC). populate_namespaces_cb() 6102 * will take care of freeing the nvme_async_probe_ctx. 6103 */ 6104 ctx->reported_bdevs = 0; 6105 populate_namespaces_cb(ctx, -EIO); 6106 } else if (ctx->namespaces_populated) { 6107 /* The namespaces for the attached controller were all 6108 * populated and the response was already sent to the 6109 * caller (usually the RPC). So free the context here. 6110 */ 6111 free_nvme_async_probe_ctx(ctx); 6112 } 6113 } 6114 6115 return SPDK_POLLER_BUSY; 6116 } 6117 6118 static bool 6119 bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 6120 uint32_t reconnect_delay_sec, 6121 uint32_t fast_io_fail_timeout_sec) 6122 { 6123 if (ctrlr_loss_timeout_sec < -1) { 6124 SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n"); 6125 return false; 6126 } else if (ctrlr_loss_timeout_sec == -1) { 6127 if (reconnect_delay_sec == 0) { 6128 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 6129 return false; 6130 } else if (fast_io_fail_timeout_sec != 0 && 6131 fast_io_fail_timeout_sec < reconnect_delay_sec) { 6132 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n"); 6133 return false; 6134 } 6135 } else if (ctrlr_loss_timeout_sec != 0) { 6136 if (reconnect_delay_sec == 0) { 6137 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 6138 return false; 6139 } else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) { 6140 SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n"); 6141 return false; 6142 } else if (fast_io_fail_timeout_sec != 0) { 6143 if (fast_io_fail_timeout_sec < reconnect_delay_sec) { 6144 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n"); 6145 return false; 6146 } else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) { 6147 SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n"); 6148 return false; 6149 } 6150 } 6151 } else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) { 6152 SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n"); 6153 return false; 6154 } 6155 6156 return true; 6157 } 6158 6159 static int 6160 bdev_nvme_load_psk(const char *fname, char *buf, size_t bufsz) 6161 { 6162 FILE *psk_file; 6163 struct stat statbuf; 6164 int rc; 6165 #define TCP_PSK_INVALID_PERMISSIONS 0177 6166 6167 if (stat(fname, &statbuf) != 0) { 6168 SPDK_ERRLOG("Could not read permissions for PSK file\n"); 6169 return -EACCES; 6170 } 6171 6172 if ((statbuf.st_mode & TCP_PSK_INVALID_PERMISSIONS) != 0) { 6173 SPDK_ERRLOG("Incorrect permissions for PSK file\n"); 6174 return -EPERM; 6175 } 6176 if ((size_t)statbuf.st_size >= bufsz) { 6177 SPDK_ERRLOG("Invalid PSK: too long\n"); 6178 return -EINVAL; 6179 } 6180 psk_file = fopen(fname, "r"); 6181 if (psk_file == NULL) { 6182 SPDK_ERRLOG("Could not open PSK file\n"); 6183 return -EINVAL; 6184 } 6185 6186 memset(buf, 0, bufsz); 6187 rc = fread(buf, 1, statbuf.st_size, psk_file); 6188 if (rc != statbuf.st_size) { 6189 SPDK_ERRLOG("Failed to read PSK\n"); 6190 fclose(psk_file); 6191 return -EINVAL; 6192 } 6193 6194 fclose(psk_file); 6195 return 0; 6196 } 6197 6198 int 6199 spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid, 6200 const char *base_name, 6201 const char **names, 6202 uint32_t count, 6203 spdk_bdev_nvme_create_cb cb_fn, 6204 void *cb_ctx, 6205 struct spdk_nvme_ctrlr_opts *drv_opts, 6206 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, 6207 bool multipath) 6208 { 6209 struct nvme_probe_skip_entry *entry, *tmp; 6210 struct nvme_async_probe_ctx *ctx; 6211 spdk_nvme_attach_cb attach_cb; 6212 int rc, len; 6213 6214 /* TODO expand this check to include both the host and target TRIDs. 6215 * Only if both are the same should we fail. 6216 */ 6217 if (nvme_ctrlr_get(trid, drv_opts->hostnqn) != NULL) { 6218 SPDK_ERRLOG("A controller with the provided trid (traddr: %s, hostnqn: %s) " 6219 "already exists.\n", trid->traddr, drv_opts->hostnqn); 6220 return -EEXIST; 6221 } 6222 6223 len = strnlen(base_name, SPDK_CONTROLLER_NAME_MAX); 6224 6225 if (len == 0 || len == SPDK_CONTROLLER_NAME_MAX) { 6226 SPDK_ERRLOG("controller name must be between 1 and %d characters\n", SPDK_CONTROLLER_NAME_MAX - 1); 6227 return -EINVAL; 6228 } 6229 6230 if (bdev_opts != NULL && 6231 !bdev_nvme_check_io_error_resiliency_params(bdev_opts->ctrlr_loss_timeout_sec, 6232 bdev_opts->reconnect_delay_sec, 6233 bdev_opts->fast_io_fail_timeout_sec)) { 6234 return -EINVAL; 6235 } 6236 6237 ctx = calloc(1, sizeof(*ctx)); 6238 if (!ctx) { 6239 return -ENOMEM; 6240 } 6241 ctx->base_name = base_name; 6242 ctx->names = names; 6243 ctx->max_bdevs = count; 6244 ctx->cb_fn = cb_fn; 6245 ctx->cb_ctx = cb_ctx; 6246 ctx->trid = *trid; 6247 6248 if (bdev_opts) { 6249 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 6250 } else { 6251 spdk_bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts); 6252 } 6253 6254 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 6255 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) { 6256 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 6257 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 6258 free(entry); 6259 break; 6260 } 6261 } 6262 } 6263 6264 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 6265 ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count; 6266 ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout; 6267 ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms; 6268 ctx->drv_opts.disable_read_ana_log_page = true; 6269 ctx->drv_opts.transport_tos = g_opts.transport_tos; 6270 6271 if (ctx->bdev_opts.psk[0] != '\0') { 6272 /* Try to use the keyring first */ 6273 ctx->drv_opts.tls_psk = spdk_keyring_get_key(ctx->bdev_opts.psk); 6274 if (ctx->drv_opts.tls_psk == NULL) { 6275 rc = bdev_nvme_load_psk(ctx->bdev_opts.psk, 6276 ctx->drv_opts.psk, sizeof(ctx->drv_opts.psk)); 6277 if (rc != 0) { 6278 SPDK_ERRLOG("Could not load PSK from %s\n", ctx->bdev_opts.psk); 6279 free_nvme_async_probe_ctx(ctx); 6280 return rc; 6281 } 6282 } 6283 } 6284 6285 if (ctx->bdev_opts.dhchap_key != NULL) { 6286 ctx->drv_opts.dhchap_key = spdk_keyring_get_key(ctx->bdev_opts.dhchap_key); 6287 if (ctx->drv_opts.dhchap_key == NULL) { 6288 SPDK_ERRLOG("Could not load DH-HMAC-CHAP key: %s\n", 6289 ctx->bdev_opts.dhchap_key); 6290 free_nvme_async_probe_ctx(ctx); 6291 return -ENOKEY; 6292 } 6293 6294 ctx->drv_opts.dhchap_digests = g_opts.dhchap_digests; 6295 ctx->drv_opts.dhchap_dhgroups = g_opts.dhchap_dhgroups; 6296 } 6297 if (ctx->bdev_opts.dhchap_ctrlr_key != NULL) { 6298 ctx->drv_opts.dhchap_ctrlr_key = 6299 spdk_keyring_get_key(ctx->bdev_opts.dhchap_ctrlr_key); 6300 if (ctx->drv_opts.dhchap_ctrlr_key == NULL) { 6301 SPDK_ERRLOG("Could not load DH-HMAC-CHAP controller key: %s\n", 6302 ctx->bdev_opts.dhchap_ctrlr_key); 6303 free_nvme_async_probe_ctx(ctx); 6304 return -ENOKEY; 6305 } 6306 } 6307 6308 if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || multipath) { 6309 attach_cb = connect_attach_cb; 6310 } else { 6311 attach_cb = connect_set_failover_cb; 6312 } 6313 6314 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb); 6315 if (ctx->probe_ctx == NULL) { 6316 SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr); 6317 free_nvme_async_probe_ctx(ctx); 6318 return -ENODEV; 6319 } 6320 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000); 6321 6322 return 0; 6323 } 6324 6325 struct bdev_nvme_delete_ctx { 6326 char *name; 6327 struct nvme_path_id path_id; 6328 bdev_nvme_delete_done_fn delete_done; 6329 void *delete_done_ctx; 6330 uint64_t timeout_ticks; 6331 struct spdk_poller *poller; 6332 }; 6333 6334 static void 6335 free_bdev_nvme_delete_ctx(struct bdev_nvme_delete_ctx *ctx) 6336 { 6337 if (ctx != NULL) { 6338 free(ctx->name); 6339 free(ctx); 6340 } 6341 } 6342 6343 static bool 6344 nvme_path_id_compare(struct nvme_path_id *p, const struct nvme_path_id *path_id) 6345 { 6346 if (path_id->trid.trtype != 0) { 6347 if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) { 6348 if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) { 6349 return false; 6350 } 6351 } else { 6352 if (path_id->trid.trtype != p->trid.trtype) { 6353 return false; 6354 } 6355 } 6356 } 6357 6358 if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) { 6359 if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) { 6360 return false; 6361 } 6362 } 6363 6364 if (path_id->trid.adrfam != 0) { 6365 if (path_id->trid.adrfam != p->trid.adrfam) { 6366 return false; 6367 } 6368 } 6369 6370 if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) { 6371 if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) { 6372 return false; 6373 } 6374 } 6375 6376 if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) { 6377 if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) { 6378 return false; 6379 } 6380 } 6381 6382 if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) { 6383 if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) { 6384 return false; 6385 } 6386 } 6387 6388 if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) { 6389 if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) { 6390 return false; 6391 } 6392 } 6393 6394 return true; 6395 } 6396 6397 static bool 6398 nvme_path_id_exists(const char *name, const struct nvme_path_id *path_id) 6399 { 6400 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6401 struct nvme_ctrlr *ctrlr; 6402 struct nvme_path_id *p; 6403 6404 pthread_mutex_lock(&g_bdev_nvme_mutex); 6405 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 6406 if (!nbdev_ctrlr) { 6407 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6408 return false; 6409 } 6410 6411 TAILQ_FOREACH(ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 6412 pthread_mutex_lock(&ctrlr->mutex); 6413 TAILQ_FOREACH(p, &ctrlr->trids, link) { 6414 if (nvme_path_id_compare(p, path_id)) { 6415 pthread_mutex_unlock(&ctrlr->mutex); 6416 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6417 return true; 6418 } 6419 } 6420 pthread_mutex_unlock(&ctrlr->mutex); 6421 } 6422 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6423 6424 return false; 6425 } 6426 6427 static int 6428 bdev_nvme_delete_complete_poll(void *arg) 6429 { 6430 struct bdev_nvme_delete_ctx *ctx = arg; 6431 int rc = 0; 6432 6433 if (nvme_path_id_exists(ctx->name, &ctx->path_id)) { 6434 if (ctx->timeout_ticks > spdk_get_ticks()) { 6435 return SPDK_POLLER_BUSY; 6436 } 6437 6438 SPDK_ERRLOG("NVMe path '%s' still exists after delete\n", ctx->name); 6439 rc = -ETIMEDOUT; 6440 } 6441 6442 spdk_poller_unregister(&ctx->poller); 6443 6444 ctx->delete_done(ctx->delete_done_ctx, rc); 6445 free_bdev_nvme_delete_ctx(ctx); 6446 6447 return SPDK_POLLER_BUSY; 6448 } 6449 6450 static int 6451 _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id) 6452 { 6453 struct nvme_path_id *p, *t; 6454 spdk_msg_fn msg_fn; 6455 int rc = -ENXIO; 6456 6457 pthread_mutex_lock(&nvme_ctrlr->mutex); 6458 6459 TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) { 6460 if (p == TAILQ_FIRST(&nvme_ctrlr->trids)) { 6461 break; 6462 } 6463 6464 if (!nvme_path_id_compare(p, path_id)) { 6465 continue; 6466 } 6467 6468 /* We are not using the specified path. */ 6469 TAILQ_REMOVE(&nvme_ctrlr->trids, p, link); 6470 free(p); 6471 rc = 0; 6472 } 6473 6474 if (p == NULL || !nvme_path_id_compare(p, path_id)) { 6475 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6476 return rc; 6477 } 6478 6479 /* If we made it here, then this path is a match! Now we need to remove it. */ 6480 6481 /* This is the active path in use right now. The active path is always the first in the list. */ 6482 assert(p == nvme_ctrlr->active_path_id); 6483 6484 if (!TAILQ_NEXT(p, link)) { 6485 /* The current path is the only path. */ 6486 msg_fn = _nvme_ctrlr_destruct; 6487 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, false); 6488 } else { 6489 /* There is an alternative path. */ 6490 msg_fn = _bdev_nvme_reset_ctrlr; 6491 rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, true); 6492 } 6493 6494 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6495 6496 if (rc == 0) { 6497 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 6498 } else if (rc == -EALREADY) { 6499 rc = 0; 6500 } 6501 6502 return rc; 6503 } 6504 6505 int 6506 bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id, 6507 bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx) 6508 { 6509 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6510 struct nvme_ctrlr *nvme_ctrlr, *tmp_nvme_ctrlr; 6511 struct bdev_nvme_delete_ctx *ctx = NULL; 6512 int rc = -ENXIO, _rc; 6513 6514 if (name == NULL || path_id == NULL) { 6515 rc = -EINVAL; 6516 goto exit; 6517 } 6518 6519 pthread_mutex_lock(&g_bdev_nvme_mutex); 6520 6521 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 6522 if (nbdev_ctrlr == NULL) { 6523 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6524 6525 SPDK_ERRLOG("Failed to find NVMe bdev controller\n"); 6526 rc = -ENODEV; 6527 goto exit; 6528 } 6529 6530 TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) { 6531 _rc = _bdev_nvme_delete(nvme_ctrlr, path_id); 6532 if (_rc < 0 && _rc != -ENXIO) { 6533 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6534 rc = _rc; 6535 goto exit; 6536 } else if (_rc == 0) { 6537 /* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr 6538 * was deleted successfully. To remember the successful deletion, 6539 * overwrite rc only if _rc is zero. 6540 */ 6541 rc = 0; 6542 } 6543 } 6544 6545 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6546 6547 if (rc != 0 || delete_done == NULL) { 6548 goto exit; 6549 } 6550 6551 ctx = calloc(1, sizeof(*ctx)); 6552 if (ctx == NULL) { 6553 SPDK_ERRLOG("Failed to allocate context for bdev_nvme_delete\n"); 6554 rc = -ENOMEM; 6555 goto exit; 6556 } 6557 6558 ctx->name = strdup(name); 6559 if (ctx->name == NULL) { 6560 SPDK_ERRLOG("Failed to copy controller name for deletion\n"); 6561 rc = -ENOMEM; 6562 goto exit; 6563 } 6564 6565 ctx->delete_done = delete_done; 6566 ctx->delete_done_ctx = delete_done_ctx; 6567 ctx->path_id = *path_id; 6568 ctx->timeout_ticks = spdk_get_ticks() + 10 * spdk_get_ticks_hz(); 6569 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_delete_complete_poll, ctx, 1000); 6570 if (ctx->poller == NULL) { 6571 SPDK_ERRLOG("Failed to register bdev_nvme_delete poller\n"); 6572 rc = -ENOMEM; 6573 goto exit; 6574 } 6575 6576 exit: 6577 if (rc != 0) { 6578 free_bdev_nvme_delete_ctx(ctx); 6579 } 6580 6581 return rc; 6582 } 6583 6584 #define DISCOVERY_INFOLOG(ctx, format, ...) \ 6585 SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 6586 6587 #define DISCOVERY_ERRLOG(ctx, format, ...) \ 6588 SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 6589 6590 struct discovery_entry_ctx { 6591 char name[128]; 6592 struct spdk_nvme_transport_id trid; 6593 struct spdk_nvme_ctrlr_opts drv_opts; 6594 struct spdk_nvmf_discovery_log_page_entry entry; 6595 TAILQ_ENTRY(discovery_entry_ctx) tailq; 6596 struct discovery_ctx *ctx; 6597 }; 6598 6599 struct discovery_ctx { 6600 char *name; 6601 spdk_bdev_nvme_start_discovery_fn start_cb_fn; 6602 spdk_bdev_nvme_stop_discovery_fn stop_cb_fn; 6603 void *cb_ctx; 6604 struct spdk_nvme_probe_ctx *probe_ctx; 6605 struct spdk_nvme_detach_ctx *detach_ctx; 6606 struct spdk_nvme_ctrlr *ctrlr; 6607 struct spdk_nvme_transport_id trid; 6608 struct discovery_entry_ctx *entry_ctx_in_use; 6609 struct spdk_poller *poller; 6610 struct spdk_nvme_ctrlr_opts drv_opts; 6611 struct spdk_bdev_nvme_ctrlr_opts bdev_opts; 6612 struct spdk_nvmf_discovery_log_page *log_page; 6613 TAILQ_ENTRY(discovery_ctx) tailq; 6614 TAILQ_HEAD(, discovery_entry_ctx) nvm_entry_ctxs; 6615 TAILQ_HEAD(, discovery_entry_ctx) discovery_entry_ctxs; 6616 int rc; 6617 bool wait_for_attach; 6618 uint64_t timeout_ticks; 6619 /* Denotes that the discovery service is being started. We're waiting 6620 * for the initial connection to the discovery controller to be 6621 * established and attach discovered NVM ctrlrs. 6622 */ 6623 bool initializing; 6624 /* Denotes if a discovery is currently in progress for this context. 6625 * That includes connecting to newly discovered subsystems. Used to 6626 * ensure we do not start a new discovery until an existing one is 6627 * complete. 6628 */ 6629 bool in_progress; 6630 6631 /* Denotes if another discovery is needed after the one in progress 6632 * completes. Set when we receive an AER completion while a discovery 6633 * is already in progress. 6634 */ 6635 bool pending; 6636 6637 /* Signal to the discovery context poller that it should stop the 6638 * discovery service, including detaching from the current discovery 6639 * controller. 6640 */ 6641 bool stop; 6642 6643 struct spdk_thread *calling_thread; 6644 uint32_t index; 6645 uint32_t attach_in_progress; 6646 char *hostnqn; 6647 6648 /* Denotes if the discovery service was started by the mdns discovery. 6649 */ 6650 bool from_mdns_discovery_service; 6651 }; 6652 6653 TAILQ_HEAD(discovery_ctxs, discovery_ctx); 6654 static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs); 6655 6656 static void get_discovery_log_page(struct discovery_ctx *ctx); 6657 6658 static void 6659 free_discovery_ctx(struct discovery_ctx *ctx) 6660 { 6661 free(ctx->log_page); 6662 free(ctx->hostnqn); 6663 free(ctx->name); 6664 free(ctx); 6665 } 6666 6667 static void 6668 discovery_complete(struct discovery_ctx *ctx) 6669 { 6670 ctx->initializing = false; 6671 ctx->in_progress = false; 6672 if (ctx->pending) { 6673 ctx->pending = false; 6674 get_discovery_log_page(ctx); 6675 } 6676 } 6677 6678 static void 6679 build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid, 6680 struct spdk_nvmf_discovery_log_page_entry *entry) 6681 { 6682 char *space; 6683 6684 trid->trtype = entry->trtype; 6685 trid->adrfam = entry->adrfam; 6686 memcpy(trid->traddr, entry->traddr, sizeof(entry->traddr)); 6687 memcpy(trid->trsvcid, entry->trsvcid, sizeof(entry->trsvcid)); 6688 /* Because the source buffer (entry->subnqn) is longer than trid->subnqn, and 6689 * before call to this function trid->subnqn is zeroed out, we need 6690 * to copy sizeof(trid->subnqn) minus one byte to make sure the last character 6691 * remains 0. Then we can shorten the string (replace ' ' with 0) if required 6692 */ 6693 memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn) - 1); 6694 6695 /* We want the traddr, trsvcid and subnqn fields to be NULL-terminated. 6696 * But the log page entries typically pad them with spaces, not zeroes. 6697 * So add a NULL terminator to each of these fields at the appropriate 6698 * location. 6699 */ 6700 space = strchr(trid->traddr, ' '); 6701 if (space) { 6702 *space = 0; 6703 } 6704 space = strchr(trid->trsvcid, ' '); 6705 if (space) { 6706 *space = 0; 6707 } 6708 space = strchr(trid->subnqn, ' '); 6709 if (space) { 6710 *space = 0; 6711 } 6712 } 6713 6714 static void 6715 _stop_discovery(void *_ctx) 6716 { 6717 struct discovery_ctx *ctx = _ctx; 6718 6719 if (ctx->attach_in_progress > 0) { 6720 spdk_thread_send_msg(spdk_get_thread(), _stop_discovery, ctx); 6721 return; 6722 } 6723 6724 ctx->stop = true; 6725 6726 while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) { 6727 struct discovery_entry_ctx *entry_ctx; 6728 struct nvme_path_id path = {}; 6729 6730 entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs); 6731 path.trid = entry_ctx->trid; 6732 bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL); 6733 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 6734 free(entry_ctx); 6735 } 6736 6737 while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) { 6738 struct discovery_entry_ctx *entry_ctx; 6739 6740 entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 6741 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 6742 free(entry_ctx); 6743 } 6744 6745 free(ctx->entry_ctx_in_use); 6746 ctx->entry_ctx_in_use = NULL; 6747 } 6748 6749 static void 6750 stop_discovery(struct discovery_ctx *ctx, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 6751 { 6752 ctx->stop_cb_fn = cb_fn; 6753 ctx->cb_ctx = cb_ctx; 6754 6755 if (ctx->attach_in_progress > 0) { 6756 DISCOVERY_INFOLOG(ctx, "stopping discovery with attach_in_progress: %"PRIu32"\n", 6757 ctx->attach_in_progress); 6758 } 6759 6760 _stop_discovery(ctx); 6761 } 6762 6763 static void 6764 remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr) 6765 { 6766 struct discovery_ctx *d_ctx; 6767 struct nvme_path_id *path_id; 6768 struct spdk_nvme_transport_id trid = {}; 6769 struct discovery_entry_ctx *entry_ctx, *tmp; 6770 6771 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 6772 6773 TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) { 6774 TAILQ_FOREACH_SAFE(entry_ctx, &d_ctx->nvm_entry_ctxs, tailq, tmp) { 6775 build_trid_from_log_page_entry(&trid, &entry_ctx->entry); 6776 if (spdk_nvme_transport_id_compare(&trid, &path_id->trid) != 0) { 6777 continue; 6778 } 6779 6780 TAILQ_REMOVE(&d_ctx->nvm_entry_ctxs, entry_ctx, tailq); 6781 free(entry_ctx); 6782 DISCOVERY_INFOLOG(d_ctx, "Remove discovery entry: %s:%s:%s\n", 6783 trid.subnqn, trid.traddr, trid.trsvcid); 6784 6785 /* Fail discovery ctrlr to force reattach attempt */ 6786 spdk_nvme_ctrlr_fail(d_ctx->ctrlr); 6787 } 6788 } 6789 } 6790 6791 static void 6792 discovery_remove_controllers(struct discovery_ctx *ctx) 6793 { 6794 struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page; 6795 struct discovery_entry_ctx *entry_ctx, *tmp; 6796 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 6797 struct spdk_nvme_transport_id old_trid = {}; 6798 uint64_t numrec, i; 6799 bool found; 6800 6801 numrec = from_le64(&log_page->numrec); 6802 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) { 6803 found = false; 6804 old_entry = &entry_ctx->entry; 6805 build_trid_from_log_page_entry(&old_trid, old_entry); 6806 for (i = 0; i < numrec; i++) { 6807 new_entry = &log_page->entries[i]; 6808 if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) { 6809 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n", 6810 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 6811 found = true; 6812 break; 6813 } 6814 } 6815 if (!found) { 6816 struct nvme_path_id path = {}; 6817 6818 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n", 6819 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 6820 6821 path.trid = entry_ctx->trid; 6822 bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL); 6823 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 6824 free(entry_ctx); 6825 } 6826 } 6827 free(log_page); 6828 ctx->log_page = NULL; 6829 discovery_complete(ctx); 6830 } 6831 6832 static void 6833 complete_discovery_start(struct discovery_ctx *ctx, int status) 6834 { 6835 ctx->timeout_ticks = 0; 6836 ctx->rc = status; 6837 if (ctx->start_cb_fn) { 6838 ctx->start_cb_fn(ctx->cb_ctx, status); 6839 ctx->start_cb_fn = NULL; 6840 ctx->cb_ctx = NULL; 6841 } 6842 } 6843 6844 static void 6845 discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc) 6846 { 6847 struct discovery_entry_ctx *entry_ctx = cb_ctx; 6848 struct discovery_ctx *ctx = entry_ctx->ctx; 6849 6850 DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name); 6851 ctx->attach_in_progress--; 6852 if (ctx->attach_in_progress == 0) { 6853 complete_discovery_start(ctx, ctx->rc); 6854 if (ctx->initializing && ctx->rc != 0) { 6855 DISCOVERY_ERRLOG(ctx, "stopping discovery due to errors: %d\n", ctx->rc); 6856 stop_discovery(ctx, NULL, ctx->cb_ctx); 6857 } else { 6858 discovery_remove_controllers(ctx); 6859 } 6860 } 6861 } 6862 6863 static struct discovery_entry_ctx * 6864 create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid) 6865 { 6866 struct discovery_entry_ctx *new_ctx; 6867 6868 new_ctx = calloc(1, sizeof(*new_ctx)); 6869 if (new_ctx == NULL) { 6870 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 6871 return NULL; 6872 } 6873 6874 new_ctx->ctx = ctx; 6875 memcpy(&new_ctx->trid, trid, sizeof(*trid)); 6876 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 6877 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 6878 return new_ctx; 6879 } 6880 6881 static void 6882 discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl, 6883 struct spdk_nvmf_discovery_log_page *log_page) 6884 { 6885 struct discovery_ctx *ctx = cb_arg; 6886 struct discovery_entry_ctx *entry_ctx, *tmp; 6887 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 6888 uint64_t numrec, i; 6889 bool found; 6890 6891 if (rc || spdk_nvme_cpl_is_error(cpl)) { 6892 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 6893 return; 6894 } 6895 6896 ctx->log_page = log_page; 6897 assert(ctx->attach_in_progress == 0); 6898 numrec = from_le64(&log_page->numrec); 6899 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) { 6900 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 6901 free(entry_ctx); 6902 } 6903 for (i = 0; i < numrec; i++) { 6904 found = false; 6905 new_entry = &log_page->entries[i]; 6906 if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY_CURRENT || 6907 new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 6908 struct discovery_entry_ctx *new_ctx; 6909 struct spdk_nvme_transport_id trid = {}; 6910 6911 build_trid_from_log_page_entry(&trid, new_entry); 6912 new_ctx = create_discovery_entry_ctx(ctx, &trid); 6913 if (new_ctx == NULL) { 6914 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 6915 break; 6916 } 6917 6918 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq); 6919 continue; 6920 } 6921 TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) { 6922 old_entry = &entry_ctx->entry; 6923 if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) { 6924 found = true; 6925 break; 6926 } 6927 } 6928 if (!found) { 6929 struct discovery_entry_ctx *subnqn_ctx = NULL, *new_ctx; 6930 struct discovery_ctx *d_ctx; 6931 6932 TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) { 6933 TAILQ_FOREACH(subnqn_ctx, &d_ctx->nvm_entry_ctxs, tailq) { 6934 if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn, 6935 sizeof(new_entry->subnqn))) { 6936 break; 6937 } 6938 } 6939 if (subnqn_ctx) { 6940 break; 6941 } 6942 } 6943 6944 new_ctx = calloc(1, sizeof(*new_ctx)); 6945 if (new_ctx == NULL) { 6946 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 6947 break; 6948 } 6949 6950 new_ctx->ctx = ctx; 6951 memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry)); 6952 build_trid_from_log_page_entry(&new_ctx->trid, new_entry); 6953 if (subnqn_ctx) { 6954 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name); 6955 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n", 6956 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 6957 new_ctx->name); 6958 } else { 6959 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++); 6960 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n", 6961 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 6962 new_ctx->name); 6963 } 6964 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 6965 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 6966 rc = spdk_bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0, 6967 discovery_attach_controller_done, new_ctx, 6968 &new_ctx->drv_opts, &ctx->bdev_opts, true); 6969 if (rc == 0) { 6970 TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq); 6971 ctx->attach_in_progress++; 6972 } else { 6973 DISCOVERY_ERRLOG(ctx, "spdk_bdev_nvme_create failed (%s)\n", spdk_strerror(-rc)); 6974 } 6975 } 6976 } 6977 6978 if (ctx->attach_in_progress == 0) { 6979 discovery_remove_controllers(ctx); 6980 } 6981 } 6982 6983 static void 6984 get_discovery_log_page(struct discovery_ctx *ctx) 6985 { 6986 int rc; 6987 6988 assert(ctx->in_progress == false); 6989 ctx->in_progress = true; 6990 rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx); 6991 if (rc != 0) { 6992 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 6993 } 6994 DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n"); 6995 } 6996 6997 static void 6998 discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 6999 { 7000 struct discovery_ctx *ctx = arg; 7001 uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16; 7002 7003 if (spdk_nvme_cpl_is_error(cpl)) { 7004 DISCOVERY_ERRLOG(ctx, "aer failed\n"); 7005 return; 7006 } 7007 7008 if (log_page_id != SPDK_NVME_LOG_DISCOVERY) { 7009 DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id); 7010 return; 7011 } 7012 7013 DISCOVERY_INFOLOG(ctx, "got aer\n"); 7014 if (ctx->in_progress) { 7015 ctx->pending = true; 7016 return; 7017 } 7018 7019 get_discovery_log_page(ctx); 7020 } 7021 7022 static void 7023 discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 7024 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 7025 { 7026 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 7027 struct discovery_ctx *ctx; 7028 7029 ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts); 7030 7031 DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n"); 7032 ctx->probe_ctx = NULL; 7033 ctx->ctrlr = ctrlr; 7034 7035 if (ctx->rc != 0) { 7036 DISCOVERY_ERRLOG(ctx, "encountered error while attaching discovery ctrlr: %d\n", 7037 ctx->rc); 7038 return; 7039 } 7040 7041 spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx); 7042 } 7043 7044 static int 7045 discovery_poller(void *arg) 7046 { 7047 struct discovery_ctx *ctx = arg; 7048 struct spdk_nvme_transport_id *trid; 7049 int rc; 7050 7051 if (ctx->detach_ctx) { 7052 rc = spdk_nvme_detach_poll_async(ctx->detach_ctx); 7053 if (rc != -EAGAIN) { 7054 ctx->detach_ctx = NULL; 7055 ctx->ctrlr = NULL; 7056 } 7057 } else if (ctx->stop) { 7058 if (ctx->ctrlr != NULL) { 7059 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 7060 if (rc == 0) { 7061 return SPDK_POLLER_BUSY; 7062 } 7063 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 7064 } 7065 spdk_poller_unregister(&ctx->poller); 7066 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 7067 assert(ctx->start_cb_fn == NULL); 7068 if (ctx->stop_cb_fn != NULL) { 7069 ctx->stop_cb_fn(ctx->cb_ctx); 7070 } 7071 free_discovery_ctx(ctx); 7072 } else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) { 7073 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7074 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 7075 assert(ctx->initializing); 7076 spdk_poller_unregister(&ctx->poller); 7077 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 7078 complete_discovery_start(ctx, -ETIMEDOUT); 7079 stop_discovery(ctx, NULL, NULL); 7080 free_discovery_ctx(ctx); 7081 return SPDK_POLLER_BUSY; 7082 } 7083 7084 assert(ctx->entry_ctx_in_use == NULL); 7085 ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 7086 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7087 trid = &ctx->entry_ctx_in_use->trid; 7088 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb); 7089 if (ctx->probe_ctx) { 7090 spdk_poller_unregister(&ctx->poller); 7091 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000); 7092 } else { 7093 DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n"); 7094 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7095 ctx->entry_ctx_in_use = NULL; 7096 } 7097 } else if (ctx->probe_ctx) { 7098 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7099 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 7100 complete_discovery_start(ctx, -ETIMEDOUT); 7101 return SPDK_POLLER_BUSY; 7102 } 7103 7104 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 7105 if (rc != -EAGAIN) { 7106 if (ctx->rc != 0) { 7107 assert(ctx->initializing); 7108 stop_discovery(ctx, NULL, ctx->cb_ctx); 7109 } else { 7110 assert(rc == 0); 7111 DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n"); 7112 ctx->rc = rc; 7113 get_discovery_log_page(ctx); 7114 } 7115 } 7116 } else { 7117 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7118 DISCOVERY_ERRLOG(ctx, "timed out while attaching NVM ctrlrs\n"); 7119 complete_discovery_start(ctx, -ETIMEDOUT); 7120 /* We need to wait until all NVM ctrlrs are attached before we stop the 7121 * discovery service to make sure we don't detach a ctrlr that is still 7122 * being attached. 7123 */ 7124 if (ctx->attach_in_progress == 0) { 7125 stop_discovery(ctx, NULL, ctx->cb_ctx); 7126 return SPDK_POLLER_BUSY; 7127 } 7128 } 7129 7130 rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr); 7131 if (rc < 0) { 7132 spdk_poller_unregister(&ctx->poller); 7133 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 7134 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7135 ctx->entry_ctx_in_use = NULL; 7136 7137 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 7138 if (rc != 0) { 7139 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 7140 ctx->ctrlr = NULL; 7141 } 7142 } 7143 } 7144 7145 return SPDK_POLLER_BUSY; 7146 } 7147 7148 static void 7149 start_discovery_poller(void *arg) 7150 { 7151 struct discovery_ctx *ctx = arg; 7152 7153 TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq); 7154 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 7155 } 7156 7157 int 7158 bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid, 7159 const char *base_name, 7160 struct spdk_nvme_ctrlr_opts *drv_opts, 7161 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, 7162 uint64_t attach_timeout, 7163 bool from_mdns, 7164 spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx) 7165 { 7166 struct discovery_ctx *ctx; 7167 struct discovery_entry_ctx *discovery_entry_ctx; 7168 7169 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 7170 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7171 if (strcmp(ctx->name, base_name) == 0) { 7172 return -EEXIST; 7173 } 7174 7175 if (ctx->entry_ctx_in_use != NULL) { 7176 if (!spdk_nvme_transport_id_compare(trid, &ctx->entry_ctx_in_use->trid)) { 7177 return -EEXIST; 7178 } 7179 } 7180 7181 TAILQ_FOREACH(discovery_entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 7182 if (!spdk_nvme_transport_id_compare(trid, &discovery_entry_ctx->trid)) { 7183 return -EEXIST; 7184 } 7185 } 7186 } 7187 7188 ctx = calloc(1, sizeof(*ctx)); 7189 if (ctx == NULL) { 7190 return -ENOMEM; 7191 } 7192 7193 ctx->name = strdup(base_name); 7194 if (ctx->name == NULL) { 7195 free_discovery_ctx(ctx); 7196 return -ENOMEM; 7197 } 7198 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 7199 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 7200 ctx->from_mdns_discovery_service = from_mdns; 7201 ctx->bdev_opts.from_discovery_service = true; 7202 ctx->calling_thread = spdk_get_thread(); 7203 ctx->start_cb_fn = cb_fn; 7204 ctx->cb_ctx = cb_ctx; 7205 ctx->initializing = true; 7206 if (ctx->start_cb_fn) { 7207 /* We can use this when dumping json to denote if this RPC parameter 7208 * was specified or not. 7209 */ 7210 ctx->wait_for_attach = true; 7211 } 7212 if (attach_timeout != 0) { 7213 ctx->timeout_ticks = spdk_get_ticks() + attach_timeout * 7214 spdk_get_ticks_hz() / 1000ull; 7215 } 7216 TAILQ_INIT(&ctx->nvm_entry_ctxs); 7217 TAILQ_INIT(&ctx->discovery_entry_ctxs); 7218 memcpy(&ctx->trid, trid, sizeof(*trid)); 7219 /* Even if user did not specify hostnqn, we can still strdup("\0"); */ 7220 ctx->hostnqn = strdup(ctx->drv_opts.hostnqn); 7221 if (ctx->hostnqn == NULL) { 7222 free_discovery_ctx(ctx); 7223 return -ENOMEM; 7224 } 7225 discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid); 7226 if (discovery_entry_ctx == NULL) { 7227 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 7228 free_discovery_ctx(ctx); 7229 return -ENOMEM; 7230 } 7231 7232 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq); 7233 spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx); 7234 return 0; 7235 } 7236 7237 int 7238 bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 7239 { 7240 struct discovery_ctx *ctx; 7241 7242 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7243 if (strcmp(name, ctx->name) == 0) { 7244 if (ctx->stop) { 7245 return -EALREADY; 7246 } 7247 /* If we're still starting the discovery service and ->rc is non-zero, we're 7248 * going to stop it as soon as we can 7249 */ 7250 if (ctx->initializing && ctx->rc != 0) { 7251 return -EALREADY; 7252 } 7253 stop_discovery(ctx, cb_fn, cb_ctx); 7254 return 0; 7255 } 7256 } 7257 7258 return -ENOENT; 7259 } 7260 7261 static int 7262 bdev_nvme_library_init(void) 7263 { 7264 g_bdev_nvme_init_thread = spdk_get_thread(); 7265 7266 spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb, 7267 bdev_nvme_destroy_poll_group_cb, 7268 sizeof(struct nvme_poll_group), "nvme_poll_groups"); 7269 7270 return 0; 7271 } 7272 7273 static void 7274 bdev_nvme_fini_destruct_ctrlrs(void) 7275 { 7276 struct nvme_bdev_ctrlr *nbdev_ctrlr; 7277 struct nvme_ctrlr *nvme_ctrlr; 7278 7279 pthread_mutex_lock(&g_bdev_nvme_mutex); 7280 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 7281 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 7282 pthread_mutex_lock(&nvme_ctrlr->mutex); 7283 if (nvme_ctrlr->destruct) { 7284 /* This controller's destruction was already started 7285 * before the application started shutting down 7286 */ 7287 pthread_mutex_unlock(&nvme_ctrlr->mutex); 7288 continue; 7289 } 7290 nvme_ctrlr->destruct = true; 7291 pthread_mutex_unlock(&nvme_ctrlr->mutex); 7292 7293 spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct, 7294 nvme_ctrlr); 7295 } 7296 } 7297 7298 g_bdev_nvme_module_finish = true; 7299 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 7300 pthread_mutex_unlock(&g_bdev_nvme_mutex); 7301 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 7302 spdk_bdev_module_fini_done(); 7303 return; 7304 } 7305 7306 pthread_mutex_unlock(&g_bdev_nvme_mutex); 7307 } 7308 7309 static void 7310 check_discovery_fini(void *arg) 7311 { 7312 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 7313 bdev_nvme_fini_destruct_ctrlrs(); 7314 } 7315 } 7316 7317 static void 7318 bdev_nvme_library_fini(void) 7319 { 7320 struct nvme_probe_skip_entry *entry, *entry_tmp; 7321 struct discovery_ctx *ctx; 7322 7323 spdk_poller_unregister(&g_hotplug_poller); 7324 free(g_hotplug_probe_ctx); 7325 g_hotplug_probe_ctx = NULL; 7326 7327 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) { 7328 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 7329 free(entry); 7330 } 7331 7332 assert(spdk_get_thread() == g_bdev_nvme_init_thread); 7333 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 7334 bdev_nvme_fini_destruct_ctrlrs(); 7335 } else { 7336 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7337 stop_discovery(ctx, check_discovery_fini, NULL); 7338 } 7339 } 7340 } 7341 7342 static void 7343 bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio) 7344 { 7345 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7346 struct spdk_bdev *bdev = bdev_io->bdev; 7347 struct spdk_dif_ctx dif_ctx; 7348 struct spdk_dif_error err_blk = {}; 7349 int rc; 7350 struct spdk_dif_ctx_init_ext_opts dif_opts; 7351 7352 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 7353 dif_opts.dif_pi_format = bdev->dif_pi_format; 7354 rc = spdk_dif_ctx_init(&dif_ctx, 7355 bdev->blocklen, bdev->md_len, bdev->md_interleave, 7356 bdev->dif_is_head_of_md, bdev->dif_type, 7357 bdev_io->u.bdev.dif_check_flags, 7358 bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0, &dif_opts); 7359 if (rc != 0) { 7360 SPDK_ERRLOG("Initialization of DIF context failed\n"); 7361 return; 7362 } 7363 7364 if (bdev->md_interleave) { 7365 rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 7366 bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 7367 } else { 7368 struct iovec md_iov = { 7369 .iov_base = bdev_io->u.bdev.md_buf, 7370 .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len, 7371 }; 7372 7373 rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 7374 &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 7375 } 7376 7377 if (rc != 0) { 7378 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", 7379 err_blk.err_type, err_blk.err_offset); 7380 } else { 7381 SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n"); 7382 } 7383 } 7384 7385 static void 7386 bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7387 { 7388 struct nvme_bdev_io *bio = ref; 7389 7390 if (spdk_nvme_cpl_is_success(cpl)) { 7391 /* Run PI verification for read data buffer. */ 7392 bdev_nvme_verify_pi_error(bio); 7393 } 7394 7395 /* Return original completion status */ 7396 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 7397 } 7398 7399 static void 7400 bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7401 { 7402 struct nvme_bdev_io *bio = ref; 7403 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7404 int ret; 7405 7406 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 7407 SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n", 7408 cpl->status.sct, cpl->status.sc); 7409 7410 /* Save completion status to use after verifying PI error. */ 7411 bio->cpl = *cpl; 7412 7413 if (spdk_likely(nvme_io_path_is_available(bio->io_path))) { 7414 /* Read without PI checking to verify PI error. */ 7415 ret = bdev_nvme_no_pi_readv(bio, 7416 bdev_io->u.bdev.iovs, 7417 bdev_io->u.bdev.iovcnt, 7418 bdev_io->u.bdev.md_buf, 7419 bdev_io->u.bdev.num_blocks, 7420 bdev_io->u.bdev.offset_blocks); 7421 if (ret == 0) { 7422 return; 7423 } 7424 } 7425 } 7426 7427 bdev_nvme_io_complete_nvme_status(bio, cpl); 7428 } 7429 7430 static void 7431 bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7432 { 7433 struct nvme_bdev_io *bio = ref; 7434 7435 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 7436 SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n", 7437 cpl->status.sct, cpl->status.sc); 7438 /* Run PI verification for write data buffer if PI error is detected. */ 7439 bdev_nvme_verify_pi_error(bio); 7440 } 7441 7442 bdev_nvme_io_complete_nvme_status(bio, cpl); 7443 } 7444 7445 static void 7446 bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7447 { 7448 struct nvme_bdev_io *bio = ref; 7449 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7450 7451 /* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks. 7452 * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error(). 7453 */ 7454 bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0; 7455 7456 if (spdk_nvme_cpl_is_pi_error(cpl)) { 7457 SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n", 7458 cpl->status.sct, cpl->status.sc); 7459 /* Run PI verification for zone append data buffer if PI error is detected. */ 7460 bdev_nvme_verify_pi_error(bio); 7461 } 7462 7463 bdev_nvme_io_complete_nvme_status(bio, cpl); 7464 } 7465 7466 static void 7467 bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7468 { 7469 struct nvme_bdev_io *bio = ref; 7470 7471 if (spdk_nvme_cpl_is_pi_error(cpl)) { 7472 SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n", 7473 cpl->status.sct, cpl->status.sc); 7474 /* Run PI verification for compare data buffer if PI error is detected. */ 7475 bdev_nvme_verify_pi_error(bio); 7476 } 7477 7478 bdev_nvme_io_complete_nvme_status(bio, cpl); 7479 } 7480 7481 static void 7482 bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7483 { 7484 struct nvme_bdev_io *bio = ref; 7485 7486 /* Compare operation completion */ 7487 if (!bio->first_fused_completed) { 7488 /* Save compare result for write callback */ 7489 bio->cpl = *cpl; 7490 bio->first_fused_completed = true; 7491 return; 7492 } 7493 7494 /* Write operation completion */ 7495 if (spdk_nvme_cpl_is_error(&bio->cpl)) { 7496 /* If bio->cpl is already an error, it means the compare operation failed. In that case, 7497 * complete the IO with the compare operation's status. 7498 */ 7499 if (!spdk_nvme_cpl_is_error(cpl)) { 7500 SPDK_ERRLOG("Unexpected write success after compare failure.\n"); 7501 } 7502 7503 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 7504 } else { 7505 bdev_nvme_io_complete_nvme_status(bio, cpl); 7506 } 7507 } 7508 7509 static void 7510 bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl) 7511 { 7512 struct nvme_bdev_io *bio = ref; 7513 7514 bdev_nvme_io_complete_nvme_status(bio, cpl); 7515 } 7516 7517 static int 7518 fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc) 7519 { 7520 switch (desc->zt) { 7521 case SPDK_NVME_ZONE_TYPE_SEQWR: 7522 info->type = SPDK_BDEV_ZONE_TYPE_SEQWR; 7523 break; 7524 default: 7525 SPDK_ERRLOG("Invalid zone type: %#x in zone report\n", desc->zt); 7526 return -EIO; 7527 } 7528 7529 switch (desc->zs) { 7530 case SPDK_NVME_ZONE_STATE_EMPTY: 7531 info->state = SPDK_BDEV_ZONE_STATE_EMPTY; 7532 break; 7533 case SPDK_NVME_ZONE_STATE_IOPEN: 7534 info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN; 7535 break; 7536 case SPDK_NVME_ZONE_STATE_EOPEN: 7537 info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN; 7538 break; 7539 case SPDK_NVME_ZONE_STATE_CLOSED: 7540 info->state = SPDK_BDEV_ZONE_STATE_CLOSED; 7541 break; 7542 case SPDK_NVME_ZONE_STATE_RONLY: 7543 info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY; 7544 break; 7545 case SPDK_NVME_ZONE_STATE_FULL: 7546 info->state = SPDK_BDEV_ZONE_STATE_FULL; 7547 break; 7548 case SPDK_NVME_ZONE_STATE_OFFLINE: 7549 info->state = SPDK_BDEV_ZONE_STATE_OFFLINE; 7550 break; 7551 default: 7552 SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs); 7553 return -EIO; 7554 } 7555 7556 info->zone_id = desc->zslba; 7557 info->write_pointer = desc->wp; 7558 info->capacity = desc->zcap; 7559 7560 return 0; 7561 } 7562 7563 static void 7564 bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl) 7565 { 7566 struct nvme_bdev_io *bio = ref; 7567 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7568 uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id; 7569 uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones; 7570 struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf; 7571 uint64_t max_zones_per_buf, i; 7572 uint32_t zone_report_bufsize; 7573 struct spdk_nvme_ns *ns; 7574 struct spdk_nvme_qpair *qpair; 7575 int ret; 7576 7577 if (spdk_nvme_cpl_is_error(cpl)) { 7578 goto out_complete_io_nvme_cpl; 7579 } 7580 7581 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 7582 ret = -ENXIO; 7583 goto out_complete_io_ret; 7584 } 7585 7586 ns = bio->io_path->nvme_ns->ns; 7587 qpair = bio->io_path->qpair->qpair; 7588 7589 zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 7590 max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) / 7591 sizeof(bio->zone_report_buf->descs[0]); 7592 7593 if (bio->zone_report_buf->nr_zones > max_zones_per_buf) { 7594 ret = -EINVAL; 7595 goto out_complete_io_ret; 7596 } 7597 7598 if (!bio->zone_report_buf->nr_zones) { 7599 ret = -EINVAL; 7600 goto out_complete_io_ret; 7601 } 7602 7603 for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) { 7604 ret = fill_zone_from_report(&info[bio->handled_zones], 7605 &bio->zone_report_buf->descs[i]); 7606 if (ret) { 7607 goto out_complete_io_ret; 7608 } 7609 bio->handled_zones++; 7610 } 7611 7612 if (bio->handled_zones < zones_to_copy) { 7613 uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 7614 uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones); 7615 7616 memset(bio->zone_report_buf, 0, zone_report_bufsize); 7617 ret = spdk_nvme_zns_report_zones(ns, qpair, 7618 bio->zone_report_buf, zone_report_bufsize, 7619 slba, SPDK_NVME_ZRA_LIST_ALL, true, 7620 bdev_nvme_get_zone_info_done, bio); 7621 if (!ret) { 7622 return; 7623 } else { 7624 goto out_complete_io_ret; 7625 } 7626 } 7627 7628 out_complete_io_nvme_cpl: 7629 free(bio->zone_report_buf); 7630 bio->zone_report_buf = NULL; 7631 bdev_nvme_io_complete_nvme_status(bio, cpl); 7632 return; 7633 7634 out_complete_io_ret: 7635 free(bio->zone_report_buf); 7636 bio->zone_report_buf = NULL; 7637 bdev_nvme_io_complete(bio, ret); 7638 } 7639 7640 static void 7641 bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl) 7642 { 7643 struct nvme_bdev_io *bio = ref; 7644 7645 bdev_nvme_io_complete_nvme_status(bio, cpl); 7646 } 7647 7648 static void 7649 bdev_nvme_admin_passthru_complete_nvme_status(void *ctx) 7650 { 7651 struct nvme_bdev_io *bio = ctx; 7652 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7653 const struct spdk_nvme_cpl *cpl = &bio->cpl; 7654 7655 assert(bdev_nvme_io_type_is_admin(bdev_io->type)); 7656 7657 __bdev_nvme_io_complete(bdev_io, 0, cpl); 7658 } 7659 7660 static void 7661 bdev_nvme_abort_complete(void *ctx) 7662 { 7663 struct nvme_bdev_io *bio = ctx; 7664 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7665 7666 if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) { 7667 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL); 7668 } else { 7669 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL); 7670 } 7671 } 7672 7673 static void 7674 bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl) 7675 { 7676 struct nvme_bdev_io *bio = ref; 7677 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7678 7679 bio->cpl = *cpl; 7680 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), bdev_nvme_abort_complete, bio); 7681 } 7682 7683 static void 7684 bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl) 7685 { 7686 struct nvme_bdev_io *bio = ref; 7687 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7688 7689 bio->cpl = *cpl; 7690 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), 7691 bdev_nvme_admin_passthru_complete_nvme_status, bio); 7692 } 7693 7694 static void 7695 bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset) 7696 { 7697 struct nvme_bdev_io *bio = ref; 7698 struct iovec *iov; 7699 7700 bio->iov_offset = sgl_offset; 7701 for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) { 7702 iov = &bio->iovs[bio->iovpos]; 7703 if (bio->iov_offset < iov->iov_len) { 7704 break; 7705 } 7706 7707 bio->iov_offset -= iov->iov_len; 7708 } 7709 } 7710 7711 static int 7712 bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length) 7713 { 7714 struct nvme_bdev_io *bio = ref; 7715 struct iovec *iov; 7716 7717 assert(bio->iovpos < bio->iovcnt); 7718 7719 iov = &bio->iovs[bio->iovpos]; 7720 7721 *address = iov->iov_base; 7722 *length = iov->iov_len; 7723 7724 if (bio->iov_offset) { 7725 assert(bio->iov_offset <= iov->iov_len); 7726 *address += bio->iov_offset; 7727 *length -= bio->iov_offset; 7728 } 7729 7730 bio->iov_offset += *length; 7731 if (bio->iov_offset == iov->iov_len) { 7732 bio->iovpos++; 7733 bio->iov_offset = 0; 7734 } 7735 7736 return 0; 7737 } 7738 7739 static void 7740 bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset) 7741 { 7742 struct nvme_bdev_io *bio = ref; 7743 struct iovec *iov; 7744 7745 bio->fused_iov_offset = sgl_offset; 7746 for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) { 7747 iov = &bio->fused_iovs[bio->fused_iovpos]; 7748 if (bio->fused_iov_offset < iov->iov_len) { 7749 break; 7750 } 7751 7752 bio->fused_iov_offset -= iov->iov_len; 7753 } 7754 } 7755 7756 static int 7757 bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length) 7758 { 7759 struct nvme_bdev_io *bio = ref; 7760 struct iovec *iov; 7761 7762 assert(bio->fused_iovpos < bio->fused_iovcnt); 7763 7764 iov = &bio->fused_iovs[bio->fused_iovpos]; 7765 7766 *address = iov->iov_base; 7767 *length = iov->iov_len; 7768 7769 if (bio->fused_iov_offset) { 7770 assert(bio->fused_iov_offset <= iov->iov_len); 7771 *address += bio->fused_iov_offset; 7772 *length -= bio->fused_iov_offset; 7773 } 7774 7775 bio->fused_iov_offset += *length; 7776 if (bio->fused_iov_offset == iov->iov_len) { 7777 bio->fused_iovpos++; 7778 bio->fused_iov_offset = 0; 7779 } 7780 7781 return 0; 7782 } 7783 7784 static int 7785 bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7786 void *md, uint64_t lba_count, uint64_t lba) 7787 { 7788 int rc; 7789 7790 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n", 7791 lba_count, lba); 7792 7793 bio->iovs = iov; 7794 bio->iovcnt = iovcnt; 7795 bio->iovpos = 0; 7796 bio->iov_offset = 0; 7797 7798 rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns, 7799 bio->io_path->qpair->qpair, 7800 lba, lba_count, 7801 bdev_nvme_no_pi_readv_done, bio, 0, 7802 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 7803 md, 0, 0); 7804 7805 if (rc != 0 && rc != -ENOMEM) { 7806 SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc); 7807 } 7808 return rc; 7809 } 7810 7811 static int 7812 bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7813 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 7814 struct spdk_memory_domain *domain, void *domain_ctx, 7815 struct spdk_accel_sequence *seq) 7816 { 7817 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7818 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7819 int rc; 7820 7821 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n", 7822 lba_count, lba); 7823 7824 bio->iovs = iov; 7825 bio->iovcnt = iovcnt; 7826 bio->iovpos = 0; 7827 bio->iov_offset = 0; 7828 7829 if (domain != NULL || seq != NULL) { 7830 bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence); 7831 bio->ext_opts.memory_domain = domain; 7832 bio->ext_opts.memory_domain_ctx = domain_ctx; 7833 bio->ext_opts.io_flags = flags; 7834 bio->ext_opts.metadata = md; 7835 bio->ext_opts.accel_sequence = seq; 7836 7837 if (iovcnt == 1) { 7838 rc = spdk_nvme_ns_cmd_read_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_readv_done, 7839 bio, &bio->ext_opts); 7840 } else { 7841 rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count, 7842 bdev_nvme_readv_done, bio, 7843 bdev_nvme_queued_reset_sgl, 7844 bdev_nvme_queued_next_sge, 7845 &bio->ext_opts); 7846 } 7847 } else if (iovcnt == 1) { 7848 rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, iov[0].iov_base, 7849 md, lba, lba_count, bdev_nvme_readv_done, 7850 bio, flags, 0, 0); 7851 } else { 7852 rc = spdk_nvme_ns_cmd_readv_with_md(ns, qpair, lba, lba_count, 7853 bdev_nvme_readv_done, bio, flags, 7854 bdev_nvme_queued_reset_sgl, 7855 bdev_nvme_queued_next_sge, md, 0, 0); 7856 } 7857 7858 if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) { 7859 SPDK_ERRLOG("readv failed: rc = %d\n", rc); 7860 } 7861 return rc; 7862 } 7863 7864 static int 7865 bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7866 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 7867 struct spdk_memory_domain *domain, void *domain_ctx, 7868 struct spdk_accel_sequence *seq, 7869 union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13) 7870 { 7871 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7872 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7873 int rc; 7874 7875 SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 7876 lba_count, lba); 7877 7878 bio->iovs = iov; 7879 bio->iovcnt = iovcnt; 7880 bio->iovpos = 0; 7881 bio->iov_offset = 0; 7882 7883 if (domain != NULL || seq != NULL) { 7884 bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence); 7885 bio->ext_opts.memory_domain = domain; 7886 bio->ext_opts.memory_domain_ctx = domain_ctx; 7887 bio->ext_opts.io_flags = flags | SPDK_NVME_IO_FLAGS_DIRECTIVE(cdw12.write.dtype); 7888 bio->ext_opts.cdw13 = cdw13.raw; 7889 bio->ext_opts.metadata = md; 7890 bio->ext_opts.accel_sequence = seq; 7891 7892 if (iovcnt == 1) { 7893 rc = spdk_nvme_ns_cmd_write_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_writev_done, 7894 bio, &bio->ext_opts); 7895 } else { 7896 rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count, 7897 bdev_nvme_writev_done, bio, 7898 bdev_nvme_queued_reset_sgl, 7899 bdev_nvme_queued_next_sge, 7900 &bio->ext_opts); 7901 } 7902 } else if (iovcnt == 1) { 7903 rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, iov[0].iov_base, 7904 md, lba, lba_count, bdev_nvme_writev_done, 7905 bio, flags, 0, 0); 7906 } else { 7907 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 7908 bdev_nvme_writev_done, bio, flags, 7909 bdev_nvme_queued_reset_sgl, 7910 bdev_nvme_queued_next_sge, md, 0, 0); 7911 } 7912 7913 if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) { 7914 SPDK_ERRLOG("writev failed: rc = %d\n", rc); 7915 } 7916 return rc; 7917 } 7918 7919 static int 7920 bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7921 void *md, uint64_t lba_count, uint64_t zslba, 7922 uint32_t flags) 7923 { 7924 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7925 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7926 int rc; 7927 7928 SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n", 7929 lba_count, zslba); 7930 7931 bio->iovs = iov; 7932 bio->iovcnt = iovcnt; 7933 bio->iovpos = 0; 7934 bio->iov_offset = 0; 7935 7936 if (iovcnt == 1) { 7937 rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba, 7938 lba_count, 7939 bdev_nvme_zone_appendv_done, bio, 7940 flags, 7941 0, 0); 7942 } else { 7943 rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count, 7944 bdev_nvme_zone_appendv_done, bio, flags, 7945 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 7946 md, 0, 0); 7947 } 7948 7949 if (rc != 0 && rc != -ENOMEM) { 7950 SPDK_ERRLOG("zone append failed: rc = %d\n", rc); 7951 } 7952 return rc; 7953 } 7954 7955 static int 7956 bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7957 void *md, uint64_t lba_count, uint64_t lba, 7958 uint32_t flags) 7959 { 7960 int rc; 7961 7962 SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n", 7963 lba_count, lba); 7964 7965 bio->iovs = iov; 7966 bio->iovcnt = iovcnt; 7967 bio->iovpos = 0; 7968 bio->iov_offset = 0; 7969 7970 rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns, 7971 bio->io_path->qpair->qpair, 7972 lba, lba_count, 7973 bdev_nvme_comparev_done, bio, flags, 7974 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 7975 md, 0, 0); 7976 7977 if (rc != 0 && rc != -ENOMEM) { 7978 SPDK_ERRLOG("comparev failed: rc = %d\n", rc); 7979 } 7980 return rc; 7981 } 7982 7983 static int 7984 bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt, 7985 struct iovec *write_iov, int write_iovcnt, 7986 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags) 7987 { 7988 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7989 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7990 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7991 int rc; 7992 7993 SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 7994 lba_count, lba); 7995 7996 bio->iovs = cmp_iov; 7997 bio->iovcnt = cmp_iovcnt; 7998 bio->iovpos = 0; 7999 bio->iov_offset = 0; 8000 bio->fused_iovs = write_iov; 8001 bio->fused_iovcnt = write_iovcnt; 8002 bio->fused_iovpos = 0; 8003 bio->fused_iov_offset = 0; 8004 8005 if (bdev_io->num_retries == 0) { 8006 bio->first_fused_submitted = false; 8007 bio->first_fused_completed = false; 8008 } 8009 8010 if (!bio->first_fused_submitted) { 8011 flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST; 8012 memset(&bio->cpl, 0, sizeof(bio->cpl)); 8013 8014 rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count, 8015 bdev_nvme_comparev_and_writev_done, bio, flags, 8016 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0); 8017 if (rc == 0) { 8018 bio->first_fused_submitted = true; 8019 flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST; 8020 } else { 8021 if (rc != -ENOMEM) { 8022 SPDK_ERRLOG("compare failed: rc = %d\n", rc); 8023 } 8024 return rc; 8025 } 8026 } 8027 8028 flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND; 8029 8030 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 8031 bdev_nvme_comparev_and_writev_done, bio, flags, 8032 bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0); 8033 if (rc != 0 && rc != -ENOMEM) { 8034 SPDK_ERRLOG("write failed: rc = %d\n", rc); 8035 rc = 0; 8036 } 8037 8038 return rc; 8039 } 8040 8041 static int 8042 bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 8043 { 8044 struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES]; 8045 struct spdk_nvme_dsm_range *range; 8046 uint64_t offset, remaining; 8047 uint64_t num_ranges_u64; 8048 uint16_t num_ranges; 8049 int rc; 8050 8051 num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) / 8052 SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8053 if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) { 8054 SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks); 8055 return -EINVAL; 8056 } 8057 num_ranges = (uint16_t)num_ranges_u64; 8058 8059 offset = offset_blocks; 8060 remaining = num_blocks; 8061 range = &dsm_ranges[0]; 8062 8063 /* Fill max-size ranges until the remaining blocks fit into one range */ 8064 while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) { 8065 range->attributes.raw = 0; 8066 range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8067 range->starting_lba = offset; 8068 8069 offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8070 remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8071 range++; 8072 } 8073 8074 /* Final range describes the remaining blocks */ 8075 range->attributes.raw = 0; 8076 range->length = remaining; 8077 range->starting_lba = offset; 8078 8079 rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns, 8080 bio->io_path->qpair->qpair, 8081 SPDK_NVME_DSM_ATTR_DEALLOCATE, 8082 dsm_ranges, num_ranges, 8083 bdev_nvme_queued_done, bio); 8084 8085 return rc; 8086 } 8087 8088 static int 8089 bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 8090 { 8091 if (num_blocks > UINT16_MAX + 1) { 8092 SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n"); 8093 return -EINVAL; 8094 } 8095 8096 return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns, 8097 bio->io_path->qpair->qpair, 8098 offset_blocks, num_blocks, 8099 bdev_nvme_queued_done, bio, 8100 0); 8101 } 8102 8103 static int 8104 bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones, 8105 struct spdk_bdev_zone_info *info) 8106 { 8107 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8108 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8109 uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 8110 uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 8111 uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns); 8112 8113 if (zone_id % zone_size != 0) { 8114 return -EINVAL; 8115 } 8116 8117 if (num_zones > total_zones || !num_zones) { 8118 return -EINVAL; 8119 } 8120 8121 assert(!bio->zone_report_buf); 8122 bio->zone_report_buf = calloc(1, zone_report_bufsize); 8123 if (!bio->zone_report_buf) { 8124 return -ENOMEM; 8125 } 8126 8127 bio->handled_zones = 0; 8128 8129 return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize, 8130 zone_id, SPDK_NVME_ZRA_LIST_ALL, true, 8131 bdev_nvme_get_zone_info_done, bio); 8132 } 8133 8134 static int 8135 bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 8136 enum spdk_bdev_zone_action action) 8137 { 8138 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8139 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8140 8141 switch (action) { 8142 case SPDK_BDEV_ZONE_CLOSE: 8143 return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false, 8144 bdev_nvme_zone_management_done, bio); 8145 case SPDK_BDEV_ZONE_FINISH: 8146 return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false, 8147 bdev_nvme_zone_management_done, bio); 8148 case SPDK_BDEV_ZONE_OPEN: 8149 return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false, 8150 bdev_nvme_zone_management_done, bio); 8151 case SPDK_BDEV_ZONE_RESET: 8152 return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false, 8153 bdev_nvme_zone_management_done, bio); 8154 case SPDK_BDEV_ZONE_OFFLINE: 8155 return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false, 8156 bdev_nvme_zone_management_done, bio); 8157 default: 8158 return -EINVAL; 8159 } 8160 } 8161 8162 static void 8163 bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 8164 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes) 8165 { 8166 struct nvme_io_path *io_path; 8167 struct nvme_ctrlr *nvme_ctrlr; 8168 uint32_t max_xfer_size; 8169 int rc = -ENXIO; 8170 8171 /* Choose the first ctrlr which is not failed. */ 8172 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 8173 nvme_ctrlr = io_path->qpair->ctrlr; 8174 8175 /* We should skip any unavailable nvme_ctrlr rather than checking 8176 * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO. 8177 */ 8178 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 8179 continue; 8180 } 8181 8182 max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr); 8183 8184 if (nbytes > max_xfer_size) { 8185 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8186 rc = -EINVAL; 8187 goto err; 8188 } 8189 8190 rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes, 8191 bdev_nvme_admin_passthru_done, bio); 8192 if (rc == 0) { 8193 return; 8194 } 8195 } 8196 8197 err: 8198 bdev_nvme_admin_complete(bio, rc); 8199 } 8200 8201 static int 8202 bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 8203 void *buf, size_t nbytes) 8204 { 8205 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8206 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8207 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8208 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8209 8210 if (nbytes > max_xfer_size) { 8211 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8212 return -EINVAL; 8213 } 8214 8215 /* 8216 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 8217 * so fill it out automatically. 8218 */ 8219 cmd->nsid = spdk_nvme_ns_get_id(ns); 8220 8221 return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf, 8222 (uint32_t)nbytes, bdev_nvme_queued_done, bio); 8223 } 8224 8225 static int 8226 bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 8227 void *buf, size_t nbytes, void *md_buf, size_t md_len) 8228 { 8229 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8230 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8231 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 8232 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8233 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8234 8235 if (nbytes > max_xfer_size) { 8236 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8237 return -EINVAL; 8238 } 8239 8240 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 8241 SPDK_ERRLOG("invalid meta data buffer size\n"); 8242 return -EINVAL; 8243 } 8244 8245 /* 8246 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 8247 * so fill it out automatically. 8248 */ 8249 cmd->nsid = spdk_nvme_ns_get_id(ns); 8250 8251 return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf, 8252 (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio); 8253 } 8254 8255 static int 8256 bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, 8257 struct spdk_nvme_cmd *cmd, struct iovec *iov, int iovcnt, 8258 size_t nbytes, void *md_buf, size_t md_len) 8259 { 8260 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8261 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8262 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 8263 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8264 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8265 8266 bio->iovs = iov; 8267 bio->iovcnt = iovcnt; 8268 bio->iovpos = 0; 8269 bio->iov_offset = 0; 8270 8271 if (nbytes > max_xfer_size) { 8272 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8273 return -EINVAL; 8274 } 8275 8276 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 8277 SPDK_ERRLOG("invalid meta data buffer size\n"); 8278 return -EINVAL; 8279 } 8280 8281 /* 8282 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands 8283 * require a nsid, so fill it out automatically. 8284 */ 8285 cmd->nsid = spdk_nvme_ns_get_id(ns); 8286 8287 return spdk_nvme_ctrlr_cmd_iov_raw_with_md( 8288 ctrlr, qpair, cmd, (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio, 8289 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge); 8290 } 8291 8292 static void 8293 bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 8294 struct nvme_bdev_io *bio_to_abort) 8295 { 8296 struct nvme_io_path *io_path; 8297 int rc = 0; 8298 8299 rc = bdev_nvme_abort_retry_io(nbdev_ch, bio_to_abort); 8300 if (rc == 0) { 8301 bdev_nvme_admin_complete(bio, 0); 8302 return; 8303 } 8304 8305 io_path = bio_to_abort->io_path; 8306 if (io_path != NULL) { 8307 rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr, 8308 io_path->qpair->qpair, 8309 bio_to_abort, 8310 bdev_nvme_abort_done, bio); 8311 } else { 8312 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 8313 rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr, 8314 NULL, 8315 bio_to_abort, 8316 bdev_nvme_abort_done, bio); 8317 8318 if (rc != -ENOENT) { 8319 break; 8320 } 8321 } 8322 } 8323 8324 if (rc != 0) { 8325 /* If no command was found or there was any error, complete the abort 8326 * request with failure. 8327 */ 8328 bdev_nvme_admin_complete(bio, rc); 8329 } 8330 } 8331 8332 static int 8333 bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, uint64_t src_offset_blocks, 8334 uint64_t num_blocks) 8335 { 8336 struct spdk_nvme_scc_source_range range = { 8337 .slba = src_offset_blocks, 8338 .nlb = num_blocks - 1 8339 }; 8340 8341 return spdk_nvme_ns_cmd_copy(bio->io_path->nvme_ns->ns, 8342 bio->io_path->qpair->qpair, 8343 &range, 1, dst_offset_blocks, 8344 bdev_nvme_queued_done, bio); 8345 } 8346 8347 static void 8348 bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w) 8349 { 8350 const char *action; 8351 uint32_t i; 8352 8353 if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) { 8354 action = "reset"; 8355 } else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) { 8356 action = "abort"; 8357 } else { 8358 action = "none"; 8359 } 8360 8361 spdk_json_write_object_begin(w); 8362 8363 spdk_json_write_named_string(w, "method", "bdev_nvme_set_options"); 8364 8365 spdk_json_write_named_object_begin(w, "params"); 8366 spdk_json_write_named_string(w, "action_on_timeout", action); 8367 spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us); 8368 spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us); 8369 spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms); 8370 spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst); 8371 spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight); 8372 spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight); 8373 spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight); 8374 spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us); 8375 spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us); 8376 spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests); 8377 spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit); 8378 spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count); 8379 spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count); 8380 spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout); 8381 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec); 8382 spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec); 8383 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec); 8384 spdk_json_write_named_bool(w, "disable_auto_failback", g_opts.disable_auto_failback); 8385 spdk_json_write_named_bool(w, "generate_uuids", g_opts.generate_uuids); 8386 spdk_json_write_named_uint8(w, "transport_tos", g_opts.transport_tos); 8387 spdk_json_write_named_bool(w, "nvme_error_stat", g_opts.nvme_error_stat); 8388 spdk_json_write_named_uint32(w, "rdma_srq_size", g_opts.rdma_srq_size); 8389 spdk_json_write_named_bool(w, "io_path_stat", g_opts.io_path_stat); 8390 spdk_json_write_named_bool(w, "allow_accel_sequence", g_opts.allow_accel_sequence); 8391 spdk_json_write_named_uint32(w, "rdma_max_cq_size", g_opts.rdma_max_cq_size); 8392 spdk_json_write_named_uint16(w, "rdma_cm_event_timeout_ms", g_opts.rdma_cm_event_timeout_ms); 8393 spdk_json_write_named_array_begin(w, "dhchap_digests"); 8394 for (i = 0; i < 32; ++i) { 8395 if (g_opts.dhchap_digests & SPDK_BIT(i)) { 8396 spdk_json_write_string(w, spdk_nvme_dhchap_get_digest_name(i)); 8397 } 8398 } 8399 spdk_json_write_array_end(w); 8400 spdk_json_write_named_array_begin(w, "dhchap_dhgroups"); 8401 for (i = 0; i < 32; ++i) { 8402 if (g_opts.dhchap_dhgroups & SPDK_BIT(i)) { 8403 spdk_json_write_string(w, spdk_nvme_dhchap_get_dhgroup_name(i)); 8404 } 8405 } 8406 8407 spdk_json_write_array_end(w); 8408 spdk_json_write_object_end(w); 8409 8410 spdk_json_write_object_end(w); 8411 } 8412 8413 static void 8414 bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx) 8415 { 8416 struct spdk_nvme_transport_id trid; 8417 8418 spdk_json_write_object_begin(w); 8419 8420 spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery"); 8421 8422 spdk_json_write_named_object_begin(w, "params"); 8423 spdk_json_write_named_string(w, "name", ctx->name); 8424 spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn); 8425 8426 trid = ctx->trid; 8427 memset(trid.subnqn, 0, sizeof(trid.subnqn)); 8428 nvme_bdev_dump_trid_json(&trid, w); 8429 8430 spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach); 8431 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec); 8432 spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec); 8433 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 8434 ctx->bdev_opts.fast_io_fail_timeout_sec); 8435 spdk_json_write_object_end(w); 8436 8437 spdk_json_write_object_end(w); 8438 } 8439 8440 #ifdef SPDK_CONFIG_NVME_CUSE 8441 static void 8442 nvme_ctrlr_cuse_config_json(struct spdk_json_write_ctx *w, 8443 struct nvme_ctrlr *nvme_ctrlr) 8444 { 8445 size_t cuse_name_size = 128; 8446 char cuse_name[cuse_name_size]; 8447 8448 if (spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, 8449 cuse_name, &cuse_name_size) != 0) { 8450 return; 8451 } 8452 8453 spdk_json_write_object_begin(w); 8454 8455 spdk_json_write_named_string(w, "method", "bdev_nvme_cuse_register"); 8456 8457 spdk_json_write_named_object_begin(w, "params"); 8458 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 8459 spdk_json_write_object_end(w); 8460 8461 spdk_json_write_object_end(w); 8462 } 8463 #endif 8464 8465 static void 8466 nvme_ctrlr_config_json(struct spdk_json_write_ctx *w, 8467 struct nvme_ctrlr *nvme_ctrlr) 8468 { 8469 struct spdk_nvme_transport_id *trid; 8470 const struct spdk_nvme_ctrlr_opts *opts; 8471 8472 if (nvme_ctrlr->opts.from_discovery_service) { 8473 /* Do not emit an RPC for this - it will be implicitly 8474 * covered by a separate bdev_nvme_start_discovery or 8475 * bdev_nvme_start_mdns_discovery RPC. 8476 */ 8477 return; 8478 } 8479 8480 trid = &nvme_ctrlr->active_path_id->trid; 8481 8482 spdk_json_write_object_begin(w); 8483 8484 spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller"); 8485 8486 spdk_json_write_named_object_begin(w, "params"); 8487 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 8488 nvme_bdev_dump_trid_json(trid, w); 8489 spdk_json_write_named_bool(w, "prchk_reftag", 8490 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0); 8491 spdk_json_write_named_bool(w, "prchk_guard", 8492 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0); 8493 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec); 8494 spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec); 8495 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 8496 nvme_ctrlr->opts.fast_io_fail_timeout_sec); 8497 if (nvme_ctrlr->psk != NULL) { 8498 spdk_json_write_named_string(w, "psk", spdk_key_get_name(nvme_ctrlr->psk)); 8499 } else if (nvme_ctrlr->opts.psk[0] != '\0') { 8500 spdk_json_write_named_string(w, "psk", nvme_ctrlr->opts.psk); 8501 } 8502 8503 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 8504 spdk_json_write_named_string(w, "hostnqn", opts->hostnqn); 8505 spdk_json_write_named_bool(w, "hdgst", opts->header_digest); 8506 spdk_json_write_named_bool(w, "ddgst", opts->data_digest); 8507 if (opts->src_addr[0] != '\0') { 8508 spdk_json_write_named_string(w, "hostaddr", opts->src_addr); 8509 } 8510 if (opts->src_svcid[0] != '\0') { 8511 spdk_json_write_named_string(w, "hostsvcid", opts->src_svcid); 8512 } 8513 8514 spdk_json_write_object_end(w); 8515 8516 spdk_json_write_object_end(w); 8517 } 8518 8519 static void 8520 bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w) 8521 { 8522 spdk_json_write_object_begin(w); 8523 spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug"); 8524 8525 spdk_json_write_named_object_begin(w, "params"); 8526 spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us); 8527 spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled); 8528 spdk_json_write_object_end(w); 8529 8530 spdk_json_write_object_end(w); 8531 } 8532 8533 static int 8534 bdev_nvme_config_json(struct spdk_json_write_ctx *w) 8535 { 8536 struct nvme_bdev_ctrlr *nbdev_ctrlr; 8537 struct nvme_ctrlr *nvme_ctrlr; 8538 struct discovery_ctx *ctx; 8539 8540 bdev_nvme_opts_config_json(w); 8541 8542 pthread_mutex_lock(&g_bdev_nvme_mutex); 8543 8544 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 8545 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 8546 nvme_ctrlr_config_json(w, nvme_ctrlr); 8547 8548 #ifdef SPDK_CONFIG_NVME_CUSE 8549 nvme_ctrlr_cuse_config_json(w, nvme_ctrlr); 8550 #endif 8551 } 8552 } 8553 8554 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 8555 if (!ctx->from_mdns_discovery_service) { 8556 bdev_nvme_discovery_config_json(w, ctx); 8557 } 8558 } 8559 8560 bdev_nvme_mdns_discovery_config_json(w); 8561 8562 /* Dump as last parameter to give all NVMe bdevs chance to be constructed 8563 * before enabling hotplug poller. 8564 */ 8565 bdev_nvme_hotplug_config_json(w); 8566 8567 pthread_mutex_unlock(&g_bdev_nvme_mutex); 8568 return 0; 8569 } 8570 8571 struct spdk_nvme_ctrlr * 8572 bdev_nvme_get_ctrlr(struct spdk_bdev *bdev) 8573 { 8574 struct nvme_bdev *nbdev; 8575 struct nvme_ns *nvme_ns; 8576 8577 if (!bdev || bdev->module != &nvme_if) { 8578 return NULL; 8579 } 8580 8581 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 8582 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 8583 assert(nvme_ns != NULL); 8584 8585 return nvme_ns->ctrlr->ctrlr; 8586 } 8587 8588 static bool 8589 nvme_io_path_is_current(struct nvme_io_path *io_path) 8590 { 8591 const struct nvme_bdev_channel *nbdev_ch; 8592 bool current; 8593 8594 if (!nvme_io_path_is_available(io_path)) { 8595 return false; 8596 } 8597 8598 nbdev_ch = io_path->nbdev_ch; 8599 if (nbdev_ch == NULL) { 8600 current = false; 8601 } else if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 8602 struct nvme_io_path *optimized_io_path = NULL; 8603 8604 STAILQ_FOREACH(optimized_io_path, &nbdev_ch->io_path_list, stailq) { 8605 if (optimized_io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) { 8606 break; 8607 } 8608 } 8609 8610 /* A non-optimized path is only current if there are no optimized paths. */ 8611 current = (io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) || 8612 (optimized_io_path == NULL); 8613 } else { 8614 if (nbdev_ch->current_io_path) { 8615 current = (io_path == nbdev_ch->current_io_path); 8616 } else { 8617 struct nvme_io_path *first_path; 8618 8619 /* We arrived here as there are no optimized paths for active-passive 8620 * mode. Check if this io_path is the first one available on the list. 8621 */ 8622 current = false; 8623 STAILQ_FOREACH(first_path, &nbdev_ch->io_path_list, stailq) { 8624 if (nvme_io_path_is_available(first_path)) { 8625 current = (io_path == first_path); 8626 break; 8627 } 8628 } 8629 } 8630 } 8631 8632 return current; 8633 } 8634 8635 void 8636 nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path) 8637 { 8638 struct nvme_ns *nvme_ns = io_path->nvme_ns; 8639 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 8640 const struct spdk_nvme_ctrlr_data *cdata; 8641 const struct spdk_nvme_transport_id *trid; 8642 const char *adrfam_str; 8643 8644 spdk_json_write_object_begin(w); 8645 8646 spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name); 8647 8648 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 8649 trid = spdk_nvme_ctrlr_get_transport_id(nvme_ctrlr->ctrlr); 8650 8651 spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid); 8652 spdk_json_write_named_bool(w, "current", nvme_io_path_is_current(io_path)); 8653 spdk_json_write_named_bool(w, "connected", nvme_qpair_is_connected(io_path->qpair)); 8654 spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns)); 8655 8656 spdk_json_write_named_object_begin(w, "transport"); 8657 spdk_json_write_named_string(w, "trtype", trid->trstring); 8658 spdk_json_write_named_string(w, "traddr", trid->traddr); 8659 if (trid->trsvcid[0] != '\0') { 8660 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 8661 } 8662 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 8663 if (adrfam_str) { 8664 spdk_json_write_named_string(w, "adrfam", adrfam_str); 8665 } 8666 spdk_json_write_object_end(w); 8667 8668 spdk_json_write_object_end(w); 8669 } 8670 8671 void 8672 bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w) 8673 { 8674 struct discovery_ctx *ctx; 8675 struct discovery_entry_ctx *entry_ctx; 8676 8677 spdk_json_write_array_begin(w); 8678 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 8679 spdk_json_write_object_begin(w); 8680 spdk_json_write_named_string(w, "name", ctx->name); 8681 8682 spdk_json_write_named_object_begin(w, "trid"); 8683 nvme_bdev_dump_trid_json(&ctx->trid, w); 8684 spdk_json_write_object_end(w); 8685 8686 spdk_json_write_named_array_begin(w, "referrals"); 8687 TAILQ_FOREACH(entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 8688 spdk_json_write_object_begin(w); 8689 spdk_json_write_named_object_begin(w, "trid"); 8690 nvme_bdev_dump_trid_json(&entry_ctx->trid, w); 8691 spdk_json_write_object_end(w); 8692 spdk_json_write_object_end(w); 8693 } 8694 spdk_json_write_array_end(w); 8695 8696 spdk_json_write_object_end(w); 8697 } 8698 spdk_json_write_array_end(w); 8699 } 8700 8701 SPDK_LOG_REGISTER_COMPONENT(bdev_nvme) 8702 8703 SPDK_TRACE_REGISTER_FN(bdev_nvme_trace, "bdev_nvme", TRACE_GROUP_BDEV_NVME) 8704 { 8705 struct spdk_trace_tpoint_opts opts[] = { 8706 { 8707 "BDEV_NVME_IO_START", TRACE_BDEV_NVME_IO_START, 8708 OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 1, 8709 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 8710 }, 8711 { 8712 "BDEV_NVME_IO_DONE", TRACE_BDEV_NVME_IO_DONE, 8713 OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 0, 8714 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 8715 } 8716 }; 8717 8718 8719 spdk_trace_register_object(OBJECT_BDEV_NVME_IO, 'N'); 8720 spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts)); 8721 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 8722 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 8723 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 8724 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 8725 } 8726