1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved. 6 */ 7 8 #include "spdk/stdinc.h" 9 10 #include "bdev_nvme.h" 11 12 #include "spdk/accel.h" 13 #include "spdk/config.h" 14 #include "spdk/endian.h" 15 #include "spdk/bdev.h" 16 #include "spdk/json.h" 17 #include "spdk/keyring.h" 18 #include "spdk/likely.h" 19 #include "spdk/nvme.h" 20 #include "spdk/nvme_ocssd.h" 21 #include "spdk/nvme_zns.h" 22 #include "spdk/opal.h" 23 #include "spdk/thread.h" 24 #include "spdk/trace.h" 25 #include "spdk/string.h" 26 #include "spdk/util.h" 27 #include "spdk/uuid.h" 28 29 #include "spdk/bdev_module.h" 30 #include "spdk/log.h" 31 32 #include "spdk_internal/usdt.h" 33 #include "spdk_internal/trace_defs.h" 34 35 #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true 36 #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS (10000) 37 38 #define NSID_STR_LEN 10 39 40 #define SPDK_CONTROLLER_NAME_MAX 512 41 42 static int bdev_nvme_config_json(struct spdk_json_write_ctx *w); 43 44 struct nvme_bdev_io { 45 /** array of iovecs to transfer. */ 46 struct iovec *iovs; 47 48 /** Number of iovecs in iovs array. */ 49 int iovcnt; 50 51 /** Current iovec position. */ 52 int iovpos; 53 54 /** Offset in current iovec. */ 55 uint32_t iov_offset; 56 57 /** Offset in current iovec. */ 58 uint32_t fused_iov_offset; 59 60 /** array of iovecs to transfer. */ 61 struct iovec *fused_iovs; 62 63 /** Number of iovecs in iovs array. */ 64 int fused_iovcnt; 65 66 /** Current iovec position. */ 67 int fused_iovpos; 68 69 /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path 70 * being reset in a reset I/O. 71 */ 72 struct nvme_io_path *io_path; 73 74 /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */ 75 struct spdk_nvme_cpl cpl; 76 77 /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */ 78 struct spdk_nvme_ns_cmd_ext_io_opts ext_opts; 79 80 /** Keeps track if first of fused commands was submitted */ 81 bool first_fused_submitted; 82 83 /** Keeps track if first of fused commands was completed */ 84 bool first_fused_completed; 85 86 /* How many times the current I/O was retried. */ 87 int32_t retry_count; 88 89 /** Expiration value in ticks to retry the current I/O. */ 90 uint64_t retry_ticks; 91 92 /** Temporary pointer to zone report buffer */ 93 struct spdk_nvme_zns_zone_report *zone_report_buf; 94 95 /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */ 96 uint64_t handled_zones; 97 98 /* Current tsc at submit time. */ 99 uint64_t submit_tsc; 100 101 /* Used to put nvme_bdev_io into the list */ 102 TAILQ_ENTRY(nvme_bdev_io) retry_link; 103 }; 104 105 struct nvme_probe_skip_entry { 106 struct spdk_nvme_transport_id trid; 107 TAILQ_ENTRY(nvme_probe_skip_entry) tailq; 108 }; 109 /* All the controllers deleted by users via RPC are skipped by hotplug monitor */ 110 static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER( 111 g_skipped_nvme_ctrlrs); 112 113 #define BDEV_NVME_DEFAULT_DIGESTS (SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA256) | \ 114 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA384) | \ 115 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA512)) 116 117 #define BDEV_NVME_DEFAULT_DHGROUPS (SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_NULL) | \ 118 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_2048) | \ 119 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_3072) | \ 120 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_4096) | \ 121 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_6144) | \ 122 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_8192)) 123 124 static struct spdk_bdev_nvme_opts g_opts = { 125 .action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE, 126 .timeout_us = 0, 127 .timeout_admin_us = 0, 128 .keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS, 129 .transport_retry_count = 4, 130 .arbitration_burst = 0, 131 .low_priority_weight = 0, 132 .medium_priority_weight = 0, 133 .high_priority_weight = 0, 134 .nvme_adminq_poll_period_us = 10000ULL, 135 .nvme_ioq_poll_period_us = 0, 136 .io_queue_requests = 0, 137 .delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT, 138 .bdev_retry_count = 3, 139 .transport_ack_timeout = 0, 140 .ctrlr_loss_timeout_sec = 0, 141 .reconnect_delay_sec = 0, 142 .fast_io_fail_timeout_sec = 0, 143 .disable_auto_failback = false, 144 .generate_uuids = false, 145 .transport_tos = 0, 146 .nvme_error_stat = false, 147 .io_path_stat = false, 148 .allow_accel_sequence = false, 149 .dhchap_digests = BDEV_NVME_DEFAULT_DIGESTS, 150 .dhchap_dhgroups = BDEV_NVME_DEFAULT_DHGROUPS, 151 }; 152 153 #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL 154 #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL 155 156 static int g_hot_insert_nvme_controller_index = 0; 157 static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT; 158 static bool g_nvme_hotplug_enabled = false; 159 struct spdk_thread *g_bdev_nvme_init_thread; 160 static struct spdk_poller *g_hotplug_poller; 161 static struct spdk_poller *g_hotplug_probe_poller; 162 static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx; 163 164 static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 165 struct nvme_async_probe_ctx *ctx); 166 static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 167 struct nvme_async_probe_ctx *ctx); 168 static int bdev_nvme_library_init(void); 169 static void bdev_nvme_library_fini(void); 170 static void _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, 171 struct spdk_bdev_io *bdev_io); 172 static void bdev_nvme_submit_request(struct spdk_io_channel *ch, 173 struct spdk_bdev_io *bdev_io); 174 static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 175 void *md, uint64_t lba_count, uint64_t lba, 176 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx, 177 struct spdk_accel_sequence *seq); 178 static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 179 void *md, uint64_t lba_count, uint64_t lba); 180 static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 181 void *md, uint64_t lba_count, uint64_t lba, 182 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx, 183 struct spdk_accel_sequence *seq, 184 union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13); 185 static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 186 void *md, uint64_t lba_count, 187 uint64_t zslba, uint32_t flags); 188 static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 189 void *md, uint64_t lba_count, uint64_t lba, 190 uint32_t flags); 191 static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, 192 struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov, 193 int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba, 194 uint32_t flags); 195 static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, 196 uint32_t num_zones, struct spdk_bdev_zone_info *info); 197 static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 198 enum spdk_bdev_zone_action action); 199 static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, 200 struct nvme_bdev_io *bio, 201 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes); 202 static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 203 void *buf, size_t nbytes); 204 static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 205 void *buf, size_t nbytes, void *md_buf, size_t md_len); 206 static int bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 207 struct iovec *iov, int iovcnt, size_t nbytes, 208 void *md_buf, size_t md_len); 209 static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, 210 struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort); 211 static void bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio); 212 static int bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 213 static int bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 214 static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr); 215 static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr); 216 217 static struct nvme_ns *nvme_ns_alloc(void); 218 static void nvme_ns_free(struct nvme_ns *ns); 219 220 static int 221 nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2) 222 { 223 return ns1->id < ns2->id ? -1 : ns1->id > ns2->id; 224 } 225 226 RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp); 227 228 struct spdk_nvme_qpair * 229 bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch) 230 { 231 struct nvme_ctrlr_channel *ctrlr_ch; 232 233 assert(ctrlr_io_ch != NULL); 234 235 ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch); 236 237 return ctrlr_ch->qpair->qpair; 238 } 239 240 static int 241 bdev_nvme_get_ctx_size(void) 242 { 243 return sizeof(struct nvme_bdev_io); 244 } 245 246 static struct spdk_bdev_module nvme_if = { 247 .name = "nvme", 248 .async_fini = true, 249 .module_init = bdev_nvme_library_init, 250 .module_fini = bdev_nvme_library_fini, 251 .config_json = bdev_nvme_config_json, 252 .get_ctx_size = bdev_nvme_get_ctx_size, 253 254 }; 255 SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if) 256 257 struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs); 258 pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER; 259 bool g_bdev_nvme_module_finish; 260 261 struct nvme_bdev_ctrlr * 262 nvme_bdev_ctrlr_get_by_name(const char *name) 263 { 264 struct nvme_bdev_ctrlr *nbdev_ctrlr; 265 266 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 267 if (strcmp(name, nbdev_ctrlr->name) == 0) { 268 break; 269 } 270 } 271 272 return nbdev_ctrlr; 273 } 274 275 static struct nvme_ctrlr * 276 nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr, 277 const struct spdk_nvme_transport_id *trid, const char *hostnqn) 278 { 279 const struct spdk_nvme_ctrlr_opts *opts; 280 struct nvme_ctrlr *nvme_ctrlr; 281 282 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 283 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 284 if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0 && 285 strcmp(hostnqn, opts->hostnqn) == 0) { 286 break; 287 } 288 } 289 290 return nvme_ctrlr; 291 } 292 293 struct nvme_ctrlr * 294 nvme_bdev_ctrlr_get_ctrlr_by_id(struct nvme_bdev_ctrlr *nbdev_ctrlr, 295 uint16_t cntlid) 296 { 297 struct nvme_ctrlr *nvme_ctrlr; 298 const struct spdk_nvme_ctrlr_data *cdata; 299 300 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 301 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 302 if (cdata->cntlid == cntlid) { 303 break; 304 } 305 } 306 307 return nvme_ctrlr; 308 } 309 310 static struct nvme_bdev * 311 nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid) 312 { 313 struct nvme_bdev *bdev; 314 315 pthread_mutex_lock(&g_bdev_nvme_mutex); 316 TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) { 317 if (bdev->nsid == nsid) { 318 break; 319 } 320 } 321 pthread_mutex_unlock(&g_bdev_nvme_mutex); 322 323 return bdev; 324 } 325 326 struct nvme_ns * 327 nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid) 328 { 329 struct nvme_ns ns; 330 331 assert(nsid > 0); 332 333 ns.id = nsid; 334 return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns); 335 } 336 337 struct nvme_ns * 338 nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr) 339 { 340 return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces); 341 } 342 343 struct nvme_ns * 344 nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns) 345 { 346 if (ns == NULL) { 347 return NULL; 348 } 349 350 return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 351 } 352 353 static struct nvme_ctrlr * 354 nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid, const char *hostnqn) 355 { 356 struct nvme_bdev_ctrlr *nbdev_ctrlr; 357 struct nvme_ctrlr *nvme_ctrlr = NULL; 358 359 pthread_mutex_lock(&g_bdev_nvme_mutex); 360 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 361 nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid, hostnqn); 362 if (nvme_ctrlr != NULL) { 363 break; 364 } 365 } 366 pthread_mutex_unlock(&g_bdev_nvme_mutex); 367 368 return nvme_ctrlr; 369 } 370 371 struct nvme_ctrlr * 372 nvme_ctrlr_get_by_name(const char *name) 373 { 374 struct nvme_bdev_ctrlr *nbdev_ctrlr; 375 struct nvme_ctrlr *nvme_ctrlr = NULL; 376 377 if (name == NULL) { 378 return NULL; 379 } 380 381 pthread_mutex_lock(&g_bdev_nvme_mutex); 382 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 383 if (nbdev_ctrlr != NULL) { 384 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 385 } 386 pthread_mutex_unlock(&g_bdev_nvme_mutex); 387 388 return nvme_ctrlr; 389 } 390 391 void 392 nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx) 393 { 394 struct nvme_bdev_ctrlr *nbdev_ctrlr; 395 396 pthread_mutex_lock(&g_bdev_nvme_mutex); 397 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 398 fn(nbdev_ctrlr, ctx); 399 } 400 pthread_mutex_unlock(&g_bdev_nvme_mutex); 401 } 402 403 struct nvme_ctrlr_channel_iter { 404 nvme_ctrlr_for_each_channel_msg fn; 405 nvme_ctrlr_for_each_channel_done cpl; 406 struct spdk_io_channel_iter *i; 407 void *ctx; 408 }; 409 410 void 411 nvme_ctrlr_for_each_channel_continue(struct nvme_ctrlr_channel_iter *iter, int status) 412 { 413 spdk_for_each_channel_continue(iter->i, status); 414 } 415 416 static void 417 nvme_ctrlr_each_channel_msg(struct spdk_io_channel_iter *i) 418 { 419 struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 420 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 421 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 422 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch); 423 424 iter->i = i; 425 iter->fn(iter, nvme_ctrlr, ctrlr_ch, iter->ctx); 426 } 427 428 static void 429 nvme_ctrlr_each_channel_cpl(struct spdk_io_channel_iter *i, int status) 430 { 431 struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 432 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 433 434 iter->i = i; 435 iter->cpl(nvme_ctrlr, iter->ctx, status); 436 437 free(iter); 438 } 439 440 void 441 nvme_ctrlr_for_each_channel(struct nvme_ctrlr *nvme_ctrlr, 442 nvme_ctrlr_for_each_channel_msg fn, void *ctx, 443 nvme_ctrlr_for_each_channel_done cpl) 444 { 445 struct nvme_ctrlr_channel_iter *iter; 446 447 assert(nvme_ctrlr != NULL && fn != NULL); 448 449 iter = calloc(1, sizeof(struct nvme_ctrlr_channel_iter)); 450 if (iter == NULL) { 451 SPDK_ERRLOG("Unable to allocate iterator\n"); 452 assert(false); 453 return; 454 } 455 456 iter->fn = fn; 457 iter->cpl = cpl; 458 iter->ctx = ctx; 459 460 spdk_for_each_channel(nvme_ctrlr, nvme_ctrlr_each_channel_msg, 461 iter, nvme_ctrlr_each_channel_cpl); 462 } 463 464 struct nvme_bdev_channel_iter { 465 nvme_bdev_for_each_channel_msg fn; 466 nvme_bdev_for_each_channel_done cpl; 467 struct spdk_io_channel_iter *i; 468 void *ctx; 469 }; 470 471 void 472 nvme_bdev_for_each_channel_continue(struct nvme_bdev_channel_iter *iter, int status) 473 { 474 spdk_for_each_channel_continue(iter->i, status); 475 } 476 477 static void 478 nvme_bdev_each_channel_msg(struct spdk_io_channel_iter *i) 479 { 480 struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 481 struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i); 482 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 483 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 484 485 iter->i = i; 486 iter->fn(iter, nbdev, nbdev_ch, iter->ctx); 487 } 488 489 static void 490 nvme_bdev_each_channel_cpl(struct spdk_io_channel_iter *i, int status) 491 { 492 struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 493 struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i); 494 495 iter->i = i; 496 iter->cpl(nbdev, iter->ctx, status); 497 498 free(iter); 499 } 500 501 void 502 nvme_bdev_for_each_channel(struct nvme_bdev *nbdev, 503 nvme_bdev_for_each_channel_msg fn, void *ctx, 504 nvme_bdev_for_each_channel_done cpl) 505 { 506 struct nvme_bdev_channel_iter *iter; 507 508 assert(nbdev != NULL && fn != NULL); 509 510 iter = calloc(1, sizeof(struct nvme_bdev_channel_iter)); 511 if (iter == NULL) { 512 SPDK_ERRLOG("Unable to allocate iterator\n"); 513 assert(false); 514 return; 515 } 516 517 iter->fn = fn; 518 iter->cpl = cpl; 519 iter->ctx = ctx; 520 521 spdk_for_each_channel(nbdev, nvme_bdev_each_channel_msg, iter, 522 nvme_bdev_each_channel_cpl); 523 } 524 525 void 526 nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w) 527 { 528 const char *trtype_str; 529 const char *adrfam_str; 530 531 trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype); 532 if (trtype_str) { 533 spdk_json_write_named_string(w, "trtype", trtype_str); 534 } 535 536 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 537 if (adrfam_str) { 538 spdk_json_write_named_string(w, "adrfam", adrfam_str); 539 } 540 541 if (trid->traddr[0] != '\0') { 542 spdk_json_write_named_string(w, "traddr", trid->traddr); 543 } 544 545 if (trid->trsvcid[0] != '\0') { 546 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 547 } 548 549 if (trid->subnqn[0] != '\0') { 550 spdk_json_write_named_string(w, "subnqn", trid->subnqn); 551 } 552 } 553 554 static void 555 nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr, 556 struct nvme_ctrlr *nvme_ctrlr) 557 { 558 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name); 559 pthread_mutex_lock(&g_bdev_nvme_mutex); 560 561 TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 562 if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) { 563 pthread_mutex_unlock(&g_bdev_nvme_mutex); 564 565 return; 566 } 567 TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 568 569 pthread_mutex_unlock(&g_bdev_nvme_mutex); 570 571 assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs)); 572 573 free(nbdev_ctrlr->name); 574 free(nbdev_ctrlr); 575 } 576 577 static void 578 _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 579 { 580 struct nvme_path_id *path_id, *tmp_path; 581 struct nvme_ns *ns, *tmp_ns; 582 583 free(nvme_ctrlr->copied_ana_desc); 584 spdk_free(nvme_ctrlr->ana_log_page); 585 586 if (nvme_ctrlr->opal_dev) { 587 spdk_opal_dev_destruct(nvme_ctrlr->opal_dev); 588 nvme_ctrlr->opal_dev = NULL; 589 } 590 591 if (nvme_ctrlr->nbdev_ctrlr) { 592 nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr); 593 } 594 595 RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) { 596 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 597 nvme_ns_free(ns); 598 } 599 600 TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) { 601 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 602 free(path_id); 603 } 604 605 pthread_mutex_destroy(&nvme_ctrlr->mutex); 606 spdk_keyring_put_key(nvme_ctrlr->psk); 607 spdk_keyring_put_key(nvme_ctrlr->dhchap_key); 608 spdk_keyring_put_key(nvme_ctrlr->dhchap_ctrlr_key); 609 free(nvme_ctrlr); 610 611 pthread_mutex_lock(&g_bdev_nvme_mutex); 612 if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 613 pthread_mutex_unlock(&g_bdev_nvme_mutex); 614 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 615 spdk_bdev_module_fini_done(); 616 return; 617 } 618 pthread_mutex_unlock(&g_bdev_nvme_mutex); 619 } 620 621 static int 622 nvme_detach_poller(void *arg) 623 { 624 struct nvme_ctrlr *nvme_ctrlr = arg; 625 int rc; 626 627 rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx); 628 if (rc != -EAGAIN) { 629 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 630 _nvme_ctrlr_delete(nvme_ctrlr); 631 } 632 633 return SPDK_POLLER_BUSY; 634 } 635 636 static void 637 nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 638 { 639 int rc; 640 641 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 642 643 /* First, unregister the adminq poller, as the driver will poll adminq if necessary */ 644 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 645 646 /* If we got here, the reset/detach poller cannot be active */ 647 assert(nvme_ctrlr->reset_detach_poller == NULL); 648 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller, 649 nvme_ctrlr, 1000); 650 if (nvme_ctrlr->reset_detach_poller == NULL) { 651 SPDK_ERRLOG("Failed to register detach poller\n"); 652 goto error; 653 } 654 655 rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx); 656 if (rc != 0) { 657 SPDK_ERRLOG("Failed to detach the NVMe controller\n"); 658 goto error; 659 } 660 661 return; 662 error: 663 /* We don't have a good way to handle errors here, so just do what we can and delete the 664 * controller without detaching the underlying NVMe device. 665 */ 666 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 667 _nvme_ctrlr_delete(nvme_ctrlr); 668 } 669 670 static void 671 nvme_ctrlr_unregister_cb(void *io_device) 672 { 673 struct nvme_ctrlr *nvme_ctrlr = io_device; 674 675 nvme_ctrlr_delete(nvme_ctrlr); 676 } 677 678 static void 679 nvme_ctrlr_unregister(void *ctx) 680 { 681 struct nvme_ctrlr *nvme_ctrlr = ctx; 682 683 spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb); 684 } 685 686 static bool 687 nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr) 688 { 689 if (!nvme_ctrlr->destruct) { 690 return false; 691 } 692 693 if (nvme_ctrlr->ref > 0) { 694 return false; 695 } 696 697 if (nvme_ctrlr->resetting) { 698 return false; 699 } 700 701 if (nvme_ctrlr->ana_log_page_updating) { 702 return false; 703 } 704 705 if (nvme_ctrlr->io_path_cache_clearing) { 706 return false; 707 } 708 709 return true; 710 } 711 712 static void 713 nvme_ctrlr_release(struct nvme_ctrlr *nvme_ctrlr) 714 { 715 pthread_mutex_lock(&nvme_ctrlr->mutex); 716 SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref); 717 718 assert(nvme_ctrlr->ref > 0); 719 nvme_ctrlr->ref--; 720 721 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 722 pthread_mutex_unlock(&nvme_ctrlr->mutex); 723 return; 724 } 725 726 pthread_mutex_unlock(&nvme_ctrlr->mutex); 727 728 spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr); 729 } 730 731 static void 732 bdev_nvme_clear_current_io_path(struct nvme_bdev_channel *nbdev_ch) 733 { 734 nbdev_ch->current_io_path = NULL; 735 nbdev_ch->rr_counter = 0; 736 } 737 738 static struct nvme_io_path * 739 _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 740 { 741 struct nvme_io_path *io_path; 742 743 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 744 if (io_path->nvme_ns == nvme_ns) { 745 break; 746 } 747 } 748 749 return io_path; 750 } 751 752 static struct nvme_io_path * 753 nvme_io_path_alloc(void) 754 { 755 struct nvme_io_path *io_path; 756 757 io_path = calloc(1, sizeof(*io_path)); 758 if (io_path == NULL) { 759 SPDK_ERRLOG("Failed to alloc io_path.\n"); 760 return NULL; 761 } 762 763 if (g_opts.io_path_stat) { 764 io_path->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 765 if (io_path->stat == NULL) { 766 free(io_path); 767 SPDK_ERRLOG("Failed to alloc io_path stat.\n"); 768 return NULL; 769 } 770 spdk_bdev_reset_io_stat(io_path->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 771 } 772 773 return io_path; 774 } 775 776 static void 777 nvme_io_path_free(struct nvme_io_path *io_path) 778 { 779 free(io_path->stat); 780 free(io_path); 781 } 782 783 static int 784 _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 785 { 786 struct nvme_io_path *io_path; 787 struct spdk_io_channel *ch; 788 struct nvme_ctrlr_channel *ctrlr_ch; 789 struct nvme_qpair *nvme_qpair; 790 791 io_path = nvme_io_path_alloc(); 792 if (io_path == NULL) { 793 return -ENOMEM; 794 } 795 796 io_path->nvme_ns = nvme_ns; 797 798 ch = spdk_get_io_channel(nvme_ns->ctrlr); 799 if (ch == NULL) { 800 nvme_io_path_free(io_path); 801 SPDK_ERRLOG("Failed to alloc io_channel.\n"); 802 return -ENOMEM; 803 } 804 805 ctrlr_ch = spdk_io_channel_get_ctx(ch); 806 807 nvme_qpair = ctrlr_ch->qpair; 808 assert(nvme_qpair != NULL); 809 810 io_path->qpair = nvme_qpair; 811 TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq); 812 813 io_path->nbdev_ch = nbdev_ch; 814 STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq); 815 816 bdev_nvme_clear_current_io_path(nbdev_ch); 817 818 return 0; 819 } 820 821 static void 822 bdev_nvme_clear_retry_io_path(struct nvme_bdev_channel *nbdev_ch, 823 struct nvme_io_path *io_path) 824 { 825 struct nvme_bdev_io *bio; 826 827 TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) { 828 if (bio->io_path == io_path) { 829 bio->io_path = NULL; 830 } 831 } 832 } 833 834 static void 835 _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path) 836 { 837 struct spdk_io_channel *ch; 838 struct nvme_qpair *nvme_qpair; 839 struct nvme_ctrlr_channel *ctrlr_ch; 840 struct nvme_bdev *nbdev; 841 842 nbdev = spdk_io_channel_get_io_device(spdk_io_channel_from_ctx(nbdev_ch)); 843 844 /* Add the statistics to nvme_ns before this path is destroyed. */ 845 pthread_mutex_lock(&nbdev->mutex); 846 if (nbdev->ref != 0 && io_path->nvme_ns->stat != NULL && io_path->stat != NULL) { 847 spdk_bdev_add_io_stat(io_path->nvme_ns->stat, io_path->stat); 848 } 849 pthread_mutex_unlock(&nbdev->mutex); 850 851 bdev_nvme_clear_current_io_path(nbdev_ch); 852 bdev_nvme_clear_retry_io_path(nbdev_ch, io_path); 853 854 STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq); 855 io_path->nbdev_ch = NULL; 856 857 nvme_qpair = io_path->qpair; 858 assert(nvme_qpair != NULL); 859 860 ctrlr_ch = nvme_qpair->ctrlr_ch; 861 assert(ctrlr_ch != NULL); 862 863 ch = spdk_io_channel_from_ctx(ctrlr_ch); 864 spdk_put_io_channel(ch); 865 866 /* After an io_path is removed, I/Os submitted to it may complete and update statistics 867 * of the io_path. To avoid heap-use-after-free error from this case, do not free the 868 * io_path here but free the io_path when the associated qpair is freed. It is ensured 869 * that all I/Os submitted to the io_path are completed when the associated qpair is freed. 870 */ 871 } 872 873 static void 874 _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch) 875 { 876 struct nvme_io_path *io_path, *tmp_io_path; 877 878 STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) { 879 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 880 } 881 } 882 883 static int 884 bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf) 885 { 886 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 887 struct nvme_bdev *nbdev = io_device; 888 struct nvme_ns *nvme_ns; 889 int rc; 890 891 STAILQ_INIT(&nbdev_ch->io_path_list); 892 TAILQ_INIT(&nbdev_ch->retry_io_list); 893 894 pthread_mutex_lock(&nbdev->mutex); 895 896 nbdev_ch->mp_policy = nbdev->mp_policy; 897 nbdev_ch->mp_selector = nbdev->mp_selector; 898 nbdev_ch->rr_min_io = nbdev->rr_min_io; 899 900 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 901 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 902 if (rc != 0) { 903 pthread_mutex_unlock(&nbdev->mutex); 904 905 _bdev_nvme_delete_io_paths(nbdev_ch); 906 return rc; 907 } 908 } 909 pthread_mutex_unlock(&nbdev->mutex); 910 911 return 0; 912 } 913 914 /* If cpl != NULL, complete the bdev_io with nvme status based on 'cpl'. 915 * If cpl == NULL, complete the bdev_io with bdev status based on 'status'. 916 */ 917 static inline void 918 __bdev_nvme_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status, 919 const struct spdk_nvme_cpl *cpl) 920 { 921 spdk_trace_record(TRACE_BDEV_NVME_IO_DONE, 0, 0, (uintptr_t)bdev_io->driver_ctx, 922 (uintptr_t)bdev_io); 923 if (cpl) { 924 spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc); 925 } else { 926 spdk_bdev_io_complete(bdev_io, status); 927 } 928 } 929 930 static void bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch); 931 932 static void 933 bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf) 934 { 935 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 936 937 bdev_nvme_abort_retry_ios(nbdev_ch); 938 _bdev_nvme_delete_io_paths(nbdev_ch); 939 } 940 941 static inline bool 942 bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type) 943 { 944 switch (io_type) { 945 case SPDK_BDEV_IO_TYPE_RESET: 946 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 947 case SPDK_BDEV_IO_TYPE_ABORT: 948 return true; 949 default: 950 break; 951 } 952 953 return false; 954 } 955 956 static inline bool 957 nvme_ns_is_active(struct nvme_ns *nvme_ns) 958 { 959 if (spdk_unlikely(nvme_ns->ana_state_updating)) { 960 return false; 961 } 962 963 if (spdk_unlikely(nvme_ns->ns == NULL)) { 964 return false; 965 } 966 967 return true; 968 } 969 970 static inline bool 971 nvme_ns_is_accessible(struct nvme_ns *nvme_ns) 972 { 973 if (spdk_unlikely(!nvme_ns_is_active(nvme_ns))) { 974 return false; 975 } 976 977 switch (nvme_ns->ana_state) { 978 case SPDK_NVME_ANA_OPTIMIZED_STATE: 979 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 980 return true; 981 default: 982 break; 983 } 984 985 return false; 986 } 987 988 static inline bool 989 nvme_qpair_is_connected(struct nvme_qpair *nvme_qpair) 990 { 991 if (spdk_unlikely(nvme_qpair->qpair == NULL)) { 992 return false; 993 } 994 995 if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 996 SPDK_NVME_QPAIR_FAILURE_NONE)) { 997 return false; 998 } 999 1000 if (spdk_unlikely(nvme_qpair->ctrlr_ch->reset_iter != NULL)) { 1001 return false; 1002 } 1003 1004 return true; 1005 } 1006 1007 static inline bool 1008 nvme_io_path_is_available(struct nvme_io_path *io_path) 1009 { 1010 if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) { 1011 return false; 1012 } 1013 1014 if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) { 1015 return false; 1016 } 1017 1018 return true; 1019 } 1020 1021 static inline bool 1022 nvme_ctrlr_is_failed(struct nvme_ctrlr *nvme_ctrlr) 1023 { 1024 if (nvme_ctrlr->destruct) { 1025 return true; 1026 } 1027 1028 if (nvme_ctrlr->fast_io_fail_timedout) { 1029 return true; 1030 } 1031 1032 if (nvme_ctrlr->resetting) { 1033 if (nvme_ctrlr->opts.reconnect_delay_sec != 0) { 1034 return false; 1035 } else { 1036 return true; 1037 } 1038 } 1039 1040 if (nvme_ctrlr->reconnect_is_delayed) { 1041 return false; 1042 } 1043 1044 if (nvme_ctrlr->disabled) { 1045 return true; 1046 } 1047 1048 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 1049 return true; 1050 } else { 1051 return false; 1052 } 1053 } 1054 1055 static bool 1056 nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr) 1057 { 1058 if (nvme_ctrlr->destruct) { 1059 return false; 1060 } 1061 1062 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 1063 return false; 1064 } 1065 1066 if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) { 1067 return false; 1068 } 1069 1070 if (nvme_ctrlr->disabled) { 1071 return false; 1072 } 1073 1074 return true; 1075 } 1076 1077 /* Simulate circular linked list. */ 1078 static inline struct nvme_io_path * 1079 nvme_io_path_get_next(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *prev_path) 1080 { 1081 struct nvme_io_path *next_path; 1082 1083 if (prev_path != NULL) { 1084 next_path = STAILQ_NEXT(prev_path, stailq); 1085 if (next_path != NULL) { 1086 return next_path; 1087 } 1088 } 1089 1090 return STAILQ_FIRST(&nbdev_ch->io_path_list); 1091 } 1092 1093 static struct nvme_io_path * 1094 _bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 1095 { 1096 struct nvme_io_path *io_path, *start, *non_optimized = NULL; 1097 1098 start = nvme_io_path_get_next(nbdev_ch, nbdev_ch->current_io_path); 1099 1100 io_path = start; 1101 do { 1102 if (spdk_likely(nvme_io_path_is_available(io_path))) { 1103 switch (io_path->nvme_ns->ana_state) { 1104 case SPDK_NVME_ANA_OPTIMIZED_STATE: 1105 nbdev_ch->current_io_path = io_path; 1106 return io_path; 1107 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 1108 if (non_optimized == NULL) { 1109 non_optimized = io_path; 1110 } 1111 break; 1112 default: 1113 assert(false); 1114 break; 1115 } 1116 } 1117 io_path = nvme_io_path_get_next(nbdev_ch, io_path); 1118 } while (io_path != start); 1119 1120 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 1121 /* We come here only if there is no optimized path. Cache even non_optimized 1122 * path for load balance across multiple non_optimized paths. 1123 */ 1124 nbdev_ch->current_io_path = non_optimized; 1125 } 1126 1127 return non_optimized; 1128 } 1129 1130 static struct nvme_io_path * 1131 _bdev_nvme_find_io_path_min_qd(struct nvme_bdev_channel *nbdev_ch) 1132 { 1133 struct nvme_io_path *io_path; 1134 struct nvme_io_path *optimized = NULL, *non_optimized = NULL; 1135 uint32_t opt_min_qd = UINT32_MAX, non_opt_min_qd = UINT32_MAX; 1136 uint32_t num_outstanding_reqs; 1137 1138 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 1139 if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) { 1140 /* The device is currently resetting. */ 1141 continue; 1142 } 1143 1144 if (spdk_unlikely(!nvme_ns_is_active(io_path->nvme_ns))) { 1145 continue; 1146 } 1147 1148 num_outstanding_reqs = spdk_nvme_qpair_get_num_outstanding_reqs(io_path->qpair->qpair); 1149 switch (io_path->nvme_ns->ana_state) { 1150 case SPDK_NVME_ANA_OPTIMIZED_STATE: 1151 if (num_outstanding_reqs < opt_min_qd) { 1152 opt_min_qd = num_outstanding_reqs; 1153 optimized = io_path; 1154 } 1155 break; 1156 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 1157 if (num_outstanding_reqs < non_opt_min_qd) { 1158 non_opt_min_qd = num_outstanding_reqs; 1159 non_optimized = io_path; 1160 } 1161 break; 1162 default: 1163 break; 1164 } 1165 } 1166 1167 /* don't cache io path for BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH selector */ 1168 if (optimized != NULL) { 1169 return optimized; 1170 } 1171 1172 return non_optimized; 1173 } 1174 1175 static inline struct nvme_io_path * 1176 bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 1177 { 1178 if (spdk_likely(nbdev_ch->current_io_path != NULL)) { 1179 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE) { 1180 return nbdev_ch->current_io_path; 1181 } else if (nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 1182 if (++nbdev_ch->rr_counter < nbdev_ch->rr_min_io) { 1183 return nbdev_ch->current_io_path; 1184 } 1185 nbdev_ch->rr_counter = 0; 1186 } 1187 } 1188 1189 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE || 1190 nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 1191 return _bdev_nvme_find_io_path(nbdev_ch); 1192 } else { 1193 return _bdev_nvme_find_io_path_min_qd(nbdev_ch); 1194 } 1195 } 1196 1197 /* Return true if there is any io_path whose qpair is active or ctrlr is not failed, 1198 * or false otherwise. 1199 * 1200 * If any io_path has an active qpair but find_io_path() returned NULL, its namespace 1201 * is likely to be non-accessible now but may become accessible. 1202 * 1203 * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr 1204 * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed 1205 * when starting to reset it but it is set to failed when the reset failed. Hence, if 1206 * a ctrlr is unfailed, it is likely that it works fine or is resetting. 1207 */ 1208 static bool 1209 any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch) 1210 { 1211 struct nvme_io_path *io_path; 1212 1213 if (nbdev_ch->resetting) { 1214 return false; 1215 } 1216 1217 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 1218 if (io_path->nvme_ns->ana_transition_timedout) { 1219 continue; 1220 } 1221 1222 if (nvme_qpair_is_connected(io_path->qpair) || 1223 !nvme_ctrlr_is_failed(io_path->qpair->ctrlr)) { 1224 return true; 1225 } 1226 } 1227 1228 return false; 1229 } 1230 1231 static void 1232 bdev_nvme_retry_io(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 1233 { 1234 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1235 struct spdk_io_channel *ch; 1236 1237 if (nbdev_io->io_path != NULL && nvme_io_path_is_available(nbdev_io->io_path)) { 1238 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 1239 } else { 1240 ch = spdk_io_channel_from_ctx(nbdev_ch); 1241 bdev_nvme_submit_request(ch, bdev_io); 1242 } 1243 } 1244 1245 static int 1246 bdev_nvme_retry_ios(void *arg) 1247 { 1248 struct nvme_bdev_channel *nbdev_ch = arg; 1249 struct nvme_bdev_io *bio, *tmp_bio; 1250 uint64_t now, delay_us; 1251 1252 now = spdk_get_ticks(); 1253 1254 TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) { 1255 if (bio->retry_ticks > now) { 1256 break; 1257 } 1258 1259 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1260 1261 bdev_nvme_retry_io(nbdev_ch, spdk_bdev_io_from_ctx(bio)); 1262 } 1263 1264 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1265 1266 bio = TAILQ_FIRST(&nbdev_ch->retry_io_list); 1267 if (bio != NULL) { 1268 delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz(); 1269 1270 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1271 delay_us); 1272 } 1273 1274 return SPDK_POLLER_BUSY; 1275 } 1276 1277 static void 1278 bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch, 1279 struct nvme_bdev_io *bio, uint64_t delay_ms) 1280 { 1281 struct nvme_bdev_io *tmp_bio; 1282 1283 bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL; 1284 1285 TAILQ_FOREACH_REVERSE(tmp_bio, &nbdev_ch->retry_io_list, retry_io_head, retry_link) { 1286 if (tmp_bio->retry_ticks <= bio->retry_ticks) { 1287 TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bio, bio, 1288 retry_link); 1289 return; 1290 } 1291 } 1292 1293 /* No earlier I/Os were found. This I/O must be the new head. */ 1294 TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bio, retry_link); 1295 1296 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1297 1298 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1299 delay_ms * 1000ULL); 1300 } 1301 1302 static void 1303 bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch) 1304 { 1305 struct nvme_bdev_io *bio, *tmp_bio; 1306 1307 TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) { 1308 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1309 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1310 } 1311 1312 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1313 } 1314 1315 static int 1316 bdev_nvme_abort_retry_io(struct nvme_bdev_channel *nbdev_ch, 1317 struct nvme_bdev_io *bio_to_abort) 1318 { 1319 struct nvme_bdev_io *bio; 1320 1321 TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) { 1322 if (bio == bio_to_abort) { 1323 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1324 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1325 return 0; 1326 } 1327 } 1328 1329 return -ENOENT; 1330 } 1331 1332 static void 1333 bdev_nvme_update_nvme_error_stat(struct spdk_bdev_io *bdev_io, const struct spdk_nvme_cpl *cpl) 1334 { 1335 struct nvme_bdev *nbdev; 1336 uint16_t sct, sc; 1337 1338 assert(spdk_nvme_cpl_is_error(cpl)); 1339 1340 nbdev = bdev_io->bdev->ctxt; 1341 1342 if (nbdev->err_stat == NULL) { 1343 return; 1344 } 1345 1346 sct = cpl->status.sct; 1347 sc = cpl->status.sc; 1348 1349 pthread_mutex_lock(&nbdev->mutex); 1350 1351 nbdev->err_stat->status_type[sct]++; 1352 switch (sct) { 1353 case SPDK_NVME_SCT_GENERIC: 1354 case SPDK_NVME_SCT_COMMAND_SPECIFIC: 1355 case SPDK_NVME_SCT_MEDIA_ERROR: 1356 case SPDK_NVME_SCT_PATH: 1357 nbdev->err_stat->status[sct][sc]++; 1358 break; 1359 default: 1360 break; 1361 } 1362 1363 pthread_mutex_unlock(&nbdev->mutex); 1364 } 1365 1366 static inline void 1367 bdev_nvme_update_io_path_stat(struct nvme_bdev_io *bio) 1368 { 1369 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1370 uint64_t num_blocks = bdev_io->u.bdev.num_blocks; 1371 uint32_t blocklen = bdev_io->bdev->blocklen; 1372 struct spdk_bdev_io_stat *stat; 1373 uint64_t tsc_diff; 1374 1375 if (bio->io_path->stat == NULL) { 1376 return; 1377 } 1378 1379 tsc_diff = spdk_get_ticks() - bio->submit_tsc; 1380 stat = bio->io_path->stat; 1381 1382 switch (bdev_io->type) { 1383 case SPDK_BDEV_IO_TYPE_READ: 1384 stat->bytes_read += num_blocks * blocklen; 1385 stat->num_read_ops++; 1386 stat->read_latency_ticks += tsc_diff; 1387 if (stat->max_read_latency_ticks < tsc_diff) { 1388 stat->max_read_latency_ticks = tsc_diff; 1389 } 1390 if (stat->min_read_latency_ticks > tsc_diff) { 1391 stat->min_read_latency_ticks = tsc_diff; 1392 } 1393 break; 1394 case SPDK_BDEV_IO_TYPE_WRITE: 1395 stat->bytes_written += num_blocks * blocklen; 1396 stat->num_write_ops++; 1397 stat->write_latency_ticks += tsc_diff; 1398 if (stat->max_write_latency_ticks < tsc_diff) { 1399 stat->max_write_latency_ticks = tsc_diff; 1400 } 1401 if (stat->min_write_latency_ticks > tsc_diff) { 1402 stat->min_write_latency_ticks = tsc_diff; 1403 } 1404 break; 1405 case SPDK_BDEV_IO_TYPE_UNMAP: 1406 stat->bytes_unmapped += num_blocks * blocklen; 1407 stat->num_unmap_ops++; 1408 stat->unmap_latency_ticks += tsc_diff; 1409 if (stat->max_unmap_latency_ticks < tsc_diff) { 1410 stat->max_unmap_latency_ticks = tsc_diff; 1411 } 1412 if (stat->min_unmap_latency_ticks > tsc_diff) { 1413 stat->min_unmap_latency_ticks = tsc_diff; 1414 } 1415 break; 1416 case SPDK_BDEV_IO_TYPE_ZCOPY: 1417 /* Track the data in the start phase only */ 1418 if (!bdev_io->u.bdev.zcopy.start) { 1419 break; 1420 } 1421 if (bdev_io->u.bdev.zcopy.populate) { 1422 stat->bytes_read += num_blocks * blocklen; 1423 stat->num_read_ops++; 1424 stat->read_latency_ticks += tsc_diff; 1425 if (stat->max_read_latency_ticks < tsc_diff) { 1426 stat->max_read_latency_ticks = tsc_diff; 1427 } 1428 if (stat->min_read_latency_ticks > tsc_diff) { 1429 stat->min_read_latency_ticks = tsc_diff; 1430 } 1431 } else { 1432 stat->bytes_written += num_blocks * blocklen; 1433 stat->num_write_ops++; 1434 stat->write_latency_ticks += tsc_diff; 1435 if (stat->max_write_latency_ticks < tsc_diff) { 1436 stat->max_write_latency_ticks = tsc_diff; 1437 } 1438 if (stat->min_write_latency_ticks > tsc_diff) { 1439 stat->min_write_latency_ticks = tsc_diff; 1440 } 1441 } 1442 break; 1443 case SPDK_BDEV_IO_TYPE_COPY: 1444 stat->bytes_copied += num_blocks * blocklen; 1445 stat->num_copy_ops++; 1446 stat->copy_latency_ticks += tsc_diff; 1447 if (stat->max_copy_latency_ticks < tsc_diff) { 1448 stat->max_copy_latency_ticks = tsc_diff; 1449 } 1450 if (stat->min_copy_latency_ticks > tsc_diff) { 1451 stat->min_copy_latency_ticks = tsc_diff; 1452 } 1453 break; 1454 default: 1455 break; 1456 } 1457 } 1458 1459 static bool 1460 bdev_nvme_check_retry_io(struct nvme_bdev_io *bio, 1461 const struct spdk_nvme_cpl *cpl, 1462 struct nvme_bdev_channel *nbdev_ch, 1463 uint64_t *_delay_ms) 1464 { 1465 struct nvme_io_path *io_path = bio->io_path; 1466 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 1467 const struct spdk_nvme_ctrlr_data *cdata; 1468 1469 if (spdk_nvme_cpl_is_path_error(cpl) || 1470 spdk_nvme_cpl_is_aborted_sq_deletion(cpl) || 1471 !nvme_io_path_is_available(io_path) || 1472 !nvme_ctrlr_is_available(nvme_ctrlr)) { 1473 bdev_nvme_clear_current_io_path(nbdev_ch); 1474 bio->io_path = NULL; 1475 if (spdk_nvme_cpl_is_ana_error(cpl)) { 1476 if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) { 1477 io_path->nvme_ns->ana_state_updating = true; 1478 } 1479 } 1480 if (!any_io_path_may_become_available(nbdev_ch)) { 1481 return false; 1482 } 1483 *_delay_ms = 0; 1484 } else { 1485 bio->retry_count++; 1486 1487 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 1488 1489 if (cpl->status.crd != 0) { 1490 *_delay_ms = cdata->crdt[cpl->status.crd] * 100; 1491 } else { 1492 *_delay_ms = 0; 1493 } 1494 } 1495 1496 return true; 1497 } 1498 1499 static inline void 1500 bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio, 1501 const struct spdk_nvme_cpl *cpl) 1502 { 1503 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1504 struct nvme_bdev_channel *nbdev_ch; 1505 uint64_t delay_ms; 1506 1507 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 1508 1509 if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) { 1510 bdev_nvme_update_io_path_stat(bio); 1511 goto complete; 1512 } 1513 1514 /* Update error counts before deciding if retry is needed. 1515 * Hence, error counts may be more than the number of I/O errors. 1516 */ 1517 bdev_nvme_update_nvme_error_stat(bdev_io, cpl); 1518 1519 if (cpl->status.dnr != 0 || spdk_nvme_cpl_is_aborted_by_request(cpl) || 1520 (g_opts.bdev_retry_count != -1 && bio->retry_count >= g_opts.bdev_retry_count)) { 1521 goto complete; 1522 } 1523 1524 /* At this point we don't know whether the sequence was successfully executed or not, so we 1525 * cannot retry the IO */ 1526 if (bdev_io->u.bdev.accel_sequence != NULL) { 1527 goto complete; 1528 } 1529 1530 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1531 1532 if (bdev_nvme_check_retry_io(bio, cpl, nbdev_ch, &delay_ms)) { 1533 bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms); 1534 return; 1535 } 1536 1537 complete: 1538 bio->retry_count = 0; 1539 bio->submit_tsc = 0; 1540 bdev_io->u.bdev.accel_sequence = NULL; 1541 __bdev_nvme_io_complete(bdev_io, 0, cpl); 1542 } 1543 1544 static inline void 1545 bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc) 1546 { 1547 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1548 struct nvme_bdev_channel *nbdev_ch; 1549 enum spdk_bdev_io_status io_status; 1550 1551 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 1552 1553 switch (rc) { 1554 case 0: 1555 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1556 break; 1557 case -ENOMEM: 1558 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1559 break; 1560 case -ENXIO: 1561 if (g_opts.bdev_retry_count == -1 || bio->retry_count < g_opts.bdev_retry_count) { 1562 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1563 1564 bdev_nvme_clear_current_io_path(nbdev_ch); 1565 bio->io_path = NULL; 1566 1567 if (any_io_path_may_become_available(nbdev_ch)) { 1568 bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL); 1569 return; 1570 } 1571 } 1572 1573 /* fallthrough */ 1574 default: 1575 spdk_accel_sequence_abort(bdev_io->u.bdev.accel_sequence); 1576 bdev_io->u.bdev.accel_sequence = NULL; 1577 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1578 break; 1579 } 1580 1581 bio->retry_count = 0; 1582 bio->submit_tsc = 0; 1583 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1584 } 1585 1586 static inline void 1587 bdev_nvme_admin_complete(struct nvme_bdev_io *bio, int rc) 1588 { 1589 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1590 enum spdk_bdev_io_status io_status; 1591 1592 switch (rc) { 1593 case 0: 1594 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1595 break; 1596 case -ENOMEM: 1597 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1598 break; 1599 case -ENXIO: 1600 /* fallthrough */ 1601 default: 1602 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1603 break; 1604 } 1605 1606 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1607 } 1608 1609 static void 1610 bdev_nvme_clear_io_path_caches_done(struct nvme_ctrlr *nvme_ctrlr, 1611 void *ctx, int status) 1612 { 1613 pthread_mutex_lock(&nvme_ctrlr->mutex); 1614 1615 assert(nvme_ctrlr->io_path_cache_clearing == true); 1616 nvme_ctrlr->io_path_cache_clearing = false; 1617 1618 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 1619 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1620 return; 1621 } 1622 1623 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1624 1625 nvme_ctrlr_unregister(nvme_ctrlr); 1626 } 1627 1628 static void 1629 _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair) 1630 { 1631 struct nvme_io_path *io_path; 1632 1633 TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) { 1634 if (io_path->nbdev_ch == NULL) { 1635 continue; 1636 } 1637 bdev_nvme_clear_current_io_path(io_path->nbdev_ch); 1638 } 1639 } 1640 1641 static void 1642 bdev_nvme_clear_io_path_cache(struct nvme_ctrlr_channel_iter *i, 1643 struct nvme_ctrlr *nvme_ctrlr, 1644 struct nvme_ctrlr_channel *ctrlr_ch, 1645 void *ctx) 1646 { 1647 assert(ctrlr_ch->qpair != NULL); 1648 1649 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 1650 1651 nvme_ctrlr_for_each_channel_continue(i, 0); 1652 } 1653 1654 static void 1655 bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr) 1656 { 1657 pthread_mutex_lock(&nvme_ctrlr->mutex); 1658 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 1659 nvme_ctrlr->io_path_cache_clearing) { 1660 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1661 return; 1662 } 1663 1664 nvme_ctrlr->io_path_cache_clearing = true; 1665 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1666 1667 nvme_ctrlr_for_each_channel(nvme_ctrlr, 1668 bdev_nvme_clear_io_path_cache, 1669 NULL, 1670 bdev_nvme_clear_io_path_caches_done); 1671 } 1672 1673 static struct nvme_qpair * 1674 nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair) 1675 { 1676 struct nvme_qpair *nvme_qpair; 1677 1678 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1679 if (nvme_qpair->qpair == qpair) { 1680 break; 1681 } 1682 } 1683 1684 return nvme_qpair; 1685 } 1686 1687 static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair); 1688 1689 static void 1690 bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx) 1691 { 1692 struct nvme_poll_group *group = poll_group_ctx; 1693 struct nvme_qpair *nvme_qpair; 1694 struct nvme_ctrlr_channel *ctrlr_ch; 1695 int status; 1696 1697 nvme_qpair = nvme_poll_group_get_qpair(group, qpair); 1698 if (nvme_qpair == NULL) { 1699 return; 1700 } 1701 1702 if (nvme_qpair->qpair != NULL) { 1703 spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair); 1704 nvme_qpair->qpair = NULL; 1705 } 1706 1707 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1708 1709 ctrlr_ch = nvme_qpair->ctrlr_ch; 1710 1711 if (ctrlr_ch != NULL) { 1712 if (ctrlr_ch->reset_iter != NULL) { 1713 /* We are in a full reset sequence. */ 1714 if (ctrlr_ch->connect_poller != NULL) { 1715 /* qpair was failed to connect. Abort the reset sequence. */ 1716 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was failed to connect. abort the reset ctrlr sequence.\n", 1717 qpair); 1718 spdk_poller_unregister(&ctrlr_ch->connect_poller); 1719 status = -1; 1720 } else { 1721 /* qpair was completed to disconnect. Just move to the next ctrlr_channel. */ 1722 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed in a reset ctrlr sequence.\n", 1723 qpair); 1724 status = 0; 1725 } 1726 nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, status); 1727 ctrlr_ch->reset_iter = NULL; 1728 } else { 1729 /* qpair was disconnected unexpectedly. Reset controller for recovery. */ 1730 SPDK_NOTICELOG("qpair %p was disconnected and freed. reset controller.\n", qpair); 1731 bdev_nvme_failover_ctrlr(nvme_qpair->ctrlr); 1732 } 1733 } else { 1734 /* In this case, ctrlr_channel is already deleted. */ 1735 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed. delete nvme_qpair.\n", qpair); 1736 nvme_qpair_delete(nvme_qpair); 1737 } 1738 } 1739 1740 static void 1741 bdev_nvme_check_io_qpairs(struct nvme_poll_group *group) 1742 { 1743 struct nvme_qpair *nvme_qpair; 1744 1745 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1746 if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) { 1747 continue; 1748 } 1749 1750 if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 1751 SPDK_NVME_QPAIR_FAILURE_NONE) { 1752 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1753 } 1754 } 1755 } 1756 1757 static int 1758 bdev_nvme_poll(void *arg) 1759 { 1760 struct nvme_poll_group *group = arg; 1761 int64_t num_completions; 1762 1763 if (group->collect_spin_stat && group->start_ticks == 0) { 1764 group->start_ticks = spdk_get_ticks(); 1765 } 1766 1767 num_completions = spdk_nvme_poll_group_process_completions(group->group, 0, 1768 bdev_nvme_disconnected_qpair_cb); 1769 if (group->collect_spin_stat) { 1770 if (num_completions > 0) { 1771 if (group->end_ticks != 0) { 1772 group->spin_ticks += (group->end_ticks - group->start_ticks); 1773 group->end_ticks = 0; 1774 } 1775 group->start_ticks = 0; 1776 } else { 1777 group->end_ticks = spdk_get_ticks(); 1778 } 1779 } 1780 1781 if (spdk_unlikely(num_completions < 0)) { 1782 bdev_nvme_check_io_qpairs(group); 1783 } 1784 1785 return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 1786 } 1787 1788 static int bdev_nvme_poll_adminq(void *arg); 1789 1790 static void 1791 bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_period_us) 1792 { 1793 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 1794 1795 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, 1796 nvme_ctrlr, new_period_us); 1797 } 1798 1799 static int 1800 bdev_nvme_poll_adminq(void *arg) 1801 { 1802 int32_t rc; 1803 struct nvme_ctrlr *nvme_ctrlr = arg; 1804 nvme_ctrlr_disconnected_cb disconnected_cb; 1805 1806 assert(nvme_ctrlr != NULL); 1807 1808 rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr); 1809 if (rc < 0) { 1810 disconnected_cb = nvme_ctrlr->disconnected_cb; 1811 nvme_ctrlr->disconnected_cb = NULL; 1812 1813 if (disconnected_cb != NULL) { 1814 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 1815 g_opts.nvme_adminq_poll_period_us); 1816 disconnected_cb(nvme_ctrlr); 1817 } else { 1818 bdev_nvme_failover_ctrlr(nvme_ctrlr); 1819 } 1820 } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) != 1821 SPDK_NVME_QPAIR_FAILURE_NONE) { 1822 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 1823 } 1824 1825 return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY; 1826 } 1827 1828 static void 1829 nvme_bdev_free(void *io_device) 1830 { 1831 struct nvme_bdev *nvme_disk = io_device; 1832 1833 pthread_mutex_destroy(&nvme_disk->mutex); 1834 free(nvme_disk->disk.name); 1835 free(nvme_disk->err_stat); 1836 free(nvme_disk); 1837 } 1838 1839 static int 1840 bdev_nvme_destruct(void *ctx) 1841 { 1842 struct nvme_bdev *nvme_disk = ctx; 1843 struct nvme_ns *nvme_ns, *tmp_nvme_ns; 1844 1845 SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid); 1846 1847 TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) { 1848 pthread_mutex_lock(&nvme_ns->ctrlr->mutex); 1849 1850 nvme_ns->bdev = NULL; 1851 1852 assert(nvme_ns->id > 0); 1853 1854 if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) { 1855 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1856 1857 nvme_ctrlr_release(nvme_ns->ctrlr); 1858 nvme_ns_free(nvme_ns); 1859 } else { 1860 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1861 } 1862 } 1863 1864 pthread_mutex_lock(&g_bdev_nvme_mutex); 1865 TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq); 1866 pthread_mutex_unlock(&g_bdev_nvme_mutex); 1867 1868 spdk_io_device_unregister(nvme_disk, nvme_bdev_free); 1869 1870 return 0; 1871 } 1872 1873 static int 1874 bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair) 1875 { 1876 struct nvme_ctrlr *nvme_ctrlr; 1877 struct spdk_nvme_io_qpair_opts opts; 1878 struct spdk_nvme_qpair *qpair; 1879 int rc; 1880 1881 nvme_ctrlr = nvme_qpair->ctrlr; 1882 1883 spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1884 opts.delay_cmd_submit = g_opts.delay_cmd_submit; 1885 opts.create_only = true; 1886 opts.async_mode = true; 1887 opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests); 1888 g_opts.io_queue_requests = opts.io_queue_requests; 1889 1890 qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1891 if (qpair == NULL) { 1892 return -1; 1893 } 1894 1895 SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name, 1896 spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread)); 1897 1898 assert(nvme_qpair->group != NULL); 1899 1900 rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair); 1901 if (rc != 0) { 1902 SPDK_ERRLOG("Unable to begin polling on NVMe Channel.\n"); 1903 goto err; 1904 } 1905 1906 rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair); 1907 if (rc != 0) { 1908 SPDK_ERRLOG("Unable to connect I/O qpair.\n"); 1909 goto err; 1910 } 1911 1912 nvme_qpair->qpair = qpair; 1913 1914 if (!g_opts.disable_auto_failback) { 1915 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1916 } 1917 1918 return 0; 1919 1920 err: 1921 spdk_nvme_ctrlr_free_io_qpair(qpair); 1922 1923 return rc; 1924 } 1925 1926 static void bdev_nvme_reset_io_continue(void *cb_arg, int rc); 1927 1928 static void 1929 bdev_nvme_complete_pending_resets(struct nvme_ctrlr_channel_iter *i, 1930 struct nvme_ctrlr *nvme_ctrlr, 1931 struct nvme_ctrlr_channel *ctrlr_ch, 1932 void *ctx) 1933 { 1934 int rc = 0; 1935 struct nvme_bdev_io *bio; 1936 1937 if (ctx != NULL) { 1938 rc = -1; 1939 } 1940 1941 while (!TAILQ_EMPTY(&ctrlr_ch->pending_resets)) { 1942 bio = TAILQ_FIRST(&ctrlr_ch->pending_resets); 1943 TAILQ_REMOVE(&ctrlr_ch->pending_resets, bio, retry_link); 1944 1945 bdev_nvme_reset_io_continue(bio, rc); 1946 } 1947 1948 nvme_ctrlr_for_each_channel_continue(i, 0); 1949 } 1950 1951 /* This function marks the current trid as failed by storing the current ticks 1952 * and then sets the next trid to the active trid within a controller if exists. 1953 * 1954 * The purpose of the boolean return value is to request the caller to disconnect 1955 * the current trid now to try connecting the next trid. 1956 */ 1957 static bool 1958 bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove, bool start) 1959 { 1960 struct nvme_path_id *path_id, *next_path; 1961 int rc __attribute__((unused)); 1962 1963 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 1964 assert(path_id); 1965 assert(path_id == nvme_ctrlr->active_path_id); 1966 next_path = TAILQ_NEXT(path_id, link); 1967 1968 /* Update the last failed time. It means the trid is failed if its last 1969 * failed time is non-zero. 1970 */ 1971 path_id->last_failed_tsc = spdk_get_ticks(); 1972 1973 if (next_path == NULL) { 1974 /* There is no alternate trid within a controller. */ 1975 return false; 1976 } 1977 1978 if (!start && nvme_ctrlr->opts.reconnect_delay_sec == 0) { 1979 /* Connect is not retried in a controller reset sequence. Connecting 1980 * the next trid will be done by the next bdev_nvme_failover_ctrlr() call. 1981 */ 1982 return false; 1983 } 1984 1985 assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE); 1986 1987 SPDK_NOTICELOG("Start failover from %s:%s to %s:%s\n", path_id->trid.traddr, 1988 path_id->trid.trsvcid, next_path->trid.traddr, next_path->trid.trsvcid); 1989 1990 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 1991 nvme_ctrlr->active_path_id = next_path; 1992 rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid); 1993 assert(rc == 0); 1994 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 1995 if (!remove) { 1996 /** Shuffle the old trid to the end of the list and use the new one. 1997 * Allows for round robin through multiple connections. 1998 */ 1999 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link); 2000 } else { 2001 free(path_id); 2002 } 2003 2004 if (start || next_path->last_failed_tsc == 0) { 2005 /* bdev_nvme_failover_ctrlr() is just called or the next trid is not failed 2006 * or used yet. Try the next trid now. 2007 */ 2008 return true; 2009 } 2010 2011 if (spdk_get_ticks() > next_path->last_failed_tsc + spdk_get_ticks_hz() * 2012 nvme_ctrlr->opts.reconnect_delay_sec) { 2013 /* Enough backoff passed since the next trid failed. Try the next trid now. */ 2014 return true; 2015 } 2016 2017 /* The next trid will be tried after reconnect_delay_sec seconds. */ 2018 return false; 2019 } 2020 2021 static bool 2022 bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr) 2023 { 2024 int32_t elapsed; 2025 2026 if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 || 2027 nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) { 2028 return false; 2029 } 2030 2031 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 2032 if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) { 2033 return true; 2034 } else { 2035 return false; 2036 } 2037 } 2038 2039 static bool 2040 bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr) 2041 { 2042 uint32_t elapsed; 2043 2044 if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) { 2045 return false; 2046 } 2047 2048 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 2049 if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) { 2050 return true; 2051 } else { 2052 return false; 2053 } 2054 } 2055 2056 static void bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success); 2057 2058 static void 2059 nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb cb_fn) 2060 { 2061 int rc; 2062 2063 rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr); 2064 if (rc != 0) { 2065 /* Disconnect fails if ctrlr is already resetting or removed. In this case, 2066 * fail the reset sequence immediately. 2067 */ 2068 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2069 return; 2070 } 2071 2072 /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq. 2073 * Set callback here to execute the specified operation after ctrlr is really disconnected. 2074 */ 2075 assert(nvme_ctrlr->disconnected_cb == NULL); 2076 nvme_ctrlr->disconnected_cb = cb_fn; 2077 2078 /* During disconnection, reduce the period to poll adminq more often. */ 2079 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0); 2080 } 2081 2082 enum bdev_nvme_op_after_reset { 2083 OP_NONE, 2084 OP_COMPLETE_PENDING_DESTRUCT, 2085 OP_DESTRUCT, 2086 OP_DELAYED_RECONNECT, 2087 OP_FAILOVER, 2088 }; 2089 2090 typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset; 2091 2092 static _bdev_nvme_op_after_reset 2093 bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success) 2094 { 2095 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 2096 /* Complete pending destruct after reset completes. */ 2097 return OP_COMPLETE_PENDING_DESTRUCT; 2098 } else if (nvme_ctrlr->pending_failover) { 2099 nvme_ctrlr->pending_failover = false; 2100 nvme_ctrlr->reset_start_tsc = 0; 2101 return OP_FAILOVER; 2102 } else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) { 2103 nvme_ctrlr->reset_start_tsc = 0; 2104 return OP_NONE; 2105 } else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 2106 return OP_DESTRUCT; 2107 } else { 2108 if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) { 2109 nvme_ctrlr->fast_io_fail_timedout = true; 2110 } 2111 return OP_DELAYED_RECONNECT; 2112 } 2113 } 2114 2115 static int bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug); 2116 static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 2117 2118 static int 2119 bdev_nvme_reconnect_delay_timer_expired(void *ctx) 2120 { 2121 struct nvme_ctrlr *nvme_ctrlr = ctx; 2122 2123 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect_delay, nvme_ctrlr->nbdev_ctrlr->name); 2124 pthread_mutex_lock(&nvme_ctrlr->mutex); 2125 2126 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2127 2128 if (!nvme_ctrlr->reconnect_is_delayed) { 2129 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2130 return SPDK_POLLER_BUSY; 2131 } 2132 2133 nvme_ctrlr->reconnect_is_delayed = false; 2134 2135 if (nvme_ctrlr->destruct) { 2136 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2137 return SPDK_POLLER_BUSY; 2138 } 2139 2140 assert(nvme_ctrlr->resetting == false); 2141 nvme_ctrlr->resetting = true; 2142 2143 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2144 2145 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 2146 2147 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2148 return SPDK_POLLER_BUSY; 2149 } 2150 2151 static void 2152 bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr) 2153 { 2154 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 2155 2156 assert(nvme_ctrlr->reconnect_is_delayed == false); 2157 nvme_ctrlr->reconnect_is_delayed = true; 2158 2159 assert(nvme_ctrlr->reconnect_delay_timer == NULL); 2160 nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired, 2161 nvme_ctrlr, 2162 nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC); 2163 } 2164 2165 static void remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr); 2166 2167 static void 2168 _bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2169 { 2170 bool success = (ctx == NULL); 2171 bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn; 2172 void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg; 2173 enum bdev_nvme_op_after_reset op_after_reset; 2174 2175 assert(nvme_ctrlr->thread == spdk_get_thread()); 2176 2177 nvme_ctrlr->ctrlr_op_cb_fn = NULL; 2178 nvme_ctrlr->ctrlr_op_cb_arg = NULL; 2179 2180 if (!success) { 2181 SPDK_ERRLOG("Resetting controller failed.\n"); 2182 } else { 2183 SPDK_NOTICELOG("Resetting controller successful.\n"); 2184 } 2185 2186 pthread_mutex_lock(&nvme_ctrlr->mutex); 2187 nvme_ctrlr->resetting = false; 2188 nvme_ctrlr->dont_retry = false; 2189 nvme_ctrlr->in_failover = false; 2190 2191 op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success); 2192 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2193 2194 /* Delay callbacks when the next operation is a failover. */ 2195 if (ctrlr_op_cb_fn && op_after_reset != OP_FAILOVER) { 2196 ctrlr_op_cb_fn(ctrlr_op_cb_arg, success ? 0 : -1); 2197 } 2198 2199 switch (op_after_reset) { 2200 case OP_COMPLETE_PENDING_DESTRUCT: 2201 nvme_ctrlr_unregister(nvme_ctrlr); 2202 break; 2203 case OP_DESTRUCT: 2204 bdev_nvme_delete_ctrlr(nvme_ctrlr, false); 2205 remove_discovery_entry(nvme_ctrlr); 2206 break; 2207 case OP_DELAYED_RECONNECT: 2208 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_start_reconnect_delay_timer); 2209 break; 2210 case OP_FAILOVER: 2211 nvme_ctrlr->ctrlr_op_cb_fn = ctrlr_op_cb_fn; 2212 nvme_ctrlr->ctrlr_op_cb_arg = ctrlr_op_cb_arg; 2213 bdev_nvme_failover_ctrlr(nvme_ctrlr); 2214 break; 2215 default: 2216 break; 2217 } 2218 } 2219 2220 static void 2221 bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success) 2222 { 2223 pthread_mutex_lock(&nvme_ctrlr->mutex); 2224 if (!success) { 2225 /* Connecting the active trid failed. Set the next alternate trid to the 2226 * active trid if it exists. 2227 */ 2228 if (bdev_nvme_failover_trid(nvme_ctrlr, false, false)) { 2229 /* The next alternate trid exists and is ready to try. Try it now. */ 2230 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2231 2232 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); 2233 return; 2234 } 2235 2236 /* We came here if there is no alternate trid or if the next trid exists but 2237 * is not ready to try. We will try the active trid after reconnect_delay_sec 2238 * seconds if it is non-zero or at the next reset call otherwise. 2239 */ 2240 } else { 2241 /* Connecting the active trid succeeded. Clear the last failed time because it 2242 * means the trid is failed if its last failed time is non-zero. 2243 */ 2244 nvme_ctrlr->active_path_id->last_failed_tsc = 0; 2245 } 2246 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2247 2248 /* Make sure we clear any pending resets before returning. */ 2249 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2250 bdev_nvme_complete_pending_resets, 2251 success ? NULL : (void *)0x1, 2252 _bdev_nvme_reset_ctrlr_complete); 2253 } 2254 2255 static void 2256 bdev_nvme_reset_create_qpairs_failed(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2257 { 2258 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2259 } 2260 2261 static void 2262 bdev_nvme_reset_destroy_qpair(struct nvme_ctrlr_channel_iter *i, 2263 struct nvme_ctrlr *nvme_ctrlr, 2264 struct nvme_ctrlr_channel *ctrlr_ch, void *ctx) 2265 { 2266 struct nvme_qpair *nvme_qpair; 2267 2268 nvme_qpair = ctrlr_ch->qpair; 2269 assert(nvme_qpair != NULL); 2270 2271 _bdev_nvme_clear_io_path_cache(nvme_qpair); 2272 2273 if (nvme_qpair->qpair != NULL) { 2274 if (nvme_qpair->ctrlr->dont_retry) { 2275 spdk_nvme_qpair_set_abort_dnr(nvme_qpair->qpair, true); 2276 } 2277 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 2278 2279 /* The current full reset sequence will move to the next 2280 * ctrlr_channel after the qpair is actually disconnected. 2281 */ 2282 assert(ctrlr_ch->reset_iter == NULL); 2283 ctrlr_ch->reset_iter = i; 2284 } else { 2285 nvme_ctrlr_for_each_channel_continue(i, 0); 2286 } 2287 } 2288 2289 static void 2290 bdev_nvme_reset_create_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2291 { 2292 if (status == 0) { 2293 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, true); 2294 } else { 2295 /* Delete the added qpairs and quiesce ctrlr to make the states clean. */ 2296 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2297 bdev_nvme_reset_destroy_qpair, 2298 NULL, 2299 bdev_nvme_reset_create_qpairs_failed); 2300 } 2301 } 2302 2303 static int 2304 bdev_nvme_reset_check_qpair_connected(void *ctx) 2305 { 2306 struct nvme_ctrlr_channel *ctrlr_ch = ctx; 2307 2308 if (ctrlr_ch->reset_iter == NULL) { 2309 /* qpair was already failed to connect and the reset sequence is being aborted. */ 2310 assert(ctrlr_ch->connect_poller == NULL); 2311 assert(ctrlr_ch->qpair->qpair == NULL); 2312 return SPDK_POLLER_BUSY; 2313 } 2314 2315 assert(ctrlr_ch->qpair->qpair != NULL); 2316 2317 if (!spdk_nvme_qpair_is_connected(ctrlr_ch->qpair->qpair)) { 2318 return SPDK_POLLER_BUSY; 2319 } 2320 2321 spdk_poller_unregister(&ctrlr_ch->connect_poller); 2322 2323 /* qpair was completed to connect. Move to the next ctrlr_channel */ 2324 nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 2325 ctrlr_ch->reset_iter = NULL; 2326 2327 if (!g_opts.disable_auto_failback) { 2328 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 2329 } 2330 2331 return SPDK_POLLER_BUSY; 2332 } 2333 2334 static void 2335 bdev_nvme_reset_create_qpair(struct nvme_ctrlr_channel_iter *i, 2336 struct nvme_ctrlr *nvme_ctrlr, 2337 struct nvme_ctrlr_channel *ctrlr_ch, 2338 void *ctx) 2339 { 2340 int rc; 2341 2342 rc = bdev_nvme_create_qpair(ctrlr_ch->qpair); 2343 if (rc == 0) { 2344 ctrlr_ch->connect_poller = SPDK_POLLER_REGISTER(bdev_nvme_reset_check_qpair_connected, 2345 ctrlr_ch, 0); 2346 2347 /* The current full reset sequence will move to the next 2348 * ctrlr_channel after the qpair is actually connected. 2349 */ 2350 assert(ctrlr_ch->reset_iter == NULL); 2351 ctrlr_ch->reset_iter = i; 2352 } else { 2353 nvme_ctrlr_for_each_channel_continue(i, rc); 2354 } 2355 } 2356 2357 static void 2358 nvme_ctrlr_check_namespaces(struct nvme_ctrlr *nvme_ctrlr) 2359 { 2360 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 2361 struct nvme_ns *nvme_ns; 2362 2363 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 2364 nvme_ns != NULL; 2365 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 2366 if (!spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 2367 SPDK_DEBUGLOG(bdev_nvme, "NSID %u was removed during reset.\n", nvme_ns->id); 2368 /* NS can be added again. Just nullify nvme_ns->ns. */ 2369 nvme_ns->ns = NULL; 2370 } 2371 } 2372 } 2373 2374 2375 static int 2376 bdev_nvme_reconnect_ctrlr_poll(void *arg) 2377 { 2378 struct nvme_ctrlr *nvme_ctrlr = arg; 2379 int rc = -ETIMEDOUT; 2380 2381 if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 2382 /* Mark the ctrlr as failed. The next call to 2383 * spdk_nvme_ctrlr_reconnect_poll_async() will then 2384 * do the necessary cleanup and return failure. 2385 */ 2386 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 2387 } 2388 2389 rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr); 2390 if (rc == -EAGAIN) { 2391 return SPDK_POLLER_BUSY; 2392 } 2393 2394 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 2395 if (rc == 0) { 2396 nvme_ctrlr_check_namespaces(nvme_ctrlr); 2397 2398 /* Recreate all of the I/O queue pairs */ 2399 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2400 bdev_nvme_reset_create_qpair, 2401 NULL, 2402 bdev_nvme_reset_create_qpairs_done); 2403 } else { 2404 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2405 } 2406 return SPDK_POLLER_BUSY; 2407 } 2408 2409 static void 2410 bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2411 { 2412 spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr); 2413 2414 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect, nvme_ctrlr->nbdev_ctrlr->name); 2415 assert(nvme_ctrlr->reset_detach_poller == NULL); 2416 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll, 2417 nvme_ctrlr, 0); 2418 } 2419 2420 static void 2421 bdev_nvme_reset_destroy_qpair_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2422 { 2423 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reset, nvme_ctrlr->nbdev_ctrlr->name); 2424 assert(status == 0); 2425 2426 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2427 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2428 } else { 2429 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); 2430 } 2431 } 2432 2433 static void 2434 bdev_nvme_reset_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr) 2435 { 2436 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2437 bdev_nvme_reset_destroy_qpair, 2438 NULL, 2439 bdev_nvme_reset_destroy_qpair_done); 2440 } 2441 2442 static void 2443 bdev_nvme_reconnect_ctrlr_now(void *ctx) 2444 { 2445 struct nvme_ctrlr *nvme_ctrlr = ctx; 2446 2447 assert(nvme_ctrlr->resetting == true); 2448 assert(nvme_ctrlr->thread == spdk_get_thread()); 2449 2450 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2451 2452 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 2453 2454 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2455 } 2456 2457 static void 2458 _bdev_nvme_reset_ctrlr(void *ctx) 2459 { 2460 struct nvme_ctrlr *nvme_ctrlr = ctx; 2461 2462 assert(nvme_ctrlr->resetting == true); 2463 assert(nvme_ctrlr->thread == spdk_get_thread()); 2464 2465 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2466 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs); 2467 } else { 2468 bdev_nvme_reset_destroy_qpairs(nvme_ctrlr); 2469 } 2470 } 2471 2472 static int 2473 bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2474 { 2475 spdk_msg_fn msg_fn; 2476 2477 pthread_mutex_lock(&nvme_ctrlr->mutex); 2478 if (nvme_ctrlr->destruct) { 2479 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2480 return -ENXIO; 2481 } 2482 2483 if (nvme_ctrlr->resetting) { 2484 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2485 SPDK_NOTICELOG("Unable to perform reset, already in progress.\n"); 2486 return -EBUSY; 2487 } 2488 2489 if (nvme_ctrlr->disabled) { 2490 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2491 SPDK_NOTICELOG("Unable to perform reset. Controller is disabled.\n"); 2492 return -EALREADY; 2493 } 2494 2495 nvme_ctrlr->resetting = true; 2496 nvme_ctrlr->dont_retry = true; 2497 2498 if (nvme_ctrlr->reconnect_is_delayed) { 2499 SPDK_DEBUGLOG(bdev_nvme, "Reconnect is already scheduled.\n"); 2500 msg_fn = bdev_nvme_reconnect_ctrlr_now; 2501 nvme_ctrlr->reconnect_is_delayed = false; 2502 } else { 2503 msg_fn = _bdev_nvme_reset_ctrlr; 2504 assert(nvme_ctrlr->reset_start_tsc == 0); 2505 } 2506 2507 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2508 2509 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2510 2511 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 2512 return 0; 2513 } 2514 2515 static int 2516 bdev_nvme_enable_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2517 { 2518 pthread_mutex_lock(&nvme_ctrlr->mutex); 2519 if (nvme_ctrlr->destruct) { 2520 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2521 return -ENXIO; 2522 } 2523 2524 if (nvme_ctrlr->resetting) { 2525 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2526 return -EBUSY; 2527 } 2528 2529 if (!nvme_ctrlr->disabled) { 2530 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2531 return -EALREADY; 2532 } 2533 2534 nvme_ctrlr->disabled = false; 2535 nvme_ctrlr->resetting = true; 2536 2537 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2538 2539 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2540 2541 spdk_thread_send_msg(nvme_ctrlr->thread, bdev_nvme_reconnect_ctrlr_now, nvme_ctrlr); 2542 return 0; 2543 } 2544 2545 static void 2546 _bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2547 { 2548 bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn; 2549 void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg; 2550 enum bdev_nvme_op_after_reset op_after_disable; 2551 2552 assert(nvme_ctrlr->thread == spdk_get_thread()); 2553 2554 nvme_ctrlr->ctrlr_op_cb_fn = NULL; 2555 nvme_ctrlr->ctrlr_op_cb_arg = NULL; 2556 2557 pthread_mutex_lock(&nvme_ctrlr->mutex); 2558 2559 nvme_ctrlr->resetting = false; 2560 nvme_ctrlr->dont_retry = false; 2561 2562 op_after_disable = bdev_nvme_check_op_after_reset(nvme_ctrlr, true); 2563 2564 nvme_ctrlr->disabled = true; 2565 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 2566 2567 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2568 2569 if (ctrlr_op_cb_fn) { 2570 ctrlr_op_cb_fn(ctrlr_op_cb_arg, 0); 2571 } 2572 2573 switch (op_after_disable) { 2574 case OP_COMPLETE_PENDING_DESTRUCT: 2575 nvme_ctrlr_unregister(nvme_ctrlr); 2576 break; 2577 default: 2578 break; 2579 } 2580 2581 } 2582 2583 static void 2584 bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr) 2585 { 2586 /* Make sure we clear any pending resets before returning. */ 2587 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2588 bdev_nvme_complete_pending_resets, 2589 NULL, 2590 _bdev_nvme_disable_ctrlr_complete); 2591 } 2592 2593 static void 2594 bdev_nvme_disable_destroy_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2595 { 2596 assert(status == 0); 2597 2598 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2599 bdev_nvme_disable_ctrlr_complete(nvme_ctrlr); 2600 } else { 2601 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_ctrlr_complete); 2602 } 2603 } 2604 2605 static void 2606 bdev_nvme_disable_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr) 2607 { 2608 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2609 bdev_nvme_reset_destroy_qpair, 2610 NULL, 2611 bdev_nvme_disable_destroy_qpairs_done); 2612 } 2613 2614 static void 2615 _bdev_nvme_cancel_reconnect_and_disable_ctrlr(void *ctx) 2616 { 2617 struct nvme_ctrlr *nvme_ctrlr = ctx; 2618 2619 assert(nvme_ctrlr->resetting == true); 2620 assert(nvme_ctrlr->thread == spdk_get_thread()); 2621 2622 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2623 2624 bdev_nvme_disable_ctrlr_complete(nvme_ctrlr); 2625 } 2626 2627 static void 2628 _bdev_nvme_disconnect_and_disable_ctrlr(void *ctx) 2629 { 2630 struct nvme_ctrlr *nvme_ctrlr = ctx; 2631 2632 assert(nvme_ctrlr->resetting == true); 2633 assert(nvme_ctrlr->thread == spdk_get_thread()); 2634 2635 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2636 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_destroy_qpairs); 2637 } else { 2638 bdev_nvme_disable_destroy_qpairs(nvme_ctrlr); 2639 } 2640 } 2641 2642 static int 2643 bdev_nvme_disable_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2644 { 2645 spdk_msg_fn msg_fn; 2646 2647 pthread_mutex_lock(&nvme_ctrlr->mutex); 2648 if (nvme_ctrlr->destruct) { 2649 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2650 return -ENXIO; 2651 } 2652 2653 if (nvme_ctrlr->resetting) { 2654 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2655 return -EBUSY; 2656 } 2657 2658 if (nvme_ctrlr->disabled) { 2659 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2660 return -EALREADY; 2661 } 2662 2663 nvme_ctrlr->resetting = true; 2664 nvme_ctrlr->dont_retry = true; 2665 2666 if (nvme_ctrlr->reconnect_is_delayed) { 2667 msg_fn = _bdev_nvme_cancel_reconnect_and_disable_ctrlr; 2668 nvme_ctrlr->reconnect_is_delayed = false; 2669 } else { 2670 msg_fn = _bdev_nvme_disconnect_and_disable_ctrlr; 2671 } 2672 2673 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2674 2675 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2676 2677 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 2678 return 0; 2679 } 2680 2681 static int 2682 nvme_ctrlr_op(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op, 2683 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2684 { 2685 int rc; 2686 2687 switch (op) { 2688 case NVME_CTRLR_OP_RESET: 2689 rc = bdev_nvme_reset_ctrlr(nvme_ctrlr); 2690 break; 2691 case NVME_CTRLR_OP_ENABLE: 2692 rc = bdev_nvme_enable_ctrlr(nvme_ctrlr); 2693 break; 2694 case NVME_CTRLR_OP_DISABLE: 2695 rc = bdev_nvme_disable_ctrlr(nvme_ctrlr); 2696 break; 2697 default: 2698 rc = -EINVAL; 2699 break; 2700 } 2701 2702 if (rc == 0) { 2703 assert(nvme_ctrlr->ctrlr_op_cb_fn == NULL); 2704 assert(nvme_ctrlr->ctrlr_op_cb_arg == NULL); 2705 nvme_ctrlr->ctrlr_op_cb_fn = cb_fn; 2706 nvme_ctrlr->ctrlr_op_cb_arg = cb_arg; 2707 } 2708 return rc; 2709 } 2710 2711 struct nvme_ctrlr_op_rpc_ctx { 2712 struct nvme_ctrlr *nvme_ctrlr; 2713 struct spdk_thread *orig_thread; 2714 enum nvme_ctrlr_op op; 2715 int rc; 2716 bdev_nvme_ctrlr_op_cb cb_fn; 2717 void *cb_arg; 2718 }; 2719 2720 static void 2721 _nvme_ctrlr_op_rpc_complete(void *_ctx) 2722 { 2723 struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx; 2724 2725 assert(ctx != NULL); 2726 assert(ctx->cb_fn != NULL); 2727 2728 ctx->cb_fn(ctx->cb_arg, ctx->rc); 2729 2730 free(ctx); 2731 } 2732 2733 static void 2734 nvme_ctrlr_op_rpc_complete(void *cb_arg, int rc) 2735 { 2736 struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg; 2737 2738 ctx->rc = rc; 2739 2740 spdk_thread_send_msg(ctx->orig_thread, _nvme_ctrlr_op_rpc_complete, ctx); 2741 } 2742 2743 void 2744 nvme_ctrlr_op_rpc(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op, 2745 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2746 { 2747 struct nvme_ctrlr_op_rpc_ctx *ctx; 2748 int rc; 2749 2750 assert(cb_fn != NULL); 2751 2752 ctx = calloc(1, sizeof(*ctx)); 2753 if (ctx == NULL) { 2754 SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n"); 2755 cb_fn(cb_arg, -ENOMEM); 2756 return; 2757 } 2758 2759 ctx->orig_thread = spdk_get_thread(); 2760 ctx->cb_fn = cb_fn; 2761 ctx->cb_arg = cb_arg; 2762 2763 rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_ctrlr_op_rpc_complete, ctx); 2764 if (rc == 0) { 2765 return; 2766 } else if (rc == -EALREADY) { 2767 rc = 0; 2768 } 2769 2770 nvme_ctrlr_op_rpc_complete(ctx, rc); 2771 } 2772 2773 static void nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc); 2774 2775 static void 2776 _nvme_bdev_ctrlr_op_rpc_continue(void *_ctx) 2777 { 2778 struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx; 2779 struct nvme_ctrlr *prev_nvme_ctrlr, *next_nvme_ctrlr; 2780 int rc; 2781 2782 prev_nvme_ctrlr = ctx->nvme_ctrlr; 2783 ctx->nvme_ctrlr = NULL; 2784 2785 if (ctx->rc != 0) { 2786 goto complete; 2787 } 2788 2789 next_nvme_ctrlr = TAILQ_NEXT(prev_nvme_ctrlr, tailq); 2790 if (next_nvme_ctrlr == NULL) { 2791 goto complete; 2792 } 2793 2794 rc = nvme_ctrlr_op(next_nvme_ctrlr, ctx->op, nvme_bdev_ctrlr_op_rpc_continue, ctx); 2795 if (rc == 0) { 2796 ctx->nvme_ctrlr = next_nvme_ctrlr; 2797 return; 2798 } else if (rc == -EALREADY) { 2799 ctx->nvme_ctrlr = next_nvme_ctrlr; 2800 rc = 0; 2801 } 2802 2803 ctx->rc = rc; 2804 2805 complete: 2806 ctx->cb_fn(ctx->cb_arg, ctx->rc); 2807 free(ctx); 2808 } 2809 2810 static void 2811 nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc) 2812 { 2813 struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg; 2814 2815 ctx->rc = rc; 2816 2817 spdk_thread_send_msg(ctx->orig_thread, _nvme_bdev_ctrlr_op_rpc_continue, ctx); 2818 } 2819 2820 void 2821 nvme_bdev_ctrlr_op_rpc(struct nvme_bdev_ctrlr *nbdev_ctrlr, enum nvme_ctrlr_op op, 2822 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2823 { 2824 struct nvme_ctrlr_op_rpc_ctx *ctx; 2825 struct nvme_ctrlr *nvme_ctrlr; 2826 int rc; 2827 2828 assert(cb_fn != NULL); 2829 2830 ctx = calloc(1, sizeof(*ctx)); 2831 if (ctx == NULL) { 2832 SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n"); 2833 cb_fn(cb_arg, -ENOMEM); 2834 return; 2835 } 2836 2837 ctx->orig_thread = spdk_get_thread(); 2838 ctx->op = op; 2839 ctx->cb_fn = cb_fn; 2840 ctx->cb_arg = cb_arg; 2841 2842 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 2843 assert(nvme_ctrlr != NULL); 2844 2845 rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_bdev_ctrlr_op_rpc_continue, ctx); 2846 if (rc == 0) { 2847 ctx->nvme_ctrlr = nvme_ctrlr; 2848 return; 2849 } else if (rc == -EALREADY) { 2850 ctx->nvme_ctrlr = nvme_ctrlr; 2851 rc = 0; 2852 } 2853 2854 nvme_bdev_ctrlr_op_rpc_continue(ctx, rc); 2855 } 2856 2857 static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio); 2858 2859 static void 2860 bdev_nvme_unfreeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status) 2861 { 2862 struct nvme_bdev_io *bio = ctx; 2863 enum spdk_bdev_io_status io_status; 2864 2865 if (bio->cpl.cdw0 == 0) { 2866 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 2867 } else { 2868 io_status = SPDK_BDEV_IO_STATUS_FAILED; 2869 } 2870 2871 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL); 2872 } 2873 2874 static void 2875 bdev_nvme_unfreeze_bdev_channel(struct nvme_bdev_channel_iter *i, 2876 struct nvme_bdev *nbdev, 2877 struct nvme_bdev_channel *nbdev_ch, void *ctx) 2878 { 2879 bdev_nvme_abort_retry_ios(nbdev_ch); 2880 nbdev_ch->resetting = false; 2881 2882 nvme_bdev_for_each_channel_continue(i, 0); 2883 } 2884 2885 static void 2886 bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio) 2887 { 2888 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2889 struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt; 2890 2891 /* Abort all queued I/Os for retry. */ 2892 nvme_bdev_for_each_channel(nbdev, 2893 bdev_nvme_unfreeze_bdev_channel, 2894 bio, 2895 bdev_nvme_unfreeze_bdev_channel_done); 2896 } 2897 2898 static void 2899 _bdev_nvme_reset_io_continue(void *ctx) 2900 { 2901 struct nvme_bdev_io *bio = ctx; 2902 struct nvme_io_path *prev_io_path, *next_io_path; 2903 int rc; 2904 2905 prev_io_path = bio->io_path; 2906 bio->io_path = NULL; 2907 2908 if (bio->cpl.cdw0 != 0) { 2909 goto complete; 2910 } 2911 2912 next_io_path = STAILQ_NEXT(prev_io_path, stailq); 2913 if (next_io_path == NULL) { 2914 goto complete; 2915 } 2916 2917 rc = _bdev_nvme_reset_io(next_io_path, bio); 2918 if (rc == 0) { 2919 return; 2920 } 2921 2922 bio->cpl.cdw0 = 1; 2923 2924 complete: 2925 bdev_nvme_reset_io_complete(bio); 2926 } 2927 2928 static void 2929 bdev_nvme_reset_io_continue(void *cb_arg, int rc) 2930 { 2931 struct nvme_bdev_io *bio = cb_arg; 2932 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2933 2934 bio->cpl.cdw0 = (rc == 0) ? 0 : 1; 2935 2936 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), _bdev_nvme_reset_io_continue, bio); 2937 } 2938 2939 static int 2940 _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio) 2941 { 2942 struct nvme_ctrlr_channel *ctrlr_ch; 2943 int rc; 2944 2945 rc = nvme_ctrlr_op(io_path->qpair->ctrlr, NVME_CTRLR_OP_RESET, 2946 bdev_nvme_reset_io_continue, bio); 2947 if (rc != 0 && rc != -EBUSY) { 2948 return rc; 2949 } 2950 2951 assert(bio->io_path == NULL); 2952 bio->io_path = io_path; 2953 2954 if (rc == -EBUSY) { 2955 ctrlr_ch = io_path->qpair->ctrlr_ch; 2956 assert(ctrlr_ch != NULL); 2957 /* 2958 * Reset call is queued only if it is from the app framework. This is on purpose so that 2959 * we don't interfere with the app framework reset strategy. i.e. we are deferring to the 2960 * upper level. If they are in the middle of a reset, we won't try to schedule another one. 2961 */ 2962 TAILQ_INSERT_TAIL(&ctrlr_ch->pending_resets, bio, retry_link); 2963 } 2964 2965 return 0; 2966 } 2967 2968 static void 2969 bdev_nvme_freeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status) 2970 { 2971 struct nvme_bdev_io *bio = ctx; 2972 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2973 struct nvme_bdev_channel *nbdev_ch; 2974 struct nvme_io_path *io_path; 2975 int rc; 2976 2977 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 2978 2979 bio->cpl.cdw0 = 0; 2980 2981 /* Reset all nvme_ctrlrs of a bdev controller sequentially. */ 2982 io_path = STAILQ_FIRST(&nbdev_ch->io_path_list); 2983 assert(io_path != NULL); 2984 2985 rc = _bdev_nvme_reset_io(io_path, bio); 2986 if (rc != 0) { 2987 /* If the current nvme_ctrlr is disabled, skip it and move to the next nvme_ctrlr. */ 2988 rc = (rc == -EALREADY) ? 0 : rc; 2989 2990 bdev_nvme_reset_io_continue(bio, rc); 2991 } 2992 } 2993 2994 static void 2995 bdev_nvme_freeze_bdev_channel(struct nvme_bdev_channel_iter *i, 2996 struct nvme_bdev *nbdev, 2997 struct nvme_bdev_channel *nbdev_ch, void *ctx) 2998 { 2999 nbdev_ch->resetting = true; 3000 3001 nvme_bdev_for_each_channel_continue(i, 0); 3002 } 3003 3004 static void 3005 bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio) 3006 { 3007 nvme_bdev_for_each_channel(nbdev, 3008 bdev_nvme_freeze_bdev_channel, 3009 bio, 3010 bdev_nvme_freeze_bdev_channel_done); 3011 } 3012 3013 static int 3014 bdev_nvme_failover_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool remove) 3015 { 3016 if (nvme_ctrlr->destruct) { 3017 /* Don't bother resetting if the controller is in the process of being destructed. */ 3018 return -ENXIO; 3019 } 3020 3021 if (nvme_ctrlr->resetting) { 3022 if (!nvme_ctrlr->in_failover) { 3023 SPDK_NOTICELOG("Reset is already in progress. Defer failover until reset completes.\n"); 3024 3025 /* Defer failover until reset completes. */ 3026 nvme_ctrlr->pending_failover = true; 3027 return -EINPROGRESS; 3028 } else { 3029 SPDK_NOTICELOG("Unable to perform failover, already in progress.\n"); 3030 return -EBUSY; 3031 } 3032 } 3033 3034 bdev_nvme_failover_trid(nvme_ctrlr, remove, true); 3035 3036 if (nvme_ctrlr->reconnect_is_delayed) { 3037 SPDK_NOTICELOG("Reconnect is already scheduled.\n"); 3038 3039 /* We rely on the next reconnect for the failover. */ 3040 return -EALREADY; 3041 } 3042 3043 if (nvme_ctrlr->disabled) { 3044 SPDK_NOTICELOG("Controller is disabled.\n"); 3045 3046 /* We rely on the enablement for the failover. */ 3047 return -EALREADY; 3048 } 3049 3050 nvme_ctrlr->resetting = true; 3051 nvme_ctrlr->in_failover = true; 3052 3053 assert(nvme_ctrlr->reset_start_tsc == 0); 3054 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 3055 3056 return 0; 3057 } 3058 3059 static int 3060 bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 3061 { 3062 int rc; 3063 3064 pthread_mutex_lock(&nvme_ctrlr->mutex); 3065 rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, false); 3066 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3067 3068 if (rc == 0) { 3069 spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset_ctrlr, nvme_ctrlr); 3070 } else if (rc == -EALREADY) { 3071 rc = 0; 3072 } 3073 3074 return rc; 3075 } 3076 3077 static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, 3078 uint64_t num_blocks); 3079 3080 static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, 3081 uint64_t num_blocks); 3082 3083 static int bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, 3084 uint64_t src_offset_blocks, 3085 uint64_t num_blocks); 3086 3087 static void 3088 bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 3089 bool success) 3090 { 3091 struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 3092 int ret; 3093 3094 if (!success) { 3095 ret = -EINVAL; 3096 goto exit; 3097 } 3098 3099 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 3100 ret = -ENXIO; 3101 goto exit; 3102 } 3103 3104 ret = bdev_nvme_readv(bio, 3105 bdev_io->u.bdev.iovs, 3106 bdev_io->u.bdev.iovcnt, 3107 bdev_io->u.bdev.md_buf, 3108 bdev_io->u.bdev.num_blocks, 3109 bdev_io->u.bdev.offset_blocks, 3110 bdev_io->u.bdev.dif_check_flags, 3111 bdev_io->u.bdev.memory_domain, 3112 bdev_io->u.bdev.memory_domain_ctx, 3113 bdev_io->u.bdev.accel_sequence); 3114 3115 exit: 3116 if (spdk_unlikely(ret != 0)) { 3117 bdev_nvme_io_complete(bio, ret); 3118 } 3119 } 3120 3121 static inline void 3122 _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 3123 { 3124 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 3125 struct spdk_bdev *bdev = bdev_io->bdev; 3126 struct nvme_bdev_io *nbdev_io_to_abort; 3127 int rc = 0; 3128 3129 switch (bdev_io->type) { 3130 case SPDK_BDEV_IO_TYPE_READ: 3131 if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) { 3132 3133 rc = bdev_nvme_readv(nbdev_io, 3134 bdev_io->u.bdev.iovs, 3135 bdev_io->u.bdev.iovcnt, 3136 bdev_io->u.bdev.md_buf, 3137 bdev_io->u.bdev.num_blocks, 3138 bdev_io->u.bdev.offset_blocks, 3139 bdev_io->u.bdev.dif_check_flags, 3140 bdev_io->u.bdev.memory_domain, 3141 bdev_io->u.bdev.memory_domain_ctx, 3142 bdev_io->u.bdev.accel_sequence); 3143 } else { 3144 spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb, 3145 bdev_io->u.bdev.num_blocks * bdev->blocklen); 3146 rc = 0; 3147 } 3148 break; 3149 case SPDK_BDEV_IO_TYPE_WRITE: 3150 rc = bdev_nvme_writev(nbdev_io, 3151 bdev_io->u.bdev.iovs, 3152 bdev_io->u.bdev.iovcnt, 3153 bdev_io->u.bdev.md_buf, 3154 bdev_io->u.bdev.num_blocks, 3155 bdev_io->u.bdev.offset_blocks, 3156 bdev_io->u.bdev.dif_check_flags, 3157 bdev_io->u.bdev.memory_domain, 3158 bdev_io->u.bdev.memory_domain_ctx, 3159 bdev_io->u.bdev.accel_sequence, 3160 bdev_io->u.bdev.nvme_cdw12, 3161 bdev_io->u.bdev.nvme_cdw13); 3162 break; 3163 case SPDK_BDEV_IO_TYPE_COMPARE: 3164 rc = bdev_nvme_comparev(nbdev_io, 3165 bdev_io->u.bdev.iovs, 3166 bdev_io->u.bdev.iovcnt, 3167 bdev_io->u.bdev.md_buf, 3168 bdev_io->u.bdev.num_blocks, 3169 bdev_io->u.bdev.offset_blocks, 3170 bdev_io->u.bdev.dif_check_flags); 3171 break; 3172 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 3173 rc = bdev_nvme_comparev_and_writev(nbdev_io, 3174 bdev_io->u.bdev.iovs, 3175 bdev_io->u.bdev.iovcnt, 3176 bdev_io->u.bdev.fused_iovs, 3177 bdev_io->u.bdev.fused_iovcnt, 3178 bdev_io->u.bdev.md_buf, 3179 bdev_io->u.bdev.num_blocks, 3180 bdev_io->u.bdev.offset_blocks, 3181 bdev_io->u.bdev.dif_check_flags); 3182 break; 3183 case SPDK_BDEV_IO_TYPE_UNMAP: 3184 rc = bdev_nvme_unmap(nbdev_io, 3185 bdev_io->u.bdev.offset_blocks, 3186 bdev_io->u.bdev.num_blocks); 3187 break; 3188 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 3189 rc = bdev_nvme_write_zeroes(nbdev_io, 3190 bdev_io->u.bdev.offset_blocks, 3191 bdev_io->u.bdev.num_blocks); 3192 break; 3193 case SPDK_BDEV_IO_TYPE_RESET: 3194 nbdev_io->io_path = NULL; 3195 bdev_nvme_reset_io(bdev->ctxt, nbdev_io); 3196 return; 3197 3198 case SPDK_BDEV_IO_TYPE_FLUSH: 3199 bdev_nvme_io_complete(nbdev_io, 0); 3200 return; 3201 3202 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 3203 rc = bdev_nvme_zone_appendv(nbdev_io, 3204 bdev_io->u.bdev.iovs, 3205 bdev_io->u.bdev.iovcnt, 3206 bdev_io->u.bdev.md_buf, 3207 bdev_io->u.bdev.num_blocks, 3208 bdev_io->u.bdev.offset_blocks, 3209 bdev_io->u.bdev.dif_check_flags); 3210 break; 3211 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 3212 rc = bdev_nvme_get_zone_info(nbdev_io, 3213 bdev_io->u.zone_mgmt.zone_id, 3214 bdev_io->u.zone_mgmt.num_zones, 3215 bdev_io->u.zone_mgmt.buf); 3216 break; 3217 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 3218 rc = bdev_nvme_zone_management(nbdev_io, 3219 bdev_io->u.zone_mgmt.zone_id, 3220 bdev_io->u.zone_mgmt.zone_action); 3221 break; 3222 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3223 nbdev_io->io_path = NULL; 3224 bdev_nvme_admin_passthru(nbdev_ch, 3225 nbdev_io, 3226 &bdev_io->u.nvme_passthru.cmd, 3227 bdev_io->u.nvme_passthru.buf, 3228 bdev_io->u.nvme_passthru.nbytes); 3229 return; 3230 3231 case SPDK_BDEV_IO_TYPE_NVME_IO: 3232 rc = bdev_nvme_io_passthru(nbdev_io, 3233 &bdev_io->u.nvme_passthru.cmd, 3234 bdev_io->u.nvme_passthru.buf, 3235 bdev_io->u.nvme_passthru.nbytes); 3236 break; 3237 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3238 rc = bdev_nvme_io_passthru_md(nbdev_io, 3239 &bdev_io->u.nvme_passthru.cmd, 3240 bdev_io->u.nvme_passthru.buf, 3241 bdev_io->u.nvme_passthru.nbytes, 3242 bdev_io->u.nvme_passthru.md_buf, 3243 bdev_io->u.nvme_passthru.md_len); 3244 break; 3245 case SPDK_BDEV_IO_TYPE_NVME_IOV_MD: 3246 rc = bdev_nvme_iov_passthru_md(nbdev_io, 3247 &bdev_io->u.nvme_passthru.cmd, 3248 bdev_io->u.nvme_passthru.iovs, 3249 bdev_io->u.nvme_passthru.iovcnt, 3250 bdev_io->u.nvme_passthru.nbytes, 3251 bdev_io->u.nvme_passthru.md_buf, 3252 bdev_io->u.nvme_passthru.md_len); 3253 break; 3254 case SPDK_BDEV_IO_TYPE_ABORT: 3255 nbdev_io->io_path = NULL; 3256 nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx; 3257 bdev_nvme_abort(nbdev_ch, 3258 nbdev_io, 3259 nbdev_io_to_abort); 3260 return; 3261 3262 case SPDK_BDEV_IO_TYPE_COPY: 3263 rc = bdev_nvme_copy(nbdev_io, 3264 bdev_io->u.bdev.offset_blocks, 3265 bdev_io->u.bdev.copy.src_offset_blocks, 3266 bdev_io->u.bdev.num_blocks); 3267 break; 3268 default: 3269 rc = -EINVAL; 3270 break; 3271 } 3272 3273 if (spdk_unlikely(rc != 0)) { 3274 bdev_nvme_io_complete(nbdev_io, rc); 3275 } 3276 } 3277 3278 static void 3279 bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 3280 { 3281 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 3282 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 3283 3284 if (spdk_likely(nbdev_io->submit_tsc == 0)) { 3285 nbdev_io->submit_tsc = spdk_bdev_io_get_submit_tsc(bdev_io); 3286 } else { 3287 /* There are cases where submit_tsc != 0, i.e. retry I/O. 3288 * We need to update submit_tsc here. 3289 */ 3290 nbdev_io->submit_tsc = spdk_get_ticks(); 3291 } 3292 3293 spdk_trace_record(TRACE_BDEV_NVME_IO_START, 0, 0, (uintptr_t)nbdev_io, (uintptr_t)bdev_io); 3294 nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch); 3295 if (spdk_unlikely(!nbdev_io->io_path)) { 3296 if (!bdev_nvme_io_type_is_admin(bdev_io->type)) { 3297 bdev_nvme_io_complete(nbdev_io, -ENXIO); 3298 return; 3299 } 3300 3301 /* Admin commands do not use the optimal I/O path. 3302 * Simply fall through even if it is not found. 3303 */ 3304 } 3305 3306 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 3307 } 3308 3309 static bool 3310 bdev_nvme_is_supported_csi(enum spdk_nvme_csi csi) 3311 { 3312 switch (csi) { 3313 case SPDK_NVME_CSI_NVM: 3314 return true; 3315 case SPDK_NVME_CSI_ZNS: 3316 return true; 3317 default: 3318 return false; 3319 } 3320 } 3321 3322 static bool 3323 bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 3324 { 3325 struct nvme_bdev *nbdev = ctx; 3326 struct nvme_ns *nvme_ns; 3327 struct spdk_nvme_ns *ns; 3328 struct spdk_nvme_ctrlr *ctrlr; 3329 const struct spdk_nvme_ctrlr_data *cdata; 3330 3331 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 3332 assert(nvme_ns != NULL); 3333 ns = nvme_ns->ns; 3334 if (ns == NULL) { 3335 return false; 3336 } 3337 3338 if (!bdev_nvme_is_supported_csi(spdk_nvme_ns_get_csi(ns))) { 3339 switch (io_type) { 3340 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3341 case SPDK_BDEV_IO_TYPE_NVME_IO: 3342 return true; 3343 3344 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3345 return spdk_nvme_ns_get_md_size(ns) ? true : false; 3346 3347 default: 3348 return false; 3349 } 3350 } 3351 3352 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 3353 3354 switch (io_type) { 3355 case SPDK_BDEV_IO_TYPE_READ: 3356 case SPDK_BDEV_IO_TYPE_WRITE: 3357 case SPDK_BDEV_IO_TYPE_RESET: 3358 case SPDK_BDEV_IO_TYPE_FLUSH: 3359 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3360 case SPDK_BDEV_IO_TYPE_NVME_IO: 3361 case SPDK_BDEV_IO_TYPE_ABORT: 3362 return true; 3363 3364 case SPDK_BDEV_IO_TYPE_COMPARE: 3365 return spdk_nvme_ns_supports_compare(ns); 3366 3367 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3368 return spdk_nvme_ns_get_md_size(ns) ? true : false; 3369 3370 case SPDK_BDEV_IO_TYPE_UNMAP: 3371 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3372 return cdata->oncs.dsm; 3373 3374 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 3375 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3376 return cdata->oncs.write_zeroes; 3377 3378 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 3379 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 3380 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) { 3381 return true; 3382 } 3383 return false; 3384 3385 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 3386 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 3387 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS; 3388 3389 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 3390 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS && 3391 spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED; 3392 3393 case SPDK_BDEV_IO_TYPE_COPY: 3394 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3395 return cdata->oncs.copy; 3396 3397 default: 3398 return false; 3399 } 3400 } 3401 3402 static int 3403 nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch) 3404 { 3405 struct nvme_qpair *nvme_qpair; 3406 struct spdk_io_channel *pg_ch; 3407 int rc; 3408 3409 nvme_qpair = calloc(1, sizeof(*nvme_qpair)); 3410 if (!nvme_qpair) { 3411 SPDK_ERRLOG("Failed to alloc nvme_qpair.\n"); 3412 return -1; 3413 } 3414 3415 TAILQ_INIT(&nvme_qpair->io_path_list); 3416 3417 nvme_qpair->ctrlr = nvme_ctrlr; 3418 nvme_qpair->ctrlr_ch = ctrlr_ch; 3419 3420 pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs); 3421 if (!pg_ch) { 3422 free(nvme_qpair); 3423 return -1; 3424 } 3425 3426 nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch); 3427 3428 #ifdef SPDK_CONFIG_VTUNE 3429 nvme_qpair->group->collect_spin_stat = true; 3430 #else 3431 nvme_qpair->group->collect_spin_stat = false; 3432 #endif 3433 3434 if (!nvme_ctrlr->disabled) { 3435 /* If a nvme_ctrlr is disabled, don't try to create qpair for it. Qpair will 3436 * be created when it's enabled. 3437 */ 3438 rc = bdev_nvme_create_qpair(nvme_qpair); 3439 if (rc != 0) { 3440 /* nvme_ctrlr can't create IO qpair if connection is down. 3441 * If reconnect_delay_sec is non-zero, creating IO qpair is retried 3442 * after reconnect_delay_sec seconds. If bdev_retry_count is non-zero, 3443 * submitted IO will be queued until IO qpair is successfully created. 3444 * 3445 * Hence, if both are satisfied, ignore the failure. 3446 */ 3447 if (nvme_ctrlr->opts.reconnect_delay_sec == 0 || g_opts.bdev_retry_count == 0) { 3448 spdk_put_io_channel(pg_ch); 3449 free(nvme_qpair); 3450 return rc; 3451 } 3452 } 3453 } 3454 3455 TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 3456 3457 ctrlr_ch->qpair = nvme_qpair; 3458 3459 pthread_mutex_lock(&nvme_qpair->ctrlr->mutex); 3460 nvme_qpair->ctrlr->ref++; 3461 pthread_mutex_unlock(&nvme_qpair->ctrlr->mutex); 3462 3463 return 0; 3464 } 3465 3466 static int 3467 bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) 3468 { 3469 struct nvme_ctrlr *nvme_ctrlr = io_device; 3470 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 3471 3472 TAILQ_INIT(&ctrlr_ch->pending_resets); 3473 3474 return nvme_qpair_create(nvme_ctrlr, ctrlr_ch); 3475 } 3476 3477 static void 3478 nvme_qpair_delete(struct nvme_qpair *nvme_qpair) 3479 { 3480 struct nvme_io_path *io_path, *next; 3481 3482 assert(nvme_qpair->group != NULL); 3483 3484 TAILQ_FOREACH_SAFE(io_path, &nvme_qpair->io_path_list, tailq, next) { 3485 TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq); 3486 nvme_io_path_free(io_path); 3487 } 3488 3489 TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 3490 3491 spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group)); 3492 3493 nvme_ctrlr_release(nvme_qpair->ctrlr); 3494 3495 free(nvme_qpair); 3496 } 3497 3498 static void 3499 bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf) 3500 { 3501 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 3502 struct nvme_qpair *nvme_qpair; 3503 3504 nvme_qpair = ctrlr_ch->qpair; 3505 assert(nvme_qpair != NULL); 3506 3507 _bdev_nvme_clear_io_path_cache(nvme_qpair); 3508 3509 if (nvme_qpair->qpair != NULL) { 3510 if (ctrlr_ch->reset_iter == NULL) { 3511 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 3512 } else { 3513 /* Skip current ctrlr_channel in a full reset sequence because 3514 * it is being deleted now. The qpair is already being disconnected. 3515 * We do not have to restart disconnecting it. 3516 */ 3517 nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 3518 } 3519 3520 /* We cannot release a reference to the poll group now. 3521 * The qpair may be disconnected asynchronously later. 3522 * We need to poll it until it is actually disconnected. 3523 * Just detach the qpair from the deleting ctrlr_channel. 3524 */ 3525 nvme_qpair->ctrlr_ch = NULL; 3526 } else { 3527 assert(ctrlr_ch->reset_iter == NULL); 3528 3529 nvme_qpair_delete(nvme_qpair); 3530 } 3531 } 3532 3533 static inline struct spdk_io_channel * 3534 bdev_nvme_get_accel_channel(struct nvme_poll_group *group) 3535 { 3536 if (spdk_unlikely(!group->accel_channel)) { 3537 group->accel_channel = spdk_accel_get_io_channel(); 3538 if (!group->accel_channel) { 3539 SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n", 3540 group); 3541 return NULL; 3542 } 3543 } 3544 3545 return group->accel_channel; 3546 } 3547 3548 static void 3549 bdev_nvme_finish_sequence(void *seq, spdk_nvme_accel_completion_cb cb_fn, void *cb_arg) 3550 { 3551 spdk_accel_sequence_finish(seq, cb_fn, cb_arg); 3552 } 3553 3554 static void 3555 bdev_nvme_abort_sequence(void *seq) 3556 { 3557 spdk_accel_sequence_abort(seq); 3558 } 3559 3560 static void 3561 bdev_nvme_reverse_sequence(void *seq) 3562 { 3563 spdk_accel_sequence_reverse(seq); 3564 } 3565 3566 static int 3567 bdev_nvme_append_crc32c(void *ctx, void **seq, uint32_t *dst, struct iovec *iovs, uint32_t iovcnt, 3568 struct spdk_memory_domain *domain, void *domain_ctx, uint32_t seed, 3569 spdk_nvme_accel_step_cb cb_fn, void *cb_arg) 3570 { 3571 struct spdk_io_channel *ch; 3572 struct nvme_poll_group *group = ctx; 3573 3574 ch = bdev_nvme_get_accel_channel(group); 3575 if (spdk_unlikely(ch == NULL)) { 3576 return -ENOMEM; 3577 } 3578 3579 return spdk_accel_append_crc32c((struct spdk_accel_sequence **)seq, ch, dst, iovs, iovcnt, 3580 domain, domain_ctx, seed, cb_fn, cb_arg); 3581 } 3582 3583 static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = { 3584 .table_size = sizeof(struct spdk_nvme_accel_fn_table), 3585 .append_crc32c = bdev_nvme_append_crc32c, 3586 .finish_sequence = bdev_nvme_finish_sequence, 3587 .reverse_sequence = bdev_nvme_reverse_sequence, 3588 .abort_sequence = bdev_nvme_abort_sequence, 3589 }; 3590 3591 static int 3592 bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf) 3593 { 3594 struct nvme_poll_group *group = ctx_buf; 3595 3596 TAILQ_INIT(&group->qpair_list); 3597 3598 group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table); 3599 if (group->group == NULL) { 3600 return -1; 3601 } 3602 3603 group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us); 3604 3605 if (group->poller == NULL) { 3606 spdk_nvme_poll_group_destroy(group->group); 3607 return -1; 3608 } 3609 3610 return 0; 3611 } 3612 3613 static void 3614 bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf) 3615 { 3616 struct nvme_poll_group *group = ctx_buf; 3617 3618 assert(TAILQ_EMPTY(&group->qpair_list)); 3619 3620 if (group->accel_channel) { 3621 spdk_put_io_channel(group->accel_channel); 3622 } 3623 3624 spdk_poller_unregister(&group->poller); 3625 if (spdk_nvme_poll_group_destroy(group->group)) { 3626 SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n"); 3627 assert(false); 3628 } 3629 } 3630 3631 static struct spdk_io_channel * 3632 bdev_nvme_get_io_channel(void *ctx) 3633 { 3634 struct nvme_bdev *nvme_bdev = ctx; 3635 3636 return spdk_get_io_channel(nvme_bdev); 3637 } 3638 3639 static void * 3640 bdev_nvme_get_module_ctx(void *ctx) 3641 { 3642 struct nvme_bdev *nvme_bdev = ctx; 3643 struct nvme_ns *nvme_ns; 3644 3645 if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) { 3646 return NULL; 3647 } 3648 3649 nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list); 3650 if (!nvme_ns) { 3651 return NULL; 3652 } 3653 3654 return nvme_ns->ns; 3655 } 3656 3657 static const char * 3658 _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state) 3659 { 3660 switch (ana_state) { 3661 case SPDK_NVME_ANA_OPTIMIZED_STATE: 3662 return "optimized"; 3663 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 3664 return "non_optimized"; 3665 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3666 return "inaccessible"; 3667 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3668 return "persistent_loss"; 3669 case SPDK_NVME_ANA_CHANGE_STATE: 3670 return "change"; 3671 default: 3672 return NULL; 3673 } 3674 } 3675 3676 static int 3677 bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size) 3678 { 3679 struct spdk_memory_domain **_domains = NULL; 3680 struct nvme_bdev *nbdev = ctx; 3681 struct nvme_ns *nvme_ns; 3682 int i = 0, _array_size = array_size; 3683 int rc = 0; 3684 3685 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 3686 if (domains && array_size >= i) { 3687 _domains = &domains[i]; 3688 } else { 3689 _domains = NULL; 3690 } 3691 rc = spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, _domains, _array_size); 3692 if (rc > 0) { 3693 i += rc; 3694 if (_array_size >= rc) { 3695 _array_size -= rc; 3696 } else { 3697 _array_size = 0; 3698 } 3699 } else if (rc < 0) { 3700 return rc; 3701 } 3702 } 3703 3704 return i; 3705 } 3706 3707 static const char * 3708 nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr) 3709 { 3710 if (nvme_ctrlr->destruct) { 3711 return "deleting"; 3712 } else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 3713 return "failed"; 3714 } else if (nvme_ctrlr->resetting) { 3715 return "resetting"; 3716 } else if (nvme_ctrlr->reconnect_is_delayed > 0) { 3717 return "reconnect_is_delayed"; 3718 } else if (nvme_ctrlr->disabled) { 3719 return "disabled"; 3720 } else { 3721 return "enabled"; 3722 } 3723 } 3724 3725 void 3726 nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr) 3727 { 3728 struct spdk_nvme_transport_id *trid; 3729 const struct spdk_nvme_ctrlr_opts *opts; 3730 const struct spdk_nvme_ctrlr_data *cdata; 3731 struct nvme_path_id *path_id; 3732 int32_t numa_id; 3733 3734 spdk_json_write_object_begin(w); 3735 3736 spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr)); 3737 3738 #ifdef SPDK_CONFIG_NVME_CUSE 3739 size_t cuse_name_size = 128; 3740 char cuse_name[cuse_name_size]; 3741 3742 int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size); 3743 if (rc == 0) { 3744 spdk_json_write_named_string(w, "cuse_device", cuse_name); 3745 } 3746 #endif 3747 trid = &nvme_ctrlr->active_path_id->trid; 3748 spdk_json_write_named_object_begin(w, "trid"); 3749 nvme_bdev_dump_trid_json(trid, w); 3750 spdk_json_write_object_end(w); 3751 3752 path_id = TAILQ_NEXT(nvme_ctrlr->active_path_id, link); 3753 if (path_id != NULL) { 3754 spdk_json_write_named_array_begin(w, "alternate_trids"); 3755 do { 3756 trid = &path_id->trid; 3757 spdk_json_write_object_begin(w); 3758 nvme_bdev_dump_trid_json(trid, w); 3759 spdk_json_write_object_end(w); 3760 3761 path_id = TAILQ_NEXT(path_id, link); 3762 } while (path_id != NULL); 3763 spdk_json_write_array_end(w); 3764 } 3765 3766 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 3767 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 3768 3769 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 3770 spdk_json_write_named_object_begin(w, "host"); 3771 spdk_json_write_named_string(w, "nqn", opts->hostnqn); 3772 spdk_json_write_named_string(w, "addr", opts->src_addr); 3773 spdk_json_write_named_string(w, "svcid", opts->src_svcid); 3774 spdk_json_write_object_end(w); 3775 3776 numa_id = spdk_nvme_ctrlr_get_numa_id(nvme_ctrlr->ctrlr); 3777 if (numa_id != SPDK_ENV_NUMA_ID_ANY) { 3778 spdk_json_write_named_uint32(w, "numa_id", numa_id); 3779 } 3780 spdk_json_write_object_end(w); 3781 } 3782 3783 static void 3784 nvme_namespace_info_json(struct spdk_json_write_ctx *w, 3785 struct nvme_ns *nvme_ns) 3786 { 3787 struct spdk_nvme_ns *ns; 3788 struct spdk_nvme_ctrlr *ctrlr; 3789 const struct spdk_nvme_ctrlr_data *cdata; 3790 const struct spdk_nvme_transport_id *trid; 3791 union spdk_nvme_vs_register vs; 3792 const struct spdk_nvme_ns_data *nsdata; 3793 char buf[128]; 3794 3795 ns = nvme_ns->ns; 3796 if (ns == NULL) { 3797 return; 3798 } 3799 3800 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 3801 3802 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3803 trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); 3804 vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr); 3805 3806 spdk_json_write_object_begin(w); 3807 3808 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 3809 spdk_json_write_named_string(w, "pci_address", trid->traddr); 3810 } 3811 3812 spdk_json_write_named_object_begin(w, "trid"); 3813 3814 nvme_bdev_dump_trid_json(trid, w); 3815 3816 spdk_json_write_object_end(w); 3817 3818 #ifdef SPDK_CONFIG_NVME_CUSE 3819 size_t cuse_name_size = 128; 3820 char cuse_name[cuse_name_size]; 3821 3822 int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns), 3823 cuse_name, &cuse_name_size); 3824 if (rc == 0) { 3825 spdk_json_write_named_string(w, "cuse_device", cuse_name); 3826 } 3827 #endif 3828 3829 spdk_json_write_named_object_begin(w, "ctrlr_data"); 3830 3831 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 3832 3833 spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid); 3834 3835 snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn); 3836 spdk_str_trim(buf); 3837 spdk_json_write_named_string(w, "model_number", buf); 3838 3839 snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn); 3840 spdk_str_trim(buf); 3841 spdk_json_write_named_string(w, "serial_number", buf); 3842 3843 snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr); 3844 spdk_str_trim(buf); 3845 spdk_json_write_named_string(w, "firmware_revision", buf); 3846 3847 if (cdata->subnqn[0] != '\0') { 3848 spdk_json_write_named_string(w, "subnqn", cdata->subnqn); 3849 } 3850 3851 spdk_json_write_named_object_begin(w, "oacs"); 3852 3853 spdk_json_write_named_uint32(w, "security", cdata->oacs.security); 3854 spdk_json_write_named_uint32(w, "format", cdata->oacs.format); 3855 spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware); 3856 spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage); 3857 3858 spdk_json_write_object_end(w); 3859 3860 spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr); 3861 spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting); 3862 3863 spdk_json_write_object_end(w); 3864 3865 spdk_json_write_named_object_begin(w, "vs"); 3866 3867 spdk_json_write_name(w, "nvme_version"); 3868 if (vs.bits.ter) { 3869 spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter); 3870 } else { 3871 spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr); 3872 } 3873 3874 spdk_json_write_object_end(w); 3875 3876 nsdata = spdk_nvme_ns_get_data(ns); 3877 3878 spdk_json_write_named_object_begin(w, "ns_data"); 3879 3880 spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns)); 3881 3882 if (cdata->cmic.ana_reporting) { 3883 spdk_json_write_named_string(w, "ana_state", 3884 _nvme_ana_state_str(nvme_ns->ana_state)); 3885 } 3886 3887 spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share); 3888 3889 spdk_json_write_object_end(w); 3890 3891 if (cdata->oacs.security) { 3892 spdk_json_write_named_object_begin(w, "security"); 3893 3894 spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal); 3895 3896 spdk_json_write_object_end(w); 3897 } 3898 3899 spdk_json_write_object_end(w); 3900 } 3901 3902 static const char * 3903 nvme_bdev_get_mp_policy_str(struct nvme_bdev *nbdev) 3904 { 3905 switch (nbdev->mp_policy) { 3906 case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE: 3907 return "active_passive"; 3908 case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE: 3909 return "active_active"; 3910 default: 3911 assert(false); 3912 return "invalid"; 3913 } 3914 } 3915 3916 static const char * 3917 nvme_bdev_get_mp_selector_str(struct nvme_bdev *nbdev) 3918 { 3919 switch (nbdev->mp_selector) { 3920 case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN: 3921 return "round_robin"; 3922 case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH: 3923 return "queue_depth"; 3924 default: 3925 assert(false); 3926 return "invalid"; 3927 } 3928 } 3929 3930 static int 3931 bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 3932 { 3933 struct nvme_bdev *nvme_bdev = ctx; 3934 struct nvme_ns *nvme_ns; 3935 3936 pthread_mutex_lock(&nvme_bdev->mutex); 3937 spdk_json_write_named_array_begin(w, "nvme"); 3938 TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) { 3939 nvme_namespace_info_json(w, nvme_ns); 3940 } 3941 spdk_json_write_array_end(w); 3942 spdk_json_write_named_string(w, "mp_policy", nvme_bdev_get_mp_policy_str(nvme_bdev)); 3943 if (nvme_bdev->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 3944 spdk_json_write_named_string(w, "selector", nvme_bdev_get_mp_selector_str(nvme_bdev)); 3945 if (nvme_bdev->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 3946 spdk_json_write_named_uint32(w, "rr_min_io", nvme_bdev->rr_min_io); 3947 } 3948 } 3949 pthread_mutex_unlock(&nvme_bdev->mutex); 3950 3951 return 0; 3952 } 3953 3954 static void 3955 bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 3956 { 3957 /* No config per bdev needed */ 3958 } 3959 3960 static uint64_t 3961 bdev_nvme_get_spin_time(struct spdk_io_channel *ch) 3962 { 3963 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 3964 struct nvme_io_path *io_path; 3965 struct nvme_poll_group *group; 3966 uint64_t spin_time = 0; 3967 3968 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 3969 group = io_path->qpair->group; 3970 3971 if (!group || !group->collect_spin_stat) { 3972 continue; 3973 } 3974 3975 if (group->end_ticks != 0) { 3976 group->spin_ticks += (group->end_ticks - group->start_ticks); 3977 group->end_ticks = 0; 3978 } 3979 3980 spin_time += group->spin_ticks; 3981 group->start_ticks = 0; 3982 group->spin_ticks = 0; 3983 } 3984 3985 return (spin_time * 1000000ULL) / spdk_get_ticks_hz(); 3986 } 3987 3988 static void 3989 bdev_nvme_reset_device_stat(void *ctx) 3990 { 3991 struct nvme_bdev *nbdev = ctx; 3992 3993 if (nbdev->err_stat != NULL) { 3994 memset(nbdev->err_stat, 0, sizeof(struct nvme_error_stat)); 3995 } 3996 } 3997 3998 /* JSON string should be lowercases and underscore delimited string. */ 3999 static void 4000 bdev_nvme_format_nvme_status(char *dst, const char *src) 4001 { 4002 char tmp[256]; 4003 4004 spdk_strcpy_replace(dst, 256, src, " - ", "_"); 4005 spdk_strcpy_replace(tmp, 256, dst, "-", "_"); 4006 spdk_strcpy_replace(dst, 256, tmp, " ", "_"); 4007 spdk_strlwr(dst); 4008 } 4009 4010 static void 4011 bdev_nvme_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w) 4012 { 4013 struct nvme_bdev *nbdev = ctx; 4014 struct spdk_nvme_status status = {}; 4015 uint16_t sct, sc; 4016 char status_json[256]; 4017 const char *status_str; 4018 4019 if (nbdev->err_stat == NULL) { 4020 return; 4021 } 4022 4023 spdk_json_write_named_object_begin(w, "nvme_error"); 4024 4025 spdk_json_write_named_object_begin(w, "status_type"); 4026 for (sct = 0; sct < 8; sct++) { 4027 if (nbdev->err_stat->status_type[sct] == 0) { 4028 continue; 4029 } 4030 status.sct = sct; 4031 4032 status_str = spdk_nvme_cpl_get_status_type_string(&status); 4033 assert(status_str != NULL); 4034 bdev_nvme_format_nvme_status(status_json, status_str); 4035 4036 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status_type[sct]); 4037 } 4038 spdk_json_write_object_end(w); 4039 4040 spdk_json_write_named_object_begin(w, "status_code"); 4041 for (sct = 0; sct < 4; sct++) { 4042 status.sct = sct; 4043 for (sc = 0; sc < 256; sc++) { 4044 if (nbdev->err_stat->status[sct][sc] == 0) { 4045 continue; 4046 } 4047 status.sc = sc; 4048 4049 status_str = spdk_nvme_cpl_get_status_string(&status); 4050 assert(status_str != NULL); 4051 bdev_nvme_format_nvme_status(status_json, status_str); 4052 4053 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status[sct][sc]); 4054 } 4055 } 4056 spdk_json_write_object_end(w); 4057 4058 spdk_json_write_object_end(w); 4059 } 4060 4061 static bool 4062 bdev_nvme_accel_sequence_supported(void *ctx, enum spdk_bdev_io_type type) 4063 { 4064 struct nvme_bdev *nbdev = ctx; 4065 struct spdk_nvme_ctrlr *ctrlr; 4066 4067 if (!g_opts.allow_accel_sequence) { 4068 return false; 4069 } 4070 4071 switch (type) { 4072 case SPDK_BDEV_IO_TYPE_WRITE: 4073 case SPDK_BDEV_IO_TYPE_READ: 4074 break; 4075 default: 4076 return false; 4077 } 4078 4079 ctrlr = bdev_nvme_get_ctrlr(&nbdev->disk); 4080 assert(ctrlr != NULL); 4081 4082 return spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ACCEL_SEQUENCE_SUPPORTED; 4083 } 4084 4085 static const struct spdk_bdev_fn_table nvmelib_fn_table = { 4086 .destruct = bdev_nvme_destruct, 4087 .submit_request = bdev_nvme_submit_request, 4088 .io_type_supported = bdev_nvme_io_type_supported, 4089 .get_io_channel = bdev_nvme_get_io_channel, 4090 .dump_info_json = bdev_nvme_dump_info_json, 4091 .write_config_json = bdev_nvme_write_config_json, 4092 .get_spin_time = bdev_nvme_get_spin_time, 4093 .get_module_ctx = bdev_nvme_get_module_ctx, 4094 .get_memory_domains = bdev_nvme_get_memory_domains, 4095 .accel_sequence_supported = bdev_nvme_accel_sequence_supported, 4096 .reset_device_stat = bdev_nvme_reset_device_stat, 4097 .dump_device_stat_json = bdev_nvme_dump_device_stat_json, 4098 }; 4099 4100 typedef int (*bdev_nvme_parse_ana_log_page_cb)( 4101 const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg); 4102 4103 static int 4104 bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 4105 bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg) 4106 { 4107 struct spdk_nvme_ana_group_descriptor *copied_desc; 4108 uint8_t *orig_desc; 4109 uint32_t i, desc_size, copy_len; 4110 int rc = 0; 4111 4112 if (nvme_ctrlr->ana_log_page == NULL) { 4113 return -EINVAL; 4114 } 4115 4116 copied_desc = nvme_ctrlr->copied_ana_desc; 4117 4118 orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page); 4119 copy_len = nvme_ctrlr->max_ana_log_page_size - sizeof(struct spdk_nvme_ana_page); 4120 4121 for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) { 4122 memcpy(copied_desc, orig_desc, copy_len); 4123 4124 rc = cb_fn(copied_desc, cb_arg); 4125 if (rc != 0) { 4126 break; 4127 } 4128 4129 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) + 4130 copied_desc->num_of_nsid * sizeof(uint32_t); 4131 orig_desc += desc_size; 4132 copy_len -= desc_size; 4133 } 4134 4135 return rc; 4136 } 4137 4138 static int 4139 nvme_ns_ana_transition_timedout(void *ctx) 4140 { 4141 struct nvme_ns *nvme_ns = ctx; 4142 4143 spdk_poller_unregister(&nvme_ns->anatt_timer); 4144 nvme_ns->ana_transition_timedout = true; 4145 4146 return SPDK_POLLER_BUSY; 4147 } 4148 4149 static void 4150 _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns, 4151 const struct spdk_nvme_ana_group_descriptor *desc) 4152 { 4153 const struct spdk_nvme_ctrlr_data *cdata; 4154 4155 nvme_ns->ana_group_id = desc->ana_group_id; 4156 nvme_ns->ana_state = desc->ana_state; 4157 nvme_ns->ana_state_updating = false; 4158 4159 switch (nvme_ns->ana_state) { 4160 case SPDK_NVME_ANA_OPTIMIZED_STATE: 4161 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 4162 nvme_ns->ana_transition_timedout = false; 4163 spdk_poller_unregister(&nvme_ns->anatt_timer); 4164 break; 4165 4166 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 4167 case SPDK_NVME_ANA_CHANGE_STATE: 4168 if (nvme_ns->anatt_timer != NULL) { 4169 break; 4170 } 4171 4172 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 4173 nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout, 4174 nvme_ns, 4175 cdata->anatt * SPDK_SEC_TO_USEC); 4176 break; 4177 default: 4178 break; 4179 } 4180 } 4181 4182 static int 4183 nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg) 4184 { 4185 struct nvme_ns *nvme_ns = cb_arg; 4186 uint32_t i; 4187 4188 assert(nvme_ns->ns != NULL); 4189 4190 for (i = 0; i < desc->num_of_nsid; i++) { 4191 if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) { 4192 continue; 4193 } 4194 4195 _nvme_ns_set_ana_state(nvme_ns, desc); 4196 return 1; 4197 } 4198 4199 return 0; 4200 } 4201 4202 static int 4203 nvme_generate_uuid(const char *sn, uint32_t nsid, struct spdk_uuid *uuid) 4204 { 4205 int rc = 0; 4206 struct spdk_uuid new_uuid, namespace_uuid; 4207 char merged_str[SPDK_NVME_CTRLR_SN_LEN + NSID_STR_LEN + 1] = {'\0'}; 4208 /* This namespace UUID was generated using uuid_generate() method. */ 4209 const char *namespace_str = {"edaed2de-24bc-4b07-b559-f47ecbe730fd"}; 4210 int size; 4211 4212 assert(strlen(sn) <= SPDK_NVME_CTRLR_SN_LEN); 4213 4214 spdk_uuid_set_null(&new_uuid); 4215 spdk_uuid_set_null(&namespace_uuid); 4216 4217 size = snprintf(merged_str, sizeof(merged_str), "%s%"PRIu32, sn, nsid); 4218 if (size <= 0 || (unsigned long)size >= sizeof(merged_str)) { 4219 return -EINVAL; 4220 } 4221 4222 spdk_uuid_parse(&namespace_uuid, namespace_str); 4223 4224 rc = spdk_uuid_generate_sha1(&new_uuid, &namespace_uuid, merged_str, size); 4225 if (rc == 0) { 4226 memcpy(uuid, &new_uuid, sizeof(struct spdk_uuid)); 4227 } 4228 4229 return rc; 4230 } 4231 4232 static int 4233 nvme_disk_create(struct spdk_bdev *disk, const char *base_name, 4234 struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns, 4235 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, void *ctx) 4236 { 4237 const struct spdk_uuid *uuid; 4238 const uint8_t *nguid; 4239 const struct spdk_nvme_ctrlr_data *cdata; 4240 const struct spdk_nvme_ns_data *nsdata; 4241 const struct spdk_nvme_ctrlr_opts *opts; 4242 enum spdk_nvme_csi csi; 4243 uint32_t atomic_bs, phys_bs, bs; 4244 char sn_tmp[SPDK_NVME_CTRLR_SN_LEN + 1] = {'\0'}; 4245 int rc; 4246 4247 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4248 csi = spdk_nvme_ns_get_csi(ns); 4249 opts = spdk_nvme_ctrlr_get_opts(ctrlr); 4250 4251 switch (csi) { 4252 case SPDK_NVME_CSI_NVM: 4253 disk->product_name = "NVMe disk"; 4254 break; 4255 case SPDK_NVME_CSI_ZNS: 4256 disk->product_name = "NVMe ZNS disk"; 4257 disk->zoned = true; 4258 disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 4259 disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) / 4260 spdk_nvme_ns_get_extended_sector_size(ns); 4261 disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns); 4262 disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns); 4263 break; 4264 default: 4265 if (bdev_opts->allow_unrecognized_csi) { 4266 disk->product_name = "NVMe Passthrough disk"; 4267 break; 4268 } 4269 SPDK_ERRLOG("unsupported CSI: %u\n", csi); 4270 return -ENOTSUP; 4271 } 4272 4273 nguid = spdk_nvme_ns_get_nguid(ns); 4274 if (!nguid) { 4275 uuid = spdk_nvme_ns_get_uuid(ns); 4276 if (uuid) { 4277 disk->uuid = *uuid; 4278 } else if (g_opts.generate_uuids) { 4279 spdk_strcpy_pad(sn_tmp, cdata->sn, SPDK_NVME_CTRLR_SN_LEN, '\0'); 4280 rc = nvme_generate_uuid(sn_tmp, spdk_nvme_ns_get_id(ns), &disk->uuid); 4281 if (rc < 0) { 4282 SPDK_ERRLOG("UUID generation failed (%s)\n", spdk_strerror(-rc)); 4283 return rc; 4284 } 4285 } 4286 } else { 4287 memcpy(&disk->uuid, nguid, sizeof(disk->uuid)); 4288 } 4289 4290 disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns)); 4291 if (!disk->name) { 4292 return -ENOMEM; 4293 } 4294 4295 disk->write_cache = 0; 4296 if (cdata->vwc.present) { 4297 /* Enable if the Volatile Write Cache exists */ 4298 disk->write_cache = 1; 4299 } 4300 if (cdata->oncs.write_zeroes) { 4301 disk->max_write_zeroes = UINT16_MAX + 1; 4302 } 4303 disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns); 4304 disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns); 4305 disk->max_segment_size = spdk_nvme_ctrlr_get_max_xfer_size(ctrlr); 4306 disk->ctratt.raw = cdata->ctratt.raw; 4307 /* NVMe driver will split one request into multiple requests 4308 * based on MDTS and stripe boundary, the bdev layer will use 4309 * max_segment_size and max_num_segments to split one big IO 4310 * into multiple requests, then small request can't run out 4311 * of NVMe internal requests data structure. 4312 */ 4313 if (opts && opts->io_queue_requests) { 4314 disk->max_num_segments = opts->io_queue_requests / 2; 4315 } 4316 if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_SGL_SUPPORTED) { 4317 /* The nvme driver will try to split I/O that have too many 4318 * SGEs, but it doesn't work if that last SGE doesn't end on 4319 * an aggregate total that is block aligned. The bdev layer has 4320 * a more robust splitting framework, so use that instead for 4321 * this case. (See issue #3269.) 4322 */ 4323 uint16_t max_sges = spdk_nvme_ctrlr_get_max_sges(ctrlr); 4324 4325 if (disk->max_num_segments == 0) { 4326 disk->max_num_segments = max_sges; 4327 } else { 4328 disk->max_num_segments = spdk_min(disk->max_num_segments, max_sges); 4329 } 4330 } 4331 disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns); 4332 4333 nsdata = spdk_nvme_ns_get_data(ns); 4334 bs = spdk_nvme_ns_get_sector_size(ns); 4335 atomic_bs = bs; 4336 phys_bs = bs; 4337 if (nsdata->nabo == 0) { 4338 if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) { 4339 atomic_bs = bs * (1 + nsdata->nawupf); 4340 } else { 4341 atomic_bs = bs * (1 + cdata->awupf); 4342 } 4343 } 4344 if (nsdata->nsfeat.optperf) { 4345 phys_bs = bs * (1 + nsdata->npwg); 4346 } 4347 disk->phys_blocklen = spdk_min(phys_bs, atomic_bs); 4348 4349 disk->md_len = spdk_nvme_ns_get_md_size(ns); 4350 if (disk->md_len != 0) { 4351 disk->md_interleave = nsdata->flbas.extended; 4352 disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns); 4353 if (disk->dif_type != SPDK_DIF_DISABLE) { 4354 disk->dif_is_head_of_md = nsdata->dps.md_start; 4355 disk->dif_check_flags = bdev_opts->prchk_flags; 4356 disk->dif_pi_format = (enum spdk_dif_pi_format)spdk_nvme_ns_get_pi_format(ns); 4357 } 4358 } 4359 4360 if (!(spdk_nvme_ctrlr_get_flags(ctrlr) & 4361 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) { 4362 disk->acwu = 0; 4363 } else if (nsdata->nsfeat.ns_atomic_write_unit) { 4364 disk->acwu = nsdata->nacwu + 1; /* 0-based */ 4365 } else { 4366 disk->acwu = cdata->acwu + 1; /* 0-based */ 4367 } 4368 4369 if (cdata->oncs.copy) { 4370 /* For now bdev interface allows only single segment copy */ 4371 disk->max_copy = nsdata->mssrl; 4372 } 4373 4374 disk->ctxt = ctx; 4375 disk->fn_table = &nvmelib_fn_table; 4376 disk->module = &nvme_if; 4377 4378 disk->numa.id_valid = 1; 4379 disk->numa.id = spdk_nvme_ctrlr_get_numa_id(ctrlr); 4380 4381 return 0; 4382 } 4383 4384 static struct nvme_bdev * 4385 nvme_bdev_alloc(void) 4386 { 4387 struct nvme_bdev *bdev; 4388 int rc; 4389 4390 bdev = calloc(1, sizeof(*bdev)); 4391 if (!bdev) { 4392 SPDK_ERRLOG("bdev calloc() failed\n"); 4393 return NULL; 4394 } 4395 4396 if (g_opts.nvme_error_stat) { 4397 bdev->err_stat = calloc(1, sizeof(struct nvme_error_stat)); 4398 if (!bdev->err_stat) { 4399 SPDK_ERRLOG("err_stat calloc() failed\n"); 4400 free(bdev); 4401 return NULL; 4402 } 4403 } 4404 4405 rc = pthread_mutex_init(&bdev->mutex, NULL); 4406 if (rc != 0) { 4407 free(bdev->err_stat); 4408 free(bdev); 4409 return NULL; 4410 } 4411 4412 bdev->ref = 1; 4413 bdev->mp_policy = BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE; 4414 bdev->mp_selector = BDEV_NVME_MP_SELECTOR_ROUND_ROBIN; 4415 bdev->rr_min_io = UINT32_MAX; 4416 TAILQ_INIT(&bdev->nvme_ns_list); 4417 4418 return bdev; 4419 } 4420 4421 static int 4422 nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4423 { 4424 struct nvme_bdev *bdev; 4425 struct nvme_bdev_ctrlr *nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr; 4426 int rc; 4427 4428 bdev = nvme_bdev_alloc(); 4429 if (bdev == NULL) { 4430 SPDK_ERRLOG("Failed to allocate NVMe bdev\n"); 4431 return -ENOMEM; 4432 } 4433 4434 bdev->opal = nvme_ctrlr->opal_dev != NULL; 4435 4436 rc = nvme_disk_create(&bdev->disk, nbdev_ctrlr->name, nvme_ctrlr->ctrlr, 4437 nvme_ns->ns, &nvme_ctrlr->opts, bdev); 4438 if (rc != 0) { 4439 SPDK_ERRLOG("Failed to create NVMe disk\n"); 4440 nvme_bdev_free(bdev); 4441 return rc; 4442 } 4443 4444 spdk_io_device_register(bdev, 4445 bdev_nvme_create_bdev_channel_cb, 4446 bdev_nvme_destroy_bdev_channel_cb, 4447 sizeof(struct nvme_bdev_channel), 4448 bdev->disk.name); 4449 4450 nvme_ns->bdev = bdev; 4451 bdev->nsid = nvme_ns->id; 4452 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 4453 4454 bdev->nbdev_ctrlr = nbdev_ctrlr; 4455 TAILQ_INSERT_TAIL(&nbdev_ctrlr->bdevs, bdev, tailq); 4456 4457 rc = spdk_bdev_register(&bdev->disk); 4458 if (rc != 0) { 4459 SPDK_ERRLOG("spdk_bdev_register() failed\n"); 4460 spdk_io_device_unregister(bdev, NULL); 4461 nvme_ns->bdev = NULL; 4462 TAILQ_REMOVE(&nbdev_ctrlr->bdevs, bdev, tailq); 4463 nvme_bdev_free(bdev); 4464 return rc; 4465 } 4466 4467 return 0; 4468 } 4469 4470 static bool 4471 bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2) 4472 { 4473 const struct spdk_nvme_ns_data *nsdata1, *nsdata2; 4474 const struct spdk_uuid *uuid1, *uuid2; 4475 4476 nsdata1 = spdk_nvme_ns_get_data(ns1); 4477 nsdata2 = spdk_nvme_ns_get_data(ns2); 4478 uuid1 = spdk_nvme_ns_get_uuid(ns1); 4479 uuid2 = spdk_nvme_ns_get_uuid(ns2); 4480 4481 return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 && 4482 nsdata1->eui64 == nsdata2->eui64 && 4483 ((uuid1 == NULL && uuid2 == NULL) || 4484 (uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) && 4485 spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2); 4486 } 4487 4488 static bool 4489 hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 4490 struct spdk_nvme_ctrlr_opts *opts) 4491 { 4492 struct nvme_probe_skip_entry *entry; 4493 4494 TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) { 4495 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 4496 return false; 4497 } 4498 } 4499 4500 opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst; 4501 opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight; 4502 opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight; 4503 opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight; 4504 opts->disable_read_ana_log_page = true; 4505 4506 SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr); 4507 4508 return true; 4509 } 4510 4511 static void 4512 nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl) 4513 { 4514 struct nvme_ctrlr *nvme_ctrlr = ctx; 4515 4516 if (spdk_nvme_cpl_is_error(cpl)) { 4517 SPDK_WARNLOG("Abort failed. Resetting controller. sc is %u, sct is %u.\n", cpl->status.sc, 4518 cpl->status.sct); 4519 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4520 } else if (cpl->cdw0 & 0x1) { 4521 SPDK_WARNLOG("Specified command could not be aborted.\n"); 4522 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4523 } 4524 } 4525 4526 static void 4527 timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, 4528 struct spdk_nvme_qpair *qpair, uint16_t cid) 4529 { 4530 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 4531 union spdk_nvme_csts_register csts; 4532 int rc; 4533 4534 assert(nvme_ctrlr->ctrlr == ctrlr); 4535 4536 SPDK_WARNLOG("Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n", ctrlr, qpair, cid); 4537 4538 /* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O 4539 * queue. (Note: qpair == NULL when there's an admin cmd timeout.) Otherwise we 4540 * would submit another fabrics cmd on the admin queue to read CSTS and check for its 4541 * completion recursively. 4542 */ 4543 if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) { 4544 csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr); 4545 if (csts.bits.cfs) { 4546 SPDK_ERRLOG("Controller Fatal Status, reset required\n"); 4547 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4548 return; 4549 } 4550 } 4551 4552 switch (g_opts.action_on_timeout) { 4553 case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT: 4554 if (qpair) { 4555 /* Don't send abort to ctrlr when ctrlr is not available. */ 4556 pthread_mutex_lock(&nvme_ctrlr->mutex); 4557 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 4558 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4559 SPDK_NOTICELOG("Quit abort. Ctrlr is not available.\n"); 4560 return; 4561 } 4562 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4563 4564 rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid, 4565 nvme_abort_cpl, nvme_ctrlr); 4566 if (rc == 0) { 4567 return; 4568 } 4569 4570 SPDK_ERRLOG("Unable to send abort. Resetting, rc is %d.\n", rc); 4571 } 4572 4573 /* FALLTHROUGH */ 4574 case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET: 4575 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4576 break; 4577 case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE: 4578 SPDK_DEBUGLOG(bdev_nvme, "No action for nvme controller timeout.\n"); 4579 break; 4580 default: 4581 SPDK_ERRLOG("An invalid timeout action value is found.\n"); 4582 break; 4583 } 4584 } 4585 4586 static struct nvme_ns * 4587 nvme_ns_alloc(void) 4588 { 4589 struct nvme_ns *nvme_ns; 4590 4591 nvme_ns = calloc(1, sizeof(struct nvme_ns)); 4592 if (nvme_ns == NULL) { 4593 return NULL; 4594 } 4595 4596 if (g_opts.io_path_stat) { 4597 nvme_ns->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 4598 if (nvme_ns->stat == NULL) { 4599 free(nvme_ns); 4600 return NULL; 4601 } 4602 spdk_bdev_reset_io_stat(nvme_ns->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 4603 } 4604 4605 return nvme_ns; 4606 } 4607 4608 static void 4609 nvme_ns_free(struct nvme_ns *nvme_ns) 4610 { 4611 free(nvme_ns->stat); 4612 free(nvme_ns); 4613 } 4614 4615 static void 4616 nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc) 4617 { 4618 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 4619 struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx; 4620 4621 if (rc == 0) { 4622 nvme_ns->probe_ctx = NULL; 4623 pthread_mutex_lock(&nvme_ctrlr->mutex); 4624 nvme_ctrlr->ref++; 4625 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4626 } else { 4627 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4628 nvme_ns_free(nvme_ns); 4629 } 4630 4631 if (ctx) { 4632 ctx->populates_in_progress--; 4633 if (ctx->populates_in_progress == 0) { 4634 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 4635 } 4636 } 4637 } 4638 4639 static void 4640 bdev_nvme_add_io_path(struct nvme_bdev_channel_iter *i, 4641 struct nvme_bdev *nbdev, 4642 struct nvme_bdev_channel *nbdev_ch, void *ctx) 4643 { 4644 struct nvme_ns *nvme_ns = ctx; 4645 int rc; 4646 4647 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 4648 if (rc != 0) { 4649 SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n"); 4650 } 4651 4652 nvme_bdev_for_each_channel_continue(i, rc); 4653 } 4654 4655 static void 4656 bdev_nvme_delete_io_path(struct nvme_bdev_channel_iter *i, 4657 struct nvme_bdev *nbdev, 4658 struct nvme_bdev_channel *nbdev_ch, void *ctx) 4659 { 4660 struct nvme_ns *nvme_ns = ctx; 4661 struct nvme_io_path *io_path; 4662 4663 io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns); 4664 if (io_path != NULL) { 4665 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 4666 } 4667 4668 nvme_bdev_for_each_channel_continue(i, 0); 4669 } 4670 4671 static void 4672 bdev_nvme_add_io_path_failed(struct nvme_bdev *nbdev, void *ctx, int status) 4673 { 4674 struct nvme_ns *nvme_ns = ctx; 4675 4676 nvme_ctrlr_populate_namespace_done(nvme_ns, -1); 4677 } 4678 4679 static void 4680 bdev_nvme_add_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status) 4681 { 4682 struct nvme_ns *nvme_ns = ctx; 4683 4684 if (status == 0) { 4685 nvme_ctrlr_populate_namespace_done(nvme_ns, 0); 4686 } else { 4687 /* Delete the added io_paths and fail populating the namespace. */ 4688 nvme_bdev_for_each_channel(nbdev, 4689 bdev_nvme_delete_io_path, 4690 nvme_ns, 4691 bdev_nvme_add_io_path_failed); 4692 } 4693 } 4694 4695 static int 4696 nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns) 4697 { 4698 struct nvme_ns *tmp_ns; 4699 const struct spdk_nvme_ns_data *nsdata; 4700 4701 nsdata = spdk_nvme_ns_get_data(nvme_ns->ns); 4702 if (!nsdata->nmic.can_share) { 4703 SPDK_ERRLOG("Namespace cannot be shared.\n"); 4704 return -EINVAL; 4705 } 4706 4707 pthread_mutex_lock(&bdev->mutex); 4708 4709 tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list); 4710 assert(tmp_ns != NULL); 4711 4712 if (tmp_ns->ns != NULL && !bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) { 4713 pthread_mutex_unlock(&bdev->mutex); 4714 SPDK_ERRLOG("Namespaces are not identical.\n"); 4715 return -EINVAL; 4716 } 4717 4718 bdev->ref++; 4719 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 4720 nvme_ns->bdev = bdev; 4721 4722 pthread_mutex_unlock(&bdev->mutex); 4723 4724 /* Add nvme_io_path to nvme_bdev_channels dynamically. */ 4725 nvme_bdev_for_each_channel(bdev, 4726 bdev_nvme_add_io_path, 4727 nvme_ns, 4728 bdev_nvme_add_io_path_done); 4729 4730 return 0; 4731 } 4732 4733 static void 4734 nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4735 { 4736 struct spdk_nvme_ns *ns; 4737 struct nvme_bdev *bdev; 4738 int rc = 0; 4739 4740 ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id); 4741 if (!ns) { 4742 SPDK_DEBUGLOG(bdev_nvme, "Invalid NS %d\n", nvme_ns->id); 4743 rc = -EINVAL; 4744 goto done; 4745 } 4746 4747 nvme_ns->ns = ns; 4748 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 4749 4750 if (nvme_ctrlr->ana_log_page != NULL) { 4751 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns); 4752 } 4753 4754 bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id); 4755 if (bdev == NULL) { 4756 rc = nvme_bdev_create(nvme_ctrlr, nvme_ns); 4757 } else { 4758 rc = nvme_bdev_add_ns(bdev, nvme_ns); 4759 if (rc == 0) { 4760 return; 4761 } 4762 } 4763 done: 4764 nvme_ctrlr_populate_namespace_done(nvme_ns, rc); 4765 } 4766 4767 static void 4768 nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns) 4769 { 4770 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 4771 4772 assert(nvme_ctrlr != NULL); 4773 4774 pthread_mutex_lock(&nvme_ctrlr->mutex); 4775 4776 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4777 4778 if (nvme_ns->bdev != NULL) { 4779 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4780 return; 4781 } 4782 4783 nvme_ns_free(nvme_ns); 4784 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4785 4786 nvme_ctrlr_release(nvme_ctrlr); 4787 } 4788 4789 static void 4790 bdev_nvme_delete_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status) 4791 { 4792 struct nvme_ns *nvme_ns = ctx; 4793 4794 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 4795 } 4796 4797 static void 4798 nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4799 { 4800 struct nvme_bdev *bdev; 4801 4802 spdk_poller_unregister(&nvme_ns->anatt_timer); 4803 4804 bdev = nvme_ns->bdev; 4805 if (bdev != NULL) { 4806 pthread_mutex_lock(&bdev->mutex); 4807 4808 assert(bdev->ref > 0); 4809 bdev->ref--; 4810 if (bdev->ref == 0) { 4811 pthread_mutex_unlock(&bdev->mutex); 4812 4813 spdk_bdev_unregister(&bdev->disk, NULL, NULL); 4814 } else { 4815 /* spdk_bdev_unregister() is not called until the last nvme_ns is 4816 * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list 4817 * and clear nvme_ns->bdev here. 4818 */ 4819 TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq); 4820 nvme_ns->bdev = NULL; 4821 4822 pthread_mutex_unlock(&bdev->mutex); 4823 4824 /* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that, 4825 * we call depopulate_namespace_done() to avoid use-after-free. 4826 */ 4827 nvme_bdev_for_each_channel(bdev, 4828 bdev_nvme_delete_io_path, 4829 nvme_ns, 4830 bdev_nvme_delete_io_path_done); 4831 return; 4832 } 4833 } 4834 4835 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 4836 } 4837 4838 static void 4839 nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 4840 struct nvme_async_probe_ctx *ctx) 4841 { 4842 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4843 struct nvme_ns *nvme_ns, *next; 4844 struct spdk_nvme_ns *ns; 4845 struct nvme_bdev *bdev; 4846 uint32_t nsid; 4847 int rc; 4848 uint64_t num_sectors; 4849 4850 if (ctx) { 4851 /* Initialize this count to 1 to handle the populate functions 4852 * calling nvme_ctrlr_populate_namespace_done() immediately. 4853 */ 4854 ctx->populates_in_progress = 1; 4855 } 4856 4857 /* First loop over our existing namespaces and see if they have been 4858 * removed. */ 4859 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4860 while (nvme_ns != NULL) { 4861 next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 4862 4863 if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 4864 /* NS is still there or added again. Its attributes may have changed. */ 4865 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id); 4866 if (nvme_ns->ns != ns) { 4867 assert(nvme_ns->ns == NULL); 4868 nvme_ns->ns = ns; 4869 SPDK_DEBUGLOG(bdev_nvme, "NSID %u was added\n", nvme_ns->id); 4870 } 4871 4872 num_sectors = spdk_nvme_ns_get_num_sectors(ns); 4873 bdev = nvme_ns->bdev; 4874 assert(bdev != NULL); 4875 if (bdev->disk.blockcnt != num_sectors) { 4876 SPDK_NOTICELOG("NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n", 4877 nvme_ns->id, 4878 bdev->disk.name, 4879 bdev->disk.blockcnt, 4880 num_sectors); 4881 rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors); 4882 if (rc != 0) { 4883 SPDK_ERRLOG("Could not change num blocks for nvme bdev: name %s, errno: %d.\n", 4884 bdev->disk.name, rc); 4885 } 4886 } 4887 } else { 4888 /* Namespace was removed */ 4889 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 4890 } 4891 4892 nvme_ns = next; 4893 } 4894 4895 /* Loop through all of the namespaces at the nvme level and see if any of them are new */ 4896 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 4897 while (nsid != 0) { 4898 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 4899 4900 if (nvme_ns == NULL) { 4901 /* Found a new one */ 4902 nvme_ns = nvme_ns_alloc(); 4903 if (nvme_ns == NULL) { 4904 SPDK_ERRLOG("Failed to allocate namespace\n"); 4905 /* This just fails to attach the namespace. It may work on a future attempt. */ 4906 continue; 4907 } 4908 4909 nvme_ns->id = nsid; 4910 nvme_ns->ctrlr = nvme_ctrlr; 4911 4912 nvme_ns->bdev = NULL; 4913 4914 if (ctx) { 4915 ctx->populates_in_progress++; 4916 } 4917 nvme_ns->probe_ctx = ctx; 4918 4919 RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4920 4921 nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns); 4922 } 4923 4924 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid); 4925 } 4926 4927 if (ctx) { 4928 /* Decrement this count now that the loop is over to account 4929 * for the one we started with. If the count is then 0, we 4930 * know any populate_namespace functions completed immediately, 4931 * so we'll kick the callback here. 4932 */ 4933 ctx->populates_in_progress--; 4934 if (ctx->populates_in_progress == 0) { 4935 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 4936 } 4937 } 4938 4939 } 4940 4941 static void 4942 nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr) 4943 { 4944 struct nvme_ns *nvme_ns, *tmp; 4945 4946 RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) { 4947 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 4948 } 4949 } 4950 4951 static uint32_t 4952 nvme_ctrlr_get_ana_log_page_size(struct nvme_ctrlr *nvme_ctrlr) 4953 { 4954 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4955 const struct spdk_nvme_ctrlr_data *cdata; 4956 uint32_t nsid, ns_count = 0; 4957 4958 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4959 4960 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 4961 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 4962 ns_count++; 4963 } 4964 4965 return sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 4966 sizeof(struct spdk_nvme_ana_group_descriptor) + ns_count * 4967 sizeof(uint32_t); 4968 } 4969 4970 static int 4971 nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc, 4972 void *cb_arg) 4973 { 4974 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 4975 struct nvme_ns *nvme_ns; 4976 uint32_t i, nsid; 4977 4978 for (i = 0; i < desc->num_of_nsid; i++) { 4979 nsid = desc->nsid[i]; 4980 if (nsid == 0) { 4981 continue; 4982 } 4983 4984 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 4985 4986 if (nvme_ns == NULL) { 4987 /* Target told us that an inactive namespace had an ANA change */ 4988 continue; 4989 } 4990 4991 _nvme_ns_set_ana_state(nvme_ns, desc); 4992 } 4993 4994 return 0; 4995 } 4996 4997 static void 4998 bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 4999 { 5000 struct nvme_ns *nvme_ns; 5001 5002 spdk_free(nvme_ctrlr->ana_log_page); 5003 nvme_ctrlr->ana_log_page = NULL; 5004 5005 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 5006 nvme_ns != NULL; 5007 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 5008 nvme_ns->ana_state_updating = false; 5009 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 5010 } 5011 } 5012 5013 static void 5014 nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl) 5015 { 5016 struct nvme_ctrlr *nvme_ctrlr = ctx; 5017 5018 if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) { 5019 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states, 5020 nvme_ctrlr); 5021 } else { 5022 bdev_nvme_disable_read_ana_log_page(nvme_ctrlr); 5023 } 5024 5025 pthread_mutex_lock(&nvme_ctrlr->mutex); 5026 5027 assert(nvme_ctrlr->ana_log_page_updating == true); 5028 nvme_ctrlr->ana_log_page_updating = false; 5029 5030 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 5031 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5032 5033 nvme_ctrlr_unregister(nvme_ctrlr); 5034 } else { 5035 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5036 5037 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 5038 } 5039 } 5040 5041 static int 5042 nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 5043 { 5044 uint32_t ana_log_page_size; 5045 int rc; 5046 5047 if (nvme_ctrlr->ana_log_page == NULL) { 5048 return -EINVAL; 5049 } 5050 5051 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 5052 5053 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 5054 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 5055 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 5056 return -EINVAL; 5057 } 5058 5059 pthread_mutex_lock(&nvme_ctrlr->mutex); 5060 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 5061 nvme_ctrlr->ana_log_page_updating) { 5062 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5063 return -EBUSY; 5064 } 5065 5066 nvme_ctrlr->ana_log_page_updating = true; 5067 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5068 5069 rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr, 5070 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 5071 SPDK_NVME_GLOBAL_NS_TAG, 5072 nvme_ctrlr->ana_log_page, 5073 ana_log_page_size, 0, 5074 nvme_ctrlr_read_ana_log_page_done, 5075 nvme_ctrlr); 5076 if (rc != 0) { 5077 nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL); 5078 } 5079 5080 return rc; 5081 } 5082 5083 static void 5084 dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 5085 { 5086 } 5087 5088 struct bdev_nvme_set_preferred_path_ctx { 5089 struct spdk_bdev_desc *desc; 5090 struct nvme_ns *nvme_ns; 5091 bdev_nvme_set_preferred_path_cb cb_fn; 5092 void *cb_arg; 5093 }; 5094 5095 static void 5096 bdev_nvme_set_preferred_path_done(struct nvme_bdev *nbdev, void *_ctx, int status) 5097 { 5098 struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx; 5099 5100 assert(ctx != NULL); 5101 assert(ctx->desc != NULL); 5102 assert(ctx->cb_fn != NULL); 5103 5104 spdk_bdev_close(ctx->desc); 5105 5106 ctx->cb_fn(ctx->cb_arg, status); 5107 5108 free(ctx); 5109 } 5110 5111 static void 5112 _bdev_nvme_set_preferred_path(struct nvme_bdev_channel_iter *i, 5113 struct nvme_bdev *nbdev, 5114 struct nvme_bdev_channel *nbdev_ch, void *_ctx) 5115 { 5116 struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx; 5117 struct nvme_io_path *io_path, *prev; 5118 5119 prev = NULL; 5120 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 5121 if (io_path->nvme_ns == ctx->nvme_ns) { 5122 break; 5123 } 5124 prev = io_path; 5125 } 5126 5127 if (io_path != NULL) { 5128 if (prev != NULL) { 5129 STAILQ_REMOVE_AFTER(&nbdev_ch->io_path_list, prev, stailq); 5130 STAILQ_INSERT_HEAD(&nbdev_ch->io_path_list, io_path, stailq); 5131 } 5132 5133 /* We can set io_path to nbdev_ch->current_io_path directly here. 5134 * However, it needs to be conditional. To simplify the code, 5135 * just clear nbdev_ch->current_io_path and let find_io_path() 5136 * fill it. 5137 * 5138 * Automatic failback may be disabled. Hence even if the io_path is 5139 * already at the head, clear nbdev_ch->current_io_path. 5140 */ 5141 bdev_nvme_clear_current_io_path(nbdev_ch); 5142 } 5143 5144 nvme_bdev_for_each_channel_continue(i, 0); 5145 } 5146 5147 static struct nvme_ns * 5148 bdev_nvme_set_preferred_ns(struct nvme_bdev *nbdev, uint16_t cntlid) 5149 { 5150 struct nvme_ns *nvme_ns, *prev; 5151 const struct spdk_nvme_ctrlr_data *cdata; 5152 5153 prev = NULL; 5154 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 5155 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 5156 5157 if (cdata->cntlid == cntlid) { 5158 break; 5159 } 5160 prev = nvme_ns; 5161 } 5162 5163 if (nvme_ns != NULL && prev != NULL) { 5164 TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq); 5165 TAILQ_INSERT_HEAD(&nbdev->nvme_ns_list, nvme_ns, tailq); 5166 } 5167 5168 return nvme_ns; 5169 } 5170 5171 /* This function supports only multipath mode. There is only a single I/O path 5172 * for each NVMe-oF controller. Hence, just move the matched I/O path to the 5173 * head of the I/O path list for each NVMe bdev channel. 5174 * 5175 * NVMe bdev channel may be acquired after completing this function. move the 5176 * matched namespace to the head of the namespace list for the NVMe bdev too. 5177 */ 5178 void 5179 bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid, 5180 bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg) 5181 { 5182 struct bdev_nvme_set_preferred_path_ctx *ctx; 5183 struct spdk_bdev *bdev; 5184 struct nvme_bdev *nbdev; 5185 int rc = 0; 5186 5187 assert(cb_fn != NULL); 5188 5189 ctx = calloc(1, sizeof(*ctx)); 5190 if (ctx == NULL) { 5191 SPDK_ERRLOG("Failed to alloc context.\n"); 5192 rc = -ENOMEM; 5193 goto err_alloc; 5194 } 5195 5196 ctx->cb_fn = cb_fn; 5197 ctx->cb_arg = cb_arg; 5198 5199 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 5200 if (rc != 0) { 5201 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 5202 goto err_open; 5203 } 5204 5205 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 5206 5207 if (bdev->module != &nvme_if) { 5208 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 5209 rc = -ENODEV; 5210 goto err_bdev; 5211 } 5212 5213 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 5214 5215 pthread_mutex_lock(&nbdev->mutex); 5216 5217 ctx->nvme_ns = bdev_nvme_set_preferred_ns(nbdev, cntlid); 5218 if (ctx->nvme_ns == NULL) { 5219 pthread_mutex_unlock(&nbdev->mutex); 5220 5221 SPDK_ERRLOG("bdev %s does not have namespace to controller %u.\n", name, cntlid); 5222 rc = -ENODEV; 5223 goto err_bdev; 5224 } 5225 5226 pthread_mutex_unlock(&nbdev->mutex); 5227 5228 nvme_bdev_for_each_channel(nbdev, 5229 _bdev_nvme_set_preferred_path, 5230 ctx, 5231 bdev_nvme_set_preferred_path_done); 5232 return; 5233 5234 err_bdev: 5235 spdk_bdev_close(ctx->desc); 5236 err_open: 5237 free(ctx); 5238 err_alloc: 5239 cb_fn(cb_arg, rc); 5240 } 5241 5242 struct bdev_nvme_set_multipath_policy_ctx { 5243 struct spdk_bdev_desc *desc; 5244 spdk_bdev_nvme_set_multipath_policy_cb cb_fn; 5245 void *cb_arg; 5246 }; 5247 5248 static void 5249 bdev_nvme_set_multipath_policy_done(struct nvme_bdev *nbdev, void *_ctx, int status) 5250 { 5251 struct bdev_nvme_set_multipath_policy_ctx *ctx = _ctx; 5252 5253 assert(ctx != NULL); 5254 assert(ctx->desc != NULL); 5255 assert(ctx->cb_fn != NULL); 5256 5257 spdk_bdev_close(ctx->desc); 5258 5259 ctx->cb_fn(ctx->cb_arg, status); 5260 5261 free(ctx); 5262 } 5263 5264 static void 5265 _bdev_nvme_set_multipath_policy(struct nvme_bdev_channel_iter *i, 5266 struct nvme_bdev *nbdev, 5267 struct nvme_bdev_channel *nbdev_ch, void *ctx) 5268 { 5269 nbdev_ch->mp_policy = nbdev->mp_policy; 5270 nbdev_ch->mp_selector = nbdev->mp_selector; 5271 nbdev_ch->rr_min_io = nbdev->rr_min_io; 5272 bdev_nvme_clear_current_io_path(nbdev_ch); 5273 5274 nvme_bdev_for_each_channel_continue(i, 0); 5275 } 5276 5277 void 5278 spdk_bdev_nvme_set_multipath_policy(const char *name, enum spdk_bdev_nvme_multipath_policy policy, 5279 enum spdk_bdev_nvme_multipath_selector selector, uint32_t rr_min_io, 5280 spdk_bdev_nvme_set_multipath_policy_cb cb_fn, void *cb_arg) 5281 { 5282 struct bdev_nvme_set_multipath_policy_ctx *ctx; 5283 struct spdk_bdev *bdev; 5284 struct nvme_bdev *nbdev; 5285 int rc; 5286 5287 assert(cb_fn != NULL); 5288 5289 switch (policy) { 5290 case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE: 5291 break; 5292 case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE: 5293 switch (selector) { 5294 case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN: 5295 if (rr_min_io == UINT32_MAX) { 5296 rr_min_io = 1; 5297 } else if (rr_min_io == 0) { 5298 rc = -EINVAL; 5299 goto exit; 5300 } 5301 break; 5302 case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH: 5303 break; 5304 default: 5305 rc = -EINVAL; 5306 goto exit; 5307 } 5308 break; 5309 default: 5310 rc = -EINVAL; 5311 goto exit; 5312 } 5313 5314 ctx = calloc(1, sizeof(*ctx)); 5315 if (ctx == NULL) { 5316 SPDK_ERRLOG("Failed to alloc context.\n"); 5317 rc = -ENOMEM; 5318 goto exit; 5319 } 5320 5321 ctx->cb_fn = cb_fn; 5322 ctx->cb_arg = cb_arg; 5323 5324 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 5325 if (rc != 0) { 5326 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 5327 rc = -ENODEV; 5328 goto err_open; 5329 } 5330 5331 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 5332 if (bdev->module != &nvme_if) { 5333 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 5334 rc = -ENODEV; 5335 goto err_module; 5336 } 5337 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 5338 5339 pthread_mutex_lock(&nbdev->mutex); 5340 nbdev->mp_policy = policy; 5341 nbdev->mp_selector = selector; 5342 nbdev->rr_min_io = rr_min_io; 5343 pthread_mutex_unlock(&nbdev->mutex); 5344 5345 nvme_bdev_for_each_channel(nbdev, 5346 _bdev_nvme_set_multipath_policy, 5347 ctx, 5348 bdev_nvme_set_multipath_policy_done); 5349 return; 5350 5351 err_module: 5352 spdk_bdev_close(ctx->desc); 5353 err_open: 5354 free(ctx); 5355 exit: 5356 cb_fn(cb_arg, rc); 5357 } 5358 5359 static void 5360 aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 5361 { 5362 struct nvme_ctrlr *nvme_ctrlr = arg; 5363 union spdk_nvme_async_event_completion event; 5364 5365 if (spdk_nvme_cpl_is_error(cpl)) { 5366 SPDK_WARNLOG("AER request execute failed\n"); 5367 return; 5368 } 5369 5370 event.raw = cpl->cdw0; 5371 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 5372 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) { 5373 nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL); 5374 } else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 5375 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) { 5376 nvme_ctrlr_read_ana_log_page(nvme_ctrlr); 5377 } 5378 } 5379 5380 static void 5381 free_nvme_async_probe_ctx(struct nvme_async_probe_ctx *ctx) 5382 { 5383 spdk_keyring_put_key(ctx->drv_opts.tls_psk); 5384 spdk_keyring_put_key(ctx->drv_opts.dhchap_key); 5385 spdk_keyring_put_key(ctx->drv_opts.dhchap_ctrlr_key); 5386 free(ctx); 5387 } 5388 5389 static void 5390 populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, int rc) 5391 { 5392 if (ctx->cb_fn) { 5393 ctx->cb_fn(ctx->cb_ctx, ctx->reported_bdevs, rc); 5394 } 5395 5396 ctx->namespaces_populated = true; 5397 if (ctx->probe_done) { 5398 /* The probe was already completed, so we need to free the context 5399 * here. This can happen for cases like OCSSD, where we need to 5400 * send additional commands to the SSD after attach. 5401 */ 5402 free_nvme_async_probe_ctx(ctx); 5403 } 5404 } 5405 5406 static int 5407 bdev_nvme_remove_poller(void *ctx) 5408 { 5409 struct spdk_nvme_transport_id trid_pcie; 5410 5411 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 5412 spdk_poller_unregister(&g_hotplug_poller); 5413 return SPDK_POLLER_IDLE; 5414 } 5415 5416 memset(&trid_pcie, 0, sizeof(trid_pcie)); 5417 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 5418 5419 if (spdk_nvme_scan_attached(&trid_pcie)) { 5420 SPDK_ERRLOG_RATELIMIT("spdk_nvme_scan_attached() failed\n"); 5421 } 5422 5423 return SPDK_POLLER_BUSY; 5424 } 5425 5426 static void 5427 nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr, 5428 struct nvme_async_probe_ctx *ctx) 5429 { 5430 spdk_io_device_register(nvme_ctrlr, 5431 bdev_nvme_create_ctrlr_channel_cb, 5432 bdev_nvme_destroy_ctrlr_channel_cb, 5433 sizeof(struct nvme_ctrlr_channel), 5434 nvme_ctrlr->nbdev_ctrlr->name); 5435 5436 nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx); 5437 5438 if (g_hotplug_poller == NULL) { 5439 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL, 5440 NVME_HOTPLUG_POLL_PERIOD_DEFAULT); 5441 } 5442 } 5443 5444 static void 5445 nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl) 5446 { 5447 struct nvme_ctrlr *nvme_ctrlr = _ctx; 5448 struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx; 5449 5450 nvme_ctrlr->probe_ctx = NULL; 5451 5452 if (spdk_nvme_cpl_is_error(cpl)) { 5453 nvme_ctrlr_delete(nvme_ctrlr); 5454 5455 if (ctx != NULL) { 5456 ctx->reported_bdevs = 0; 5457 populate_namespaces_cb(ctx, -1); 5458 } 5459 return; 5460 } 5461 5462 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 5463 } 5464 5465 static int 5466 nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 5467 struct nvme_async_probe_ctx *ctx) 5468 { 5469 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 5470 const struct spdk_nvme_ctrlr_data *cdata; 5471 uint32_t ana_log_page_size; 5472 5473 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5474 5475 /* Set buffer size enough to include maximum number of allowed namespaces. */ 5476 ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 5477 sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->mnan * 5478 sizeof(uint32_t); 5479 5480 nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL, 5481 SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_DMA); 5482 if (nvme_ctrlr->ana_log_page == NULL) { 5483 SPDK_ERRLOG("could not allocate ANA log page buffer\n"); 5484 return -ENXIO; 5485 } 5486 5487 /* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned. 5488 * Hence copy each descriptor to a temporary area when parsing it. 5489 * 5490 * Allocate a buffer whose size is as large as ANA log page buffer because 5491 * we do not know the size of a descriptor until actually reading it. 5492 */ 5493 nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size); 5494 if (nvme_ctrlr->copied_ana_desc == NULL) { 5495 SPDK_ERRLOG("could not allocate a buffer to parse ANA descriptor\n"); 5496 return -ENOMEM; 5497 } 5498 5499 nvme_ctrlr->max_ana_log_page_size = ana_log_page_size; 5500 5501 nvme_ctrlr->probe_ctx = ctx; 5502 5503 /* Then, set the read size only to include the current active namespaces. */ 5504 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 5505 5506 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 5507 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 5508 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 5509 return -EINVAL; 5510 } 5511 5512 return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, 5513 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 5514 SPDK_NVME_GLOBAL_NS_TAG, 5515 nvme_ctrlr->ana_log_page, 5516 ana_log_page_size, 0, 5517 nvme_ctrlr_init_ana_log_page_done, 5518 nvme_ctrlr); 5519 } 5520 5521 /* hostnqn and subnqn were already verified before attaching a controller. 5522 * Hence check only the multipath capability and cntlid here. 5523 */ 5524 static bool 5525 bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr) 5526 { 5527 struct nvme_ctrlr *tmp; 5528 const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata; 5529 5530 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5531 5532 if (!cdata->cmic.multi_ctrlr) { 5533 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 5534 return false; 5535 } 5536 5537 TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) { 5538 tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr); 5539 5540 if (!tmp_cdata->cmic.multi_ctrlr) { 5541 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 5542 return false; 5543 } 5544 if (cdata->cntlid == tmp_cdata->cntlid) { 5545 SPDK_ERRLOG("cntlid %u are duplicated.\n", tmp_cdata->cntlid); 5546 return false; 5547 } 5548 } 5549 5550 return true; 5551 } 5552 5553 SPDK_LOG_DEPRECATION_REGISTER(multipath_config, 5554 "bdev_nvme_attach_controller.multipath configuration mismatch", "v25.01", 0); 5555 5556 static int 5557 nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr) 5558 { 5559 struct nvme_bdev_ctrlr *nbdev_ctrlr; 5560 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 5561 struct nvme_ctrlr *nctrlr; 5562 int rc = 0; 5563 5564 pthread_mutex_lock(&g_bdev_nvme_mutex); 5565 5566 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 5567 if (nbdev_ctrlr != NULL) { 5568 if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) { 5569 rc = -EINVAL; 5570 goto exit; 5571 } 5572 TAILQ_FOREACH(nctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 5573 if (nctrlr->opts.multipath != nvme_ctrlr->opts.multipath) { 5574 /* All controllers created with the same name must be configured either 5575 * for multipath or for failover. Otherwise we have configuration mismatch. 5576 * While this is currently still supported, support for configuration where some 5577 * controllers with the same name are configured for multipath, while others 5578 * are configured for failover will be removed in release 25.01. 5579 * Default mode change: starting from SPDK 25.01, if the user will not provide 5580 * '-x <mode>' parameter in the bdev_nvme_attach_controller RPC call, default 5581 * mode assigned to the controller will be 'multipath' 5582 */ 5583 SPDK_LOG_DEPRECATED(multipath_config); 5584 break; 5585 } 5586 } 5587 } else { 5588 nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr)); 5589 if (nbdev_ctrlr == NULL) { 5590 SPDK_ERRLOG("Failed to allocate nvme_bdev_ctrlr.\n"); 5591 rc = -ENOMEM; 5592 goto exit; 5593 } 5594 nbdev_ctrlr->name = strdup(name); 5595 if (nbdev_ctrlr->name == NULL) { 5596 SPDK_ERRLOG("Failed to allocate name of nvme_bdev_ctrlr.\n"); 5597 free(nbdev_ctrlr); 5598 goto exit; 5599 } 5600 TAILQ_INIT(&nbdev_ctrlr->ctrlrs); 5601 TAILQ_INIT(&nbdev_ctrlr->bdevs); 5602 TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 5603 } 5604 nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr; 5605 TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 5606 exit: 5607 pthread_mutex_unlock(&g_bdev_nvme_mutex); 5608 return rc; 5609 } 5610 5611 static int 5612 nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr, 5613 const char *name, 5614 const struct spdk_nvme_transport_id *trid, 5615 struct nvme_async_probe_ctx *ctx) 5616 { 5617 struct nvme_ctrlr *nvme_ctrlr; 5618 struct nvme_path_id *path_id; 5619 const struct spdk_nvme_ctrlr_data *cdata; 5620 int rc; 5621 5622 nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr)); 5623 if (nvme_ctrlr == NULL) { 5624 SPDK_ERRLOG("Failed to allocate device struct\n"); 5625 return -ENOMEM; 5626 } 5627 5628 rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL); 5629 if (rc != 0) { 5630 free(nvme_ctrlr); 5631 return rc; 5632 } 5633 5634 TAILQ_INIT(&nvme_ctrlr->trids); 5635 RB_INIT(&nvme_ctrlr->namespaces); 5636 5637 /* Get another reference to the key, so the first one can be released from probe_ctx */ 5638 if (ctx != NULL) { 5639 if (ctx->drv_opts.tls_psk != NULL) { 5640 nvme_ctrlr->psk = spdk_keyring_get_key( 5641 spdk_key_get_name(ctx->drv_opts.tls_psk)); 5642 if (nvme_ctrlr->psk == NULL) { 5643 /* Could only happen if the key was removed in the meantime */ 5644 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5645 spdk_key_get_name(ctx->drv_opts.tls_psk)); 5646 rc = -ENOKEY; 5647 goto err; 5648 } 5649 } 5650 5651 if (ctx->drv_opts.dhchap_key != NULL) { 5652 nvme_ctrlr->dhchap_key = spdk_keyring_get_key( 5653 spdk_key_get_name(ctx->drv_opts.dhchap_key)); 5654 if (nvme_ctrlr->dhchap_key == NULL) { 5655 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5656 spdk_key_get_name(ctx->drv_opts.dhchap_key)); 5657 rc = -ENOKEY; 5658 goto err; 5659 } 5660 } 5661 5662 if (ctx->drv_opts.dhchap_ctrlr_key != NULL) { 5663 nvme_ctrlr->dhchap_ctrlr_key = 5664 spdk_keyring_get_key( 5665 spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key)); 5666 if (nvme_ctrlr->dhchap_ctrlr_key == NULL) { 5667 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5668 spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key)); 5669 rc = -ENOKEY; 5670 goto err; 5671 } 5672 } 5673 } 5674 5675 path_id = calloc(1, sizeof(*path_id)); 5676 if (path_id == NULL) { 5677 SPDK_ERRLOG("Failed to allocate trid entry pointer\n"); 5678 rc = -ENOMEM; 5679 goto err; 5680 } 5681 5682 path_id->trid = *trid; 5683 if (ctx != NULL) { 5684 memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr)); 5685 memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid)); 5686 } 5687 nvme_ctrlr->active_path_id = path_id; 5688 TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link); 5689 5690 nvme_ctrlr->thread = spdk_get_thread(); 5691 nvme_ctrlr->ctrlr = ctrlr; 5692 nvme_ctrlr->ref = 1; 5693 5694 if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) { 5695 SPDK_ERRLOG("OCSSDs are not supported"); 5696 rc = -ENOTSUP; 5697 goto err; 5698 } 5699 5700 if (ctx != NULL) { 5701 memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts)); 5702 } else { 5703 spdk_bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts); 5704 } 5705 5706 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr, 5707 g_opts.nvme_adminq_poll_period_us); 5708 5709 if (g_opts.timeout_us > 0) { 5710 /* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */ 5711 /* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */ 5712 uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ? 5713 g_opts.timeout_us : g_opts.timeout_admin_us; 5714 spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us, 5715 adm_timeout_us, timeout_cb, nvme_ctrlr); 5716 } 5717 5718 spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr); 5719 spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr); 5720 5721 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 5722 SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) { 5723 nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr); 5724 } 5725 5726 rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr); 5727 if (rc != 0) { 5728 goto err; 5729 } 5730 5731 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5732 5733 if (cdata->cmic.ana_reporting) { 5734 rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx); 5735 if (rc == 0) { 5736 return 0; 5737 } 5738 } else { 5739 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 5740 return 0; 5741 } 5742 5743 err: 5744 nvme_ctrlr_delete(nvme_ctrlr); 5745 return rc; 5746 } 5747 5748 void 5749 spdk_bdev_nvme_get_default_ctrlr_opts(struct spdk_bdev_nvme_ctrlr_opts *opts) 5750 { 5751 opts->prchk_flags = 0; 5752 opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec; 5753 opts->reconnect_delay_sec = g_opts.reconnect_delay_sec; 5754 opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec; 5755 opts->multipath = false; 5756 } 5757 5758 static void 5759 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 5760 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts) 5761 { 5762 char *name; 5763 5764 name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++); 5765 if (!name) { 5766 SPDK_ERRLOG("Failed to assign name to NVMe device\n"); 5767 return; 5768 } 5769 5770 if (nvme_ctrlr_create(ctrlr, name, trid, NULL) == 0) { 5771 SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name); 5772 } else { 5773 SPDK_ERRLOG("Failed to attach to %s (%s)\n", trid->traddr, name); 5774 } 5775 5776 free(name); 5777 } 5778 5779 static void 5780 _nvme_ctrlr_destruct(void *ctx) 5781 { 5782 struct nvme_ctrlr *nvme_ctrlr = ctx; 5783 5784 nvme_ctrlr_depopulate_namespaces(nvme_ctrlr); 5785 nvme_ctrlr_release(nvme_ctrlr); 5786 } 5787 5788 static int 5789 bdev_nvme_delete_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 5790 { 5791 struct nvme_probe_skip_entry *entry; 5792 5793 /* The controller's destruction was already started */ 5794 if (nvme_ctrlr->destruct) { 5795 return -EALREADY; 5796 } 5797 5798 if (!hotplug && 5799 nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 5800 entry = calloc(1, sizeof(*entry)); 5801 if (!entry) { 5802 return -ENOMEM; 5803 } 5804 entry->trid = nvme_ctrlr->active_path_id->trid; 5805 TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq); 5806 } 5807 5808 nvme_ctrlr->destruct = true; 5809 return 0; 5810 } 5811 5812 static int 5813 bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 5814 { 5815 int rc; 5816 5817 pthread_mutex_lock(&nvme_ctrlr->mutex); 5818 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, hotplug); 5819 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5820 5821 if (rc == 0) { 5822 _nvme_ctrlr_destruct(nvme_ctrlr); 5823 } else if (rc == -EALREADY) { 5824 rc = 0; 5825 } 5826 5827 return rc; 5828 } 5829 5830 static void 5831 remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr) 5832 { 5833 struct nvme_ctrlr *nvme_ctrlr = cb_ctx; 5834 5835 bdev_nvme_delete_ctrlr(nvme_ctrlr, true); 5836 } 5837 5838 static int 5839 bdev_nvme_hotplug_probe(void *arg) 5840 { 5841 if (g_hotplug_probe_ctx == NULL) { 5842 spdk_poller_unregister(&g_hotplug_probe_poller); 5843 return SPDK_POLLER_IDLE; 5844 } 5845 5846 if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) { 5847 g_hotplug_probe_ctx = NULL; 5848 spdk_poller_unregister(&g_hotplug_probe_poller); 5849 } 5850 5851 return SPDK_POLLER_BUSY; 5852 } 5853 5854 static int 5855 bdev_nvme_hotplug(void *arg) 5856 { 5857 struct spdk_nvme_transport_id trid_pcie; 5858 5859 if (g_hotplug_probe_ctx) { 5860 return SPDK_POLLER_BUSY; 5861 } 5862 5863 memset(&trid_pcie, 0, sizeof(trid_pcie)); 5864 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 5865 5866 g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL, 5867 hotplug_probe_cb, attach_cb, NULL); 5868 5869 if (g_hotplug_probe_ctx) { 5870 assert(g_hotplug_probe_poller == NULL); 5871 g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000); 5872 } 5873 5874 return SPDK_POLLER_BUSY; 5875 } 5876 5877 void 5878 bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts) 5879 { 5880 *opts = g_opts; 5881 } 5882 5883 static bool bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 5884 uint32_t reconnect_delay_sec, 5885 uint32_t fast_io_fail_timeout_sec); 5886 5887 static int 5888 bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts) 5889 { 5890 if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) { 5891 /* Can't set timeout_admin_us without also setting timeout_us */ 5892 SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n"); 5893 return -EINVAL; 5894 } 5895 5896 if (opts->bdev_retry_count < -1) { 5897 SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n"); 5898 return -EINVAL; 5899 } 5900 5901 if (!bdev_nvme_check_io_error_resiliency_params(opts->ctrlr_loss_timeout_sec, 5902 opts->reconnect_delay_sec, 5903 opts->fast_io_fail_timeout_sec)) { 5904 return -EINVAL; 5905 } 5906 5907 return 0; 5908 } 5909 5910 int 5911 bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) 5912 { 5913 int ret; 5914 5915 ret = bdev_nvme_validate_opts(opts); 5916 if (ret) { 5917 SPDK_WARNLOG("Failed to set nvme opts.\n"); 5918 return ret; 5919 } 5920 5921 if (g_bdev_nvme_init_thread != NULL) { 5922 if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 5923 return -EPERM; 5924 } 5925 } 5926 5927 if (opts->rdma_srq_size != 0 || 5928 opts->rdma_max_cq_size != 0 || 5929 opts->rdma_cm_event_timeout_ms != 0) { 5930 struct spdk_nvme_transport_opts drv_opts; 5931 5932 spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts)); 5933 if (opts->rdma_srq_size != 0) { 5934 drv_opts.rdma_srq_size = opts->rdma_srq_size; 5935 } 5936 if (opts->rdma_max_cq_size != 0) { 5937 drv_opts.rdma_max_cq_size = opts->rdma_max_cq_size; 5938 } 5939 if (opts->rdma_cm_event_timeout_ms != 0) { 5940 drv_opts.rdma_cm_event_timeout_ms = opts->rdma_cm_event_timeout_ms; 5941 } 5942 5943 ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts)); 5944 if (ret) { 5945 SPDK_ERRLOG("Failed to set NVMe transport opts.\n"); 5946 return ret; 5947 } 5948 } 5949 5950 g_opts = *opts; 5951 5952 return 0; 5953 } 5954 5955 struct set_nvme_hotplug_ctx { 5956 uint64_t period_us; 5957 bool enabled; 5958 spdk_msg_fn fn; 5959 void *fn_ctx; 5960 }; 5961 5962 static void 5963 set_nvme_hotplug_period_cb(void *_ctx) 5964 { 5965 struct set_nvme_hotplug_ctx *ctx = _ctx; 5966 5967 spdk_poller_unregister(&g_hotplug_poller); 5968 if (ctx->enabled) { 5969 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us); 5970 } else { 5971 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL, 5972 NVME_HOTPLUG_POLL_PERIOD_DEFAULT); 5973 } 5974 5975 g_nvme_hotplug_poll_period_us = ctx->period_us; 5976 g_nvme_hotplug_enabled = ctx->enabled; 5977 if (ctx->fn) { 5978 ctx->fn(ctx->fn_ctx); 5979 } 5980 5981 free(ctx); 5982 } 5983 5984 int 5985 bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx) 5986 { 5987 struct set_nvme_hotplug_ctx *ctx; 5988 5989 if (enabled == true && !spdk_process_is_primary()) { 5990 return -EPERM; 5991 } 5992 5993 ctx = calloc(1, sizeof(*ctx)); 5994 if (ctx == NULL) { 5995 return -ENOMEM; 5996 } 5997 5998 period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us; 5999 ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX); 6000 ctx->enabled = enabled; 6001 ctx->fn = cb; 6002 ctx->fn_ctx = cb_ctx; 6003 6004 spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx); 6005 return 0; 6006 } 6007 6008 static void 6009 nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 6010 struct nvme_async_probe_ctx *ctx) 6011 { 6012 struct nvme_ns *nvme_ns; 6013 struct nvme_bdev *nvme_bdev; 6014 size_t j; 6015 6016 assert(nvme_ctrlr != NULL); 6017 6018 if (ctx->names == NULL) { 6019 ctx->reported_bdevs = 0; 6020 populate_namespaces_cb(ctx, 0); 6021 return; 6022 } 6023 6024 /* 6025 * Report the new bdevs that were created in this call. 6026 * There can be more than one bdev per NVMe controller. 6027 */ 6028 j = 0; 6029 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 6030 while (nvme_ns != NULL) { 6031 nvme_bdev = nvme_ns->bdev; 6032 if (j < ctx->max_bdevs) { 6033 ctx->names[j] = nvme_bdev->disk.name; 6034 j++; 6035 } else { 6036 SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n", 6037 ctx->max_bdevs); 6038 ctx->reported_bdevs = 0; 6039 populate_namespaces_cb(ctx, -ERANGE); 6040 return; 6041 } 6042 6043 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 6044 } 6045 6046 ctx->reported_bdevs = j; 6047 populate_namespaces_cb(ctx, 0); 6048 } 6049 6050 static int 6051 bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 6052 struct spdk_nvme_ctrlr *new_ctrlr, 6053 struct spdk_nvme_transport_id *trid) 6054 { 6055 struct nvme_path_id *tmp_trid; 6056 6057 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 6058 SPDK_ERRLOG("PCIe failover is not supported.\n"); 6059 return -ENOTSUP; 6060 } 6061 6062 /* Currently we only support failover to the same transport type. */ 6063 if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) { 6064 SPDK_WARNLOG("Failover from trtype: %s to a different trtype: %s is not supported currently\n", 6065 spdk_nvme_transport_id_trtype_str(nvme_ctrlr->active_path_id->trid.trtype), 6066 spdk_nvme_transport_id_trtype_str(trid->trtype)); 6067 return -EINVAL; 6068 } 6069 6070 6071 /* Currently we only support failover to the same NQN. */ 6072 if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) { 6073 SPDK_WARNLOG("Failover from subnqn: %s to a different subnqn: %s is not supported currently\n", 6074 nvme_ctrlr->active_path_id->trid.subnqn, trid->subnqn); 6075 return -EINVAL; 6076 } 6077 6078 /* Skip all the other checks if we've already registered this path. */ 6079 TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) { 6080 if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) { 6081 SPDK_WARNLOG("This path (traddr: %s subnqn: %s) is already registered\n", trid->traddr, 6082 trid->subnqn); 6083 return -EALREADY; 6084 } 6085 } 6086 6087 return 0; 6088 } 6089 6090 static int 6091 bdev_nvme_check_secondary_namespace(struct nvme_ctrlr *nvme_ctrlr, 6092 struct spdk_nvme_ctrlr *new_ctrlr) 6093 { 6094 struct nvme_ns *nvme_ns; 6095 struct spdk_nvme_ns *new_ns; 6096 6097 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 6098 while (nvme_ns != NULL) { 6099 new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id); 6100 assert(new_ns != NULL); 6101 6102 if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) { 6103 return -EINVAL; 6104 } 6105 6106 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 6107 } 6108 6109 return 0; 6110 } 6111 6112 static int 6113 _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 6114 struct spdk_nvme_transport_id *trid) 6115 { 6116 struct nvme_path_id *active_id, *new_trid, *tmp_trid; 6117 6118 new_trid = calloc(1, sizeof(*new_trid)); 6119 if (new_trid == NULL) { 6120 return -ENOMEM; 6121 } 6122 new_trid->trid = *trid; 6123 6124 active_id = nvme_ctrlr->active_path_id; 6125 assert(active_id != NULL); 6126 assert(active_id == TAILQ_FIRST(&nvme_ctrlr->trids)); 6127 6128 /* Skip the active trid not to replace it until it is failed. */ 6129 tmp_trid = TAILQ_NEXT(active_id, link); 6130 if (tmp_trid == NULL) { 6131 goto add_tail; 6132 } 6133 6134 /* It means the trid is faled if its last failed time is non-zero. 6135 * Insert the new alternate trid before any failed trid. 6136 */ 6137 TAILQ_FOREACH_FROM(tmp_trid, &nvme_ctrlr->trids, link) { 6138 if (tmp_trid->last_failed_tsc != 0) { 6139 TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link); 6140 return 0; 6141 } 6142 } 6143 6144 add_tail: 6145 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link); 6146 return 0; 6147 } 6148 6149 /* This is the case that a secondary path is added to an existing 6150 * nvme_ctrlr for failover. After checking if it can access the same 6151 * namespaces as the primary path, it is disconnected until failover occurs. 6152 */ 6153 static int 6154 bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 6155 struct spdk_nvme_ctrlr *new_ctrlr, 6156 struct spdk_nvme_transport_id *trid) 6157 { 6158 int rc; 6159 6160 assert(nvme_ctrlr != NULL); 6161 6162 pthread_mutex_lock(&nvme_ctrlr->mutex); 6163 6164 rc = bdev_nvme_check_secondary_trid(nvme_ctrlr, new_ctrlr, trid); 6165 if (rc != 0) { 6166 goto exit; 6167 } 6168 6169 rc = bdev_nvme_check_secondary_namespace(nvme_ctrlr, new_ctrlr); 6170 if (rc != 0) { 6171 goto exit; 6172 } 6173 6174 rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid); 6175 6176 exit: 6177 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6178 6179 spdk_nvme_detach(new_ctrlr); 6180 6181 return rc; 6182 } 6183 6184 static void 6185 connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 6186 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 6187 { 6188 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 6189 struct nvme_async_probe_ctx *ctx; 6190 int rc; 6191 6192 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 6193 ctx->ctrlr_attached = true; 6194 6195 rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx); 6196 if (rc != 0) { 6197 ctx->reported_bdevs = 0; 6198 populate_namespaces_cb(ctx, rc); 6199 } 6200 } 6201 6202 SPDK_LOG_DEPRECATION_REGISTER(failover_config, 6203 "bdev_nvme_attach_controller.failover configuration mismatch", "v25.01", 0); 6204 6205 static void 6206 connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 6207 struct spdk_nvme_ctrlr *ctrlr, 6208 const struct spdk_nvme_ctrlr_opts *opts) 6209 { 6210 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 6211 struct nvme_ctrlr *nvme_ctrlr; 6212 struct nvme_async_probe_ctx *ctx; 6213 int rc; 6214 6215 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 6216 ctx->ctrlr_attached = true; 6217 6218 nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name); 6219 if (nvme_ctrlr) { 6220 if (nvme_ctrlr->opts.multipath) { 6221 /* All controllers created with the same name must be configured either 6222 * for multipath or for failover. Otherwise we have configuration mismatch. 6223 * While this is currently still supported, support for configuration where some 6224 * controllers with the same name are configured for multipath, while others 6225 * are configured for failover will be removed in release 25.01. 6226 * Default mode change: starting from SPDK 25.01, if the user will not provide 6227 * '-x <mode>' parameter in the bdev_nvme_attach_controller RPC call, default 6228 * mode assigned to the controller will be 'multipath' 6229 */ 6230 SPDK_LOG_DEPRECATED(failover_config); 6231 } 6232 rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid); 6233 } else { 6234 rc = -ENODEV; 6235 } 6236 6237 ctx->reported_bdevs = 0; 6238 populate_namespaces_cb(ctx, rc); 6239 } 6240 6241 static int 6242 bdev_nvme_async_poll(void *arg) 6243 { 6244 struct nvme_async_probe_ctx *ctx = arg; 6245 int rc; 6246 6247 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 6248 if (spdk_unlikely(rc != -EAGAIN)) { 6249 ctx->probe_done = true; 6250 spdk_poller_unregister(&ctx->poller); 6251 if (!ctx->ctrlr_attached) { 6252 /* The probe is done, but no controller was attached. 6253 * That means we had a failure, so report -EIO back to 6254 * the caller (usually the RPC). populate_namespaces_cb() 6255 * will take care of freeing the nvme_async_probe_ctx. 6256 */ 6257 ctx->reported_bdevs = 0; 6258 populate_namespaces_cb(ctx, -EIO); 6259 } else if (ctx->namespaces_populated) { 6260 /* The namespaces for the attached controller were all 6261 * populated and the response was already sent to the 6262 * caller (usually the RPC). So free the context here. 6263 */ 6264 free_nvme_async_probe_ctx(ctx); 6265 } 6266 } 6267 6268 return SPDK_POLLER_BUSY; 6269 } 6270 6271 static bool 6272 bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 6273 uint32_t reconnect_delay_sec, 6274 uint32_t fast_io_fail_timeout_sec) 6275 { 6276 if (ctrlr_loss_timeout_sec < -1) { 6277 SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n"); 6278 return false; 6279 } else if (ctrlr_loss_timeout_sec == -1) { 6280 if (reconnect_delay_sec == 0) { 6281 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 6282 return false; 6283 } else if (fast_io_fail_timeout_sec != 0 && 6284 fast_io_fail_timeout_sec < reconnect_delay_sec) { 6285 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n"); 6286 return false; 6287 } 6288 } else if (ctrlr_loss_timeout_sec != 0) { 6289 if (reconnect_delay_sec == 0) { 6290 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 6291 return false; 6292 } else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) { 6293 SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n"); 6294 return false; 6295 } else if (fast_io_fail_timeout_sec != 0) { 6296 if (fast_io_fail_timeout_sec < reconnect_delay_sec) { 6297 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n"); 6298 return false; 6299 } else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) { 6300 SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n"); 6301 return false; 6302 } 6303 } 6304 } else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) { 6305 SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n"); 6306 return false; 6307 } 6308 6309 return true; 6310 } 6311 6312 int 6313 spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid, 6314 const char *base_name, 6315 const char **names, 6316 uint32_t count, 6317 spdk_bdev_nvme_create_cb cb_fn, 6318 void *cb_ctx, 6319 struct spdk_nvme_ctrlr_opts *drv_opts, 6320 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, 6321 bool multipath) 6322 { 6323 struct nvme_probe_skip_entry *entry, *tmp; 6324 struct nvme_async_probe_ctx *ctx; 6325 spdk_nvme_attach_cb attach_cb; 6326 int len; 6327 6328 /* TODO expand this check to include both the host and target TRIDs. 6329 * Only if both are the same should we fail. 6330 */ 6331 if (nvme_ctrlr_get(trid, drv_opts->hostnqn) != NULL) { 6332 SPDK_ERRLOG("A controller with the provided trid (traddr: %s, hostnqn: %s) " 6333 "already exists.\n", trid->traddr, drv_opts->hostnqn); 6334 return -EEXIST; 6335 } 6336 6337 len = strnlen(base_name, SPDK_CONTROLLER_NAME_MAX); 6338 6339 if (len == 0 || len == SPDK_CONTROLLER_NAME_MAX) { 6340 SPDK_ERRLOG("controller name must be between 1 and %d characters\n", SPDK_CONTROLLER_NAME_MAX - 1); 6341 return -EINVAL; 6342 } 6343 6344 if (bdev_opts != NULL && 6345 !bdev_nvme_check_io_error_resiliency_params(bdev_opts->ctrlr_loss_timeout_sec, 6346 bdev_opts->reconnect_delay_sec, 6347 bdev_opts->fast_io_fail_timeout_sec)) { 6348 return -EINVAL; 6349 } 6350 6351 ctx = calloc(1, sizeof(*ctx)); 6352 if (!ctx) { 6353 return -ENOMEM; 6354 } 6355 ctx->base_name = base_name; 6356 ctx->names = names; 6357 ctx->max_bdevs = count; 6358 ctx->cb_fn = cb_fn; 6359 ctx->cb_ctx = cb_ctx; 6360 ctx->trid = *trid; 6361 6362 if (bdev_opts) { 6363 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 6364 } else { 6365 spdk_bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts); 6366 } 6367 6368 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 6369 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) { 6370 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 6371 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 6372 free(entry); 6373 break; 6374 } 6375 } 6376 } 6377 6378 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 6379 ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count; 6380 ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout; 6381 ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms; 6382 ctx->drv_opts.disable_read_ana_log_page = true; 6383 ctx->drv_opts.transport_tos = g_opts.transport_tos; 6384 6385 if (ctx->bdev_opts.psk != NULL) { 6386 ctx->drv_opts.tls_psk = spdk_keyring_get_key(ctx->bdev_opts.psk); 6387 if (ctx->drv_opts.tls_psk == NULL) { 6388 SPDK_ERRLOG("Could not load PSK: %s\n", ctx->bdev_opts.psk); 6389 free_nvme_async_probe_ctx(ctx); 6390 return -ENOKEY; 6391 } 6392 } 6393 6394 if (ctx->bdev_opts.dhchap_key != NULL) { 6395 ctx->drv_opts.dhchap_key = spdk_keyring_get_key(ctx->bdev_opts.dhchap_key); 6396 if (ctx->drv_opts.dhchap_key == NULL) { 6397 SPDK_ERRLOG("Could not load DH-HMAC-CHAP key: %s\n", 6398 ctx->bdev_opts.dhchap_key); 6399 free_nvme_async_probe_ctx(ctx); 6400 return -ENOKEY; 6401 } 6402 6403 ctx->drv_opts.dhchap_digests = g_opts.dhchap_digests; 6404 ctx->drv_opts.dhchap_dhgroups = g_opts.dhchap_dhgroups; 6405 } 6406 if (ctx->bdev_opts.dhchap_ctrlr_key != NULL) { 6407 ctx->drv_opts.dhchap_ctrlr_key = 6408 spdk_keyring_get_key(ctx->bdev_opts.dhchap_ctrlr_key); 6409 if (ctx->drv_opts.dhchap_ctrlr_key == NULL) { 6410 SPDK_ERRLOG("Could not load DH-HMAC-CHAP controller key: %s\n", 6411 ctx->bdev_opts.dhchap_ctrlr_key); 6412 free_nvme_async_probe_ctx(ctx); 6413 return -ENOKEY; 6414 } 6415 } 6416 6417 if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || multipath) { 6418 attach_cb = connect_attach_cb; 6419 } else { 6420 attach_cb = connect_set_failover_cb; 6421 } 6422 6423 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb); 6424 if (ctx->probe_ctx == NULL) { 6425 SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr); 6426 free_nvme_async_probe_ctx(ctx); 6427 return -ENODEV; 6428 } 6429 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000); 6430 6431 return 0; 6432 } 6433 6434 struct bdev_nvme_delete_ctx { 6435 char *name; 6436 struct nvme_path_id path_id; 6437 bdev_nvme_delete_done_fn delete_done; 6438 void *delete_done_ctx; 6439 uint64_t timeout_ticks; 6440 struct spdk_poller *poller; 6441 }; 6442 6443 static void 6444 free_bdev_nvme_delete_ctx(struct bdev_nvme_delete_ctx *ctx) 6445 { 6446 if (ctx != NULL) { 6447 free(ctx->name); 6448 free(ctx); 6449 } 6450 } 6451 6452 static bool 6453 nvme_path_id_compare(struct nvme_path_id *p, const struct nvme_path_id *path_id) 6454 { 6455 if (path_id->trid.trtype != 0) { 6456 if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) { 6457 if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) { 6458 return false; 6459 } 6460 } else { 6461 if (path_id->trid.trtype != p->trid.trtype) { 6462 return false; 6463 } 6464 } 6465 } 6466 6467 if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) { 6468 if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) { 6469 return false; 6470 } 6471 } 6472 6473 if (path_id->trid.adrfam != 0) { 6474 if (path_id->trid.adrfam != p->trid.adrfam) { 6475 return false; 6476 } 6477 } 6478 6479 if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) { 6480 if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) { 6481 return false; 6482 } 6483 } 6484 6485 if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) { 6486 if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) { 6487 return false; 6488 } 6489 } 6490 6491 if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) { 6492 if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) { 6493 return false; 6494 } 6495 } 6496 6497 if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) { 6498 if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) { 6499 return false; 6500 } 6501 } 6502 6503 return true; 6504 } 6505 6506 static bool 6507 nvme_path_id_exists(const char *name, const struct nvme_path_id *path_id) 6508 { 6509 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6510 struct nvme_ctrlr *ctrlr; 6511 struct nvme_path_id *p; 6512 6513 pthread_mutex_lock(&g_bdev_nvme_mutex); 6514 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 6515 if (!nbdev_ctrlr) { 6516 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6517 return false; 6518 } 6519 6520 TAILQ_FOREACH(ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 6521 pthread_mutex_lock(&ctrlr->mutex); 6522 TAILQ_FOREACH(p, &ctrlr->trids, link) { 6523 if (nvme_path_id_compare(p, path_id)) { 6524 pthread_mutex_unlock(&ctrlr->mutex); 6525 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6526 return true; 6527 } 6528 } 6529 pthread_mutex_unlock(&ctrlr->mutex); 6530 } 6531 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6532 6533 return false; 6534 } 6535 6536 static int 6537 bdev_nvme_delete_complete_poll(void *arg) 6538 { 6539 struct bdev_nvme_delete_ctx *ctx = arg; 6540 int rc = 0; 6541 6542 if (nvme_path_id_exists(ctx->name, &ctx->path_id)) { 6543 if (ctx->timeout_ticks > spdk_get_ticks()) { 6544 return SPDK_POLLER_BUSY; 6545 } 6546 6547 SPDK_ERRLOG("NVMe path '%s' still exists after delete\n", ctx->name); 6548 rc = -ETIMEDOUT; 6549 } 6550 6551 spdk_poller_unregister(&ctx->poller); 6552 6553 ctx->delete_done(ctx->delete_done_ctx, rc); 6554 free_bdev_nvme_delete_ctx(ctx); 6555 6556 return SPDK_POLLER_BUSY; 6557 } 6558 6559 static int 6560 _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id) 6561 { 6562 struct nvme_path_id *p, *t; 6563 spdk_msg_fn msg_fn; 6564 int rc = -ENXIO; 6565 6566 pthread_mutex_lock(&nvme_ctrlr->mutex); 6567 6568 TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) { 6569 if (p == TAILQ_FIRST(&nvme_ctrlr->trids)) { 6570 break; 6571 } 6572 6573 if (!nvme_path_id_compare(p, path_id)) { 6574 continue; 6575 } 6576 6577 /* We are not using the specified path. */ 6578 TAILQ_REMOVE(&nvme_ctrlr->trids, p, link); 6579 free(p); 6580 rc = 0; 6581 } 6582 6583 if (p == NULL || !nvme_path_id_compare(p, path_id)) { 6584 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6585 return rc; 6586 } 6587 6588 /* If we made it here, then this path is a match! Now we need to remove it. */ 6589 6590 /* This is the active path in use right now. The active path is always the first in the list. */ 6591 assert(p == nvme_ctrlr->active_path_id); 6592 6593 if (!TAILQ_NEXT(p, link)) { 6594 /* The current path is the only path. */ 6595 msg_fn = _nvme_ctrlr_destruct; 6596 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, false); 6597 } else { 6598 /* There is an alternative path. */ 6599 msg_fn = _bdev_nvme_reset_ctrlr; 6600 rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, true); 6601 } 6602 6603 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6604 6605 if (rc == 0) { 6606 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 6607 } else if (rc == -EALREADY) { 6608 rc = 0; 6609 } 6610 6611 return rc; 6612 } 6613 6614 int 6615 bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id, 6616 bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx) 6617 { 6618 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6619 struct nvme_ctrlr *nvme_ctrlr, *tmp_nvme_ctrlr; 6620 struct bdev_nvme_delete_ctx *ctx = NULL; 6621 int rc = -ENXIO, _rc; 6622 6623 if (name == NULL || path_id == NULL) { 6624 rc = -EINVAL; 6625 goto exit; 6626 } 6627 6628 pthread_mutex_lock(&g_bdev_nvme_mutex); 6629 6630 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 6631 if (nbdev_ctrlr == NULL) { 6632 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6633 6634 SPDK_ERRLOG("Failed to find NVMe bdev controller\n"); 6635 rc = -ENODEV; 6636 goto exit; 6637 } 6638 6639 TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) { 6640 _rc = _bdev_nvme_delete(nvme_ctrlr, path_id); 6641 if (_rc < 0 && _rc != -ENXIO) { 6642 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6643 rc = _rc; 6644 goto exit; 6645 } else if (_rc == 0) { 6646 /* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr 6647 * was deleted successfully. To remember the successful deletion, 6648 * overwrite rc only if _rc is zero. 6649 */ 6650 rc = 0; 6651 } 6652 } 6653 6654 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6655 6656 if (rc != 0 || delete_done == NULL) { 6657 goto exit; 6658 } 6659 6660 ctx = calloc(1, sizeof(*ctx)); 6661 if (ctx == NULL) { 6662 SPDK_ERRLOG("Failed to allocate context for bdev_nvme_delete\n"); 6663 rc = -ENOMEM; 6664 goto exit; 6665 } 6666 6667 ctx->name = strdup(name); 6668 if (ctx->name == NULL) { 6669 SPDK_ERRLOG("Failed to copy controller name for deletion\n"); 6670 rc = -ENOMEM; 6671 goto exit; 6672 } 6673 6674 ctx->delete_done = delete_done; 6675 ctx->delete_done_ctx = delete_done_ctx; 6676 ctx->path_id = *path_id; 6677 ctx->timeout_ticks = spdk_get_ticks() + 10 * spdk_get_ticks_hz(); 6678 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_delete_complete_poll, ctx, 1000); 6679 if (ctx->poller == NULL) { 6680 SPDK_ERRLOG("Failed to register bdev_nvme_delete poller\n"); 6681 rc = -ENOMEM; 6682 goto exit; 6683 } 6684 6685 exit: 6686 if (rc != 0) { 6687 free_bdev_nvme_delete_ctx(ctx); 6688 } 6689 6690 return rc; 6691 } 6692 6693 #define DISCOVERY_INFOLOG(ctx, format, ...) \ 6694 SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 6695 6696 #define DISCOVERY_ERRLOG(ctx, format, ...) \ 6697 SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 6698 6699 struct discovery_entry_ctx { 6700 char name[128]; 6701 struct spdk_nvme_transport_id trid; 6702 struct spdk_nvme_ctrlr_opts drv_opts; 6703 struct spdk_nvmf_discovery_log_page_entry entry; 6704 TAILQ_ENTRY(discovery_entry_ctx) tailq; 6705 struct discovery_ctx *ctx; 6706 }; 6707 6708 struct discovery_ctx { 6709 char *name; 6710 spdk_bdev_nvme_start_discovery_fn start_cb_fn; 6711 spdk_bdev_nvme_stop_discovery_fn stop_cb_fn; 6712 void *cb_ctx; 6713 struct spdk_nvme_probe_ctx *probe_ctx; 6714 struct spdk_nvme_detach_ctx *detach_ctx; 6715 struct spdk_nvme_ctrlr *ctrlr; 6716 struct spdk_nvme_transport_id trid; 6717 struct discovery_entry_ctx *entry_ctx_in_use; 6718 struct spdk_poller *poller; 6719 struct spdk_nvme_ctrlr_opts drv_opts; 6720 struct spdk_bdev_nvme_ctrlr_opts bdev_opts; 6721 struct spdk_nvmf_discovery_log_page *log_page; 6722 TAILQ_ENTRY(discovery_ctx) tailq; 6723 TAILQ_HEAD(, discovery_entry_ctx) nvm_entry_ctxs; 6724 TAILQ_HEAD(, discovery_entry_ctx) discovery_entry_ctxs; 6725 int rc; 6726 bool wait_for_attach; 6727 uint64_t timeout_ticks; 6728 /* Denotes that the discovery service is being started. We're waiting 6729 * for the initial connection to the discovery controller to be 6730 * established and attach discovered NVM ctrlrs. 6731 */ 6732 bool initializing; 6733 /* Denotes if a discovery is currently in progress for this context. 6734 * That includes connecting to newly discovered subsystems. Used to 6735 * ensure we do not start a new discovery until an existing one is 6736 * complete. 6737 */ 6738 bool in_progress; 6739 6740 /* Denotes if another discovery is needed after the one in progress 6741 * completes. Set when we receive an AER completion while a discovery 6742 * is already in progress. 6743 */ 6744 bool pending; 6745 6746 /* Signal to the discovery context poller that it should stop the 6747 * discovery service, including detaching from the current discovery 6748 * controller. 6749 */ 6750 bool stop; 6751 6752 struct spdk_thread *calling_thread; 6753 uint32_t index; 6754 uint32_t attach_in_progress; 6755 char *hostnqn; 6756 6757 /* Denotes if the discovery service was started by the mdns discovery. 6758 */ 6759 bool from_mdns_discovery_service; 6760 }; 6761 6762 TAILQ_HEAD(discovery_ctxs, discovery_ctx); 6763 static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs); 6764 6765 static void get_discovery_log_page(struct discovery_ctx *ctx); 6766 6767 static void 6768 free_discovery_ctx(struct discovery_ctx *ctx) 6769 { 6770 free(ctx->log_page); 6771 free(ctx->hostnqn); 6772 free(ctx->name); 6773 free(ctx); 6774 } 6775 6776 static void 6777 discovery_complete(struct discovery_ctx *ctx) 6778 { 6779 ctx->initializing = false; 6780 ctx->in_progress = false; 6781 if (ctx->pending) { 6782 ctx->pending = false; 6783 get_discovery_log_page(ctx); 6784 } 6785 } 6786 6787 static void 6788 build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid, 6789 struct spdk_nvmf_discovery_log_page_entry *entry) 6790 { 6791 char *space; 6792 6793 trid->trtype = entry->trtype; 6794 trid->adrfam = entry->adrfam; 6795 memcpy(trid->traddr, entry->traddr, sizeof(entry->traddr)); 6796 memcpy(trid->trsvcid, entry->trsvcid, sizeof(entry->trsvcid)); 6797 /* Because the source buffer (entry->subnqn) is longer than trid->subnqn, and 6798 * before call to this function trid->subnqn is zeroed out, we need 6799 * to copy sizeof(trid->subnqn) minus one byte to make sure the last character 6800 * remains 0. Then we can shorten the string (replace ' ' with 0) if required 6801 */ 6802 memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn) - 1); 6803 6804 /* We want the traddr, trsvcid and subnqn fields to be NULL-terminated. 6805 * But the log page entries typically pad them with spaces, not zeroes. 6806 * So add a NULL terminator to each of these fields at the appropriate 6807 * location. 6808 */ 6809 space = strchr(trid->traddr, ' '); 6810 if (space) { 6811 *space = 0; 6812 } 6813 space = strchr(trid->trsvcid, ' '); 6814 if (space) { 6815 *space = 0; 6816 } 6817 space = strchr(trid->subnqn, ' '); 6818 if (space) { 6819 *space = 0; 6820 } 6821 } 6822 6823 static void 6824 _stop_discovery(void *_ctx) 6825 { 6826 struct discovery_ctx *ctx = _ctx; 6827 6828 if (ctx->attach_in_progress > 0) { 6829 spdk_thread_send_msg(spdk_get_thread(), _stop_discovery, ctx); 6830 return; 6831 } 6832 6833 ctx->stop = true; 6834 6835 while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) { 6836 struct discovery_entry_ctx *entry_ctx; 6837 struct nvme_path_id path = {}; 6838 6839 entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs); 6840 path.trid = entry_ctx->trid; 6841 bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL); 6842 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 6843 free(entry_ctx); 6844 } 6845 6846 while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) { 6847 struct discovery_entry_ctx *entry_ctx; 6848 6849 entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 6850 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 6851 free(entry_ctx); 6852 } 6853 6854 free(ctx->entry_ctx_in_use); 6855 ctx->entry_ctx_in_use = NULL; 6856 } 6857 6858 static void 6859 stop_discovery(struct discovery_ctx *ctx, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 6860 { 6861 ctx->stop_cb_fn = cb_fn; 6862 ctx->cb_ctx = cb_ctx; 6863 6864 if (ctx->attach_in_progress > 0) { 6865 DISCOVERY_INFOLOG(ctx, "stopping discovery with attach_in_progress: %"PRIu32"\n", 6866 ctx->attach_in_progress); 6867 } 6868 6869 _stop_discovery(ctx); 6870 } 6871 6872 static void 6873 remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr) 6874 { 6875 struct discovery_ctx *d_ctx; 6876 struct nvme_path_id *path_id; 6877 struct spdk_nvme_transport_id trid = {}; 6878 struct discovery_entry_ctx *entry_ctx, *tmp; 6879 6880 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 6881 6882 TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) { 6883 TAILQ_FOREACH_SAFE(entry_ctx, &d_ctx->nvm_entry_ctxs, tailq, tmp) { 6884 build_trid_from_log_page_entry(&trid, &entry_ctx->entry); 6885 if (spdk_nvme_transport_id_compare(&trid, &path_id->trid) != 0) { 6886 continue; 6887 } 6888 6889 TAILQ_REMOVE(&d_ctx->nvm_entry_ctxs, entry_ctx, tailq); 6890 free(entry_ctx); 6891 DISCOVERY_INFOLOG(d_ctx, "Remove discovery entry: %s:%s:%s\n", 6892 trid.subnqn, trid.traddr, trid.trsvcid); 6893 6894 /* Fail discovery ctrlr to force reattach attempt */ 6895 spdk_nvme_ctrlr_fail(d_ctx->ctrlr); 6896 } 6897 } 6898 } 6899 6900 static void 6901 discovery_remove_controllers(struct discovery_ctx *ctx) 6902 { 6903 struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page; 6904 struct discovery_entry_ctx *entry_ctx, *tmp; 6905 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 6906 struct spdk_nvme_transport_id old_trid = {}; 6907 uint64_t numrec, i; 6908 bool found; 6909 6910 numrec = from_le64(&log_page->numrec); 6911 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) { 6912 found = false; 6913 old_entry = &entry_ctx->entry; 6914 build_trid_from_log_page_entry(&old_trid, old_entry); 6915 for (i = 0; i < numrec; i++) { 6916 new_entry = &log_page->entries[i]; 6917 if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) { 6918 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n", 6919 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 6920 found = true; 6921 break; 6922 } 6923 } 6924 if (!found) { 6925 struct nvme_path_id path = {}; 6926 6927 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n", 6928 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 6929 6930 path.trid = entry_ctx->trid; 6931 bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL); 6932 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 6933 free(entry_ctx); 6934 } 6935 } 6936 free(log_page); 6937 ctx->log_page = NULL; 6938 discovery_complete(ctx); 6939 } 6940 6941 static void 6942 complete_discovery_start(struct discovery_ctx *ctx, int status) 6943 { 6944 ctx->timeout_ticks = 0; 6945 ctx->rc = status; 6946 if (ctx->start_cb_fn) { 6947 ctx->start_cb_fn(ctx->cb_ctx, status); 6948 ctx->start_cb_fn = NULL; 6949 ctx->cb_ctx = NULL; 6950 } 6951 } 6952 6953 static void 6954 discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc) 6955 { 6956 struct discovery_entry_ctx *entry_ctx = cb_ctx; 6957 struct discovery_ctx *ctx = entry_ctx->ctx; 6958 6959 DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name); 6960 ctx->attach_in_progress--; 6961 if (ctx->attach_in_progress == 0) { 6962 complete_discovery_start(ctx, ctx->rc); 6963 if (ctx->initializing && ctx->rc != 0) { 6964 DISCOVERY_ERRLOG(ctx, "stopping discovery due to errors: %d\n", ctx->rc); 6965 stop_discovery(ctx, NULL, ctx->cb_ctx); 6966 } else { 6967 discovery_remove_controllers(ctx); 6968 } 6969 } 6970 } 6971 6972 static struct discovery_entry_ctx * 6973 create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid) 6974 { 6975 struct discovery_entry_ctx *new_ctx; 6976 6977 new_ctx = calloc(1, sizeof(*new_ctx)); 6978 if (new_ctx == NULL) { 6979 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 6980 return NULL; 6981 } 6982 6983 new_ctx->ctx = ctx; 6984 memcpy(&new_ctx->trid, trid, sizeof(*trid)); 6985 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 6986 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 6987 return new_ctx; 6988 } 6989 6990 static void 6991 discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl, 6992 struct spdk_nvmf_discovery_log_page *log_page) 6993 { 6994 struct discovery_ctx *ctx = cb_arg; 6995 struct discovery_entry_ctx *entry_ctx, *tmp; 6996 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 6997 uint64_t numrec, i; 6998 bool found; 6999 7000 if (rc || spdk_nvme_cpl_is_error(cpl)) { 7001 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 7002 return; 7003 } 7004 7005 ctx->log_page = log_page; 7006 assert(ctx->attach_in_progress == 0); 7007 numrec = from_le64(&log_page->numrec); 7008 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) { 7009 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 7010 free(entry_ctx); 7011 } 7012 for (i = 0; i < numrec; i++) { 7013 found = false; 7014 new_entry = &log_page->entries[i]; 7015 if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY_CURRENT || 7016 new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 7017 struct discovery_entry_ctx *new_ctx; 7018 struct spdk_nvme_transport_id trid = {}; 7019 7020 build_trid_from_log_page_entry(&trid, new_entry); 7021 new_ctx = create_discovery_entry_ctx(ctx, &trid); 7022 if (new_ctx == NULL) { 7023 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 7024 break; 7025 } 7026 7027 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq); 7028 continue; 7029 } 7030 TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) { 7031 old_entry = &entry_ctx->entry; 7032 if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) { 7033 found = true; 7034 break; 7035 } 7036 } 7037 if (!found) { 7038 struct discovery_entry_ctx *subnqn_ctx = NULL, *new_ctx; 7039 struct discovery_ctx *d_ctx; 7040 7041 TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) { 7042 TAILQ_FOREACH(subnqn_ctx, &d_ctx->nvm_entry_ctxs, tailq) { 7043 if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn, 7044 sizeof(new_entry->subnqn))) { 7045 break; 7046 } 7047 } 7048 if (subnqn_ctx) { 7049 break; 7050 } 7051 } 7052 7053 new_ctx = calloc(1, sizeof(*new_ctx)); 7054 if (new_ctx == NULL) { 7055 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 7056 break; 7057 } 7058 7059 new_ctx->ctx = ctx; 7060 memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry)); 7061 build_trid_from_log_page_entry(&new_ctx->trid, new_entry); 7062 if (subnqn_ctx) { 7063 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name); 7064 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n", 7065 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 7066 new_ctx->name); 7067 } else { 7068 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++); 7069 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n", 7070 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 7071 new_ctx->name); 7072 } 7073 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 7074 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 7075 rc = spdk_bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0, 7076 discovery_attach_controller_done, new_ctx, 7077 &new_ctx->drv_opts, &ctx->bdev_opts, true); 7078 if (rc == 0) { 7079 TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq); 7080 ctx->attach_in_progress++; 7081 } else { 7082 DISCOVERY_ERRLOG(ctx, "spdk_bdev_nvme_create failed (%s)\n", spdk_strerror(-rc)); 7083 } 7084 } 7085 } 7086 7087 if (ctx->attach_in_progress == 0) { 7088 discovery_remove_controllers(ctx); 7089 } 7090 } 7091 7092 static void 7093 get_discovery_log_page(struct discovery_ctx *ctx) 7094 { 7095 int rc; 7096 7097 assert(ctx->in_progress == false); 7098 ctx->in_progress = true; 7099 rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx); 7100 if (rc != 0) { 7101 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 7102 } 7103 DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n"); 7104 } 7105 7106 static void 7107 discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 7108 { 7109 struct discovery_ctx *ctx = arg; 7110 uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16; 7111 7112 if (spdk_nvme_cpl_is_error(cpl)) { 7113 DISCOVERY_ERRLOG(ctx, "aer failed\n"); 7114 return; 7115 } 7116 7117 if (log_page_id != SPDK_NVME_LOG_DISCOVERY) { 7118 DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id); 7119 return; 7120 } 7121 7122 DISCOVERY_INFOLOG(ctx, "got aer\n"); 7123 if (ctx->in_progress) { 7124 ctx->pending = true; 7125 return; 7126 } 7127 7128 get_discovery_log_page(ctx); 7129 } 7130 7131 static void 7132 discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 7133 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 7134 { 7135 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 7136 struct discovery_ctx *ctx; 7137 7138 ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts); 7139 7140 DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n"); 7141 ctx->probe_ctx = NULL; 7142 ctx->ctrlr = ctrlr; 7143 7144 if (ctx->rc != 0) { 7145 DISCOVERY_ERRLOG(ctx, "encountered error while attaching discovery ctrlr: %d\n", 7146 ctx->rc); 7147 return; 7148 } 7149 7150 spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx); 7151 } 7152 7153 static int 7154 discovery_poller(void *arg) 7155 { 7156 struct discovery_ctx *ctx = arg; 7157 struct spdk_nvme_transport_id *trid; 7158 int rc; 7159 7160 if (ctx->detach_ctx) { 7161 rc = spdk_nvme_detach_poll_async(ctx->detach_ctx); 7162 if (rc != -EAGAIN) { 7163 ctx->detach_ctx = NULL; 7164 ctx->ctrlr = NULL; 7165 } 7166 } else if (ctx->stop) { 7167 if (ctx->ctrlr != NULL) { 7168 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 7169 if (rc == 0) { 7170 return SPDK_POLLER_BUSY; 7171 } 7172 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 7173 } 7174 spdk_poller_unregister(&ctx->poller); 7175 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 7176 assert(ctx->start_cb_fn == NULL); 7177 if (ctx->stop_cb_fn != NULL) { 7178 ctx->stop_cb_fn(ctx->cb_ctx); 7179 } 7180 free_discovery_ctx(ctx); 7181 } else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) { 7182 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7183 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 7184 assert(ctx->initializing); 7185 spdk_poller_unregister(&ctx->poller); 7186 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 7187 complete_discovery_start(ctx, -ETIMEDOUT); 7188 stop_discovery(ctx, NULL, NULL); 7189 free_discovery_ctx(ctx); 7190 return SPDK_POLLER_BUSY; 7191 } 7192 7193 assert(ctx->entry_ctx_in_use == NULL); 7194 ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 7195 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7196 trid = &ctx->entry_ctx_in_use->trid; 7197 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb); 7198 if (ctx->probe_ctx) { 7199 spdk_poller_unregister(&ctx->poller); 7200 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000); 7201 } else { 7202 DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n"); 7203 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7204 ctx->entry_ctx_in_use = NULL; 7205 } 7206 } else if (ctx->probe_ctx) { 7207 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7208 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 7209 complete_discovery_start(ctx, -ETIMEDOUT); 7210 return SPDK_POLLER_BUSY; 7211 } 7212 7213 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 7214 if (rc != -EAGAIN) { 7215 if (ctx->rc != 0) { 7216 assert(ctx->initializing); 7217 stop_discovery(ctx, NULL, ctx->cb_ctx); 7218 } else { 7219 assert(rc == 0); 7220 DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n"); 7221 ctx->rc = rc; 7222 get_discovery_log_page(ctx); 7223 } 7224 } 7225 } else { 7226 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7227 DISCOVERY_ERRLOG(ctx, "timed out while attaching NVM ctrlrs\n"); 7228 complete_discovery_start(ctx, -ETIMEDOUT); 7229 /* We need to wait until all NVM ctrlrs are attached before we stop the 7230 * discovery service to make sure we don't detach a ctrlr that is still 7231 * being attached. 7232 */ 7233 if (ctx->attach_in_progress == 0) { 7234 stop_discovery(ctx, NULL, ctx->cb_ctx); 7235 return SPDK_POLLER_BUSY; 7236 } 7237 } 7238 7239 rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr); 7240 if (rc < 0) { 7241 spdk_poller_unregister(&ctx->poller); 7242 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 7243 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7244 ctx->entry_ctx_in_use = NULL; 7245 7246 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 7247 if (rc != 0) { 7248 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 7249 ctx->ctrlr = NULL; 7250 } 7251 } 7252 } 7253 7254 return SPDK_POLLER_BUSY; 7255 } 7256 7257 static void 7258 start_discovery_poller(void *arg) 7259 { 7260 struct discovery_ctx *ctx = arg; 7261 7262 TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq); 7263 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 7264 } 7265 7266 int 7267 bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid, 7268 const char *base_name, 7269 struct spdk_nvme_ctrlr_opts *drv_opts, 7270 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, 7271 uint64_t attach_timeout, 7272 bool from_mdns, 7273 spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx) 7274 { 7275 struct discovery_ctx *ctx; 7276 struct discovery_entry_ctx *discovery_entry_ctx; 7277 7278 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 7279 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7280 if (strcmp(ctx->name, base_name) == 0) { 7281 return -EEXIST; 7282 } 7283 7284 if (ctx->entry_ctx_in_use != NULL) { 7285 if (!spdk_nvme_transport_id_compare(trid, &ctx->entry_ctx_in_use->trid)) { 7286 return -EEXIST; 7287 } 7288 } 7289 7290 TAILQ_FOREACH(discovery_entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 7291 if (!spdk_nvme_transport_id_compare(trid, &discovery_entry_ctx->trid)) { 7292 return -EEXIST; 7293 } 7294 } 7295 } 7296 7297 ctx = calloc(1, sizeof(*ctx)); 7298 if (ctx == NULL) { 7299 return -ENOMEM; 7300 } 7301 7302 ctx->name = strdup(base_name); 7303 if (ctx->name == NULL) { 7304 free_discovery_ctx(ctx); 7305 return -ENOMEM; 7306 } 7307 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 7308 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 7309 ctx->from_mdns_discovery_service = from_mdns; 7310 ctx->bdev_opts.from_discovery_service = true; 7311 ctx->calling_thread = spdk_get_thread(); 7312 ctx->start_cb_fn = cb_fn; 7313 ctx->cb_ctx = cb_ctx; 7314 ctx->initializing = true; 7315 if (ctx->start_cb_fn) { 7316 /* We can use this when dumping json to denote if this RPC parameter 7317 * was specified or not. 7318 */ 7319 ctx->wait_for_attach = true; 7320 } 7321 if (attach_timeout != 0) { 7322 ctx->timeout_ticks = spdk_get_ticks() + attach_timeout * 7323 spdk_get_ticks_hz() / 1000ull; 7324 } 7325 TAILQ_INIT(&ctx->nvm_entry_ctxs); 7326 TAILQ_INIT(&ctx->discovery_entry_ctxs); 7327 memcpy(&ctx->trid, trid, sizeof(*trid)); 7328 /* Even if user did not specify hostnqn, we can still strdup("\0"); */ 7329 ctx->hostnqn = strdup(ctx->drv_opts.hostnqn); 7330 if (ctx->hostnqn == NULL) { 7331 free_discovery_ctx(ctx); 7332 return -ENOMEM; 7333 } 7334 discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid); 7335 if (discovery_entry_ctx == NULL) { 7336 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 7337 free_discovery_ctx(ctx); 7338 return -ENOMEM; 7339 } 7340 7341 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq); 7342 spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx); 7343 return 0; 7344 } 7345 7346 int 7347 bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 7348 { 7349 struct discovery_ctx *ctx; 7350 7351 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7352 if (strcmp(name, ctx->name) == 0) { 7353 if (ctx->stop) { 7354 return -EALREADY; 7355 } 7356 /* If we're still starting the discovery service and ->rc is non-zero, we're 7357 * going to stop it as soon as we can 7358 */ 7359 if (ctx->initializing && ctx->rc != 0) { 7360 return -EALREADY; 7361 } 7362 stop_discovery(ctx, cb_fn, cb_ctx); 7363 return 0; 7364 } 7365 } 7366 7367 return -ENOENT; 7368 } 7369 7370 static int 7371 bdev_nvme_library_init(void) 7372 { 7373 g_bdev_nvme_init_thread = spdk_get_thread(); 7374 7375 spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb, 7376 bdev_nvme_destroy_poll_group_cb, 7377 sizeof(struct nvme_poll_group), "nvme_poll_groups"); 7378 7379 return 0; 7380 } 7381 7382 static void 7383 bdev_nvme_fini_destruct_ctrlrs(void) 7384 { 7385 struct nvme_bdev_ctrlr *nbdev_ctrlr; 7386 struct nvme_ctrlr *nvme_ctrlr; 7387 7388 pthread_mutex_lock(&g_bdev_nvme_mutex); 7389 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 7390 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 7391 pthread_mutex_lock(&nvme_ctrlr->mutex); 7392 if (nvme_ctrlr->destruct) { 7393 /* This controller's destruction was already started 7394 * before the application started shutting down 7395 */ 7396 pthread_mutex_unlock(&nvme_ctrlr->mutex); 7397 continue; 7398 } 7399 nvme_ctrlr->destruct = true; 7400 pthread_mutex_unlock(&nvme_ctrlr->mutex); 7401 7402 spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct, 7403 nvme_ctrlr); 7404 } 7405 } 7406 7407 g_bdev_nvme_module_finish = true; 7408 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 7409 pthread_mutex_unlock(&g_bdev_nvme_mutex); 7410 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 7411 spdk_bdev_module_fini_done(); 7412 return; 7413 } 7414 7415 pthread_mutex_unlock(&g_bdev_nvme_mutex); 7416 } 7417 7418 static void 7419 check_discovery_fini(void *arg) 7420 { 7421 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 7422 bdev_nvme_fini_destruct_ctrlrs(); 7423 } 7424 } 7425 7426 static void 7427 bdev_nvme_library_fini(void) 7428 { 7429 struct nvme_probe_skip_entry *entry, *entry_tmp; 7430 struct discovery_ctx *ctx; 7431 7432 spdk_poller_unregister(&g_hotplug_poller); 7433 free(g_hotplug_probe_ctx); 7434 g_hotplug_probe_ctx = NULL; 7435 7436 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) { 7437 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 7438 free(entry); 7439 } 7440 7441 assert(spdk_get_thread() == g_bdev_nvme_init_thread); 7442 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 7443 bdev_nvme_fini_destruct_ctrlrs(); 7444 } else { 7445 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7446 stop_discovery(ctx, check_discovery_fini, NULL); 7447 } 7448 } 7449 } 7450 7451 static void 7452 bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio) 7453 { 7454 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7455 struct spdk_bdev *bdev = bdev_io->bdev; 7456 struct spdk_dif_ctx dif_ctx; 7457 struct spdk_dif_error err_blk = {}; 7458 int rc; 7459 struct spdk_dif_ctx_init_ext_opts dif_opts; 7460 7461 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 7462 dif_opts.dif_pi_format = bdev->dif_pi_format; 7463 rc = spdk_dif_ctx_init(&dif_ctx, 7464 bdev->blocklen, bdev->md_len, bdev->md_interleave, 7465 bdev->dif_is_head_of_md, bdev->dif_type, 7466 bdev_io->u.bdev.dif_check_flags, 7467 bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0, &dif_opts); 7468 if (rc != 0) { 7469 SPDK_ERRLOG("Initialization of DIF context failed\n"); 7470 return; 7471 } 7472 7473 if (bdev->md_interleave) { 7474 rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 7475 bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 7476 } else { 7477 struct iovec md_iov = { 7478 .iov_base = bdev_io->u.bdev.md_buf, 7479 .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len, 7480 }; 7481 7482 rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 7483 &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 7484 } 7485 7486 if (rc != 0) { 7487 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", 7488 err_blk.err_type, err_blk.err_offset); 7489 } else { 7490 SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n"); 7491 } 7492 } 7493 7494 static void 7495 bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7496 { 7497 struct nvme_bdev_io *bio = ref; 7498 7499 if (spdk_nvme_cpl_is_success(cpl)) { 7500 /* Run PI verification for read data buffer. */ 7501 bdev_nvme_verify_pi_error(bio); 7502 } 7503 7504 /* Return original completion status */ 7505 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 7506 } 7507 7508 static void 7509 bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7510 { 7511 struct nvme_bdev_io *bio = ref; 7512 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7513 int ret; 7514 7515 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 7516 SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n", 7517 cpl->status.sct, cpl->status.sc); 7518 7519 /* Save completion status to use after verifying PI error. */ 7520 bio->cpl = *cpl; 7521 7522 if (spdk_likely(nvme_io_path_is_available(bio->io_path))) { 7523 /* Read without PI checking to verify PI error. */ 7524 ret = bdev_nvme_no_pi_readv(bio, 7525 bdev_io->u.bdev.iovs, 7526 bdev_io->u.bdev.iovcnt, 7527 bdev_io->u.bdev.md_buf, 7528 bdev_io->u.bdev.num_blocks, 7529 bdev_io->u.bdev.offset_blocks); 7530 if (ret == 0) { 7531 return; 7532 } 7533 } 7534 } 7535 7536 bdev_nvme_io_complete_nvme_status(bio, cpl); 7537 } 7538 7539 static void 7540 bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7541 { 7542 struct nvme_bdev_io *bio = ref; 7543 7544 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 7545 SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n", 7546 cpl->status.sct, cpl->status.sc); 7547 /* Run PI verification for write data buffer if PI error is detected. */ 7548 bdev_nvme_verify_pi_error(bio); 7549 } 7550 7551 bdev_nvme_io_complete_nvme_status(bio, cpl); 7552 } 7553 7554 static void 7555 bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7556 { 7557 struct nvme_bdev_io *bio = ref; 7558 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7559 7560 /* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks. 7561 * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error(). 7562 */ 7563 bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0; 7564 7565 if (spdk_nvme_cpl_is_pi_error(cpl)) { 7566 SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n", 7567 cpl->status.sct, cpl->status.sc); 7568 /* Run PI verification for zone append data buffer if PI error is detected. */ 7569 bdev_nvme_verify_pi_error(bio); 7570 } 7571 7572 bdev_nvme_io_complete_nvme_status(bio, cpl); 7573 } 7574 7575 static void 7576 bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7577 { 7578 struct nvme_bdev_io *bio = ref; 7579 7580 if (spdk_nvme_cpl_is_pi_error(cpl)) { 7581 SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n", 7582 cpl->status.sct, cpl->status.sc); 7583 /* Run PI verification for compare data buffer if PI error is detected. */ 7584 bdev_nvme_verify_pi_error(bio); 7585 } 7586 7587 bdev_nvme_io_complete_nvme_status(bio, cpl); 7588 } 7589 7590 static void 7591 bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7592 { 7593 struct nvme_bdev_io *bio = ref; 7594 7595 /* Compare operation completion */ 7596 if (!bio->first_fused_completed) { 7597 /* Save compare result for write callback */ 7598 bio->cpl = *cpl; 7599 bio->first_fused_completed = true; 7600 return; 7601 } 7602 7603 /* Write operation completion */ 7604 if (spdk_nvme_cpl_is_error(&bio->cpl)) { 7605 /* If bio->cpl is already an error, it means the compare operation failed. In that case, 7606 * complete the IO with the compare operation's status. 7607 */ 7608 if (!spdk_nvme_cpl_is_error(cpl)) { 7609 SPDK_ERRLOG("Unexpected write success after compare failure.\n"); 7610 } 7611 7612 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 7613 } else { 7614 bdev_nvme_io_complete_nvme_status(bio, cpl); 7615 } 7616 } 7617 7618 static void 7619 bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl) 7620 { 7621 struct nvme_bdev_io *bio = ref; 7622 7623 bdev_nvme_io_complete_nvme_status(bio, cpl); 7624 } 7625 7626 static int 7627 fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc) 7628 { 7629 switch (desc->zt) { 7630 case SPDK_NVME_ZONE_TYPE_SEQWR: 7631 info->type = SPDK_BDEV_ZONE_TYPE_SEQWR; 7632 break; 7633 default: 7634 SPDK_ERRLOG("Invalid zone type: %#x in zone report\n", desc->zt); 7635 return -EIO; 7636 } 7637 7638 switch (desc->zs) { 7639 case SPDK_NVME_ZONE_STATE_EMPTY: 7640 info->state = SPDK_BDEV_ZONE_STATE_EMPTY; 7641 break; 7642 case SPDK_NVME_ZONE_STATE_IOPEN: 7643 info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN; 7644 break; 7645 case SPDK_NVME_ZONE_STATE_EOPEN: 7646 info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN; 7647 break; 7648 case SPDK_NVME_ZONE_STATE_CLOSED: 7649 info->state = SPDK_BDEV_ZONE_STATE_CLOSED; 7650 break; 7651 case SPDK_NVME_ZONE_STATE_RONLY: 7652 info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY; 7653 break; 7654 case SPDK_NVME_ZONE_STATE_FULL: 7655 info->state = SPDK_BDEV_ZONE_STATE_FULL; 7656 break; 7657 case SPDK_NVME_ZONE_STATE_OFFLINE: 7658 info->state = SPDK_BDEV_ZONE_STATE_OFFLINE; 7659 break; 7660 default: 7661 SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs); 7662 return -EIO; 7663 } 7664 7665 info->zone_id = desc->zslba; 7666 info->write_pointer = desc->wp; 7667 info->capacity = desc->zcap; 7668 7669 return 0; 7670 } 7671 7672 static void 7673 bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl) 7674 { 7675 struct nvme_bdev_io *bio = ref; 7676 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7677 uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id; 7678 uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones; 7679 struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf; 7680 uint64_t max_zones_per_buf, i; 7681 uint32_t zone_report_bufsize; 7682 struct spdk_nvme_ns *ns; 7683 struct spdk_nvme_qpair *qpair; 7684 int ret; 7685 7686 if (spdk_nvme_cpl_is_error(cpl)) { 7687 goto out_complete_io_nvme_cpl; 7688 } 7689 7690 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 7691 ret = -ENXIO; 7692 goto out_complete_io_ret; 7693 } 7694 7695 ns = bio->io_path->nvme_ns->ns; 7696 qpair = bio->io_path->qpair->qpair; 7697 7698 zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 7699 max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) / 7700 sizeof(bio->zone_report_buf->descs[0]); 7701 7702 if (bio->zone_report_buf->nr_zones > max_zones_per_buf) { 7703 ret = -EINVAL; 7704 goto out_complete_io_ret; 7705 } 7706 7707 if (!bio->zone_report_buf->nr_zones) { 7708 ret = -EINVAL; 7709 goto out_complete_io_ret; 7710 } 7711 7712 for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) { 7713 ret = fill_zone_from_report(&info[bio->handled_zones], 7714 &bio->zone_report_buf->descs[i]); 7715 if (ret) { 7716 goto out_complete_io_ret; 7717 } 7718 bio->handled_zones++; 7719 } 7720 7721 if (bio->handled_zones < zones_to_copy) { 7722 uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 7723 uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones); 7724 7725 memset(bio->zone_report_buf, 0, zone_report_bufsize); 7726 ret = spdk_nvme_zns_report_zones(ns, qpair, 7727 bio->zone_report_buf, zone_report_bufsize, 7728 slba, SPDK_NVME_ZRA_LIST_ALL, true, 7729 bdev_nvme_get_zone_info_done, bio); 7730 if (!ret) { 7731 return; 7732 } else { 7733 goto out_complete_io_ret; 7734 } 7735 } 7736 7737 out_complete_io_nvme_cpl: 7738 free(bio->zone_report_buf); 7739 bio->zone_report_buf = NULL; 7740 bdev_nvme_io_complete_nvme_status(bio, cpl); 7741 return; 7742 7743 out_complete_io_ret: 7744 free(bio->zone_report_buf); 7745 bio->zone_report_buf = NULL; 7746 bdev_nvme_io_complete(bio, ret); 7747 } 7748 7749 static void 7750 bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl) 7751 { 7752 struct nvme_bdev_io *bio = ref; 7753 7754 bdev_nvme_io_complete_nvme_status(bio, cpl); 7755 } 7756 7757 static void 7758 bdev_nvme_admin_passthru_complete_nvme_status(void *ctx) 7759 { 7760 struct nvme_bdev_io *bio = ctx; 7761 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7762 const struct spdk_nvme_cpl *cpl = &bio->cpl; 7763 7764 assert(bdev_nvme_io_type_is_admin(bdev_io->type)); 7765 7766 __bdev_nvme_io_complete(bdev_io, 0, cpl); 7767 } 7768 7769 static void 7770 bdev_nvme_abort_complete(void *ctx) 7771 { 7772 struct nvme_bdev_io *bio = ctx; 7773 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7774 7775 if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) { 7776 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL); 7777 } else { 7778 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL); 7779 } 7780 } 7781 7782 static void 7783 bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl) 7784 { 7785 struct nvme_bdev_io *bio = ref; 7786 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7787 7788 bio->cpl = *cpl; 7789 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), bdev_nvme_abort_complete, bio); 7790 } 7791 7792 static void 7793 bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl) 7794 { 7795 struct nvme_bdev_io *bio = ref; 7796 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7797 7798 bio->cpl = *cpl; 7799 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), 7800 bdev_nvme_admin_passthru_complete_nvme_status, bio); 7801 } 7802 7803 static void 7804 bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset) 7805 { 7806 struct nvme_bdev_io *bio = ref; 7807 struct iovec *iov; 7808 7809 bio->iov_offset = sgl_offset; 7810 for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) { 7811 iov = &bio->iovs[bio->iovpos]; 7812 if (bio->iov_offset < iov->iov_len) { 7813 break; 7814 } 7815 7816 bio->iov_offset -= iov->iov_len; 7817 } 7818 } 7819 7820 static int 7821 bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length) 7822 { 7823 struct nvme_bdev_io *bio = ref; 7824 struct iovec *iov; 7825 7826 assert(bio->iovpos < bio->iovcnt); 7827 7828 iov = &bio->iovs[bio->iovpos]; 7829 7830 *address = iov->iov_base; 7831 *length = iov->iov_len; 7832 7833 if (bio->iov_offset) { 7834 assert(bio->iov_offset <= iov->iov_len); 7835 *address += bio->iov_offset; 7836 *length -= bio->iov_offset; 7837 } 7838 7839 bio->iov_offset += *length; 7840 if (bio->iov_offset == iov->iov_len) { 7841 bio->iovpos++; 7842 bio->iov_offset = 0; 7843 } 7844 7845 return 0; 7846 } 7847 7848 static void 7849 bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset) 7850 { 7851 struct nvme_bdev_io *bio = ref; 7852 struct iovec *iov; 7853 7854 bio->fused_iov_offset = sgl_offset; 7855 for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) { 7856 iov = &bio->fused_iovs[bio->fused_iovpos]; 7857 if (bio->fused_iov_offset < iov->iov_len) { 7858 break; 7859 } 7860 7861 bio->fused_iov_offset -= iov->iov_len; 7862 } 7863 } 7864 7865 static int 7866 bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length) 7867 { 7868 struct nvme_bdev_io *bio = ref; 7869 struct iovec *iov; 7870 7871 assert(bio->fused_iovpos < bio->fused_iovcnt); 7872 7873 iov = &bio->fused_iovs[bio->fused_iovpos]; 7874 7875 *address = iov->iov_base; 7876 *length = iov->iov_len; 7877 7878 if (bio->fused_iov_offset) { 7879 assert(bio->fused_iov_offset <= iov->iov_len); 7880 *address += bio->fused_iov_offset; 7881 *length -= bio->fused_iov_offset; 7882 } 7883 7884 bio->fused_iov_offset += *length; 7885 if (bio->fused_iov_offset == iov->iov_len) { 7886 bio->fused_iovpos++; 7887 bio->fused_iov_offset = 0; 7888 } 7889 7890 return 0; 7891 } 7892 7893 static int 7894 bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7895 void *md, uint64_t lba_count, uint64_t lba) 7896 { 7897 int rc; 7898 7899 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n", 7900 lba_count, lba); 7901 7902 bio->iovs = iov; 7903 bio->iovcnt = iovcnt; 7904 bio->iovpos = 0; 7905 bio->iov_offset = 0; 7906 7907 rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns, 7908 bio->io_path->qpair->qpair, 7909 lba, lba_count, 7910 bdev_nvme_no_pi_readv_done, bio, 0, 7911 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 7912 md, 0, 0); 7913 7914 if (rc != 0 && rc != -ENOMEM) { 7915 SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc); 7916 } 7917 return rc; 7918 } 7919 7920 static int 7921 bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7922 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 7923 struct spdk_memory_domain *domain, void *domain_ctx, 7924 struct spdk_accel_sequence *seq) 7925 { 7926 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7927 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7928 int rc; 7929 7930 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n", 7931 lba_count, lba); 7932 7933 bio->iovs = iov; 7934 bio->iovcnt = iovcnt; 7935 bio->iovpos = 0; 7936 bio->iov_offset = 0; 7937 7938 if (domain != NULL || seq != NULL) { 7939 bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence); 7940 bio->ext_opts.memory_domain = domain; 7941 bio->ext_opts.memory_domain_ctx = domain_ctx; 7942 bio->ext_opts.io_flags = flags; 7943 bio->ext_opts.metadata = md; 7944 bio->ext_opts.accel_sequence = seq; 7945 7946 if (iovcnt == 1) { 7947 rc = spdk_nvme_ns_cmd_read_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_readv_done, 7948 bio, &bio->ext_opts); 7949 } else { 7950 rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count, 7951 bdev_nvme_readv_done, bio, 7952 bdev_nvme_queued_reset_sgl, 7953 bdev_nvme_queued_next_sge, 7954 &bio->ext_opts); 7955 } 7956 } else if (iovcnt == 1) { 7957 rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, iov[0].iov_base, 7958 md, lba, lba_count, bdev_nvme_readv_done, 7959 bio, flags, 0, 0); 7960 } else { 7961 rc = spdk_nvme_ns_cmd_readv_with_md(ns, qpair, lba, lba_count, 7962 bdev_nvme_readv_done, bio, flags, 7963 bdev_nvme_queued_reset_sgl, 7964 bdev_nvme_queued_next_sge, md, 0, 0); 7965 } 7966 7967 if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) { 7968 SPDK_ERRLOG("readv failed: rc = %d\n", rc); 7969 } 7970 return rc; 7971 } 7972 7973 static int 7974 bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7975 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 7976 struct spdk_memory_domain *domain, void *domain_ctx, 7977 struct spdk_accel_sequence *seq, 7978 union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13) 7979 { 7980 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7981 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7982 int rc; 7983 7984 SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 7985 lba_count, lba); 7986 7987 bio->iovs = iov; 7988 bio->iovcnt = iovcnt; 7989 bio->iovpos = 0; 7990 bio->iov_offset = 0; 7991 7992 if (domain != NULL || seq != NULL) { 7993 bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence); 7994 bio->ext_opts.memory_domain = domain; 7995 bio->ext_opts.memory_domain_ctx = domain_ctx; 7996 bio->ext_opts.io_flags = flags | SPDK_NVME_IO_FLAGS_DIRECTIVE(cdw12.write.dtype); 7997 bio->ext_opts.cdw13 = cdw13.raw; 7998 bio->ext_opts.metadata = md; 7999 bio->ext_opts.accel_sequence = seq; 8000 8001 if (iovcnt == 1) { 8002 rc = spdk_nvme_ns_cmd_write_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_writev_done, 8003 bio, &bio->ext_opts); 8004 } else { 8005 rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count, 8006 bdev_nvme_writev_done, bio, 8007 bdev_nvme_queued_reset_sgl, 8008 bdev_nvme_queued_next_sge, 8009 &bio->ext_opts); 8010 } 8011 } else if (iovcnt == 1) { 8012 rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, iov[0].iov_base, 8013 md, lba, lba_count, bdev_nvme_writev_done, 8014 bio, flags, 0, 0); 8015 } else { 8016 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 8017 bdev_nvme_writev_done, bio, flags, 8018 bdev_nvme_queued_reset_sgl, 8019 bdev_nvme_queued_next_sge, md, 0, 0); 8020 } 8021 8022 if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) { 8023 SPDK_ERRLOG("writev failed: rc = %d\n", rc); 8024 } 8025 return rc; 8026 } 8027 8028 static int 8029 bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 8030 void *md, uint64_t lba_count, uint64_t zslba, 8031 uint32_t flags) 8032 { 8033 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8034 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8035 int rc; 8036 8037 SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n", 8038 lba_count, zslba); 8039 8040 bio->iovs = iov; 8041 bio->iovcnt = iovcnt; 8042 bio->iovpos = 0; 8043 bio->iov_offset = 0; 8044 8045 if (iovcnt == 1) { 8046 rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba, 8047 lba_count, 8048 bdev_nvme_zone_appendv_done, bio, 8049 flags, 8050 0, 0); 8051 } else { 8052 rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count, 8053 bdev_nvme_zone_appendv_done, bio, flags, 8054 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 8055 md, 0, 0); 8056 } 8057 8058 if (rc != 0 && rc != -ENOMEM) { 8059 SPDK_ERRLOG("zone append failed: rc = %d\n", rc); 8060 } 8061 return rc; 8062 } 8063 8064 static int 8065 bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 8066 void *md, uint64_t lba_count, uint64_t lba, 8067 uint32_t flags) 8068 { 8069 int rc; 8070 8071 SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n", 8072 lba_count, lba); 8073 8074 bio->iovs = iov; 8075 bio->iovcnt = iovcnt; 8076 bio->iovpos = 0; 8077 bio->iov_offset = 0; 8078 8079 rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns, 8080 bio->io_path->qpair->qpair, 8081 lba, lba_count, 8082 bdev_nvme_comparev_done, bio, flags, 8083 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 8084 md, 0, 0); 8085 8086 if (rc != 0 && rc != -ENOMEM) { 8087 SPDK_ERRLOG("comparev failed: rc = %d\n", rc); 8088 } 8089 return rc; 8090 } 8091 8092 static int 8093 bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt, 8094 struct iovec *write_iov, int write_iovcnt, 8095 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags) 8096 { 8097 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8098 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8099 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 8100 int rc; 8101 8102 SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 8103 lba_count, lba); 8104 8105 bio->iovs = cmp_iov; 8106 bio->iovcnt = cmp_iovcnt; 8107 bio->iovpos = 0; 8108 bio->iov_offset = 0; 8109 bio->fused_iovs = write_iov; 8110 bio->fused_iovcnt = write_iovcnt; 8111 bio->fused_iovpos = 0; 8112 bio->fused_iov_offset = 0; 8113 8114 if (bdev_io->num_retries == 0) { 8115 bio->first_fused_submitted = false; 8116 bio->first_fused_completed = false; 8117 } 8118 8119 if (!bio->first_fused_submitted) { 8120 flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST; 8121 memset(&bio->cpl, 0, sizeof(bio->cpl)); 8122 8123 rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count, 8124 bdev_nvme_comparev_and_writev_done, bio, flags, 8125 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0); 8126 if (rc == 0) { 8127 bio->first_fused_submitted = true; 8128 flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST; 8129 } else { 8130 if (rc != -ENOMEM) { 8131 SPDK_ERRLOG("compare failed: rc = %d\n", rc); 8132 } 8133 return rc; 8134 } 8135 } 8136 8137 flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND; 8138 8139 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 8140 bdev_nvme_comparev_and_writev_done, bio, flags, 8141 bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0); 8142 if (rc != 0 && rc != -ENOMEM) { 8143 SPDK_ERRLOG("write failed: rc = %d\n", rc); 8144 rc = 0; 8145 } 8146 8147 return rc; 8148 } 8149 8150 static int 8151 bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 8152 { 8153 struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES]; 8154 struct spdk_nvme_dsm_range *range; 8155 uint64_t offset, remaining; 8156 uint64_t num_ranges_u64; 8157 uint16_t num_ranges; 8158 int rc; 8159 8160 num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) / 8161 SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8162 if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) { 8163 SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks); 8164 return -EINVAL; 8165 } 8166 num_ranges = (uint16_t)num_ranges_u64; 8167 8168 offset = offset_blocks; 8169 remaining = num_blocks; 8170 range = &dsm_ranges[0]; 8171 8172 /* Fill max-size ranges until the remaining blocks fit into one range */ 8173 while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) { 8174 range->attributes.raw = 0; 8175 range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8176 range->starting_lba = offset; 8177 8178 offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8179 remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8180 range++; 8181 } 8182 8183 /* Final range describes the remaining blocks */ 8184 range->attributes.raw = 0; 8185 range->length = remaining; 8186 range->starting_lba = offset; 8187 8188 rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns, 8189 bio->io_path->qpair->qpair, 8190 SPDK_NVME_DSM_ATTR_DEALLOCATE, 8191 dsm_ranges, num_ranges, 8192 bdev_nvme_queued_done, bio); 8193 8194 return rc; 8195 } 8196 8197 static int 8198 bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 8199 { 8200 if (num_blocks > UINT16_MAX + 1) { 8201 SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n"); 8202 return -EINVAL; 8203 } 8204 8205 return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns, 8206 bio->io_path->qpair->qpair, 8207 offset_blocks, num_blocks, 8208 bdev_nvme_queued_done, bio, 8209 0); 8210 } 8211 8212 static int 8213 bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones, 8214 struct spdk_bdev_zone_info *info) 8215 { 8216 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8217 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8218 uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 8219 uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 8220 uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns); 8221 8222 if (zone_id % zone_size != 0) { 8223 return -EINVAL; 8224 } 8225 8226 if (num_zones > total_zones || !num_zones) { 8227 return -EINVAL; 8228 } 8229 8230 assert(!bio->zone_report_buf); 8231 bio->zone_report_buf = calloc(1, zone_report_bufsize); 8232 if (!bio->zone_report_buf) { 8233 return -ENOMEM; 8234 } 8235 8236 bio->handled_zones = 0; 8237 8238 return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize, 8239 zone_id, SPDK_NVME_ZRA_LIST_ALL, true, 8240 bdev_nvme_get_zone_info_done, bio); 8241 } 8242 8243 static int 8244 bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 8245 enum spdk_bdev_zone_action action) 8246 { 8247 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8248 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8249 8250 switch (action) { 8251 case SPDK_BDEV_ZONE_CLOSE: 8252 return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false, 8253 bdev_nvme_zone_management_done, bio); 8254 case SPDK_BDEV_ZONE_FINISH: 8255 return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false, 8256 bdev_nvme_zone_management_done, bio); 8257 case SPDK_BDEV_ZONE_OPEN: 8258 return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false, 8259 bdev_nvme_zone_management_done, bio); 8260 case SPDK_BDEV_ZONE_RESET: 8261 return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false, 8262 bdev_nvme_zone_management_done, bio); 8263 case SPDK_BDEV_ZONE_OFFLINE: 8264 return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false, 8265 bdev_nvme_zone_management_done, bio); 8266 default: 8267 return -EINVAL; 8268 } 8269 } 8270 8271 static void 8272 bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 8273 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes) 8274 { 8275 struct nvme_io_path *io_path; 8276 struct nvme_ctrlr *nvme_ctrlr; 8277 uint32_t max_xfer_size; 8278 int rc = -ENXIO; 8279 8280 /* Choose the first ctrlr which is not failed. */ 8281 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 8282 nvme_ctrlr = io_path->qpair->ctrlr; 8283 8284 /* We should skip any unavailable nvme_ctrlr rather than checking 8285 * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO. 8286 */ 8287 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 8288 continue; 8289 } 8290 8291 max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr); 8292 8293 if (nbytes > max_xfer_size) { 8294 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8295 rc = -EINVAL; 8296 goto err; 8297 } 8298 8299 rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes, 8300 bdev_nvme_admin_passthru_done, bio); 8301 if (rc == 0) { 8302 return; 8303 } 8304 } 8305 8306 err: 8307 bdev_nvme_admin_complete(bio, rc); 8308 } 8309 8310 static int 8311 bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 8312 void *buf, size_t nbytes) 8313 { 8314 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8315 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8316 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8317 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8318 8319 if (nbytes > max_xfer_size) { 8320 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8321 return -EINVAL; 8322 } 8323 8324 /* 8325 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 8326 * so fill it out automatically. 8327 */ 8328 cmd->nsid = spdk_nvme_ns_get_id(ns); 8329 8330 return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf, 8331 (uint32_t)nbytes, bdev_nvme_queued_done, bio); 8332 } 8333 8334 static int 8335 bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 8336 void *buf, size_t nbytes, void *md_buf, size_t md_len) 8337 { 8338 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8339 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8340 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 8341 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8342 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8343 8344 if (nbytes > max_xfer_size) { 8345 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8346 return -EINVAL; 8347 } 8348 8349 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 8350 SPDK_ERRLOG("invalid meta data buffer size\n"); 8351 return -EINVAL; 8352 } 8353 8354 /* 8355 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 8356 * so fill it out automatically. 8357 */ 8358 cmd->nsid = spdk_nvme_ns_get_id(ns); 8359 8360 return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf, 8361 (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio); 8362 } 8363 8364 static int 8365 bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, 8366 struct spdk_nvme_cmd *cmd, struct iovec *iov, int iovcnt, 8367 size_t nbytes, void *md_buf, size_t md_len) 8368 { 8369 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8370 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8371 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 8372 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8373 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8374 8375 bio->iovs = iov; 8376 bio->iovcnt = iovcnt; 8377 bio->iovpos = 0; 8378 bio->iov_offset = 0; 8379 8380 if (nbytes > max_xfer_size) { 8381 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8382 return -EINVAL; 8383 } 8384 8385 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 8386 SPDK_ERRLOG("invalid meta data buffer size\n"); 8387 return -EINVAL; 8388 } 8389 8390 /* 8391 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands 8392 * require a nsid, so fill it out automatically. 8393 */ 8394 cmd->nsid = spdk_nvme_ns_get_id(ns); 8395 8396 return spdk_nvme_ctrlr_cmd_iov_raw_with_md( 8397 ctrlr, qpair, cmd, (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio, 8398 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge); 8399 } 8400 8401 static void 8402 bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 8403 struct nvme_bdev_io *bio_to_abort) 8404 { 8405 struct nvme_io_path *io_path; 8406 int rc = 0; 8407 8408 rc = bdev_nvme_abort_retry_io(nbdev_ch, bio_to_abort); 8409 if (rc == 0) { 8410 bdev_nvme_admin_complete(bio, 0); 8411 return; 8412 } 8413 8414 io_path = bio_to_abort->io_path; 8415 if (io_path != NULL) { 8416 rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr, 8417 io_path->qpair->qpair, 8418 bio_to_abort, 8419 bdev_nvme_abort_done, bio); 8420 } else { 8421 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 8422 rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr, 8423 NULL, 8424 bio_to_abort, 8425 bdev_nvme_abort_done, bio); 8426 8427 if (rc != -ENOENT) { 8428 break; 8429 } 8430 } 8431 } 8432 8433 if (rc != 0) { 8434 /* If no command was found or there was any error, complete the abort 8435 * request with failure. 8436 */ 8437 bdev_nvme_admin_complete(bio, rc); 8438 } 8439 } 8440 8441 static int 8442 bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, uint64_t src_offset_blocks, 8443 uint64_t num_blocks) 8444 { 8445 struct spdk_nvme_scc_source_range range = { 8446 .slba = src_offset_blocks, 8447 .nlb = num_blocks - 1 8448 }; 8449 8450 return spdk_nvme_ns_cmd_copy(bio->io_path->nvme_ns->ns, 8451 bio->io_path->qpair->qpair, 8452 &range, 1, dst_offset_blocks, 8453 bdev_nvme_queued_done, bio); 8454 } 8455 8456 static void 8457 bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w) 8458 { 8459 const char *action; 8460 uint32_t i; 8461 8462 if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) { 8463 action = "reset"; 8464 } else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) { 8465 action = "abort"; 8466 } else { 8467 action = "none"; 8468 } 8469 8470 spdk_json_write_object_begin(w); 8471 8472 spdk_json_write_named_string(w, "method", "bdev_nvme_set_options"); 8473 8474 spdk_json_write_named_object_begin(w, "params"); 8475 spdk_json_write_named_string(w, "action_on_timeout", action); 8476 spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us); 8477 spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us); 8478 spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms); 8479 spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst); 8480 spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight); 8481 spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight); 8482 spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight); 8483 spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us); 8484 spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us); 8485 spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests); 8486 spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit); 8487 spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count); 8488 spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count); 8489 spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout); 8490 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec); 8491 spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec); 8492 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec); 8493 spdk_json_write_named_bool(w, "disable_auto_failback", g_opts.disable_auto_failback); 8494 spdk_json_write_named_bool(w, "generate_uuids", g_opts.generate_uuids); 8495 spdk_json_write_named_uint8(w, "transport_tos", g_opts.transport_tos); 8496 spdk_json_write_named_bool(w, "nvme_error_stat", g_opts.nvme_error_stat); 8497 spdk_json_write_named_uint32(w, "rdma_srq_size", g_opts.rdma_srq_size); 8498 spdk_json_write_named_bool(w, "io_path_stat", g_opts.io_path_stat); 8499 spdk_json_write_named_bool(w, "allow_accel_sequence", g_opts.allow_accel_sequence); 8500 spdk_json_write_named_uint32(w, "rdma_max_cq_size", g_opts.rdma_max_cq_size); 8501 spdk_json_write_named_uint16(w, "rdma_cm_event_timeout_ms", g_opts.rdma_cm_event_timeout_ms); 8502 spdk_json_write_named_array_begin(w, "dhchap_digests"); 8503 for (i = 0; i < 32; ++i) { 8504 if (g_opts.dhchap_digests & SPDK_BIT(i)) { 8505 spdk_json_write_string(w, spdk_nvme_dhchap_get_digest_name(i)); 8506 } 8507 } 8508 spdk_json_write_array_end(w); 8509 spdk_json_write_named_array_begin(w, "dhchap_dhgroups"); 8510 for (i = 0; i < 32; ++i) { 8511 if (g_opts.dhchap_dhgroups & SPDK_BIT(i)) { 8512 spdk_json_write_string(w, spdk_nvme_dhchap_get_dhgroup_name(i)); 8513 } 8514 } 8515 8516 spdk_json_write_array_end(w); 8517 spdk_json_write_object_end(w); 8518 8519 spdk_json_write_object_end(w); 8520 } 8521 8522 static void 8523 bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx) 8524 { 8525 struct spdk_nvme_transport_id trid; 8526 8527 spdk_json_write_object_begin(w); 8528 8529 spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery"); 8530 8531 spdk_json_write_named_object_begin(w, "params"); 8532 spdk_json_write_named_string(w, "name", ctx->name); 8533 spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn); 8534 8535 trid = ctx->trid; 8536 memset(trid.subnqn, 0, sizeof(trid.subnqn)); 8537 nvme_bdev_dump_trid_json(&trid, w); 8538 8539 spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach); 8540 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec); 8541 spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec); 8542 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 8543 ctx->bdev_opts.fast_io_fail_timeout_sec); 8544 spdk_json_write_object_end(w); 8545 8546 spdk_json_write_object_end(w); 8547 } 8548 8549 #ifdef SPDK_CONFIG_NVME_CUSE 8550 static void 8551 nvme_ctrlr_cuse_config_json(struct spdk_json_write_ctx *w, 8552 struct nvme_ctrlr *nvme_ctrlr) 8553 { 8554 size_t cuse_name_size = 128; 8555 char cuse_name[cuse_name_size]; 8556 8557 if (spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, 8558 cuse_name, &cuse_name_size) != 0) { 8559 return; 8560 } 8561 8562 spdk_json_write_object_begin(w); 8563 8564 spdk_json_write_named_string(w, "method", "bdev_nvme_cuse_register"); 8565 8566 spdk_json_write_named_object_begin(w, "params"); 8567 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 8568 spdk_json_write_object_end(w); 8569 8570 spdk_json_write_object_end(w); 8571 } 8572 #endif 8573 8574 static void 8575 nvme_ctrlr_config_json(struct spdk_json_write_ctx *w, 8576 struct nvme_ctrlr *nvme_ctrlr, 8577 struct nvme_path_id *path_id) 8578 { 8579 struct spdk_nvme_transport_id *trid; 8580 const struct spdk_nvme_ctrlr_opts *opts; 8581 8582 if (nvme_ctrlr->opts.from_discovery_service) { 8583 /* Do not emit an RPC for this - it will be implicitly 8584 * covered by a separate bdev_nvme_start_discovery or 8585 * bdev_nvme_start_mdns_discovery RPC. 8586 */ 8587 return; 8588 } 8589 8590 trid = &path_id->trid; 8591 8592 spdk_json_write_object_begin(w); 8593 8594 spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller"); 8595 8596 spdk_json_write_named_object_begin(w, "params"); 8597 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 8598 nvme_bdev_dump_trid_json(trid, w); 8599 spdk_json_write_named_bool(w, "prchk_reftag", 8600 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0); 8601 spdk_json_write_named_bool(w, "prchk_guard", 8602 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0); 8603 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec); 8604 spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec); 8605 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 8606 nvme_ctrlr->opts.fast_io_fail_timeout_sec); 8607 if (nvme_ctrlr->psk != NULL) { 8608 spdk_json_write_named_string(w, "psk", spdk_key_get_name(nvme_ctrlr->psk)); 8609 } 8610 if (nvme_ctrlr->dhchap_key != NULL) { 8611 spdk_json_write_named_string(w, "dhchap_key", 8612 spdk_key_get_name(nvme_ctrlr->dhchap_key)); 8613 } 8614 if (nvme_ctrlr->dhchap_ctrlr_key != NULL) { 8615 spdk_json_write_named_string(w, "dhchap_ctrlr_key", 8616 spdk_key_get_name(nvme_ctrlr->dhchap_ctrlr_key)); 8617 } 8618 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 8619 spdk_json_write_named_string(w, "hostnqn", opts->hostnqn); 8620 spdk_json_write_named_bool(w, "hdgst", opts->header_digest); 8621 spdk_json_write_named_bool(w, "ddgst", opts->data_digest); 8622 if (opts->src_addr[0] != '\0') { 8623 spdk_json_write_named_string(w, "hostaddr", opts->src_addr); 8624 } 8625 if (opts->src_svcid[0] != '\0') { 8626 spdk_json_write_named_string(w, "hostsvcid", opts->src_svcid); 8627 } 8628 8629 if (nvme_ctrlr->opts.multipath) { 8630 spdk_json_write_named_string(w, "multipath", "multipath"); 8631 } 8632 spdk_json_write_object_end(w); 8633 8634 spdk_json_write_object_end(w); 8635 } 8636 8637 static void 8638 bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w) 8639 { 8640 spdk_json_write_object_begin(w); 8641 spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug"); 8642 8643 spdk_json_write_named_object_begin(w, "params"); 8644 spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us); 8645 spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled); 8646 spdk_json_write_object_end(w); 8647 8648 spdk_json_write_object_end(w); 8649 } 8650 8651 static int 8652 bdev_nvme_config_json(struct spdk_json_write_ctx *w) 8653 { 8654 struct nvme_bdev_ctrlr *nbdev_ctrlr; 8655 struct nvme_ctrlr *nvme_ctrlr; 8656 struct discovery_ctx *ctx; 8657 struct nvme_path_id *path_id; 8658 8659 bdev_nvme_opts_config_json(w); 8660 8661 pthread_mutex_lock(&g_bdev_nvme_mutex); 8662 8663 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 8664 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 8665 path_id = nvme_ctrlr->active_path_id; 8666 assert(path_id == TAILQ_FIRST(&nvme_ctrlr->trids)); 8667 nvme_ctrlr_config_json(w, nvme_ctrlr, path_id); 8668 8669 path_id = TAILQ_NEXT(path_id, link); 8670 while (path_id != NULL) { 8671 nvme_ctrlr_config_json(w, nvme_ctrlr, path_id); 8672 path_id = TAILQ_NEXT(path_id, link); 8673 } 8674 8675 #ifdef SPDK_CONFIG_NVME_CUSE 8676 nvme_ctrlr_cuse_config_json(w, nvme_ctrlr); 8677 #endif 8678 } 8679 } 8680 8681 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 8682 if (!ctx->from_mdns_discovery_service) { 8683 bdev_nvme_discovery_config_json(w, ctx); 8684 } 8685 } 8686 8687 bdev_nvme_mdns_discovery_config_json(w); 8688 8689 /* Dump as last parameter to give all NVMe bdevs chance to be constructed 8690 * before enabling hotplug poller. 8691 */ 8692 bdev_nvme_hotplug_config_json(w); 8693 8694 pthread_mutex_unlock(&g_bdev_nvme_mutex); 8695 return 0; 8696 } 8697 8698 struct spdk_nvme_ctrlr * 8699 bdev_nvme_get_ctrlr(struct spdk_bdev *bdev) 8700 { 8701 struct nvme_bdev *nbdev; 8702 struct nvme_ns *nvme_ns; 8703 8704 if (!bdev || bdev->module != &nvme_if) { 8705 return NULL; 8706 } 8707 8708 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 8709 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 8710 assert(nvme_ns != NULL); 8711 8712 return nvme_ns->ctrlr->ctrlr; 8713 } 8714 8715 static bool 8716 nvme_io_path_is_current(struct nvme_io_path *io_path) 8717 { 8718 const struct nvme_bdev_channel *nbdev_ch; 8719 bool current; 8720 8721 if (!nvme_io_path_is_available(io_path)) { 8722 return false; 8723 } 8724 8725 nbdev_ch = io_path->nbdev_ch; 8726 if (nbdev_ch == NULL) { 8727 current = false; 8728 } else if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 8729 struct nvme_io_path *optimized_io_path = NULL; 8730 8731 STAILQ_FOREACH(optimized_io_path, &nbdev_ch->io_path_list, stailq) { 8732 if (optimized_io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) { 8733 break; 8734 } 8735 } 8736 8737 /* A non-optimized path is only current if there are no optimized paths. */ 8738 current = (io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) || 8739 (optimized_io_path == NULL); 8740 } else { 8741 if (nbdev_ch->current_io_path) { 8742 current = (io_path == nbdev_ch->current_io_path); 8743 } else { 8744 struct nvme_io_path *first_path; 8745 8746 /* We arrived here as there are no optimized paths for active-passive 8747 * mode. Check if this io_path is the first one available on the list. 8748 */ 8749 current = false; 8750 STAILQ_FOREACH(first_path, &nbdev_ch->io_path_list, stailq) { 8751 if (nvme_io_path_is_available(first_path)) { 8752 current = (io_path == first_path); 8753 break; 8754 } 8755 } 8756 } 8757 } 8758 8759 return current; 8760 } 8761 8762 void 8763 nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path) 8764 { 8765 struct nvme_ns *nvme_ns = io_path->nvme_ns; 8766 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 8767 const struct spdk_nvme_ctrlr_data *cdata; 8768 const struct spdk_nvme_transport_id *trid; 8769 const char *adrfam_str; 8770 8771 spdk_json_write_object_begin(w); 8772 8773 spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name); 8774 8775 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 8776 trid = spdk_nvme_ctrlr_get_transport_id(nvme_ctrlr->ctrlr); 8777 8778 spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid); 8779 spdk_json_write_named_bool(w, "current", nvme_io_path_is_current(io_path)); 8780 spdk_json_write_named_bool(w, "connected", nvme_qpair_is_connected(io_path->qpair)); 8781 spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns)); 8782 8783 spdk_json_write_named_object_begin(w, "transport"); 8784 spdk_json_write_named_string(w, "trtype", trid->trstring); 8785 spdk_json_write_named_string(w, "traddr", trid->traddr); 8786 if (trid->trsvcid[0] != '\0') { 8787 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 8788 } 8789 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 8790 if (adrfam_str) { 8791 spdk_json_write_named_string(w, "adrfam", adrfam_str); 8792 } 8793 spdk_json_write_object_end(w); 8794 8795 spdk_json_write_object_end(w); 8796 } 8797 8798 void 8799 bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w) 8800 { 8801 struct discovery_ctx *ctx; 8802 struct discovery_entry_ctx *entry_ctx; 8803 8804 spdk_json_write_array_begin(w); 8805 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 8806 spdk_json_write_object_begin(w); 8807 spdk_json_write_named_string(w, "name", ctx->name); 8808 8809 spdk_json_write_named_object_begin(w, "trid"); 8810 nvme_bdev_dump_trid_json(&ctx->trid, w); 8811 spdk_json_write_object_end(w); 8812 8813 spdk_json_write_named_array_begin(w, "referrals"); 8814 TAILQ_FOREACH(entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 8815 spdk_json_write_object_begin(w); 8816 spdk_json_write_named_object_begin(w, "trid"); 8817 nvme_bdev_dump_trid_json(&entry_ctx->trid, w); 8818 spdk_json_write_object_end(w); 8819 spdk_json_write_object_end(w); 8820 } 8821 spdk_json_write_array_end(w); 8822 8823 spdk_json_write_object_end(w); 8824 } 8825 spdk_json_write_array_end(w); 8826 } 8827 8828 SPDK_LOG_REGISTER_COMPONENT(bdev_nvme) 8829 8830 SPDK_TRACE_REGISTER_FN(bdev_nvme_trace, "bdev_nvme", TRACE_GROUP_BDEV_NVME) 8831 { 8832 struct spdk_trace_tpoint_opts opts[] = { 8833 { 8834 "BDEV_NVME_IO_START", TRACE_BDEV_NVME_IO_START, 8835 OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 1, 8836 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 8837 }, 8838 { 8839 "BDEV_NVME_IO_DONE", TRACE_BDEV_NVME_IO_DONE, 8840 OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 0, 8841 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 8842 } 8843 }; 8844 8845 8846 spdk_trace_register_object(OBJECT_BDEV_NVME_IO, 'N'); 8847 spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts)); 8848 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 8849 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 8850 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 8851 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 8852 } 8853