1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved. 6 */ 7 8 #include "spdk/stdinc.h" 9 10 #include "bdev_nvme.h" 11 12 #include "spdk/accel.h" 13 #include "spdk/config.h" 14 #include "spdk/endian.h" 15 #include "spdk/bdev.h" 16 #include "spdk/json.h" 17 #include "spdk/keyring.h" 18 #include "spdk/likely.h" 19 #include "spdk/nvme.h" 20 #include "spdk/nvme_ocssd.h" 21 #include "spdk/nvme_zns.h" 22 #include "spdk/opal.h" 23 #include "spdk/thread.h" 24 #include "spdk/trace.h" 25 #include "spdk/string.h" 26 #include "spdk/util.h" 27 #include "spdk/uuid.h" 28 29 #include "spdk/bdev_module.h" 30 #include "spdk/log.h" 31 32 #include "spdk_internal/usdt.h" 33 #include "spdk_internal/trace_defs.h" 34 35 #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true 36 #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS (10000) 37 38 #define NSID_STR_LEN 10 39 40 #define SPDK_CONTROLLER_NAME_MAX 512 41 42 static int bdev_nvme_config_json(struct spdk_json_write_ctx *w); 43 44 struct nvme_bdev_io { 45 /** array of iovecs to transfer. */ 46 struct iovec *iovs; 47 48 /** Number of iovecs in iovs array. */ 49 int iovcnt; 50 51 /** Current iovec position. */ 52 int iovpos; 53 54 /** Offset in current iovec. */ 55 uint32_t iov_offset; 56 57 /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path 58 * being reset in a reset I/O. 59 */ 60 struct nvme_io_path *io_path; 61 62 /** array of iovecs to transfer. */ 63 struct iovec *fused_iovs; 64 65 /** Number of iovecs in iovs array. */ 66 int fused_iovcnt; 67 68 /** Current iovec position. */ 69 int fused_iovpos; 70 71 /** Offset in current iovec. */ 72 uint32_t fused_iov_offset; 73 74 /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */ 75 struct spdk_nvme_cpl cpl; 76 77 /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */ 78 struct spdk_nvme_ns_cmd_ext_io_opts ext_opts; 79 80 /** Keeps track if first of fused commands was submitted */ 81 bool first_fused_submitted; 82 83 /** Keeps track if first of fused commands was completed */ 84 bool first_fused_completed; 85 86 /** Temporary pointer to zone report buffer */ 87 struct spdk_nvme_zns_zone_report *zone_report_buf; 88 89 /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */ 90 uint64_t handled_zones; 91 92 /** Expiration value in ticks to retry the current I/O. */ 93 uint64_t retry_ticks; 94 95 /* How many times the current I/O was retried. */ 96 int32_t retry_count; 97 98 /* Current tsc at submit time. */ 99 uint64_t submit_tsc; 100 101 /* Used to put nvme_bdev_io into the list */ 102 TAILQ_ENTRY(nvme_bdev_io) retry_link; 103 }; 104 105 struct nvme_probe_skip_entry { 106 struct spdk_nvme_transport_id trid; 107 TAILQ_ENTRY(nvme_probe_skip_entry) tailq; 108 }; 109 /* All the controllers deleted by users via RPC are skipped by hotplug monitor */ 110 static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER( 111 g_skipped_nvme_ctrlrs); 112 113 #define BDEV_NVME_DEFAULT_DIGESTS (SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA256) | \ 114 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA384) | \ 115 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA512)) 116 117 #define BDEV_NVME_DEFAULT_DHGROUPS (SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_NULL) | \ 118 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_2048) | \ 119 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_3072) | \ 120 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_4096) | \ 121 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_6144) | \ 122 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_8192)) 123 124 static struct spdk_bdev_nvme_opts g_opts = { 125 .action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE, 126 .timeout_us = 0, 127 .timeout_admin_us = 0, 128 .keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS, 129 .transport_retry_count = 4, 130 .arbitration_burst = 0, 131 .low_priority_weight = 0, 132 .medium_priority_weight = 0, 133 .high_priority_weight = 0, 134 .nvme_adminq_poll_period_us = 10000ULL, 135 .nvme_ioq_poll_period_us = 0, 136 .io_queue_requests = 0, 137 .delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT, 138 .bdev_retry_count = 3, 139 .transport_ack_timeout = 0, 140 .ctrlr_loss_timeout_sec = 0, 141 .reconnect_delay_sec = 0, 142 .fast_io_fail_timeout_sec = 0, 143 .disable_auto_failback = false, 144 .generate_uuids = false, 145 .transport_tos = 0, 146 .nvme_error_stat = false, 147 .io_path_stat = false, 148 .allow_accel_sequence = false, 149 .dhchap_digests = BDEV_NVME_DEFAULT_DIGESTS, 150 .dhchap_dhgroups = BDEV_NVME_DEFAULT_DHGROUPS, 151 }; 152 153 #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL 154 #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL 155 156 static int g_hot_insert_nvme_controller_index = 0; 157 static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT; 158 static bool g_nvme_hotplug_enabled = false; 159 struct spdk_thread *g_bdev_nvme_init_thread; 160 static struct spdk_poller *g_hotplug_poller; 161 static struct spdk_poller *g_hotplug_probe_poller; 162 static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx; 163 164 static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 165 struct nvme_async_probe_ctx *ctx); 166 static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 167 struct nvme_async_probe_ctx *ctx); 168 static int bdev_nvme_library_init(void); 169 static void bdev_nvme_library_fini(void); 170 static void _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, 171 struct spdk_bdev_io *bdev_io); 172 static void bdev_nvme_submit_request(struct spdk_io_channel *ch, 173 struct spdk_bdev_io *bdev_io); 174 static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 175 void *md, uint64_t lba_count, uint64_t lba, 176 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx, 177 struct spdk_accel_sequence *seq); 178 static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 179 void *md, uint64_t lba_count, uint64_t lba); 180 static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 181 void *md, uint64_t lba_count, uint64_t lba, 182 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx, 183 struct spdk_accel_sequence *seq, 184 union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13); 185 static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 186 void *md, uint64_t lba_count, 187 uint64_t zslba, uint32_t flags); 188 static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 189 void *md, uint64_t lba_count, uint64_t lba, 190 uint32_t flags); 191 static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, 192 struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov, 193 int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba, 194 uint32_t flags); 195 static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, 196 uint32_t num_zones, struct spdk_bdev_zone_info *info); 197 static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 198 enum spdk_bdev_zone_action action); 199 static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, 200 struct nvme_bdev_io *bio, 201 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes); 202 static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 203 void *buf, size_t nbytes); 204 static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 205 void *buf, size_t nbytes, void *md_buf, size_t md_len); 206 static int bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 207 struct iovec *iov, int iovcnt, size_t nbytes, 208 void *md_buf, size_t md_len); 209 static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, 210 struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort); 211 static void bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio); 212 static int bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 213 static int bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 214 static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr); 215 static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr); 216 217 static struct nvme_ns *nvme_ns_alloc(void); 218 static void nvme_ns_free(struct nvme_ns *ns); 219 220 static int 221 nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2) 222 { 223 return ns1->id < ns2->id ? -1 : ns1->id > ns2->id; 224 } 225 226 RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp); 227 228 struct spdk_nvme_qpair * 229 bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch) 230 { 231 struct nvme_ctrlr_channel *ctrlr_ch; 232 233 assert(ctrlr_io_ch != NULL); 234 235 ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch); 236 237 return ctrlr_ch->qpair->qpair; 238 } 239 240 static int 241 bdev_nvme_get_ctx_size(void) 242 { 243 return sizeof(struct nvme_bdev_io); 244 } 245 246 static struct spdk_bdev_module nvme_if = { 247 .name = "nvme", 248 .async_fini = true, 249 .module_init = bdev_nvme_library_init, 250 .module_fini = bdev_nvme_library_fini, 251 .config_json = bdev_nvme_config_json, 252 .get_ctx_size = bdev_nvme_get_ctx_size, 253 254 }; 255 SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if) 256 257 struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs); 258 pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER; 259 bool g_bdev_nvme_module_finish; 260 261 struct nvme_bdev_ctrlr * 262 nvme_bdev_ctrlr_get_by_name(const char *name) 263 { 264 struct nvme_bdev_ctrlr *nbdev_ctrlr; 265 266 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 267 if (strcmp(name, nbdev_ctrlr->name) == 0) { 268 break; 269 } 270 } 271 272 return nbdev_ctrlr; 273 } 274 275 static struct nvme_ctrlr * 276 nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr, 277 const struct spdk_nvme_transport_id *trid, const char *hostnqn) 278 { 279 const struct spdk_nvme_ctrlr_opts *opts; 280 struct nvme_ctrlr *nvme_ctrlr; 281 282 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 283 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 284 if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0 && 285 strcmp(hostnqn, opts->hostnqn) == 0) { 286 break; 287 } 288 } 289 290 return nvme_ctrlr; 291 } 292 293 struct nvme_ctrlr * 294 nvme_bdev_ctrlr_get_ctrlr_by_id(struct nvme_bdev_ctrlr *nbdev_ctrlr, 295 uint16_t cntlid) 296 { 297 struct nvme_ctrlr *nvme_ctrlr; 298 const struct spdk_nvme_ctrlr_data *cdata; 299 300 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 301 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 302 if (cdata->cntlid == cntlid) { 303 break; 304 } 305 } 306 307 return nvme_ctrlr; 308 } 309 310 static struct nvme_bdev * 311 nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid) 312 { 313 struct nvme_bdev *bdev; 314 315 pthread_mutex_lock(&g_bdev_nvme_mutex); 316 TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) { 317 if (bdev->nsid == nsid) { 318 break; 319 } 320 } 321 pthread_mutex_unlock(&g_bdev_nvme_mutex); 322 323 return bdev; 324 } 325 326 struct nvme_ns * 327 nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid) 328 { 329 struct nvme_ns ns; 330 331 assert(nsid > 0); 332 333 ns.id = nsid; 334 return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns); 335 } 336 337 struct nvme_ns * 338 nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr) 339 { 340 return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces); 341 } 342 343 struct nvme_ns * 344 nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns) 345 { 346 if (ns == NULL) { 347 return NULL; 348 } 349 350 return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 351 } 352 353 static struct nvme_ctrlr * 354 nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid, const char *hostnqn) 355 { 356 struct nvme_bdev_ctrlr *nbdev_ctrlr; 357 struct nvme_ctrlr *nvme_ctrlr = NULL; 358 359 pthread_mutex_lock(&g_bdev_nvme_mutex); 360 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 361 nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid, hostnqn); 362 if (nvme_ctrlr != NULL) { 363 break; 364 } 365 } 366 pthread_mutex_unlock(&g_bdev_nvme_mutex); 367 368 return nvme_ctrlr; 369 } 370 371 struct nvme_ctrlr * 372 nvme_ctrlr_get_by_name(const char *name) 373 { 374 struct nvme_bdev_ctrlr *nbdev_ctrlr; 375 struct nvme_ctrlr *nvme_ctrlr = NULL; 376 377 if (name == NULL) { 378 return NULL; 379 } 380 381 pthread_mutex_lock(&g_bdev_nvme_mutex); 382 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 383 if (nbdev_ctrlr != NULL) { 384 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 385 } 386 pthread_mutex_unlock(&g_bdev_nvme_mutex); 387 388 return nvme_ctrlr; 389 } 390 391 void 392 nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx) 393 { 394 struct nvme_bdev_ctrlr *nbdev_ctrlr; 395 396 pthread_mutex_lock(&g_bdev_nvme_mutex); 397 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 398 fn(nbdev_ctrlr, ctx); 399 } 400 pthread_mutex_unlock(&g_bdev_nvme_mutex); 401 } 402 403 struct nvme_ctrlr_channel_iter { 404 nvme_ctrlr_for_each_channel_msg fn; 405 nvme_ctrlr_for_each_channel_done cpl; 406 struct spdk_io_channel_iter *i; 407 void *ctx; 408 }; 409 410 void 411 nvme_ctrlr_for_each_channel_continue(struct nvme_ctrlr_channel_iter *iter, int status) 412 { 413 spdk_for_each_channel_continue(iter->i, status); 414 } 415 416 static void 417 nvme_ctrlr_each_channel_msg(struct spdk_io_channel_iter *i) 418 { 419 struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 420 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 421 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 422 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch); 423 424 iter->i = i; 425 iter->fn(iter, nvme_ctrlr, ctrlr_ch, iter->ctx); 426 } 427 428 static void 429 nvme_ctrlr_each_channel_cpl(struct spdk_io_channel_iter *i, int status) 430 { 431 struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 432 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 433 434 iter->i = i; 435 iter->cpl(nvme_ctrlr, iter->ctx, status); 436 437 free(iter); 438 } 439 440 void 441 nvme_ctrlr_for_each_channel(struct nvme_ctrlr *nvme_ctrlr, 442 nvme_ctrlr_for_each_channel_msg fn, void *ctx, 443 nvme_ctrlr_for_each_channel_done cpl) 444 { 445 struct nvme_ctrlr_channel_iter *iter; 446 447 assert(nvme_ctrlr != NULL && fn != NULL); 448 449 iter = calloc(1, sizeof(struct nvme_ctrlr_channel_iter)); 450 if (iter == NULL) { 451 SPDK_ERRLOG("Unable to allocate iterator\n"); 452 assert(false); 453 return; 454 } 455 456 iter->fn = fn; 457 iter->cpl = cpl; 458 iter->ctx = ctx; 459 460 spdk_for_each_channel(nvme_ctrlr, nvme_ctrlr_each_channel_msg, 461 iter, nvme_ctrlr_each_channel_cpl); 462 } 463 464 struct nvme_bdev_channel_iter { 465 nvme_bdev_for_each_channel_msg fn; 466 nvme_bdev_for_each_channel_done cpl; 467 struct spdk_io_channel_iter *i; 468 void *ctx; 469 }; 470 471 void 472 nvme_bdev_for_each_channel_continue(struct nvme_bdev_channel_iter *iter, int status) 473 { 474 spdk_for_each_channel_continue(iter->i, status); 475 } 476 477 static void 478 nvme_bdev_each_channel_msg(struct spdk_io_channel_iter *i) 479 { 480 struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 481 struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i); 482 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 483 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 484 485 iter->i = i; 486 iter->fn(iter, nbdev, nbdev_ch, iter->ctx); 487 } 488 489 static void 490 nvme_bdev_each_channel_cpl(struct spdk_io_channel_iter *i, int status) 491 { 492 struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 493 struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i); 494 495 iter->i = i; 496 iter->cpl(nbdev, iter->ctx, status); 497 498 free(iter); 499 } 500 501 void 502 nvme_bdev_for_each_channel(struct nvme_bdev *nbdev, 503 nvme_bdev_for_each_channel_msg fn, void *ctx, 504 nvme_bdev_for_each_channel_done cpl) 505 { 506 struct nvme_bdev_channel_iter *iter; 507 508 assert(nbdev != NULL && fn != NULL); 509 510 iter = calloc(1, sizeof(struct nvme_bdev_channel_iter)); 511 if (iter == NULL) { 512 SPDK_ERRLOG("Unable to allocate iterator\n"); 513 assert(false); 514 return; 515 } 516 517 iter->fn = fn; 518 iter->cpl = cpl; 519 iter->ctx = ctx; 520 521 spdk_for_each_channel(nbdev, nvme_bdev_each_channel_msg, iter, 522 nvme_bdev_each_channel_cpl); 523 } 524 525 void 526 nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w) 527 { 528 const char *trtype_str; 529 const char *adrfam_str; 530 531 trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype); 532 if (trtype_str) { 533 spdk_json_write_named_string(w, "trtype", trtype_str); 534 } 535 536 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 537 if (adrfam_str) { 538 spdk_json_write_named_string(w, "adrfam", adrfam_str); 539 } 540 541 if (trid->traddr[0] != '\0') { 542 spdk_json_write_named_string(w, "traddr", trid->traddr); 543 } 544 545 if (trid->trsvcid[0] != '\0') { 546 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 547 } 548 549 if (trid->subnqn[0] != '\0') { 550 spdk_json_write_named_string(w, "subnqn", trid->subnqn); 551 } 552 } 553 554 static void 555 nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr, 556 struct nvme_ctrlr *nvme_ctrlr) 557 { 558 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name); 559 pthread_mutex_lock(&g_bdev_nvme_mutex); 560 561 TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 562 if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) { 563 pthread_mutex_unlock(&g_bdev_nvme_mutex); 564 565 return; 566 } 567 TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 568 569 pthread_mutex_unlock(&g_bdev_nvme_mutex); 570 571 assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs)); 572 573 free(nbdev_ctrlr->name); 574 free(nbdev_ctrlr); 575 } 576 577 static void 578 _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 579 { 580 struct nvme_path_id *path_id, *tmp_path; 581 struct nvme_ns *ns, *tmp_ns; 582 583 free(nvme_ctrlr->copied_ana_desc); 584 spdk_free(nvme_ctrlr->ana_log_page); 585 586 if (nvme_ctrlr->opal_dev) { 587 spdk_opal_dev_destruct(nvme_ctrlr->opal_dev); 588 nvme_ctrlr->opal_dev = NULL; 589 } 590 591 if (nvme_ctrlr->nbdev_ctrlr) { 592 nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr); 593 } 594 595 RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) { 596 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 597 nvme_ns_free(ns); 598 } 599 600 TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) { 601 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 602 free(path_id); 603 } 604 605 pthread_mutex_destroy(&nvme_ctrlr->mutex); 606 spdk_keyring_put_key(nvme_ctrlr->psk); 607 spdk_keyring_put_key(nvme_ctrlr->dhchap_key); 608 spdk_keyring_put_key(nvme_ctrlr->dhchap_ctrlr_key); 609 free(nvme_ctrlr); 610 611 pthread_mutex_lock(&g_bdev_nvme_mutex); 612 if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 613 pthread_mutex_unlock(&g_bdev_nvme_mutex); 614 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 615 spdk_bdev_module_fini_done(); 616 return; 617 } 618 pthread_mutex_unlock(&g_bdev_nvme_mutex); 619 } 620 621 static int 622 nvme_detach_poller(void *arg) 623 { 624 struct nvme_ctrlr *nvme_ctrlr = arg; 625 int rc; 626 627 rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx); 628 if (rc != -EAGAIN) { 629 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 630 _nvme_ctrlr_delete(nvme_ctrlr); 631 } 632 633 return SPDK_POLLER_BUSY; 634 } 635 636 static void 637 nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 638 { 639 int rc; 640 641 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 642 643 /* First, unregister the adminq poller, as the driver will poll adminq if necessary */ 644 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 645 646 /* If we got here, the reset/detach poller cannot be active */ 647 assert(nvme_ctrlr->reset_detach_poller == NULL); 648 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller, 649 nvme_ctrlr, 1000); 650 if (nvme_ctrlr->reset_detach_poller == NULL) { 651 SPDK_ERRLOG("Failed to register detach poller\n"); 652 goto error; 653 } 654 655 rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx); 656 if (rc != 0) { 657 SPDK_ERRLOG("Failed to detach the NVMe controller\n"); 658 goto error; 659 } 660 661 return; 662 error: 663 /* We don't have a good way to handle errors here, so just do what we can and delete the 664 * controller without detaching the underlying NVMe device. 665 */ 666 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 667 _nvme_ctrlr_delete(nvme_ctrlr); 668 } 669 670 static void 671 nvme_ctrlr_unregister_cb(void *io_device) 672 { 673 struct nvme_ctrlr *nvme_ctrlr = io_device; 674 675 nvme_ctrlr_delete(nvme_ctrlr); 676 } 677 678 static void 679 nvme_ctrlr_unregister(void *ctx) 680 { 681 struct nvme_ctrlr *nvme_ctrlr = ctx; 682 683 spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb); 684 } 685 686 static bool 687 nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr) 688 { 689 if (!nvme_ctrlr->destruct) { 690 return false; 691 } 692 693 if (nvme_ctrlr->ref > 0) { 694 return false; 695 } 696 697 if (nvme_ctrlr->resetting) { 698 return false; 699 } 700 701 if (nvme_ctrlr->ana_log_page_updating) { 702 return false; 703 } 704 705 if (nvme_ctrlr->io_path_cache_clearing) { 706 return false; 707 } 708 709 return true; 710 } 711 712 static void 713 nvme_ctrlr_release(struct nvme_ctrlr *nvme_ctrlr) 714 { 715 pthread_mutex_lock(&nvme_ctrlr->mutex); 716 SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref); 717 718 assert(nvme_ctrlr->ref > 0); 719 nvme_ctrlr->ref--; 720 721 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 722 pthread_mutex_unlock(&nvme_ctrlr->mutex); 723 return; 724 } 725 726 pthread_mutex_unlock(&nvme_ctrlr->mutex); 727 728 spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr); 729 } 730 731 static void 732 bdev_nvme_clear_current_io_path(struct nvme_bdev_channel *nbdev_ch) 733 { 734 nbdev_ch->current_io_path = NULL; 735 nbdev_ch->rr_counter = 0; 736 } 737 738 static struct nvme_io_path * 739 _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 740 { 741 struct nvme_io_path *io_path; 742 743 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 744 if (io_path->nvme_ns == nvme_ns) { 745 break; 746 } 747 } 748 749 return io_path; 750 } 751 752 static struct nvme_io_path * 753 nvme_io_path_alloc(void) 754 { 755 struct nvme_io_path *io_path; 756 757 io_path = calloc(1, sizeof(*io_path)); 758 if (io_path == NULL) { 759 SPDK_ERRLOG("Failed to alloc io_path.\n"); 760 return NULL; 761 } 762 763 if (g_opts.io_path_stat) { 764 io_path->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 765 if (io_path->stat == NULL) { 766 free(io_path); 767 SPDK_ERRLOG("Failed to alloc io_path stat.\n"); 768 return NULL; 769 } 770 spdk_bdev_reset_io_stat(io_path->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 771 } 772 773 return io_path; 774 } 775 776 static void 777 nvme_io_path_free(struct nvme_io_path *io_path) 778 { 779 free(io_path->stat); 780 free(io_path); 781 } 782 783 static int 784 _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 785 { 786 struct nvme_io_path *io_path; 787 struct spdk_io_channel *ch; 788 struct nvme_ctrlr_channel *ctrlr_ch; 789 struct nvme_qpair *nvme_qpair; 790 791 io_path = nvme_io_path_alloc(); 792 if (io_path == NULL) { 793 return -ENOMEM; 794 } 795 796 io_path->nvme_ns = nvme_ns; 797 798 ch = spdk_get_io_channel(nvme_ns->ctrlr); 799 if (ch == NULL) { 800 nvme_io_path_free(io_path); 801 SPDK_ERRLOG("Failed to alloc io_channel.\n"); 802 return -ENOMEM; 803 } 804 805 ctrlr_ch = spdk_io_channel_get_ctx(ch); 806 807 nvme_qpair = ctrlr_ch->qpair; 808 assert(nvme_qpair != NULL); 809 810 io_path->qpair = nvme_qpair; 811 TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq); 812 813 io_path->nbdev_ch = nbdev_ch; 814 STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq); 815 816 bdev_nvme_clear_current_io_path(nbdev_ch); 817 818 return 0; 819 } 820 821 static void 822 bdev_nvme_clear_retry_io_path(struct nvme_bdev_channel *nbdev_ch, 823 struct nvme_io_path *io_path) 824 { 825 struct nvme_bdev_io *bio; 826 827 TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) { 828 if (bio->io_path == io_path) { 829 bio->io_path = NULL; 830 } 831 } 832 } 833 834 static void 835 _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path) 836 { 837 struct spdk_io_channel *ch; 838 struct nvme_qpair *nvme_qpair; 839 struct nvme_ctrlr_channel *ctrlr_ch; 840 struct nvme_bdev *nbdev; 841 842 nbdev = spdk_io_channel_get_io_device(spdk_io_channel_from_ctx(nbdev_ch)); 843 844 /* Add the statistics to nvme_ns before this path is destroyed. */ 845 pthread_mutex_lock(&nbdev->mutex); 846 if (nbdev->ref != 0 && io_path->nvme_ns->stat != NULL && io_path->stat != NULL) { 847 spdk_bdev_add_io_stat(io_path->nvme_ns->stat, io_path->stat); 848 } 849 pthread_mutex_unlock(&nbdev->mutex); 850 851 bdev_nvme_clear_current_io_path(nbdev_ch); 852 bdev_nvme_clear_retry_io_path(nbdev_ch, io_path); 853 854 STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq); 855 io_path->nbdev_ch = NULL; 856 857 nvme_qpair = io_path->qpair; 858 assert(nvme_qpair != NULL); 859 860 ctrlr_ch = nvme_qpair->ctrlr_ch; 861 assert(ctrlr_ch != NULL); 862 863 ch = spdk_io_channel_from_ctx(ctrlr_ch); 864 spdk_put_io_channel(ch); 865 866 /* After an io_path is removed, I/Os submitted to it may complete and update statistics 867 * of the io_path. To avoid heap-use-after-free error from this case, do not free the 868 * io_path here but free the io_path when the associated qpair is freed. It is ensured 869 * that all I/Os submitted to the io_path are completed when the associated qpair is freed. 870 */ 871 } 872 873 static void 874 _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch) 875 { 876 struct nvme_io_path *io_path, *tmp_io_path; 877 878 STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) { 879 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 880 } 881 } 882 883 static int 884 bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf) 885 { 886 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 887 struct nvme_bdev *nbdev = io_device; 888 struct nvme_ns *nvme_ns; 889 int rc; 890 891 STAILQ_INIT(&nbdev_ch->io_path_list); 892 TAILQ_INIT(&nbdev_ch->retry_io_list); 893 894 pthread_mutex_lock(&nbdev->mutex); 895 896 nbdev_ch->mp_policy = nbdev->mp_policy; 897 nbdev_ch->mp_selector = nbdev->mp_selector; 898 nbdev_ch->rr_min_io = nbdev->rr_min_io; 899 900 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 901 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 902 if (rc != 0) { 903 pthread_mutex_unlock(&nbdev->mutex); 904 905 _bdev_nvme_delete_io_paths(nbdev_ch); 906 return rc; 907 } 908 } 909 pthread_mutex_unlock(&nbdev->mutex); 910 911 return 0; 912 } 913 914 /* If cpl != NULL, complete the bdev_io with nvme status based on 'cpl'. 915 * If cpl == NULL, complete the bdev_io with bdev status based on 'status'. 916 */ 917 static inline void 918 __bdev_nvme_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status, 919 const struct spdk_nvme_cpl *cpl) 920 { 921 spdk_trace_record(TRACE_BDEV_NVME_IO_DONE, 0, 0, (uintptr_t)bdev_io->driver_ctx, 922 (uintptr_t)bdev_io); 923 if (cpl) { 924 spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc); 925 } else { 926 spdk_bdev_io_complete(bdev_io, status); 927 } 928 } 929 930 static void bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch); 931 932 static void 933 bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf) 934 { 935 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 936 937 bdev_nvme_abort_retry_ios(nbdev_ch); 938 _bdev_nvme_delete_io_paths(nbdev_ch); 939 } 940 941 static inline bool 942 bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type) 943 { 944 switch (io_type) { 945 case SPDK_BDEV_IO_TYPE_RESET: 946 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 947 case SPDK_BDEV_IO_TYPE_ABORT: 948 return true; 949 default: 950 break; 951 } 952 953 return false; 954 } 955 956 static inline bool 957 nvme_ns_is_active(struct nvme_ns *nvme_ns) 958 { 959 if (spdk_unlikely(nvme_ns->ana_state_updating)) { 960 return false; 961 } 962 963 if (spdk_unlikely(nvme_ns->ns == NULL)) { 964 return false; 965 } 966 967 return true; 968 } 969 970 static inline bool 971 nvme_ns_is_accessible(struct nvme_ns *nvme_ns) 972 { 973 if (spdk_unlikely(!nvme_ns_is_active(nvme_ns))) { 974 return false; 975 } 976 977 switch (nvme_ns->ana_state) { 978 case SPDK_NVME_ANA_OPTIMIZED_STATE: 979 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 980 return true; 981 default: 982 break; 983 } 984 985 return false; 986 } 987 988 static inline bool 989 nvme_qpair_is_connected(struct nvme_qpair *nvme_qpair) 990 { 991 if (spdk_unlikely(nvme_qpair->qpair == NULL)) { 992 return false; 993 } 994 995 if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 996 SPDK_NVME_QPAIR_FAILURE_NONE)) { 997 return false; 998 } 999 1000 if (spdk_unlikely(nvme_qpair->ctrlr_ch->reset_iter != NULL)) { 1001 return false; 1002 } 1003 1004 return true; 1005 } 1006 1007 static inline bool 1008 nvme_io_path_is_available(struct nvme_io_path *io_path) 1009 { 1010 if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) { 1011 return false; 1012 } 1013 1014 if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) { 1015 return false; 1016 } 1017 1018 return true; 1019 } 1020 1021 static inline bool 1022 nvme_ctrlr_is_failed(struct nvme_ctrlr *nvme_ctrlr) 1023 { 1024 if (nvme_ctrlr->destruct) { 1025 return true; 1026 } 1027 1028 if (nvme_ctrlr->fast_io_fail_timedout) { 1029 return true; 1030 } 1031 1032 if (nvme_ctrlr->resetting) { 1033 if (nvme_ctrlr->opts.reconnect_delay_sec != 0) { 1034 return false; 1035 } else { 1036 return true; 1037 } 1038 } 1039 1040 if (nvme_ctrlr->reconnect_is_delayed) { 1041 return false; 1042 } 1043 1044 if (nvme_ctrlr->disabled) { 1045 return true; 1046 } 1047 1048 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 1049 return true; 1050 } else { 1051 return false; 1052 } 1053 } 1054 1055 static bool 1056 nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr) 1057 { 1058 if (nvme_ctrlr->destruct) { 1059 return false; 1060 } 1061 1062 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 1063 return false; 1064 } 1065 1066 if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) { 1067 return false; 1068 } 1069 1070 if (nvme_ctrlr->disabled) { 1071 return false; 1072 } 1073 1074 return true; 1075 } 1076 1077 /* Simulate circular linked list. */ 1078 static inline struct nvme_io_path * 1079 nvme_io_path_get_next(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *prev_path) 1080 { 1081 struct nvme_io_path *next_path; 1082 1083 if (prev_path != NULL) { 1084 next_path = STAILQ_NEXT(prev_path, stailq); 1085 if (next_path != NULL) { 1086 return next_path; 1087 } 1088 } 1089 1090 return STAILQ_FIRST(&nbdev_ch->io_path_list); 1091 } 1092 1093 static struct nvme_io_path * 1094 _bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 1095 { 1096 struct nvme_io_path *io_path, *start, *non_optimized = NULL; 1097 1098 start = nvme_io_path_get_next(nbdev_ch, nbdev_ch->current_io_path); 1099 1100 io_path = start; 1101 do { 1102 if (spdk_likely(nvme_io_path_is_available(io_path))) { 1103 switch (io_path->nvme_ns->ana_state) { 1104 case SPDK_NVME_ANA_OPTIMIZED_STATE: 1105 nbdev_ch->current_io_path = io_path; 1106 return io_path; 1107 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 1108 if (non_optimized == NULL) { 1109 non_optimized = io_path; 1110 } 1111 break; 1112 default: 1113 assert(false); 1114 break; 1115 } 1116 } 1117 io_path = nvme_io_path_get_next(nbdev_ch, io_path); 1118 } while (io_path != start); 1119 1120 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 1121 /* We come here only if there is no optimized path. Cache even non_optimized 1122 * path for load balance across multiple non_optimized paths. 1123 */ 1124 nbdev_ch->current_io_path = non_optimized; 1125 } 1126 1127 return non_optimized; 1128 } 1129 1130 static struct nvme_io_path * 1131 _bdev_nvme_find_io_path_min_qd(struct nvme_bdev_channel *nbdev_ch) 1132 { 1133 struct nvme_io_path *io_path; 1134 struct nvme_io_path *optimized = NULL, *non_optimized = NULL; 1135 uint32_t opt_min_qd = UINT32_MAX, non_opt_min_qd = UINT32_MAX; 1136 uint32_t num_outstanding_reqs; 1137 1138 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 1139 if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) { 1140 /* The device is currently resetting. */ 1141 continue; 1142 } 1143 1144 if (spdk_unlikely(!nvme_ns_is_active(io_path->nvme_ns))) { 1145 continue; 1146 } 1147 1148 num_outstanding_reqs = spdk_nvme_qpair_get_num_outstanding_reqs(io_path->qpair->qpair); 1149 switch (io_path->nvme_ns->ana_state) { 1150 case SPDK_NVME_ANA_OPTIMIZED_STATE: 1151 if (num_outstanding_reqs < opt_min_qd) { 1152 opt_min_qd = num_outstanding_reqs; 1153 optimized = io_path; 1154 } 1155 break; 1156 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 1157 if (num_outstanding_reqs < non_opt_min_qd) { 1158 non_opt_min_qd = num_outstanding_reqs; 1159 non_optimized = io_path; 1160 } 1161 break; 1162 default: 1163 break; 1164 } 1165 } 1166 1167 /* don't cache io path for BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH selector */ 1168 if (optimized != NULL) { 1169 return optimized; 1170 } 1171 1172 return non_optimized; 1173 } 1174 1175 static inline struct nvme_io_path * 1176 bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 1177 { 1178 if (spdk_likely(nbdev_ch->current_io_path != NULL)) { 1179 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE) { 1180 return nbdev_ch->current_io_path; 1181 } else if (nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 1182 if (++nbdev_ch->rr_counter < nbdev_ch->rr_min_io) { 1183 return nbdev_ch->current_io_path; 1184 } 1185 nbdev_ch->rr_counter = 0; 1186 } 1187 } 1188 1189 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE || 1190 nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 1191 return _bdev_nvme_find_io_path(nbdev_ch); 1192 } else { 1193 return _bdev_nvme_find_io_path_min_qd(nbdev_ch); 1194 } 1195 } 1196 1197 /* Return true if there is any io_path whose qpair is active or ctrlr is not failed, 1198 * or false otherwise. 1199 * 1200 * If any io_path has an active qpair but find_io_path() returned NULL, its namespace 1201 * is likely to be non-accessible now but may become accessible. 1202 * 1203 * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr 1204 * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed 1205 * when starting to reset it but it is set to failed when the reset failed. Hence, if 1206 * a ctrlr is unfailed, it is likely that it works fine or is resetting. 1207 */ 1208 static bool 1209 any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch) 1210 { 1211 struct nvme_io_path *io_path; 1212 1213 if (nbdev_ch->resetting) { 1214 return false; 1215 } 1216 1217 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 1218 if (io_path->nvme_ns->ana_transition_timedout) { 1219 continue; 1220 } 1221 1222 if (nvme_qpair_is_connected(io_path->qpair) || 1223 !nvme_ctrlr_is_failed(io_path->qpair->ctrlr)) { 1224 return true; 1225 } 1226 } 1227 1228 return false; 1229 } 1230 1231 static void 1232 bdev_nvme_retry_io(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 1233 { 1234 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1235 struct spdk_io_channel *ch; 1236 1237 if (nbdev_io->io_path != NULL && nvme_io_path_is_available(nbdev_io->io_path)) { 1238 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 1239 } else { 1240 ch = spdk_io_channel_from_ctx(nbdev_ch); 1241 bdev_nvme_submit_request(ch, bdev_io); 1242 } 1243 } 1244 1245 static int 1246 bdev_nvme_retry_ios(void *arg) 1247 { 1248 struct nvme_bdev_channel *nbdev_ch = arg; 1249 struct nvme_bdev_io *bio, *tmp_bio; 1250 uint64_t now, delay_us; 1251 1252 now = spdk_get_ticks(); 1253 1254 TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) { 1255 if (bio->retry_ticks > now) { 1256 break; 1257 } 1258 1259 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1260 1261 bdev_nvme_retry_io(nbdev_ch, spdk_bdev_io_from_ctx(bio)); 1262 } 1263 1264 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1265 1266 bio = TAILQ_FIRST(&nbdev_ch->retry_io_list); 1267 if (bio != NULL) { 1268 delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz(); 1269 1270 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1271 delay_us); 1272 } 1273 1274 return SPDK_POLLER_BUSY; 1275 } 1276 1277 static void 1278 bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch, 1279 struct nvme_bdev_io *bio, uint64_t delay_ms) 1280 { 1281 struct nvme_bdev_io *tmp_bio; 1282 1283 bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL; 1284 1285 TAILQ_FOREACH_REVERSE(tmp_bio, &nbdev_ch->retry_io_list, retry_io_head, retry_link) { 1286 if (tmp_bio->retry_ticks <= bio->retry_ticks) { 1287 TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bio, bio, 1288 retry_link); 1289 return; 1290 } 1291 } 1292 1293 /* No earlier I/Os were found. This I/O must be the new head. */ 1294 TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bio, retry_link); 1295 1296 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1297 1298 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1299 delay_ms * 1000ULL); 1300 } 1301 1302 static void 1303 bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch) 1304 { 1305 struct nvme_bdev_io *bio, *tmp_bio; 1306 1307 TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) { 1308 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1309 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1310 } 1311 1312 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1313 } 1314 1315 static int 1316 bdev_nvme_abort_retry_io(struct nvme_bdev_channel *nbdev_ch, 1317 struct nvme_bdev_io *bio_to_abort) 1318 { 1319 struct nvme_bdev_io *bio; 1320 1321 TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) { 1322 if (bio == bio_to_abort) { 1323 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1324 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1325 return 0; 1326 } 1327 } 1328 1329 return -ENOENT; 1330 } 1331 1332 static void 1333 bdev_nvme_update_nvme_error_stat(struct spdk_bdev_io *bdev_io, const struct spdk_nvme_cpl *cpl) 1334 { 1335 struct nvme_bdev *nbdev; 1336 uint16_t sct, sc; 1337 1338 assert(spdk_nvme_cpl_is_error(cpl)); 1339 1340 nbdev = bdev_io->bdev->ctxt; 1341 1342 if (nbdev->err_stat == NULL) { 1343 return; 1344 } 1345 1346 sct = cpl->status.sct; 1347 sc = cpl->status.sc; 1348 1349 pthread_mutex_lock(&nbdev->mutex); 1350 1351 nbdev->err_stat->status_type[sct]++; 1352 switch (sct) { 1353 case SPDK_NVME_SCT_GENERIC: 1354 case SPDK_NVME_SCT_COMMAND_SPECIFIC: 1355 case SPDK_NVME_SCT_MEDIA_ERROR: 1356 case SPDK_NVME_SCT_PATH: 1357 nbdev->err_stat->status[sct][sc]++; 1358 break; 1359 default: 1360 break; 1361 } 1362 1363 pthread_mutex_unlock(&nbdev->mutex); 1364 } 1365 1366 static inline void 1367 bdev_nvme_update_io_path_stat(struct nvme_bdev_io *bio) 1368 { 1369 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1370 uint64_t num_blocks = bdev_io->u.bdev.num_blocks; 1371 uint32_t blocklen = bdev_io->bdev->blocklen; 1372 struct spdk_bdev_io_stat *stat; 1373 uint64_t tsc_diff; 1374 1375 if (bio->io_path->stat == NULL) { 1376 return; 1377 } 1378 1379 tsc_diff = spdk_get_ticks() - bio->submit_tsc; 1380 stat = bio->io_path->stat; 1381 1382 switch (bdev_io->type) { 1383 case SPDK_BDEV_IO_TYPE_READ: 1384 stat->bytes_read += num_blocks * blocklen; 1385 stat->num_read_ops++; 1386 stat->read_latency_ticks += tsc_diff; 1387 if (stat->max_read_latency_ticks < tsc_diff) { 1388 stat->max_read_latency_ticks = tsc_diff; 1389 } 1390 if (stat->min_read_latency_ticks > tsc_diff) { 1391 stat->min_read_latency_ticks = tsc_diff; 1392 } 1393 break; 1394 case SPDK_BDEV_IO_TYPE_WRITE: 1395 stat->bytes_written += num_blocks * blocklen; 1396 stat->num_write_ops++; 1397 stat->write_latency_ticks += tsc_diff; 1398 if (stat->max_write_latency_ticks < tsc_diff) { 1399 stat->max_write_latency_ticks = tsc_diff; 1400 } 1401 if (stat->min_write_latency_ticks > tsc_diff) { 1402 stat->min_write_latency_ticks = tsc_diff; 1403 } 1404 break; 1405 case SPDK_BDEV_IO_TYPE_UNMAP: 1406 stat->bytes_unmapped += num_blocks * blocklen; 1407 stat->num_unmap_ops++; 1408 stat->unmap_latency_ticks += tsc_diff; 1409 if (stat->max_unmap_latency_ticks < tsc_diff) { 1410 stat->max_unmap_latency_ticks = tsc_diff; 1411 } 1412 if (stat->min_unmap_latency_ticks > tsc_diff) { 1413 stat->min_unmap_latency_ticks = tsc_diff; 1414 } 1415 break; 1416 case SPDK_BDEV_IO_TYPE_ZCOPY: 1417 /* Track the data in the start phase only */ 1418 if (!bdev_io->u.bdev.zcopy.start) { 1419 break; 1420 } 1421 if (bdev_io->u.bdev.zcopy.populate) { 1422 stat->bytes_read += num_blocks * blocklen; 1423 stat->num_read_ops++; 1424 stat->read_latency_ticks += tsc_diff; 1425 if (stat->max_read_latency_ticks < tsc_diff) { 1426 stat->max_read_latency_ticks = tsc_diff; 1427 } 1428 if (stat->min_read_latency_ticks > tsc_diff) { 1429 stat->min_read_latency_ticks = tsc_diff; 1430 } 1431 } else { 1432 stat->bytes_written += num_blocks * blocklen; 1433 stat->num_write_ops++; 1434 stat->write_latency_ticks += tsc_diff; 1435 if (stat->max_write_latency_ticks < tsc_diff) { 1436 stat->max_write_latency_ticks = tsc_diff; 1437 } 1438 if (stat->min_write_latency_ticks > tsc_diff) { 1439 stat->min_write_latency_ticks = tsc_diff; 1440 } 1441 } 1442 break; 1443 case SPDK_BDEV_IO_TYPE_COPY: 1444 stat->bytes_copied += num_blocks * blocklen; 1445 stat->num_copy_ops++; 1446 stat->copy_latency_ticks += tsc_diff; 1447 if (stat->max_copy_latency_ticks < tsc_diff) { 1448 stat->max_copy_latency_ticks = tsc_diff; 1449 } 1450 if (stat->min_copy_latency_ticks > tsc_diff) { 1451 stat->min_copy_latency_ticks = tsc_diff; 1452 } 1453 break; 1454 default: 1455 break; 1456 } 1457 } 1458 1459 static bool 1460 bdev_nvme_check_retry_io(struct nvme_bdev_io *bio, 1461 const struct spdk_nvme_cpl *cpl, 1462 struct nvme_bdev_channel *nbdev_ch, 1463 uint64_t *_delay_ms) 1464 { 1465 struct nvme_io_path *io_path = bio->io_path; 1466 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 1467 const struct spdk_nvme_ctrlr_data *cdata; 1468 1469 if (spdk_nvme_cpl_is_path_error(cpl) || 1470 spdk_nvme_cpl_is_aborted_sq_deletion(cpl) || 1471 !nvme_io_path_is_available(io_path) || 1472 !nvme_ctrlr_is_available(nvme_ctrlr)) { 1473 bdev_nvme_clear_current_io_path(nbdev_ch); 1474 bio->io_path = NULL; 1475 if (spdk_nvme_cpl_is_ana_error(cpl)) { 1476 if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) { 1477 io_path->nvme_ns->ana_state_updating = true; 1478 } 1479 } 1480 if (!any_io_path_may_become_available(nbdev_ch)) { 1481 return false; 1482 } 1483 *_delay_ms = 0; 1484 } else { 1485 bio->retry_count++; 1486 1487 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 1488 1489 if (cpl->status.crd != 0) { 1490 *_delay_ms = cdata->crdt[cpl->status.crd] * 100; 1491 } else { 1492 *_delay_ms = 0; 1493 } 1494 } 1495 1496 return true; 1497 } 1498 1499 static inline void 1500 bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio, 1501 const struct spdk_nvme_cpl *cpl) 1502 { 1503 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1504 struct nvme_bdev_channel *nbdev_ch; 1505 uint64_t delay_ms; 1506 1507 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 1508 1509 if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) { 1510 bdev_nvme_update_io_path_stat(bio); 1511 goto complete; 1512 } 1513 1514 /* Update error counts before deciding if retry is needed. 1515 * Hence, error counts may be more than the number of I/O errors. 1516 */ 1517 bdev_nvme_update_nvme_error_stat(bdev_io, cpl); 1518 1519 if (cpl->status.dnr != 0 || spdk_nvme_cpl_is_aborted_by_request(cpl) || 1520 (g_opts.bdev_retry_count != -1 && bio->retry_count >= g_opts.bdev_retry_count)) { 1521 goto complete; 1522 } 1523 1524 /* At this point we don't know whether the sequence was successfully executed or not, so we 1525 * cannot retry the IO */ 1526 if (bdev_io->u.bdev.accel_sequence != NULL) { 1527 goto complete; 1528 } 1529 1530 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1531 1532 if (bdev_nvme_check_retry_io(bio, cpl, nbdev_ch, &delay_ms)) { 1533 bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms); 1534 return; 1535 } 1536 1537 complete: 1538 bio->retry_count = 0; 1539 bio->submit_tsc = 0; 1540 bdev_io->u.bdev.accel_sequence = NULL; 1541 __bdev_nvme_io_complete(bdev_io, 0, cpl); 1542 } 1543 1544 static inline void 1545 bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc) 1546 { 1547 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1548 struct nvme_bdev_channel *nbdev_ch; 1549 enum spdk_bdev_io_status io_status; 1550 1551 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 1552 1553 switch (rc) { 1554 case 0: 1555 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1556 break; 1557 case -ENOMEM: 1558 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1559 break; 1560 case -ENXIO: 1561 if (g_opts.bdev_retry_count == -1 || bio->retry_count < g_opts.bdev_retry_count) { 1562 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1563 1564 bdev_nvme_clear_current_io_path(nbdev_ch); 1565 bio->io_path = NULL; 1566 1567 if (any_io_path_may_become_available(nbdev_ch)) { 1568 bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL); 1569 return; 1570 } 1571 } 1572 1573 /* fallthrough */ 1574 default: 1575 spdk_accel_sequence_abort(bdev_io->u.bdev.accel_sequence); 1576 bdev_io->u.bdev.accel_sequence = NULL; 1577 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1578 break; 1579 } 1580 1581 bio->retry_count = 0; 1582 bio->submit_tsc = 0; 1583 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1584 } 1585 1586 static inline void 1587 bdev_nvme_admin_complete(struct nvme_bdev_io *bio, int rc) 1588 { 1589 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1590 enum spdk_bdev_io_status io_status; 1591 1592 switch (rc) { 1593 case 0: 1594 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1595 break; 1596 case -ENOMEM: 1597 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1598 break; 1599 case -ENXIO: 1600 /* fallthrough */ 1601 default: 1602 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1603 break; 1604 } 1605 1606 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1607 } 1608 1609 static void 1610 bdev_nvme_clear_io_path_caches_done(struct nvme_ctrlr *nvme_ctrlr, 1611 void *ctx, int status) 1612 { 1613 pthread_mutex_lock(&nvme_ctrlr->mutex); 1614 1615 assert(nvme_ctrlr->io_path_cache_clearing == true); 1616 nvme_ctrlr->io_path_cache_clearing = false; 1617 1618 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 1619 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1620 return; 1621 } 1622 1623 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1624 1625 nvme_ctrlr_unregister(nvme_ctrlr); 1626 } 1627 1628 static void 1629 _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair) 1630 { 1631 struct nvme_io_path *io_path; 1632 1633 TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) { 1634 if (io_path->nbdev_ch == NULL) { 1635 continue; 1636 } 1637 bdev_nvme_clear_current_io_path(io_path->nbdev_ch); 1638 } 1639 } 1640 1641 static void 1642 bdev_nvme_clear_io_path_cache(struct nvme_ctrlr_channel_iter *i, 1643 struct nvme_ctrlr *nvme_ctrlr, 1644 struct nvme_ctrlr_channel *ctrlr_ch, 1645 void *ctx) 1646 { 1647 assert(ctrlr_ch->qpair != NULL); 1648 1649 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 1650 1651 nvme_ctrlr_for_each_channel_continue(i, 0); 1652 } 1653 1654 static void 1655 bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr) 1656 { 1657 pthread_mutex_lock(&nvme_ctrlr->mutex); 1658 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 1659 nvme_ctrlr->io_path_cache_clearing) { 1660 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1661 return; 1662 } 1663 1664 nvme_ctrlr->io_path_cache_clearing = true; 1665 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1666 1667 nvme_ctrlr_for_each_channel(nvme_ctrlr, 1668 bdev_nvme_clear_io_path_cache, 1669 NULL, 1670 bdev_nvme_clear_io_path_caches_done); 1671 } 1672 1673 static struct nvme_qpair * 1674 nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair) 1675 { 1676 struct nvme_qpair *nvme_qpair; 1677 1678 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1679 if (nvme_qpair->qpair == qpair) { 1680 break; 1681 } 1682 } 1683 1684 return nvme_qpair; 1685 } 1686 1687 static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair); 1688 1689 static void 1690 bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx) 1691 { 1692 struct nvme_poll_group *group = poll_group_ctx; 1693 struct nvme_qpair *nvme_qpair; 1694 struct nvme_ctrlr_channel *ctrlr_ch; 1695 int status; 1696 1697 nvme_qpair = nvme_poll_group_get_qpair(group, qpair); 1698 if (nvme_qpair == NULL) { 1699 return; 1700 } 1701 1702 if (nvme_qpair->qpair != NULL) { 1703 spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair); 1704 nvme_qpair->qpair = NULL; 1705 } 1706 1707 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1708 1709 ctrlr_ch = nvme_qpair->ctrlr_ch; 1710 1711 if (ctrlr_ch != NULL) { 1712 if (ctrlr_ch->reset_iter != NULL) { 1713 /* We are in a full reset sequence. */ 1714 if (ctrlr_ch->connect_poller != NULL) { 1715 /* qpair was failed to connect. Abort the reset sequence. */ 1716 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was failed to connect. abort the reset ctrlr sequence.\n", 1717 qpair); 1718 spdk_poller_unregister(&ctrlr_ch->connect_poller); 1719 status = -1; 1720 } else { 1721 /* qpair was completed to disconnect. Just move to the next ctrlr_channel. */ 1722 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed in a reset ctrlr sequence.\n", 1723 qpair); 1724 status = 0; 1725 } 1726 nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, status); 1727 ctrlr_ch->reset_iter = NULL; 1728 } else { 1729 /* qpair was disconnected unexpectedly. Reset controller for recovery. */ 1730 SPDK_NOTICELOG("qpair %p was disconnected and freed. reset controller.\n", qpair); 1731 bdev_nvme_failover_ctrlr(nvme_qpair->ctrlr); 1732 } 1733 } else { 1734 /* In this case, ctrlr_channel is already deleted. */ 1735 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed. delete nvme_qpair.\n", qpair); 1736 nvme_qpair_delete(nvme_qpair); 1737 } 1738 } 1739 1740 static void 1741 bdev_nvme_check_io_qpairs(struct nvme_poll_group *group) 1742 { 1743 struct nvme_qpair *nvme_qpair; 1744 1745 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1746 if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) { 1747 continue; 1748 } 1749 1750 if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 1751 SPDK_NVME_QPAIR_FAILURE_NONE) { 1752 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1753 } 1754 } 1755 } 1756 1757 static int 1758 bdev_nvme_poll(void *arg) 1759 { 1760 struct nvme_poll_group *group = arg; 1761 int64_t num_completions; 1762 1763 if (group->collect_spin_stat && group->start_ticks == 0) { 1764 group->start_ticks = spdk_get_ticks(); 1765 } 1766 1767 num_completions = spdk_nvme_poll_group_process_completions(group->group, 0, 1768 bdev_nvme_disconnected_qpair_cb); 1769 if (group->collect_spin_stat) { 1770 if (num_completions > 0) { 1771 if (group->end_ticks != 0) { 1772 group->spin_ticks += (group->end_ticks - group->start_ticks); 1773 group->end_ticks = 0; 1774 } 1775 group->start_ticks = 0; 1776 } else { 1777 group->end_ticks = spdk_get_ticks(); 1778 } 1779 } 1780 1781 if (spdk_unlikely(num_completions < 0)) { 1782 bdev_nvme_check_io_qpairs(group); 1783 } 1784 1785 return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 1786 } 1787 1788 static int bdev_nvme_poll_adminq(void *arg); 1789 1790 static void 1791 bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_period_us) 1792 { 1793 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 1794 1795 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, 1796 nvme_ctrlr, new_period_us); 1797 } 1798 1799 static int 1800 bdev_nvme_poll_adminq(void *arg) 1801 { 1802 int32_t rc; 1803 struct nvme_ctrlr *nvme_ctrlr = arg; 1804 nvme_ctrlr_disconnected_cb disconnected_cb; 1805 1806 assert(nvme_ctrlr != NULL); 1807 1808 rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr); 1809 if (rc < 0) { 1810 disconnected_cb = nvme_ctrlr->disconnected_cb; 1811 nvme_ctrlr->disconnected_cb = NULL; 1812 1813 if (disconnected_cb != NULL) { 1814 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 1815 g_opts.nvme_adminq_poll_period_us); 1816 disconnected_cb(nvme_ctrlr); 1817 } else { 1818 bdev_nvme_failover_ctrlr(nvme_ctrlr); 1819 } 1820 } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) != 1821 SPDK_NVME_QPAIR_FAILURE_NONE) { 1822 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 1823 } 1824 1825 return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY; 1826 } 1827 1828 static void 1829 nvme_bdev_free(void *io_device) 1830 { 1831 struct nvme_bdev *nvme_disk = io_device; 1832 1833 pthread_mutex_destroy(&nvme_disk->mutex); 1834 free(nvme_disk->disk.name); 1835 free(nvme_disk->err_stat); 1836 free(nvme_disk); 1837 } 1838 1839 static int 1840 bdev_nvme_destruct(void *ctx) 1841 { 1842 struct nvme_bdev *nvme_disk = ctx; 1843 struct nvme_ns *nvme_ns, *tmp_nvme_ns; 1844 1845 SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid); 1846 1847 TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) { 1848 pthread_mutex_lock(&nvme_ns->ctrlr->mutex); 1849 1850 nvme_ns->bdev = NULL; 1851 1852 assert(nvme_ns->id > 0); 1853 1854 if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) { 1855 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1856 1857 nvme_ctrlr_release(nvme_ns->ctrlr); 1858 nvme_ns_free(nvme_ns); 1859 } else { 1860 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1861 } 1862 } 1863 1864 pthread_mutex_lock(&g_bdev_nvme_mutex); 1865 TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq); 1866 pthread_mutex_unlock(&g_bdev_nvme_mutex); 1867 1868 spdk_io_device_unregister(nvme_disk, nvme_bdev_free); 1869 1870 return 0; 1871 } 1872 1873 static int 1874 bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair) 1875 { 1876 struct nvme_ctrlr *nvme_ctrlr; 1877 struct spdk_nvme_io_qpair_opts opts; 1878 struct spdk_nvme_qpair *qpair; 1879 int rc; 1880 1881 nvme_ctrlr = nvme_qpair->ctrlr; 1882 1883 spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1884 opts.delay_cmd_submit = g_opts.delay_cmd_submit; 1885 opts.create_only = true; 1886 opts.async_mode = true; 1887 opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests); 1888 g_opts.io_queue_requests = opts.io_queue_requests; 1889 1890 qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1891 if (qpair == NULL) { 1892 return -1; 1893 } 1894 1895 SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name, 1896 spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread)); 1897 1898 assert(nvme_qpair->group != NULL); 1899 1900 rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair); 1901 if (rc != 0) { 1902 SPDK_ERRLOG("Unable to begin polling on NVMe Channel.\n"); 1903 goto err; 1904 } 1905 1906 rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair); 1907 if (rc != 0) { 1908 SPDK_ERRLOG("Unable to connect I/O qpair.\n"); 1909 goto err; 1910 } 1911 1912 nvme_qpair->qpair = qpair; 1913 1914 if (!g_opts.disable_auto_failback) { 1915 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1916 } 1917 1918 return 0; 1919 1920 err: 1921 spdk_nvme_ctrlr_free_io_qpair(qpair); 1922 1923 return rc; 1924 } 1925 1926 static void bdev_nvme_reset_io_continue(void *cb_arg, int rc); 1927 1928 static void 1929 bdev_nvme_complete_pending_resets(struct nvme_ctrlr_channel_iter *i, 1930 struct nvme_ctrlr *nvme_ctrlr, 1931 struct nvme_ctrlr_channel *ctrlr_ch, 1932 void *ctx) 1933 { 1934 int rc = 0; 1935 struct nvme_bdev_io *bio; 1936 1937 if (ctx != NULL) { 1938 rc = -1; 1939 } 1940 1941 while (!TAILQ_EMPTY(&ctrlr_ch->pending_resets)) { 1942 bio = TAILQ_FIRST(&ctrlr_ch->pending_resets); 1943 TAILQ_REMOVE(&ctrlr_ch->pending_resets, bio, retry_link); 1944 1945 bdev_nvme_reset_io_continue(bio, rc); 1946 } 1947 1948 nvme_ctrlr_for_each_channel_continue(i, 0); 1949 } 1950 1951 /* This function marks the current trid as failed by storing the current ticks 1952 * and then sets the next trid to the active trid within a controller if exists. 1953 * 1954 * The purpose of the boolean return value is to request the caller to disconnect 1955 * the current trid now to try connecting the next trid. 1956 */ 1957 static bool 1958 bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove, bool start) 1959 { 1960 struct nvme_path_id *path_id, *next_path; 1961 int rc __attribute__((unused)); 1962 1963 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 1964 assert(path_id); 1965 assert(path_id == nvme_ctrlr->active_path_id); 1966 next_path = TAILQ_NEXT(path_id, link); 1967 1968 /* Update the last failed time. It means the trid is failed if its last 1969 * failed time is non-zero. 1970 */ 1971 path_id->last_failed_tsc = spdk_get_ticks(); 1972 1973 if (next_path == NULL) { 1974 /* There is no alternate trid within a controller. */ 1975 return false; 1976 } 1977 1978 if (!start && nvme_ctrlr->opts.reconnect_delay_sec == 0) { 1979 /* Connect is not retried in a controller reset sequence. Connecting 1980 * the next trid will be done by the next bdev_nvme_failover_ctrlr() call. 1981 */ 1982 return false; 1983 } 1984 1985 assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE); 1986 1987 SPDK_NOTICELOG("Start failover from %s:%s to %s:%s\n", path_id->trid.traddr, 1988 path_id->trid.trsvcid, next_path->trid.traddr, next_path->trid.trsvcid); 1989 1990 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 1991 nvme_ctrlr->active_path_id = next_path; 1992 rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid); 1993 assert(rc == 0); 1994 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 1995 if (!remove) { 1996 /** Shuffle the old trid to the end of the list and use the new one. 1997 * Allows for round robin through multiple connections. 1998 */ 1999 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link); 2000 } else { 2001 free(path_id); 2002 } 2003 2004 if (start || next_path->last_failed_tsc == 0) { 2005 /* bdev_nvme_failover_ctrlr() is just called or the next trid is not failed 2006 * or used yet. Try the next trid now. 2007 */ 2008 return true; 2009 } 2010 2011 if (spdk_get_ticks() > next_path->last_failed_tsc + spdk_get_ticks_hz() * 2012 nvme_ctrlr->opts.reconnect_delay_sec) { 2013 /* Enough backoff passed since the next trid failed. Try the next trid now. */ 2014 return true; 2015 } 2016 2017 /* The next trid will be tried after reconnect_delay_sec seconds. */ 2018 return false; 2019 } 2020 2021 static bool 2022 bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr) 2023 { 2024 int32_t elapsed; 2025 2026 if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 || 2027 nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) { 2028 return false; 2029 } 2030 2031 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 2032 if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) { 2033 return true; 2034 } else { 2035 return false; 2036 } 2037 } 2038 2039 static bool 2040 bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr) 2041 { 2042 uint32_t elapsed; 2043 2044 if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) { 2045 return false; 2046 } 2047 2048 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 2049 if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) { 2050 return true; 2051 } else { 2052 return false; 2053 } 2054 } 2055 2056 static void bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success); 2057 2058 static void 2059 nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb cb_fn) 2060 { 2061 int rc; 2062 2063 rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr); 2064 if (rc != 0) { 2065 /* Disconnect fails if ctrlr is already resetting or removed. In this case, 2066 * fail the reset sequence immediately. 2067 */ 2068 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2069 return; 2070 } 2071 2072 /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq. 2073 * Set callback here to execute the specified operation after ctrlr is really disconnected. 2074 */ 2075 assert(nvme_ctrlr->disconnected_cb == NULL); 2076 nvme_ctrlr->disconnected_cb = cb_fn; 2077 2078 /* During disconnection, reduce the period to poll adminq more often. */ 2079 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0); 2080 } 2081 2082 enum bdev_nvme_op_after_reset { 2083 OP_NONE, 2084 OP_COMPLETE_PENDING_DESTRUCT, 2085 OP_DESTRUCT, 2086 OP_DELAYED_RECONNECT, 2087 OP_FAILOVER, 2088 }; 2089 2090 typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset; 2091 2092 static _bdev_nvme_op_after_reset 2093 bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success) 2094 { 2095 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 2096 /* Complete pending destruct after reset completes. */ 2097 return OP_COMPLETE_PENDING_DESTRUCT; 2098 } else if (nvme_ctrlr->pending_failover) { 2099 nvme_ctrlr->pending_failover = false; 2100 nvme_ctrlr->reset_start_tsc = 0; 2101 return OP_FAILOVER; 2102 } else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) { 2103 nvme_ctrlr->reset_start_tsc = 0; 2104 return OP_NONE; 2105 } else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 2106 return OP_DESTRUCT; 2107 } else { 2108 if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) { 2109 nvme_ctrlr->fast_io_fail_timedout = true; 2110 } 2111 return OP_DELAYED_RECONNECT; 2112 } 2113 } 2114 2115 static int bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug); 2116 static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 2117 2118 static int 2119 bdev_nvme_reconnect_delay_timer_expired(void *ctx) 2120 { 2121 struct nvme_ctrlr *nvme_ctrlr = ctx; 2122 2123 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect_delay, nvme_ctrlr->nbdev_ctrlr->name); 2124 pthread_mutex_lock(&nvme_ctrlr->mutex); 2125 2126 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2127 2128 if (!nvme_ctrlr->reconnect_is_delayed) { 2129 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2130 return SPDK_POLLER_BUSY; 2131 } 2132 2133 nvme_ctrlr->reconnect_is_delayed = false; 2134 2135 if (nvme_ctrlr->destruct) { 2136 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2137 return SPDK_POLLER_BUSY; 2138 } 2139 2140 assert(nvme_ctrlr->resetting == false); 2141 nvme_ctrlr->resetting = true; 2142 2143 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2144 2145 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 2146 2147 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2148 return SPDK_POLLER_BUSY; 2149 } 2150 2151 static void 2152 bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr) 2153 { 2154 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 2155 2156 assert(nvme_ctrlr->reconnect_is_delayed == false); 2157 nvme_ctrlr->reconnect_is_delayed = true; 2158 2159 assert(nvme_ctrlr->reconnect_delay_timer == NULL); 2160 nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired, 2161 nvme_ctrlr, 2162 nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC); 2163 } 2164 2165 static void remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr); 2166 2167 static void 2168 _bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2169 { 2170 bool success = (ctx == NULL); 2171 bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn; 2172 void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg; 2173 enum bdev_nvme_op_after_reset op_after_reset; 2174 2175 assert(nvme_ctrlr->thread == spdk_get_thread()); 2176 2177 nvme_ctrlr->ctrlr_op_cb_fn = NULL; 2178 nvme_ctrlr->ctrlr_op_cb_arg = NULL; 2179 2180 if (!success) { 2181 SPDK_ERRLOG("Resetting controller failed.\n"); 2182 } else { 2183 SPDK_NOTICELOG("Resetting controller successful.\n"); 2184 } 2185 2186 pthread_mutex_lock(&nvme_ctrlr->mutex); 2187 nvme_ctrlr->resetting = false; 2188 nvme_ctrlr->dont_retry = false; 2189 nvme_ctrlr->in_failover = false; 2190 2191 op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success); 2192 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2193 2194 /* Delay callbacks when the next operation is a failover. */ 2195 if (ctrlr_op_cb_fn && op_after_reset != OP_FAILOVER) { 2196 ctrlr_op_cb_fn(ctrlr_op_cb_arg, success ? 0 : -1); 2197 } 2198 2199 switch (op_after_reset) { 2200 case OP_COMPLETE_PENDING_DESTRUCT: 2201 nvme_ctrlr_unregister(nvme_ctrlr); 2202 break; 2203 case OP_DESTRUCT: 2204 bdev_nvme_delete_ctrlr(nvme_ctrlr, false); 2205 remove_discovery_entry(nvme_ctrlr); 2206 break; 2207 case OP_DELAYED_RECONNECT: 2208 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_start_reconnect_delay_timer); 2209 break; 2210 case OP_FAILOVER: 2211 nvme_ctrlr->ctrlr_op_cb_fn = ctrlr_op_cb_fn; 2212 nvme_ctrlr->ctrlr_op_cb_arg = ctrlr_op_cb_arg; 2213 bdev_nvme_failover_ctrlr(nvme_ctrlr); 2214 break; 2215 default: 2216 break; 2217 } 2218 } 2219 2220 static void 2221 bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success) 2222 { 2223 pthread_mutex_lock(&nvme_ctrlr->mutex); 2224 if (!success) { 2225 /* Connecting the active trid failed. Set the next alternate trid to the 2226 * active trid if it exists. 2227 */ 2228 if (bdev_nvme_failover_trid(nvme_ctrlr, false, false)) { 2229 /* The next alternate trid exists and is ready to try. Try it now. */ 2230 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2231 2232 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); 2233 return; 2234 } 2235 2236 /* We came here if there is no alternate trid or if the next trid exists but 2237 * is not ready to try. We will try the active trid after reconnect_delay_sec 2238 * seconds if it is non-zero or at the next reset call otherwise. 2239 */ 2240 } else { 2241 /* Connecting the active trid succeeded. Clear the last failed time because it 2242 * means the trid is failed if its last failed time is non-zero. 2243 */ 2244 nvme_ctrlr->active_path_id->last_failed_tsc = 0; 2245 } 2246 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2247 2248 /* Make sure we clear any pending resets before returning. */ 2249 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2250 bdev_nvme_complete_pending_resets, 2251 success ? NULL : (void *)0x1, 2252 _bdev_nvme_reset_ctrlr_complete); 2253 } 2254 2255 static void 2256 bdev_nvme_reset_create_qpairs_failed(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2257 { 2258 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2259 } 2260 2261 static void 2262 bdev_nvme_reset_destroy_qpair(struct nvme_ctrlr_channel_iter *i, 2263 struct nvme_ctrlr *nvme_ctrlr, 2264 struct nvme_ctrlr_channel *ctrlr_ch, void *ctx) 2265 { 2266 struct nvme_qpair *nvme_qpair; 2267 2268 nvme_qpair = ctrlr_ch->qpair; 2269 assert(nvme_qpair != NULL); 2270 2271 _bdev_nvme_clear_io_path_cache(nvme_qpair); 2272 2273 if (nvme_qpair->qpair != NULL) { 2274 if (nvme_qpair->ctrlr->dont_retry) { 2275 spdk_nvme_qpair_set_abort_dnr(nvme_qpair->qpair, true); 2276 } 2277 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 2278 2279 /* The current full reset sequence will move to the next 2280 * ctrlr_channel after the qpair is actually disconnected. 2281 */ 2282 assert(ctrlr_ch->reset_iter == NULL); 2283 ctrlr_ch->reset_iter = i; 2284 } else { 2285 nvme_ctrlr_for_each_channel_continue(i, 0); 2286 } 2287 } 2288 2289 static void 2290 bdev_nvme_reset_create_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2291 { 2292 if (status == 0) { 2293 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, true); 2294 } else { 2295 /* Delete the added qpairs and quiesce ctrlr to make the states clean. */ 2296 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2297 bdev_nvme_reset_destroy_qpair, 2298 NULL, 2299 bdev_nvme_reset_create_qpairs_failed); 2300 } 2301 } 2302 2303 static int 2304 bdev_nvme_reset_check_qpair_connected(void *ctx) 2305 { 2306 struct nvme_ctrlr_channel *ctrlr_ch = ctx; 2307 2308 if (ctrlr_ch->reset_iter == NULL) { 2309 /* qpair was already failed to connect and the reset sequence is being aborted. */ 2310 assert(ctrlr_ch->connect_poller == NULL); 2311 assert(ctrlr_ch->qpair->qpair == NULL); 2312 return SPDK_POLLER_BUSY; 2313 } 2314 2315 assert(ctrlr_ch->qpair->qpair != NULL); 2316 2317 if (!spdk_nvme_qpair_is_connected(ctrlr_ch->qpair->qpair)) { 2318 return SPDK_POLLER_BUSY; 2319 } 2320 2321 spdk_poller_unregister(&ctrlr_ch->connect_poller); 2322 2323 /* qpair was completed to connect. Move to the next ctrlr_channel */ 2324 nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 2325 ctrlr_ch->reset_iter = NULL; 2326 2327 if (!g_opts.disable_auto_failback) { 2328 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 2329 } 2330 2331 return SPDK_POLLER_BUSY; 2332 } 2333 2334 static void 2335 bdev_nvme_reset_create_qpair(struct nvme_ctrlr_channel_iter *i, 2336 struct nvme_ctrlr *nvme_ctrlr, 2337 struct nvme_ctrlr_channel *ctrlr_ch, 2338 void *ctx) 2339 { 2340 int rc; 2341 2342 rc = bdev_nvme_create_qpair(ctrlr_ch->qpair); 2343 if (rc == 0) { 2344 ctrlr_ch->connect_poller = SPDK_POLLER_REGISTER(bdev_nvme_reset_check_qpair_connected, 2345 ctrlr_ch, 0); 2346 2347 /* The current full reset sequence will move to the next 2348 * ctrlr_channel after the qpair is actually connected. 2349 */ 2350 assert(ctrlr_ch->reset_iter == NULL); 2351 ctrlr_ch->reset_iter = i; 2352 } else { 2353 nvme_ctrlr_for_each_channel_continue(i, rc); 2354 } 2355 } 2356 2357 static void 2358 nvme_ctrlr_check_namespaces(struct nvme_ctrlr *nvme_ctrlr) 2359 { 2360 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 2361 struct nvme_ns *nvme_ns; 2362 2363 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 2364 nvme_ns != NULL; 2365 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 2366 if (!spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 2367 SPDK_DEBUGLOG(bdev_nvme, "NSID %u was removed during reset.\n", nvme_ns->id); 2368 /* NS can be added again. Just nullify nvme_ns->ns. */ 2369 nvme_ns->ns = NULL; 2370 } 2371 } 2372 } 2373 2374 2375 static int 2376 bdev_nvme_reconnect_ctrlr_poll(void *arg) 2377 { 2378 struct nvme_ctrlr *nvme_ctrlr = arg; 2379 int rc = -ETIMEDOUT; 2380 2381 if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 2382 /* Mark the ctrlr as failed. The next call to 2383 * spdk_nvme_ctrlr_reconnect_poll_async() will then 2384 * do the necessary cleanup and return failure. 2385 */ 2386 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 2387 } 2388 2389 rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr); 2390 if (rc == -EAGAIN) { 2391 return SPDK_POLLER_BUSY; 2392 } 2393 2394 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 2395 if (rc == 0) { 2396 nvme_ctrlr_check_namespaces(nvme_ctrlr); 2397 2398 /* Recreate all of the I/O queue pairs */ 2399 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2400 bdev_nvme_reset_create_qpair, 2401 NULL, 2402 bdev_nvme_reset_create_qpairs_done); 2403 } else { 2404 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2405 } 2406 return SPDK_POLLER_BUSY; 2407 } 2408 2409 static void 2410 bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2411 { 2412 spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr); 2413 2414 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect, nvme_ctrlr->nbdev_ctrlr->name); 2415 assert(nvme_ctrlr->reset_detach_poller == NULL); 2416 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll, 2417 nvme_ctrlr, 0); 2418 } 2419 2420 static void 2421 bdev_nvme_reset_destroy_qpair_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2422 { 2423 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reset, nvme_ctrlr->nbdev_ctrlr->name); 2424 assert(status == 0); 2425 2426 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2427 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2428 } else { 2429 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); 2430 } 2431 } 2432 2433 static void 2434 bdev_nvme_reset_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr) 2435 { 2436 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2437 bdev_nvme_reset_destroy_qpair, 2438 NULL, 2439 bdev_nvme_reset_destroy_qpair_done); 2440 } 2441 2442 static void 2443 bdev_nvme_reconnect_ctrlr_now(void *ctx) 2444 { 2445 struct nvme_ctrlr *nvme_ctrlr = ctx; 2446 2447 assert(nvme_ctrlr->resetting == true); 2448 assert(nvme_ctrlr->thread == spdk_get_thread()); 2449 2450 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2451 2452 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 2453 2454 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2455 } 2456 2457 static void 2458 _bdev_nvme_reset_ctrlr(void *ctx) 2459 { 2460 struct nvme_ctrlr *nvme_ctrlr = ctx; 2461 2462 assert(nvme_ctrlr->resetting == true); 2463 assert(nvme_ctrlr->thread == spdk_get_thread()); 2464 2465 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2466 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs); 2467 } else { 2468 bdev_nvme_reset_destroy_qpairs(nvme_ctrlr); 2469 } 2470 } 2471 2472 static int 2473 bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2474 { 2475 spdk_msg_fn msg_fn; 2476 2477 pthread_mutex_lock(&nvme_ctrlr->mutex); 2478 if (nvme_ctrlr->destruct) { 2479 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2480 return -ENXIO; 2481 } 2482 2483 if (nvme_ctrlr->resetting) { 2484 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2485 SPDK_NOTICELOG("Unable to perform reset, already in progress.\n"); 2486 return -EBUSY; 2487 } 2488 2489 if (nvme_ctrlr->disabled) { 2490 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2491 SPDK_NOTICELOG("Unable to perform reset. Controller is disabled.\n"); 2492 return -EALREADY; 2493 } 2494 2495 nvme_ctrlr->resetting = true; 2496 nvme_ctrlr->dont_retry = true; 2497 2498 if (nvme_ctrlr->reconnect_is_delayed) { 2499 SPDK_DEBUGLOG(bdev_nvme, "Reconnect is already scheduled.\n"); 2500 msg_fn = bdev_nvme_reconnect_ctrlr_now; 2501 nvme_ctrlr->reconnect_is_delayed = false; 2502 } else { 2503 msg_fn = _bdev_nvme_reset_ctrlr; 2504 assert(nvme_ctrlr->reset_start_tsc == 0); 2505 } 2506 2507 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2508 2509 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2510 2511 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 2512 return 0; 2513 } 2514 2515 static int 2516 bdev_nvme_enable_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2517 { 2518 pthread_mutex_lock(&nvme_ctrlr->mutex); 2519 if (nvme_ctrlr->destruct) { 2520 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2521 return -ENXIO; 2522 } 2523 2524 if (nvme_ctrlr->resetting) { 2525 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2526 return -EBUSY; 2527 } 2528 2529 if (!nvme_ctrlr->disabled) { 2530 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2531 return -EALREADY; 2532 } 2533 2534 nvme_ctrlr->disabled = false; 2535 nvme_ctrlr->resetting = true; 2536 2537 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2538 2539 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2540 2541 spdk_thread_send_msg(nvme_ctrlr->thread, bdev_nvme_reconnect_ctrlr_now, nvme_ctrlr); 2542 return 0; 2543 } 2544 2545 static void 2546 _bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2547 { 2548 bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn; 2549 void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg; 2550 enum bdev_nvme_op_after_reset op_after_disable; 2551 2552 assert(nvme_ctrlr->thread == spdk_get_thread()); 2553 2554 nvme_ctrlr->ctrlr_op_cb_fn = NULL; 2555 nvme_ctrlr->ctrlr_op_cb_arg = NULL; 2556 2557 pthread_mutex_lock(&nvme_ctrlr->mutex); 2558 2559 nvme_ctrlr->resetting = false; 2560 nvme_ctrlr->dont_retry = false; 2561 2562 op_after_disable = bdev_nvme_check_op_after_reset(nvme_ctrlr, true); 2563 2564 nvme_ctrlr->disabled = true; 2565 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 2566 2567 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2568 2569 if (ctrlr_op_cb_fn) { 2570 ctrlr_op_cb_fn(ctrlr_op_cb_arg, 0); 2571 } 2572 2573 switch (op_after_disable) { 2574 case OP_COMPLETE_PENDING_DESTRUCT: 2575 nvme_ctrlr_unregister(nvme_ctrlr); 2576 break; 2577 default: 2578 break; 2579 } 2580 2581 } 2582 2583 static void 2584 bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr) 2585 { 2586 /* Make sure we clear any pending resets before returning. */ 2587 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2588 bdev_nvme_complete_pending_resets, 2589 NULL, 2590 _bdev_nvme_disable_ctrlr_complete); 2591 } 2592 2593 static void 2594 bdev_nvme_disable_destroy_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2595 { 2596 assert(status == 0); 2597 2598 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2599 bdev_nvme_disable_ctrlr_complete(nvme_ctrlr); 2600 } else { 2601 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_ctrlr_complete); 2602 } 2603 } 2604 2605 static void 2606 bdev_nvme_disable_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr) 2607 { 2608 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2609 bdev_nvme_reset_destroy_qpair, 2610 NULL, 2611 bdev_nvme_disable_destroy_qpairs_done); 2612 } 2613 2614 static void 2615 _bdev_nvme_cancel_reconnect_and_disable_ctrlr(void *ctx) 2616 { 2617 struct nvme_ctrlr *nvme_ctrlr = ctx; 2618 2619 assert(nvme_ctrlr->resetting == true); 2620 assert(nvme_ctrlr->thread == spdk_get_thread()); 2621 2622 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2623 2624 bdev_nvme_disable_ctrlr_complete(nvme_ctrlr); 2625 } 2626 2627 static void 2628 _bdev_nvme_disconnect_and_disable_ctrlr(void *ctx) 2629 { 2630 struct nvme_ctrlr *nvme_ctrlr = ctx; 2631 2632 assert(nvme_ctrlr->resetting == true); 2633 assert(nvme_ctrlr->thread == spdk_get_thread()); 2634 2635 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2636 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_destroy_qpairs); 2637 } else { 2638 bdev_nvme_disable_destroy_qpairs(nvme_ctrlr); 2639 } 2640 } 2641 2642 static int 2643 bdev_nvme_disable_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2644 { 2645 spdk_msg_fn msg_fn; 2646 2647 pthread_mutex_lock(&nvme_ctrlr->mutex); 2648 if (nvme_ctrlr->destruct) { 2649 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2650 return -ENXIO; 2651 } 2652 2653 if (nvme_ctrlr->resetting) { 2654 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2655 return -EBUSY; 2656 } 2657 2658 if (nvme_ctrlr->disabled) { 2659 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2660 return -EALREADY; 2661 } 2662 2663 nvme_ctrlr->resetting = true; 2664 nvme_ctrlr->dont_retry = true; 2665 2666 if (nvme_ctrlr->reconnect_is_delayed) { 2667 msg_fn = _bdev_nvme_cancel_reconnect_and_disable_ctrlr; 2668 nvme_ctrlr->reconnect_is_delayed = false; 2669 } else { 2670 msg_fn = _bdev_nvme_disconnect_and_disable_ctrlr; 2671 } 2672 2673 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2674 2675 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2676 2677 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 2678 return 0; 2679 } 2680 2681 static int 2682 nvme_ctrlr_op(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op, 2683 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2684 { 2685 int rc; 2686 2687 switch (op) { 2688 case NVME_CTRLR_OP_RESET: 2689 rc = bdev_nvme_reset_ctrlr(nvme_ctrlr); 2690 break; 2691 case NVME_CTRLR_OP_ENABLE: 2692 rc = bdev_nvme_enable_ctrlr(nvme_ctrlr); 2693 break; 2694 case NVME_CTRLR_OP_DISABLE: 2695 rc = bdev_nvme_disable_ctrlr(nvme_ctrlr); 2696 break; 2697 default: 2698 rc = -EINVAL; 2699 break; 2700 } 2701 2702 if (rc == 0) { 2703 assert(nvme_ctrlr->ctrlr_op_cb_fn == NULL); 2704 assert(nvme_ctrlr->ctrlr_op_cb_arg == NULL); 2705 nvme_ctrlr->ctrlr_op_cb_fn = cb_fn; 2706 nvme_ctrlr->ctrlr_op_cb_arg = cb_arg; 2707 } 2708 return rc; 2709 } 2710 2711 struct nvme_ctrlr_op_rpc_ctx { 2712 struct nvme_ctrlr *nvme_ctrlr; 2713 struct spdk_thread *orig_thread; 2714 enum nvme_ctrlr_op op; 2715 int rc; 2716 bdev_nvme_ctrlr_op_cb cb_fn; 2717 void *cb_arg; 2718 }; 2719 2720 static void 2721 _nvme_ctrlr_op_rpc_complete(void *_ctx) 2722 { 2723 struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx; 2724 2725 assert(ctx != NULL); 2726 assert(ctx->cb_fn != NULL); 2727 2728 ctx->cb_fn(ctx->cb_arg, ctx->rc); 2729 2730 free(ctx); 2731 } 2732 2733 static void 2734 nvme_ctrlr_op_rpc_complete(void *cb_arg, int rc) 2735 { 2736 struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg; 2737 2738 ctx->rc = rc; 2739 2740 spdk_thread_send_msg(ctx->orig_thread, _nvme_ctrlr_op_rpc_complete, ctx); 2741 } 2742 2743 void 2744 nvme_ctrlr_op_rpc(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op, 2745 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2746 { 2747 struct nvme_ctrlr_op_rpc_ctx *ctx; 2748 int rc; 2749 2750 assert(cb_fn != NULL); 2751 2752 ctx = calloc(1, sizeof(*ctx)); 2753 if (ctx == NULL) { 2754 SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n"); 2755 cb_fn(cb_arg, -ENOMEM); 2756 return; 2757 } 2758 2759 ctx->orig_thread = spdk_get_thread(); 2760 ctx->cb_fn = cb_fn; 2761 ctx->cb_arg = cb_arg; 2762 2763 rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_ctrlr_op_rpc_complete, ctx); 2764 if (rc == 0) { 2765 return; 2766 } else if (rc == -EALREADY) { 2767 rc = 0; 2768 } 2769 2770 nvme_ctrlr_op_rpc_complete(ctx, rc); 2771 } 2772 2773 static void nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc); 2774 2775 static void 2776 _nvme_bdev_ctrlr_op_rpc_continue(void *_ctx) 2777 { 2778 struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx; 2779 struct nvme_ctrlr *prev_nvme_ctrlr, *next_nvme_ctrlr; 2780 int rc; 2781 2782 prev_nvme_ctrlr = ctx->nvme_ctrlr; 2783 ctx->nvme_ctrlr = NULL; 2784 2785 if (ctx->rc != 0) { 2786 goto complete; 2787 } 2788 2789 next_nvme_ctrlr = TAILQ_NEXT(prev_nvme_ctrlr, tailq); 2790 if (next_nvme_ctrlr == NULL) { 2791 goto complete; 2792 } 2793 2794 rc = nvme_ctrlr_op(next_nvme_ctrlr, ctx->op, nvme_bdev_ctrlr_op_rpc_continue, ctx); 2795 if (rc == 0) { 2796 ctx->nvme_ctrlr = next_nvme_ctrlr; 2797 return; 2798 } else if (rc == -EALREADY) { 2799 ctx->nvme_ctrlr = next_nvme_ctrlr; 2800 rc = 0; 2801 } 2802 2803 ctx->rc = rc; 2804 2805 complete: 2806 ctx->cb_fn(ctx->cb_arg, ctx->rc); 2807 free(ctx); 2808 } 2809 2810 static void 2811 nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc) 2812 { 2813 struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg; 2814 2815 ctx->rc = rc; 2816 2817 spdk_thread_send_msg(ctx->orig_thread, _nvme_bdev_ctrlr_op_rpc_continue, ctx); 2818 } 2819 2820 void 2821 nvme_bdev_ctrlr_op_rpc(struct nvme_bdev_ctrlr *nbdev_ctrlr, enum nvme_ctrlr_op op, 2822 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2823 { 2824 struct nvme_ctrlr_op_rpc_ctx *ctx; 2825 struct nvme_ctrlr *nvme_ctrlr; 2826 int rc; 2827 2828 assert(cb_fn != NULL); 2829 2830 ctx = calloc(1, sizeof(*ctx)); 2831 if (ctx == NULL) { 2832 SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n"); 2833 cb_fn(cb_arg, -ENOMEM); 2834 return; 2835 } 2836 2837 ctx->orig_thread = spdk_get_thread(); 2838 ctx->op = op; 2839 ctx->cb_fn = cb_fn; 2840 ctx->cb_arg = cb_arg; 2841 2842 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 2843 assert(nvme_ctrlr != NULL); 2844 2845 rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_bdev_ctrlr_op_rpc_continue, ctx); 2846 if (rc == 0) { 2847 ctx->nvme_ctrlr = nvme_ctrlr; 2848 return; 2849 } else if (rc == -EALREADY) { 2850 ctx->nvme_ctrlr = nvme_ctrlr; 2851 rc = 0; 2852 } 2853 2854 nvme_bdev_ctrlr_op_rpc_continue(ctx, rc); 2855 } 2856 2857 static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio); 2858 2859 static void 2860 bdev_nvme_unfreeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status) 2861 { 2862 struct nvme_bdev_io *bio = ctx; 2863 enum spdk_bdev_io_status io_status; 2864 2865 if (bio->cpl.cdw0 == 0) { 2866 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 2867 } else { 2868 io_status = SPDK_BDEV_IO_STATUS_FAILED; 2869 } 2870 2871 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL); 2872 } 2873 2874 static void 2875 bdev_nvme_unfreeze_bdev_channel(struct nvme_bdev_channel_iter *i, 2876 struct nvme_bdev *nbdev, 2877 struct nvme_bdev_channel *nbdev_ch, void *ctx) 2878 { 2879 bdev_nvme_abort_retry_ios(nbdev_ch); 2880 nbdev_ch->resetting = false; 2881 2882 nvme_bdev_for_each_channel_continue(i, 0); 2883 } 2884 2885 static void 2886 bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio) 2887 { 2888 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2889 struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt; 2890 2891 /* Abort all queued I/Os for retry. */ 2892 nvme_bdev_for_each_channel(nbdev, 2893 bdev_nvme_unfreeze_bdev_channel, 2894 bio, 2895 bdev_nvme_unfreeze_bdev_channel_done); 2896 } 2897 2898 static void 2899 _bdev_nvme_reset_io_continue(void *ctx) 2900 { 2901 struct nvme_bdev_io *bio = ctx; 2902 struct nvme_io_path *prev_io_path, *next_io_path; 2903 int rc; 2904 2905 prev_io_path = bio->io_path; 2906 bio->io_path = NULL; 2907 2908 if (bio->cpl.cdw0 != 0) { 2909 goto complete; 2910 } 2911 2912 next_io_path = STAILQ_NEXT(prev_io_path, stailq); 2913 if (next_io_path == NULL) { 2914 goto complete; 2915 } 2916 2917 rc = _bdev_nvme_reset_io(next_io_path, bio); 2918 if (rc == 0) { 2919 return; 2920 } 2921 2922 bio->cpl.cdw0 = 1; 2923 2924 complete: 2925 bdev_nvme_reset_io_complete(bio); 2926 } 2927 2928 static void 2929 bdev_nvme_reset_io_continue(void *cb_arg, int rc) 2930 { 2931 struct nvme_bdev_io *bio = cb_arg; 2932 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2933 2934 bio->cpl.cdw0 = (rc == 0) ? 0 : 1; 2935 2936 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), _bdev_nvme_reset_io_continue, bio); 2937 } 2938 2939 static int 2940 _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio) 2941 { 2942 struct nvme_ctrlr_channel *ctrlr_ch; 2943 int rc; 2944 2945 rc = nvme_ctrlr_op(io_path->qpair->ctrlr, NVME_CTRLR_OP_RESET, 2946 bdev_nvme_reset_io_continue, bio); 2947 if (rc != 0 && rc != -EBUSY) { 2948 return rc; 2949 } 2950 2951 assert(bio->io_path == NULL); 2952 bio->io_path = io_path; 2953 2954 if (rc == -EBUSY) { 2955 ctrlr_ch = io_path->qpair->ctrlr_ch; 2956 assert(ctrlr_ch != NULL); 2957 /* 2958 * Reset call is queued only if it is from the app framework. This is on purpose so that 2959 * we don't interfere with the app framework reset strategy. i.e. we are deferring to the 2960 * upper level. If they are in the middle of a reset, we won't try to schedule another one. 2961 */ 2962 TAILQ_INSERT_TAIL(&ctrlr_ch->pending_resets, bio, retry_link); 2963 } 2964 2965 return 0; 2966 } 2967 2968 static void 2969 bdev_nvme_freeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status) 2970 { 2971 struct nvme_bdev_io *bio = ctx; 2972 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2973 struct nvme_bdev_channel *nbdev_ch; 2974 struct nvme_io_path *io_path; 2975 int rc; 2976 2977 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 2978 2979 bio->cpl.cdw0 = 0; 2980 2981 /* Reset all nvme_ctrlrs of a bdev controller sequentially. */ 2982 io_path = STAILQ_FIRST(&nbdev_ch->io_path_list); 2983 assert(io_path != NULL); 2984 2985 rc = _bdev_nvme_reset_io(io_path, bio); 2986 if (rc != 0) { 2987 /* If the current nvme_ctrlr is disabled, skip it and move to the next nvme_ctrlr. */ 2988 rc = (rc == -EALREADY) ? 0 : rc; 2989 2990 bdev_nvme_reset_io_continue(bio, rc); 2991 } 2992 } 2993 2994 static void 2995 bdev_nvme_freeze_bdev_channel(struct nvme_bdev_channel_iter *i, 2996 struct nvme_bdev *nbdev, 2997 struct nvme_bdev_channel *nbdev_ch, void *ctx) 2998 { 2999 nbdev_ch->resetting = true; 3000 3001 nvme_bdev_for_each_channel_continue(i, 0); 3002 } 3003 3004 static void 3005 bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio) 3006 { 3007 nvme_bdev_for_each_channel(nbdev, 3008 bdev_nvme_freeze_bdev_channel, 3009 bio, 3010 bdev_nvme_freeze_bdev_channel_done); 3011 } 3012 3013 static int 3014 bdev_nvme_failover_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool remove) 3015 { 3016 if (nvme_ctrlr->destruct) { 3017 /* Don't bother resetting if the controller is in the process of being destructed. */ 3018 return -ENXIO; 3019 } 3020 3021 if (nvme_ctrlr->resetting) { 3022 if (!nvme_ctrlr->in_failover) { 3023 SPDK_NOTICELOG("Reset is already in progress. Defer failover until reset completes.\n"); 3024 3025 /* Defer failover until reset completes. */ 3026 nvme_ctrlr->pending_failover = true; 3027 return -EINPROGRESS; 3028 } else { 3029 SPDK_NOTICELOG("Unable to perform failover, already in progress.\n"); 3030 return -EBUSY; 3031 } 3032 } 3033 3034 bdev_nvme_failover_trid(nvme_ctrlr, remove, true); 3035 3036 if (nvme_ctrlr->reconnect_is_delayed) { 3037 SPDK_NOTICELOG("Reconnect is already scheduled.\n"); 3038 3039 /* We rely on the next reconnect for the failover. */ 3040 return -EALREADY; 3041 } 3042 3043 if (nvme_ctrlr->disabled) { 3044 SPDK_NOTICELOG("Controller is disabled.\n"); 3045 3046 /* We rely on the enablement for the failover. */ 3047 return -EALREADY; 3048 } 3049 3050 nvme_ctrlr->resetting = true; 3051 nvme_ctrlr->in_failover = true; 3052 3053 assert(nvme_ctrlr->reset_start_tsc == 0); 3054 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 3055 3056 return 0; 3057 } 3058 3059 static int 3060 bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 3061 { 3062 int rc; 3063 3064 pthread_mutex_lock(&nvme_ctrlr->mutex); 3065 rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, false); 3066 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3067 3068 if (rc == 0) { 3069 spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset_ctrlr, nvme_ctrlr); 3070 } else if (rc == -EALREADY) { 3071 rc = 0; 3072 } 3073 3074 return rc; 3075 } 3076 3077 static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, 3078 uint64_t num_blocks); 3079 3080 static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, 3081 uint64_t num_blocks); 3082 3083 static int bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, 3084 uint64_t src_offset_blocks, 3085 uint64_t num_blocks); 3086 3087 static void 3088 bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 3089 bool success) 3090 { 3091 struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 3092 int ret; 3093 3094 if (!success) { 3095 ret = -EINVAL; 3096 goto exit; 3097 } 3098 3099 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 3100 ret = -ENXIO; 3101 goto exit; 3102 } 3103 3104 ret = bdev_nvme_readv(bio, 3105 bdev_io->u.bdev.iovs, 3106 bdev_io->u.bdev.iovcnt, 3107 bdev_io->u.bdev.md_buf, 3108 bdev_io->u.bdev.num_blocks, 3109 bdev_io->u.bdev.offset_blocks, 3110 bdev_io->u.bdev.dif_check_flags, 3111 bdev_io->u.bdev.memory_domain, 3112 bdev_io->u.bdev.memory_domain_ctx, 3113 bdev_io->u.bdev.accel_sequence); 3114 3115 exit: 3116 if (spdk_unlikely(ret != 0)) { 3117 bdev_nvme_io_complete(bio, ret); 3118 } 3119 } 3120 3121 static inline void 3122 _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 3123 { 3124 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 3125 struct spdk_bdev *bdev = bdev_io->bdev; 3126 struct nvme_bdev_io *nbdev_io_to_abort; 3127 int rc = 0; 3128 3129 switch (bdev_io->type) { 3130 case SPDK_BDEV_IO_TYPE_READ: 3131 if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) { 3132 3133 rc = bdev_nvme_readv(nbdev_io, 3134 bdev_io->u.bdev.iovs, 3135 bdev_io->u.bdev.iovcnt, 3136 bdev_io->u.bdev.md_buf, 3137 bdev_io->u.bdev.num_blocks, 3138 bdev_io->u.bdev.offset_blocks, 3139 bdev_io->u.bdev.dif_check_flags, 3140 bdev_io->u.bdev.memory_domain, 3141 bdev_io->u.bdev.memory_domain_ctx, 3142 bdev_io->u.bdev.accel_sequence); 3143 } else { 3144 spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb, 3145 bdev_io->u.bdev.num_blocks * bdev->blocklen); 3146 rc = 0; 3147 } 3148 break; 3149 case SPDK_BDEV_IO_TYPE_WRITE: 3150 rc = bdev_nvme_writev(nbdev_io, 3151 bdev_io->u.bdev.iovs, 3152 bdev_io->u.bdev.iovcnt, 3153 bdev_io->u.bdev.md_buf, 3154 bdev_io->u.bdev.num_blocks, 3155 bdev_io->u.bdev.offset_blocks, 3156 bdev_io->u.bdev.dif_check_flags, 3157 bdev_io->u.bdev.memory_domain, 3158 bdev_io->u.bdev.memory_domain_ctx, 3159 bdev_io->u.bdev.accel_sequence, 3160 bdev_io->u.bdev.nvme_cdw12, 3161 bdev_io->u.bdev.nvme_cdw13); 3162 break; 3163 case SPDK_BDEV_IO_TYPE_COMPARE: 3164 rc = bdev_nvme_comparev(nbdev_io, 3165 bdev_io->u.bdev.iovs, 3166 bdev_io->u.bdev.iovcnt, 3167 bdev_io->u.bdev.md_buf, 3168 bdev_io->u.bdev.num_blocks, 3169 bdev_io->u.bdev.offset_blocks, 3170 bdev_io->u.bdev.dif_check_flags); 3171 break; 3172 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 3173 rc = bdev_nvme_comparev_and_writev(nbdev_io, 3174 bdev_io->u.bdev.iovs, 3175 bdev_io->u.bdev.iovcnt, 3176 bdev_io->u.bdev.fused_iovs, 3177 bdev_io->u.bdev.fused_iovcnt, 3178 bdev_io->u.bdev.md_buf, 3179 bdev_io->u.bdev.num_blocks, 3180 bdev_io->u.bdev.offset_blocks, 3181 bdev_io->u.bdev.dif_check_flags); 3182 break; 3183 case SPDK_BDEV_IO_TYPE_UNMAP: 3184 rc = bdev_nvme_unmap(nbdev_io, 3185 bdev_io->u.bdev.offset_blocks, 3186 bdev_io->u.bdev.num_blocks); 3187 break; 3188 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 3189 rc = bdev_nvme_write_zeroes(nbdev_io, 3190 bdev_io->u.bdev.offset_blocks, 3191 bdev_io->u.bdev.num_blocks); 3192 break; 3193 case SPDK_BDEV_IO_TYPE_RESET: 3194 nbdev_io->io_path = NULL; 3195 bdev_nvme_reset_io(bdev->ctxt, nbdev_io); 3196 return; 3197 3198 case SPDK_BDEV_IO_TYPE_FLUSH: 3199 bdev_nvme_io_complete(nbdev_io, 0); 3200 return; 3201 3202 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 3203 rc = bdev_nvme_zone_appendv(nbdev_io, 3204 bdev_io->u.bdev.iovs, 3205 bdev_io->u.bdev.iovcnt, 3206 bdev_io->u.bdev.md_buf, 3207 bdev_io->u.bdev.num_blocks, 3208 bdev_io->u.bdev.offset_blocks, 3209 bdev_io->u.bdev.dif_check_flags); 3210 break; 3211 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 3212 rc = bdev_nvme_get_zone_info(nbdev_io, 3213 bdev_io->u.zone_mgmt.zone_id, 3214 bdev_io->u.zone_mgmt.num_zones, 3215 bdev_io->u.zone_mgmt.buf); 3216 break; 3217 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 3218 rc = bdev_nvme_zone_management(nbdev_io, 3219 bdev_io->u.zone_mgmt.zone_id, 3220 bdev_io->u.zone_mgmt.zone_action); 3221 break; 3222 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3223 nbdev_io->io_path = NULL; 3224 bdev_nvme_admin_passthru(nbdev_ch, 3225 nbdev_io, 3226 &bdev_io->u.nvme_passthru.cmd, 3227 bdev_io->u.nvme_passthru.buf, 3228 bdev_io->u.nvme_passthru.nbytes); 3229 return; 3230 3231 case SPDK_BDEV_IO_TYPE_NVME_IO: 3232 rc = bdev_nvme_io_passthru(nbdev_io, 3233 &bdev_io->u.nvme_passthru.cmd, 3234 bdev_io->u.nvme_passthru.buf, 3235 bdev_io->u.nvme_passthru.nbytes); 3236 break; 3237 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3238 rc = bdev_nvme_io_passthru_md(nbdev_io, 3239 &bdev_io->u.nvme_passthru.cmd, 3240 bdev_io->u.nvme_passthru.buf, 3241 bdev_io->u.nvme_passthru.nbytes, 3242 bdev_io->u.nvme_passthru.md_buf, 3243 bdev_io->u.nvme_passthru.md_len); 3244 break; 3245 case SPDK_BDEV_IO_TYPE_NVME_IOV_MD: 3246 rc = bdev_nvme_iov_passthru_md(nbdev_io, 3247 &bdev_io->u.nvme_passthru.cmd, 3248 bdev_io->u.nvme_passthru.iovs, 3249 bdev_io->u.nvme_passthru.iovcnt, 3250 bdev_io->u.nvme_passthru.nbytes, 3251 bdev_io->u.nvme_passthru.md_buf, 3252 bdev_io->u.nvme_passthru.md_len); 3253 break; 3254 case SPDK_BDEV_IO_TYPE_ABORT: 3255 nbdev_io->io_path = NULL; 3256 nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx; 3257 bdev_nvme_abort(nbdev_ch, 3258 nbdev_io, 3259 nbdev_io_to_abort); 3260 return; 3261 3262 case SPDK_BDEV_IO_TYPE_COPY: 3263 rc = bdev_nvme_copy(nbdev_io, 3264 bdev_io->u.bdev.offset_blocks, 3265 bdev_io->u.bdev.copy.src_offset_blocks, 3266 bdev_io->u.bdev.num_blocks); 3267 break; 3268 default: 3269 rc = -EINVAL; 3270 break; 3271 } 3272 3273 if (spdk_unlikely(rc != 0)) { 3274 bdev_nvme_io_complete(nbdev_io, rc); 3275 } 3276 } 3277 3278 static void 3279 bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 3280 { 3281 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 3282 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 3283 3284 if (spdk_likely(nbdev_io->submit_tsc == 0)) { 3285 nbdev_io->submit_tsc = spdk_bdev_io_get_submit_tsc(bdev_io); 3286 } else { 3287 /* There are cases where submit_tsc != 0, i.e. retry I/O. 3288 * We need to update submit_tsc here. 3289 */ 3290 nbdev_io->submit_tsc = spdk_get_ticks(); 3291 } 3292 3293 spdk_trace_record(TRACE_BDEV_NVME_IO_START, 0, 0, (uintptr_t)nbdev_io, (uintptr_t)bdev_io); 3294 nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch); 3295 if (spdk_unlikely(!nbdev_io->io_path)) { 3296 if (!bdev_nvme_io_type_is_admin(bdev_io->type)) { 3297 bdev_nvme_io_complete(nbdev_io, -ENXIO); 3298 return; 3299 } 3300 3301 /* Admin commands do not use the optimal I/O path. 3302 * Simply fall through even if it is not found. 3303 */ 3304 } 3305 3306 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 3307 } 3308 3309 static bool 3310 bdev_nvme_is_supported_csi(enum spdk_nvme_csi csi) 3311 { 3312 switch (csi) { 3313 case SPDK_NVME_CSI_NVM: 3314 return true; 3315 case SPDK_NVME_CSI_ZNS: 3316 return true; 3317 default: 3318 return false; 3319 } 3320 } 3321 3322 static bool 3323 bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 3324 { 3325 struct nvme_bdev *nbdev = ctx; 3326 struct nvme_ns *nvme_ns; 3327 struct spdk_nvme_ns *ns; 3328 struct spdk_nvme_ctrlr *ctrlr; 3329 const struct spdk_nvme_ctrlr_data *cdata; 3330 3331 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 3332 assert(nvme_ns != NULL); 3333 ns = nvme_ns->ns; 3334 if (ns == NULL) { 3335 return false; 3336 } 3337 3338 if (!bdev_nvme_is_supported_csi(spdk_nvme_ns_get_csi(ns))) { 3339 switch (io_type) { 3340 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3341 case SPDK_BDEV_IO_TYPE_NVME_IO: 3342 return true; 3343 3344 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3345 return spdk_nvme_ns_get_md_size(ns) ? true : false; 3346 3347 default: 3348 return false; 3349 } 3350 } 3351 3352 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 3353 3354 switch (io_type) { 3355 case SPDK_BDEV_IO_TYPE_READ: 3356 case SPDK_BDEV_IO_TYPE_WRITE: 3357 case SPDK_BDEV_IO_TYPE_RESET: 3358 case SPDK_BDEV_IO_TYPE_FLUSH: 3359 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3360 case SPDK_BDEV_IO_TYPE_NVME_IO: 3361 case SPDK_BDEV_IO_TYPE_ABORT: 3362 return true; 3363 3364 case SPDK_BDEV_IO_TYPE_COMPARE: 3365 return spdk_nvme_ns_supports_compare(ns); 3366 3367 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3368 return spdk_nvme_ns_get_md_size(ns) ? true : false; 3369 3370 case SPDK_BDEV_IO_TYPE_UNMAP: 3371 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3372 return cdata->oncs.dsm; 3373 3374 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 3375 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3376 return cdata->oncs.write_zeroes; 3377 3378 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 3379 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 3380 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) { 3381 return true; 3382 } 3383 return false; 3384 3385 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 3386 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 3387 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS; 3388 3389 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 3390 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS && 3391 spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED; 3392 3393 case SPDK_BDEV_IO_TYPE_COPY: 3394 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3395 return cdata->oncs.copy; 3396 3397 default: 3398 return false; 3399 } 3400 } 3401 3402 static int 3403 nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch) 3404 { 3405 struct nvme_qpair *nvme_qpair; 3406 struct spdk_io_channel *pg_ch; 3407 int rc; 3408 3409 nvme_qpair = calloc(1, sizeof(*nvme_qpair)); 3410 if (!nvme_qpair) { 3411 SPDK_ERRLOG("Failed to alloc nvme_qpair.\n"); 3412 return -1; 3413 } 3414 3415 TAILQ_INIT(&nvme_qpair->io_path_list); 3416 3417 nvme_qpair->ctrlr = nvme_ctrlr; 3418 nvme_qpair->ctrlr_ch = ctrlr_ch; 3419 3420 pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs); 3421 if (!pg_ch) { 3422 free(nvme_qpair); 3423 return -1; 3424 } 3425 3426 nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch); 3427 3428 #ifdef SPDK_CONFIG_VTUNE 3429 nvme_qpair->group->collect_spin_stat = true; 3430 #else 3431 nvme_qpair->group->collect_spin_stat = false; 3432 #endif 3433 3434 if (!nvme_ctrlr->disabled) { 3435 /* If a nvme_ctrlr is disabled, don't try to create qpair for it. Qpair will 3436 * be created when it's enabled. 3437 */ 3438 rc = bdev_nvme_create_qpair(nvme_qpair); 3439 if (rc != 0) { 3440 /* nvme_ctrlr can't create IO qpair if connection is down. 3441 * If reconnect_delay_sec is non-zero, creating IO qpair is retried 3442 * after reconnect_delay_sec seconds. If bdev_retry_count is non-zero, 3443 * submitted IO will be queued until IO qpair is successfully created. 3444 * 3445 * Hence, if both are satisfied, ignore the failure. 3446 */ 3447 if (nvme_ctrlr->opts.reconnect_delay_sec == 0 || g_opts.bdev_retry_count == 0) { 3448 spdk_put_io_channel(pg_ch); 3449 free(nvme_qpair); 3450 return rc; 3451 } 3452 } 3453 } 3454 3455 TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 3456 3457 ctrlr_ch->qpair = nvme_qpair; 3458 3459 pthread_mutex_lock(&nvme_qpair->ctrlr->mutex); 3460 nvme_qpair->ctrlr->ref++; 3461 pthread_mutex_unlock(&nvme_qpair->ctrlr->mutex); 3462 3463 return 0; 3464 } 3465 3466 static int 3467 bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) 3468 { 3469 struct nvme_ctrlr *nvme_ctrlr = io_device; 3470 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 3471 3472 TAILQ_INIT(&ctrlr_ch->pending_resets); 3473 3474 return nvme_qpair_create(nvme_ctrlr, ctrlr_ch); 3475 } 3476 3477 static void 3478 nvme_qpair_delete(struct nvme_qpair *nvme_qpair) 3479 { 3480 struct nvme_io_path *io_path, *next; 3481 3482 assert(nvme_qpair->group != NULL); 3483 3484 TAILQ_FOREACH_SAFE(io_path, &nvme_qpair->io_path_list, tailq, next) { 3485 TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq); 3486 nvme_io_path_free(io_path); 3487 } 3488 3489 TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 3490 3491 spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group)); 3492 3493 nvme_ctrlr_release(nvme_qpair->ctrlr); 3494 3495 free(nvme_qpair); 3496 } 3497 3498 static void 3499 bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf) 3500 { 3501 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 3502 struct nvme_qpair *nvme_qpair; 3503 3504 nvme_qpair = ctrlr_ch->qpair; 3505 assert(nvme_qpair != NULL); 3506 3507 _bdev_nvme_clear_io_path_cache(nvme_qpair); 3508 3509 if (nvme_qpair->qpair != NULL) { 3510 if (ctrlr_ch->reset_iter == NULL) { 3511 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 3512 } else { 3513 /* Skip current ctrlr_channel in a full reset sequence because 3514 * it is being deleted now. The qpair is already being disconnected. 3515 * We do not have to restart disconnecting it. 3516 */ 3517 nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 3518 } 3519 3520 /* We cannot release a reference to the poll group now. 3521 * The qpair may be disconnected asynchronously later. 3522 * We need to poll it until it is actually disconnected. 3523 * Just detach the qpair from the deleting ctrlr_channel. 3524 */ 3525 nvme_qpair->ctrlr_ch = NULL; 3526 } else { 3527 assert(ctrlr_ch->reset_iter == NULL); 3528 3529 nvme_qpair_delete(nvme_qpair); 3530 } 3531 } 3532 3533 static inline struct spdk_io_channel * 3534 bdev_nvme_get_accel_channel(struct nvme_poll_group *group) 3535 { 3536 if (spdk_unlikely(!group->accel_channel)) { 3537 group->accel_channel = spdk_accel_get_io_channel(); 3538 if (!group->accel_channel) { 3539 SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n", 3540 group); 3541 return NULL; 3542 } 3543 } 3544 3545 return group->accel_channel; 3546 } 3547 3548 static void 3549 bdev_nvme_submit_accel_crc32c(void *ctx, uint32_t *dst, struct iovec *iov, 3550 uint32_t iov_cnt, uint32_t seed, 3551 spdk_nvme_accel_completion_cb cb_fn, void *cb_arg) 3552 { 3553 struct spdk_io_channel *accel_ch; 3554 struct nvme_poll_group *group = ctx; 3555 int rc; 3556 3557 assert(cb_fn != NULL); 3558 3559 accel_ch = bdev_nvme_get_accel_channel(group); 3560 if (spdk_unlikely(accel_ch == NULL)) { 3561 cb_fn(cb_arg, -ENOMEM); 3562 return; 3563 } 3564 3565 rc = spdk_accel_submit_crc32cv(accel_ch, dst, iov, iov_cnt, seed, cb_fn, cb_arg); 3566 if (rc) { 3567 /* For the two cases, spdk_accel_submit_crc32cv does not call the user's cb_fn */ 3568 if (rc == -ENOMEM || rc == -EINVAL) { 3569 cb_fn(cb_arg, rc); 3570 } 3571 SPDK_ERRLOG("Cannot complete the accelerated crc32c operation with iov=%p\n", iov); 3572 } 3573 } 3574 3575 static void 3576 bdev_nvme_finish_sequence(void *seq, spdk_nvme_accel_completion_cb cb_fn, void *cb_arg) 3577 { 3578 spdk_accel_sequence_finish(seq, cb_fn, cb_arg); 3579 } 3580 3581 static void 3582 bdev_nvme_abort_sequence(void *seq) 3583 { 3584 spdk_accel_sequence_abort(seq); 3585 } 3586 3587 static void 3588 bdev_nvme_reverse_sequence(void *seq) 3589 { 3590 spdk_accel_sequence_reverse(seq); 3591 } 3592 3593 static int 3594 bdev_nvme_append_crc32c(void *ctx, void **seq, uint32_t *dst, struct iovec *iovs, uint32_t iovcnt, 3595 struct spdk_memory_domain *domain, void *domain_ctx, uint32_t seed, 3596 spdk_nvme_accel_step_cb cb_fn, void *cb_arg) 3597 { 3598 struct spdk_io_channel *ch; 3599 struct nvme_poll_group *group = ctx; 3600 3601 ch = bdev_nvme_get_accel_channel(group); 3602 if (spdk_unlikely(ch == NULL)) { 3603 return -ENOMEM; 3604 } 3605 3606 return spdk_accel_append_crc32c((struct spdk_accel_sequence **)seq, ch, dst, iovs, iovcnt, 3607 domain, domain_ctx, seed, cb_fn, cb_arg); 3608 } 3609 3610 static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = { 3611 .table_size = sizeof(struct spdk_nvme_accel_fn_table), 3612 .submit_accel_crc32c = bdev_nvme_submit_accel_crc32c, 3613 .append_crc32c = bdev_nvme_append_crc32c, 3614 .finish_sequence = bdev_nvme_finish_sequence, 3615 .reverse_sequence = bdev_nvme_reverse_sequence, 3616 .abort_sequence = bdev_nvme_abort_sequence, 3617 }; 3618 3619 static int 3620 bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf) 3621 { 3622 struct nvme_poll_group *group = ctx_buf; 3623 3624 TAILQ_INIT(&group->qpair_list); 3625 3626 group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table); 3627 if (group->group == NULL) { 3628 return -1; 3629 } 3630 3631 group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us); 3632 3633 if (group->poller == NULL) { 3634 spdk_nvme_poll_group_destroy(group->group); 3635 return -1; 3636 } 3637 3638 return 0; 3639 } 3640 3641 static void 3642 bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf) 3643 { 3644 struct nvme_poll_group *group = ctx_buf; 3645 3646 assert(TAILQ_EMPTY(&group->qpair_list)); 3647 3648 if (group->accel_channel) { 3649 spdk_put_io_channel(group->accel_channel); 3650 } 3651 3652 spdk_poller_unregister(&group->poller); 3653 if (spdk_nvme_poll_group_destroy(group->group)) { 3654 SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n"); 3655 assert(false); 3656 } 3657 } 3658 3659 static struct spdk_io_channel * 3660 bdev_nvme_get_io_channel(void *ctx) 3661 { 3662 struct nvme_bdev *nvme_bdev = ctx; 3663 3664 return spdk_get_io_channel(nvme_bdev); 3665 } 3666 3667 static void * 3668 bdev_nvme_get_module_ctx(void *ctx) 3669 { 3670 struct nvme_bdev *nvme_bdev = ctx; 3671 struct nvme_ns *nvme_ns; 3672 3673 if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) { 3674 return NULL; 3675 } 3676 3677 nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list); 3678 if (!nvme_ns) { 3679 return NULL; 3680 } 3681 3682 return nvme_ns->ns; 3683 } 3684 3685 static const char * 3686 _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state) 3687 { 3688 switch (ana_state) { 3689 case SPDK_NVME_ANA_OPTIMIZED_STATE: 3690 return "optimized"; 3691 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 3692 return "non_optimized"; 3693 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3694 return "inaccessible"; 3695 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3696 return "persistent_loss"; 3697 case SPDK_NVME_ANA_CHANGE_STATE: 3698 return "change"; 3699 default: 3700 return NULL; 3701 } 3702 } 3703 3704 static int 3705 bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size) 3706 { 3707 struct spdk_memory_domain **_domains = NULL; 3708 struct nvme_bdev *nbdev = ctx; 3709 struct nvme_ns *nvme_ns; 3710 int i = 0, _array_size = array_size; 3711 int rc = 0; 3712 3713 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 3714 if (domains && array_size >= i) { 3715 _domains = &domains[i]; 3716 } else { 3717 _domains = NULL; 3718 } 3719 rc = spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, _domains, _array_size); 3720 if (rc > 0) { 3721 i += rc; 3722 if (_array_size >= rc) { 3723 _array_size -= rc; 3724 } else { 3725 _array_size = 0; 3726 } 3727 } else if (rc < 0) { 3728 return rc; 3729 } 3730 } 3731 3732 return i; 3733 } 3734 3735 static const char * 3736 nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr) 3737 { 3738 if (nvme_ctrlr->destruct) { 3739 return "deleting"; 3740 } else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 3741 return "failed"; 3742 } else if (nvme_ctrlr->resetting) { 3743 return "resetting"; 3744 } else if (nvme_ctrlr->reconnect_is_delayed > 0) { 3745 return "reconnect_is_delayed"; 3746 } else if (nvme_ctrlr->disabled) { 3747 return "disabled"; 3748 } else { 3749 return "enabled"; 3750 } 3751 } 3752 3753 void 3754 nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr) 3755 { 3756 struct spdk_nvme_transport_id *trid; 3757 const struct spdk_nvme_ctrlr_opts *opts; 3758 const struct spdk_nvme_ctrlr_data *cdata; 3759 struct nvme_path_id *path_id; 3760 int32_t numa_id; 3761 3762 spdk_json_write_object_begin(w); 3763 3764 spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr)); 3765 3766 #ifdef SPDK_CONFIG_NVME_CUSE 3767 size_t cuse_name_size = 128; 3768 char cuse_name[cuse_name_size]; 3769 3770 int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size); 3771 if (rc == 0) { 3772 spdk_json_write_named_string(w, "cuse_device", cuse_name); 3773 } 3774 #endif 3775 trid = &nvme_ctrlr->active_path_id->trid; 3776 spdk_json_write_named_object_begin(w, "trid"); 3777 nvme_bdev_dump_trid_json(trid, w); 3778 spdk_json_write_object_end(w); 3779 3780 path_id = TAILQ_NEXT(nvme_ctrlr->active_path_id, link); 3781 if (path_id != NULL) { 3782 spdk_json_write_named_array_begin(w, "alternate_trids"); 3783 do { 3784 trid = &path_id->trid; 3785 spdk_json_write_object_begin(w); 3786 nvme_bdev_dump_trid_json(trid, w); 3787 spdk_json_write_object_end(w); 3788 3789 path_id = TAILQ_NEXT(path_id, link); 3790 } while (path_id != NULL); 3791 spdk_json_write_array_end(w); 3792 } 3793 3794 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 3795 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 3796 3797 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 3798 spdk_json_write_named_object_begin(w, "host"); 3799 spdk_json_write_named_string(w, "nqn", opts->hostnqn); 3800 spdk_json_write_named_string(w, "addr", opts->src_addr); 3801 spdk_json_write_named_string(w, "svcid", opts->src_svcid); 3802 spdk_json_write_object_end(w); 3803 3804 numa_id = spdk_nvme_ctrlr_get_numa_id(nvme_ctrlr->ctrlr); 3805 if (numa_id != SPDK_ENV_NUMA_ID_ANY) { 3806 spdk_json_write_named_uint32(w, "numa_id", numa_id); 3807 } 3808 spdk_json_write_object_end(w); 3809 } 3810 3811 static void 3812 nvme_namespace_info_json(struct spdk_json_write_ctx *w, 3813 struct nvme_ns *nvme_ns) 3814 { 3815 struct spdk_nvme_ns *ns; 3816 struct spdk_nvme_ctrlr *ctrlr; 3817 const struct spdk_nvme_ctrlr_data *cdata; 3818 const struct spdk_nvme_transport_id *trid; 3819 union spdk_nvme_vs_register vs; 3820 const struct spdk_nvme_ns_data *nsdata; 3821 char buf[128]; 3822 3823 ns = nvme_ns->ns; 3824 if (ns == NULL) { 3825 return; 3826 } 3827 3828 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 3829 3830 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3831 trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); 3832 vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr); 3833 3834 spdk_json_write_object_begin(w); 3835 3836 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 3837 spdk_json_write_named_string(w, "pci_address", trid->traddr); 3838 } 3839 3840 spdk_json_write_named_object_begin(w, "trid"); 3841 3842 nvme_bdev_dump_trid_json(trid, w); 3843 3844 spdk_json_write_object_end(w); 3845 3846 #ifdef SPDK_CONFIG_NVME_CUSE 3847 size_t cuse_name_size = 128; 3848 char cuse_name[cuse_name_size]; 3849 3850 int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns), 3851 cuse_name, &cuse_name_size); 3852 if (rc == 0) { 3853 spdk_json_write_named_string(w, "cuse_device", cuse_name); 3854 } 3855 #endif 3856 3857 spdk_json_write_named_object_begin(w, "ctrlr_data"); 3858 3859 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 3860 3861 spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid); 3862 3863 snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn); 3864 spdk_str_trim(buf); 3865 spdk_json_write_named_string(w, "model_number", buf); 3866 3867 snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn); 3868 spdk_str_trim(buf); 3869 spdk_json_write_named_string(w, "serial_number", buf); 3870 3871 snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr); 3872 spdk_str_trim(buf); 3873 spdk_json_write_named_string(w, "firmware_revision", buf); 3874 3875 if (cdata->subnqn[0] != '\0') { 3876 spdk_json_write_named_string(w, "subnqn", cdata->subnqn); 3877 } 3878 3879 spdk_json_write_named_object_begin(w, "oacs"); 3880 3881 spdk_json_write_named_uint32(w, "security", cdata->oacs.security); 3882 spdk_json_write_named_uint32(w, "format", cdata->oacs.format); 3883 spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware); 3884 spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage); 3885 3886 spdk_json_write_object_end(w); 3887 3888 spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr); 3889 spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting); 3890 3891 spdk_json_write_object_end(w); 3892 3893 spdk_json_write_named_object_begin(w, "vs"); 3894 3895 spdk_json_write_name(w, "nvme_version"); 3896 if (vs.bits.ter) { 3897 spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter); 3898 } else { 3899 spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr); 3900 } 3901 3902 spdk_json_write_object_end(w); 3903 3904 nsdata = spdk_nvme_ns_get_data(ns); 3905 3906 spdk_json_write_named_object_begin(w, "ns_data"); 3907 3908 spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns)); 3909 3910 if (cdata->cmic.ana_reporting) { 3911 spdk_json_write_named_string(w, "ana_state", 3912 _nvme_ana_state_str(nvme_ns->ana_state)); 3913 } 3914 3915 spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share); 3916 3917 spdk_json_write_object_end(w); 3918 3919 if (cdata->oacs.security) { 3920 spdk_json_write_named_object_begin(w, "security"); 3921 3922 spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal); 3923 3924 spdk_json_write_object_end(w); 3925 } 3926 3927 spdk_json_write_object_end(w); 3928 } 3929 3930 static const char * 3931 nvme_bdev_get_mp_policy_str(struct nvme_bdev *nbdev) 3932 { 3933 switch (nbdev->mp_policy) { 3934 case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE: 3935 return "active_passive"; 3936 case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE: 3937 return "active_active"; 3938 default: 3939 assert(false); 3940 return "invalid"; 3941 } 3942 } 3943 3944 static const char * 3945 nvme_bdev_get_mp_selector_str(struct nvme_bdev *nbdev) 3946 { 3947 switch (nbdev->mp_selector) { 3948 case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN: 3949 return "round_robin"; 3950 case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH: 3951 return "queue_depth"; 3952 default: 3953 assert(false); 3954 return "invalid"; 3955 } 3956 } 3957 3958 static int 3959 bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 3960 { 3961 struct nvme_bdev *nvme_bdev = ctx; 3962 struct nvme_ns *nvme_ns; 3963 3964 pthread_mutex_lock(&nvme_bdev->mutex); 3965 spdk_json_write_named_array_begin(w, "nvme"); 3966 TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) { 3967 nvme_namespace_info_json(w, nvme_ns); 3968 } 3969 spdk_json_write_array_end(w); 3970 spdk_json_write_named_string(w, "mp_policy", nvme_bdev_get_mp_policy_str(nvme_bdev)); 3971 if (nvme_bdev->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 3972 spdk_json_write_named_string(w, "selector", nvme_bdev_get_mp_selector_str(nvme_bdev)); 3973 if (nvme_bdev->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 3974 spdk_json_write_named_uint32(w, "rr_min_io", nvme_bdev->rr_min_io); 3975 } 3976 } 3977 pthread_mutex_unlock(&nvme_bdev->mutex); 3978 3979 return 0; 3980 } 3981 3982 static void 3983 bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 3984 { 3985 /* No config per bdev needed */ 3986 } 3987 3988 static uint64_t 3989 bdev_nvme_get_spin_time(struct spdk_io_channel *ch) 3990 { 3991 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 3992 struct nvme_io_path *io_path; 3993 struct nvme_poll_group *group; 3994 uint64_t spin_time = 0; 3995 3996 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 3997 group = io_path->qpair->group; 3998 3999 if (!group || !group->collect_spin_stat) { 4000 continue; 4001 } 4002 4003 if (group->end_ticks != 0) { 4004 group->spin_ticks += (group->end_ticks - group->start_ticks); 4005 group->end_ticks = 0; 4006 } 4007 4008 spin_time += group->spin_ticks; 4009 group->start_ticks = 0; 4010 group->spin_ticks = 0; 4011 } 4012 4013 return (spin_time * 1000000ULL) / spdk_get_ticks_hz(); 4014 } 4015 4016 static void 4017 bdev_nvme_reset_device_stat(void *ctx) 4018 { 4019 struct nvme_bdev *nbdev = ctx; 4020 4021 if (nbdev->err_stat != NULL) { 4022 memset(nbdev->err_stat, 0, sizeof(struct nvme_error_stat)); 4023 } 4024 } 4025 4026 /* JSON string should be lowercases and underscore delimited string. */ 4027 static void 4028 bdev_nvme_format_nvme_status(char *dst, const char *src) 4029 { 4030 char tmp[256]; 4031 4032 spdk_strcpy_replace(dst, 256, src, " - ", "_"); 4033 spdk_strcpy_replace(tmp, 256, dst, "-", "_"); 4034 spdk_strcpy_replace(dst, 256, tmp, " ", "_"); 4035 spdk_strlwr(dst); 4036 } 4037 4038 static void 4039 bdev_nvme_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w) 4040 { 4041 struct nvme_bdev *nbdev = ctx; 4042 struct spdk_nvme_status status = {}; 4043 uint16_t sct, sc; 4044 char status_json[256]; 4045 const char *status_str; 4046 4047 if (nbdev->err_stat == NULL) { 4048 return; 4049 } 4050 4051 spdk_json_write_named_object_begin(w, "nvme_error"); 4052 4053 spdk_json_write_named_object_begin(w, "status_type"); 4054 for (sct = 0; sct < 8; sct++) { 4055 if (nbdev->err_stat->status_type[sct] == 0) { 4056 continue; 4057 } 4058 status.sct = sct; 4059 4060 status_str = spdk_nvme_cpl_get_status_type_string(&status); 4061 assert(status_str != NULL); 4062 bdev_nvme_format_nvme_status(status_json, status_str); 4063 4064 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status_type[sct]); 4065 } 4066 spdk_json_write_object_end(w); 4067 4068 spdk_json_write_named_object_begin(w, "status_code"); 4069 for (sct = 0; sct < 4; sct++) { 4070 status.sct = sct; 4071 for (sc = 0; sc < 256; sc++) { 4072 if (nbdev->err_stat->status[sct][sc] == 0) { 4073 continue; 4074 } 4075 status.sc = sc; 4076 4077 status_str = spdk_nvme_cpl_get_status_string(&status); 4078 assert(status_str != NULL); 4079 bdev_nvme_format_nvme_status(status_json, status_str); 4080 4081 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status[sct][sc]); 4082 } 4083 } 4084 spdk_json_write_object_end(w); 4085 4086 spdk_json_write_object_end(w); 4087 } 4088 4089 static bool 4090 bdev_nvme_accel_sequence_supported(void *ctx, enum spdk_bdev_io_type type) 4091 { 4092 struct nvme_bdev *nbdev = ctx; 4093 struct spdk_nvme_ctrlr *ctrlr; 4094 4095 if (!g_opts.allow_accel_sequence) { 4096 return false; 4097 } 4098 4099 switch (type) { 4100 case SPDK_BDEV_IO_TYPE_WRITE: 4101 case SPDK_BDEV_IO_TYPE_READ: 4102 break; 4103 default: 4104 return false; 4105 } 4106 4107 ctrlr = bdev_nvme_get_ctrlr(&nbdev->disk); 4108 assert(ctrlr != NULL); 4109 4110 return spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ACCEL_SEQUENCE_SUPPORTED; 4111 } 4112 4113 static const struct spdk_bdev_fn_table nvmelib_fn_table = { 4114 .destruct = bdev_nvme_destruct, 4115 .submit_request = bdev_nvme_submit_request, 4116 .io_type_supported = bdev_nvme_io_type_supported, 4117 .get_io_channel = bdev_nvme_get_io_channel, 4118 .dump_info_json = bdev_nvme_dump_info_json, 4119 .write_config_json = bdev_nvme_write_config_json, 4120 .get_spin_time = bdev_nvme_get_spin_time, 4121 .get_module_ctx = bdev_nvme_get_module_ctx, 4122 .get_memory_domains = bdev_nvme_get_memory_domains, 4123 .accel_sequence_supported = bdev_nvme_accel_sequence_supported, 4124 .reset_device_stat = bdev_nvme_reset_device_stat, 4125 .dump_device_stat_json = bdev_nvme_dump_device_stat_json, 4126 }; 4127 4128 typedef int (*bdev_nvme_parse_ana_log_page_cb)( 4129 const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg); 4130 4131 static int 4132 bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 4133 bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg) 4134 { 4135 struct spdk_nvme_ana_group_descriptor *copied_desc; 4136 uint8_t *orig_desc; 4137 uint32_t i, desc_size, copy_len; 4138 int rc = 0; 4139 4140 if (nvme_ctrlr->ana_log_page == NULL) { 4141 return -EINVAL; 4142 } 4143 4144 copied_desc = nvme_ctrlr->copied_ana_desc; 4145 4146 orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page); 4147 copy_len = nvme_ctrlr->max_ana_log_page_size - sizeof(struct spdk_nvme_ana_page); 4148 4149 for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) { 4150 memcpy(copied_desc, orig_desc, copy_len); 4151 4152 rc = cb_fn(copied_desc, cb_arg); 4153 if (rc != 0) { 4154 break; 4155 } 4156 4157 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) + 4158 copied_desc->num_of_nsid * sizeof(uint32_t); 4159 orig_desc += desc_size; 4160 copy_len -= desc_size; 4161 } 4162 4163 return rc; 4164 } 4165 4166 static int 4167 nvme_ns_ana_transition_timedout(void *ctx) 4168 { 4169 struct nvme_ns *nvme_ns = ctx; 4170 4171 spdk_poller_unregister(&nvme_ns->anatt_timer); 4172 nvme_ns->ana_transition_timedout = true; 4173 4174 return SPDK_POLLER_BUSY; 4175 } 4176 4177 static void 4178 _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns, 4179 const struct spdk_nvme_ana_group_descriptor *desc) 4180 { 4181 const struct spdk_nvme_ctrlr_data *cdata; 4182 4183 nvme_ns->ana_group_id = desc->ana_group_id; 4184 nvme_ns->ana_state = desc->ana_state; 4185 nvme_ns->ana_state_updating = false; 4186 4187 switch (nvme_ns->ana_state) { 4188 case SPDK_NVME_ANA_OPTIMIZED_STATE: 4189 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 4190 nvme_ns->ana_transition_timedout = false; 4191 spdk_poller_unregister(&nvme_ns->anatt_timer); 4192 break; 4193 4194 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 4195 case SPDK_NVME_ANA_CHANGE_STATE: 4196 if (nvme_ns->anatt_timer != NULL) { 4197 break; 4198 } 4199 4200 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 4201 nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout, 4202 nvme_ns, 4203 cdata->anatt * SPDK_SEC_TO_USEC); 4204 break; 4205 default: 4206 break; 4207 } 4208 } 4209 4210 static int 4211 nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg) 4212 { 4213 struct nvme_ns *nvme_ns = cb_arg; 4214 uint32_t i; 4215 4216 assert(nvme_ns->ns != NULL); 4217 4218 for (i = 0; i < desc->num_of_nsid; i++) { 4219 if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) { 4220 continue; 4221 } 4222 4223 _nvme_ns_set_ana_state(nvme_ns, desc); 4224 return 1; 4225 } 4226 4227 return 0; 4228 } 4229 4230 static int 4231 nvme_generate_uuid(const char *sn, uint32_t nsid, struct spdk_uuid *uuid) 4232 { 4233 int rc = 0; 4234 struct spdk_uuid new_uuid, namespace_uuid; 4235 char merged_str[SPDK_NVME_CTRLR_SN_LEN + NSID_STR_LEN + 1] = {'\0'}; 4236 /* This namespace UUID was generated using uuid_generate() method. */ 4237 const char *namespace_str = {"edaed2de-24bc-4b07-b559-f47ecbe730fd"}; 4238 int size; 4239 4240 assert(strlen(sn) <= SPDK_NVME_CTRLR_SN_LEN); 4241 4242 spdk_uuid_set_null(&new_uuid); 4243 spdk_uuid_set_null(&namespace_uuid); 4244 4245 size = snprintf(merged_str, sizeof(merged_str), "%s%"PRIu32, sn, nsid); 4246 if (size <= 0 || (unsigned long)size >= sizeof(merged_str)) { 4247 return -EINVAL; 4248 } 4249 4250 spdk_uuid_parse(&namespace_uuid, namespace_str); 4251 4252 rc = spdk_uuid_generate_sha1(&new_uuid, &namespace_uuid, merged_str, size); 4253 if (rc == 0) { 4254 memcpy(uuid, &new_uuid, sizeof(struct spdk_uuid)); 4255 } 4256 4257 return rc; 4258 } 4259 4260 static int 4261 nvme_disk_create(struct spdk_bdev *disk, const char *base_name, 4262 struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns, 4263 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, void *ctx) 4264 { 4265 const struct spdk_uuid *uuid; 4266 const uint8_t *nguid; 4267 const struct spdk_nvme_ctrlr_data *cdata; 4268 const struct spdk_nvme_ns_data *nsdata; 4269 const struct spdk_nvme_ctrlr_opts *opts; 4270 enum spdk_nvme_csi csi; 4271 uint32_t atomic_bs, phys_bs, bs; 4272 char sn_tmp[SPDK_NVME_CTRLR_SN_LEN + 1] = {'\0'}; 4273 int rc; 4274 4275 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4276 csi = spdk_nvme_ns_get_csi(ns); 4277 opts = spdk_nvme_ctrlr_get_opts(ctrlr); 4278 4279 switch (csi) { 4280 case SPDK_NVME_CSI_NVM: 4281 disk->product_name = "NVMe disk"; 4282 break; 4283 case SPDK_NVME_CSI_ZNS: 4284 disk->product_name = "NVMe ZNS disk"; 4285 disk->zoned = true; 4286 disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 4287 disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) / 4288 spdk_nvme_ns_get_extended_sector_size(ns); 4289 disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns); 4290 disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns); 4291 break; 4292 default: 4293 if (bdev_opts->allow_unrecognized_csi) { 4294 disk->product_name = "NVMe Passthrough disk"; 4295 break; 4296 } 4297 SPDK_ERRLOG("unsupported CSI: %u\n", csi); 4298 return -ENOTSUP; 4299 } 4300 4301 nguid = spdk_nvme_ns_get_nguid(ns); 4302 if (!nguid) { 4303 uuid = spdk_nvme_ns_get_uuid(ns); 4304 if (uuid) { 4305 disk->uuid = *uuid; 4306 } else if (g_opts.generate_uuids) { 4307 spdk_strcpy_pad(sn_tmp, cdata->sn, SPDK_NVME_CTRLR_SN_LEN, '\0'); 4308 rc = nvme_generate_uuid(sn_tmp, spdk_nvme_ns_get_id(ns), &disk->uuid); 4309 if (rc < 0) { 4310 SPDK_ERRLOG("UUID generation failed (%s)\n", spdk_strerror(-rc)); 4311 return rc; 4312 } 4313 } 4314 } else { 4315 memcpy(&disk->uuid, nguid, sizeof(disk->uuid)); 4316 } 4317 4318 disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns)); 4319 if (!disk->name) { 4320 return -ENOMEM; 4321 } 4322 4323 disk->write_cache = 0; 4324 if (cdata->vwc.present) { 4325 /* Enable if the Volatile Write Cache exists */ 4326 disk->write_cache = 1; 4327 } 4328 if (cdata->oncs.write_zeroes) { 4329 disk->max_write_zeroes = UINT16_MAX + 1; 4330 } 4331 disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns); 4332 disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns); 4333 disk->max_segment_size = spdk_nvme_ctrlr_get_max_xfer_size(ctrlr); 4334 disk->ctratt.raw = cdata->ctratt.raw; 4335 /* NVMe driver will split one request into multiple requests 4336 * based on MDTS and stripe boundary, the bdev layer will use 4337 * max_segment_size and max_num_segments to split one big IO 4338 * into multiple requests, then small request can't run out 4339 * of NVMe internal requests data structure. 4340 */ 4341 if (opts && opts->io_queue_requests) { 4342 disk->max_num_segments = opts->io_queue_requests / 2; 4343 } 4344 if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_SGL_SUPPORTED) { 4345 /* The nvme driver will try to split I/O that have too many 4346 * SGEs, but it doesn't work if that last SGE doesn't end on 4347 * an aggregate total that is block aligned. The bdev layer has 4348 * a more robust splitting framework, so use that instead for 4349 * this case. (See issue #3269.) 4350 */ 4351 uint16_t max_sges = spdk_nvme_ctrlr_get_max_sges(ctrlr); 4352 4353 if (disk->max_num_segments == 0) { 4354 disk->max_num_segments = max_sges; 4355 } else { 4356 disk->max_num_segments = spdk_min(disk->max_num_segments, max_sges); 4357 } 4358 } 4359 disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns); 4360 4361 nsdata = spdk_nvme_ns_get_data(ns); 4362 bs = spdk_nvme_ns_get_sector_size(ns); 4363 atomic_bs = bs; 4364 phys_bs = bs; 4365 if (nsdata->nabo == 0) { 4366 if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) { 4367 atomic_bs = bs * (1 + nsdata->nawupf); 4368 } else { 4369 atomic_bs = bs * (1 + cdata->awupf); 4370 } 4371 } 4372 if (nsdata->nsfeat.optperf) { 4373 phys_bs = bs * (1 + nsdata->npwg); 4374 } 4375 disk->phys_blocklen = spdk_min(phys_bs, atomic_bs); 4376 4377 disk->md_len = spdk_nvme_ns_get_md_size(ns); 4378 if (disk->md_len != 0) { 4379 disk->md_interleave = nsdata->flbas.extended; 4380 disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns); 4381 if (disk->dif_type != SPDK_DIF_DISABLE) { 4382 disk->dif_is_head_of_md = nsdata->dps.md_start; 4383 disk->dif_check_flags = bdev_opts->prchk_flags; 4384 disk->dif_pi_format = (enum spdk_dif_pi_format)spdk_nvme_ns_get_pi_format(ns); 4385 } 4386 } 4387 4388 if (!(spdk_nvme_ctrlr_get_flags(ctrlr) & 4389 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) { 4390 disk->acwu = 0; 4391 } else if (nsdata->nsfeat.ns_atomic_write_unit) { 4392 disk->acwu = nsdata->nacwu + 1; /* 0-based */ 4393 } else { 4394 disk->acwu = cdata->acwu + 1; /* 0-based */ 4395 } 4396 4397 if (cdata->oncs.copy) { 4398 /* For now bdev interface allows only single segment copy */ 4399 disk->max_copy = nsdata->mssrl; 4400 } 4401 4402 disk->ctxt = ctx; 4403 disk->fn_table = &nvmelib_fn_table; 4404 disk->module = &nvme_if; 4405 4406 disk->numa.id_valid = 1; 4407 disk->numa.id = spdk_nvme_ctrlr_get_numa_id(ctrlr); 4408 4409 return 0; 4410 } 4411 4412 static struct nvme_bdev * 4413 nvme_bdev_alloc(void) 4414 { 4415 struct nvme_bdev *bdev; 4416 int rc; 4417 4418 bdev = calloc(1, sizeof(*bdev)); 4419 if (!bdev) { 4420 SPDK_ERRLOG("bdev calloc() failed\n"); 4421 return NULL; 4422 } 4423 4424 if (g_opts.nvme_error_stat) { 4425 bdev->err_stat = calloc(1, sizeof(struct nvme_error_stat)); 4426 if (!bdev->err_stat) { 4427 SPDK_ERRLOG("err_stat calloc() failed\n"); 4428 free(bdev); 4429 return NULL; 4430 } 4431 } 4432 4433 rc = pthread_mutex_init(&bdev->mutex, NULL); 4434 if (rc != 0) { 4435 free(bdev->err_stat); 4436 free(bdev); 4437 return NULL; 4438 } 4439 4440 bdev->ref = 1; 4441 bdev->mp_policy = BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE; 4442 bdev->mp_selector = BDEV_NVME_MP_SELECTOR_ROUND_ROBIN; 4443 bdev->rr_min_io = UINT32_MAX; 4444 TAILQ_INIT(&bdev->nvme_ns_list); 4445 4446 return bdev; 4447 } 4448 4449 static int 4450 nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4451 { 4452 struct nvme_bdev *bdev; 4453 struct nvme_bdev_ctrlr *nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr; 4454 int rc; 4455 4456 bdev = nvme_bdev_alloc(); 4457 if (bdev == NULL) { 4458 SPDK_ERRLOG("Failed to allocate NVMe bdev\n"); 4459 return -ENOMEM; 4460 } 4461 4462 bdev->opal = nvme_ctrlr->opal_dev != NULL; 4463 4464 rc = nvme_disk_create(&bdev->disk, nbdev_ctrlr->name, nvme_ctrlr->ctrlr, 4465 nvme_ns->ns, &nvme_ctrlr->opts, bdev); 4466 if (rc != 0) { 4467 SPDK_ERRLOG("Failed to create NVMe disk\n"); 4468 nvme_bdev_free(bdev); 4469 return rc; 4470 } 4471 4472 spdk_io_device_register(bdev, 4473 bdev_nvme_create_bdev_channel_cb, 4474 bdev_nvme_destroy_bdev_channel_cb, 4475 sizeof(struct nvme_bdev_channel), 4476 bdev->disk.name); 4477 4478 nvme_ns->bdev = bdev; 4479 bdev->nsid = nvme_ns->id; 4480 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 4481 4482 bdev->nbdev_ctrlr = nbdev_ctrlr; 4483 TAILQ_INSERT_TAIL(&nbdev_ctrlr->bdevs, bdev, tailq); 4484 4485 rc = spdk_bdev_register(&bdev->disk); 4486 if (rc != 0) { 4487 SPDK_ERRLOG("spdk_bdev_register() failed\n"); 4488 spdk_io_device_unregister(bdev, NULL); 4489 nvme_ns->bdev = NULL; 4490 TAILQ_REMOVE(&nbdev_ctrlr->bdevs, bdev, tailq); 4491 nvme_bdev_free(bdev); 4492 return rc; 4493 } 4494 4495 return 0; 4496 } 4497 4498 static bool 4499 bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2) 4500 { 4501 const struct spdk_nvme_ns_data *nsdata1, *nsdata2; 4502 const struct spdk_uuid *uuid1, *uuid2; 4503 4504 nsdata1 = spdk_nvme_ns_get_data(ns1); 4505 nsdata2 = spdk_nvme_ns_get_data(ns2); 4506 uuid1 = spdk_nvme_ns_get_uuid(ns1); 4507 uuid2 = spdk_nvme_ns_get_uuid(ns2); 4508 4509 return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 && 4510 nsdata1->eui64 == nsdata2->eui64 && 4511 ((uuid1 == NULL && uuid2 == NULL) || 4512 (uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) && 4513 spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2); 4514 } 4515 4516 static bool 4517 hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 4518 struct spdk_nvme_ctrlr_opts *opts) 4519 { 4520 struct nvme_probe_skip_entry *entry; 4521 4522 TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) { 4523 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 4524 return false; 4525 } 4526 } 4527 4528 opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst; 4529 opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight; 4530 opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight; 4531 opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight; 4532 opts->disable_read_ana_log_page = true; 4533 4534 SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr); 4535 4536 return true; 4537 } 4538 4539 static void 4540 nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl) 4541 { 4542 struct nvme_ctrlr *nvme_ctrlr = ctx; 4543 4544 if (spdk_nvme_cpl_is_error(cpl)) { 4545 SPDK_WARNLOG("Abort failed. Resetting controller. sc is %u, sct is %u.\n", cpl->status.sc, 4546 cpl->status.sct); 4547 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4548 } else if (cpl->cdw0 & 0x1) { 4549 SPDK_WARNLOG("Specified command could not be aborted.\n"); 4550 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4551 } 4552 } 4553 4554 static void 4555 timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, 4556 struct spdk_nvme_qpair *qpair, uint16_t cid) 4557 { 4558 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 4559 union spdk_nvme_csts_register csts; 4560 int rc; 4561 4562 assert(nvme_ctrlr->ctrlr == ctrlr); 4563 4564 SPDK_WARNLOG("Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n", ctrlr, qpair, cid); 4565 4566 /* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O 4567 * queue. (Note: qpair == NULL when there's an admin cmd timeout.) Otherwise we 4568 * would submit another fabrics cmd on the admin queue to read CSTS and check for its 4569 * completion recursively. 4570 */ 4571 if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) { 4572 csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr); 4573 if (csts.bits.cfs) { 4574 SPDK_ERRLOG("Controller Fatal Status, reset required\n"); 4575 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4576 return; 4577 } 4578 } 4579 4580 switch (g_opts.action_on_timeout) { 4581 case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT: 4582 if (qpair) { 4583 /* Don't send abort to ctrlr when ctrlr is not available. */ 4584 pthread_mutex_lock(&nvme_ctrlr->mutex); 4585 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 4586 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4587 SPDK_NOTICELOG("Quit abort. Ctrlr is not available.\n"); 4588 return; 4589 } 4590 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4591 4592 rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid, 4593 nvme_abort_cpl, nvme_ctrlr); 4594 if (rc == 0) { 4595 return; 4596 } 4597 4598 SPDK_ERRLOG("Unable to send abort. Resetting, rc is %d.\n", rc); 4599 } 4600 4601 /* FALLTHROUGH */ 4602 case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET: 4603 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4604 break; 4605 case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE: 4606 SPDK_DEBUGLOG(bdev_nvme, "No action for nvme controller timeout.\n"); 4607 break; 4608 default: 4609 SPDK_ERRLOG("An invalid timeout action value is found.\n"); 4610 break; 4611 } 4612 } 4613 4614 static struct nvme_ns * 4615 nvme_ns_alloc(void) 4616 { 4617 struct nvme_ns *nvme_ns; 4618 4619 nvme_ns = calloc(1, sizeof(struct nvme_ns)); 4620 if (nvme_ns == NULL) { 4621 return NULL; 4622 } 4623 4624 if (g_opts.io_path_stat) { 4625 nvme_ns->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 4626 if (nvme_ns->stat == NULL) { 4627 free(nvme_ns); 4628 return NULL; 4629 } 4630 spdk_bdev_reset_io_stat(nvme_ns->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 4631 } 4632 4633 return nvme_ns; 4634 } 4635 4636 static void 4637 nvme_ns_free(struct nvme_ns *nvme_ns) 4638 { 4639 free(nvme_ns->stat); 4640 free(nvme_ns); 4641 } 4642 4643 static void 4644 nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc) 4645 { 4646 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 4647 struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx; 4648 4649 if (rc == 0) { 4650 nvme_ns->probe_ctx = NULL; 4651 pthread_mutex_lock(&nvme_ctrlr->mutex); 4652 nvme_ctrlr->ref++; 4653 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4654 } else { 4655 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4656 nvme_ns_free(nvme_ns); 4657 } 4658 4659 if (ctx) { 4660 ctx->populates_in_progress--; 4661 if (ctx->populates_in_progress == 0) { 4662 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 4663 } 4664 } 4665 } 4666 4667 static void 4668 bdev_nvme_add_io_path(struct nvme_bdev_channel_iter *i, 4669 struct nvme_bdev *nbdev, 4670 struct nvme_bdev_channel *nbdev_ch, void *ctx) 4671 { 4672 struct nvme_ns *nvme_ns = ctx; 4673 int rc; 4674 4675 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 4676 if (rc != 0) { 4677 SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n"); 4678 } 4679 4680 nvme_bdev_for_each_channel_continue(i, rc); 4681 } 4682 4683 static void 4684 bdev_nvme_delete_io_path(struct nvme_bdev_channel_iter *i, 4685 struct nvme_bdev *nbdev, 4686 struct nvme_bdev_channel *nbdev_ch, void *ctx) 4687 { 4688 struct nvme_ns *nvme_ns = ctx; 4689 struct nvme_io_path *io_path; 4690 4691 io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns); 4692 if (io_path != NULL) { 4693 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 4694 } 4695 4696 nvme_bdev_for_each_channel_continue(i, 0); 4697 } 4698 4699 static void 4700 bdev_nvme_add_io_path_failed(struct nvme_bdev *nbdev, void *ctx, int status) 4701 { 4702 struct nvme_ns *nvme_ns = ctx; 4703 4704 nvme_ctrlr_populate_namespace_done(nvme_ns, -1); 4705 } 4706 4707 static void 4708 bdev_nvme_add_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status) 4709 { 4710 struct nvme_ns *nvme_ns = ctx; 4711 4712 if (status == 0) { 4713 nvme_ctrlr_populate_namespace_done(nvme_ns, 0); 4714 } else { 4715 /* Delete the added io_paths and fail populating the namespace. */ 4716 nvme_bdev_for_each_channel(nbdev, 4717 bdev_nvme_delete_io_path, 4718 nvme_ns, 4719 bdev_nvme_add_io_path_failed); 4720 } 4721 } 4722 4723 static int 4724 nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns) 4725 { 4726 struct nvme_ns *tmp_ns; 4727 const struct spdk_nvme_ns_data *nsdata; 4728 4729 nsdata = spdk_nvme_ns_get_data(nvme_ns->ns); 4730 if (!nsdata->nmic.can_share) { 4731 SPDK_ERRLOG("Namespace cannot be shared.\n"); 4732 return -EINVAL; 4733 } 4734 4735 pthread_mutex_lock(&bdev->mutex); 4736 4737 tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list); 4738 assert(tmp_ns != NULL); 4739 4740 if (tmp_ns->ns != NULL && !bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) { 4741 pthread_mutex_unlock(&bdev->mutex); 4742 SPDK_ERRLOG("Namespaces are not identical.\n"); 4743 return -EINVAL; 4744 } 4745 4746 bdev->ref++; 4747 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 4748 nvme_ns->bdev = bdev; 4749 4750 pthread_mutex_unlock(&bdev->mutex); 4751 4752 /* Add nvme_io_path to nvme_bdev_channels dynamically. */ 4753 nvme_bdev_for_each_channel(bdev, 4754 bdev_nvme_add_io_path, 4755 nvme_ns, 4756 bdev_nvme_add_io_path_done); 4757 4758 return 0; 4759 } 4760 4761 static void 4762 nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4763 { 4764 struct spdk_nvme_ns *ns; 4765 struct nvme_bdev *bdev; 4766 int rc = 0; 4767 4768 ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id); 4769 if (!ns) { 4770 SPDK_DEBUGLOG(bdev_nvme, "Invalid NS %d\n", nvme_ns->id); 4771 rc = -EINVAL; 4772 goto done; 4773 } 4774 4775 nvme_ns->ns = ns; 4776 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 4777 4778 if (nvme_ctrlr->ana_log_page != NULL) { 4779 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns); 4780 } 4781 4782 bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id); 4783 if (bdev == NULL) { 4784 rc = nvme_bdev_create(nvme_ctrlr, nvme_ns); 4785 } else { 4786 rc = nvme_bdev_add_ns(bdev, nvme_ns); 4787 if (rc == 0) { 4788 return; 4789 } 4790 } 4791 done: 4792 nvme_ctrlr_populate_namespace_done(nvme_ns, rc); 4793 } 4794 4795 static void 4796 nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns) 4797 { 4798 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 4799 4800 assert(nvme_ctrlr != NULL); 4801 4802 pthread_mutex_lock(&nvme_ctrlr->mutex); 4803 4804 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4805 4806 if (nvme_ns->bdev != NULL) { 4807 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4808 return; 4809 } 4810 4811 nvme_ns_free(nvme_ns); 4812 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4813 4814 nvme_ctrlr_release(nvme_ctrlr); 4815 } 4816 4817 static void 4818 bdev_nvme_delete_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status) 4819 { 4820 struct nvme_ns *nvme_ns = ctx; 4821 4822 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 4823 } 4824 4825 static void 4826 nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4827 { 4828 struct nvme_bdev *bdev; 4829 4830 spdk_poller_unregister(&nvme_ns->anatt_timer); 4831 4832 bdev = nvme_ns->bdev; 4833 if (bdev != NULL) { 4834 pthread_mutex_lock(&bdev->mutex); 4835 4836 assert(bdev->ref > 0); 4837 bdev->ref--; 4838 if (bdev->ref == 0) { 4839 pthread_mutex_unlock(&bdev->mutex); 4840 4841 spdk_bdev_unregister(&bdev->disk, NULL, NULL); 4842 } else { 4843 /* spdk_bdev_unregister() is not called until the last nvme_ns is 4844 * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list 4845 * and clear nvme_ns->bdev here. 4846 */ 4847 TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq); 4848 nvme_ns->bdev = NULL; 4849 4850 pthread_mutex_unlock(&bdev->mutex); 4851 4852 /* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that, 4853 * we call depopulate_namespace_done() to avoid use-after-free. 4854 */ 4855 nvme_bdev_for_each_channel(bdev, 4856 bdev_nvme_delete_io_path, 4857 nvme_ns, 4858 bdev_nvme_delete_io_path_done); 4859 return; 4860 } 4861 } 4862 4863 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 4864 } 4865 4866 static void 4867 nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 4868 struct nvme_async_probe_ctx *ctx) 4869 { 4870 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4871 struct nvme_ns *nvme_ns, *next; 4872 struct spdk_nvme_ns *ns; 4873 struct nvme_bdev *bdev; 4874 uint32_t nsid; 4875 int rc; 4876 uint64_t num_sectors; 4877 4878 if (ctx) { 4879 /* Initialize this count to 1 to handle the populate functions 4880 * calling nvme_ctrlr_populate_namespace_done() immediately. 4881 */ 4882 ctx->populates_in_progress = 1; 4883 } 4884 4885 /* First loop over our existing namespaces and see if they have been 4886 * removed. */ 4887 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4888 while (nvme_ns != NULL) { 4889 next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 4890 4891 if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 4892 /* NS is still there or added again. Its attributes may have changed. */ 4893 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id); 4894 if (nvme_ns->ns != ns) { 4895 assert(nvme_ns->ns == NULL); 4896 nvme_ns->ns = ns; 4897 SPDK_DEBUGLOG(bdev_nvme, "NSID %u was added\n", nvme_ns->id); 4898 } 4899 4900 num_sectors = spdk_nvme_ns_get_num_sectors(ns); 4901 bdev = nvme_ns->bdev; 4902 assert(bdev != NULL); 4903 if (bdev->disk.blockcnt != num_sectors) { 4904 SPDK_NOTICELOG("NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n", 4905 nvme_ns->id, 4906 bdev->disk.name, 4907 bdev->disk.blockcnt, 4908 num_sectors); 4909 rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors); 4910 if (rc != 0) { 4911 SPDK_ERRLOG("Could not change num blocks for nvme bdev: name %s, errno: %d.\n", 4912 bdev->disk.name, rc); 4913 } 4914 } 4915 } else { 4916 /* Namespace was removed */ 4917 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 4918 } 4919 4920 nvme_ns = next; 4921 } 4922 4923 /* Loop through all of the namespaces at the nvme level and see if any of them are new */ 4924 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 4925 while (nsid != 0) { 4926 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 4927 4928 if (nvme_ns == NULL) { 4929 /* Found a new one */ 4930 nvme_ns = nvme_ns_alloc(); 4931 if (nvme_ns == NULL) { 4932 SPDK_ERRLOG("Failed to allocate namespace\n"); 4933 /* This just fails to attach the namespace. It may work on a future attempt. */ 4934 continue; 4935 } 4936 4937 nvme_ns->id = nsid; 4938 nvme_ns->ctrlr = nvme_ctrlr; 4939 4940 nvme_ns->bdev = NULL; 4941 4942 if (ctx) { 4943 ctx->populates_in_progress++; 4944 } 4945 nvme_ns->probe_ctx = ctx; 4946 4947 RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4948 4949 nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns); 4950 } 4951 4952 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid); 4953 } 4954 4955 if (ctx) { 4956 /* Decrement this count now that the loop is over to account 4957 * for the one we started with. If the count is then 0, we 4958 * know any populate_namespace functions completed immediately, 4959 * so we'll kick the callback here. 4960 */ 4961 ctx->populates_in_progress--; 4962 if (ctx->populates_in_progress == 0) { 4963 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 4964 } 4965 } 4966 4967 } 4968 4969 static void 4970 nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr) 4971 { 4972 struct nvme_ns *nvme_ns, *tmp; 4973 4974 RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) { 4975 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 4976 } 4977 } 4978 4979 static uint32_t 4980 nvme_ctrlr_get_ana_log_page_size(struct nvme_ctrlr *nvme_ctrlr) 4981 { 4982 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4983 const struct spdk_nvme_ctrlr_data *cdata; 4984 uint32_t nsid, ns_count = 0; 4985 4986 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4987 4988 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 4989 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 4990 ns_count++; 4991 } 4992 4993 return sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 4994 sizeof(struct spdk_nvme_ana_group_descriptor) + ns_count * 4995 sizeof(uint32_t); 4996 } 4997 4998 static int 4999 nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc, 5000 void *cb_arg) 5001 { 5002 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 5003 struct nvme_ns *nvme_ns; 5004 uint32_t i, nsid; 5005 5006 for (i = 0; i < desc->num_of_nsid; i++) { 5007 nsid = desc->nsid[i]; 5008 if (nsid == 0) { 5009 continue; 5010 } 5011 5012 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 5013 5014 if (nvme_ns == NULL) { 5015 /* Target told us that an inactive namespace had an ANA change */ 5016 continue; 5017 } 5018 5019 _nvme_ns_set_ana_state(nvme_ns, desc); 5020 } 5021 5022 return 0; 5023 } 5024 5025 static void 5026 bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 5027 { 5028 struct nvme_ns *nvme_ns; 5029 5030 spdk_free(nvme_ctrlr->ana_log_page); 5031 nvme_ctrlr->ana_log_page = NULL; 5032 5033 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 5034 nvme_ns != NULL; 5035 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 5036 nvme_ns->ana_state_updating = false; 5037 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 5038 } 5039 } 5040 5041 static void 5042 nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl) 5043 { 5044 struct nvme_ctrlr *nvme_ctrlr = ctx; 5045 5046 if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) { 5047 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states, 5048 nvme_ctrlr); 5049 } else { 5050 bdev_nvme_disable_read_ana_log_page(nvme_ctrlr); 5051 } 5052 5053 pthread_mutex_lock(&nvme_ctrlr->mutex); 5054 5055 assert(nvme_ctrlr->ana_log_page_updating == true); 5056 nvme_ctrlr->ana_log_page_updating = false; 5057 5058 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 5059 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5060 5061 nvme_ctrlr_unregister(nvme_ctrlr); 5062 } else { 5063 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5064 5065 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 5066 } 5067 } 5068 5069 static int 5070 nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 5071 { 5072 uint32_t ana_log_page_size; 5073 int rc; 5074 5075 if (nvme_ctrlr->ana_log_page == NULL) { 5076 return -EINVAL; 5077 } 5078 5079 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 5080 5081 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 5082 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 5083 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 5084 return -EINVAL; 5085 } 5086 5087 pthread_mutex_lock(&nvme_ctrlr->mutex); 5088 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 5089 nvme_ctrlr->ana_log_page_updating) { 5090 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5091 return -EBUSY; 5092 } 5093 5094 nvme_ctrlr->ana_log_page_updating = true; 5095 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5096 5097 rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr, 5098 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 5099 SPDK_NVME_GLOBAL_NS_TAG, 5100 nvme_ctrlr->ana_log_page, 5101 ana_log_page_size, 0, 5102 nvme_ctrlr_read_ana_log_page_done, 5103 nvme_ctrlr); 5104 if (rc != 0) { 5105 nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL); 5106 } 5107 5108 return rc; 5109 } 5110 5111 static void 5112 dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 5113 { 5114 } 5115 5116 struct bdev_nvme_set_preferred_path_ctx { 5117 struct spdk_bdev_desc *desc; 5118 struct nvme_ns *nvme_ns; 5119 bdev_nvme_set_preferred_path_cb cb_fn; 5120 void *cb_arg; 5121 }; 5122 5123 static void 5124 bdev_nvme_set_preferred_path_done(struct nvme_bdev *nbdev, void *_ctx, int status) 5125 { 5126 struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx; 5127 5128 assert(ctx != NULL); 5129 assert(ctx->desc != NULL); 5130 assert(ctx->cb_fn != NULL); 5131 5132 spdk_bdev_close(ctx->desc); 5133 5134 ctx->cb_fn(ctx->cb_arg, status); 5135 5136 free(ctx); 5137 } 5138 5139 static void 5140 _bdev_nvme_set_preferred_path(struct nvme_bdev_channel_iter *i, 5141 struct nvme_bdev *nbdev, 5142 struct nvme_bdev_channel *nbdev_ch, void *_ctx) 5143 { 5144 struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx; 5145 struct nvme_io_path *io_path, *prev; 5146 5147 prev = NULL; 5148 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 5149 if (io_path->nvme_ns == ctx->nvme_ns) { 5150 break; 5151 } 5152 prev = io_path; 5153 } 5154 5155 if (io_path != NULL) { 5156 if (prev != NULL) { 5157 STAILQ_REMOVE_AFTER(&nbdev_ch->io_path_list, prev, stailq); 5158 STAILQ_INSERT_HEAD(&nbdev_ch->io_path_list, io_path, stailq); 5159 } 5160 5161 /* We can set io_path to nbdev_ch->current_io_path directly here. 5162 * However, it needs to be conditional. To simplify the code, 5163 * just clear nbdev_ch->current_io_path and let find_io_path() 5164 * fill it. 5165 * 5166 * Automatic failback may be disabled. Hence even if the io_path is 5167 * already at the head, clear nbdev_ch->current_io_path. 5168 */ 5169 bdev_nvme_clear_current_io_path(nbdev_ch); 5170 } 5171 5172 nvme_bdev_for_each_channel_continue(i, 0); 5173 } 5174 5175 static struct nvme_ns * 5176 bdev_nvme_set_preferred_ns(struct nvme_bdev *nbdev, uint16_t cntlid) 5177 { 5178 struct nvme_ns *nvme_ns, *prev; 5179 const struct spdk_nvme_ctrlr_data *cdata; 5180 5181 prev = NULL; 5182 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 5183 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 5184 5185 if (cdata->cntlid == cntlid) { 5186 break; 5187 } 5188 prev = nvme_ns; 5189 } 5190 5191 if (nvme_ns != NULL && prev != NULL) { 5192 TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq); 5193 TAILQ_INSERT_HEAD(&nbdev->nvme_ns_list, nvme_ns, tailq); 5194 } 5195 5196 return nvme_ns; 5197 } 5198 5199 /* This function supports only multipath mode. There is only a single I/O path 5200 * for each NVMe-oF controller. Hence, just move the matched I/O path to the 5201 * head of the I/O path list for each NVMe bdev channel. 5202 * 5203 * NVMe bdev channel may be acquired after completing this function. move the 5204 * matched namespace to the head of the namespace list for the NVMe bdev too. 5205 */ 5206 void 5207 bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid, 5208 bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg) 5209 { 5210 struct bdev_nvme_set_preferred_path_ctx *ctx; 5211 struct spdk_bdev *bdev; 5212 struct nvme_bdev *nbdev; 5213 int rc = 0; 5214 5215 assert(cb_fn != NULL); 5216 5217 ctx = calloc(1, sizeof(*ctx)); 5218 if (ctx == NULL) { 5219 SPDK_ERRLOG("Failed to alloc context.\n"); 5220 rc = -ENOMEM; 5221 goto err_alloc; 5222 } 5223 5224 ctx->cb_fn = cb_fn; 5225 ctx->cb_arg = cb_arg; 5226 5227 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 5228 if (rc != 0) { 5229 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 5230 goto err_open; 5231 } 5232 5233 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 5234 5235 if (bdev->module != &nvme_if) { 5236 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 5237 rc = -ENODEV; 5238 goto err_bdev; 5239 } 5240 5241 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 5242 5243 pthread_mutex_lock(&nbdev->mutex); 5244 5245 ctx->nvme_ns = bdev_nvme_set_preferred_ns(nbdev, cntlid); 5246 if (ctx->nvme_ns == NULL) { 5247 pthread_mutex_unlock(&nbdev->mutex); 5248 5249 SPDK_ERRLOG("bdev %s does not have namespace to controller %u.\n", name, cntlid); 5250 rc = -ENODEV; 5251 goto err_bdev; 5252 } 5253 5254 pthread_mutex_unlock(&nbdev->mutex); 5255 5256 nvme_bdev_for_each_channel(nbdev, 5257 _bdev_nvme_set_preferred_path, 5258 ctx, 5259 bdev_nvme_set_preferred_path_done); 5260 return; 5261 5262 err_bdev: 5263 spdk_bdev_close(ctx->desc); 5264 err_open: 5265 free(ctx); 5266 err_alloc: 5267 cb_fn(cb_arg, rc); 5268 } 5269 5270 struct bdev_nvme_set_multipath_policy_ctx { 5271 struct spdk_bdev_desc *desc; 5272 spdk_bdev_nvme_set_multipath_policy_cb cb_fn; 5273 void *cb_arg; 5274 }; 5275 5276 static void 5277 bdev_nvme_set_multipath_policy_done(struct nvme_bdev *nbdev, void *_ctx, int status) 5278 { 5279 struct bdev_nvme_set_multipath_policy_ctx *ctx = _ctx; 5280 5281 assert(ctx != NULL); 5282 assert(ctx->desc != NULL); 5283 assert(ctx->cb_fn != NULL); 5284 5285 spdk_bdev_close(ctx->desc); 5286 5287 ctx->cb_fn(ctx->cb_arg, status); 5288 5289 free(ctx); 5290 } 5291 5292 static void 5293 _bdev_nvme_set_multipath_policy(struct nvme_bdev_channel_iter *i, 5294 struct nvme_bdev *nbdev, 5295 struct nvme_bdev_channel *nbdev_ch, void *ctx) 5296 { 5297 nbdev_ch->mp_policy = nbdev->mp_policy; 5298 nbdev_ch->mp_selector = nbdev->mp_selector; 5299 nbdev_ch->rr_min_io = nbdev->rr_min_io; 5300 bdev_nvme_clear_current_io_path(nbdev_ch); 5301 5302 nvme_bdev_for_each_channel_continue(i, 0); 5303 } 5304 5305 void 5306 spdk_bdev_nvme_set_multipath_policy(const char *name, enum spdk_bdev_nvme_multipath_policy policy, 5307 enum spdk_bdev_nvme_multipath_selector selector, uint32_t rr_min_io, 5308 spdk_bdev_nvme_set_multipath_policy_cb cb_fn, void *cb_arg) 5309 { 5310 struct bdev_nvme_set_multipath_policy_ctx *ctx; 5311 struct spdk_bdev *bdev; 5312 struct nvme_bdev *nbdev; 5313 int rc; 5314 5315 assert(cb_fn != NULL); 5316 5317 switch (policy) { 5318 case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE: 5319 break; 5320 case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE: 5321 switch (selector) { 5322 case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN: 5323 if (rr_min_io == UINT32_MAX) { 5324 rr_min_io = 1; 5325 } else if (rr_min_io == 0) { 5326 rc = -EINVAL; 5327 goto exit; 5328 } 5329 break; 5330 case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH: 5331 break; 5332 default: 5333 rc = -EINVAL; 5334 goto exit; 5335 } 5336 break; 5337 default: 5338 rc = -EINVAL; 5339 goto exit; 5340 } 5341 5342 ctx = calloc(1, sizeof(*ctx)); 5343 if (ctx == NULL) { 5344 SPDK_ERRLOG("Failed to alloc context.\n"); 5345 rc = -ENOMEM; 5346 goto exit; 5347 } 5348 5349 ctx->cb_fn = cb_fn; 5350 ctx->cb_arg = cb_arg; 5351 5352 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 5353 if (rc != 0) { 5354 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 5355 rc = -ENODEV; 5356 goto err_open; 5357 } 5358 5359 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 5360 if (bdev->module != &nvme_if) { 5361 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 5362 rc = -ENODEV; 5363 goto err_module; 5364 } 5365 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 5366 5367 pthread_mutex_lock(&nbdev->mutex); 5368 nbdev->mp_policy = policy; 5369 nbdev->mp_selector = selector; 5370 nbdev->rr_min_io = rr_min_io; 5371 pthread_mutex_unlock(&nbdev->mutex); 5372 5373 nvme_bdev_for_each_channel(nbdev, 5374 _bdev_nvme_set_multipath_policy, 5375 ctx, 5376 bdev_nvme_set_multipath_policy_done); 5377 return; 5378 5379 err_module: 5380 spdk_bdev_close(ctx->desc); 5381 err_open: 5382 free(ctx); 5383 exit: 5384 cb_fn(cb_arg, rc); 5385 } 5386 5387 static void 5388 aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 5389 { 5390 struct nvme_ctrlr *nvme_ctrlr = arg; 5391 union spdk_nvme_async_event_completion event; 5392 5393 if (spdk_nvme_cpl_is_error(cpl)) { 5394 SPDK_WARNLOG("AER request execute failed\n"); 5395 return; 5396 } 5397 5398 event.raw = cpl->cdw0; 5399 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 5400 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) { 5401 nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL); 5402 } else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 5403 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) { 5404 nvme_ctrlr_read_ana_log_page(nvme_ctrlr); 5405 } 5406 } 5407 5408 static void 5409 free_nvme_async_probe_ctx(struct nvme_async_probe_ctx *ctx) 5410 { 5411 spdk_keyring_put_key(ctx->drv_opts.tls_psk); 5412 spdk_keyring_put_key(ctx->drv_opts.dhchap_key); 5413 spdk_keyring_put_key(ctx->drv_opts.dhchap_ctrlr_key); 5414 free(ctx); 5415 } 5416 5417 static void 5418 populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, int rc) 5419 { 5420 if (ctx->cb_fn) { 5421 ctx->cb_fn(ctx->cb_ctx, ctx->reported_bdevs, rc); 5422 } 5423 5424 ctx->namespaces_populated = true; 5425 if (ctx->probe_done) { 5426 /* The probe was already completed, so we need to free the context 5427 * here. This can happen for cases like OCSSD, where we need to 5428 * send additional commands to the SSD after attach. 5429 */ 5430 free_nvme_async_probe_ctx(ctx); 5431 } 5432 } 5433 5434 static int 5435 bdev_nvme_remove_poller(void *ctx) 5436 { 5437 struct spdk_nvme_transport_id trid_pcie; 5438 5439 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 5440 spdk_poller_unregister(&g_hotplug_poller); 5441 return SPDK_POLLER_IDLE; 5442 } 5443 5444 memset(&trid_pcie, 0, sizeof(trid_pcie)); 5445 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 5446 5447 if (spdk_nvme_scan_attached(&trid_pcie)) { 5448 SPDK_ERRLOG_RATELIMIT("spdk_nvme_scan_attached() failed\n"); 5449 } 5450 5451 return SPDK_POLLER_BUSY; 5452 } 5453 5454 static void 5455 nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr, 5456 struct nvme_async_probe_ctx *ctx) 5457 { 5458 spdk_io_device_register(nvme_ctrlr, 5459 bdev_nvme_create_ctrlr_channel_cb, 5460 bdev_nvme_destroy_ctrlr_channel_cb, 5461 sizeof(struct nvme_ctrlr_channel), 5462 nvme_ctrlr->nbdev_ctrlr->name); 5463 5464 nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx); 5465 5466 if (g_hotplug_poller == NULL) { 5467 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL, 5468 NVME_HOTPLUG_POLL_PERIOD_DEFAULT); 5469 } 5470 } 5471 5472 static void 5473 nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl) 5474 { 5475 struct nvme_ctrlr *nvme_ctrlr = _ctx; 5476 struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx; 5477 5478 nvme_ctrlr->probe_ctx = NULL; 5479 5480 if (spdk_nvme_cpl_is_error(cpl)) { 5481 nvme_ctrlr_delete(nvme_ctrlr); 5482 5483 if (ctx != NULL) { 5484 ctx->reported_bdevs = 0; 5485 populate_namespaces_cb(ctx, -1); 5486 } 5487 return; 5488 } 5489 5490 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 5491 } 5492 5493 static int 5494 nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 5495 struct nvme_async_probe_ctx *ctx) 5496 { 5497 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 5498 const struct spdk_nvme_ctrlr_data *cdata; 5499 uint32_t ana_log_page_size; 5500 5501 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5502 5503 /* Set buffer size enough to include maximum number of allowed namespaces. */ 5504 ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 5505 sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->mnan * 5506 sizeof(uint32_t); 5507 5508 nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL, 5509 SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_DMA); 5510 if (nvme_ctrlr->ana_log_page == NULL) { 5511 SPDK_ERRLOG("could not allocate ANA log page buffer\n"); 5512 return -ENXIO; 5513 } 5514 5515 /* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned. 5516 * Hence copy each descriptor to a temporary area when parsing it. 5517 * 5518 * Allocate a buffer whose size is as large as ANA log page buffer because 5519 * we do not know the size of a descriptor until actually reading it. 5520 */ 5521 nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size); 5522 if (nvme_ctrlr->copied_ana_desc == NULL) { 5523 SPDK_ERRLOG("could not allocate a buffer to parse ANA descriptor\n"); 5524 return -ENOMEM; 5525 } 5526 5527 nvme_ctrlr->max_ana_log_page_size = ana_log_page_size; 5528 5529 nvme_ctrlr->probe_ctx = ctx; 5530 5531 /* Then, set the read size only to include the current active namespaces. */ 5532 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 5533 5534 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 5535 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 5536 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 5537 return -EINVAL; 5538 } 5539 5540 return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, 5541 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 5542 SPDK_NVME_GLOBAL_NS_TAG, 5543 nvme_ctrlr->ana_log_page, 5544 ana_log_page_size, 0, 5545 nvme_ctrlr_init_ana_log_page_done, 5546 nvme_ctrlr); 5547 } 5548 5549 /* hostnqn and subnqn were already verified before attaching a controller. 5550 * Hence check only the multipath capability and cntlid here. 5551 */ 5552 static bool 5553 bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr) 5554 { 5555 struct nvme_ctrlr *tmp; 5556 const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata; 5557 5558 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5559 5560 if (!cdata->cmic.multi_ctrlr) { 5561 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 5562 return false; 5563 } 5564 5565 TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) { 5566 tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr); 5567 5568 if (!tmp_cdata->cmic.multi_ctrlr) { 5569 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 5570 return false; 5571 } 5572 if (cdata->cntlid == tmp_cdata->cntlid) { 5573 SPDK_ERRLOG("cntlid %u are duplicated.\n", tmp_cdata->cntlid); 5574 return false; 5575 } 5576 } 5577 5578 return true; 5579 } 5580 5581 static int 5582 nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr) 5583 { 5584 struct nvme_bdev_ctrlr *nbdev_ctrlr; 5585 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 5586 int rc = 0; 5587 5588 pthread_mutex_lock(&g_bdev_nvme_mutex); 5589 5590 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 5591 if (nbdev_ctrlr != NULL) { 5592 if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) { 5593 rc = -EINVAL; 5594 goto exit; 5595 } 5596 } else { 5597 nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr)); 5598 if (nbdev_ctrlr == NULL) { 5599 SPDK_ERRLOG("Failed to allocate nvme_bdev_ctrlr.\n"); 5600 rc = -ENOMEM; 5601 goto exit; 5602 } 5603 nbdev_ctrlr->name = strdup(name); 5604 if (nbdev_ctrlr->name == NULL) { 5605 SPDK_ERRLOG("Failed to allocate name of nvme_bdev_ctrlr.\n"); 5606 free(nbdev_ctrlr); 5607 goto exit; 5608 } 5609 TAILQ_INIT(&nbdev_ctrlr->ctrlrs); 5610 TAILQ_INIT(&nbdev_ctrlr->bdevs); 5611 TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 5612 } 5613 nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr; 5614 TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 5615 exit: 5616 pthread_mutex_unlock(&g_bdev_nvme_mutex); 5617 return rc; 5618 } 5619 5620 static int 5621 nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr, 5622 const char *name, 5623 const struct spdk_nvme_transport_id *trid, 5624 struct nvme_async_probe_ctx *ctx) 5625 { 5626 struct nvme_ctrlr *nvme_ctrlr; 5627 struct nvme_path_id *path_id; 5628 const struct spdk_nvme_ctrlr_data *cdata; 5629 int rc; 5630 5631 nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr)); 5632 if (nvme_ctrlr == NULL) { 5633 SPDK_ERRLOG("Failed to allocate device struct\n"); 5634 return -ENOMEM; 5635 } 5636 5637 rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL); 5638 if (rc != 0) { 5639 free(nvme_ctrlr); 5640 return rc; 5641 } 5642 5643 TAILQ_INIT(&nvme_ctrlr->trids); 5644 RB_INIT(&nvme_ctrlr->namespaces); 5645 5646 /* Get another reference to the key, so the first one can be released from probe_ctx */ 5647 if (ctx != NULL) { 5648 if (ctx->drv_opts.tls_psk != NULL) { 5649 nvme_ctrlr->psk = spdk_keyring_get_key( 5650 spdk_key_get_name(ctx->drv_opts.tls_psk)); 5651 if (nvme_ctrlr->psk == NULL) { 5652 /* Could only happen if the key was removed in the meantime */ 5653 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5654 spdk_key_get_name(ctx->drv_opts.tls_psk)); 5655 rc = -ENOKEY; 5656 goto err; 5657 } 5658 } 5659 5660 if (ctx->drv_opts.dhchap_key != NULL) { 5661 nvme_ctrlr->dhchap_key = spdk_keyring_get_key( 5662 spdk_key_get_name(ctx->drv_opts.dhchap_key)); 5663 if (nvme_ctrlr->dhchap_key == NULL) { 5664 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5665 spdk_key_get_name(ctx->drv_opts.dhchap_key)); 5666 rc = -ENOKEY; 5667 goto err; 5668 } 5669 } 5670 5671 if (ctx->drv_opts.dhchap_ctrlr_key != NULL) { 5672 nvme_ctrlr->dhchap_ctrlr_key = 5673 spdk_keyring_get_key( 5674 spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key)); 5675 if (nvme_ctrlr->dhchap_ctrlr_key == NULL) { 5676 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5677 spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key)); 5678 rc = -ENOKEY; 5679 goto err; 5680 } 5681 } 5682 } 5683 5684 path_id = calloc(1, sizeof(*path_id)); 5685 if (path_id == NULL) { 5686 SPDK_ERRLOG("Failed to allocate trid entry pointer\n"); 5687 rc = -ENOMEM; 5688 goto err; 5689 } 5690 5691 path_id->trid = *trid; 5692 if (ctx != NULL) { 5693 memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr)); 5694 memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid)); 5695 } 5696 nvme_ctrlr->active_path_id = path_id; 5697 TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link); 5698 5699 nvme_ctrlr->thread = spdk_get_thread(); 5700 nvme_ctrlr->ctrlr = ctrlr; 5701 nvme_ctrlr->ref = 1; 5702 5703 if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) { 5704 SPDK_ERRLOG("OCSSDs are not supported"); 5705 rc = -ENOTSUP; 5706 goto err; 5707 } 5708 5709 if (ctx != NULL) { 5710 memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts)); 5711 } else { 5712 spdk_bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts); 5713 } 5714 5715 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr, 5716 g_opts.nvme_adminq_poll_period_us); 5717 5718 if (g_opts.timeout_us > 0) { 5719 /* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */ 5720 /* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */ 5721 uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ? 5722 g_opts.timeout_us : g_opts.timeout_admin_us; 5723 spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us, 5724 adm_timeout_us, timeout_cb, nvme_ctrlr); 5725 } 5726 5727 spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr); 5728 spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr); 5729 5730 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 5731 SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) { 5732 nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr); 5733 } 5734 5735 rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr); 5736 if (rc != 0) { 5737 goto err; 5738 } 5739 5740 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5741 5742 if (cdata->cmic.ana_reporting) { 5743 rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx); 5744 if (rc == 0) { 5745 return 0; 5746 } 5747 } else { 5748 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 5749 return 0; 5750 } 5751 5752 err: 5753 nvme_ctrlr_delete(nvme_ctrlr); 5754 return rc; 5755 } 5756 5757 void 5758 spdk_bdev_nvme_get_default_ctrlr_opts(struct spdk_bdev_nvme_ctrlr_opts *opts) 5759 { 5760 opts->prchk_flags = 0; 5761 opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec; 5762 opts->reconnect_delay_sec = g_opts.reconnect_delay_sec; 5763 opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec; 5764 opts->multipath = false; 5765 } 5766 5767 static void 5768 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 5769 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts) 5770 { 5771 char *name; 5772 5773 name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++); 5774 if (!name) { 5775 SPDK_ERRLOG("Failed to assign name to NVMe device\n"); 5776 return; 5777 } 5778 5779 if (nvme_ctrlr_create(ctrlr, name, trid, NULL) == 0) { 5780 SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name); 5781 } else { 5782 SPDK_ERRLOG("Failed to attach to %s (%s)\n", trid->traddr, name); 5783 } 5784 5785 free(name); 5786 } 5787 5788 static void 5789 _nvme_ctrlr_destruct(void *ctx) 5790 { 5791 struct nvme_ctrlr *nvme_ctrlr = ctx; 5792 5793 nvme_ctrlr_depopulate_namespaces(nvme_ctrlr); 5794 nvme_ctrlr_release(nvme_ctrlr); 5795 } 5796 5797 static int 5798 bdev_nvme_delete_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 5799 { 5800 struct nvme_probe_skip_entry *entry; 5801 5802 /* The controller's destruction was already started */ 5803 if (nvme_ctrlr->destruct) { 5804 return -EALREADY; 5805 } 5806 5807 if (!hotplug && 5808 nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 5809 entry = calloc(1, sizeof(*entry)); 5810 if (!entry) { 5811 return -ENOMEM; 5812 } 5813 entry->trid = nvme_ctrlr->active_path_id->trid; 5814 TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq); 5815 } 5816 5817 nvme_ctrlr->destruct = true; 5818 return 0; 5819 } 5820 5821 static int 5822 bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 5823 { 5824 int rc; 5825 5826 pthread_mutex_lock(&nvme_ctrlr->mutex); 5827 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, hotplug); 5828 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5829 5830 if (rc == 0) { 5831 _nvme_ctrlr_destruct(nvme_ctrlr); 5832 } else if (rc == -EALREADY) { 5833 rc = 0; 5834 } 5835 5836 return rc; 5837 } 5838 5839 static void 5840 remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr) 5841 { 5842 struct nvme_ctrlr *nvme_ctrlr = cb_ctx; 5843 5844 bdev_nvme_delete_ctrlr(nvme_ctrlr, true); 5845 } 5846 5847 static int 5848 bdev_nvme_hotplug_probe(void *arg) 5849 { 5850 if (g_hotplug_probe_ctx == NULL) { 5851 spdk_poller_unregister(&g_hotplug_probe_poller); 5852 return SPDK_POLLER_IDLE; 5853 } 5854 5855 if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) { 5856 g_hotplug_probe_ctx = NULL; 5857 spdk_poller_unregister(&g_hotplug_probe_poller); 5858 } 5859 5860 return SPDK_POLLER_BUSY; 5861 } 5862 5863 static int 5864 bdev_nvme_hotplug(void *arg) 5865 { 5866 struct spdk_nvme_transport_id trid_pcie; 5867 5868 if (g_hotplug_probe_ctx) { 5869 return SPDK_POLLER_BUSY; 5870 } 5871 5872 memset(&trid_pcie, 0, sizeof(trid_pcie)); 5873 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 5874 5875 g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL, 5876 hotplug_probe_cb, attach_cb, NULL); 5877 5878 if (g_hotplug_probe_ctx) { 5879 assert(g_hotplug_probe_poller == NULL); 5880 g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000); 5881 } 5882 5883 return SPDK_POLLER_BUSY; 5884 } 5885 5886 void 5887 bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts) 5888 { 5889 *opts = g_opts; 5890 } 5891 5892 static bool bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 5893 uint32_t reconnect_delay_sec, 5894 uint32_t fast_io_fail_timeout_sec); 5895 5896 static int 5897 bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts) 5898 { 5899 if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) { 5900 /* Can't set timeout_admin_us without also setting timeout_us */ 5901 SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n"); 5902 return -EINVAL; 5903 } 5904 5905 if (opts->bdev_retry_count < -1) { 5906 SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n"); 5907 return -EINVAL; 5908 } 5909 5910 if (!bdev_nvme_check_io_error_resiliency_params(opts->ctrlr_loss_timeout_sec, 5911 opts->reconnect_delay_sec, 5912 opts->fast_io_fail_timeout_sec)) { 5913 return -EINVAL; 5914 } 5915 5916 return 0; 5917 } 5918 5919 int 5920 bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) 5921 { 5922 int ret; 5923 5924 ret = bdev_nvme_validate_opts(opts); 5925 if (ret) { 5926 SPDK_WARNLOG("Failed to set nvme opts.\n"); 5927 return ret; 5928 } 5929 5930 if (g_bdev_nvme_init_thread != NULL) { 5931 if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 5932 return -EPERM; 5933 } 5934 } 5935 5936 if (opts->rdma_srq_size != 0 || 5937 opts->rdma_max_cq_size != 0 || 5938 opts->rdma_cm_event_timeout_ms != 0) { 5939 struct spdk_nvme_transport_opts drv_opts; 5940 5941 spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts)); 5942 if (opts->rdma_srq_size != 0) { 5943 drv_opts.rdma_srq_size = opts->rdma_srq_size; 5944 } 5945 if (opts->rdma_max_cq_size != 0) { 5946 drv_opts.rdma_max_cq_size = opts->rdma_max_cq_size; 5947 } 5948 if (opts->rdma_cm_event_timeout_ms != 0) { 5949 drv_opts.rdma_cm_event_timeout_ms = opts->rdma_cm_event_timeout_ms; 5950 } 5951 5952 ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts)); 5953 if (ret) { 5954 SPDK_ERRLOG("Failed to set NVMe transport opts.\n"); 5955 return ret; 5956 } 5957 } 5958 5959 g_opts = *opts; 5960 5961 return 0; 5962 } 5963 5964 struct set_nvme_hotplug_ctx { 5965 uint64_t period_us; 5966 bool enabled; 5967 spdk_msg_fn fn; 5968 void *fn_ctx; 5969 }; 5970 5971 static void 5972 set_nvme_hotplug_period_cb(void *_ctx) 5973 { 5974 struct set_nvme_hotplug_ctx *ctx = _ctx; 5975 5976 spdk_poller_unregister(&g_hotplug_poller); 5977 if (ctx->enabled) { 5978 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us); 5979 } else { 5980 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL, 5981 NVME_HOTPLUG_POLL_PERIOD_DEFAULT); 5982 } 5983 5984 g_nvme_hotplug_poll_period_us = ctx->period_us; 5985 g_nvme_hotplug_enabled = ctx->enabled; 5986 if (ctx->fn) { 5987 ctx->fn(ctx->fn_ctx); 5988 } 5989 5990 free(ctx); 5991 } 5992 5993 int 5994 bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx) 5995 { 5996 struct set_nvme_hotplug_ctx *ctx; 5997 5998 if (enabled == true && !spdk_process_is_primary()) { 5999 return -EPERM; 6000 } 6001 6002 ctx = calloc(1, sizeof(*ctx)); 6003 if (ctx == NULL) { 6004 return -ENOMEM; 6005 } 6006 6007 period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us; 6008 ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX); 6009 ctx->enabled = enabled; 6010 ctx->fn = cb; 6011 ctx->fn_ctx = cb_ctx; 6012 6013 spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx); 6014 return 0; 6015 } 6016 6017 static void 6018 nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 6019 struct nvme_async_probe_ctx *ctx) 6020 { 6021 struct nvme_ns *nvme_ns; 6022 struct nvme_bdev *nvme_bdev; 6023 size_t j; 6024 6025 assert(nvme_ctrlr != NULL); 6026 6027 if (ctx->names == NULL) { 6028 ctx->reported_bdevs = 0; 6029 populate_namespaces_cb(ctx, 0); 6030 return; 6031 } 6032 6033 /* 6034 * Report the new bdevs that were created in this call. 6035 * There can be more than one bdev per NVMe controller. 6036 */ 6037 j = 0; 6038 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 6039 while (nvme_ns != NULL) { 6040 nvme_bdev = nvme_ns->bdev; 6041 if (j < ctx->max_bdevs) { 6042 ctx->names[j] = nvme_bdev->disk.name; 6043 j++; 6044 } else { 6045 SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n", 6046 ctx->max_bdevs); 6047 ctx->reported_bdevs = 0; 6048 populate_namespaces_cb(ctx, -ERANGE); 6049 return; 6050 } 6051 6052 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 6053 } 6054 6055 ctx->reported_bdevs = j; 6056 populate_namespaces_cb(ctx, 0); 6057 } 6058 6059 static int 6060 bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 6061 struct spdk_nvme_ctrlr *new_ctrlr, 6062 struct spdk_nvme_transport_id *trid) 6063 { 6064 struct nvme_path_id *tmp_trid; 6065 6066 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 6067 SPDK_ERRLOG("PCIe failover is not supported.\n"); 6068 return -ENOTSUP; 6069 } 6070 6071 /* Currently we only support failover to the same transport type. */ 6072 if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) { 6073 SPDK_WARNLOG("Failover from trtype: %s to a different trtype: %s is not supported currently\n", 6074 spdk_nvme_transport_id_trtype_str(nvme_ctrlr->active_path_id->trid.trtype), 6075 spdk_nvme_transport_id_trtype_str(trid->trtype)); 6076 return -EINVAL; 6077 } 6078 6079 6080 /* Currently we only support failover to the same NQN. */ 6081 if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) { 6082 SPDK_WARNLOG("Failover from subnqn: %s to a different subnqn: %s is not supported currently\n", 6083 nvme_ctrlr->active_path_id->trid.subnqn, trid->subnqn); 6084 return -EINVAL; 6085 } 6086 6087 /* Skip all the other checks if we've already registered this path. */ 6088 TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) { 6089 if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) { 6090 SPDK_WARNLOG("This path (traddr: %s subnqn: %s) is already registered\n", trid->traddr, 6091 trid->subnqn); 6092 return -EALREADY; 6093 } 6094 } 6095 6096 return 0; 6097 } 6098 6099 static int 6100 bdev_nvme_check_secondary_namespace(struct nvme_ctrlr *nvme_ctrlr, 6101 struct spdk_nvme_ctrlr *new_ctrlr) 6102 { 6103 struct nvme_ns *nvme_ns; 6104 struct spdk_nvme_ns *new_ns; 6105 6106 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 6107 while (nvme_ns != NULL) { 6108 new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id); 6109 assert(new_ns != NULL); 6110 6111 if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) { 6112 return -EINVAL; 6113 } 6114 6115 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 6116 } 6117 6118 return 0; 6119 } 6120 6121 static int 6122 _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 6123 struct spdk_nvme_transport_id *trid) 6124 { 6125 struct nvme_path_id *active_id, *new_trid, *tmp_trid; 6126 6127 new_trid = calloc(1, sizeof(*new_trid)); 6128 if (new_trid == NULL) { 6129 return -ENOMEM; 6130 } 6131 new_trid->trid = *trid; 6132 6133 active_id = nvme_ctrlr->active_path_id; 6134 assert(active_id != NULL); 6135 assert(active_id == TAILQ_FIRST(&nvme_ctrlr->trids)); 6136 6137 /* Skip the active trid not to replace it until it is failed. */ 6138 tmp_trid = TAILQ_NEXT(active_id, link); 6139 if (tmp_trid == NULL) { 6140 goto add_tail; 6141 } 6142 6143 /* It means the trid is faled if its last failed time is non-zero. 6144 * Insert the new alternate trid before any failed trid. 6145 */ 6146 TAILQ_FOREACH_FROM(tmp_trid, &nvme_ctrlr->trids, link) { 6147 if (tmp_trid->last_failed_tsc != 0) { 6148 TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link); 6149 return 0; 6150 } 6151 } 6152 6153 add_tail: 6154 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link); 6155 return 0; 6156 } 6157 6158 /* This is the case that a secondary path is added to an existing 6159 * nvme_ctrlr for failover. After checking if it can access the same 6160 * namespaces as the primary path, it is disconnected until failover occurs. 6161 */ 6162 static int 6163 bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 6164 struct spdk_nvme_ctrlr *new_ctrlr, 6165 struct spdk_nvme_transport_id *trid) 6166 { 6167 int rc; 6168 6169 assert(nvme_ctrlr != NULL); 6170 6171 pthread_mutex_lock(&nvme_ctrlr->mutex); 6172 6173 rc = bdev_nvme_check_secondary_trid(nvme_ctrlr, new_ctrlr, trid); 6174 if (rc != 0) { 6175 goto exit; 6176 } 6177 6178 rc = bdev_nvme_check_secondary_namespace(nvme_ctrlr, new_ctrlr); 6179 if (rc != 0) { 6180 goto exit; 6181 } 6182 6183 rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid); 6184 6185 exit: 6186 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6187 6188 spdk_nvme_detach(new_ctrlr); 6189 6190 return rc; 6191 } 6192 6193 static void 6194 connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 6195 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 6196 { 6197 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 6198 struct nvme_async_probe_ctx *ctx; 6199 int rc; 6200 6201 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 6202 ctx->ctrlr_attached = true; 6203 6204 rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx); 6205 if (rc != 0) { 6206 ctx->reported_bdevs = 0; 6207 populate_namespaces_cb(ctx, rc); 6208 } 6209 } 6210 6211 static void 6212 connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 6213 struct spdk_nvme_ctrlr *ctrlr, 6214 const struct spdk_nvme_ctrlr_opts *opts) 6215 { 6216 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 6217 struct nvme_ctrlr *nvme_ctrlr; 6218 struct nvme_async_probe_ctx *ctx; 6219 int rc; 6220 6221 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 6222 ctx->ctrlr_attached = true; 6223 6224 nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name); 6225 if (nvme_ctrlr) { 6226 rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid); 6227 } else { 6228 rc = -ENODEV; 6229 } 6230 6231 ctx->reported_bdevs = 0; 6232 populate_namespaces_cb(ctx, rc); 6233 } 6234 6235 static int 6236 bdev_nvme_async_poll(void *arg) 6237 { 6238 struct nvme_async_probe_ctx *ctx = arg; 6239 int rc; 6240 6241 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 6242 if (spdk_unlikely(rc != -EAGAIN)) { 6243 ctx->probe_done = true; 6244 spdk_poller_unregister(&ctx->poller); 6245 if (!ctx->ctrlr_attached) { 6246 /* The probe is done, but no controller was attached. 6247 * That means we had a failure, so report -EIO back to 6248 * the caller (usually the RPC). populate_namespaces_cb() 6249 * will take care of freeing the nvme_async_probe_ctx. 6250 */ 6251 ctx->reported_bdevs = 0; 6252 populate_namespaces_cb(ctx, -EIO); 6253 } else if (ctx->namespaces_populated) { 6254 /* The namespaces for the attached controller were all 6255 * populated and the response was already sent to the 6256 * caller (usually the RPC). So free the context here. 6257 */ 6258 free_nvme_async_probe_ctx(ctx); 6259 } 6260 } 6261 6262 return SPDK_POLLER_BUSY; 6263 } 6264 6265 static bool 6266 bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 6267 uint32_t reconnect_delay_sec, 6268 uint32_t fast_io_fail_timeout_sec) 6269 { 6270 if (ctrlr_loss_timeout_sec < -1) { 6271 SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n"); 6272 return false; 6273 } else if (ctrlr_loss_timeout_sec == -1) { 6274 if (reconnect_delay_sec == 0) { 6275 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 6276 return false; 6277 } else if (fast_io_fail_timeout_sec != 0 && 6278 fast_io_fail_timeout_sec < reconnect_delay_sec) { 6279 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n"); 6280 return false; 6281 } 6282 } else if (ctrlr_loss_timeout_sec != 0) { 6283 if (reconnect_delay_sec == 0) { 6284 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 6285 return false; 6286 } else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) { 6287 SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n"); 6288 return false; 6289 } else if (fast_io_fail_timeout_sec != 0) { 6290 if (fast_io_fail_timeout_sec < reconnect_delay_sec) { 6291 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n"); 6292 return false; 6293 } else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) { 6294 SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n"); 6295 return false; 6296 } 6297 } 6298 } else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) { 6299 SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n"); 6300 return false; 6301 } 6302 6303 return true; 6304 } 6305 6306 static int 6307 bdev_nvme_load_psk(const char *fname, char *buf, size_t bufsz) 6308 { 6309 FILE *psk_file; 6310 struct stat statbuf; 6311 int rc; 6312 #define TCP_PSK_INVALID_PERMISSIONS 0177 6313 6314 if (stat(fname, &statbuf) != 0) { 6315 SPDK_ERRLOG("Could not read permissions for PSK file\n"); 6316 return -EACCES; 6317 } 6318 6319 if ((statbuf.st_mode & TCP_PSK_INVALID_PERMISSIONS) != 0) { 6320 SPDK_ERRLOG("Incorrect permissions for PSK file\n"); 6321 return -EPERM; 6322 } 6323 if ((size_t)statbuf.st_size >= bufsz) { 6324 SPDK_ERRLOG("Invalid PSK: too long\n"); 6325 return -EINVAL; 6326 } 6327 psk_file = fopen(fname, "r"); 6328 if (psk_file == NULL) { 6329 SPDK_ERRLOG("Could not open PSK file\n"); 6330 return -EINVAL; 6331 } 6332 6333 memset(buf, 0, bufsz); 6334 rc = fread(buf, 1, statbuf.st_size, psk_file); 6335 if (rc != statbuf.st_size) { 6336 SPDK_ERRLOG("Failed to read PSK\n"); 6337 fclose(psk_file); 6338 return -EINVAL; 6339 } 6340 6341 fclose(psk_file); 6342 return 0; 6343 } 6344 6345 int 6346 spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid, 6347 const char *base_name, 6348 const char **names, 6349 uint32_t count, 6350 spdk_bdev_nvme_create_cb cb_fn, 6351 void *cb_ctx, 6352 struct spdk_nvme_ctrlr_opts *drv_opts, 6353 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, 6354 bool multipath) 6355 { 6356 struct nvme_probe_skip_entry *entry, *tmp; 6357 struct nvme_async_probe_ctx *ctx; 6358 spdk_nvme_attach_cb attach_cb; 6359 int rc, len; 6360 6361 /* TODO expand this check to include both the host and target TRIDs. 6362 * Only if both are the same should we fail. 6363 */ 6364 if (nvme_ctrlr_get(trid, drv_opts->hostnqn) != NULL) { 6365 SPDK_ERRLOG("A controller with the provided trid (traddr: %s, hostnqn: %s) " 6366 "already exists.\n", trid->traddr, drv_opts->hostnqn); 6367 return -EEXIST; 6368 } 6369 6370 len = strnlen(base_name, SPDK_CONTROLLER_NAME_MAX); 6371 6372 if (len == 0 || len == SPDK_CONTROLLER_NAME_MAX) { 6373 SPDK_ERRLOG("controller name must be between 1 and %d characters\n", SPDK_CONTROLLER_NAME_MAX - 1); 6374 return -EINVAL; 6375 } 6376 6377 if (bdev_opts != NULL && 6378 !bdev_nvme_check_io_error_resiliency_params(bdev_opts->ctrlr_loss_timeout_sec, 6379 bdev_opts->reconnect_delay_sec, 6380 bdev_opts->fast_io_fail_timeout_sec)) { 6381 return -EINVAL; 6382 } 6383 6384 ctx = calloc(1, sizeof(*ctx)); 6385 if (!ctx) { 6386 return -ENOMEM; 6387 } 6388 ctx->base_name = base_name; 6389 ctx->names = names; 6390 ctx->max_bdevs = count; 6391 ctx->cb_fn = cb_fn; 6392 ctx->cb_ctx = cb_ctx; 6393 ctx->trid = *trid; 6394 6395 if (bdev_opts) { 6396 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 6397 } else { 6398 spdk_bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts); 6399 } 6400 6401 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 6402 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) { 6403 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 6404 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 6405 free(entry); 6406 break; 6407 } 6408 } 6409 } 6410 6411 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 6412 ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count; 6413 ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout; 6414 ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms; 6415 ctx->drv_opts.disable_read_ana_log_page = true; 6416 ctx->drv_opts.transport_tos = g_opts.transport_tos; 6417 6418 if (ctx->bdev_opts.psk[0] != '\0') { 6419 /* Try to use the keyring first */ 6420 ctx->drv_opts.tls_psk = spdk_keyring_get_key(ctx->bdev_opts.psk); 6421 if (ctx->drv_opts.tls_psk == NULL) { 6422 rc = bdev_nvme_load_psk(ctx->bdev_opts.psk, 6423 ctx->drv_opts.psk, sizeof(ctx->drv_opts.psk)); 6424 if (rc != 0) { 6425 SPDK_ERRLOG("Could not load PSK from %s\n", ctx->bdev_opts.psk); 6426 free_nvme_async_probe_ctx(ctx); 6427 return rc; 6428 } 6429 } 6430 } 6431 6432 if (ctx->bdev_opts.dhchap_key != NULL) { 6433 ctx->drv_opts.dhchap_key = spdk_keyring_get_key(ctx->bdev_opts.dhchap_key); 6434 if (ctx->drv_opts.dhchap_key == NULL) { 6435 SPDK_ERRLOG("Could not load DH-HMAC-CHAP key: %s\n", 6436 ctx->bdev_opts.dhchap_key); 6437 free_nvme_async_probe_ctx(ctx); 6438 return -ENOKEY; 6439 } 6440 6441 ctx->drv_opts.dhchap_digests = g_opts.dhchap_digests; 6442 ctx->drv_opts.dhchap_dhgroups = g_opts.dhchap_dhgroups; 6443 } 6444 if (ctx->bdev_opts.dhchap_ctrlr_key != NULL) { 6445 ctx->drv_opts.dhchap_ctrlr_key = 6446 spdk_keyring_get_key(ctx->bdev_opts.dhchap_ctrlr_key); 6447 if (ctx->drv_opts.dhchap_ctrlr_key == NULL) { 6448 SPDK_ERRLOG("Could not load DH-HMAC-CHAP controller key: %s\n", 6449 ctx->bdev_opts.dhchap_ctrlr_key); 6450 free_nvme_async_probe_ctx(ctx); 6451 return -ENOKEY; 6452 } 6453 } 6454 6455 if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || multipath) { 6456 attach_cb = connect_attach_cb; 6457 } else { 6458 attach_cb = connect_set_failover_cb; 6459 } 6460 6461 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb); 6462 if (ctx->probe_ctx == NULL) { 6463 SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr); 6464 free_nvme_async_probe_ctx(ctx); 6465 return -ENODEV; 6466 } 6467 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000); 6468 6469 return 0; 6470 } 6471 6472 struct bdev_nvme_delete_ctx { 6473 char *name; 6474 struct nvme_path_id path_id; 6475 bdev_nvme_delete_done_fn delete_done; 6476 void *delete_done_ctx; 6477 uint64_t timeout_ticks; 6478 struct spdk_poller *poller; 6479 }; 6480 6481 static void 6482 free_bdev_nvme_delete_ctx(struct bdev_nvme_delete_ctx *ctx) 6483 { 6484 if (ctx != NULL) { 6485 free(ctx->name); 6486 free(ctx); 6487 } 6488 } 6489 6490 static bool 6491 nvme_path_id_compare(struct nvme_path_id *p, const struct nvme_path_id *path_id) 6492 { 6493 if (path_id->trid.trtype != 0) { 6494 if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) { 6495 if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) { 6496 return false; 6497 } 6498 } else { 6499 if (path_id->trid.trtype != p->trid.trtype) { 6500 return false; 6501 } 6502 } 6503 } 6504 6505 if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) { 6506 if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) { 6507 return false; 6508 } 6509 } 6510 6511 if (path_id->trid.adrfam != 0) { 6512 if (path_id->trid.adrfam != p->trid.adrfam) { 6513 return false; 6514 } 6515 } 6516 6517 if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) { 6518 if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) { 6519 return false; 6520 } 6521 } 6522 6523 if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) { 6524 if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) { 6525 return false; 6526 } 6527 } 6528 6529 if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) { 6530 if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) { 6531 return false; 6532 } 6533 } 6534 6535 if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) { 6536 if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) { 6537 return false; 6538 } 6539 } 6540 6541 return true; 6542 } 6543 6544 static bool 6545 nvme_path_id_exists(const char *name, const struct nvme_path_id *path_id) 6546 { 6547 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6548 struct nvme_ctrlr *ctrlr; 6549 struct nvme_path_id *p; 6550 6551 pthread_mutex_lock(&g_bdev_nvme_mutex); 6552 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 6553 if (!nbdev_ctrlr) { 6554 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6555 return false; 6556 } 6557 6558 TAILQ_FOREACH(ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 6559 pthread_mutex_lock(&ctrlr->mutex); 6560 TAILQ_FOREACH(p, &ctrlr->trids, link) { 6561 if (nvme_path_id_compare(p, path_id)) { 6562 pthread_mutex_unlock(&ctrlr->mutex); 6563 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6564 return true; 6565 } 6566 } 6567 pthread_mutex_unlock(&ctrlr->mutex); 6568 } 6569 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6570 6571 return false; 6572 } 6573 6574 static int 6575 bdev_nvme_delete_complete_poll(void *arg) 6576 { 6577 struct bdev_nvme_delete_ctx *ctx = arg; 6578 int rc = 0; 6579 6580 if (nvme_path_id_exists(ctx->name, &ctx->path_id)) { 6581 if (ctx->timeout_ticks > spdk_get_ticks()) { 6582 return SPDK_POLLER_BUSY; 6583 } 6584 6585 SPDK_ERRLOG("NVMe path '%s' still exists after delete\n", ctx->name); 6586 rc = -ETIMEDOUT; 6587 } 6588 6589 spdk_poller_unregister(&ctx->poller); 6590 6591 ctx->delete_done(ctx->delete_done_ctx, rc); 6592 free_bdev_nvme_delete_ctx(ctx); 6593 6594 return SPDK_POLLER_BUSY; 6595 } 6596 6597 static int 6598 _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id) 6599 { 6600 struct nvme_path_id *p, *t; 6601 spdk_msg_fn msg_fn; 6602 int rc = -ENXIO; 6603 6604 pthread_mutex_lock(&nvme_ctrlr->mutex); 6605 6606 TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) { 6607 if (p == TAILQ_FIRST(&nvme_ctrlr->trids)) { 6608 break; 6609 } 6610 6611 if (!nvme_path_id_compare(p, path_id)) { 6612 continue; 6613 } 6614 6615 /* We are not using the specified path. */ 6616 TAILQ_REMOVE(&nvme_ctrlr->trids, p, link); 6617 free(p); 6618 rc = 0; 6619 } 6620 6621 if (p == NULL || !nvme_path_id_compare(p, path_id)) { 6622 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6623 return rc; 6624 } 6625 6626 /* If we made it here, then this path is a match! Now we need to remove it. */ 6627 6628 /* This is the active path in use right now. The active path is always the first in the list. */ 6629 assert(p == nvme_ctrlr->active_path_id); 6630 6631 if (!TAILQ_NEXT(p, link)) { 6632 /* The current path is the only path. */ 6633 msg_fn = _nvme_ctrlr_destruct; 6634 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, false); 6635 } else { 6636 /* There is an alternative path. */ 6637 msg_fn = _bdev_nvme_reset_ctrlr; 6638 rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, true); 6639 } 6640 6641 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6642 6643 if (rc == 0) { 6644 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 6645 } else if (rc == -EALREADY) { 6646 rc = 0; 6647 } 6648 6649 return rc; 6650 } 6651 6652 int 6653 bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id, 6654 bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx) 6655 { 6656 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6657 struct nvme_ctrlr *nvme_ctrlr, *tmp_nvme_ctrlr; 6658 struct bdev_nvme_delete_ctx *ctx = NULL; 6659 int rc = -ENXIO, _rc; 6660 6661 if (name == NULL || path_id == NULL) { 6662 rc = -EINVAL; 6663 goto exit; 6664 } 6665 6666 pthread_mutex_lock(&g_bdev_nvme_mutex); 6667 6668 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 6669 if (nbdev_ctrlr == NULL) { 6670 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6671 6672 SPDK_ERRLOG("Failed to find NVMe bdev controller\n"); 6673 rc = -ENODEV; 6674 goto exit; 6675 } 6676 6677 TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) { 6678 _rc = _bdev_nvme_delete(nvme_ctrlr, path_id); 6679 if (_rc < 0 && _rc != -ENXIO) { 6680 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6681 rc = _rc; 6682 goto exit; 6683 } else if (_rc == 0) { 6684 /* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr 6685 * was deleted successfully. To remember the successful deletion, 6686 * overwrite rc only if _rc is zero. 6687 */ 6688 rc = 0; 6689 } 6690 } 6691 6692 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6693 6694 if (rc != 0 || delete_done == NULL) { 6695 goto exit; 6696 } 6697 6698 ctx = calloc(1, sizeof(*ctx)); 6699 if (ctx == NULL) { 6700 SPDK_ERRLOG("Failed to allocate context for bdev_nvme_delete\n"); 6701 rc = -ENOMEM; 6702 goto exit; 6703 } 6704 6705 ctx->name = strdup(name); 6706 if (ctx->name == NULL) { 6707 SPDK_ERRLOG("Failed to copy controller name for deletion\n"); 6708 rc = -ENOMEM; 6709 goto exit; 6710 } 6711 6712 ctx->delete_done = delete_done; 6713 ctx->delete_done_ctx = delete_done_ctx; 6714 ctx->path_id = *path_id; 6715 ctx->timeout_ticks = spdk_get_ticks() + 10 * spdk_get_ticks_hz(); 6716 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_delete_complete_poll, ctx, 1000); 6717 if (ctx->poller == NULL) { 6718 SPDK_ERRLOG("Failed to register bdev_nvme_delete poller\n"); 6719 rc = -ENOMEM; 6720 goto exit; 6721 } 6722 6723 exit: 6724 if (rc != 0) { 6725 free_bdev_nvme_delete_ctx(ctx); 6726 } 6727 6728 return rc; 6729 } 6730 6731 #define DISCOVERY_INFOLOG(ctx, format, ...) \ 6732 SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 6733 6734 #define DISCOVERY_ERRLOG(ctx, format, ...) \ 6735 SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 6736 6737 struct discovery_entry_ctx { 6738 char name[128]; 6739 struct spdk_nvme_transport_id trid; 6740 struct spdk_nvme_ctrlr_opts drv_opts; 6741 struct spdk_nvmf_discovery_log_page_entry entry; 6742 TAILQ_ENTRY(discovery_entry_ctx) tailq; 6743 struct discovery_ctx *ctx; 6744 }; 6745 6746 struct discovery_ctx { 6747 char *name; 6748 spdk_bdev_nvme_start_discovery_fn start_cb_fn; 6749 spdk_bdev_nvme_stop_discovery_fn stop_cb_fn; 6750 void *cb_ctx; 6751 struct spdk_nvme_probe_ctx *probe_ctx; 6752 struct spdk_nvme_detach_ctx *detach_ctx; 6753 struct spdk_nvme_ctrlr *ctrlr; 6754 struct spdk_nvme_transport_id trid; 6755 struct discovery_entry_ctx *entry_ctx_in_use; 6756 struct spdk_poller *poller; 6757 struct spdk_nvme_ctrlr_opts drv_opts; 6758 struct spdk_bdev_nvme_ctrlr_opts bdev_opts; 6759 struct spdk_nvmf_discovery_log_page *log_page; 6760 TAILQ_ENTRY(discovery_ctx) tailq; 6761 TAILQ_HEAD(, discovery_entry_ctx) nvm_entry_ctxs; 6762 TAILQ_HEAD(, discovery_entry_ctx) discovery_entry_ctxs; 6763 int rc; 6764 bool wait_for_attach; 6765 uint64_t timeout_ticks; 6766 /* Denotes that the discovery service is being started. We're waiting 6767 * for the initial connection to the discovery controller to be 6768 * established and attach discovered NVM ctrlrs. 6769 */ 6770 bool initializing; 6771 /* Denotes if a discovery is currently in progress for this context. 6772 * That includes connecting to newly discovered subsystems. Used to 6773 * ensure we do not start a new discovery until an existing one is 6774 * complete. 6775 */ 6776 bool in_progress; 6777 6778 /* Denotes if another discovery is needed after the one in progress 6779 * completes. Set when we receive an AER completion while a discovery 6780 * is already in progress. 6781 */ 6782 bool pending; 6783 6784 /* Signal to the discovery context poller that it should stop the 6785 * discovery service, including detaching from the current discovery 6786 * controller. 6787 */ 6788 bool stop; 6789 6790 struct spdk_thread *calling_thread; 6791 uint32_t index; 6792 uint32_t attach_in_progress; 6793 char *hostnqn; 6794 6795 /* Denotes if the discovery service was started by the mdns discovery. 6796 */ 6797 bool from_mdns_discovery_service; 6798 }; 6799 6800 TAILQ_HEAD(discovery_ctxs, discovery_ctx); 6801 static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs); 6802 6803 static void get_discovery_log_page(struct discovery_ctx *ctx); 6804 6805 static void 6806 free_discovery_ctx(struct discovery_ctx *ctx) 6807 { 6808 free(ctx->log_page); 6809 free(ctx->hostnqn); 6810 free(ctx->name); 6811 free(ctx); 6812 } 6813 6814 static void 6815 discovery_complete(struct discovery_ctx *ctx) 6816 { 6817 ctx->initializing = false; 6818 ctx->in_progress = false; 6819 if (ctx->pending) { 6820 ctx->pending = false; 6821 get_discovery_log_page(ctx); 6822 } 6823 } 6824 6825 static void 6826 build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid, 6827 struct spdk_nvmf_discovery_log_page_entry *entry) 6828 { 6829 char *space; 6830 6831 trid->trtype = entry->trtype; 6832 trid->adrfam = entry->adrfam; 6833 memcpy(trid->traddr, entry->traddr, sizeof(entry->traddr)); 6834 memcpy(trid->trsvcid, entry->trsvcid, sizeof(entry->trsvcid)); 6835 /* Because the source buffer (entry->subnqn) is longer than trid->subnqn, and 6836 * before call to this function trid->subnqn is zeroed out, we need 6837 * to copy sizeof(trid->subnqn) minus one byte to make sure the last character 6838 * remains 0. Then we can shorten the string (replace ' ' with 0) if required 6839 */ 6840 memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn) - 1); 6841 6842 /* We want the traddr, trsvcid and subnqn fields to be NULL-terminated. 6843 * But the log page entries typically pad them with spaces, not zeroes. 6844 * So add a NULL terminator to each of these fields at the appropriate 6845 * location. 6846 */ 6847 space = strchr(trid->traddr, ' '); 6848 if (space) { 6849 *space = 0; 6850 } 6851 space = strchr(trid->trsvcid, ' '); 6852 if (space) { 6853 *space = 0; 6854 } 6855 space = strchr(trid->subnqn, ' '); 6856 if (space) { 6857 *space = 0; 6858 } 6859 } 6860 6861 static void 6862 _stop_discovery(void *_ctx) 6863 { 6864 struct discovery_ctx *ctx = _ctx; 6865 6866 if (ctx->attach_in_progress > 0) { 6867 spdk_thread_send_msg(spdk_get_thread(), _stop_discovery, ctx); 6868 return; 6869 } 6870 6871 ctx->stop = true; 6872 6873 while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) { 6874 struct discovery_entry_ctx *entry_ctx; 6875 struct nvme_path_id path = {}; 6876 6877 entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs); 6878 path.trid = entry_ctx->trid; 6879 bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL); 6880 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 6881 free(entry_ctx); 6882 } 6883 6884 while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) { 6885 struct discovery_entry_ctx *entry_ctx; 6886 6887 entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 6888 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 6889 free(entry_ctx); 6890 } 6891 6892 free(ctx->entry_ctx_in_use); 6893 ctx->entry_ctx_in_use = NULL; 6894 } 6895 6896 static void 6897 stop_discovery(struct discovery_ctx *ctx, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 6898 { 6899 ctx->stop_cb_fn = cb_fn; 6900 ctx->cb_ctx = cb_ctx; 6901 6902 if (ctx->attach_in_progress > 0) { 6903 DISCOVERY_INFOLOG(ctx, "stopping discovery with attach_in_progress: %"PRIu32"\n", 6904 ctx->attach_in_progress); 6905 } 6906 6907 _stop_discovery(ctx); 6908 } 6909 6910 static void 6911 remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr) 6912 { 6913 struct discovery_ctx *d_ctx; 6914 struct nvme_path_id *path_id; 6915 struct spdk_nvme_transport_id trid = {}; 6916 struct discovery_entry_ctx *entry_ctx, *tmp; 6917 6918 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 6919 6920 TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) { 6921 TAILQ_FOREACH_SAFE(entry_ctx, &d_ctx->nvm_entry_ctxs, tailq, tmp) { 6922 build_trid_from_log_page_entry(&trid, &entry_ctx->entry); 6923 if (spdk_nvme_transport_id_compare(&trid, &path_id->trid) != 0) { 6924 continue; 6925 } 6926 6927 TAILQ_REMOVE(&d_ctx->nvm_entry_ctxs, entry_ctx, tailq); 6928 free(entry_ctx); 6929 DISCOVERY_INFOLOG(d_ctx, "Remove discovery entry: %s:%s:%s\n", 6930 trid.subnqn, trid.traddr, trid.trsvcid); 6931 6932 /* Fail discovery ctrlr to force reattach attempt */ 6933 spdk_nvme_ctrlr_fail(d_ctx->ctrlr); 6934 } 6935 } 6936 } 6937 6938 static void 6939 discovery_remove_controllers(struct discovery_ctx *ctx) 6940 { 6941 struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page; 6942 struct discovery_entry_ctx *entry_ctx, *tmp; 6943 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 6944 struct spdk_nvme_transport_id old_trid = {}; 6945 uint64_t numrec, i; 6946 bool found; 6947 6948 numrec = from_le64(&log_page->numrec); 6949 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) { 6950 found = false; 6951 old_entry = &entry_ctx->entry; 6952 build_trid_from_log_page_entry(&old_trid, old_entry); 6953 for (i = 0; i < numrec; i++) { 6954 new_entry = &log_page->entries[i]; 6955 if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) { 6956 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n", 6957 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 6958 found = true; 6959 break; 6960 } 6961 } 6962 if (!found) { 6963 struct nvme_path_id path = {}; 6964 6965 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n", 6966 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 6967 6968 path.trid = entry_ctx->trid; 6969 bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL); 6970 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 6971 free(entry_ctx); 6972 } 6973 } 6974 free(log_page); 6975 ctx->log_page = NULL; 6976 discovery_complete(ctx); 6977 } 6978 6979 static void 6980 complete_discovery_start(struct discovery_ctx *ctx, int status) 6981 { 6982 ctx->timeout_ticks = 0; 6983 ctx->rc = status; 6984 if (ctx->start_cb_fn) { 6985 ctx->start_cb_fn(ctx->cb_ctx, status); 6986 ctx->start_cb_fn = NULL; 6987 ctx->cb_ctx = NULL; 6988 } 6989 } 6990 6991 static void 6992 discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc) 6993 { 6994 struct discovery_entry_ctx *entry_ctx = cb_ctx; 6995 struct discovery_ctx *ctx = entry_ctx->ctx; 6996 6997 DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name); 6998 ctx->attach_in_progress--; 6999 if (ctx->attach_in_progress == 0) { 7000 complete_discovery_start(ctx, ctx->rc); 7001 if (ctx->initializing && ctx->rc != 0) { 7002 DISCOVERY_ERRLOG(ctx, "stopping discovery due to errors: %d\n", ctx->rc); 7003 stop_discovery(ctx, NULL, ctx->cb_ctx); 7004 } else { 7005 discovery_remove_controllers(ctx); 7006 } 7007 } 7008 } 7009 7010 static struct discovery_entry_ctx * 7011 create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid) 7012 { 7013 struct discovery_entry_ctx *new_ctx; 7014 7015 new_ctx = calloc(1, sizeof(*new_ctx)); 7016 if (new_ctx == NULL) { 7017 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 7018 return NULL; 7019 } 7020 7021 new_ctx->ctx = ctx; 7022 memcpy(&new_ctx->trid, trid, sizeof(*trid)); 7023 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 7024 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 7025 return new_ctx; 7026 } 7027 7028 static void 7029 discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl, 7030 struct spdk_nvmf_discovery_log_page *log_page) 7031 { 7032 struct discovery_ctx *ctx = cb_arg; 7033 struct discovery_entry_ctx *entry_ctx, *tmp; 7034 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 7035 uint64_t numrec, i; 7036 bool found; 7037 7038 if (rc || spdk_nvme_cpl_is_error(cpl)) { 7039 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 7040 return; 7041 } 7042 7043 ctx->log_page = log_page; 7044 assert(ctx->attach_in_progress == 0); 7045 numrec = from_le64(&log_page->numrec); 7046 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) { 7047 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 7048 free(entry_ctx); 7049 } 7050 for (i = 0; i < numrec; i++) { 7051 found = false; 7052 new_entry = &log_page->entries[i]; 7053 if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY_CURRENT || 7054 new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 7055 struct discovery_entry_ctx *new_ctx; 7056 struct spdk_nvme_transport_id trid = {}; 7057 7058 build_trid_from_log_page_entry(&trid, new_entry); 7059 new_ctx = create_discovery_entry_ctx(ctx, &trid); 7060 if (new_ctx == NULL) { 7061 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 7062 break; 7063 } 7064 7065 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq); 7066 continue; 7067 } 7068 TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) { 7069 old_entry = &entry_ctx->entry; 7070 if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) { 7071 found = true; 7072 break; 7073 } 7074 } 7075 if (!found) { 7076 struct discovery_entry_ctx *subnqn_ctx = NULL, *new_ctx; 7077 struct discovery_ctx *d_ctx; 7078 7079 TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) { 7080 TAILQ_FOREACH(subnqn_ctx, &d_ctx->nvm_entry_ctxs, tailq) { 7081 if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn, 7082 sizeof(new_entry->subnqn))) { 7083 break; 7084 } 7085 } 7086 if (subnqn_ctx) { 7087 break; 7088 } 7089 } 7090 7091 new_ctx = calloc(1, sizeof(*new_ctx)); 7092 if (new_ctx == NULL) { 7093 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 7094 break; 7095 } 7096 7097 new_ctx->ctx = ctx; 7098 memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry)); 7099 build_trid_from_log_page_entry(&new_ctx->trid, new_entry); 7100 if (subnqn_ctx) { 7101 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name); 7102 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n", 7103 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 7104 new_ctx->name); 7105 } else { 7106 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++); 7107 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n", 7108 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 7109 new_ctx->name); 7110 } 7111 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 7112 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 7113 rc = spdk_bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0, 7114 discovery_attach_controller_done, new_ctx, 7115 &new_ctx->drv_opts, &ctx->bdev_opts, true); 7116 if (rc == 0) { 7117 TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq); 7118 ctx->attach_in_progress++; 7119 } else { 7120 DISCOVERY_ERRLOG(ctx, "spdk_bdev_nvme_create failed (%s)\n", spdk_strerror(-rc)); 7121 } 7122 } 7123 } 7124 7125 if (ctx->attach_in_progress == 0) { 7126 discovery_remove_controllers(ctx); 7127 } 7128 } 7129 7130 static void 7131 get_discovery_log_page(struct discovery_ctx *ctx) 7132 { 7133 int rc; 7134 7135 assert(ctx->in_progress == false); 7136 ctx->in_progress = true; 7137 rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx); 7138 if (rc != 0) { 7139 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 7140 } 7141 DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n"); 7142 } 7143 7144 static void 7145 discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 7146 { 7147 struct discovery_ctx *ctx = arg; 7148 uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16; 7149 7150 if (spdk_nvme_cpl_is_error(cpl)) { 7151 DISCOVERY_ERRLOG(ctx, "aer failed\n"); 7152 return; 7153 } 7154 7155 if (log_page_id != SPDK_NVME_LOG_DISCOVERY) { 7156 DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id); 7157 return; 7158 } 7159 7160 DISCOVERY_INFOLOG(ctx, "got aer\n"); 7161 if (ctx->in_progress) { 7162 ctx->pending = true; 7163 return; 7164 } 7165 7166 get_discovery_log_page(ctx); 7167 } 7168 7169 static void 7170 discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 7171 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 7172 { 7173 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 7174 struct discovery_ctx *ctx; 7175 7176 ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts); 7177 7178 DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n"); 7179 ctx->probe_ctx = NULL; 7180 ctx->ctrlr = ctrlr; 7181 7182 if (ctx->rc != 0) { 7183 DISCOVERY_ERRLOG(ctx, "encountered error while attaching discovery ctrlr: %d\n", 7184 ctx->rc); 7185 return; 7186 } 7187 7188 spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx); 7189 } 7190 7191 static int 7192 discovery_poller(void *arg) 7193 { 7194 struct discovery_ctx *ctx = arg; 7195 struct spdk_nvme_transport_id *trid; 7196 int rc; 7197 7198 if (ctx->detach_ctx) { 7199 rc = spdk_nvme_detach_poll_async(ctx->detach_ctx); 7200 if (rc != -EAGAIN) { 7201 ctx->detach_ctx = NULL; 7202 ctx->ctrlr = NULL; 7203 } 7204 } else if (ctx->stop) { 7205 if (ctx->ctrlr != NULL) { 7206 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 7207 if (rc == 0) { 7208 return SPDK_POLLER_BUSY; 7209 } 7210 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 7211 } 7212 spdk_poller_unregister(&ctx->poller); 7213 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 7214 assert(ctx->start_cb_fn == NULL); 7215 if (ctx->stop_cb_fn != NULL) { 7216 ctx->stop_cb_fn(ctx->cb_ctx); 7217 } 7218 free_discovery_ctx(ctx); 7219 } else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) { 7220 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7221 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 7222 assert(ctx->initializing); 7223 spdk_poller_unregister(&ctx->poller); 7224 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 7225 complete_discovery_start(ctx, -ETIMEDOUT); 7226 stop_discovery(ctx, NULL, NULL); 7227 free_discovery_ctx(ctx); 7228 return SPDK_POLLER_BUSY; 7229 } 7230 7231 assert(ctx->entry_ctx_in_use == NULL); 7232 ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 7233 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7234 trid = &ctx->entry_ctx_in_use->trid; 7235 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb); 7236 if (ctx->probe_ctx) { 7237 spdk_poller_unregister(&ctx->poller); 7238 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000); 7239 } else { 7240 DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n"); 7241 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7242 ctx->entry_ctx_in_use = NULL; 7243 } 7244 } else if (ctx->probe_ctx) { 7245 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7246 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 7247 complete_discovery_start(ctx, -ETIMEDOUT); 7248 return SPDK_POLLER_BUSY; 7249 } 7250 7251 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 7252 if (rc != -EAGAIN) { 7253 if (ctx->rc != 0) { 7254 assert(ctx->initializing); 7255 stop_discovery(ctx, NULL, ctx->cb_ctx); 7256 } else { 7257 assert(rc == 0); 7258 DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n"); 7259 ctx->rc = rc; 7260 get_discovery_log_page(ctx); 7261 } 7262 } 7263 } else { 7264 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7265 DISCOVERY_ERRLOG(ctx, "timed out while attaching NVM ctrlrs\n"); 7266 complete_discovery_start(ctx, -ETIMEDOUT); 7267 /* We need to wait until all NVM ctrlrs are attached before we stop the 7268 * discovery service to make sure we don't detach a ctrlr that is still 7269 * being attached. 7270 */ 7271 if (ctx->attach_in_progress == 0) { 7272 stop_discovery(ctx, NULL, ctx->cb_ctx); 7273 return SPDK_POLLER_BUSY; 7274 } 7275 } 7276 7277 rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr); 7278 if (rc < 0) { 7279 spdk_poller_unregister(&ctx->poller); 7280 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 7281 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7282 ctx->entry_ctx_in_use = NULL; 7283 7284 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 7285 if (rc != 0) { 7286 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 7287 ctx->ctrlr = NULL; 7288 } 7289 } 7290 } 7291 7292 return SPDK_POLLER_BUSY; 7293 } 7294 7295 static void 7296 start_discovery_poller(void *arg) 7297 { 7298 struct discovery_ctx *ctx = arg; 7299 7300 TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq); 7301 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 7302 } 7303 7304 int 7305 bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid, 7306 const char *base_name, 7307 struct spdk_nvme_ctrlr_opts *drv_opts, 7308 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, 7309 uint64_t attach_timeout, 7310 bool from_mdns, 7311 spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx) 7312 { 7313 struct discovery_ctx *ctx; 7314 struct discovery_entry_ctx *discovery_entry_ctx; 7315 7316 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 7317 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7318 if (strcmp(ctx->name, base_name) == 0) { 7319 return -EEXIST; 7320 } 7321 7322 if (ctx->entry_ctx_in_use != NULL) { 7323 if (!spdk_nvme_transport_id_compare(trid, &ctx->entry_ctx_in_use->trid)) { 7324 return -EEXIST; 7325 } 7326 } 7327 7328 TAILQ_FOREACH(discovery_entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 7329 if (!spdk_nvme_transport_id_compare(trid, &discovery_entry_ctx->trid)) { 7330 return -EEXIST; 7331 } 7332 } 7333 } 7334 7335 ctx = calloc(1, sizeof(*ctx)); 7336 if (ctx == NULL) { 7337 return -ENOMEM; 7338 } 7339 7340 ctx->name = strdup(base_name); 7341 if (ctx->name == NULL) { 7342 free_discovery_ctx(ctx); 7343 return -ENOMEM; 7344 } 7345 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 7346 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 7347 ctx->from_mdns_discovery_service = from_mdns; 7348 ctx->bdev_opts.from_discovery_service = true; 7349 ctx->calling_thread = spdk_get_thread(); 7350 ctx->start_cb_fn = cb_fn; 7351 ctx->cb_ctx = cb_ctx; 7352 ctx->initializing = true; 7353 if (ctx->start_cb_fn) { 7354 /* We can use this when dumping json to denote if this RPC parameter 7355 * was specified or not. 7356 */ 7357 ctx->wait_for_attach = true; 7358 } 7359 if (attach_timeout != 0) { 7360 ctx->timeout_ticks = spdk_get_ticks() + attach_timeout * 7361 spdk_get_ticks_hz() / 1000ull; 7362 } 7363 TAILQ_INIT(&ctx->nvm_entry_ctxs); 7364 TAILQ_INIT(&ctx->discovery_entry_ctxs); 7365 memcpy(&ctx->trid, trid, sizeof(*trid)); 7366 /* Even if user did not specify hostnqn, we can still strdup("\0"); */ 7367 ctx->hostnqn = strdup(ctx->drv_opts.hostnqn); 7368 if (ctx->hostnqn == NULL) { 7369 free_discovery_ctx(ctx); 7370 return -ENOMEM; 7371 } 7372 discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid); 7373 if (discovery_entry_ctx == NULL) { 7374 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 7375 free_discovery_ctx(ctx); 7376 return -ENOMEM; 7377 } 7378 7379 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq); 7380 spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx); 7381 return 0; 7382 } 7383 7384 int 7385 bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 7386 { 7387 struct discovery_ctx *ctx; 7388 7389 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7390 if (strcmp(name, ctx->name) == 0) { 7391 if (ctx->stop) { 7392 return -EALREADY; 7393 } 7394 /* If we're still starting the discovery service and ->rc is non-zero, we're 7395 * going to stop it as soon as we can 7396 */ 7397 if (ctx->initializing && ctx->rc != 0) { 7398 return -EALREADY; 7399 } 7400 stop_discovery(ctx, cb_fn, cb_ctx); 7401 return 0; 7402 } 7403 } 7404 7405 return -ENOENT; 7406 } 7407 7408 static int 7409 bdev_nvme_library_init(void) 7410 { 7411 g_bdev_nvme_init_thread = spdk_get_thread(); 7412 7413 spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb, 7414 bdev_nvme_destroy_poll_group_cb, 7415 sizeof(struct nvme_poll_group), "nvme_poll_groups"); 7416 7417 return 0; 7418 } 7419 7420 static void 7421 bdev_nvme_fini_destruct_ctrlrs(void) 7422 { 7423 struct nvme_bdev_ctrlr *nbdev_ctrlr; 7424 struct nvme_ctrlr *nvme_ctrlr; 7425 7426 pthread_mutex_lock(&g_bdev_nvme_mutex); 7427 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 7428 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 7429 pthread_mutex_lock(&nvme_ctrlr->mutex); 7430 if (nvme_ctrlr->destruct) { 7431 /* This controller's destruction was already started 7432 * before the application started shutting down 7433 */ 7434 pthread_mutex_unlock(&nvme_ctrlr->mutex); 7435 continue; 7436 } 7437 nvme_ctrlr->destruct = true; 7438 pthread_mutex_unlock(&nvme_ctrlr->mutex); 7439 7440 spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct, 7441 nvme_ctrlr); 7442 } 7443 } 7444 7445 g_bdev_nvme_module_finish = true; 7446 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 7447 pthread_mutex_unlock(&g_bdev_nvme_mutex); 7448 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 7449 spdk_bdev_module_fini_done(); 7450 return; 7451 } 7452 7453 pthread_mutex_unlock(&g_bdev_nvme_mutex); 7454 } 7455 7456 static void 7457 check_discovery_fini(void *arg) 7458 { 7459 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 7460 bdev_nvme_fini_destruct_ctrlrs(); 7461 } 7462 } 7463 7464 static void 7465 bdev_nvme_library_fini(void) 7466 { 7467 struct nvme_probe_skip_entry *entry, *entry_tmp; 7468 struct discovery_ctx *ctx; 7469 7470 spdk_poller_unregister(&g_hotplug_poller); 7471 free(g_hotplug_probe_ctx); 7472 g_hotplug_probe_ctx = NULL; 7473 7474 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) { 7475 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 7476 free(entry); 7477 } 7478 7479 assert(spdk_get_thread() == g_bdev_nvme_init_thread); 7480 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 7481 bdev_nvme_fini_destruct_ctrlrs(); 7482 } else { 7483 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7484 stop_discovery(ctx, check_discovery_fini, NULL); 7485 } 7486 } 7487 } 7488 7489 static void 7490 bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio) 7491 { 7492 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7493 struct spdk_bdev *bdev = bdev_io->bdev; 7494 struct spdk_dif_ctx dif_ctx; 7495 struct spdk_dif_error err_blk = {}; 7496 int rc; 7497 struct spdk_dif_ctx_init_ext_opts dif_opts; 7498 7499 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 7500 dif_opts.dif_pi_format = bdev->dif_pi_format; 7501 rc = spdk_dif_ctx_init(&dif_ctx, 7502 bdev->blocklen, bdev->md_len, bdev->md_interleave, 7503 bdev->dif_is_head_of_md, bdev->dif_type, 7504 bdev_io->u.bdev.dif_check_flags, 7505 bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0, &dif_opts); 7506 if (rc != 0) { 7507 SPDK_ERRLOG("Initialization of DIF context failed\n"); 7508 return; 7509 } 7510 7511 if (bdev->md_interleave) { 7512 rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 7513 bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 7514 } else { 7515 struct iovec md_iov = { 7516 .iov_base = bdev_io->u.bdev.md_buf, 7517 .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len, 7518 }; 7519 7520 rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 7521 &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 7522 } 7523 7524 if (rc != 0) { 7525 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", 7526 err_blk.err_type, err_blk.err_offset); 7527 } else { 7528 SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n"); 7529 } 7530 } 7531 7532 static void 7533 bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7534 { 7535 struct nvme_bdev_io *bio = ref; 7536 7537 if (spdk_nvme_cpl_is_success(cpl)) { 7538 /* Run PI verification for read data buffer. */ 7539 bdev_nvme_verify_pi_error(bio); 7540 } 7541 7542 /* Return original completion status */ 7543 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 7544 } 7545 7546 static void 7547 bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7548 { 7549 struct nvme_bdev_io *bio = ref; 7550 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7551 int ret; 7552 7553 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 7554 SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n", 7555 cpl->status.sct, cpl->status.sc); 7556 7557 /* Save completion status to use after verifying PI error. */ 7558 bio->cpl = *cpl; 7559 7560 if (spdk_likely(nvme_io_path_is_available(bio->io_path))) { 7561 /* Read without PI checking to verify PI error. */ 7562 ret = bdev_nvme_no_pi_readv(bio, 7563 bdev_io->u.bdev.iovs, 7564 bdev_io->u.bdev.iovcnt, 7565 bdev_io->u.bdev.md_buf, 7566 bdev_io->u.bdev.num_blocks, 7567 bdev_io->u.bdev.offset_blocks); 7568 if (ret == 0) { 7569 return; 7570 } 7571 } 7572 } 7573 7574 bdev_nvme_io_complete_nvme_status(bio, cpl); 7575 } 7576 7577 static void 7578 bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7579 { 7580 struct nvme_bdev_io *bio = ref; 7581 7582 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 7583 SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n", 7584 cpl->status.sct, cpl->status.sc); 7585 /* Run PI verification for write data buffer if PI error is detected. */ 7586 bdev_nvme_verify_pi_error(bio); 7587 } 7588 7589 bdev_nvme_io_complete_nvme_status(bio, cpl); 7590 } 7591 7592 static void 7593 bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7594 { 7595 struct nvme_bdev_io *bio = ref; 7596 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7597 7598 /* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks. 7599 * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error(). 7600 */ 7601 bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0; 7602 7603 if (spdk_nvme_cpl_is_pi_error(cpl)) { 7604 SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n", 7605 cpl->status.sct, cpl->status.sc); 7606 /* Run PI verification for zone append data buffer if PI error is detected. */ 7607 bdev_nvme_verify_pi_error(bio); 7608 } 7609 7610 bdev_nvme_io_complete_nvme_status(bio, cpl); 7611 } 7612 7613 static void 7614 bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7615 { 7616 struct nvme_bdev_io *bio = ref; 7617 7618 if (spdk_nvme_cpl_is_pi_error(cpl)) { 7619 SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n", 7620 cpl->status.sct, cpl->status.sc); 7621 /* Run PI verification for compare data buffer if PI error is detected. */ 7622 bdev_nvme_verify_pi_error(bio); 7623 } 7624 7625 bdev_nvme_io_complete_nvme_status(bio, cpl); 7626 } 7627 7628 static void 7629 bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7630 { 7631 struct nvme_bdev_io *bio = ref; 7632 7633 /* Compare operation completion */ 7634 if (!bio->first_fused_completed) { 7635 /* Save compare result for write callback */ 7636 bio->cpl = *cpl; 7637 bio->first_fused_completed = true; 7638 return; 7639 } 7640 7641 /* Write operation completion */ 7642 if (spdk_nvme_cpl_is_error(&bio->cpl)) { 7643 /* If bio->cpl is already an error, it means the compare operation failed. In that case, 7644 * complete the IO with the compare operation's status. 7645 */ 7646 if (!spdk_nvme_cpl_is_error(cpl)) { 7647 SPDK_ERRLOG("Unexpected write success after compare failure.\n"); 7648 } 7649 7650 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 7651 } else { 7652 bdev_nvme_io_complete_nvme_status(bio, cpl); 7653 } 7654 } 7655 7656 static void 7657 bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl) 7658 { 7659 struct nvme_bdev_io *bio = ref; 7660 7661 bdev_nvme_io_complete_nvme_status(bio, cpl); 7662 } 7663 7664 static int 7665 fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc) 7666 { 7667 switch (desc->zt) { 7668 case SPDK_NVME_ZONE_TYPE_SEQWR: 7669 info->type = SPDK_BDEV_ZONE_TYPE_SEQWR; 7670 break; 7671 default: 7672 SPDK_ERRLOG("Invalid zone type: %#x in zone report\n", desc->zt); 7673 return -EIO; 7674 } 7675 7676 switch (desc->zs) { 7677 case SPDK_NVME_ZONE_STATE_EMPTY: 7678 info->state = SPDK_BDEV_ZONE_STATE_EMPTY; 7679 break; 7680 case SPDK_NVME_ZONE_STATE_IOPEN: 7681 info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN; 7682 break; 7683 case SPDK_NVME_ZONE_STATE_EOPEN: 7684 info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN; 7685 break; 7686 case SPDK_NVME_ZONE_STATE_CLOSED: 7687 info->state = SPDK_BDEV_ZONE_STATE_CLOSED; 7688 break; 7689 case SPDK_NVME_ZONE_STATE_RONLY: 7690 info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY; 7691 break; 7692 case SPDK_NVME_ZONE_STATE_FULL: 7693 info->state = SPDK_BDEV_ZONE_STATE_FULL; 7694 break; 7695 case SPDK_NVME_ZONE_STATE_OFFLINE: 7696 info->state = SPDK_BDEV_ZONE_STATE_OFFLINE; 7697 break; 7698 default: 7699 SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs); 7700 return -EIO; 7701 } 7702 7703 info->zone_id = desc->zslba; 7704 info->write_pointer = desc->wp; 7705 info->capacity = desc->zcap; 7706 7707 return 0; 7708 } 7709 7710 static void 7711 bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl) 7712 { 7713 struct nvme_bdev_io *bio = ref; 7714 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7715 uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id; 7716 uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones; 7717 struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf; 7718 uint64_t max_zones_per_buf, i; 7719 uint32_t zone_report_bufsize; 7720 struct spdk_nvme_ns *ns; 7721 struct spdk_nvme_qpair *qpair; 7722 int ret; 7723 7724 if (spdk_nvme_cpl_is_error(cpl)) { 7725 goto out_complete_io_nvme_cpl; 7726 } 7727 7728 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 7729 ret = -ENXIO; 7730 goto out_complete_io_ret; 7731 } 7732 7733 ns = bio->io_path->nvme_ns->ns; 7734 qpair = bio->io_path->qpair->qpair; 7735 7736 zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 7737 max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) / 7738 sizeof(bio->zone_report_buf->descs[0]); 7739 7740 if (bio->zone_report_buf->nr_zones > max_zones_per_buf) { 7741 ret = -EINVAL; 7742 goto out_complete_io_ret; 7743 } 7744 7745 if (!bio->zone_report_buf->nr_zones) { 7746 ret = -EINVAL; 7747 goto out_complete_io_ret; 7748 } 7749 7750 for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) { 7751 ret = fill_zone_from_report(&info[bio->handled_zones], 7752 &bio->zone_report_buf->descs[i]); 7753 if (ret) { 7754 goto out_complete_io_ret; 7755 } 7756 bio->handled_zones++; 7757 } 7758 7759 if (bio->handled_zones < zones_to_copy) { 7760 uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 7761 uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones); 7762 7763 memset(bio->zone_report_buf, 0, zone_report_bufsize); 7764 ret = spdk_nvme_zns_report_zones(ns, qpair, 7765 bio->zone_report_buf, zone_report_bufsize, 7766 slba, SPDK_NVME_ZRA_LIST_ALL, true, 7767 bdev_nvme_get_zone_info_done, bio); 7768 if (!ret) { 7769 return; 7770 } else { 7771 goto out_complete_io_ret; 7772 } 7773 } 7774 7775 out_complete_io_nvme_cpl: 7776 free(bio->zone_report_buf); 7777 bio->zone_report_buf = NULL; 7778 bdev_nvme_io_complete_nvme_status(bio, cpl); 7779 return; 7780 7781 out_complete_io_ret: 7782 free(bio->zone_report_buf); 7783 bio->zone_report_buf = NULL; 7784 bdev_nvme_io_complete(bio, ret); 7785 } 7786 7787 static void 7788 bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl) 7789 { 7790 struct nvme_bdev_io *bio = ref; 7791 7792 bdev_nvme_io_complete_nvme_status(bio, cpl); 7793 } 7794 7795 static void 7796 bdev_nvme_admin_passthru_complete_nvme_status(void *ctx) 7797 { 7798 struct nvme_bdev_io *bio = ctx; 7799 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7800 const struct spdk_nvme_cpl *cpl = &bio->cpl; 7801 7802 assert(bdev_nvme_io_type_is_admin(bdev_io->type)); 7803 7804 __bdev_nvme_io_complete(bdev_io, 0, cpl); 7805 } 7806 7807 static void 7808 bdev_nvme_abort_complete(void *ctx) 7809 { 7810 struct nvme_bdev_io *bio = ctx; 7811 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7812 7813 if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) { 7814 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL); 7815 } else { 7816 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL); 7817 } 7818 } 7819 7820 static void 7821 bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl) 7822 { 7823 struct nvme_bdev_io *bio = ref; 7824 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7825 7826 bio->cpl = *cpl; 7827 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), bdev_nvme_abort_complete, bio); 7828 } 7829 7830 static void 7831 bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl) 7832 { 7833 struct nvme_bdev_io *bio = ref; 7834 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7835 7836 bio->cpl = *cpl; 7837 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), 7838 bdev_nvme_admin_passthru_complete_nvme_status, bio); 7839 } 7840 7841 static void 7842 bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset) 7843 { 7844 struct nvme_bdev_io *bio = ref; 7845 struct iovec *iov; 7846 7847 bio->iov_offset = sgl_offset; 7848 for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) { 7849 iov = &bio->iovs[bio->iovpos]; 7850 if (bio->iov_offset < iov->iov_len) { 7851 break; 7852 } 7853 7854 bio->iov_offset -= iov->iov_len; 7855 } 7856 } 7857 7858 static int 7859 bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length) 7860 { 7861 struct nvme_bdev_io *bio = ref; 7862 struct iovec *iov; 7863 7864 assert(bio->iovpos < bio->iovcnt); 7865 7866 iov = &bio->iovs[bio->iovpos]; 7867 7868 *address = iov->iov_base; 7869 *length = iov->iov_len; 7870 7871 if (bio->iov_offset) { 7872 assert(bio->iov_offset <= iov->iov_len); 7873 *address += bio->iov_offset; 7874 *length -= bio->iov_offset; 7875 } 7876 7877 bio->iov_offset += *length; 7878 if (bio->iov_offset == iov->iov_len) { 7879 bio->iovpos++; 7880 bio->iov_offset = 0; 7881 } 7882 7883 return 0; 7884 } 7885 7886 static void 7887 bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset) 7888 { 7889 struct nvme_bdev_io *bio = ref; 7890 struct iovec *iov; 7891 7892 bio->fused_iov_offset = sgl_offset; 7893 for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) { 7894 iov = &bio->fused_iovs[bio->fused_iovpos]; 7895 if (bio->fused_iov_offset < iov->iov_len) { 7896 break; 7897 } 7898 7899 bio->fused_iov_offset -= iov->iov_len; 7900 } 7901 } 7902 7903 static int 7904 bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length) 7905 { 7906 struct nvme_bdev_io *bio = ref; 7907 struct iovec *iov; 7908 7909 assert(bio->fused_iovpos < bio->fused_iovcnt); 7910 7911 iov = &bio->fused_iovs[bio->fused_iovpos]; 7912 7913 *address = iov->iov_base; 7914 *length = iov->iov_len; 7915 7916 if (bio->fused_iov_offset) { 7917 assert(bio->fused_iov_offset <= iov->iov_len); 7918 *address += bio->fused_iov_offset; 7919 *length -= bio->fused_iov_offset; 7920 } 7921 7922 bio->fused_iov_offset += *length; 7923 if (bio->fused_iov_offset == iov->iov_len) { 7924 bio->fused_iovpos++; 7925 bio->fused_iov_offset = 0; 7926 } 7927 7928 return 0; 7929 } 7930 7931 static int 7932 bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7933 void *md, uint64_t lba_count, uint64_t lba) 7934 { 7935 int rc; 7936 7937 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n", 7938 lba_count, lba); 7939 7940 bio->iovs = iov; 7941 bio->iovcnt = iovcnt; 7942 bio->iovpos = 0; 7943 bio->iov_offset = 0; 7944 7945 rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns, 7946 bio->io_path->qpair->qpair, 7947 lba, lba_count, 7948 bdev_nvme_no_pi_readv_done, bio, 0, 7949 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 7950 md, 0, 0); 7951 7952 if (rc != 0 && rc != -ENOMEM) { 7953 SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc); 7954 } 7955 return rc; 7956 } 7957 7958 static int 7959 bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7960 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 7961 struct spdk_memory_domain *domain, void *domain_ctx, 7962 struct spdk_accel_sequence *seq) 7963 { 7964 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7965 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7966 int rc; 7967 7968 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n", 7969 lba_count, lba); 7970 7971 bio->iovs = iov; 7972 bio->iovcnt = iovcnt; 7973 bio->iovpos = 0; 7974 bio->iov_offset = 0; 7975 7976 if (domain != NULL || seq != NULL) { 7977 bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence); 7978 bio->ext_opts.memory_domain = domain; 7979 bio->ext_opts.memory_domain_ctx = domain_ctx; 7980 bio->ext_opts.io_flags = flags; 7981 bio->ext_opts.metadata = md; 7982 bio->ext_opts.accel_sequence = seq; 7983 7984 if (iovcnt == 1) { 7985 rc = spdk_nvme_ns_cmd_read_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_readv_done, 7986 bio, &bio->ext_opts); 7987 } else { 7988 rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count, 7989 bdev_nvme_readv_done, bio, 7990 bdev_nvme_queued_reset_sgl, 7991 bdev_nvme_queued_next_sge, 7992 &bio->ext_opts); 7993 } 7994 } else if (iovcnt == 1) { 7995 rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, iov[0].iov_base, 7996 md, lba, lba_count, bdev_nvme_readv_done, 7997 bio, flags, 0, 0); 7998 } else { 7999 rc = spdk_nvme_ns_cmd_readv_with_md(ns, qpair, lba, lba_count, 8000 bdev_nvme_readv_done, bio, flags, 8001 bdev_nvme_queued_reset_sgl, 8002 bdev_nvme_queued_next_sge, md, 0, 0); 8003 } 8004 8005 if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) { 8006 SPDK_ERRLOG("readv failed: rc = %d\n", rc); 8007 } 8008 return rc; 8009 } 8010 8011 static int 8012 bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 8013 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 8014 struct spdk_memory_domain *domain, void *domain_ctx, 8015 struct spdk_accel_sequence *seq, 8016 union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13) 8017 { 8018 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8019 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8020 int rc; 8021 8022 SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 8023 lba_count, lba); 8024 8025 bio->iovs = iov; 8026 bio->iovcnt = iovcnt; 8027 bio->iovpos = 0; 8028 bio->iov_offset = 0; 8029 8030 if (domain != NULL || seq != NULL) { 8031 bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence); 8032 bio->ext_opts.memory_domain = domain; 8033 bio->ext_opts.memory_domain_ctx = domain_ctx; 8034 bio->ext_opts.io_flags = flags | SPDK_NVME_IO_FLAGS_DIRECTIVE(cdw12.write.dtype); 8035 bio->ext_opts.cdw13 = cdw13.raw; 8036 bio->ext_opts.metadata = md; 8037 bio->ext_opts.accel_sequence = seq; 8038 8039 if (iovcnt == 1) { 8040 rc = spdk_nvme_ns_cmd_write_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_writev_done, 8041 bio, &bio->ext_opts); 8042 } else { 8043 rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count, 8044 bdev_nvme_writev_done, bio, 8045 bdev_nvme_queued_reset_sgl, 8046 bdev_nvme_queued_next_sge, 8047 &bio->ext_opts); 8048 } 8049 } else if (iovcnt == 1) { 8050 rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, iov[0].iov_base, 8051 md, lba, lba_count, bdev_nvme_writev_done, 8052 bio, flags, 0, 0); 8053 } else { 8054 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 8055 bdev_nvme_writev_done, bio, flags, 8056 bdev_nvme_queued_reset_sgl, 8057 bdev_nvme_queued_next_sge, md, 0, 0); 8058 } 8059 8060 if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) { 8061 SPDK_ERRLOG("writev failed: rc = %d\n", rc); 8062 } 8063 return rc; 8064 } 8065 8066 static int 8067 bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 8068 void *md, uint64_t lba_count, uint64_t zslba, 8069 uint32_t flags) 8070 { 8071 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8072 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8073 int rc; 8074 8075 SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n", 8076 lba_count, zslba); 8077 8078 bio->iovs = iov; 8079 bio->iovcnt = iovcnt; 8080 bio->iovpos = 0; 8081 bio->iov_offset = 0; 8082 8083 if (iovcnt == 1) { 8084 rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba, 8085 lba_count, 8086 bdev_nvme_zone_appendv_done, bio, 8087 flags, 8088 0, 0); 8089 } else { 8090 rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count, 8091 bdev_nvme_zone_appendv_done, bio, flags, 8092 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 8093 md, 0, 0); 8094 } 8095 8096 if (rc != 0 && rc != -ENOMEM) { 8097 SPDK_ERRLOG("zone append failed: rc = %d\n", rc); 8098 } 8099 return rc; 8100 } 8101 8102 static int 8103 bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 8104 void *md, uint64_t lba_count, uint64_t lba, 8105 uint32_t flags) 8106 { 8107 int rc; 8108 8109 SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n", 8110 lba_count, lba); 8111 8112 bio->iovs = iov; 8113 bio->iovcnt = iovcnt; 8114 bio->iovpos = 0; 8115 bio->iov_offset = 0; 8116 8117 rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns, 8118 bio->io_path->qpair->qpair, 8119 lba, lba_count, 8120 bdev_nvme_comparev_done, bio, flags, 8121 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 8122 md, 0, 0); 8123 8124 if (rc != 0 && rc != -ENOMEM) { 8125 SPDK_ERRLOG("comparev failed: rc = %d\n", rc); 8126 } 8127 return rc; 8128 } 8129 8130 static int 8131 bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt, 8132 struct iovec *write_iov, int write_iovcnt, 8133 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags) 8134 { 8135 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8136 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8137 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 8138 int rc; 8139 8140 SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 8141 lba_count, lba); 8142 8143 bio->iovs = cmp_iov; 8144 bio->iovcnt = cmp_iovcnt; 8145 bio->iovpos = 0; 8146 bio->iov_offset = 0; 8147 bio->fused_iovs = write_iov; 8148 bio->fused_iovcnt = write_iovcnt; 8149 bio->fused_iovpos = 0; 8150 bio->fused_iov_offset = 0; 8151 8152 if (bdev_io->num_retries == 0) { 8153 bio->first_fused_submitted = false; 8154 bio->first_fused_completed = false; 8155 } 8156 8157 if (!bio->first_fused_submitted) { 8158 flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST; 8159 memset(&bio->cpl, 0, sizeof(bio->cpl)); 8160 8161 rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count, 8162 bdev_nvme_comparev_and_writev_done, bio, flags, 8163 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0); 8164 if (rc == 0) { 8165 bio->first_fused_submitted = true; 8166 flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST; 8167 } else { 8168 if (rc != -ENOMEM) { 8169 SPDK_ERRLOG("compare failed: rc = %d\n", rc); 8170 } 8171 return rc; 8172 } 8173 } 8174 8175 flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND; 8176 8177 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 8178 bdev_nvme_comparev_and_writev_done, bio, flags, 8179 bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0); 8180 if (rc != 0 && rc != -ENOMEM) { 8181 SPDK_ERRLOG("write failed: rc = %d\n", rc); 8182 rc = 0; 8183 } 8184 8185 return rc; 8186 } 8187 8188 static int 8189 bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 8190 { 8191 struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES]; 8192 struct spdk_nvme_dsm_range *range; 8193 uint64_t offset, remaining; 8194 uint64_t num_ranges_u64; 8195 uint16_t num_ranges; 8196 int rc; 8197 8198 num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) / 8199 SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8200 if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) { 8201 SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks); 8202 return -EINVAL; 8203 } 8204 num_ranges = (uint16_t)num_ranges_u64; 8205 8206 offset = offset_blocks; 8207 remaining = num_blocks; 8208 range = &dsm_ranges[0]; 8209 8210 /* Fill max-size ranges until the remaining blocks fit into one range */ 8211 while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) { 8212 range->attributes.raw = 0; 8213 range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8214 range->starting_lba = offset; 8215 8216 offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8217 remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8218 range++; 8219 } 8220 8221 /* Final range describes the remaining blocks */ 8222 range->attributes.raw = 0; 8223 range->length = remaining; 8224 range->starting_lba = offset; 8225 8226 rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns, 8227 bio->io_path->qpair->qpair, 8228 SPDK_NVME_DSM_ATTR_DEALLOCATE, 8229 dsm_ranges, num_ranges, 8230 bdev_nvme_queued_done, bio); 8231 8232 return rc; 8233 } 8234 8235 static int 8236 bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 8237 { 8238 if (num_blocks > UINT16_MAX + 1) { 8239 SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n"); 8240 return -EINVAL; 8241 } 8242 8243 return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns, 8244 bio->io_path->qpair->qpair, 8245 offset_blocks, num_blocks, 8246 bdev_nvme_queued_done, bio, 8247 0); 8248 } 8249 8250 static int 8251 bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones, 8252 struct spdk_bdev_zone_info *info) 8253 { 8254 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8255 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8256 uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 8257 uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 8258 uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns); 8259 8260 if (zone_id % zone_size != 0) { 8261 return -EINVAL; 8262 } 8263 8264 if (num_zones > total_zones || !num_zones) { 8265 return -EINVAL; 8266 } 8267 8268 assert(!bio->zone_report_buf); 8269 bio->zone_report_buf = calloc(1, zone_report_bufsize); 8270 if (!bio->zone_report_buf) { 8271 return -ENOMEM; 8272 } 8273 8274 bio->handled_zones = 0; 8275 8276 return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize, 8277 zone_id, SPDK_NVME_ZRA_LIST_ALL, true, 8278 bdev_nvme_get_zone_info_done, bio); 8279 } 8280 8281 static int 8282 bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 8283 enum spdk_bdev_zone_action action) 8284 { 8285 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8286 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8287 8288 switch (action) { 8289 case SPDK_BDEV_ZONE_CLOSE: 8290 return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false, 8291 bdev_nvme_zone_management_done, bio); 8292 case SPDK_BDEV_ZONE_FINISH: 8293 return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false, 8294 bdev_nvme_zone_management_done, bio); 8295 case SPDK_BDEV_ZONE_OPEN: 8296 return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false, 8297 bdev_nvme_zone_management_done, bio); 8298 case SPDK_BDEV_ZONE_RESET: 8299 return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false, 8300 bdev_nvme_zone_management_done, bio); 8301 case SPDK_BDEV_ZONE_OFFLINE: 8302 return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false, 8303 bdev_nvme_zone_management_done, bio); 8304 default: 8305 return -EINVAL; 8306 } 8307 } 8308 8309 static void 8310 bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 8311 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes) 8312 { 8313 struct nvme_io_path *io_path; 8314 struct nvme_ctrlr *nvme_ctrlr; 8315 uint32_t max_xfer_size; 8316 int rc = -ENXIO; 8317 8318 /* Choose the first ctrlr which is not failed. */ 8319 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 8320 nvme_ctrlr = io_path->qpair->ctrlr; 8321 8322 /* We should skip any unavailable nvme_ctrlr rather than checking 8323 * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO. 8324 */ 8325 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 8326 continue; 8327 } 8328 8329 max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr); 8330 8331 if (nbytes > max_xfer_size) { 8332 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8333 rc = -EINVAL; 8334 goto err; 8335 } 8336 8337 rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes, 8338 bdev_nvme_admin_passthru_done, bio); 8339 if (rc == 0) { 8340 return; 8341 } 8342 } 8343 8344 err: 8345 bdev_nvme_admin_complete(bio, rc); 8346 } 8347 8348 static int 8349 bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 8350 void *buf, size_t nbytes) 8351 { 8352 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8353 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8354 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8355 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8356 8357 if (nbytes > max_xfer_size) { 8358 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8359 return -EINVAL; 8360 } 8361 8362 /* 8363 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 8364 * so fill it out automatically. 8365 */ 8366 cmd->nsid = spdk_nvme_ns_get_id(ns); 8367 8368 return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf, 8369 (uint32_t)nbytes, bdev_nvme_queued_done, bio); 8370 } 8371 8372 static int 8373 bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 8374 void *buf, size_t nbytes, void *md_buf, size_t md_len) 8375 { 8376 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8377 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8378 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 8379 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8380 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8381 8382 if (nbytes > max_xfer_size) { 8383 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8384 return -EINVAL; 8385 } 8386 8387 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 8388 SPDK_ERRLOG("invalid meta data buffer size\n"); 8389 return -EINVAL; 8390 } 8391 8392 /* 8393 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 8394 * so fill it out automatically. 8395 */ 8396 cmd->nsid = spdk_nvme_ns_get_id(ns); 8397 8398 return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf, 8399 (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio); 8400 } 8401 8402 static int 8403 bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, 8404 struct spdk_nvme_cmd *cmd, struct iovec *iov, int iovcnt, 8405 size_t nbytes, void *md_buf, size_t md_len) 8406 { 8407 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8408 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8409 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 8410 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8411 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8412 8413 bio->iovs = iov; 8414 bio->iovcnt = iovcnt; 8415 bio->iovpos = 0; 8416 bio->iov_offset = 0; 8417 8418 if (nbytes > max_xfer_size) { 8419 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8420 return -EINVAL; 8421 } 8422 8423 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 8424 SPDK_ERRLOG("invalid meta data buffer size\n"); 8425 return -EINVAL; 8426 } 8427 8428 /* 8429 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands 8430 * require a nsid, so fill it out automatically. 8431 */ 8432 cmd->nsid = spdk_nvme_ns_get_id(ns); 8433 8434 return spdk_nvme_ctrlr_cmd_iov_raw_with_md( 8435 ctrlr, qpair, cmd, (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio, 8436 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge); 8437 } 8438 8439 static void 8440 bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 8441 struct nvme_bdev_io *bio_to_abort) 8442 { 8443 struct nvme_io_path *io_path; 8444 int rc = 0; 8445 8446 rc = bdev_nvme_abort_retry_io(nbdev_ch, bio_to_abort); 8447 if (rc == 0) { 8448 bdev_nvme_admin_complete(bio, 0); 8449 return; 8450 } 8451 8452 io_path = bio_to_abort->io_path; 8453 if (io_path != NULL) { 8454 rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr, 8455 io_path->qpair->qpair, 8456 bio_to_abort, 8457 bdev_nvme_abort_done, bio); 8458 } else { 8459 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 8460 rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr, 8461 NULL, 8462 bio_to_abort, 8463 bdev_nvme_abort_done, bio); 8464 8465 if (rc != -ENOENT) { 8466 break; 8467 } 8468 } 8469 } 8470 8471 if (rc != 0) { 8472 /* If no command was found or there was any error, complete the abort 8473 * request with failure. 8474 */ 8475 bdev_nvme_admin_complete(bio, rc); 8476 } 8477 } 8478 8479 static int 8480 bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, uint64_t src_offset_blocks, 8481 uint64_t num_blocks) 8482 { 8483 struct spdk_nvme_scc_source_range range = { 8484 .slba = src_offset_blocks, 8485 .nlb = num_blocks - 1 8486 }; 8487 8488 return spdk_nvme_ns_cmd_copy(bio->io_path->nvme_ns->ns, 8489 bio->io_path->qpair->qpair, 8490 &range, 1, dst_offset_blocks, 8491 bdev_nvme_queued_done, bio); 8492 } 8493 8494 static void 8495 bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w) 8496 { 8497 const char *action; 8498 uint32_t i; 8499 8500 if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) { 8501 action = "reset"; 8502 } else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) { 8503 action = "abort"; 8504 } else { 8505 action = "none"; 8506 } 8507 8508 spdk_json_write_object_begin(w); 8509 8510 spdk_json_write_named_string(w, "method", "bdev_nvme_set_options"); 8511 8512 spdk_json_write_named_object_begin(w, "params"); 8513 spdk_json_write_named_string(w, "action_on_timeout", action); 8514 spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us); 8515 spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us); 8516 spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms); 8517 spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst); 8518 spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight); 8519 spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight); 8520 spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight); 8521 spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us); 8522 spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us); 8523 spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests); 8524 spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit); 8525 spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count); 8526 spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count); 8527 spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout); 8528 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec); 8529 spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec); 8530 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec); 8531 spdk_json_write_named_bool(w, "disable_auto_failback", g_opts.disable_auto_failback); 8532 spdk_json_write_named_bool(w, "generate_uuids", g_opts.generate_uuids); 8533 spdk_json_write_named_uint8(w, "transport_tos", g_opts.transport_tos); 8534 spdk_json_write_named_bool(w, "nvme_error_stat", g_opts.nvme_error_stat); 8535 spdk_json_write_named_uint32(w, "rdma_srq_size", g_opts.rdma_srq_size); 8536 spdk_json_write_named_bool(w, "io_path_stat", g_opts.io_path_stat); 8537 spdk_json_write_named_bool(w, "allow_accel_sequence", g_opts.allow_accel_sequence); 8538 spdk_json_write_named_uint32(w, "rdma_max_cq_size", g_opts.rdma_max_cq_size); 8539 spdk_json_write_named_uint16(w, "rdma_cm_event_timeout_ms", g_opts.rdma_cm_event_timeout_ms); 8540 spdk_json_write_named_array_begin(w, "dhchap_digests"); 8541 for (i = 0; i < 32; ++i) { 8542 if (g_opts.dhchap_digests & SPDK_BIT(i)) { 8543 spdk_json_write_string(w, spdk_nvme_dhchap_get_digest_name(i)); 8544 } 8545 } 8546 spdk_json_write_array_end(w); 8547 spdk_json_write_named_array_begin(w, "dhchap_dhgroups"); 8548 for (i = 0; i < 32; ++i) { 8549 if (g_opts.dhchap_dhgroups & SPDK_BIT(i)) { 8550 spdk_json_write_string(w, spdk_nvme_dhchap_get_dhgroup_name(i)); 8551 } 8552 } 8553 8554 spdk_json_write_array_end(w); 8555 spdk_json_write_object_end(w); 8556 8557 spdk_json_write_object_end(w); 8558 } 8559 8560 static void 8561 bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx) 8562 { 8563 struct spdk_nvme_transport_id trid; 8564 8565 spdk_json_write_object_begin(w); 8566 8567 spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery"); 8568 8569 spdk_json_write_named_object_begin(w, "params"); 8570 spdk_json_write_named_string(w, "name", ctx->name); 8571 spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn); 8572 8573 trid = ctx->trid; 8574 memset(trid.subnqn, 0, sizeof(trid.subnqn)); 8575 nvme_bdev_dump_trid_json(&trid, w); 8576 8577 spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach); 8578 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec); 8579 spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec); 8580 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 8581 ctx->bdev_opts.fast_io_fail_timeout_sec); 8582 spdk_json_write_object_end(w); 8583 8584 spdk_json_write_object_end(w); 8585 } 8586 8587 #ifdef SPDK_CONFIG_NVME_CUSE 8588 static void 8589 nvme_ctrlr_cuse_config_json(struct spdk_json_write_ctx *w, 8590 struct nvme_ctrlr *nvme_ctrlr) 8591 { 8592 size_t cuse_name_size = 128; 8593 char cuse_name[cuse_name_size]; 8594 8595 if (spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, 8596 cuse_name, &cuse_name_size) != 0) { 8597 return; 8598 } 8599 8600 spdk_json_write_object_begin(w); 8601 8602 spdk_json_write_named_string(w, "method", "bdev_nvme_cuse_register"); 8603 8604 spdk_json_write_named_object_begin(w, "params"); 8605 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 8606 spdk_json_write_object_end(w); 8607 8608 spdk_json_write_object_end(w); 8609 } 8610 #endif 8611 8612 static void 8613 nvme_ctrlr_config_json(struct spdk_json_write_ctx *w, 8614 struct nvme_ctrlr *nvme_ctrlr, 8615 struct nvme_path_id *path_id) 8616 { 8617 struct spdk_nvme_transport_id *trid; 8618 const struct spdk_nvme_ctrlr_opts *opts; 8619 8620 if (nvme_ctrlr->opts.from_discovery_service) { 8621 /* Do not emit an RPC for this - it will be implicitly 8622 * covered by a separate bdev_nvme_start_discovery or 8623 * bdev_nvme_start_mdns_discovery RPC. 8624 */ 8625 return; 8626 } 8627 8628 trid = &path_id->trid; 8629 8630 spdk_json_write_object_begin(w); 8631 8632 spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller"); 8633 8634 spdk_json_write_named_object_begin(w, "params"); 8635 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 8636 nvme_bdev_dump_trid_json(trid, w); 8637 spdk_json_write_named_bool(w, "prchk_reftag", 8638 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0); 8639 spdk_json_write_named_bool(w, "prchk_guard", 8640 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0); 8641 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec); 8642 spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec); 8643 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 8644 nvme_ctrlr->opts.fast_io_fail_timeout_sec); 8645 if (nvme_ctrlr->psk != NULL) { 8646 spdk_json_write_named_string(w, "psk", spdk_key_get_name(nvme_ctrlr->psk)); 8647 } else if (nvme_ctrlr->opts.psk[0] != '\0') { 8648 spdk_json_write_named_string(w, "psk", nvme_ctrlr->opts.psk); 8649 } 8650 8651 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 8652 spdk_json_write_named_string(w, "hostnqn", opts->hostnqn); 8653 spdk_json_write_named_bool(w, "hdgst", opts->header_digest); 8654 spdk_json_write_named_bool(w, "ddgst", opts->data_digest); 8655 if (opts->src_addr[0] != '\0') { 8656 spdk_json_write_named_string(w, "hostaddr", opts->src_addr); 8657 } 8658 if (opts->src_svcid[0] != '\0') { 8659 spdk_json_write_named_string(w, "hostsvcid", opts->src_svcid); 8660 } 8661 8662 if (nvme_ctrlr->opts.multipath) { 8663 spdk_json_write_named_string(w, "multipath", "multipath"); 8664 } 8665 spdk_json_write_object_end(w); 8666 8667 spdk_json_write_object_end(w); 8668 } 8669 8670 static void 8671 bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w) 8672 { 8673 spdk_json_write_object_begin(w); 8674 spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug"); 8675 8676 spdk_json_write_named_object_begin(w, "params"); 8677 spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us); 8678 spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled); 8679 spdk_json_write_object_end(w); 8680 8681 spdk_json_write_object_end(w); 8682 } 8683 8684 static int 8685 bdev_nvme_config_json(struct spdk_json_write_ctx *w) 8686 { 8687 struct nvme_bdev_ctrlr *nbdev_ctrlr; 8688 struct nvme_ctrlr *nvme_ctrlr; 8689 struct discovery_ctx *ctx; 8690 struct nvme_path_id *path_id; 8691 8692 bdev_nvme_opts_config_json(w); 8693 8694 pthread_mutex_lock(&g_bdev_nvme_mutex); 8695 8696 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 8697 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 8698 path_id = nvme_ctrlr->active_path_id; 8699 assert(path_id == TAILQ_FIRST(&nvme_ctrlr->trids)); 8700 nvme_ctrlr_config_json(w, nvme_ctrlr, path_id); 8701 8702 path_id = TAILQ_NEXT(path_id, link); 8703 while (path_id != NULL) { 8704 nvme_ctrlr_config_json(w, nvme_ctrlr, path_id); 8705 path_id = TAILQ_NEXT(path_id, link); 8706 } 8707 8708 #ifdef SPDK_CONFIG_NVME_CUSE 8709 nvme_ctrlr_cuse_config_json(w, nvme_ctrlr); 8710 #endif 8711 } 8712 } 8713 8714 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 8715 if (!ctx->from_mdns_discovery_service) { 8716 bdev_nvme_discovery_config_json(w, ctx); 8717 } 8718 } 8719 8720 bdev_nvme_mdns_discovery_config_json(w); 8721 8722 /* Dump as last parameter to give all NVMe bdevs chance to be constructed 8723 * before enabling hotplug poller. 8724 */ 8725 bdev_nvme_hotplug_config_json(w); 8726 8727 pthread_mutex_unlock(&g_bdev_nvme_mutex); 8728 return 0; 8729 } 8730 8731 struct spdk_nvme_ctrlr * 8732 bdev_nvme_get_ctrlr(struct spdk_bdev *bdev) 8733 { 8734 struct nvme_bdev *nbdev; 8735 struct nvme_ns *nvme_ns; 8736 8737 if (!bdev || bdev->module != &nvme_if) { 8738 return NULL; 8739 } 8740 8741 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 8742 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 8743 assert(nvme_ns != NULL); 8744 8745 return nvme_ns->ctrlr->ctrlr; 8746 } 8747 8748 static bool 8749 nvme_io_path_is_current(struct nvme_io_path *io_path) 8750 { 8751 const struct nvme_bdev_channel *nbdev_ch; 8752 bool current; 8753 8754 if (!nvme_io_path_is_available(io_path)) { 8755 return false; 8756 } 8757 8758 nbdev_ch = io_path->nbdev_ch; 8759 if (nbdev_ch == NULL) { 8760 current = false; 8761 } else if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 8762 struct nvme_io_path *optimized_io_path = NULL; 8763 8764 STAILQ_FOREACH(optimized_io_path, &nbdev_ch->io_path_list, stailq) { 8765 if (optimized_io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) { 8766 break; 8767 } 8768 } 8769 8770 /* A non-optimized path is only current if there are no optimized paths. */ 8771 current = (io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) || 8772 (optimized_io_path == NULL); 8773 } else { 8774 if (nbdev_ch->current_io_path) { 8775 current = (io_path == nbdev_ch->current_io_path); 8776 } else { 8777 struct nvme_io_path *first_path; 8778 8779 /* We arrived here as there are no optimized paths for active-passive 8780 * mode. Check if this io_path is the first one available on the list. 8781 */ 8782 current = false; 8783 STAILQ_FOREACH(first_path, &nbdev_ch->io_path_list, stailq) { 8784 if (nvme_io_path_is_available(first_path)) { 8785 current = (io_path == first_path); 8786 break; 8787 } 8788 } 8789 } 8790 } 8791 8792 return current; 8793 } 8794 8795 void 8796 nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path) 8797 { 8798 struct nvme_ns *nvme_ns = io_path->nvme_ns; 8799 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 8800 const struct spdk_nvme_ctrlr_data *cdata; 8801 const struct spdk_nvme_transport_id *trid; 8802 const char *adrfam_str; 8803 8804 spdk_json_write_object_begin(w); 8805 8806 spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name); 8807 8808 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 8809 trid = spdk_nvme_ctrlr_get_transport_id(nvme_ctrlr->ctrlr); 8810 8811 spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid); 8812 spdk_json_write_named_bool(w, "current", nvme_io_path_is_current(io_path)); 8813 spdk_json_write_named_bool(w, "connected", nvme_qpair_is_connected(io_path->qpair)); 8814 spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns)); 8815 8816 spdk_json_write_named_object_begin(w, "transport"); 8817 spdk_json_write_named_string(w, "trtype", trid->trstring); 8818 spdk_json_write_named_string(w, "traddr", trid->traddr); 8819 if (trid->trsvcid[0] != '\0') { 8820 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 8821 } 8822 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 8823 if (adrfam_str) { 8824 spdk_json_write_named_string(w, "adrfam", adrfam_str); 8825 } 8826 spdk_json_write_object_end(w); 8827 8828 spdk_json_write_object_end(w); 8829 } 8830 8831 void 8832 bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w) 8833 { 8834 struct discovery_ctx *ctx; 8835 struct discovery_entry_ctx *entry_ctx; 8836 8837 spdk_json_write_array_begin(w); 8838 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 8839 spdk_json_write_object_begin(w); 8840 spdk_json_write_named_string(w, "name", ctx->name); 8841 8842 spdk_json_write_named_object_begin(w, "trid"); 8843 nvme_bdev_dump_trid_json(&ctx->trid, w); 8844 spdk_json_write_object_end(w); 8845 8846 spdk_json_write_named_array_begin(w, "referrals"); 8847 TAILQ_FOREACH(entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 8848 spdk_json_write_object_begin(w); 8849 spdk_json_write_named_object_begin(w, "trid"); 8850 nvme_bdev_dump_trid_json(&entry_ctx->trid, w); 8851 spdk_json_write_object_end(w); 8852 spdk_json_write_object_end(w); 8853 } 8854 spdk_json_write_array_end(w); 8855 8856 spdk_json_write_object_end(w); 8857 } 8858 spdk_json_write_array_end(w); 8859 } 8860 8861 SPDK_LOG_REGISTER_COMPONENT(bdev_nvme) 8862 8863 SPDK_TRACE_REGISTER_FN(bdev_nvme_trace, "bdev_nvme", TRACE_GROUP_BDEV_NVME) 8864 { 8865 struct spdk_trace_tpoint_opts opts[] = { 8866 { 8867 "BDEV_NVME_IO_START", TRACE_BDEV_NVME_IO_START, 8868 OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 1, 8869 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 8870 }, 8871 { 8872 "BDEV_NVME_IO_DONE", TRACE_BDEV_NVME_IO_DONE, 8873 OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 0, 8874 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 8875 } 8876 }; 8877 8878 8879 spdk_trace_register_object(OBJECT_BDEV_NVME_IO, 'N'); 8880 spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts)); 8881 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 8882 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 8883 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 8884 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 8885 } 8886