1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved. 6 */ 7 8 #include "spdk/stdinc.h" 9 10 #include "bdev_nvme.h" 11 12 #include "spdk/accel.h" 13 #include "spdk/config.h" 14 #include "spdk/endian.h" 15 #include "spdk/bdev.h" 16 #include "spdk/json.h" 17 #include "spdk/keyring.h" 18 #include "spdk/likely.h" 19 #include "spdk/nvme.h" 20 #include "spdk/nvme_ocssd.h" 21 #include "spdk/nvme_zns.h" 22 #include "spdk/opal.h" 23 #include "spdk/thread.h" 24 #include "spdk/trace.h" 25 #include "spdk/string.h" 26 #include "spdk/util.h" 27 #include "spdk/uuid.h" 28 29 #include "spdk/bdev_module.h" 30 #include "spdk/log.h" 31 32 #include "spdk_internal/usdt.h" 33 #include "spdk_internal/trace_defs.h" 34 35 #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true 36 #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS (10000) 37 38 #define NSID_STR_LEN 10 39 40 #define SPDK_CONTROLLER_NAME_MAX 512 41 42 static int bdev_nvme_config_json(struct spdk_json_write_ctx *w); 43 44 struct nvme_bdev_io { 45 /** array of iovecs to transfer. */ 46 struct iovec *iovs; 47 48 /** Number of iovecs in iovs array. */ 49 int iovcnt; 50 51 /** Current iovec position. */ 52 int iovpos; 53 54 /** Offset in current iovec. */ 55 uint32_t iov_offset; 56 57 /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path 58 * being reset in a reset I/O. 59 */ 60 struct nvme_io_path *io_path; 61 62 /** array of iovecs to transfer. */ 63 struct iovec *fused_iovs; 64 65 /** Number of iovecs in iovs array. */ 66 int fused_iovcnt; 67 68 /** Current iovec position. */ 69 int fused_iovpos; 70 71 /** Offset in current iovec. */ 72 uint32_t fused_iov_offset; 73 74 /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */ 75 struct spdk_nvme_cpl cpl; 76 77 /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */ 78 struct spdk_nvme_ns_cmd_ext_io_opts ext_opts; 79 80 /** Keeps track if first of fused commands was submitted */ 81 bool first_fused_submitted; 82 83 /** Keeps track if first of fused commands was completed */ 84 bool first_fused_completed; 85 86 /** Temporary pointer to zone report buffer */ 87 struct spdk_nvme_zns_zone_report *zone_report_buf; 88 89 /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */ 90 uint64_t handled_zones; 91 92 /** Expiration value in ticks to retry the current I/O. */ 93 uint64_t retry_ticks; 94 95 /* How many times the current I/O was retried. */ 96 int32_t retry_count; 97 98 /* Current tsc at submit time. */ 99 uint64_t submit_tsc; 100 101 /* Used to put nvme_bdev_io into the list */ 102 TAILQ_ENTRY(nvme_bdev_io) retry_link; 103 }; 104 105 struct nvme_probe_skip_entry { 106 struct spdk_nvme_transport_id trid; 107 TAILQ_ENTRY(nvme_probe_skip_entry) tailq; 108 }; 109 /* All the controllers deleted by users via RPC are skipped by hotplug monitor */ 110 static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER( 111 g_skipped_nvme_ctrlrs); 112 113 #define BDEV_NVME_DEFAULT_DIGESTS (SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA256) | \ 114 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA384) | \ 115 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA512)) 116 117 #define BDEV_NVME_DEFAULT_DHGROUPS (SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_NULL) | \ 118 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_2048) | \ 119 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_3072) | \ 120 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_4096) | \ 121 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_6144) | \ 122 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_8192)) 123 124 static struct spdk_bdev_nvme_opts g_opts = { 125 .action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE, 126 .timeout_us = 0, 127 .timeout_admin_us = 0, 128 .keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS, 129 .transport_retry_count = 4, 130 .arbitration_burst = 0, 131 .low_priority_weight = 0, 132 .medium_priority_weight = 0, 133 .high_priority_weight = 0, 134 .nvme_adminq_poll_period_us = 10000ULL, 135 .nvme_ioq_poll_period_us = 0, 136 .io_queue_requests = 0, 137 .delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT, 138 .bdev_retry_count = 3, 139 .transport_ack_timeout = 0, 140 .ctrlr_loss_timeout_sec = 0, 141 .reconnect_delay_sec = 0, 142 .fast_io_fail_timeout_sec = 0, 143 .disable_auto_failback = false, 144 .generate_uuids = false, 145 .transport_tos = 0, 146 .nvme_error_stat = false, 147 .io_path_stat = false, 148 .allow_accel_sequence = false, 149 .dhchap_digests = BDEV_NVME_DEFAULT_DIGESTS, 150 .dhchap_dhgroups = BDEV_NVME_DEFAULT_DHGROUPS, 151 }; 152 153 #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL 154 #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL 155 156 static int g_hot_insert_nvme_controller_index = 0; 157 static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT; 158 static bool g_nvme_hotplug_enabled = false; 159 struct spdk_thread *g_bdev_nvme_init_thread; 160 static struct spdk_poller *g_hotplug_poller; 161 static struct spdk_poller *g_hotplug_probe_poller; 162 static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx; 163 164 static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 165 struct nvme_async_probe_ctx *ctx); 166 static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 167 struct nvme_async_probe_ctx *ctx); 168 static int bdev_nvme_library_init(void); 169 static void bdev_nvme_library_fini(void); 170 static void _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, 171 struct spdk_bdev_io *bdev_io); 172 static void bdev_nvme_submit_request(struct spdk_io_channel *ch, 173 struct spdk_bdev_io *bdev_io); 174 static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 175 void *md, uint64_t lba_count, uint64_t lba, 176 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx, 177 struct spdk_accel_sequence *seq); 178 static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 179 void *md, uint64_t lba_count, uint64_t lba); 180 static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 181 void *md, uint64_t lba_count, uint64_t lba, 182 uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx, 183 struct spdk_accel_sequence *seq, 184 union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13); 185 static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 186 void *md, uint64_t lba_count, 187 uint64_t zslba, uint32_t flags); 188 static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 189 void *md, uint64_t lba_count, uint64_t lba, 190 uint32_t flags); 191 static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, 192 struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov, 193 int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba, 194 uint32_t flags); 195 static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, 196 uint32_t num_zones, struct spdk_bdev_zone_info *info); 197 static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 198 enum spdk_bdev_zone_action action); 199 static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, 200 struct nvme_bdev_io *bio, 201 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes); 202 static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 203 void *buf, size_t nbytes); 204 static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 205 void *buf, size_t nbytes, void *md_buf, size_t md_len); 206 static int bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 207 struct iovec *iov, int iovcnt, size_t nbytes, 208 void *md_buf, size_t md_len); 209 static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, 210 struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort); 211 static void bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio); 212 static int bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 213 static int bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 214 static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr); 215 static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr); 216 217 static struct nvme_ns *nvme_ns_alloc(void); 218 static void nvme_ns_free(struct nvme_ns *ns); 219 220 static int 221 nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2) 222 { 223 return ns1->id < ns2->id ? -1 : ns1->id > ns2->id; 224 } 225 226 RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp); 227 228 struct spdk_nvme_qpair * 229 bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch) 230 { 231 struct nvme_ctrlr_channel *ctrlr_ch; 232 233 assert(ctrlr_io_ch != NULL); 234 235 ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch); 236 237 return ctrlr_ch->qpair->qpair; 238 } 239 240 static int 241 bdev_nvme_get_ctx_size(void) 242 { 243 return sizeof(struct nvme_bdev_io); 244 } 245 246 static struct spdk_bdev_module nvme_if = { 247 .name = "nvme", 248 .async_fini = true, 249 .module_init = bdev_nvme_library_init, 250 .module_fini = bdev_nvme_library_fini, 251 .config_json = bdev_nvme_config_json, 252 .get_ctx_size = bdev_nvme_get_ctx_size, 253 254 }; 255 SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if) 256 257 struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs); 258 pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER; 259 bool g_bdev_nvme_module_finish; 260 261 struct nvme_bdev_ctrlr * 262 nvme_bdev_ctrlr_get_by_name(const char *name) 263 { 264 struct nvme_bdev_ctrlr *nbdev_ctrlr; 265 266 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 267 if (strcmp(name, nbdev_ctrlr->name) == 0) { 268 break; 269 } 270 } 271 272 return nbdev_ctrlr; 273 } 274 275 static struct nvme_ctrlr * 276 nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr, 277 const struct spdk_nvme_transport_id *trid, const char *hostnqn) 278 { 279 const struct spdk_nvme_ctrlr_opts *opts; 280 struct nvme_ctrlr *nvme_ctrlr; 281 282 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 283 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 284 if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0 && 285 strcmp(hostnqn, opts->hostnqn) == 0) { 286 break; 287 } 288 } 289 290 return nvme_ctrlr; 291 } 292 293 struct nvme_ctrlr * 294 nvme_bdev_ctrlr_get_ctrlr_by_id(struct nvme_bdev_ctrlr *nbdev_ctrlr, 295 uint16_t cntlid) 296 { 297 struct nvme_ctrlr *nvme_ctrlr; 298 const struct spdk_nvme_ctrlr_data *cdata; 299 300 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 301 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 302 if (cdata->cntlid == cntlid) { 303 break; 304 } 305 } 306 307 return nvme_ctrlr; 308 } 309 310 static struct nvme_bdev * 311 nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid) 312 { 313 struct nvme_bdev *bdev; 314 315 pthread_mutex_lock(&g_bdev_nvme_mutex); 316 TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) { 317 if (bdev->nsid == nsid) { 318 break; 319 } 320 } 321 pthread_mutex_unlock(&g_bdev_nvme_mutex); 322 323 return bdev; 324 } 325 326 struct nvme_ns * 327 nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid) 328 { 329 struct nvme_ns ns; 330 331 assert(nsid > 0); 332 333 ns.id = nsid; 334 return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns); 335 } 336 337 struct nvme_ns * 338 nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr) 339 { 340 return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces); 341 } 342 343 struct nvme_ns * 344 nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns) 345 { 346 if (ns == NULL) { 347 return NULL; 348 } 349 350 return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 351 } 352 353 static struct nvme_ctrlr * 354 nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid, const char *hostnqn) 355 { 356 struct nvme_bdev_ctrlr *nbdev_ctrlr; 357 struct nvme_ctrlr *nvme_ctrlr = NULL; 358 359 pthread_mutex_lock(&g_bdev_nvme_mutex); 360 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 361 nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid, hostnqn); 362 if (nvme_ctrlr != NULL) { 363 break; 364 } 365 } 366 pthread_mutex_unlock(&g_bdev_nvme_mutex); 367 368 return nvme_ctrlr; 369 } 370 371 struct nvme_ctrlr * 372 nvme_ctrlr_get_by_name(const char *name) 373 { 374 struct nvme_bdev_ctrlr *nbdev_ctrlr; 375 struct nvme_ctrlr *nvme_ctrlr = NULL; 376 377 if (name == NULL) { 378 return NULL; 379 } 380 381 pthread_mutex_lock(&g_bdev_nvme_mutex); 382 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 383 if (nbdev_ctrlr != NULL) { 384 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 385 } 386 pthread_mutex_unlock(&g_bdev_nvme_mutex); 387 388 return nvme_ctrlr; 389 } 390 391 void 392 nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx) 393 { 394 struct nvme_bdev_ctrlr *nbdev_ctrlr; 395 396 pthread_mutex_lock(&g_bdev_nvme_mutex); 397 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 398 fn(nbdev_ctrlr, ctx); 399 } 400 pthread_mutex_unlock(&g_bdev_nvme_mutex); 401 } 402 403 struct nvme_ctrlr_channel_iter { 404 nvme_ctrlr_for_each_channel_msg fn; 405 nvme_ctrlr_for_each_channel_done cpl; 406 struct spdk_io_channel_iter *i; 407 void *ctx; 408 }; 409 410 void 411 nvme_ctrlr_for_each_channel_continue(struct nvme_ctrlr_channel_iter *iter, int status) 412 { 413 spdk_for_each_channel_continue(iter->i, status); 414 } 415 416 static void 417 nvme_ctrlr_each_channel_msg(struct spdk_io_channel_iter *i) 418 { 419 struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 420 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 421 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 422 struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch); 423 424 iter->i = i; 425 iter->fn(iter, nvme_ctrlr, ctrlr_ch, iter->ctx); 426 } 427 428 static void 429 nvme_ctrlr_each_channel_cpl(struct spdk_io_channel_iter *i, int status) 430 { 431 struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 432 struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i); 433 434 iter->i = i; 435 iter->cpl(nvme_ctrlr, iter->ctx, status); 436 437 free(iter); 438 } 439 440 void 441 nvme_ctrlr_for_each_channel(struct nvme_ctrlr *nvme_ctrlr, 442 nvme_ctrlr_for_each_channel_msg fn, void *ctx, 443 nvme_ctrlr_for_each_channel_done cpl) 444 { 445 struct nvme_ctrlr_channel_iter *iter; 446 447 assert(nvme_ctrlr != NULL && fn != NULL); 448 449 iter = calloc(1, sizeof(struct nvme_ctrlr_channel_iter)); 450 if (iter == NULL) { 451 SPDK_ERRLOG("Unable to allocate iterator\n"); 452 assert(false); 453 return; 454 } 455 456 iter->fn = fn; 457 iter->cpl = cpl; 458 iter->ctx = ctx; 459 460 spdk_for_each_channel(nvme_ctrlr, nvme_ctrlr_each_channel_msg, 461 iter, nvme_ctrlr_each_channel_cpl); 462 } 463 464 struct nvme_bdev_channel_iter { 465 nvme_bdev_for_each_channel_msg fn; 466 nvme_bdev_for_each_channel_done cpl; 467 struct spdk_io_channel_iter *i; 468 void *ctx; 469 }; 470 471 void 472 nvme_bdev_for_each_channel_continue(struct nvme_bdev_channel_iter *iter, int status) 473 { 474 spdk_for_each_channel_continue(iter->i, status); 475 } 476 477 static void 478 nvme_bdev_each_channel_msg(struct spdk_io_channel_iter *i) 479 { 480 struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 481 struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i); 482 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 483 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 484 485 iter->i = i; 486 iter->fn(iter, nbdev, nbdev_ch, iter->ctx); 487 } 488 489 static void 490 nvme_bdev_each_channel_cpl(struct spdk_io_channel_iter *i, int status) 491 { 492 struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i); 493 struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i); 494 495 iter->i = i; 496 iter->cpl(nbdev, iter->ctx, status); 497 498 free(iter); 499 } 500 501 void 502 nvme_bdev_for_each_channel(struct nvme_bdev *nbdev, 503 nvme_bdev_for_each_channel_msg fn, void *ctx, 504 nvme_bdev_for_each_channel_done cpl) 505 { 506 struct nvme_bdev_channel_iter *iter; 507 508 assert(nbdev != NULL && fn != NULL); 509 510 iter = calloc(1, sizeof(struct nvme_bdev_channel_iter)); 511 if (iter == NULL) { 512 SPDK_ERRLOG("Unable to allocate iterator\n"); 513 assert(false); 514 return; 515 } 516 517 iter->fn = fn; 518 iter->cpl = cpl; 519 iter->ctx = ctx; 520 521 spdk_for_each_channel(nbdev, nvme_bdev_each_channel_msg, iter, 522 nvme_bdev_each_channel_cpl); 523 } 524 525 void 526 nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w) 527 { 528 const char *trtype_str; 529 const char *adrfam_str; 530 531 trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype); 532 if (trtype_str) { 533 spdk_json_write_named_string(w, "trtype", trtype_str); 534 } 535 536 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 537 if (adrfam_str) { 538 spdk_json_write_named_string(w, "adrfam", adrfam_str); 539 } 540 541 if (trid->traddr[0] != '\0') { 542 spdk_json_write_named_string(w, "traddr", trid->traddr); 543 } 544 545 if (trid->trsvcid[0] != '\0') { 546 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 547 } 548 549 if (trid->subnqn[0] != '\0') { 550 spdk_json_write_named_string(w, "subnqn", trid->subnqn); 551 } 552 } 553 554 static void 555 nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr, 556 struct nvme_ctrlr *nvme_ctrlr) 557 { 558 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name); 559 pthread_mutex_lock(&g_bdev_nvme_mutex); 560 561 TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 562 if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) { 563 pthread_mutex_unlock(&g_bdev_nvme_mutex); 564 565 return; 566 } 567 TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 568 569 pthread_mutex_unlock(&g_bdev_nvme_mutex); 570 571 assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs)); 572 573 free(nbdev_ctrlr->name); 574 free(nbdev_ctrlr); 575 } 576 577 static void 578 _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 579 { 580 struct nvme_path_id *path_id, *tmp_path; 581 struct nvme_ns *ns, *tmp_ns; 582 583 free(nvme_ctrlr->copied_ana_desc); 584 spdk_free(nvme_ctrlr->ana_log_page); 585 586 if (nvme_ctrlr->opal_dev) { 587 spdk_opal_dev_destruct(nvme_ctrlr->opal_dev); 588 nvme_ctrlr->opal_dev = NULL; 589 } 590 591 if (nvme_ctrlr->nbdev_ctrlr) { 592 nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr); 593 } 594 595 RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) { 596 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns); 597 nvme_ns_free(ns); 598 } 599 600 TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) { 601 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 602 free(path_id); 603 } 604 605 pthread_mutex_destroy(&nvme_ctrlr->mutex); 606 spdk_keyring_put_key(nvme_ctrlr->psk); 607 spdk_keyring_put_key(nvme_ctrlr->dhchap_key); 608 spdk_keyring_put_key(nvme_ctrlr->dhchap_ctrlr_key); 609 free(nvme_ctrlr); 610 611 pthread_mutex_lock(&g_bdev_nvme_mutex); 612 if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 613 pthread_mutex_unlock(&g_bdev_nvme_mutex); 614 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 615 spdk_bdev_module_fini_done(); 616 return; 617 } 618 pthread_mutex_unlock(&g_bdev_nvme_mutex); 619 } 620 621 static int 622 nvme_detach_poller(void *arg) 623 { 624 struct nvme_ctrlr *nvme_ctrlr = arg; 625 int rc; 626 627 rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx); 628 if (rc != -EAGAIN) { 629 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 630 _nvme_ctrlr_delete(nvme_ctrlr); 631 } 632 633 return SPDK_POLLER_BUSY; 634 } 635 636 static void 637 nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr) 638 { 639 int rc; 640 641 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 642 643 /* First, unregister the adminq poller, as the driver will poll adminq if necessary */ 644 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 645 646 /* If we got here, the reset/detach poller cannot be active */ 647 assert(nvme_ctrlr->reset_detach_poller == NULL); 648 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller, 649 nvme_ctrlr, 1000); 650 if (nvme_ctrlr->reset_detach_poller == NULL) { 651 SPDK_ERRLOG("Failed to register detach poller\n"); 652 goto error; 653 } 654 655 rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx); 656 if (rc != 0) { 657 SPDK_ERRLOG("Failed to detach the NVMe controller\n"); 658 goto error; 659 } 660 661 return; 662 error: 663 /* We don't have a good way to handle errors here, so just do what we can and delete the 664 * controller without detaching the underlying NVMe device. 665 */ 666 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 667 _nvme_ctrlr_delete(nvme_ctrlr); 668 } 669 670 static void 671 nvme_ctrlr_unregister_cb(void *io_device) 672 { 673 struct nvme_ctrlr *nvme_ctrlr = io_device; 674 675 nvme_ctrlr_delete(nvme_ctrlr); 676 } 677 678 static void 679 nvme_ctrlr_unregister(void *ctx) 680 { 681 struct nvme_ctrlr *nvme_ctrlr = ctx; 682 683 spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb); 684 } 685 686 static bool 687 nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr) 688 { 689 if (!nvme_ctrlr->destruct) { 690 return false; 691 } 692 693 if (nvme_ctrlr->ref > 0) { 694 return false; 695 } 696 697 if (nvme_ctrlr->resetting) { 698 return false; 699 } 700 701 if (nvme_ctrlr->ana_log_page_updating) { 702 return false; 703 } 704 705 if (nvme_ctrlr->io_path_cache_clearing) { 706 return false; 707 } 708 709 return true; 710 } 711 712 static void 713 nvme_ctrlr_release(struct nvme_ctrlr *nvme_ctrlr) 714 { 715 pthread_mutex_lock(&nvme_ctrlr->mutex); 716 SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref); 717 718 assert(nvme_ctrlr->ref > 0); 719 nvme_ctrlr->ref--; 720 721 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 722 pthread_mutex_unlock(&nvme_ctrlr->mutex); 723 return; 724 } 725 726 pthread_mutex_unlock(&nvme_ctrlr->mutex); 727 728 spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr); 729 } 730 731 static void 732 bdev_nvme_clear_current_io_path(struct nvme_bdev_channel *nbdev_ch) 733 { 734 nbdev_ch->current_io_path = NULL; 735 nbdev_ch->rr_counter = 0; 736 } 737 738 static struct nvme_io_path * 739 _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 740 { 741 struct nvme_io_path *io_path; 742 743 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 744 if (io_path->nvme_ns == nvme_ns) { 745 break; 746 } 747 } 748 749 return io_path; 750 } 751 752 static struct nvme_io_path * 753 nvme_io_path_alloc(void) 754 { 755 struct nvme_io_path *io_path; 756 757 io_path = calloc(1, sizeof(*io_path)); 758 if (io_path == NULL) { 759 SPDK_ERRLOG("Failed to alloc io_path.\n"); 760 return NULL; 761 } 762 763 if (g_opts.io_path_stat) { 764 io_path->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 765 if (io_path->stat == NULL) { 766 free(io_path); 767 SPDK_ERRLOG("Failed to alloc io_path stat.\n"); 768 return NULL; 769 } 770 spdk_bdev_reset_io_stat(io_path->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 771 } 772 773 return io_path; 774 } 775 776 static void 777 nvme_io_path_free(struct nvme_io_path *io_path) 778 { 779 free(io_path->stat); 780 free(io_path); 781 } 782 783 static int 784 _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns) 785 { 786 struct nvme_io_path *io_path; 787 struct spdk_io_channel *ch; 788 struct nvme_ctrlr_channel *ctrlr_ch; 789 struct nvme_qpair *nvme_qpair; 790 791 io_path = nvme_io_path_alloc(); 792 if (io_path == NULL) { 793 return -ENOMEM; 794 } 795 796 io_path->nvme_ns = nvme_ns; 797 798 ch = spdk_get_io_channel(nvme_ns->ctrlr); 799 if (ch == NULL) { 800 nvme_io_path_free(io_path); 801 SPDK_ERRLOG("Failed to alloc io_channel.\n"); 802 return -ENOMEM; 803 } 804 805 ctrlr_ch = spdk_io_channel_get_ctx(ch); 806 807 nvme_qpair = ctrlr_ch->qpair; 808 assert(nvme_qpair != NULL); 809 810 io_path->qpair = nvme_qpair; 811 TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq); 812 813 io_path->nbdev_ch = nbdev_ch; 814 STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq); 815 816 bdev_nvme_clear_current_io_path(nbdev_ch); 817 818 return 0; 819 } 820 821 static void 822 bdev_nvme_clear_retry_io_path(struct nvme_bdev_channel *nbdev_ch, 823 struct nvme_io_path *io_path) 824 { 825 struct nvme_bdev_io *bio; 826 827 TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) { 828 if (bio->io_path == io_path) { 829 bio->io_path = NULL; 830 } 831 } 832 } 833 834 static void 835 _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path) 836 { 837 struct spdk_io_channel *ch; 838 struct nvme_qpair *nvme_qpair; 839 struct nvme_ctrlr_channel *ctrlr_ch; 840 struct nvme_bdev *nbdev; 841 842 nbdev = spdk_io_channel_get_io_device(spdk_io_channel_from_ctx(nbdev_ch)); 843 844 /* Add the statistics to nvme_ns before this path is destroyed. */ 845 pthread_mutex_lock(&nbdev->mutex); 846 if (nbdev->ref != 0 && io_path->nvme_ns->stat != NULL && io_path->stat != NULL) { 847 spdk_bdev_add_io_stat(io_path->nvme_ns->stat, io_path->stat); 848 } 849 pthread_mutex_unlock(&nbdev->mutex); 850 851 bdev_nvme_clear_current_io_path(nbdev_ch); 852 bdev_nvme_clear_retry_io_path(nbdev_ch, io_path); 853 854 STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq); 855 io_path->nbdev_ch = NULL; 856 857 nvme_qpair = io_path->qpair; 858 assert(nvme_qpair != NULL); 859 860 ctrlr_ch = nvme_qpair->ctrlr_ch; 861 assert(ctrlr_ch != NULL); 862 863 ch = spdk_io_channel_from_ctx(ctrlr_ch); 864 spdk_put_io_channel(ch); 865 866 /* After an io_path is removed, I/Os submitted to it may complete and update statistics 867 * of the io_path. To avoid heap-use-after-free error from this case, do not free the 868 * io_path here but free the io_path when the associated qpair is freed. It is ensured 869 * that all I/Os submitted to the io_path are completed when the associated qpair is freed. 870 */ 871 } 872 873 static void 874 _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch) 875 { 876 struct nvme_io_path *io_path, *tmp_io_path; 877 878 STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) { 879 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 880 } 881 } 882 883 static int 884 bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf) 885 { 886 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 887 struct nvme_bdev *nbdev = io_device; 888 struct nvme_ns *nvme_ns; 889 int rc; 890 891 STAILQ_INIT(&nbdev_ch->io_path_list); 892 TAILQ_INIT(&nbdev_ch->retry_io_list); 893 894 pthread_mutex_lock(&nbdev->mutex); 895 896 nbdev_ch->mp_policy = nbdev->mp_policy; 897 nbdev_ch->mp_selector = nbdev->mp_selector; 898 nbdev_ch->rr_min_io = nbdev->rr_min_io; 899 900 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 901 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 902 if (rc != 0) { 903 pthread_mutex_unlock(&nbdev->mutex); 904 905 _bdev_nvme_delete_io_paths(nbdev_ch); 906 return rc; 907 } 908 } 909 pthread_mutex_unlock(&nbdev->mutex); 910 911 return 0; 912 } 913 914 /* If cpl != NULL, complete the bdev_io with nvme status based on 'cpl'. 915 * If cpl == NULL, complete the bdev_io with bdev status based on 'status'. 916 */ 917 static inline void 918 __bdev_nvme_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status, 919 const struct spdk_nvme_cpl *cpl) 920 { 921 spdk_trace_record(TRACE_BDEV_NVME_IO_DONE, 0, 0, (uintptr_t)bdev_io->driver_ctx, 922 (uintptr_t)bdev_io); 923 if (cpl) { 924 spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc); 925 } else { 926 spdk_bdev_io_complete(bdev_io, status); 927 } 928 } 929 930 static void bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch); 931 932 static void 933 bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf) 934 { 935 struct nvme_bdev_channel *nbdev_ch = ctx_buf; 936 937 bdev_nvme_abort_retry_ios(nbdev_ch); 938 _bdev_nvme_delete_io_paths(nbdev_ch); 939 } 940 941 static inline bool 942 bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type) 943 { 944 switch (io_type) { 945 case SPDK_BDEV_IO_TYPE_RESET: 946 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 947 case SPDK_BDEV_IO_TYPE_ABORT: 948 return true; 949 default: 950 break; 951 } 952 953 return false; 954 } 955 956 static inline bool 957 nvme_ns_is_active(struct nvme_ns *nvme_ns) 958 { 959 if (spdk_unlikely(nvme_ns->ana_state_updating)) { 960 return false; 961 } 962 963 if (spdk_unlikely(nvme_ns->ns == NULL)) { 964 return false; 965 } 966 967 return true; 968 } 969 970 static inline bool 971 nvme_ns_is_accessible(struct nvme_ns *nvme_ns) 972 { 973 if (spdk_unlikely(!nvme_ns_is_active(nvme_ns))) { 974 return false; 975 } 976 977 switch (nvme_ns->ana_state) { 978 case SPDK_NVME_ANA_OPTIMIZED_STATE: 979 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 980 return true; 981 default: 982 break; 983 } 984 985 return false; 986 } 987 988 static inline bool 989 nvme_qpair_is_connected(struct nvme_qpair *nvme_qpair) 990 { 991 if (spdk_unlikely(nvme_qpair->qpair == NULL)) { 992 return false; 993 } 994 995 if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 996 SPDK_NVME_QPAIR_FAILURE_NONE)) { 997 return false; 998 } 999 1000 if (spdk_unlikely(nvme_qpair->ctrlr_ch->reset_iter != NULL)) { 1001 return false; 1002 } 1003 1004 return true; 1005 } 1006 1007 static inline bool 1008 nvme_io_path_is_available(struct nvme_io_path *io_path) 1009 { 1010 if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) { 1011 return false; 1012 } 1013 1014 if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) { 1015 return false; 1016 } 1017 1018 return true; 1019 } 1020 1021 static inline bool 1022 nvme_ctrlr_is_failed(struct nvme_ctrlr *nvme_ctrlr) 1023 { 1024 if (nvme_ctrlr->destruct) { 1025 return true; 1026 } 1027 1028 if (nvme_ctrlr->fast_io_fail_timedout) { 1029 return true; 1030 } 1031 1032 if (nvme_ctrlr->resetting) { 1033 if (nvme_ctrlr->opts.reconnect_delay_sec != 0) { 1034 return false; 1035 } else { 1036 return true; 1037 } 1038 } 1039 1040 if (nvme_ctrlr->reconnect_is_delayed) { 1041 return false; 1042 } 1043 1044 if (nvme_ctrlr->disabled) { 1045 return true; 1046 } 1047 1048 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 1049 return true; 1050 } else { 1051 return false; 1052 } 1053 } 1054 1055 static bool 1056 nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr) 1057 { 1058 if (nvme_ctrlr->destruct) { 1059 return false; 1060 } 1061 1062 if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 1063 return false; 1064 } 1065 1066 if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) { 1067 return false; 1068 } 1069 1070 if (nvme_ctrlr->disabled) { 1071 return false; 1072 } 1073 1074 return true; 1075 } 1076 1077 /* Simulate circular linked list. */ 1078 static inline struct nvme_io_path * 1079 nvme_io_path_get_next(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *prev_path) 1080 { 1081 struct nvme_io_path *next_path; 1082 1083 if (prev_path != NULL) { 1084 next_path = STAILQ_NEXT(prev_path, stailq); 1085 if (next_path != NULL) { 1086 return next_path; 1087 } 1088 } 1089 1090 return STAILQ_FIRST(&nbdev_ch->io_path_list); 1091 } 1092 1093 static struct nvme_io_path * 1094 _bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 1095 { 1096 struct nvme_io_path *io_path, *start, *non_optimized = NULL; 1097 1098 start = nvme_io_path_get_next(nbdev_ch, nbdev_ch->current_io_path); 1099 1100 io_path = start; 1101 do { 1102 if (spdk_likely(nvme_io_path_is_available(io_path))) { 1103 switch (io_path->nvme_ns->ana_state) { 1104 case SPDK_NVME_ANA_OPTIMIZED_STATE: 1105 nbdev_ch->current_io_path = io_path; 1106 return io_path; 1107 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 1108 if (non_optimized == NULL) { 1109 non_optimized = io_path; 1110 } 1111 break; 1112 default: 1113 assert(false); 1114 break; 1115 } 1116 } 1117 io_path = nvme_io_path_get_next(nbdev_ch, io_path); 1118 } while (io_path != start); 1119 1120 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 1121 /* We come here only if there is no optimized path. Cache even non_optimized 1122 * path for load balance across multiple non_optimized paths. 1123 */ 1124 nbdev_ch->current_io_path = non_optimized; 1125 } 1126 1127 return non_optimized; 1128 } 1129 1130 static struct nvme_io_path * 1131 _bdev_nvme_find_io_path_min_qd(struct nvme_bdev_channel *nbdev_ch) 1132 { 1133 struct nvme_io_path *io_path; 1134 struct nvme_io_path *optimized = NULL, *non_optimized = NULL; 1135 uint32_t opt_min_qd = UINT32_MAX, non_opt_min_qd = UINT32_MAX; 1136 uint32_t num_outstanding_reqs; 1137 1138 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 1139 if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) { 1140 /* The device is currently resetting. */ 1141 continue; 1142 } 1143 1144 if (spdk_unlikely(!nvme_ns_is_active(io_path->nvme_ns))) { 1145 continue; 1146 } 1147 1148 num_outstanding_reqs = spdk_nvme_qpair_get_num_outstanding_reqs(io_path->qpair->qpair); 1149 switch (io_path->nvme_ns->ana_state) { 1150 case SPDK_NVME_ANA_OPTIMIZED_STATE: 1151 if (num_outstanding_reqs < opt_min_qd) { 1152 opt_min_qd = num_outstanding_reqs; 1153 optimized = io_path; 1154 } 1155 break; 1156 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 1157 if (num_outstanding_reqs < non_opt_min_qd) { 1158 non_opt_min_qd = num_outstanding_reqs; 1159 non_optimized = io_path; 1160 } 1161 break; 1162 default: 1163 break; 1164 } 1165 } 1166 1167 /* don't cache io path for BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH selector */ 1168 if (optimized != NULL) { 1169 return optimized; 1170 } 1171 1172 return non_optimized; 1173 } 1174 1175 static inline struct nvme_io_path * 1176 bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch) 1177 { 1178 if (spdk_likely(nbdev_ch->current_io_path != NULL)) { 1179 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE) { 1180 return nbdev_ch->current_io_path; 1181 } else if (nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 1182 if (++nbdev_ch->rr_counter < nbdev_ch->rr_min_io) { 1183 return nbdev_ch->current_io_path; 1184 } 1185 nbdev_ch->rr_counter = 0; 1186 } 1187 } 1188 1189 if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE || 1190 nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 1191 return _bdev_nvme_find_io_path(nbdev_ch); 1192 } else { 1193 return _bdev_nvme_find_io_path_min_qd(nbdev_ch); 1194 } 1195 } 1196 1197 /* Return true if there is any io_path whose qpair is active or ctrlr is not failed, 1198 * or false otherwise. 1199 * 1200 * If any io_path has an active qpair but find_io_path() returned NULL, its namespace 1201 * is likely to be non-accessible now but may become accessible. 1202 * 1203 * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr 1204 * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed 1205 * when starting to reset it but it is set to failed when the reset failed. Hence, if 1206 * a ctrlr is unfailed, it is likely that it works fine or is resetting. 1207 */ 1208 static bool 1209 any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch) 1210 { 1211 struct nvme_io_path *io_path; 1212 1213 if (nbdev_ch->resetting) { 1214 return false; 1215 } 1216 1217 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 1218 if (io_path->nvme_ns->ana_transition_timedout) { 1219 continue; 1220 } 1221 1222 if (nvme_qpair_is_connected(io_path->qpair) || 1223 !nvme_ctrlr_is_failed(io_path->qpair->ctrlr)) { 1224 return true; 1225 } 1226 } 1227 1228 return false; 1229 } 1230 1231 static void 1232 bdev_nvme_retry_io(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 1233 { 1234 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 1235 struct spdk_io_channel *ch; 1236 1237 if (nbdev_io->io_path != NULL && nvme_io_path_is_available(nbdev_io->io_path)) { 1238 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 1239 } else { 1240 ch = spdk_io_channel_from_ctx(nbdev_ch); 1241 bdev_nvme_submit_request(ch, bdev_io); 1242 } 1243 } 1244 1245 static int 1246 bdev_nvme_retry_ios(void *arg) 1247 { 1248 struct nvme_bdev_channel *nbdev_ch = arg; 1249 struct nvme_bdev_io *bio, *tmp_bio; 1250 uint64_t now, delay_us; 1251 1252 now = spdk_get_ticks(); 1253 1254 TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) { 1255 if (bio->retry_ticks > now) { 1256 break; 1257 } 1258 1259 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1260 1261 bdev_nvme_retry_io(nbdev_ch, spdk_bdev_io_from_ctx(bio)); 1262 } 1263 1264 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1265 1266 bio = TAILQ_FIRST(&nbdev_ch->retry_io_list); 1267 if (bio != NULL) { 1268 delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz(); 1269 1270 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1271 delay_us); 1272 } 1273 1274 return SPDK_POLLER_BUSY; 1275 } 1276 1277 static void 1278 bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch, 1279 struct nvme_bdev_io *bio, uint64_t delay_ms) 1280 { 1281 struct nvme_bdev_io *tmp_bio; 1282 1283 bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL; 1284 1285 TAILQ_FOREACH_REVERSE(tmp_bio, &nbdev_ch->retry_io_list, retry_io_head, retry_link) { 1286 if (tmp_bio->retry_ticks <= bio->retry_ticks) { 1287 TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bio, bio, 1288 retry_link); 1289 return; 1290 } 1291 } 1292 1293 /* No earlier I/Os were found. This I/O must be the new head. */ 1294 TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bio, retry_link); 1295 1296 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1297 1298 nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch, 1299 delay_ms * 1000ULL); 1300 } 1301 1302 static void 1303 bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch) 1304 { 1305 struct nvme_bdev_io *bio, *tmp_bio; 1306 1307 TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) { 1308 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1309 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1310 } 1311 1312 spdk_poller_unregister(&nbdev_ch->retry_io_poller); 1313 } 1314 1315 static int 1316 bdev_nvme_abort_retry_io(struct nvme_bdev_channel *nbdev_ch, 1317 struct nvme_bdev_io *bio_to_abort) 1318 { 1319 struct nvme_bdev_io *bio; 1320 1321 TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) { 1322 if (bio == bio_to_abort) { 1323 TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link); 1324 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL); 1325 return 0; 1326 } 1327 } 1328 1329 return -ENOENT; 1330 } 1331 1332 static void 1333 bdev_nvme_update_nvme_error_stat(struct spdk_bdev_io *bdev_io, const struct spdk_nvme_cpl *cpl) 1334 { 1335 struct nvme_bdev *nbdev; 1336 uint16_t sct, sc; 1337 1338 assert(spdk_nvme_cpl_is_error(cpl)); 1339 1340 nbdev = bdev_io->bdev->ctxt; 1341 1342 if (nbdev->err_stat == NULL) { 1343 return; 1344 } 1345 1346 sct = cpl->status.sct; 1347 sc = cpl->status.sc; 1348 1349 pthread_mutex_lock(&nbdev->mutex); 1350 1351 nbdev->err_stat->status_type[sct]++; 1352 switch (sct) { 1353 case SPDK_NVME_SCT_GENERIC: 1354 case SPDK_NVME_SCT_COMMAND_SPECIFIC: 1355 case SPDK_NVME_SCT_MEDIA_ERROR: 1356 case SPDK_NVME_SCT_PATH: 1357 nbdev->err_stat->status[sct][sc]++; 1358 break; 1359 default: 1360 break; 1361 } 1362 1363 pthread_mutex_unlock(&nbdev->mutex); 1364 } 1365 1366 static inline void 1367 bdev_nvme_update_io_path_stat(struct nvme_bdev_io *bio) 1368 { 1369 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1370 uint64_t num_blocks = bdev_io->u.bdev.num_blocks; 1371 uint32_t blocklen = bdev_io->bdev->blocklen; 1372 struct spdk_bdev_io_stat *stat; 1373 uint64_t tsc_diff; 1374 1375 if (bio->io_path->stat == NULL) { 1376 return; 1377 } 1378 1379 tsc_diff = spdk_get_ticks() - bio->submit_tsc; 1380 stat = bio->io_path->stat; 1381 1382 switch (bdev_io->type) { 1383 case SPDK_BDEV_IO_TYPE_READ: 1384 stat->bytes_read += num_blocks * blocklen; 1385 stat->num_read_ops++; 1386 stat->read_latency_ticks += tsc_diff; 1387 if (stat->max_read_latency_ticks < tsc_diff) { 1388 stat->max_read_latency_ticks = tsc_diff; 1389 } 1390 if (stat->min_read_latency_ticks > tsc_diff) { 1391 stat->min_read_latency_ticks = tsc_diff; 1392 } 1393 break; 1394 case SPDK_BDEV_IO_TYPE_WRITE: 1395 stat->bytes_written += num_blocks * blocklen; 1396 stat->num_write_ops++; 1397 stat->write_latency_ticks += tsc_diff; 1398 if (stat->max_write_latency_ticks < tsc_diff) { 1399 stat->max_write_latency_ticks = tsc_diff; 1400 } 1401 if (stat->min_write_latency_ticks > tsc_diff) { 1402 stat->min_write_latency_ticks = tsc_diff; 1403 } 1404 break; 1405 case SPDK_BDEV_IO_TYPE_UNMAP: 1406 stat->bytes_unmapped += num_blocks * blocklen; 1407 stat->num_unmap_ops++; 1408 stat->unmap_latency_ticks += tsc_diff; 1409 if (stat->max_unmap_latency_ticks < tsc_diff) { 1410 stat->max_unmap_latency_ticks = tsc_diff; 1411 } 1412 if (stat->min_unmap_latency_ticks > tsc_diff) { 1413 stat->min_unmap_latency_ticks = tsc_diff; 1414 } 1415 break; 1416 case SPDK_BDEV_IO_TYPE_ZCOPY: 1417 /* Track the data in the start phase only */ 1418 if (!bdev_io->u.bdev.zcopy.start) { 1419 break; 1420 } 1421 if (bdev_io->u.bdev.zcopy.populate) { 1422 stat->bytes_read += num_blocks * blocklen; 1423 stat->num_read_ops++; 1424 stat->read_latency_ticks += tsc_diff; 1425 if (stat->max_read_latency_ticks < tsc_diff) { 1426 stat->max_read_latency_ticks = tsc_diff; 1427 } 1428 if (stat->min_read_latency_ticks > tsc_diff) { 1429 stat->min_read_latency_ticks = tsc_diff; 1430 } 1431 } else { 1432 stat->bytes_written += num_blocks * blocklen; 1433 stat->num_write_ops++; 1434 stat->write_latency_ticks += tsc_diff; 1435 if (stat->max_write_latency_ticks < tsc_diff) { 1436 stat->max_write_latency_ticks = tsc_diff; 1437 } 1438 if (stat->min_write_latency_ticks > tsc_diff) { 1439 stat->min_write_latency_ticks = tsc_diff; 1440 } 1441 } 1442 break; 1443 case SPDK_BDEV_IO_TYPE_COPY: 1444 stat->bytes_copied += num_blocks * blocklen; 1445 stat->num_copy_ops++; 1446 stat->copy_latency_ticks += tsc_diff; 1447 if (stat->max_copy_latency_ticks < tsc_diff) { 1448 stat->max_copy_latency_ticks = tsc_diff; 1449 } 1450 if (stat->min_copy_latency_ticks > tsc_diff) { 1451 stat->min_copy_latency_ticks = tsc_diff; 1452 } 1453 break; 1454 default: 1455 break; 1456 } 1457 } 1458 1459 static bool 1460 bdev_nvme_check_retry_io(struct nvme_bdev_io *bio, 1461 const struct spdk_nvme_cpl *cpl, 1462 struct nvme_bdev_channel *nbdev_ch, 1463 uint64_t *_delay_ms) 1464 { 1465 struct nvme_io_path *io_path = bio->io_path; 1466 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 1467 const struct spdk_nvme_ctrlr_data *cdata; 1468 1469 if (spdk_nvme_cpl_is_path_error(cpl) || 1470 spdk_nvme_cpl_is_aborted_sq_deletion(cpl) || 1471 !nvme_io_path_is_available(io_path) || 1472 !nvme_ctrlr_is_available(nvme_ctrlr)) { 1473 bdev_nvme_clear_current_io_path(nbdev_ch); 1474 bio->io_path = NULL; 1475 if (spdk_nvme_cpl_is_ana_error(cpl)) { 1476 if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) { 1477 io_path->nvme_ns->ana_state_updating = true; 1478 } 1479 } 1480 if (!any_io_path_may_become_available(nbdev_ch)) { 1481 return false; 1482 } 1483 *_delay_ms = 0; 1484 } else { 1485 bio->retry_count++; 1486 1487 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 1488 1489 if (cpl->status.crd != 0) { 1490 *_delay_ms = cdata->crdt[cpl->status.crd] * 100; 1491 } else { 1492 *_delay_ms = 0; 1493 } 1494 } 1495 1496 return true; 1497 } 1498 1499 static inline void 1500 bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio, 1501 const struct spdk_nvme_cpl *cpl) 1502 { 1503 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1504 struct nvme_bdev_channel *nbdev_ch; 1505 uint64_t delay_ms; 1506 1507 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 1508 1509 if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) { 1510 bdev_nvme_update_io_path_stat(bio); 1511 goto complete; 1512 } 1513 1514 /* Update error counts before deciding if retry is needed. 1515 * Hence, error counts may be more than the number of I/O errors. 1516 */ 1517 bdev_nvme_update_nvme_error_stat(bdev_io, cpl); 1518 1519 if (cpl->status.dnr != 0 || spdk_nvme_cpl_is_aborted_by_request(cpl) || 1520 (g_opts.bdev_retry_count != -1 && bio->retry_count >= g_opts.bdev_retry_count)) { 1521 goto complete; 1522 } 1523 1524 /* At this point we don't know whether the sequence was successfully executed or not, so we 1525 * cannot retry the IO */ 1526 if (bdev_io->u.bdev.accel_sequence != NULL) { 1527 goto complete; 1528 } 1529 1530 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1531 1532 if (bdev_nvme_check_retry_io(bio, cpl, nbdev_ch, &delay_ms)) { 1533 bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms); 1534 return; 1535 } 1536 1537 complete: 1538 bio->retry_count = 0; 1539 bio->submit_tsc = 0; 1540 bdev_io->u.bdev.accel_sequence = NULL; 1541 __bdev_nvme_io_complete(bdev_io, 0, cpl); 1542 } 1543 1544 static inline void 1545 bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc) 1546 { 1547 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1548 struct nvme_bdev_channel *nbdev_ch; 1549 enum spdk_bdev_io_status io_status; 1550 1551 assert(!bdev_nvme_io_type_is_admin(bdev_io->type)); 1552 1553 switch (rc) { 1554 case 0: 1555 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1556 break; 1557 case -ENOMEM: 1558 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1559 break; 1560 case -ENXIO: 1561 if (g_opts.bdev_retry_count == -1 || bio->retry_count < g_opts.bdev_retry_count) { 1562 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 1563 1564 bdev_nvme_clear_current_io_path(nbdev_ch); 1565 bio->io_path = NULL; 1566 1567 if (any_io_path_may_become_available(nbdev_ch)) { 1568 bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL); 1569 return; 1570 } 1571 } 1572 1573 /* fallthrough */ 1574 default: 1575 spdk_accel_sequence_abort(bdev_io->u.bdev.accel_sequence); 1576 bdev_io->u.bdev.accel_sequence = NULL; 1577 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1578 break; 1579 } 1580 1581 bio->retry_count = 0; 1582 bio->submit_tsc = 0; 1583 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1584 } 1585 1586 static inline void 1587 bdev_nvme_admin_complete(struct nvme_bdev_io *bio, int rc) 1588 { 1589 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 1590 enum spdk_bdev_io_status io_status; 1591 1592 switch (rc) { 1593 case 0: 1594 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1595 break; 1596 case -ENOMEM: 1597 io_status = SPDK_BDEV_IO_STATUS_NOMEM; 1598 break; 1599 case -ENXIO: 1600 /* fallthrough */ 1601 default: 1602 io_status = SPDK_BDEV_IO_STATUS_FAILED; 1603 break; 1604 } 1605 1606 __bdev_nvme_io_complete(bdev_io, io_status, NULL); 1607 } 1608 1609 static void 1610 bdev_nvme_clear_io_path_caches_done(struct nvme_ctrlr *nvme_ctrlr, 1611 void *ctx, int status) 1612 { 1613 pthread_mutex_lock(&nvme_ctrlr->mutex); 1614 1615 assert(nvme_ctrlr->io_path_cache_clearing == true); 1616 nvme_ctrlr->io_path_cache_clearing = false; 1617 1618 if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 1619 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1620 return; 1621 } 1622 1623 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1624 1625 nvme_ctrlr_unregister(nvme_ctrlr); 1626 } 1627 1628 static void 1629 _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair) 1630 { 1631 struct nvme_io_path *io_path; 1632 1633 TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) { 1634 if (io_path->nbdev_ch == NULL) { 1635 continue; 1636 } 1637 bdev_nvme_clear_current_io_path(io_path->nbdev_ch); 1638 } 1639 } 1640 1641 static void 1642 bdev_nvme_clear_io_path_cache(struct nvme_ctrlr_channel_iter *i, 1643 struct nvme_ctrlr *nvme_ctrlr, 1644 struct nvme_ctrlr_channel *ctrlr_ch, 1645 void *ctx) 1646 { 1647 assert(ctrlr_ch->qpair != NULL); 1648 1649 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 1650 1651 nvme_ctrlr_for_each_channel_continue(i, 0); 1652 } 1653 1654 static void 1655 bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr) 1656 { 1657 pthread_mutex_lock(&nvme_ctrlr->mutex); 1658 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 1659 nvme_ctrlr->io_path_cache_clearing) { 1660 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1661 return; 1662 } 1663 1664 nvme_ctrlr->io_path_cache_clearing = true; 1665 pthread_mutex_unlock(&nvme_ctrlr->mutex); 1666 1667 nvme_ctrlr_for_each_channel(nvme_ctrlr, 1668 bdev_nvme_clear_io_path_cache, 1669 NULL, 1670 bdev_nvme_clear_io_path_caches_done); 1671 } 1672 1673 static struct nvme_qpair * 1674 nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair) 1675 { 1676 struct nvme_qpair *nvme_qpair; 1677 1678 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1679 if (nvme_qpair->qpair == qpair) { 1680 break; 1681 } 1682 } 1683 1684 return nvme_qpair; 1685 } 1686 1687 static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair); 1688 1689 static void 1690 bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx) 1691 { 1692 struct nvme_poll_group *group = poll_group_ctx; 1693 struct nvme_qpair *nvme_qpair; 1694 struct nvme_ctrlr_channel *ctrlr_ch; 1695 int status; 1696 1697 nvme_qpair = nvme_poll_group_get_qpair(group, qpair); 1698 if (nvme_qpair == NULL) { 1699 return; 1700 } 1701 1702 if (nvme_qpair->qpair != NULL) { 1703 spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair); 1704 nvme_qpair->qpair = NULL; 1705 } 1706 1707 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1708 1709 ctrlr_ch = nvme_qpair->ctrlr_ch; 1710 1711 if (ctrlr_ch != NULL) { 1712 if (ctrlr_ch->reset_iter != NULL) { 1713 /* We are in a full reset sequence. */ 1714 if (ctrlr_ch->connect_poller != NULL) { 1715 /* qpair was failed to connect. Abort the reset sequence. */ 1716 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was failed to connect. abort the reset ctrlr sequence.\n", 1717 qpair); 1718 spdk_poller_unregister(&ctrlr_ch->connect_poller); 1719 status = -1; 1720 } else { 1721 /* qpair was completed to disconnect. Just move to the next ctrlr_channel. */ 1722 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed in a reset ctrlr sequence.\n", 1723 qpair); 1724 status = 0; 1725 } 1726 nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, status); 1727 ctrlr_ch->reset_iter = NULL; 1728 } else { 1729 /* qpair was disconnected unexpectedly. Reset controller for recovery. */ 1730 SPDK_NOTICELOG("qpair %p was disconnected and freed. reset controller.\n", qpair); 1731 bdev_nvme_failover_ctrlr(nvme_qpair->ctrlr); 1732 } 1733 } else { 1734 /* In this case, ctrlr_channel is already deleted. */ 1735 SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed. delete nvme_qpair.\n", qpair); 1736 nvme_qpair_delete(nvme_qpair); 1737 } 1738 } 1739 1740 static void 1741 bdev_nvme_check_io_qpairs(struct nvme_poll_group *group) 1742 { 1743 struct nvme_qpair *nvme_qpair; 1744 1745 TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) { 1746 if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) { 1747 continue; 1748 } 1749 1750 if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) != 1751 SPDK_NVME_QPAIR_FAILURE_NONE) { 1752 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1753 } 1754 } 1755 } 1756 1757 static int 1758 bdev_nvme_poll(void *arg) 1759 { 1760 struct nvme_poll_group *group = arg; 1761 int64_t num_completions; 1762 1763 if (group->collect_spin_stat && group->start_ticks == 0) { 1764 group->start_ticks = spdk_get_ticks(); 1765 } 1766 1767 num_completions = spdk_nvme_poll_group_process_completions(group->group, 0, 1768 bdev_nvme_disconnected_qpair_cb); 1769 if (group->collect_spin_stat) { 1770 if (num_completions > 0) { 1771 if (group->end_ticks != 0) { 1772 group->spin_ticks += (group->end_ticks - group->start_ticks); 1773 group->end_ticks = 0; 1774 } 1775 group->start_ticks = 0; 1776 } else { 1777 group->end_ticks = spdk_get_ticks(); 1778 } 1779 } 1780 1781 if (spdk_unlikely(num_completions < 0)) { 1782 bdev_nvme_check_io_qpairs(group); 1783 } 1784 1785 return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 1786 } 1787 1788 static int bdev_nvme_poll_adminq(void *arg); 1789 1790 static void 1791 bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_period_us) 1792 { 1793 spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller); 1794 1795 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, 1796 nvme_ctrlr, new_period_us); 1797 } 1798 1799 static int 1800 bdev_nvme_poll_adminq(void *arg) 1801 { 1802 int32_t rc; 1803 struct nvme_ctrlr *nvme_ctrlr = arg; 1804 nvme_ctrlr_disconnected_cb disconnected_cb; 1805 1806 assert(nvme_ctrlr != NULL); 1807 1808 rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr); 1809 if (rc < 0) { 1810 disconnected_cb = nvme_ctrlr->disconnected_cb; 1811 nvme_ctrlr->disconnected_cb = NULL; 1812 1813 if (disconnected_cb != NULL) { 1814 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 1815 g_opts.nvme_adminq_poll_period_us); 1816 disconnected_cb(nvme_ctrlr); 1817 } else { 1818 bdev_nvme_failover_ctrlr(nvme_ctrlr); 1819 } 1820 } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) != 1821 SPDK_NVME_QPAIR_FAILURE_NONE) { 1822 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 1823 } 1824 1825 return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY; 1826 } 1827 1828 static void 1829 nvme_bdev_free(void *io_device) 1830 { 1831 struct nvme_bdev *nvme_disk = io_device; 1832 1833 pthread_mutex_destroy(&nvme_disk->mutex); 1834 free(nvme_disk->disk.name); 1835 free(nvme_disk->err_stat); 1836 free(nvme_disk); 1837 } 1838 1839 static int 1840 bdev_nvme_destruct(void *ctx) 1841 { 1842 struct nvme_bdev *nvme_disk = ctx; 1843 struct nvme_ns *nvme_ns, *tmp_nvme_ns; 1844 1845 SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid); 1846 1847 TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) { 1848 pthread_mutex_lock(&nvme_ns->ctrlr->mutex); 1849 1850 nvme_ns->bdev = NULL; 1851 1852 assert(nvme_ns->id > 0); 1853 1854 if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) { 1855 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1856 1857 nvme_ctrlr_release(nvme_ns->ctrlr); 1858 nvme_ns_free(nvme_ns); 1859 } else { 1860 pthread_mutex_unlock(&nvme_ns->ctrlr->mutex); 1861 } 1862 } 1863 1864 pthread_mutex_lock(&g_bdev_nvme_mutex); 1865 TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq); 1866 pthread_mutex_unlock(&g_bdev_nvme_mutex); 1867 1868 spdk_io_device_unregister(nvme_disk, nvme_bdev_free); 1869 1870 return 0; 1871 } 1872 1873 static int 1874 bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair) 1875 { 1876 struct nvme_ctrlr *nvme_ctrlr; 1877 struct spdk_nvme_io_qpair_opts opts; 1878 struct spdk_nvme_qpair *qpair; 1879 int rc; 1880 1881 nvme_ctrlr = nvme_qpair->ctrlr; 1882 1883 spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1884 opts.delay_cmd_submit = g_opts.delay_cmd_submit; 1885 opts.create_only = true; 1886 opts.async_mode = true; 1887 opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests); 1888 g_opts.io_queue_requests = opts.io_queue_requests; 1889 1890 qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts)); 1891 if (qpair == NULL) { 1892 return -1; 1893 } 1894 1895 SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name, 1896 spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread)); 1897 1898 assert(nvme_qpair->group != NULL); 1899 1900 rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair); 1901 if (rc != 0) { 1902 SPDK_ERRLOG("Unable to begin polling on NVMe Channel.\n"); 1903 goto err; 1904 } 1905 1906 rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair); 1907 if (rc != 0) { 1908 SPDK_ERRLOG("Unable to connect I/O qpair.\n"); 1909 goto err; 1910 } 1911 1912 nvme_qpair->qpair = qpair; 1913 1914 if (!g_opts.disable_auto_failback) { 1915 _bdev_nvme_clear_io_path_cache(nvme_qpair); 1916 } 1917 1918 return 0; 1919 1920 err: 1921 spdk_nvme_ctrlr_free_io_qpair(qpair); 1922 1923 return rc; 1924 } 1925 1926 static void bdev_nvme_reset_io_continue(void *cb_arg, int rc); 1927 1928 static void 1929 bdev_nvme_complete_pending_resets(struct nvme_ctrlr_channel_iter *i, 1930 struct nvme_ctrlr *nvme_ctrlr, 1931 struct nvme_ctrlr_channel *ctrlr_ch, 1932 void *ctx) 1933 { 1934 int rc = 0; 1935 struct nvme_bdev_io *bio; 1936 1937 if (ctx != NULL) { 1938 rc = -1; 1939 } 1940 1941 while (!TAILQ_EMPTY(&ctrlr_ch->pending_resets)) { 1942 bio = TAILQ_FIRST(&ctrlr_ch->pending_resets); 1943 TAILQ_REMOVE(&ctrlr_ch->pending_resets, bio, retry_link); 1944 1945 bdev_nvme_reset_io_continue(bio, rc); 1946 } 1947 1948 nvme_ctrlr_for_each_channel_continue(i, 0); 1949 } 1950 1951 /* This function marks the current trid as failed by storing the current ticks 1952 * and then sets the next trid to the active trid within a controller if exists. 1953 * 1954 * The purpose of the boolean return value is to request the caller to disconnect 1955 * the current trid now to try connecting the next trid. 1956 */ 1957 static bool 1958 bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove, bool start) 1959 { 1960 struct nvme_path_id *path_id, *next_path; 1961 int rc __attribute__((unused)); 1962 1963 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 1964 assert(path_id); 1965 assert(path_id == nvme_ctrlr->active_path_id); 1966 next_path = TAILQ_NEXT(path_id, link); 1967 1968 /* Update the last failed time. It means the trid is failed if its last 1969 * failed time is non-zero. 1970 */ 1971 path_id->last_failed_tsc = spdk_get_ticks(); 1972 1973 if (next_path == NULL) { 1974 /* There is no alternate trid within a controller. */ 1975 return false; 1976 } 1977 1978 if (!start && nvme_ctrlr->opts.reconnect_delay_sec == 0) { 1979 /* Connect is not retried in a controller reset sequence. Connecting 1980 * the next trid will be done by the next bdev_nvme_failover_ctrlr() call. 1981 */ 1982 return false; 1983 } 1984 1985 assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE); 1986 1987 SPDK_NOTICELOG("Start failover from %s:%s to %s:%s\n", path_id->trid.traddr, 1988 path_id->trid.trsvcid, next_path->trid.traddr, next_path->trid.trsvcid); 1989 1990 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 1991 nvme_ctrlr->active_path_id = next_path; 1992 rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid); 1993 assert(rc == 0); 1994 TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link); 1995 if (!remove) { 1996 /** Shuffle the old trid to the end of the list and use the new one. 1997 * Allows for round robin through multiple connections. 1998 */ 1999 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link); 2000 } else { 2001 free(path_id); 2002 } 2003 2004 if (start || next_path->last_failed_tsc == 0) { 2005 /* bdev_nvme_failover_ctrlr() is just called or the next trid is not failed 2006 * or used yet. Try the next trid now. 2007 */ 2008 return true; 2009 } 2010 2011 if (spdk_get_ticks() > next_path->last_failed_tsc + spdk_get_ticks_hz() * 2012 nvme_ctrlr->opts.reconnect_delay_sec) { 2013 /* Enough backoff passed since the next trid failed. Try the next trid now. */ 2014 return true; 2015 } 2016 2017 /* The next trid will be tried after reconnect_delay_sec seconds. */ 2018 return false; 2019 } 2020 2021 static bool 2022 bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr) 2023 { 2024 int32_t elapsed; 2025 2026 if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 || 2027 nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) { 2028 return false; 2029 } 2030 2031 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 2032 if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) { 2033 return true; 2034 } else { 2035 return false; 2036 } 2037 } 2038 2039 static bool 2040 bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr) 2041 { 2042 uint32_t elapsed; 2043 2044 if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) { 2045 return false; 2046 } 2047 2048 elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz(); 2049 if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) { 2050 return true; 2051 } else { 2052 return false; 2053 } 2054 } 2055 2056 static void bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success); 2057 2058 static void 2059 nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb cb_fn) 2060 { 2061 int rc; 2062 2063 rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr); 2064 if (rc != 0) { 2065 /* Disconnect fails if ctrlr is already resetting or removed. In this case, 2066 * fail the reset sequence immediately. 2067 */ 2068 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2069 return; 2070 } 2071 2072 /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq. 2073 * Set callback here to execute the specified operation after ctrlr is really disconnected. 2074 */ 2075 assert(nvme_ctrlr->disconnected_cb == NULL); 2076 nvme_ctrlr->disconnected_cb = cb_fn; 2077 2078 /* During disconnection, reduce the period to poll adminq more often. */ 2079 bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0); 2080 } 2081 2082 enum bdev_nvme_op_after_reset { 2083 OP_NONE, 2084 OP_COMPLETE_PENDING_DESTRUCT, 2085 OP_DESTRUCT, 2086 OP_DELAYED_RECONNECT, 2087 OP_FAILOVER, 2088 }; 2089 2090 typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset; 2091 2092 static _bdev_nvme_op_after_reset 2093 bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success) 2094 { 2095 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 2096 /* Complete pending destruct after reset completes. */ 2097 return OP_COMPLETE_PENDING_DESTRUCT; 2098 } else if (nvme_ctrlr->pending_failover) { 2099 nvme_ctrlr->pending_failover = false; 2100 nvme_ctrlr->reset_start_tsc = 0; 2101 return OP_FAILOVER; 2102 } else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) { 2103 nvme_ctrlr->reset_start_tsc = 0; 2104 return OP_NONE; 2105 } else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 2106 return OP_DESTRUCT; 2107 } else { 2108 if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) { 2109 nvme_ctrlr->fast_io_fail_timedout = true; 2110 } 2111 return OP_DELAYED_RECONNECT; 2112 } 2113 } 2114 2115 static int bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug); 2116 static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr); 2117 2118 static int 2119 bdev_nvme_reconnect_delay_timer_expired(void *ctx) 2120 { 2121 struct nvme_ctrlr *nvme_ctrlr = ctx; 2122 2123 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect_delay, nvme_ctrlr->nbdev_ctrlr->name); 2124 pthread_mutex_lock(&nvme_ctrlr->mutex); 2125 2126 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2127 2128 if (!nvme_ctrlr->reconnect_is_delayed) { 2129 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2130 return SPDK_POLLER_BUSY; 2131 } 2132 2133 nvme_ctrlr->reconnect_is_delayed = false; 2134 2135 if (nvme_ctrlr->destruct) { 2136 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2137 return SPDK_POLLER_BUSY; 2138 } 2139 2140 assert(nvme_ctrlr->resetting == false); 2141 nvme_ctrlr->resetting = true; 2142 2143 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2144 2145 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 2146 2147 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2148 return SPDK_POLLER_BUSY; 2149 } 2150 2151 static void 2152 bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr) 2153 { 2154 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 2155 2156 assert(nvme_ctrlr->reconnect_is_delayed == false); 2157 nvme_ctrlr->reconnect_is_delayed = true; 2158 2159 assert(nvme_ctrlr->reconnect_delay_timer == NULL); 2160 nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired, 2161 nvme_ctrlr, 2162 nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC); 2163 } 2164 2165 static void remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr); 2166 2167 static void 2168 _bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2169 { 2170 bool success = (ctx == NULL); 2171 bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn; 2172 void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg; 2173 enum bdev_nvme_op_after_reset op_after_reset; 2174 2175 assert(nvme_ctrlr->thread == spdk_get_thread()); 2176 2177 nvme_ctrlr->ctrlr_op_cb_fn = NULL; 2178 nvme_ctrlr->ctrlr_op_cb_arg = NULL; 2179 2180 if (!success) { 2181 SPDK_ERRLOG("Resetting controller failed.\n"); 2182 } else { 2183 SPDK_NOTICELOG("Resetting controller successful.\n"); 2184 } 2185 2186 pthread_mutex_lock(&nvme_ctrlr->mutex); 2187 nvme_ctrlr->resetting = false; 2188 nvme_ctrlr->dont_retry = false; 2189 nvme_ctrlr->in_failover = false; 2190 2191 op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success); 2192 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2193 2194 /* Delay callbacks when the next operation is a failover. */ 2195 if (ctrlr_op_cb_fn && op_after_reset != OP_FAILOVER) { 2196 ctrlr_op_cb_fn(ctrlr_op_cb_arg, success ? 0 : -1); 2197 } 2198 2199 switch (op_after_reset) { 2200 case OP_COMPLETE_PENDING_DESTRUCT: 2201 nvme_ctrlr_unregister(nvme_ctrlr); 2202 break; 2203 case OP_DESTRUCT: 2204 bdev_nvme_delete_ctrlr(nvme_ctrlr, false); 2205 remove_discovery_entry(nvme_ctrlr); 2206 break; 2207 case OP_DELAYED_RECONNECT: 2208 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_start_reconnect_delay_timer); 2209 break; 2210 case OP_FAILOVER: 2211 nvme_ctrlr->ctrlr_op_cb_fn = ctrlr_op_cb_fn; 2212 nvme_ctrlr->ctrlr_op_cb_arg = ctrlr_op_cb_arg; 2213 bdev_nvme_failover_ctrlr(nvme_ctrlr); 2214 break; 2215 default: 2216 break; 2217 } 2218 } 2219 2220 static void 2221 bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success) 2222 { 2223 pthread_mutex_lock(&nvme_ctrlr->mutex); 2224 if (!success) { 2225 /* Connecting the active trid failed. Set the next alternate trid to the 2226 * active trid if it exists. 2227 */ 2228 if (bdev_nvme_failover_trid(nvme_ctrlr, false, false)) { 2229 /* The next alternate trid exists and is ready to try. Try it now. */ 2230 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2231 2232 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); 2233 return; 2234 } 2235 2236 /* We came here if there is no alternate trid or if the next trid exists but 2237 * is not ready to try. We will try the active trid after reconnect_delay_sec 2238 * seconds if it is non-zero or at the next reset call otherwise. 2239 */ 2240 } else { 2241 /* Connecting the active trid succeeded. Clear the last failed time because it 2242 * means the trid is failed if its last failed time is non-zero. 2243 */ 2244 nvme_ctrlr->active_path_id->last_failed_tsc = 0; 2245 } 2246 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2247 2248 /* Make sure we clear any pending resets before returning. */ 2249 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2250 bdev_nvme_complete_pending_resets, 2251 success ? NULL : (void *)0x1, 2252 _bdev_nvme_reset_ctrlr_complete); 2253 } 2254 2255 static void 2256 bdev_nvme_reset_create_qpairs_failed(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2257 { 2258 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2259 } 2260 2261 static void 2262 bdev_nvme_reset_destroy_qpair(struct nvme_ctrlr_channel_iter *i, 2263 struct nvme_ctrlr *nvme_ctrlr, 2264 struct nvme_ctrlr_channel *ctrlr_ch, void *ctx) 2265 { 2266 struct nvme_qpair *nvme_qpair; 2267 2268 nvme_qpair = ctrlr_ch->qpair; 2269 assert(nvme_qpair != NULL); 2270 2271 _bdev_nvme_clear_io_path_cache(nvme_qpair); 2272 2273 if (nvme_qpair->qpair != NULL) { 2274 if (nvme_qpair->ctrlr->dont_retry) { 2275 spdk_nvme_qpair_set_abort_dnr(nvme_qpair->qpair, true); 2276 } 2277 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 2278 2279 /* The current full reset sequence will move to the next 2280 * ctrlr_channel after the qpair is actually disconnected. 2281 */ 2282 assert(ctrlr_ch->reset_iter == NULL); 2283 ctrlr_ch->reset_iter = i; 2284 } else { 2285 nvme_ctrlr_for_each_channel_continue(i, 0); 2286 } 2287 } 2288 2289 static void 2290 bdev_nvme_reset_create_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2291 { 2292 if (status == 0) { 2293 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, true); 2294 } else { 2295 /* Delete the added qpairs and quiesce ctrlr to make the states clean. */ 2296 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2297 bdev_nvme_reset_destroy_qpair, 2298 NULL, 2299 bdev_nvme_reset_create_qpairs_failed); 2300 } 2301 } 2302 2303 static int 2304 bdev_nvme_reset_check_qpair_connected(void *ctx) 2305 { 2306 struct nvme_ctrlr_channel *ctrlr_ch = ctx; 2307 2308 if (ctrlr_ch->reset_iter == NULL) { 2309 /* qpair was already failed to connect and the reset sequence is being aborted. */ 2310 assert(ctrlr_ch->connect_poller == NULL); 2311 assert(ctrlr_ch->qpair->qpair == NULL); 2312 return SPDK_POLLER_BUSY; 2313 } 2314 2315 assert(ctrlr_ch->qpair->qpair != NULL); 2316 2317 if (!spdk_nvme_qpair_is_connected(ctrlr_ch->qpair->qpair)) { 2318 return SPDK_POLLER_BUSY; 2319 } 2320 2321 spdk_poller_unregister(&ctrlr_ch->connect_poller); 2322 2323 /* qpair was completed to connect. Move to the next ctrlr_channel */ 2324 nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 2325 ctrlr_ch->reset_iter = NULL; 2326 2327 if (!g_opts.disable_auto_failback) { 2328 _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair); 2329 } 2330 2331 return SPDK_POLLER_BUSY; 2332 } 2333 2334 static void 2335 bdev_nvme_reset_create_qpair(struct nvme_ctrlr_channel_iter *i, 2336 struct nvme_ctrlr *nvme_ctrlr, 2337 struct nvme_ctrlr_channel *ctrlr_ch, 2338 void *ctx) 2339 { 2340 int rc; 2341 2342 rc = bdev_nvme_create_qpair(ctrlr_ch->qpair); 2343 if (rc == 0) { 2344 ctrlr_ch->connect_poller = SPDK_POLLER_REGISTER(bdev_nvme_reset_check_qpair_connected, 2345 ctrlr_ch, 0); 2346 2347 /* The current full reset sequence will move to the next 2348 * ctrlr_channel after the qpair is actually connected. 2349 */ 2350 assert(ctrlr_ch->reset_iter == NULL); 2351 ctrlr_ch->reset_iter = i; 2352 } else { 2353 nvme_ctrlr_for_each_channel_continue(i, rc); 2354 } 2355 } 2356 2357 static void 2358 nvme_ctrlr_check_namespaces(struct nvme_ctrlr *nvme_ctrlr) 2359 { 2360 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 2361 struct nvme_ns *nvme_ns; 2362 2363 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 2364 nvme_ns != NULL; 2365 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 2366 if (!spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 2367 SPDK_DEBUGLOG(bdev_nvme, "NSID %u was removed during reset.\n", nvme_ns->id); 2368 /* NS can be added again. Just nullify nvme_ns->ns. */ 2369 nvme_ns->ns = NULL; 2370 } 2371 } 2372 } 2373 2374 2375 static int 2376 bdev_nvme_reconnect_ctrlr_poll(void *arg) 2377 { 2378 struct nvme_ctrlr *nvme_ctrlr = arg; 2379 int rc = -ETIMEDOUT; 2380 2381 if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) { 2382 /* Mark the ctrlr as failed. The next call to 2383 * spdk_nvme_ctrlr_reconnect_poll_async() will then 2384 * do the necessary cleanup and return failure. 2385 */ 2386 spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr); 2387 } 2388 2389 rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr); 2390 if (rc == -EAGAIN) { 2391 return SPDK_POLLER_BUSY; 2392 } 2393 2394 spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller); 2395 if (rc == 0) { 2396 nvme_ctrlr_check_namespaces(nvme_ctrlr); 2397 2398 /* Recreate all of the I/O queue pairs */ 2399 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2400 bdev_nvme_reset_create_qpair, 2401 NULL, 2402 bdev_nvme_reset_create_qpairs_done); 2403 } else { 2404 bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false); 2405 } 2406 return SPDK_POLLER_BUSY; 2407 } 2408 2409 static void 2410 bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2411 { 2412 spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr); 2413 2414 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect, nvme_ctrlr->nbdev_ctrlr->name); 2415 assert(nvme_ctrlr->reset_detach_poller == NULL); 2416 nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll, 2417 nvme_ctrlr, 0); 2418 } 2419 2420 static void 2421 bdev_nvme_reset_destroy_qpair_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2422 { 2423 SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reset, nvme_ctrlr->nbdev_ctrlr->name); 2424 assert(status == 0); 2425 2426 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2427 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2428 } else { 2429 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); 2430 } 2431 } 2432 2433 static void 2434 bdev_nvme_reset_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr) 2435 { 2436 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2437 bdev_nvme_reset_destroy_qpair, 2438 NULL, 2439 bdev_nvme_reset_destroy_qpair_done); 2440 } 2441 2442 static void 2443 bdev_nvme_reconnect_ctrlr_now(void *ctx) 2444 { 2445 struct nvme_ctrlr *nvme_ctrlr = ctx; 2446 2447 assert(nvme_ctrlr->resetting == true); 2448 assert(nvme_ctrlr->thread == spdk_get_thread()); 2449 2450 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2451 2452 spdk_poller_resume(nvme_ctrlr->adminq_timer_poller); 2453 2454 bdev_nvme_reconnect_ctrlr(nvme_ctrlr); 2455 } 2456 2457 static void 2458 _bdev_nvme_reset_ctrlr(void *ctx) 2459 { 2460 struct nvme_ctrlr *nvme_ctrlr = ctx; 2461 2462 assert(nvme_ctrlr->resetting == true); 2463 assert(nvme_ctrlr->thread == spdk_get_thread()); 2464 2465 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2466 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs); 2467 } else { 2468 bdev_nvme_reset_destroy_qpairs(nvme_ctrlr); 2469 } 2470 } 2471 2472 static int 2473 bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2474 { 2475 spdk_msg_fn msg_fn; 2476 2477 pthread_mutex_lock(&nvme_ctrlr->mutex); 2478 if (nvme_ctrlr->destruct) { 2479 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2480 return -ENXIO; 2481 } 2482 2483 if (nvme_ctrlr->resetting) { 2484 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2485 SPDK_NOTICELOG("Unable to perform reset, already in progress.\n"); 2486 return -EBUSY; 2487 } 2488 2489 if (nvme_ctrlr->disabled) { 2490 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2491 SPDK_NOTICELOG("Unable to perform reset. Controller is disabled.\n"); 2492 return -EALREADY; 2493 } 2494 2495 nvme_ctrlr->resetting = true; 2496 nvme_ctrlr->dont_retry = true; 2497 2498 if (nvme_ctrlr->reconnect_is_delayed) { 2499 SPDK_DEBUGLOG(bdev_nvme, "Reconnect is already scheduled.\n"); 2500 msg_fn = bdev_nvme_reconnect_ctrlr_now; 2501 nvme_ctrlr->reconnect_is_delayed = false; 2502 } else { 2503 msg_fn = _bdev_nvme_reset_ctrlr; 2504 assert(nvme_ctrlr->reset_start_tsc == 0); 2505 } 2506 2507 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2508 2509 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2510 2511 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 2512 return 0; 2513 } 2514 2515 static int 2516 bdev_nvme_enable_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2517 { 2518 pthread_mutex_lock(&nvme_ctrlr->mutex); 2519 if (nvme_ctrlr->destruct) { 2520 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2521 return -ENXIO; 2522 } 2523 2524 if (nvme_ctrlr->resetting) { 2525 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2526 return -EBUSY; 2527 } 2528 2529 if (!nvme_ctrlr->disabled) { 2530 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2531 return -EALREADY; 2532 } 2533 2534 nvme_ctrlr->disabled = false; 2535 nvme_ctrlr->resetting = true; 2536 2537 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2538 2539 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2540 2541 spdk_thread_send_msg(nvme_ctrlr->thread, bdev_nvme_reconnect_ctrlr_now, nvme_ctrlr); 2542 return 0; 2543 } 2544 2545 static void 2546 _bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2547 { 2548 bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn; 2549 void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg; 2550 enum bdev_nvme_op_after_reset op_after_disable; 2551 2552 assert(nvme_ctrlr->thread == spdk_get_thread()); 2553 2554 nvme_ctrlr->ctrlr_op_cb_fn = NULL; 2555 nvme_ctrlr->ctrlr_op_cb_arg = NULL; 2556 2557 pthread_mutex_lock(&nvme_ctrlr->mutex); 2558 2559 nvme_ctrlr->resetting = false; 2560 nvme_ctrlr->dont_retry = false; 2561 2562 op_after_disable = bdev_nvme_check_op_after_reset(nvme_ctrlr, true); 2563 2564 nvme_ctrlr->disabled = true; 2565 spdk_poller_pause(nvme_ctrlr->adminq_timer_poller); 2566 2567 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2568 2569 if (ctrlr_op_cb_fn) { 2570 ctrlr_op_cb_fn(ctrlr_op_cb_arg, 0); 2571 } 2572 2573 switch (op_after_disable) { 2574 case OP_COMPLETE_PENDING_DESTRUCT: 2575 nvme_ctrlr_unregister(nvme_ctrlr); 2576 break; 2577 default: 2578 break; 2579 } 2580 2581 } 2582 2583 static void 2584 bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr) 2585 { 2586 /* Make sure we clear any pending resets before returning. */ 2587 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2588 bdev_nvme_complete_pending_resets, 2589 NULL, 2590 _bdev_nvme_disable_ctrlr_complete); 2591 } 2592 2593 static void 2594 bdev_nvme_disable_destroy_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status) 2595 { 2596 assert(status == 0); 2597 2598 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2599 bdev_nvme_disable_ctrlr_complete(nvme_ctrlr); 2600 } else { 2601 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_ctrlr_complete); 2602 } 2603 } 2604 2605 static void 2606 bdev_nvme_disable_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr) 2607 { 2608 nvme_ctrlr_for_each_channel(nvme_ctrlr, 2609 bdev_nvme_reset_destroy_qpair, 2610 NULL, 2611 bdev_nvme_disable_destroy_qpairs_done); 2612 } 2613 2614 static void 2615 _bdev_nvme_cancel_reconnect_and_disable_ctrlr(void *ctx) 2616 { 2617 struct nvme_ctrlr *nvme_ctrlr = ctx; 2618 2619 assert(nvme_ctrlr->resetting == true); 2620 assert(nvme_ctrlr->thread == spdk_get_thread()); 2621 2622 spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer); 2623 2624 bdev_nvme_disable_ctrlr_complete(nvme_ctrlr); 2625 } 2626 2627 static void 2628 _bdev_nvme_disconnect_and_disable_ctrlr(void *ctx) 2629 { 2630 struct nvme_ctrlr *nvme_ctrlr = ctx; 2631 2632 assert(nvme_ctrlr->resetting == true); 2633 assert(nvme_ctrlr->thread == spdk_get_thread()); 2634 2635 if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { 2636 nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_destroy_qpairs); 2637 } else { 2638 bdev_nvme_disable_destroy_qpairs(nvme_ctrlr); 2639 } 2640 } 2641 2642 static int 2643 bdev_nvme_disable_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 2644 { 2645 spdk_msg_fn msg_fn; 2646 2647 pthread_mutex_lock(&nvme_ctrlr->mutex); 2648 if (nvme_ctrlr->destruct) { 2649 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2650 return -ENXIO; 2651 } 2652 2653 if (nvme_ctrlr->resetting) { 2654 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2655 return -EBUSY; 2656 } 2657 2658 if (nvme_ctrlr->disabled) { 2659 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2660 return -EALREADY; 2661 } 2662 2663 nvme_ctrlr->resetting = true; 2664 nvme_ctrlr->dont_retry = true; 2665 2666 if (nvme_ctrlr->reconnect_is_delayed) { 2667 msg_fn = _bdev_nvme_cancel_reconnect_and_disable_ctrlr; 2668 nvme_ctrlr->reconnect_is_delayed = false; 2669 } else { 2670 msg_fn = _bdev_nvme_disconnect_and_disable_ctrlr; 2671 } 2672 2673 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 2674 2675 pthread_mutex_unlock(&nvme_ctrlr->mutex); 2676 2677 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 2678 return 0; 2679 } 2680 2681 static int 2682 nvme_ctrlr_op(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op, 2683 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2684 { 2685 int rc; 2686 2687 switch (op) { 2688 case NVME_CTRLR_OP_RESET: 2689 rc = bdev_nvme_reset_ctrlr(nvme_ctrlr); 2690 break; 2691 case NVME_CTRLR_OP_ENABLE: 2692 rc = bdev_nvme_enable_ctrlr(nvme_ctrlr); 2693 break; 2694 case NVME_CTRLR_OP_DISABLE: 2695 rc = bdev_nvme_disable_ctrlr(nvme_ctrlr); 2696 break; 2697 default: 2698 rc = -EINVAL; 2699 break; 2700 } 2701 2702 if (rc == 0) { 2703 assert(nvme_ctrlr->ctrlr_op_cb_fn == NULL); 2704 assert(nvme_ctrlr->ctrlr_op_cb_arg == NULL); 2705 nvme_ctrlr->ctrlr_op_cb_fn = cb_fn; 2706 nvme_ctrlr->ctrlr_op_cb_arg = cb_arg; 2707 } 2708 return rc; 2709 } 2710 2711 struct nvme_ctrlr_op_rpc_ctx { 2712 struct nvme_ctrlr *nvme_ctrlr; 2713 struct spdk_thread *orig_thread; 2714 enum nvme_ctrlr_op op; 2715 int rc; 2716 bdev_nvme_ctrlr_op_cb cb_fn; 2717 void *cb_arg; 2718 }; 2719 2720 static void 2721 _nvme_ctrlr_op_rpc_complete(void *_ctx) 2722 { 2723 struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx; 2724 2725 assert(ctx != NULL); 2726 assert(ctx->cb_fn != NULL); 2727 2728 ctx->cb_fn(ctx->cb_arg, ctx->rc); 2729 2730 free(ctx); 2731 } 2732 2733 static void 2734 nvme_ctrlr_op_rpc_complete(void *cb_arg, int rc) 2735 { 2736 struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg; 2737 2738 ctx->rc = rc; 2739 2740 spdk_thread_send_msg(ctx->orig_thread, _nvme_ctrlr_op_rpc_complete, ctx); 2741 } 2742 2743 void 2744 nvme_ctrlr_op_rpc(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op, 2745 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2746 { 2747 struct nvme_ctrlr_op_rpc_ctx *ctx; 2748 int rc; 2749 2750 assert(cb_fn != NULL); 2751 2752 ctx = calloc(1, sizeof(*ctx)); 2753 if (ctx == NULL) { 2754 SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n"); 2755 cb_fn(cb_arg, -ENOMEM); 2756 return; 2757 } 2758 2759 ctx->orig_thread = spdk_get_thread(); 2760 ctx->cb_fn = cb_fn; 2761 ctx->cb_arg = cb_arg; 2762 2763 rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_ctrlr_op_rpc_complete, ctx); 2764 if (rc == 0) { 2765 return; 2766 } else if (rc == -EALREADY) { 2767 rc = 0; 2768 } 2769 2770 nvme_ctrlr_op_rpc_complete(ctx, rc); 2771 } 2772 2773 static void nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc); 2774 2775 static void 2776 _nvme_bdev_ctrlr_op_rpc_continue(void *_ctx) 2777 { 2778 struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx; 2779 struct nvme_ctrlr *prev_nvme_ctrlr, *next_nvme_ctrlr; 2780 int rc; 2781 2782 prev_nvme_ctrlr = ctx->nvme_ctrlr; 2783 ctx->nvme_ctrlr = NULL; 2784 2785 if (ctx->rc != 0) { 2786 goto complete; 2787 } 2788 2789 next_nvme_ctrlr = TAILQ_NEXT(prev_nvme_ctrlr, tailq); 2790 if (next_nvme_ctrlr == NULL) { 2791 goto complete; 2792 } 2793 2794 rc = nvme_ctrlr_op(next_nvme_ctrlr, ctx->op, nvme_bdev_ctrlr_op_rpc_continue, ctx); 2795 if (rc == 0) { 2796 ctx->nvme_ctrlr = next_nvme_ctrlr; 2797 return; 2798 } else if (rc == -EALREADY) { 2799 ctx->nvme_ctrlr = next_nvme_ctrlr; 2800 rc = 0; 2801 } 2802 2803 ctx->rc = rc; 2804 2805 complete: 2806 ctx->cb_fn(ctx->cb_arg, ctx->rc); 2807 free(ctx); 2808 } 2809 2810 static void 2811 nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc) 2812 { 2813 struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg; 2814 2815 ctx->rc = rc; 2816 2817 spdk_thread_send_msg(ctx->orig_thread, _nvme_bdev_ctrlr_op_rpc_continue, ctx); 2818 } 2819 2820 void 2821 nvme_bdev_ctrlr_op_rpc(struct nvme_bdev_ctrlr *nbdev_ctrlr, enum nvme_ctrlr_op op, 2822 bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg) 2823 { 2824 struct nvme_ctrlr_op_rpc_ctx *ctx; 2825 struct nvme_ctrlr *nvme_ctrlr; 2826 int rc; 2827 2828 assert(cb_fn != NULL); 2829 2830 ctx = calloc(1, sizeof(*ctx)); 2831 if (ctx == NULL) { 2832 SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n"); 2833 cb_fn(cb_arg, -ENOMEM); 2834 return; 2835 } 2836 2837 ctx->orig_thread = spdk_get_thread(); 2838 ctx->op = op; 2839 ctx->cb_fn = cb_fn; 2840 ctx->cb_arg = cb_arg; 2841 2842 nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs); 2843 assert(nvme_ctrlr != NULL); 2844 2845 rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_bdev_ctrlr_op_rpc_continue, ctx); 2846 if (rc == 0) { 2847 ctx->nvme_ctrlr = nvme_ctrlr; 2848 return; 2849 } else if (rc == -EALREADY) { 2850 ctx->nvme_ctrlr = nvme_ctrlr; 2851 rc = 0; 2852 } 2853 2854 nvme_bdev_ctrlr_op_rpc_continue(ctx, rc); 2855 } 2856 2857 static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio); 2858 2859 static void 2860 bdev_nvme_unfreeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status) 2861 { 2862 struct nvme_bdev_io *bio = ctx; 2863 enum spdk_bdev_io_status io_status; 2864 2865 if (bio->cpl.cdw0 == 0) { 2866 io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 2867 } else { 2868 io_status = SPDK_BDEV_IO_STATUS_FAILED; 2869 } 2870 2871 __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL); 2872 } 2873 2874 static void 2875 bdev_nvme_unfreeze_bdev_channel(struct nvme_bdev_channel_iter *i, 2876 struct nvme_bdev *nbdev, 2877 struct nvme_bdev_channel *nbdev_ch, void *ctx) 2878 { 2879 bdev_nvme_abort_retry_ios(nbdev_ch); 2880 nbdev_ch->resetting = false; 2881 2882 nvme_bdev_for_each_channel_continue(i, 0); 2883 } 2884 2885 static void 2886 bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio) 2887 { 2888 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2889 struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt; 2890 2891 /* Abort all queued I/Os for retry. */ 2892 nvme_bdev_for_each_channel(nbdev, 2893 bdev_nvme_unfreeze_bdev_channel, 2894 bio, 2895 bdev_nvme_unfreeze_bdev_channel_done); 2896 } 2897 2898 static void 2899 _bdev_nvme_reset_io_continue(void *ctx) 2900 { 2901 struct nvme_bdev_io *bio = ctx; 2902 struct nvme_io_path *prev_io_path, *next_io_path; 2903 int rc; 2904 2905 prev_io_path = bio->io_path; 2906 bio->io_path = NULL; 2907 2908 if (bio->cpl.cdw0 != 0) { 2909 goto complete; 2910 } 2911 2912 next_io_path = STAILQ_NEXT(prev_io_path, stailq); 2913 if (next_io_path == NULL) { 2914 goto complete; 2915 } 2916 2917 rc = _bdev_nvme_reset_io(next_io_path, bio); 2918 if (rc == 0) { 2919 return; 2920 } 2921 2922 bio->cpl.cdw0 = 1; 2923 2924 complete: 2925 bdev_nvme_reset_io_complete(bio); 2926 } 2927 2928 static void 2929 bdev_nvme_reset_io_continue(void *cb_arg, int rc) 2930 { 2931 struct nvme_bdev_io *bio = cb_arg; 2932 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2933 2934 bio->cpl.cdw0 = (rc == 0) ? 0 : 1; 2935 2936 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), _bdev_nvme_reset_io_continue, bio); 2937 } 2938 2939 static int 2940 _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio) 2941 { 2942 struct nvme_ctrlr_channel *ctrlr_ch; 2943 int rc; 2944 2945 rc = nvme_ctrlr_op(io_path->qpair->ctrlr, NVME_CTRLR_OP_RESET, 2946 bdev_nvme_reset_io_continue, bio); 2947 if (rc != 0 && rc != -EBUSY) { 2948 return rc; 2949 } 2950 2951 assert(bio->io_path == NULL); 2952 bio->io_path = io_path; 2953 2954 if (rc == -EBUSY) { 2955 ctrlr_ch = io_path->qpair->ctrlr_ch; 2956 assert(ctrlr_ch != NULL); 2957 /* 2958 * Reset call is queued only if it is from the app framework. This is on purpose so that 2959 * we don't interfere with the app framework reset strategy. i.e. we are deferring to the 2960 * upper level. If they are in the middle of a reset, we won't try to schedule another one. 2961 */ 2962 TAILQ_INSERT_TAIL(&ctrlr_ch->pending_resets, bio, retry_link); 2963 } 2964 2965 return 0; 2966 } 2967 2968 static void 2969 bdev_nvme_freeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status) 2970 { 2971 struct nvme_bdev_io *bio = ctx; 2972 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 2973 struct nvme_bdev_channel *nbdev_ch; 2974 struct nvme_io_path *io_path; 2975 int rc; 2976 2977 nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io)); 2978 2979 bio->cpl.cdw0 = 0; 2980 2981 /* Reset all nvme_ctrlrs of a bdev controller sequentially. */ 2982 io_path = STAILQ_FIRST(&nbdev_ch->io_path_list); 2983 assert(io_path != NULL); 2984 2985 rc = _bdev_nvme_reset_io(io_path, bio); 2986 if (rc != 0) { 2987 /* If the current nvme_ctrlr is disabled, skip it and move to the next nvme_ctrlr. */ 2988 rc = (rc == -EALREADY) ? 0 : rc; 2989 2990 bdev_nvme_reset_io_continue(bio, rc); 2991 } 2992 } 2993 2994 static void 2995 bdev_nvme_freeze_bdev_channel(struct nvme_bdev_channel_iter *i, 2996 struct nvme_bdev *nbdev, 2997 struct nvme_bdev_channel *nbdev_ch, void *ctx) 2998 { 2999 nbdev_ch->resetting = true; 3000 3001 nvme_bdev_for_each_channel_continue(i, 0); 3002 } 3003 3004 static void 3005 bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio) 3006 { 3007 nvme_bdev_for_each_channel(nbdev, 3008 bdev_nvme_freeze_bdev_channel, 3009 bio, 3010 bdev_nvme_freeze_bdev_channel_done); 3011 } 3012 3013 static int 3014 bdev_nvme_failover_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool remove) 3015 { 3016 if (nvme_ctrlr->destruct) { 3017 /* Don't bother resetting if the controller is in the process of being destructed. */ 3018 return -ENXIO; 3019 } 3020 3021 if (nvme_ctrlr->resetting) { 3022 if (!nvme_ctrlr->in_failover) { 3023 SPDK_NOTICELOG("Reset is already in progress. Defer failover until reset completes.\n"); 3024 3025 /* Defer failover until reset completes. */ 3026 nvme_ctrlr->pending_failover = true; 3027 return -EINPROGRESS; 3028 } else { 3029 SPDK_NOTICELOG("Unable to perform failover, already in progress.\n"); 3030 return -EBUSY; 3031 } 3032 } 3033 3034 bdev_nvme_failover_trid(nvme_ctrlr, remove, true); 3035 3036 if (nvme_ctrlr->reconnect_is_delayed) { 3037 SPDK_NOTICELOG("Reconnect is already scheduled.\n"); 3038 3039 /* We rely on the next reconnect for the failover. */ 3040 return -EALREADY; 3041 } 3042 3043 if (nvme_ctrlr->disabled) { 3044 SPDK_NOTICELOG("Controller is disabled.\n"); 3045 3046 /* We rely on the enablement for the failover. */ 3047 return -EALREADY; 3048 } 3049 3050 nvme_ctrlr->resetting = true; 3051 nvme_ctrlr->in_failover = true; 3052 3053 assert(nvme_ctrlr->reset_start_tsc == 0); 3054 nvme_ctrlr->reset_start_tsc = spdk_get_ticks(); 3055 3056 return 0; 3057 } 3058 3059 static int 3060 bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr) 3061 { 3062 int rc; 3063 3064 pthread_mutex_lock(&nvme_ctrlr->mutex); 3065 rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, false); 3066 pthread_mutex_unlock(&nvme_ctrlr->mutex); 3067 3068 if (rc == 0) { 3069 spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset_ctrlr, nvme_ctrlr); 3070 } else if (rc == -EALREADY) { 3071 rc = 0; 3072 } 3073 3074 return rc; 3075 } 3076 3077 static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, 3078 uint64_t num_blocks); 3079 3080 static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, 3081 uint64_t num_blocks); 3082 3083 static int bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, 3084 uint64_t src_offset_blocks, 3085 uint64_t num_blocks); 3086 3087 static void 3088 bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 3089 bool success) 3090 { 3091 struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx; 3092 int ret; 3093 3094 if (!success) { 3095 ret = -EINVAL; 3096 goto exit; 3097 } 3098 3099 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 3100 ret = -ENXIO; 3101 goto exit; 3102 } 3103 3104 ret = bdev_nvme_readv(bio, 3105 bdev_io->u.bdev.iovs, 3106 bdev_io->u.bdev.iovcnt, 3107 bdev_io->u.bdev.md_buf, 3108 bdev_io->u.bdev.num_blocks, 3109 bdev_io->u.bdev.offset_blocks, 3110 bdev_io->u.bdev.dif_check_flags, 3111 bdev_io->u.bdev.memory_domain, 3112 bdev_io->u.bdev.memory_domain_ctx, 3113 bdev_io->u.bdev.accel_sequence); 3114 3115 exit: 3116 if (spdk_unlikely(ret != 0)) { 3117 bdev_nvme_io_complete(bio, ret); 3118 } 3119 } 3120 3121 static inline void 3122 _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io) 3123 { 3124 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 3125 struct spdk_bdev *bdev = bdev_io->bdev; 3126 struct nvme_bdev_io *nbdev_io_to_abort; 3127 int rc = 0; 3128 3129 switch (bdev_io->type) { 3130 case SPDK_BDEV_IO_TYPE_READ: 3131 if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) { 3132 3133 rc = bdev_nvme_readv(nbdev_io, 3134 bdev_io->u.bdev.iovs, 3135 bdev_io->u.bdev.iovcnt, 3136 bdev_io->u.bdev.md_buf, 3137 bdev_io->u.bdev.num_blocks, 3138 bdev_io->u.bdev.offset_blocks, 3139 bdev_io->u.bdev.dif_check_flags, 3140 bdev_io->u.bdev.memory_domain, 3141 bdev_io->u.bdev.memory_domain_ctx, 3142 bdev_io->u.bdev.accel_sequence); 3143 } else { 3144 spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb, 3145 bdev_io->u.bdev.num_blocks * bdev->blocklen); 3146 rc = 0; 3147 } 3148 break; 3149 case SPDK_BDEV_IO_TYPE_WRITE: 3150 rc = bdev_nvme_writev(nbdev_io, 3151 bdev_io->u.bdev.iovs, 3152 bdev_io->u.bdev.iovcnt, 3153 bdev_io->u.bdev.md_buf, 3154 bdev_io->u.bdev.num_blocks, 3155 bdev_io->u.bdev.offset_blocks, 3156 bdev_io->u.bdev.dif_check_flags, 3157 bdev_io->u.bdev.memory_domain, 3158 bdev_io->u.bdev.memory_domain_ctx, 3159 bdev_io->u.bdev.accel_sequence, 3160 bdev_io->u.bdev.nvme_cdw12, 3161 bdev_io->u.bdev.nvme_cdw13); 3162 break; 3163 case SPDK_BDEV_IO_TYPE_COMPARE: 3164 rc = bdev_nvme_comparev(nbdev_io, 3165 bdev_io->u.bdev.iovs, 3166 bdev_io->u.bdev.iovcnt, 3167 bdev_io->u.bdev.md_buf, 3168 bdev_io->u.bdev.num_blocks, 3169 bdev_io->u.bdev.offset_blocks, 3170 bdev_io->u.bdev.dif_check_flags); 3171 break; 3172 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 3173 rc = bdev_nvme_comparev_and_writev(nbdev_io, 3174 bdev_io->u.bdev.iovs, 3175 bdev_io->u.bdev.iovcnt, 3176 bdev_io->u.bdev.fused_iovs, 3177 bdev_io->u.bdev.fused_iovcnt, 3178 bdev_io->u.bdev.md_buf, 3179 bdev_io->u.bdev.num_blocks, 3180 bdev_io->u.bdev.offset_blocks, 3181 bdev_io->u.bdev.dif_check_flags); 3182 break; 3183 case SPDK_BDEV_IO_TYPE_UNMAP: 3184 rc = bdev_nvme_unmap(nbdev_io, 3185 bdev_io->u.bdev.offset_blocks, 3186 bdev_io->u.bdev.num_blocks); 3187 break; 3188 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 3189 rc = bdev_nvme_write_zeroes(nbdev_io, 3190 bdev_io->u.bdev.offset_blocks, 3191 bdev_io->u.bdev.num_blocks); 3192 break; 3193 case SPDK_BDEV_IO_TYPE_RESET: 3194 nbdev_io->io_path = NULL; 3195 bdev_nvme_reset_io(bdev->ctxt, nbdev_io); 3196 return; 3197 3198 case SPDK_BDEV_IO_TYPE_FLUSH: 3199 bdev_nvme_io_complete(nbdev_io, 0); 3200 return; 3201 3202 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 3203 rc = bdev_nvme_zone_appendv(nbdev_io, 3204 bdev_io->u.bdev.iovs, 3205 bdev_io->u.bdev.iovcnt, 3206 bdev_io->u.bdev.md_buf, 3207 bdev_io->u.bdev.num_blocks, 3208 bdev_io->u.bdev.offset_blocks, 3209 bdev_io->u.bdev.dif_check_flags); 3210 break; 3211 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 3212 rc = bdev_nvme_get_zone_info(nbdev_io, 3213 bdev_io->u.zone_mgmt.zone_id, 3214 bdev_io->u.zone_mgmt.num_zones, 3215 bdev_io->u.zone_mgmt.buf); 3216 break; 3217 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 3218 rc = bdev_nvme_zone_management(nbdev_io, 3219 bdev_io->u.zone_mgmt.zone_id, 3220 bdev_io->u.zone_mgmt.zone_action); 3221 break; 3222 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3223 nbdev_io->io_path = NULL; 3224 bdev_nvme_admin_passthru(nbdev_ch, 3225 nbdev_io, 3226 &bdev_io->u.nvme_passthru.cmd, 3227 bdev_io->u.nvme_passthru.buf, 3228 bdev_io->u.nvme_passthru.nbytes); 3229 return; 3230 3231 case SPDK_BDEV_IO_TYPE_NVME_IO: 3232 rc = bdev_nvme_io_passthru(nbdev_io, 3233 &bdev_io->u.nvme_passthru.cmd, 3234 bdev_io->u.nvme_passthru.buf, 3235 bdev_io->u.nvme_passthru.nbytes); 3236 break; 3237 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3238 rc = bdev_nvme_io_passthru_md(nbdev_io, 3239 &bdev_io->u.nvme_passthru.cmd, 3240 bdev_io->u.nvme_passthru.buf, 3241 bdev_io->u.nvme_passthru.nbytes, 3242 bdev_io->u.nvme_passthru.md_buf, 3243 bdev_io->u.nvme_passthru.md_len); 3244 break; 3245 case SPDK_BDEV_IO_TYPE_NVME_IOV_MD: 3246 rc = bdev_nvme_iov_passthru_md(nbdev_io, 3247 &bdev_io->u.nvme_passthru.cmd, 3248 bdev_io->u.nvme_passthru.iovs, 3249 bdev_io->u.nvme_passthru.iovcnt, 3250 bdev_io->u.nvme_passthru.nbytes, 3251 bdev_io->u.nvme_passthru.md_buf, 3252 bdev_io->u.nvme_passthru.md_len); 3253 break; 3254 case SPDK_BDEV_IO_TYPE_ABORT: 3255 nbdev_io->io_path = NULL; 3256 nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx; 3257 bdev_nvme_abort(nbdev_ch, 3258 nbdev_io, 3259 nbdev_io_to_abort); 3260 return; 3261 3262 case SPDK_BDEV_IO_TYPE_COPY: 3263 rc = bdev_nvme_copy(nbdev_io, 3264 bdev_io->u.bdev.offset_blocks, 3265 bdev_io->u.bdev.copy.src_offset_blocks, 3266 bdev_io->u.bdev.num_blocks); 3267 break; 3268 default: 3269 rc = -EINVAL; 3270 break; 3271 } 3272 3273 if (spdk_unlikely(rc != 0)) { 3274 bdev_nvme_io_complete(nbdev_io, rc); 3275 } 3276 } 3277 3278 static void 3279 bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 3280 { 3281 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 3282 struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx; 3283 3284 if (spdk_likely(nbdev_io->submit_tsc == 0)) { 3285 nbdev_io->submit_tsc = spdk_bdev_io_get_submit_tsc(bdev_io); 3286 } else { 3287 /* There are cases where submit_tsc != 0, i.e. retry I/O. 3288 * We need to update submit_tsc here. 3289 */ 3290 nbdev_io->submit_tsc = spdk_get_ticks(); 3291 } 3292 3293 spdk_trace_record(TRACE_BDEV_NVME_IO_START, 0, 0, (uintptr_t)nbdev_io, (uintptr_t)bdev_io); 3294 nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch); 3295 if (spdk_unlikely(!nbdev_io->io_path)) { 3296 if (!bdev_nvme_io_type_is_admin(bdev_io->type)) { 3297 bdev_nvme_io_complete(nbdev_io, -ENXIO); 3298 return; 3299 } 3300 3301 /* Admin commands do not use the optimal I/O path. 3302 * Simply fall through even if it is not found. 3303 */ 3304 } 3305 3306 _bdev_nvme_submit_request(nbdev_ch, bdev_io); 3307 } 3308 3309 static bool 3310 bdev_nvme_is_supported_csi(enum spdk_nvme_csi csi) 3311 { 3312 switch (csi) { 3313 case SPDK_NVME_CSI_NVM: 3314 return true; 3315 case SPDK_NVME_CSI_ZNS: 3316 return true; 3317 default: 3318 return false; 3319 } 3320 } 3321 3322 static bool 3323 bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 3324 { 3325 struct nvme_bdev *nbdev = ctx; 3326 struct nvme_ns *nvme_ns; 3327 struct spdk_nvme_ns *ns; 3328 struct spdk_nvme_ctrlr *ctrlr; 3329 const struct spdk_nvme_ctrlr_data *cdata; 3330 3331 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 3332 assert(nvme_ns != NULL); 3333 ns = nvme_ns->ns; 3334 if (ns == NULL) { 3335 return false; 3336 } 3337 3338 if (!bdev_nvme_is_supported_csi(spdk_nvme_ns_get_csi(ns))) { 3339 switch (io_type) { 3340 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3341 case SPDK_BDEV_IO_TYPE_NVME_IO: 3342 return true; 3343 3344 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3345 return spdk_nvme_ns_get_md_size(ns) ? true : false; 3346 3347 default: 3348 return false; 3349 } 3350 } 3351 3352 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 3353 3354 switch (io_type) { 3355 case SPDK_BDEV_IO_TYPE_READ: 3356 case SPDK_BDEV_IO_TYPE_WRITE: 3357 case SPDK_BDEV_IO_TYPE_RESET: 3358 case SPDK_BDEV_IO_TYPE_FLUSH: 3359 case SPDK_BDEV_IO_TYPE_NVME_ADMIN: 3360 case SPDK_BDEV_IO_TYPE_NVME_IO: 3361 case SPDK_BDEV_IO_TYPE_ABORT: 3362 return true; 3363 3364 case SPDK_BDEV_IO_TYPE_COMPARE: 3365 return spdk_nvme_ns_supports_compare(ns); 3366 3367 case SPDK_BDEV_IO_TYPE_NVME_IO_MD: 3368 return spdk_nvme_ns_get_md_size(ns) ? true : false; 3369 3370 case SPDK_BDEV_IO_TYPE_UNMAP: 3371 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3372 return cdata->oncs.dsm; 3373 3374 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 3375 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3376 return cdata->oncs.write_zeroes; 3377 3378 case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE: 3379 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 3380 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) { 3381 return true; 3382 } 3383 return false; 3384 3385 case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO: 3386 case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT: 3387 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS; 3388 3389 case SPDK_BDEV_IO_TYPE_ZONE_APPEND: 3390 return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS && 3391 spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED; 3392 3393 case SPDK_BDEV_IO_TYPE_COPY: 3394 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3395 return cdata->oncs.copy; 3396 3397 default: 3398 return false; 3399 } 3400 } 3401 3402 static int 3403 nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch) 3404 { 3405 struct nvme_qpair *nvme_qpair; 3406 struct spdk_io_channel *pg_ch; 3407 int rc; 3408 3409 nvme_qpair = calloc(1, sizeof(*nvme_qpair)); 3410 if (!nvme_qpair) { 3411 SPDK_ERRLOG("Failed to alloc nvme_qpair.\n"); 3412 return -1; 3413 } 3414 3415 TAILQ_INIT(&nvme_qpair->io_path_list); 3416 3417 nvme_qpair->ctrlr = nvme_ctrlr; 3418 nvme_qpair->ctrlr_ch = ctrlr_ch; 3419 3420 pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs); 3421 if (!pg_ch) { 3422 free(nvme_qpair); 3423 return -1; 3424 } 3425 3426 nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch); 3427 3428 #ifdef SPDK_CONFIG_VTUNE 3429 nvme_qpair->group->collect_spin_stat = true; 3430 #else 3431 nvme_qpair->group->collect_spin_stat = false; 3432 #endif 3433 3434 if (!nvme_ctrlr->disabled) { 3435 /* If a nvme_ctrlr is disabled, don't try to create qpair for it. Qpair will 3436 * be created when it's enabled. 3437 */ 3438 rc = bdev_nvme_create_qpair(nvme_qpair); 3439 if (rc != 0) { 3440 /* nvme_ctrlr can't create IO qpair if connection is down. 3441 * If reconnect_delay_sec is non-zero, creating IO qpair is retried 3442 * after reconnect_delay_sec seconds. If bdev_retry_count is non-zero, 3443 * submitted IO will be queued until IO qpair is successfully created. 3444 * 3445 * Hence, if both are satisfied, ignore the failure. 3446 */ 3447 if (nvme_ctrlr->opts.reconnect_delay_sec == 0 || g_opts.bdev_retry_count == 0) { 3448 spdk_put_io_channel(pg_ch); 3449 free(nvme_qpair); 3450 return rc; 3451 } 3452 } 3453 } 3454 3455 TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 3456 3457 ctrlr_ch->qpair = nvme_qpair; 3458 3459 pthread_mutex_lock(&nvme_qpair->ctrlr->mutex); 3460 nvme_qpair->ctrlr->ref++; 3461 pthread_mutex_unlock(&nvme_qpair->ctrlr->mutex); 3462 3463 return 0; 3464 } 3465 3466 static int 3467 bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) 3468 { 3469 struct nvme_ctrlr *nvme_ctrlr = io_device; 3470 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 3471 3472 TAILQ_INIT(&ctrlr_ch->pending_resets); 3473 3474 return nvme_qpair_create(nvme_ctrlr, ctrlr_ch); 3475 } 3476 3477 static void 3478 nvme_qpair_delete(struct nvme_qpair *nvme_qpair) 3479 { 3480 struct nvme_io_path *io_path, *next; 3481 3482 assert(nvme_qpair->group != NULL); 3483 3484 TAILQ_FOREACH_SAFE(io_path, &nvme_qpair->io_path_list, tailq, next) { 3485 TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq); 3486 nvme_io_path_free(io_path); 3487 } 3488 3489 TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq); 3490 3491 spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group)); 3492 3493 nvme_ctrlr_release(nvme_qpair->ctrlr); 3494 3495 free(nvme_qpair); 3496 } 3497 3498 static void 3499 bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf) 3500 { 3501 struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; 3502 struct nvme_qpair *nvme_qpair; 3503 3504 nvme_qpair = ctrlr_ch->qpair; 3505 assert(nvme_qpair != NULL); 3506 3507 _bdev_nvme_clear_io_path_cache(nvme_qpair); 3508 3509 if (nvme_qpair->qpair != NULL) { 3510 if (ctrlr_ch->reset_iter == NULL) { 3511 spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair); 3512 } else { 3513 /* Skip current ctrlr_channel in a full reset sequence because 3514 * it is being deleted now. The qpair is already being disconnected. 3515 * We do not have to restart disconnecting it. 3516 */ 3517 nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0); 3518 } 3519 3520 /* We cannot release a reference to the poll group now. 3521 * The qpair may be disconnected asynchronously later. 3522 * We need to poll it until it is actually disconnected. 3523 * Just detach the qpair from the deleting ctrlr_channel. 3524 */ 3525 nvme_qpair->ctrlr_ch = NULL; 3526 } else { 3527 assert(ctrlr_ch->reset_iter == NULL); 3528 3529 nvme_qpair_delete(nvme_qpair); 3530 } 3531 } 3532 3533 static inline struct spdk_io_channel * 3534 bdev_nvme_get_accel_channel(struct nvme_poll_group *group) 3535 { 3536 if (spdk_unlikely(!group->accel_channel)) { 3537 group->accel_channel = spdk_accel_get_io_channel(); 3538 if (!group->accel_channel) { 3539 SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n", 3540 group); 3541 return NULL; 3542 } 3543 } 3544 3545 return group->accel_channel; 3546 } 3547 3548 static void 3549 bdev_nvme_finish_sequence(void *seq, spdk_nvme_accel_completion_cb cb_fn, void *cb_arg) 3550 { 3551 spdk_accel_sequence_finish(seq, cb_fn, cb_arg); 3552 } 3553 3554 static void 3555 bdev_nvme_abort_sequence(void *seq) 3556 { 3557 spdk_accel_sequence_abort(seq); 3558 } 3559 3560 static void 3561 bdev_nvme_reverse_sequence(void *seq) 3562 { 3563 spdk_accel_sequence_reverse(seq); 3564 } 3565 3566 static int 3567 bdev_nvme_append_crc32c(void *ctx, void **seq, uint32_t *dst, struct iovec *iovs, uint32_t iovcnt, 3568 struct spdk_memory_domain *domain, void *domain_ctx, uint32_t seed, 3569 spdk_nvme_accel_step_cb cb_fn, void *cb_arg) 3570 { 3571 struct spdk_io_channel *ch; 3572 struct nvme_poll_group *group = ctx; 3573 3574 ch = bdev_nvme_get_accel_channel(group); 3575 if (spdk_unlikely(ch == NULL)) { 3576 return -ENOMEM; 3577 } 3578 3579 return spdk_accel_append_crc32c((struct spdk_accel_sequence **)seq, ch, dst, iovs, iovcnt, 3580 domain, domain_ctx, seed, cb_fn, cb_arg); 3581 } 3582 3583 static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = { 3584 .table_size = sizeof(struct spdk_nvme_accel_fn_table), 3585 .append_crc32c = bdev_nvme_append_crc32c, 3586 .finish_sequence = bdev_nvme_finish_sequence, 3587 .reverse_sequence = bdev_nvme_reverse_sequence, 3588 .abort_sequence = bdev_nvme_abort_sequence, 3589 }; 3590 3591 static int 3592 bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf) 3593 { 3594 struct nvme_poll_group *group = ctx_buf; 3595 3596 TAILQ_INIT(&group->qpair_list); 3597 3598 group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table); 3599 if (group->group == NULL) { 3600 return -1; 3601 } 3602 3603 group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us); 3604 3605 if (group->poller == NULL) { 3606 spdk_nvme_poll_group_destroy(group->group); 3607 return -1; 3608 } 3609 3610 return 0; 3611 } 3612 3613 static void 3614 bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf) 3615 { 3616 struct nvme_poll_group *group = ctx_buf; 3617 3618 assert(TAILQ_EMPTY(&group->qpair_list)); 3619 3620 if (group->accel_channel) { 3621 spdk_put_io_channel(group->accel_channel); 3622 } 3623 3624 spdk_poller_unregister(&group->poller); 3625 if (spdk_nvme_poll_group_destroy(group->group)) { 3626 SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n"); 3627 assert(false); 3628 } 3629 } 3630 3631 static struct spdk_io_channel * 3632 bdev_nvme_get_io_channel(void *ctx) 3633 { 3634 struct nvme_bdev *nvme_bdev = ctx; 3635 3636 return spdk_get_io_channel(nvme_bdev); 3637 } 3638 3639 static void * 3640 bdev_nvme_get_module_ctx(void *ctx) 3641 { 3642 struct nvme_bdev *nvme_bdev = ctx; 3643 struct nvme_ns *nvme_ns; 3644 3645 if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) { 3646 return NULL; 3647 } 3648 3649 nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list); 3650 if (!nvme_ns) { 3651 return NULL; 3652 } 3653 3654 return nvme_ns->ns; 3655 } 3656 3657 static const char * 3658 _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state) 3659 { 3660 switch (ana_state) { 3661 case SPDK_NVME_ANA_OPTIMIZED_STATE: 3662 return "optimized"; 3663 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 3664 return "non_optimized"; 3665 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 3666 return "inaccessible"; 3667 case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE: 3668 return "persistent_loss"; 3669 case SPDK_NVME_ANA_CHANGE_STATE: 3670 return "change"; 3671 default: 3672 return NULL; 3673 } 3674 } 3675 3676 static int 3677 bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size) 3678 { 3679 struct spdk_memory_domain **_domains = NULL; 3680 struct nvme_bdev *nbdev = ctx; 3681 struct nvme_ns *nvme_ns; 3682 int i = 0, _array_size = array_size; 3683 int rc = 0; 3684 3685 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 3686 if (domains && array_size >= i) { 3687 _domains = &domains[i]; 3688 } else { 3689 _domains = NULL; 3690 } 3691 rc = spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, _domains, _array_size); 3692 if (rc > 0) { 3693 i += rc; 3694 if (_array_size >= rc) { 3695 _array_size -= rc; 3696 } else { 3697 _array_size = 0; 3698 } 3699 } else if (rc < 0) { 3700 return rc; 3701 } 3702 } 3703 3704 return i; 3705 } 3706 3707 static const char * 3708 nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr) 3709 { 3710 if (nvme_ctrlr->destruct) { 3711 return "deleting"; 3712 } else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) { 3713 return "failed"; 3714 } else if (nvme_ctrlr->resetting) { 3715 return "resetting"; 3716 } else if (nvme_ctrlr->reconnect_is_delayed > 0) { 3717 return "reconnect_is_delayed"; 3718 } else if (nvme_ctrlr->disabled) { 3719 return "disabled"; 3720 } else { 3721 return "enabled"; 3722 } 3723 } 3724 3725 void 3726 nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr) 3727 { 3728 struct spdk_nvme_transport_id *trid; 3729 const struct spdk_nvme_ctrlr_opts *opts; 3730 const struct spdk_nvme_ctrlr_data *cdata; 3731 struct nvme_path_id *path_id; 3732 int32_t numa_id; 3733 3734 spdk_json_write_object_begin(w); 3735 3736 spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr)); 3737 3738 #ifdef SPDK_CONFIG_NVME_CUSE 3739 size_t cuse_name_size = 128; 3740 char cuse_name[cuse_name_size]; 3741 3742 int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size); 3743 if (rc == 0) { 3744 spdk_json_write_named_string(w, "cuse_device", cuse_name); 3745 } 3746 #endif 3747 trid = &nvme_ctrlr->active_path_id->trid; 3748 spdk_json_write_named_object_begin(w, "trid"); 3749 nvme_bdev_dump_trid_json(trid, w); 3750 spdk_json_write_object_end(w); 3751 3752 path_id = TAILQ_NEXT(nvme_ctrlr->active_path_id, link); 3753 if (path_id != NULL) { 3754 spdk_json_write_named_array_begin(w, "alternate_trids"); 3755 do { 3756 trid = &path_id->trid; 3757 spdk_json_write_object_begin(w); 3758 nvme_bdev_dump_trid_json(trid, w); 3759 spdk_json_write_object_end(w); 3760 3761 path_id = TAILQ_NEXT(path_id, link); 3762 } while (path_id != NULL); 3763 spdk_json_write_array_end(w); 3764 } 3765 3766 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 3767 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 3768 3769 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 3770 spdk_json_write_named_object_begin(w, "host"); 3771 spdk_json_write_named_string(w, "nqn", opts->hostnqn); 3772 spdk_json_write_named_string(w, "addr", opts->src_addr); 3773 spdk_json_write_named_string(w, "svcid", opts->src_svcid); 3774 spdk_json_write_object_end(w); 3775 3776 numa_id = spdk_nvme_ctrlr_get_numa_id(nvme_ctrlr->ctrlr); 3777 if (numa_id != SPDK_ENV_NUMA_ID_ANY) { 3778 spdk_json_write_named_uint32(w, "numa_id", numa_id); 3779 } 3780 spdk_json_write_object_end(w); 3781 } 3782 3783 static void 3784 nvme_namespace_info_json(struct spdk_json_write_ctx *w, 3785 struct nvme_ns *nvme_ns) 3786 { 3787 struct spdk_nvme_ns *ns; 3788 struct spdk_nvme_ctrlr *ctrlr; 3789 const struct spdk_nvme_ctrlr_data *cdata; 3790 const struct spdk_nvme_transport_id *trid; 3791 union spdk_nvme_vs_register vs; 3792 const struct spdk_nvme_ns_data *nsdata; 3793 char buf[128]; 3794 3795 ns = nvme_ns->ns; 3796 if (ns == NULL) { 3797 return; 3798 } 3799 3800 ctrlr = spdk_nvme_ns_get_ctrlr(ns); 3801 3802 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 3803 trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); 3804 vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr); 3805 3806 spdk_json_write_object_begin(w); 3807 3808 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 3809 spdk_json_write_named_string(w, "pci_address", trid->traddr); 3810 } 3811 3812 spdk_json_write_named_object_begin(w, "trid"); 3813 3814 nvme_bdev_dump_trid_json(trid, w); 3815 3816 spdk_json_write_object_end(w); 3817 3818 #ifdef SPDK_CONFIG_NVME_CUSE 3819 size_t cuse_name_size = 128; 3820 char cuse_name[cuse_name_size]; 3821 3822 int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns), 3823 cuse_name, &cuse_name_size); 3824 if (rc == 0) { 3825 spdk_json_write_named_string(w, "cuse_device", cuse_name); 3826 } 3827 #endif 3828 3829 spdk_json_write_named_object_begin(w, "ctrlr_data"); 3830 3831 spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid); 3832 3833 spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid); 3834 3835 snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn); 3836 spdk_str_trim(buf); 3837 spdk_json_write_named_string(w, "model_number", buf); 3838 3839 snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn); 3840 spdk_str_trim(buf); 3841 spdk_json_write_named_string(w, "serial_number", buf); 3842 3843 snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr); 3844 spdk_str_trim(buf); 3845 spdk_json_write_named_string(w, "firmware_revision", buf); 3846 3847 if (cdata->subnqn[0] != '\0') { 3848 spdk_json_write_named_string(w, "subnqn", cdata->subnqn); 3849 } 3850 3851 spdk_json_write_named_object_begin(w, "oacs"); 3852 3853 spdk_json_write_named_uint32(w, "security", cdata->oacs.security); 3854 spdk_json_write_named_uint32(w, "format", cdata->oacs.format); 3855 spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware); 3856 spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage); 3857 3858 spdk_json_write_object_end(w); 3859 3860 spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr); 3861 spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting); 3862 3863 spdk_json_write_object_end(w); 3864 3865 spdk_json_write_named_object_begin(w, "vs"); 3866 3867 spdk_json_write_name(w, "nvme_version"); 3868 if (vs.bits.ter) { 3869 spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter); 3870 } else { 3871 spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr); 3872 } 3873 3874 spdk_json_write_object_end(w); 3875 3876 nsdata = spdk_nvme_ns_get_data(ns); 3877 3878 spdk_json_write_named_object_begin(w, "ns_data"); 3879 3880 spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns)); 3881 3882 if (cdata->cmic.ana_reporting) { 3883 spdk_json_write_named_string(w, "ana_state", 3884 _nvme_ana_state_str(nvme_ns->ana_state)); 3885 } 3886 3887 spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share); 3888 3889 spdk_json_write_object_end(w); 3890 3891 if (cdata->oacs.security) { 3892 spdk_json_write_named_object_begin(w, "security"); 3893 3894 spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal); 3895 3896 spdk_json_write_object_end(w); 3897 } 3898 3899 spdk_json_write_object_end(w); 3900 } 3901 3902 static const char * 3903 nvme_bdev_get_mp_policy_str(struct nvme_bdev *nbdev) 3904 { 3905 switch (nbdev->mp_policy) { 3906 case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE: 3907 return "active_passive"; 3908 case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE: 3909 return "active_active"; 3910 default: 3911 assert(false); 3912 return "invalid"; 3913 } 3914 } 3915 3916 static const char * 3917 nvme_bdev_get_mp_selector_str(struct nvme_bdev *nbdev) 3918 { 3919 switch (nbdev->mp_selector) { 3920 case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN: 3921 return "round_robin"; 3922 case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH: 3923 return "queue_depth"; 3924 default: 3925 assert(false); 3926 return "invalid"; 3927 } 3928 } 3929 3930 static int 3931 bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 3932 { 3933 struct nvme_bdev *nvme_bdev = ctx; 3934 struct nvme_ns *nvme_ns; 3935 3936 pthread_mutex_lock(&nvme_bdev->mutex); 3937 spdk_json_write_named_array_begin(w, "nvme"); 3938 TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) { 3939 nvme_namespace_info_json(w, nvme_ns); 3940 } 3941 spdk_json_write_array_end(w); 3942 spdk_json_write_named_string(w, "mp_policy", nvme_bdev_get_mp_policy_str(nvme_bdev)); 3943 if (nvme_bdev->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 3944 spdk_json_write_named_string(w, "selector", nvme_bdev_get_mp_selector_str(nvme_bdev)); 3945 if (nvme_bdev->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) { 3946 spdk_json_write_named_uint32(w, "rr_min_io", nvme_bdev->rr_min_io); 3947 } 3948 } 3949 pthread_mutex_unlock(&nvme_bdev->mutex); 3950 3951 return 0; 3952 } 3953 3954 static void 3955 bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 3956 { 3957 /* No config per bdev needed */ 3958 } 3959 3960 static uint64_t 3961 bdev_nvme_get_spin_time(struct spdk_io_channel *ch) 3962 { 3963 struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); 3964 struct nvme_io_path *io_path; 3965 struct nvme_poll_group *group; 3966 uint64_t spin_time = 0; 3967 3968 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 3969 group = io_path->qpair->group; 3970 3971 if (!group || !group->collect_spin_stat) { 3972 continue; 3973 } 3974 3975 if (group->end_ticks != 0) { 3976 group->spin_ticks += (group->end_ticks - group->start_ticks); 3977 group->end_ticks = 0; 3978 } 3979 3980 spin_time += group->spin_ticks; 3981 group->start_ticks = 0; 3982 group->spin_ticks = 0; 3983 } 3984 3985 return (spin_time * 1000000ULL) / spdk_get_ticks_hz(); 3986 } 3987 3988 static void 3989 bdev_nvme_reset_device_stat(void *ctx) 3990 { 3991 struct nvme_bdev *nbdev = ctx; 3992 3993 if (nbdev->err_stat != NULL) { 3994 memset(nbdev->err_stat, 0, sizeof(struct nvme_error_stat)); 3995 } 3996 } 3997 3998 /* JSON string should be lowercases and underscore delimited string. */ 3999 static void 4000 bdev_nvme_format_nvme_status(char *dst, const char *src) 4001 { 4002 char tmp[256]; 4003 4004 spdk_strcpy_replace(dst, 256, src, " - ", "_"); 4005 spdk_strcpy_replace(tmp, 256, dst, "-", "_"); 4006 spdk_strcpy_replace(dst, 256, tmp, " ", "_"); 4007 spdk_strlwr(dst); 4008 } 4009 4010 static void 4011 bdev_nvme_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w) 4012 { 4013 struct nvme_bdev *nbdev = ctx; 4014 struct spdk_nvme_status status = {}; 4015 uint16_t sct, sc; 4016 char status_json[256]; 4017 const char *status_str; 4018 4019 if (nbdev->err_stat == NULL) { 4020 return; 4021 } 4022 4023 spdk_json_write_named_object_begin(w, "nvme_error"); 4024 4025 spdk_json_write_named_object_begin(w, "status_type"); 4026 for (sct = 0; sct < 8; sct++) { 4027 if (nbdev->err_stat->status_type[sct] == 0) { 4028 continue; 4029 } 4030 status.sct = sct; 4031 4032 status_str = spdk_nvme_cpl_get_status_type_string(&status); 4033 assert(status_str != NULL); 4034 bdev_nvme_format_nvme_status(status_json, status_str); 4035 4036 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status_type[sct]); 4037 } 4038 spdk_json_write_object_end(w); 4039 4040 spdk_json_write_named_object_begin(w, "status_code"); 4041 for (sct = 0; sct < 4; sct++) { 4042 status.sct = sct; 4043 for (sc = 0; sc < 256; sc++) { 4044 if (nbdev->err_stat->status[sct][sc] == 0) { 4045 continue; 4046 } 4047 status.sc = sc; 4048 4049 status_str = spdk_nvme_cpl_get_status_string(&status); 4050 assert(status_str != NULL); 4051 bdev_nvme_format_nvme_status(status_json, status_str); 4052 4053 spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status[sct][sc]); 4054 } 4055 } 4056 spdk_json_write_object_end(w); 4057 4058 spdk_json_write_object_end(w); 4059 } 4060 4061 static bool 4062 bdev_nvme_accel_sequence_supported(void *ctx, enum spdk_bdev_io_type type) 4063 { 4064 struct nvme_bdev *nbdev = ctx; 4065 struct spdk_nvme_ctrlr *ctrlr; 4066 4067 if (!g_opts.allow_accel_sequence) { 4068 return false; 4069 } 4070 4071 switch (type) { 4072 case SPDK_BDEV_IO_TYPE_WRITE: 4073 case SPDK_BDEV_IO_TYPE_READ: 4074 break; 4075 default: 4076 return false; 4077 } 4078 4079 ctrlr = bdev_nvme_get_ctrlr(&nbdev->disk); 4080 assert(ctrlr != NULL); 4081 4082 return spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ACCEL_SEQUENCE_SUPPORTED; 4083 } 4084 4085 static const struct spdk_bdev_fn_table nvmelib_fn_table = { 4086 .destruct = bdev_nvme_destruct, 4087 .submit_request = bdev_nvme_submit_request, 4088 .io_type_supported = bdev_nvme_io_type_supported, 4089 .get_io_channel = bdev_nvme_get_io_channel, 4090 .dump_info_json = bdev_nvme_dump_info_json, 4091 .write_config_json = bdev_nvme_write_config_json, 4092 .get_spin_time = bdev_nvme_get_spin_time, 4093 .get_module_ctx = bdev_nvme_get_module_ctx, 4094 .get_memory_domains = bdev_nvme_get_memory_domains, 4095 .accel_sequence_supported = bdev_nvme_accel_sequence_supported, 4096 .reset_device_stat = bdev_nvme_reset_device_stat, 4097 .dump_device_stat_json = bdev_nvme_dump_device_stat_json, 4098 }; 4099 4100 typedef int (*bdev_nvme_parse_ana_log_page_cb)( 4101 const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg); 4102 4103 static int 4104 bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 4105 bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg) 4106 { 4107 struct spdk_nvme_ana_group_descriptor *copied_desc; 4108 uint8_t *orig_desc; 4109 uint32_t i, desc_size, copy_len; 4110 int rc = 0; 4111 4112 if (nvme_ctrlr->ana_log_page == NULL) { 4113 return -EINVAL; 4114 } 4115 4116 copied_desc = nvme_ctrlr->copied_ana_desc; 4117 4118 orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page); 4119 copy_len = nvme_ctrlr->max_ana_log_page_size - sizeof(struct spdk_nvme_ana_page); 4120 4121 for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) { 4122 memcpy(copied_desc, orig_desc, copy_len); 4123 4124 rc = cb_fn(copied_desc, cb_arg); 4125 if (rc != 0) { 4126 break; 4127 } 4128 4129 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) + 4130 copied_desc->num_of_nsid * sizeof(uint32_t); 4131 orig_desc += desc_size; 4132 copy_len -= desc_size; 4133 } 4134 4135 return rc; 4136 } 4137 4138 static int 4139 nvme_ns_ana_transition_timedout(void *ctx) 4140 { 4141 struct nvme_ns *nvme_ns = ctx; 4142 4143 spdk_poller_unregister(&nvme_ns->anatt_timer); 4144 nvme_ns->ana_transition_timedout = true; 4145 4146 return SPDK_POLLER_BUSY; 4147 } 4148 4149 static void 4150 _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns, 4151 const struct spdk_nvme_ana_group_descriptor *desc) 4152 { 4153 const struct spdk_nvme_ctrlr_data *cdata; 4154 4155 nvme_ns->ana_group_id = desc->ana_group_id; 4156 nvme_ns->ana_state = desc->ana_state; 4157 nvme_ns->ana_state_updating = false; 4158 4159 switch (nvme_ns->ana_state) { 4160 case SPDK_NVME_ANA_OPTIMIZED_STATE: 4161 case SPDK_NVME_ANA_NON_OPTIMIZED_STATE: 4162 nvme_ns->ana_transition_timedout = false; 4163 spdk_poller_unregister(&nvme_ns->anatt_timer); 4164 break; 4165 4166 case SPDK_NVME_ANA_INACCESSIBLE_STATE: 4167 case SPDK_NVME_ANA_CHANGE_STATE: 4168 if (nvme_ns->anatt_timer != NULL) { 4169 break; 4170 } 4171 4172 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 4173 nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout, 4174 nvme_ns, 4175 cdata->anatt * SPDK_SEC_TO_USEC); 4176 break; 4177 default: 4178 break; 4179 } 4180 } 4181 4182 static int 4183 nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg) 4184 { 4185 struct nvme_ns *nvme_ns = cb_arg; 4186 uint32_t i; 4187 4188 assert(nvme_ns->ns != NULL); 4189 4190 for (i = 0; i < desc->num_of_nsid; i++) { 4191 if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) { 4192 continue; 4193 } 4194 4195 _nvme_ns_set_ana_state(nvme_ns, desc); 4196 return 1; 4197 } 4198 4199 return 0; 4200 } 4201 4202 static int 4203 nvme_generate_uuid(const char *sn, uint32_t nsid, struct spdk_uuid *uuid) 4204 { 4205 int rc = 0; 4206 struct spdk_uuid new_uuid, namespace_uuid; 4207 char merged_str[SPDK_NVME_CTRLR_SN_LEN + NSID_STR_LEN + 1] = {'\0'}; 4208 /* This namespace UUID was generated using uuid_generate() method. */ 4209 const char *namespace_str = {"edaed2de-24bc-4b07-b559-f47ecbe730fd"}; 4210 int size; 4211 4212 assert(strlen(sn) <= SPDK_NVME_CTRLR_SN_LEN); 4213 4214 spdk_uuid_set_null(&new_uuid); 4215 spdk_uuid_set_null(&namespace_uuid); 4216 4217 size = snprintf(merged_str, sizeof(merged_str), "%s%"PRIu32, sn, nsid); 4218 if (size <= 0 || (unsigned long)size >= sizeof(merged_str)) { 4219 return -EINVAL; 4220 } 4221 4222 spdk_uuid_parse(&namespace_uuid, namespace_str); 4223 4224 rc = spdk_uuid_generate_sha1(&new_uuid, &namespace_uuid, merged_str, size); 4225 if (rc == 0) { 4226 memcpy(uuid, &new_uuid, sizeof(struct spdk_uuid)); 4227 } 4228 4229 return rc; 4230 } 4231 4232 static int 4233 nvme_disk_create(struct spdk_bdev *disk, const char *base_name, 4234 struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns, 4235 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, void *ctx) 4236 { 4237 const struct spdk_uuid *uuid; 4238 const uint8_t *nguid; 4239 const struct spdk_nvme_ctrlr_data *cdata; 4240 const struct spdk_nvme_ns_data *nsdata; 4241 const struct spdk_nvme_ctrlr_opts *opts; 4242 enum spdk_nvme_csi csi; 4243 uint32_t atomic_bs, phys_bs, bs; 4244 char sn_tmp[SPDK_NVME_CTRLR_SN_LEN + 1] = {'\0'}; 4245 int rc; 4246 4247 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4248 csi = spdk_nvme_ns_get_csi(ns); 4249 opts = spdk_nvme_ctrlr_get_opts(ctrlr); 4250 4251 switch (csi) { 4252 case SPDK_NVME_CSI_NVM: 4253 disk->product_name = "NVMe disk"; 4254 break; 4255 case SPDK_NVME_CSI_ZNS: 4256 disk->product_name = "NVMe ZNS disk"; 4257 disk->zoned = true; 4258 disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 4259 disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) / 4260 spdk_nvme_ns_get_extended_sector_size(ns); 4261 disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns); 4262 disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns); 4263 break; 4264 default: 4265 if (bdev_opts->allow_unrecognized_csi) { 4266 disk->product_name = "NVMe Passthrough disk"; 4267 break; 4268 } 4269 SPDK_ERRLOG("unsupported CSI: %u\n", csi); 4270 return -ENOTSUP; 4271 } 4272 4273 nguid = spdk_nvme_ns_get_nguid(ns); 4274 if (!nguid) { 4275 uuid = spdk_nvme_ns_get_uuid(ns); 4276 if (uuid) { 4277 disk->uuid = *uuid; 4278 } else if (g_opts.generate_uuids) { 4279 spdk_strcpy_pad(sn_tmp, cdata->sn, SPDK_NVME_CTRLR_SN_LEN, '\0'); 4280 rc = nvme_generate_uuid(sn_tmp, spdk_nvme_ns_get_id(ns), &disk->uuid); 4281 if (rc < 0) { 4282 SPDK_ERRLOG("UUID generation failed (%s)\n", spdk_strerror(-rc)); 4283 return rc; 4284 } 4285 } 4286 } else { 4287 memcpy(&disk->uuid, nguid, sizeof(disk->uuid)); 4288 } 4289 4290 disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns)); 4291 if (!disk->name) { 4292 return -ENOMEM; 4293 } 4294 4295 disk->write_cache = 0; 4296 if (cdata->vwc.present) { 4297 /* Enable if the Volatile Write Cache exists */ 4298 disk->write_cache = 1; 4299 } 4300 if (cdata->oncs.write_zeroes) { 4301 disk->max_write_zeroes = UINT16_MAX + 1; 4302 } 4303 disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns); 4304 disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns); 4305 disk->max_segment_size = spdk_nvme_ctrlr_get_max_xfer_size(ctrlr); 4306 disk->ctratt.raw = cdata->ctratt.raw; 4307 /* NVMe driver will split one request into multiple requests 4308 * based on MDTS and stripe boundary, the bdev layer will use 4309 * max_segment_size and max_num_segments to split one big IO 4310 * into multiple requests, then small request can't run out 4311 * of NVMe internal requests data structure. 4312 */ 4313 if (opts && opts->io_queue_requests) { 4314 disk->max_num_segments = opts->io_queue_requests / 2; 4315 } 4316 if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_SGL_SUPPORTED) { 4317 /* The nvme driver will try to split I/O that have too many 4318 * SGEs, but it doesn't work if that last SGE doesn't end on 4319 * an aggregate total that is block aligned. The bdev layer has 4320 * a more robust splitting framework, so use that instead for 4321 * this case. (See issue #3269.) 4322 */ 4323 uint16_t max_sges = spdk_nvme_ctrlr_get_max_sges(ctrlr); 4324 4325 if (disk->max_num_segments == 0) { 4326 disk->max_num_segments = max_sges; 4327 } else { 4328 disk->max_num_segments = spdk_min(disk->max_num_segments, max_sges); 4329 } 4330 } 4331 disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns); 4332 4333 nsdata = spdk_nvme_ns_get_data(ns); 4334 bs = spdk_nvme_ns_get_sector_size(ns); 4335 atomic_bs = bs; 4336 phys_bs = bs; 4337 if (nsdata->nabo == 0) { 4338 if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) { 4339 atomic_bs = bs * (1 + nsdata->nawupf); 4340 } else { 4341 atomic_bs = bs * (1 + cdata->awupf); 4342 } 4343 } 4344 if (nsdata->nsfeat.optperf) { 4345 phys_bs = bs * (1 + nsdata->npwg); 4346 } 4347 disk->phys_blocklen = spdk_min(phys_bs, atomic_bs); 4348 4349 disk->md_len = spdk_nvme_ns_get_md_size(ns); 4350 if (disk->md_len != 0) { 4351 disk->md_interleave = nsdata->flbas.extended; 4352 disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns); 4353 if (disk->dif_type != SPDK_DIF_DISABLE) { 4354 disk->dif_is_head_of_md = nsdata->dps.md_start; 4355 disk->dif_check_flags = bdev_opts->prchk_flags; 4356 disk->dif_pi_format = (enum spdk_dif_pi_format)spdk_nvme_ns_get_pi_format(ns); 4357 } 4358 } 4359 4360 if (!(spdk_nvme_ctrlr_get_flags(ctrlr) & 4361 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) { 4362 disk->acwu = 0; 4363 } else if (nsdata->nsfeat.ns_atomic_write_unit) { 4364 disk->acwu = nsdata->nacwu + 1; /* 0-based */ 4365 } else { 4366 disk->acwu = cdata->acwu + 1; /* 0-based */ 4367 } 4368 4369 if (cdata->oncs.copy) { 4370 /* For now bdev interface allows only single segment copy */ 4371 disk->max_copy = nsdata->mssrl; 4372 } 4373 4374 disk->ctxt = ctx; 4375 disk->fn_table = &nvmelib_fn_table; 4376 disk->module = &nvme_if; 4377 4378 disk->numa.id_valid = 1; 4379 disk->numa.id = spdk_nvme_ctrlr_get_numa_id(ctrlr); 4380 4381 return 0; 4382 } 4383 4384 static struct nvme_bdev * 4385 nvme_bdev_alloc(void) 4386 { 4387 struct nvme_bdev *bdev; 4388 int rc; 4389 4390 bdev = calloc(1, sizeof(*bdev)); 4391 if (!bdev) { 4392 SPDK_ERRLOG("bdev calloc() failed\n"); 4393 return NULL; 4394 } 4395 4396 if (g_opts.nvme_error_stat) { 4397 bdev->err_stat = calloc(1, sizeof(struct nvme_error_stat)); 4398 if (!bdev->err_stat) { 4399 SPDK_ERRLOG("err_stat calloc() failed\n"); 4400 free(bdev); 4401 return NULL; 4402 } 4403 } 4404 4405 rc = pthread_mutex_init(&bdev->mutex, NULL); 4406 if (rc != 0) { 4407 free(bdev->err_stat); 4408 free(bdev); 4409 return NULL; 4410 } 4411 4412 bdev->ref = 1; 4413 bdev->mp_policy = BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE; 4414 bdev->mp_selector = BDEV_NVME_MP_SELECTOR_ROUND_ROBIN; 4415 bdev->rr_min_io = UINT32_MAX; 4416 TAILQ_INIT(&bdev->nvme_ns_list); 4417 4418 return bdev; 4419 } 4420 4421 static int 4422 nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4423 { 4424 struct nvme_bdev *bdev; 4425 struct nvme_bdev_ctrlr *nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr; 4426 int rc; 4427 4428 bdev = nvme_bdev_alloc(); 4429 if (bdev == NULL) { 4430 SPDK_ERRLOG("Failed to allocate NVMe bdev\n"); 4431 return -ENOMEM; 4432 } 4433 4434 bdev->opal = nvme_ctrlr->opal_dev != NULL; 4435 4436 rc = nvme_disk_create(&bdev->disk, nbdev_ctrlr->name, nvme_ctrlr->ctrlr, 4437 nvme_ns->ns, &nvme_ctrlr->opts, bdev); 4438 if (rc != 0) { 4439 SPDK_ERRLOG("Failed to create NVMe disk\n"); 4440 nvme_bdev_free(bdev); 4441 return rc; 4442 } 4443 4444 spdk_io_device_register(bdev, 4445 bdev_nvme_create_bdev_channel_cb, 4446 bdev_nvme_destroy_bdev_channel_cb, 4447 sizeof(struct nvme_bdev_channel), 4448 bdev->disk.name); 4449 4450 nvme_ns->bdev = bdev; 4451 bdev->nsid = nvme_ns->id; 4452 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 4453 4454 bdev->nbdev_ctrlr = nbdev_ctrlr; 4455 TAILQ_INSERT_TAIL(&nbdev_ctrlr->bdevs, bdev, tailq); 4456 4457 rc = spdk_bdev_register(&bdev->disk); 4458 if (rc != 0) { 4459 SPDK_ERRLOG("spdk_bdev_register() failed\n"); 4460 spdk_io_device_unregister(bdev, NULL); 4461 nvme_ns->bdev = NULL; 4462 TAILQ_REMOVE(&nbdev_ctrlr->bdevs, bdev, tailq); 4463 nvme_bdev_free(bdev); 4464 return rc; 4465 } 4466 4467 return 0; 4468 } 4469 4470 static bool 4471 bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2) 4472 { 4473 const struct spdk_nvme_ns_data *nsdata1, *nsdata2; 4474 const struct spdk_uuid *uuid1, *uuid2; 4475 4476 nsdata1 = spdk_nvme_ns_get_data(ns1); 4477 nsdata2 = spdk_nvme_ns_get_data(ns2); 4478 uuid1 = spdk_nvme_ns_get_uuid(ns1); 4479 uuid2 = spdk_nvme_ns_get_uuid(ns2); 4480 4481 return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 && 4482 nsdata1->eui64 == nsdata2->eui64 && 4483 ((uuid1 == NULL && uuid2 == NULL) || 4484 (uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) && 4485 spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2); 4486 } 4487 4488 static bool 4489 hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 4490 struct spdk_nvme_ctrlr_opts *opts) 4491 { 4492 struct nvme_probe_skip_entry *entry; 4493 4494 TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) { 4495 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 4496 return false; 4497 } 4498 } 4499 4500 opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst; 4501 opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight; 4502 opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight; 4503 opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight; 4504 opts->disable_read_ana_log_page = true; 4505 4506 SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr); 4507 4508 return true; 4509 } 4510 4511 static void 4512 nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl) 4513 { 4514 struct nvme_ctrlr *nvme_ctrlr = ctx; 4515 4516 if (spdk_nvme_cpl_is_error(cpl)) { 4517 SPDK_WARNLOG("Abort failed. Resetting controller. sc is %u, sct is %u.\n", cpl->status.sc, 4518 cpl->status.sct); 4519 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4520 } else if (cpl->cdw0 & 0x1) { 4521 SPDK_WARNLOG("Specified command could not be aborted.\n"); 4522 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4523 } 4524 } 4525 4526 static void 4527 timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, 4528 struct spdk_nvme_qpair *qpair, uint16_t cid) 4529 { 4530 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 4531 union spdk_nvme_csts_register csts; 4532 int rc; 4533 4534 assert(nvme_ctrlr->ctrlr == ctrlr); 4535 4536 SPDK_WARNLOG("Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n", ctrlr, qpair, cid); 4537 4538 /* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O 4539 * queue. (Note: qpair == NULL when there's an admin cmd timeout.) Otherwise we 4540 * would submit another fabrics cmd on the admin queue to read CSTS and check for its 4541 * completion recursively. 4542 */ 4543 if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) { 4544 csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr); 4545 if (csts.bits.cfs) { 4546 SPDK_ERRLOG("Controller Fatal Status, reset required\n"); 4547 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4548 return; 4549 } 4550 } 4551 4552 switch (g_opts.action_on_timeout) { 4553 case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT: 4554 if (qpair) { 4555 /* Don't send abort to ctrlr when ctrlr is not available. */ 4556 pthread_mutex_lock(&nvme_ctrlr->mutex); 4557 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 4558 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4559 SPDK_NOTICELOG("Quit abort. Ctrlr is not available.\n"); 4560 return; 4561 } 4562 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4563 4564 rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid, 4565 nvme_abort_cpl, nvme_ctrlr); 4566 if (rc == 0) { 4567 return; 4568 } 4569 4570 SPDK_ERRLOG("Unable to send abort. Resetting, rc is %d.\n", rc); 4571 } 4572 4573 /* FALLTHROUGH */ 4574 case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET: 4575 bdev_nvme_reset_ctrlr(nvme_ctrlr); 4576 break; 4577 case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE: 4578 SPDK_DEBUGLOG(bdev_nvme, "No action for nvme controller timeout.\n"); 4579 break; 4580 default: 4581 SPDK_ERRLOG("An invalid timeout action value is found.\n"); 4582 break; 4583 } 4584 } 4585 4586 static struct nvme_ns * 4587 nvme_ns_alloc(void) 4588 { 4589 struct nvme_ns *nvme_ns; 4590 4591 nvme_ns = calloc(1, sizeof(struct nvme_ns)); 4592 if (nvme_ns == NULL) { 4593 return NULL; 4594 } 4595 4596 if (g_opts.io_path_stat) { 4597 nvme_ns->stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 4598 if (nvme_ns->stat == NULL) { 4599 free(nvme_ns); 4600 return NULL; 4601 } 4602 spdk_bdev_reset_io_stat(nvme_ns->stat, SPDK_BDEV_RESET_STAT_MAXMIN); 4603 } 4604 4605 return nvme_ns; 4606 } 4607 4608 static void 4609 nvme_ns_free(struct nvme_ns *nvme_ns) 4610 { 4611 free(nvme_ns->stat); 4612 free(nvme_ns); 4613 } 4614 4615 static void 4616 nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc) 4617 { 4618 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 4619 struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx; 4620 4621 if (rc == 0) { 4622 nvme_ns->probe_ctx = NULL; 4623 pthread_mutex_lock(&nvme_ctrlr->mutex); 4624 nvme_ctrlr->ref++; 4625 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4626 } else { 4627 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4628 nvme_ns_free(nvme_ns); 4629 } 4630 4631 if (ctx) { 4632 ctx->populates_in_progress--; 4633 if (ctx->populates_in_progress == 0) { 4634 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 4635 } 4636 } 4637 } 4638 4639 static void 4640 bdev_nvme_add_io_path(struct nvme_bdev_channel_iter *i, 4641 struct nvme_bdev *nbdev, 4642 struct nvme_bdev_channel *nbdev_ch, void *ctx) 4643 { 4644 struct nvme_ns *nvme_ns = ctx; 4645 int rc; 4646 4647 rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns); 4648 if (rc != 0) { 4649 SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n"); 4650 } 4651 4652 nvme_bdev_for_each_channel_continue(i, rc); 4653 } 4654 4655 static void 4656 bdev_nvme_delete_io_path(struct nvme_bdev_channel_iter *i, 4657 struct nvme_bdev *nbdev, 4658 struct nvme_bdev_channel *nbdev_ch, void *ctx) 4659 { 4660 struct nvme_ns *nvme_ns = ctx; 4661 struct nvme_io_path *io_path; 4662 4663 io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns); 4664 if (io_path != NULL) { 4665 _bdev_nvme_delete_io_path(nbdev_ch, io_path); 4666 } 4667 4668 nvme_bdev_for_each_channel_continue(i, 0); 4669 } 4670 4671 static void 4672 bdev_nvme_add_io_path_failed(struct nvme_bdev *nbdev, void *ctx, int status) 4673 { 4674 struct nvme_ns *nvme_ns = ctx; 4675 4676 nvme_ctrlr_populate_namespace_done(nvme_ns, -1); 4677 } 4678 4679 static void 4680 bdev_nvme_add_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status) 4681 { 4682 struct nvme_ns *nvme_ns = ctx; 4683 4684 if (status == 0) { 4685 nvme_ctrlr_populate_namespace_done(nvme_ns, 0); 4686 } else { 4687 /* Delete the added io_paths and fail populating the namespace. */ 4688 nvme_bdev_for_each_channel(nbdev, 4689 bdev_nvme_delete_io_path, 4690 nvme_ns, 4691 bdev_nvme_add_io_path_failed); 4692 } 4693 } 4694 4695 static int 4696 nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns) 4697 { 4698 struct nvme_ns *tmp_ns; 4699 const struct spdk_nvme_ns_data *nsdata; 4700 4701 nsdata = spdk_nvme_ns_get_data(nvme_ns->ns); 4702 if (!nsdata->nmic.can_share) { 4703 SPDK_ERRLOG("Namespace cannot be shared.\n"); 4704 return -EINVAL; 4705 } 4706 4707 pthread_mutex_lock(&bdev->mutex); 4708 4709 tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list); 4710 assert(tmp_ns != NULL); 4711 4712 if (tmp_ns->ns != NULL && !bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) { 4713 pthread_mutex_unlock(&bdev->mutex); 4714 SPDK_ERRLOG("Namespaces are not identical.\n"); 4715 return -EINVAL; 4716 } 4717 4718 bdev->ref++; 4719 TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq); 4720 nvme_ns->bdev = bdev; 4721 4722 pthread_mutex_unlock(&bdev->mutex); 4723 4724 /* Add nvme_io_path to nvme_bdev_channels dynamically. */ 4725 nvme_bdev_for_each_channel(bdev, 4726 bdev_nvme_add_io_path, 4727 nvme_ns, 4728 bdev_nvme_add_io_path_done); 4729 4730 return 0; 4731 } 4732 4733 static void 4734 nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4735 { 4736 struct spdk_nvme_ns *ns; 4737 struct nvme_bdev *bdev; 4738 int rc = 0; 4739 4740 ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id); 4741 if (!ns) { 4742 SPDK_DEBUGLOG(bdev_nvme, "Invalid NS %d\n", nvme_ns->id); 4743 rc = -EINVAL; 4744 goto done; 4745 } 4746 4747 nvme_ns->ns = ns; 4748 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 4749 4750 if (nvme_ctrlr->ana_log_page != NULL) { 4751 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns); 4752 } 4753 4754 bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id); 4755 if (bdev == NULL) { 4756 rc = nvme_bdev_create(nvme_ctrlr, nvme_ns); 4757 } else { 4758 rc = nvme_bdev_add_ns(bdev, nvme_ns); 4759 if (rc == 0) { 4760 return; 4761 } 4762 } 4763 done: 4764 nvme_ctrlr_populate_namespace_done(nvme_ns, rc); 4765 } 4766 4767 static void 4768 nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns) 4769 { 4770 struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr; 4771 4772 assert(nvme_ctrlr != NULL); 4773 4774 pthread_mutex_lock(&nvme_ctrlr->mutex); 4775 4776 RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4777 4778 if (nvme_ns->bdev != NULL) { 4779 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4780 return; 4781 } 4782 4783 nvme_ns_free(nvme_ns); 4784 pthread_mutex_unlock(&nvme_ctrlr->mutex); 4785 4786 nvme_ctrlr_release(nvme_ctrlr); 4787 } 4788 4789 static void 4790 bdev_nvme_delete_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status) 4791 { 4792 struct nvme_ns *nvme_ns = ctx; 4793 4794 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 4795 } 4796 4797 static void 4798 nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 4799 { 4800 struct nvme_bdev *bdev; 4801 4802 spdk_poller_unregister(&nvme_ns->anatt_timer); 4803 4804 bdev = nvme_ns->bdev; 4805 if (bdev != NULL) { 4806 pthread_mutex_lock(&bdev->mutex); 4807 4808 assert(bdev->ref > 0); 4809 bdev->ref--; 4810 if (bdev->ref == 0) { 4811 pthread_mutex_unlock(&bdev->mutex); 4812 4813 spdk_bdev_unregister(&bdev->disk, NULL, NULL); 4814 } else { 4815 /* spdk_bdev_unregister() is not called until the last nvme_ns is 4816 * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list 4817 * and clear nvme_ns->bdev here. 4818 */ 4819 TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq); 4820 nvme_ns->bdev = NULL; 4821 4822 pthread_mutex_unlock(&bdev->mutex); 4823 4824 /* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that, 4825 * we call depopulate_namespace_done() to avoid use-after-free. 4826 */ 4827 nvme_bdev_for_each_channel(bdev, 4828 bdev_nvme_delete_io_path, 4829 nvme_ns, 4830 bdev_nvme_delete_io_path_done); 4831 return; 4832 } 4833 } 4834 4835 nvme_ctrlr_depopulate_namespace_done(nvme_ns); 4836 } 4837 4838 static void 4839 nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr, 4840 struct nvme_async_probe_ctx *ctx) 4841 { 4842 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4843 struct nvme_ns *nvme_ns, *next; 4844 struct spdk_nvme_ns *ns; 4845 struct nvme_bdev *bdev; 4846 uint32_t nsid; 4847 int rc; 4848 uint64_t num_sectors; 4849 4850 if (ctx) { 4851 /* Initialize this count to 1 to handle the populate functions 4852 * calling nvme_ctrlr_populate_namespace_done() immediately. 4853 */ 4854 ctx->populates_in_progress = 1; 4855 } 4856 4857 /* First loop over our existing namespaces and see if they have been 4858 * removed. */ 4859 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 4860 while (nvme_ns != NULL) { 4861 next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 4862 4863 if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) { 4864 /* NS is still there or added again. Its attributes may have changed. */ 4865 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id); 4866 if (nvme_ns->ns != ns) { 4867 assert(nvme_ns->ns == NULL); 4868 nvme_ns->ns = ns; 4869 SPDK_DEBUGLOG(bdev_nvme, "NSID %u was added\n", nvme_ns->id); 4870 } 4871 4872 num_sectors = spdk_nvme_ns_get_num_sectors(ns); 4873 bdev = nvme_ns->bdev; 4874 assert(bdev != NULL); 4875 if (bdev->disk.blockcnt != num_sectors) { 4876 SPDK_NOTICELOG("NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n", 4877 nvme_ns->id, 4878 bdev->disk.name, 4879 bdev->disk.blockcnt, 4880 num_sectors); 4881 rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors); 4882 if (rc != 0) { 4883 SPDK_ERRLOG("Could not change num blocks for nvme bdev: name %s, errno: %d.\n", 4884 bdev->disk.name, rc); 4885 } 4886 } 4887 } else { 4888 /* Namespace was removed */ 4889 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 4890 } 4891 4892 nvme_ns = next; 4893 } 4894 4895 /* Loop through all of the namespaces at the nvme level and see if any of them are new */ 4896 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 4897 while (nsid != 0) { 4898 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 4899 4900 if (nvme_ns == NULL) { 4901 /* Found a new one */ 4902 nvme_ns = nvme_ns_alloc(); 4903 if (nvme_ns == NULL) { 4904 SPDK_ERRLOG("Failed to allocate namespace\n"); 4905 /* This just fails to attach the namespace. It may work on a future attempt. */ 4906 continue; 4907 } 4908 4909 nvme_ns->id = nsid; 4910 nvme_ns->ctrlr = nvme_ctrlr; 4911 4912 nvme_ns->bdev = NULL; 4913 4914 if (ctx) { 4915 ctx->populates_in_progress++; 4916 } 4917 nvme_ns->probe_ctx = ctx; 4918 4919 RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns); 4920 4921 nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns); 4922 } 4923 4924 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid); 4925 } 4926 4927 if (ctx) { 4928 /* Decrement this count now that the loop is over to account 4929 * for the one we started with. If the count is then 0, we 4930 * know any populate_namespace functions completed immediately, 4931 * so we'll kick the callback here. 4932 */ 4933 ctx->populates_in_progress--; 4934 if (ctx->populates_in_progress == 0) { 4935 nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx); 4936 } 4937 } 4938 4939 } 4940 4941 static void 4942 nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr) 4943 { 4944 struct nvme_ns *nvme_ns, *tmp; 4945 4946 RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) { 4947 nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns); 4948 } 4949 } 4950 4951 static uint32_t 4952 nvme_ctrlr_get_ana_log_page_size(struct nvme_ctrlr *nvme_ctrlr) 4953 { 4954 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 4955 const struct spdk_nvme_ctrlr_data *cdata; 4956 uint32_t nsid, ns_count = 0; 4957 4958 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 4959 4960 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 4961 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 4962 ns_count++; 4963 } 4964 4965 return sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 4966 sizeof(struct spdk_nvme_ana_group_descriptor) + ns_count * 4967 sizeof(uint32_t); 4968 } 4969 4970 static int 4971 nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc, 4972 void *cb_arg) 4973 { 4974 struct nvme_ctrlr *nvme_ctrlr = cb_arg; 4975 struct nvme_ns *nvme_ns; 4976 uint32_t i, nsid; 4977 4978 for (i = 0; i < desc->num_of_nsid; i++) { 4979 nsid = desc->nsid[i]; 4980 if (nsid == 0) { 4981 continue; 4982 } 4983 4984 nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid); 4985 4986 if (nvme_ns == NULL) { 4987 /* Target told us that an inactive namespace had an ANA change */ 4988 continue; 4989 } 4990 4991 _nvme_ns_set_ana_state(nvme_ns, desc); 4992 } 4993 4994 return 0; 4995 } 4996 4997 static void 4998 bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 4999 { 5000 struct nvme_ns *nvme_ns; 5001 5002 spdk_free(nvme_ctrlr->ana_log_page); 5003 nvme_ctrlr->ana_log_page = NULL; 5004 5005 for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 5006 nvme_ns != NULL; 5007 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) { 5008 nvme_ns->ana_state_updating = false; 5009 nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 5010 } 5011 } 5012 5013 static void 5014 nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl) 5015 { 5016 struct nvme_ctrlr *nvme_ctrlr = ctx; 5017 5018 if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) { 5019 bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states, 5020 nvme_ctrlr); 5021 } else { 5022 bdev_nvme_disable_read_ana_log_page(nvme_ctrlr); 5023 } 5024 5025 pthread_mutex_lock(&nvme_ctrlr->mutex); 5026 5027 assert(nvme_ctrlr->ana_log_page_updating == true); 5028 nvme_ctrlr->ana_log_page_updating = false; 5029 5030 if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) { 5031 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5032 5033 nvme_ctrlr_unregister(nvme_ctrlr); 5034 } else { 5035 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5036 5037 bdev_nvme_clear_io_path_caches(nvme_ctrlr); 5038 } 5039 } 5040 5041 static int 5042 nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr) 5043 { 5044 uint32_t ana_log_page_size; 5045 int rc; 5046 5047 if (nvme_ctrlr->ana_log_page == NULL) { 5048 return -EINVAL; 5049 } 5050 5051 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 5052 5053 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 5054 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 5055 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 5056 return -EINVAL; 5057 } 5058 5059 pthread_mutex_lock(&nvme_ctrlr->mutex); 5060 if (!nvme_ctrlr_is_available(nvme_ctrlr) || 5061 nvme_ctrlr->ana_log_page_updating) { 5062 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5063 return -EBUSY; 5064 } 5065 5066 nvme_ctrlr->ana_log_page_updating = true; 5067 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5068 5069 rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr, 5070 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 5071 SPDK_NVME_GLOBAL_NS_TAG, 5072 nvme_ctrlr->ana_log_page, 5073 ana_log_page_size, 0, 5074 nvme_ctrlr_read_ana_log_page_done, 5075 nvme_ctrlr); 5076 if (rc != 0) { 5077 nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL); 5078 } 5079 5080 return rc; 5081 } 5082 5083 static void 5084 dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 5085 { 5086 } 5087 5088 struct bdev_nvme_set_preferred_path_ctx { 5089 struct spdk_bdev_desc *desc; 5090 struct nvme_ns *nvme_ns; 5091 bdev_nvme_set_preferred_path_cb cb_fn; 5092 void *cb_arg; 5093 }; 5094 5095 static void 5096 bdev_nvme_set_preferred_path_done(struct nvme_bdev *nbdev, void *_ctx, int status) 5097 { 5098 struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx; 5099 5100 assert(ctx != NULL); 5101 assert(ctx->desc != NULL); 5102 assert(ctx->cb_fn != NULL); 5103 5104 spdk_bdev_close(ctx->desc); 5105 5106 ctx->cb_fn(ctx->cb_arg, status); 5107 5108 free(ctx); 5109 } 5110 5111 static void 5112 _bdev_nvme_set_preferred_path(struct nvme_bdev_channel_iter *i, 5113 struct nvme_bdev *nbdev, 5114 struct nvme_bdev_channel *nbdev_ch, void *_ctx) 5115 { 5116 struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx; 5117 struct nvme_io_path *io_path, *prev; 5118 5119 prev = NULL; 5120 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 5121 if (io_path->nvme_ns == ctx->nvme_ns) { 5122 break; 5123 } 5124 prev = io_path; 5125 } 5126 5127 if (io_path != NULL) { 5128 if (prev != NULL) { 5129 STAILQ_REMOVE_AFTER(&nbdev_ch->io_path_list, prev, stailq); 5130 STAILQ_INSERT_HEAD(&nbdev_ch->io_path_list, io_path, stailq); 5131 } 5132 5133 /* We can set io_path to nbdev_ch->current_io_path directly here. 5134 * However, it needs to be conditional. To simplify the code, 5135 * just clear nbdev_ch->current_io_path and let find_io_path() 5136 * fill it. 5137 * 5138 * Automatic failback may be disabled. Hence even if the io_path is 5139 * already at the head, clear nbdev_ch->current_io_path. 5140 */ 5141 bdev_nvme_clear_current_io_path(nbdev_ch); 5142 } 5143 5144 nvme_bdev_for_each_channel_continue(i, 0); 5145 } 5146 5147 static struct nvme_ns * 5148 bdev_nvme_set_preferred_ns(struct nvme_bdev *nbdev, uint16_t cntlid) 5149 { 5150 struct nvme_ns *nvme_ns, *prev; 5151 const struct spdk_nvme_ctrlr_data *cdata; 5152 5153 prev = NULL; 5154 TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { 5155 cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr); 5156 5157 if (cdata->cntlid == cntlid) { 5158 break; 5159 } 5160 prev = nvme_ns; 5161 } 5162 5163 if (nvme_ns != NULL && prev != NULL) { 5164 TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq); 5165 TAILQ_INSERT_HEAD(&nbdev->nvme_ns_list, nvme_ns, tailq); 5166 } 5167 5168 return nvme_ns; 5169 } 5170 5171 /* This function supports only multipath mode. There is only a single I/O path 5172 * for each NVMe-oF controller. Hence, just move the matched I/O path to the 5173 * head of the I/O path list for each NVMe bdev channel. 5174 * 5175 * NVMe bdev channel may be acquired after completing this function. move the 5176 * matched namespace to the head of the namespace list for the NVMe bdev too. 5177 */ 5178 void 5179 bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid, 5180 bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg) 5181 { 5182 struct bdev_nvme_set_preferred_path_ctx *ctx; 5183 struct spdk_bdev *bdev; 5184 struct nvme_bdev *nbdev; 5185 int rc = 0; 5186 5187 assert(cb_fn != NULL); 5188 5189 ctx = calloc(1, sizeof(*ctx)); 5190 if (ctx == NULL) { 5191 SPDK_ERRLOG("Failed to alloc context.\n"); 5192 rc = -ENOMEM; 5193 goto err_alloc; 5194 } 5195 5196 ctx->cb_fn = cb_fn; 5197 ctx->cb_arg = cb_arg; 5198 5199 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 5200 if (rc != 0) { 5201 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 5202 goto err_open; 5203 } 5204 5205 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 5206 5207 if (bdev->module != &nvme_if) { 5208 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 5209 rc = -ENODEV; 5210 goto err_bdev; 5211 } 5212 5213 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 5214 5215 pthread_mutex_lock(&nbdev->mutex); 5216 5217 ctx->nvme_ns = bdev_nvme_set_preferred_ns(nbdev, cntlid); 5218 if (ctx->nvme_ns == NULL) { 5219 pthread_mutex_unlock(&nbdev->mutex); 5220 5221 SPDK_ERRLOG("bdev %s does not have namespace to controller %u.\n", name, cntlid); 5222 rc = -ENODEV; 5223 goto err_bdev; 5224 } 5225 5226 pthread_mutex_unlock(&nbdev->mutex); 5227 5228 nvme_bdev_for_each_channel(nbdev, 5229 _bdev_nvme_set_preferred_path, 5230 ctx, 5231 bdev_nvme_set_preferred_path_done); 5232 return; 5233 5234 err_bdev: 5235 spdk_bdev_close(ctx->desc); 5236 err_open: 5237 free(ctx); 5238 err_alloc: 5239 cb_fn(cb_arg, rc); 5240 } 5241 5242 struct bdev_nvme_set_multipath_policy_ctx { 5243 struct spdk_bdev_desc *desc; 5244 spdk_bdev_nvme_set_multipath_policy_cb cb_fn; 5245 void *cb_arg; 5246 }; 5247 5248 static void 5249 bdev_nvme_set_multipath_policy_done(struct nvme_bdev *nbdev, void *_ctx, int status) 5250 { 5251 struct bdev_nvme_set_multipath_policy_ctx *ctx = _ctx; 5252 5253 assert(ctx != NULL); 5254 assert(ctx->desc != NULL); 5255 assert(ctx->cb_fn != NULL); 5256 5257 spdk_bdev_close(ctx->desc); 5258 5259 ctx->cb_fn(ctx->cb_arg, status); 5260 5261 free(ctx); 5262 } 5263 5264 static void 5265 _bdev_nvme_set_multipath_policy(struct nvme_bdev_channel_iter *i, 5266 struct nvme_bdev *nbdev, 5267 struct nvme_bdev_channel *nbdev_ch, void *ctx) 5268 { 5269 nbdev_ch->mp_policy = nbdev->mp_policy; 5270 nbdev_ch->mp_selector = nbdev->mp_selector; 5271 nbdev_ch->rr_min_io = nbdev->rr_min_io; 5272 bdev_nvme_clear_current_io_path(nbdev_ch); 5273 5274 nvme_bdev_for_each_channel_continue(i, 0); 5275 } 5276 5277 void 5278 spdk_bdev_nvme_set_multipath_policy(const char *name, enum spdk_bdev_nvme_multipath_policy policy, 5279 enum spdk_bdev_nvme_multipath_selector selector, uint32_t rr_min_io, 5280 spdk_bdev_nvme_set_multipath_policy_cb cb_fn, void *cb_arg) 5281 { 5282 struct bdev_nvme_set_multipath_policy_ctx *ctx; 5283 struct spdk_bdev *bdev; 5284 struct nvme_bdev *nbdev; 5285 int rc; 5286 5287 assert(cb_fn != NULL); 5288 5289 switch (policy) { 5290 case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE: 5291 break; 5292 case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE: 5293 switch (selector) { 5294 case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN: 5295 if (rr_min_io == UINT32_MAX) { 5296 rr_min_io = 1; 5297 } else if (rr_min_io == 0) { 5298 rc = -EINVAL; 5299 goto exit; 5300 } 5301 break; 5302 case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH: 5303 break; 5304 default: 5305 rc = -EINVAL; 5306 goto exit; 5307 } 5308 break; 5309 default: 5310 rc = -EINVAL; 5311 goto exit; 5312 } 5313 5314 ctx = calloc(1, sizeof(*ctx)); 5315 if (ctx == NULL) { 5316 SPDK_ERRLOG("Failed to alloc context.\n"); 5317 rc = -ENOMEM; 5318 goto exit; 5319 } 5320 5321 ctx->cb_fn = cb_fn; 5322 ctx->cb_arg = cb_arg; 5323 5324 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc); 5325 if (rc != 0) { 5326 SPDK_ERRLOG("Failed to open bdev %s.\n", name); 5327 rc = -ENODEV; 5328 goto err_open; 5329 } 5330 5331 bdev = spdk_bdev_desc_get_bdev(ctx->desc); 5332 if (bdev->module != &nvme_if) { 5333 SPDK_ERRLOG("bdev %s is not registered in this module.\n", name); 5334 rc = -ENODEV; 5335 goto err_module; 5336 } 5337 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 5338 5339 pthread_mutex_lock(&nbdev->mutex); 5340 nbdev->mp_policy = policy; 5341 nbdev->mp_selector = selector; 5342 nbdev->rr_min_io = rr_min_io; 5343 pthread_mutex_unlock(&nbdev->mutex); 5344 5345 nvme_bdev_for_each_channel(nbdev, 5346 _bdev_nvme_set_multipath_policy, 5347 ctx, 5348 bdev_nvme_set_multipath_policy_done); 5349 return; 5350 5351 err_module: 5352 spdk_bdev_close(ctx->desc); 5353 err_open: 5354 free(ctx); 5355 exit: 5356 cb_fn(cb_arg, rc); 5357 } 5358 5359 static void 5360 aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 5361 { 5362 struct nvme_ctrlr *nvme_ctrlr = arg; 5363 union spdk_nvme_async_event_completion event; 5364 5365 if (spdk_nvme_cpl_is_error(cpl)) { 5366 SPDK_WARNLOG("AER request execute failed\n"); 5367 return; 5368 } 5369 5370 event.raw = cpl->cdw0; 5371 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 5372 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) { 5373 nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL); 5374 } else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 5375 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) { 5376 nvme_ctrlr_read_ana_log_page(nvme_ctrlr); 5377 } 5378 } 5379 5380 static void 5381 free_nvme_async_probe_ctx(struct nvme_async_probe_ctx *ctx) 5382 { 5383 spdk_keyring_put_key(ctx->drv_opts.tls_psk); 5384 spdk_keyring_put_key(ctx->drv_opts.dhchap_key); 5385 spdk_keyring_put_key(ctx->drv_opts.dhchap_ctrlr_key); 5386 free(ctx); 5387 } 5388 5389 static void 5390 populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, int rc) 5391 { 5392 if (ctx->cb_fn) { 5393 ctx->cb_fn(ctx->cb_ctx, ctx->reported_bdevs, rc); 5394 } 5395 5396 ctx->namespaces_populated = true; 5397 if (ctx->probe_done) { 5398 /* The probe was already completed, so we need to free the context 5399 * here. This can happen for cases like OCSSD, where we need to 5400 * send additional commands to the SSD after attach. 5401 */ 5402 free_nvme_async_probe_ctx(ctx); 5403 } 5404 } 5405 5406 static int 5407 bdev_nvme_remove_poller(void *ctx) 5408 { 5409 struct spdk_nvme_transport_id trid_pcie; 5410 5411 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 5412 spdk_poller_unregister(&g_hotplug_poller); 5413 return SPDK_POLLER_IDLE; 5414 } 5415 5416 memset(&trid_pcie, 0, sizeof(trid_pcie)); 5417 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 5418 5419 if (spdk_nvme_scan_attached(&trid_pcie)) { 5420 SPDK_ERRLOG_RATELIMIT("spdk_nvme_scan_attached() failed\n"); 5421 } 5422 5423 return SPDK_POLLER_BUSY; 5424 } 5425 5426 static void 5427 nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr, 5428 struct nvme_async_probe_ctx *ctx) 5429 { 5430 spdk_io_device_register(nvme_ctrlr, 5431 bdev_nvme_create_ctrlr_channel_cb, 5432 bdev_nvme_destroy_ctrlr_channel_cb, 5433 sizeof(struct nvme_ctrlr_channel), 5434 nvme_ctrlr->nbdev_ctrlr->name); 5435 5436 nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx); 5437 5438 if (g_hotplug_poller == NULL) { 5439 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL, 5440 NVME_HOTPLUG_POLL_PERIOD_DEFAULT); 5441 } 5442 } 5443 5444 static void 5445 nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl) 5446 { 5447 struct nvme_ctrlr *nvme_ctrlr = _ctx; 5448 struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx; 5449 5450 nvme_ctrlr->probe_ctx = NULL; 5451 5452 if (spdk_nvme_cpl_is_error(cpl)) { 5453 nvme_ctrlr_delete(nvme_ctrlr); 5454 5455 if (ctx != NULL) { 5456 ctx->reported_bdevs = 0; 5457 populate_namespaces_cb(ctx, -1); 5458 } 5459 return; 5460 } 5461 5462 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 5463 } 5464 5465 static int 5466 nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr, 5467 struct nvme_async_probe_ctx *ctx) 5468 { 5469 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 5470 const struct spdk_nvme_ctrlr_data *cdata; 5471 uint32_t ana_log_page_size; 5472 5473 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5474 5475 /* Set buffer size enough to include maximum number of allowed namespaces. */ 5476 ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid * 5477 sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->mnan * 5478 sizeof(uint32_t); 5479 5480 nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL, 5481 SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_DMA); 5482 if (nvme_ctrlr->ana_log_page == NULL) { 5483 SPDK_ERRLOG("could not allocate ANA log page buffer\n"); 5484 return -ENXIO; 5485 } 5486 5487 /* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned. 5488 * Hence copy each descriptor to a temporary area when parsing it. 5489 * 5490 * Allocate a buffer whose size is as large as ANA log page buffer because 5491 * we do not know the size of a descriptor until actually reading it. 5492 */ 5493 nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size); 5494 if (nvme_ctrlr->copied_ana_desc == NULL) { 5495 SPDK_ERRLOG("could not allocate a buffer to parse ANA descriptor\n"); 5496 return -ENOMEM; 5497 } 5498 5499 nvme_ctrlr->max_ana_log_page_size = ana_log_page_size; 5500 5501 nvme_ctrlr->probe_ctx = ctx; 5502 5503 /* Then, set the read size only to include the current active namespaces. */ 5504 ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr); 5505 5506 if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) { 5507 SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n", 5508 ana_log_page_size, nvme_ctrlr->max_ana_log_page_size); 5509 return -EINVAL; 5510 } 5511 5512 return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, 5513 SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 5514 SPDK_NVME_GLOBAL_NS_TAG, 5515 nvme_ctrlr->ana_log_page, 5516 ana_log_page_size, 0, 5517 nvme_ctrlr_init_ana_log_page_done, 5518 nvme_ctrlr); 5519 } 5520 5521 /* hostnqn and subnqn were already verified before attaching a controller. 5522 * Hence check only the multipath capability and cntlid here. 5523 */ 5524 static bool 5525 bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr) 5526 { 5527 struct nvme_ctrlr *tmp; 5528 const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata; 5529 5530 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5531 5532 if (!cdata->cmic.multi_ctrlr) { 5533 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 5534 return false; 5535 } 5536 5537 TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) { 5538 tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr); 5539 5540 if (!tmp_cdata->cmic.multi_ctrlr) { 5541 SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid); 5542 return false; 5543 } 5544 if (cdata->cntlid == tmp_cdata->cntlid) { 5545 SPDK_ERRLOG("cntlid %u are duplicated.\n", tmp_cdata->cntlid); 5546 return false; 5547 } 5548 } 5549 5550 return true; 5551 } 5552 5553 static int 5554 nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr) 5555 { 5556 struct nvme_bdev_ctrlr *nbdev_ctrlr; 5557 struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr; 5558 int rc = 0; 5559 5560 pthread_mutex_lock(&g_bdev_nvme_mutex); 5561 5562 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 5563 if (nbdev_ctrlr != NULL) { 5564 if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) { 5565 rc = -EINVAL; 5566 goto exit; 5567 } 5568 } else { 5569 nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr)); 5570 if (nbdev_ctrlr == NULL) { 5571 SPDK_ERRLOG("Failed to allocate nvme_bdev_ctrlr.\n"); 5572 rc = -ENOMEM; 5573 goto exit; 5574 } 5575 nbdev_ctrlr->name = strdup(name); 5576 if (nbdev_ctrlr->name == NULL) { 5577 SPDK_ERRLOG("Failed to allocate name of nvme_bdev_ctrlr.\n"); 5578 free(nbdev_ctrlr); 5579 goto exit; 5580 } 5581 TAILQ_INIT(&nbdev_ctrlr->ctrlrs); 5582 TAILQ_INIT(&nbdev_ctrlr->bdevs); 5583 TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq); 5584 } 5585 nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr; 5586 TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq); 5587 exit: 5588 pthread_mutex_unlock(&g_bdev_nvme_mutex); 5589 return rc; 5590 } 5591 5592 static int 5593 nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr, 5594 const char *name, 5595 const struct spdk_nvme_transport_id *trid, 5596 struct nvme_async_probe_ctx *ctx) 5597 { 5598 struct nvme_ctrlr *nvme_ctrlr; 5599 struct nvme_path_id *path_id; 5600 const struct spdk_nvme_ctrlr_data *cdata; 5601 int rc; 5602 5603 nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr)); 5604 if (nvme_ctrlr == NULL) { 5605 SPDK_ERRLOG("Failed to allocate device struct\n"); 5606 return -ENOMEM; 5607 } 5608 5609 rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL); 5610 if (rc != 0) { 5611 free(nvme_ctrlr); 5612 return rc; 5613 } 5614 5615 TAILQ_INIT(&nvme_ctrlr->trids); 5616 RB_INIT(&nvme_ctrlr->namespaces); 5617 5618 /* Get another reference to the key, so the first one can be released from probe_ctx */ 5619 if (ctx != NULL) { 5620 if (ctx->drv_opts.tls_psk != NULL) { 5621 nvme_ctrlr->psk = spdk_keyring_get_key( 5622 spdk_key_get_name(ctx->drv_opts.tls_psk)); 5623 if (nvme_ctrlr->psk == NULL) { 5624 /* Could only happen if the key was removed in the meantime */ 5625 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5626 spdk_key_get_name(ctx->drv_opts.tls_psk)); 5627 rc = -ENOKEY; 5628 goto err; 5629 } 5630 } 5631 5632 if (ctx->drv_opts.dhchap_key != NULL) { 5633 nvme_ctrlr->dhchap_key = spdk_keyring_get_key( 5634 spdk_key_get_name(ctx->drv_opts.dhchap_key)); 5635 if (nvme_ctrlr->dhchap_key == NULL) { 5636 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5637 spdk_key_get_name(ctx->drv_opts.dhchap_key)); 5638 rc = -ENOKEY; 5639 goto err; 5640 } 5641 } 5642 5643 if (ctx->drv_opts.dhchap_ctrlr_key != NULL) { 5644 nvme_ctrlr->dhchap_ctrlr_key = 5645 spdk_keyring_get_key( 5646 spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key)); 5647 if (nvme_ctrlr->dhchap_ctrlr_key == NULL) { 5648 SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n", 5649 spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key)); 5650 rc = -ENOKEY; 5651 goto err; 5652 } 5653 } 5654 } 5655 5656 path_id = calloc(1, sizeof(*path_id)); 5657 if (path_id == NULL) { 5658 SPDK_ERRLOG("Failed to allocate trid entry pointer\n"); 5659 rc = -ENOMEM; 5660 goto err; 5661 } 5662 5663 path_id->trid = *trid; 5664 if (ctx != NULL) { 5665 memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr)); 5666 memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid)); 5667 } 5668 nvme_ctrlr->active_path_id = path_id; 5669 TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link); 5670 5671 nvme_ctrlr->thread = spdk_get_thread(); 5672 nvme_ctrlr->ctrlr = ctrlr; 5673 nvme_ctrlr->ref = 1; 5674 5675 if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) { 5676 SPDK_ERRLOG("OCSSDs are not supported"); 5677 rc = -ENOTSUP; 5678 goto err; 5679 } 5680 5681 if (ctx != NULL) { 5682 memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts)); 5683 } else { 5684 spdk_bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts); 5685 } 5686 5687 nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr, 5688 g_opts.nvme_adminq_poll_period_us); 5689 5690 if (g_opts.timeout_us > 0) { 5691 /* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */ 5692 /* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */ 5693 uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ? 5694 g_opts.timeout_us : g_opts.timeout_admin_us; 5695 spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us, 5696 adm_timeout_us, timeout_cb, nvme_ctrlr); 5697 } 5698 5699 spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr); 5700 spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr); 5701 5702 if (spdk_nvme_ctrlr_get_flags(ctrlr) & 5703 SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) { 5704 nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr); 5705 } 5706 5707 rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr); 5708 if (rc != 0) { 5709 goto err; 5710 } 5711 5712 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 5713 5714 if (cdata->cmic.ana_reporting) { 5715 rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx); 5716 if (rc == 0) { 5717 return 0; 5718 } 5719 } else { 5720 nvme_ctrlr_create_done(nvme_ctrlr, ctx); 5721 return 0; 5722 } 5723 5724 err: 5725 nvme_ctrlr_delete(nvme_ctrlr); 5726 return rc; 5727 } 5728 5729 void 5730 spdk_bdev_nvme_get_default_ctrlr_opts(struct spdk_bdev_nvme_ctrlr_opts *opts) 5731 { 5732 opts->prchk_flags = 0; 5733 opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec; 5734 opts->reconnect_delay_sec = g_opts.reconnect_delay_sec; 5735 opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec; 5736 opts->multipath = false; 5737 } 5738 5739 static void 5740 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 5741 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts) 5742 { 5743 char *name; 5744 5745 name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++); 5746 if (!name) { 5747 SPDK_ERRLOG("Failed to assign name to NVMe device\n"); 5748 return; 5749 } 5750 5751 if (nvme_ctrlr_create(ctrlr, name, trid, NULL) == 0) { 5752 SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name); 5753 } else { 5754 SPDK_ERRLOG("Failed to attach to %s (%s)\n", trid->traddr, name); 5755 } 5756 5757 free(name); 5758 } 5759 5760 static void 5761 _nvme_ctrlr_destruct(void *ctx) 5762 { 5763 struct nvme_ctrlr *nvme_ctrlr = ctx; 5764 5765 nvme_ctrlr_depopulate_namespaces(nvme_ctrlr); 5766 nvme_ctrlr_release(nvme_ctrlr); 5767 } 5768 5769 static int 5770 bdev_nvme_delete_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 5771 { 5772 struct nvme_probe_skip_entry *entry; 5773 5774 /* The controller's destruction was already started */ 5775 if (nvme_ctrlr->destruct) { 5776 return -EALREADY; 5777 } 5778 5779 if (!hotplug && 5780 nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 5781 entry = calloc(1, sizeof(*entry)); 5782 if (!entry) { 5783 return -ENOMEM; 5784 } 5785 entry->trid = nvme_ctrlr->active_path_id->trid; 5786 TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq); 5787 } 5788 5789 nvme_ctrlr->destruct = true; 5790 return 0; 5791 } 5792 5793 static int 5794 bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug) 5795 { 5796 int rc; 5797 5798 pthread_mutex_lock(&nvme_ctrlr->mutex); 5799 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, hotplug); 5800 pthread_mutex_unlock(&nvme_ctrlr->mutex); 5801 5802 if (rc == 0) { 5803 _nvme_ctrlr_destruct(nvme_ctrlr); 5804 } else if (rc == -EALREADY) { 5805 rc = 0; 5806 } 5807 5808 return rc; 5809 } 5810 5811 static void 5812 remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr) 5813 { 5814 struct nvme_ctrlr *nvme_ctrlr = cb_ctx; 5815 5816 bdev_nvme_delete_ctrlr(nvme_ctrlr, true); 5817 } 5818 5819 static int 5820 bdev_nvme_hotplug_probe(void *arg) 5821 { 5822 if (g_hotplug_probe_ctx == NULL) { 5823 spdk_poller_unregister(&g_hotplug_probe_poller); 5824 return SPDK_POLLER_IDLE; 5825 } 5826 5827 if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) { 5828 g_hotplug_probe_ctx = NULL; 5829 spdk_poller_unregister(&g_hotplug_probe_poller); 5830 } 5831 5832 return SPDK_POLLER_BUSY; 5833 } 5834 5835 static int 5836 bdev_nvme_hotplug(void *arg) 5837 { 5838 struct spdk_nvme_transport_id trid_pcie; 5839 5840 if (g_hotplug_probe_ctx) { 5841 return SPDK_POLLER_BUSY; 5842 } 5843 5844 memset(&trid_pcie, 0, sizeof(trid_pcie)); 5845 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 5846 5847 g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL, 5848 hotplug_probe_cb, attach_cb, NULL); 5849 5850 if (g_hotplug_probe_ctx) { 5851 assert(g_hotplug_probe_poller == NULL); 5852 g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000); 5853 } 5854 5855 return SPDK_POLLER_BUSY; 5856 } 5857 5858 void 5859 bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts) 5860 { 5861 *opts = g_opts; 5862 } 5863 5864 static bool bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 5865 uint32_t reconnect_delay_sec, 5866 uint32_t fast_io_fail_timeout_sec); 5867 5868 static int 5869 bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts) 5870 { 5871 if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) { 5872 /* Can't set timeout_admin_us without also setting timeout_us */ 5873 SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n"); 5874 return -EINVAL; 5875 } 5876 5877 if (opts->bdev_retry_count < -1) { 5878 SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n"); 5879 return -EINVAL; 5880 } 5881 5882 if (!bdev_nvme_check_io_error_resiliency_params(opts->ctrlr_loss_timeout_sec, 5883 opts->reconnect_delay_sec, 5884 opts->fast_io_fail_timeout_sec)) { 5885 return -EINVAL; 5886 } 5887 5888 return 0; 5889 } 5890 5891 int 5892 bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) 5893 { 5894 int ret; 5895 5896 ret = bdev_nvme_validate_opts(opts); 5897 if (ret) { 5898 SPDK_WARNLOG("Failed to set nvme opts.\n"); 5899 return ret; 5900 } 5901 5902 if (g_bdev_nvme_init_thread != NULL) { 5903 if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 5904 return -EPERM; 5905 } 5906 } 5907 5908 if (opts->rdma_srq_size != 0 || 5909 opts->rdma_max_cq_size != 0 || 5910 opts->rdma_cm_event_timeout_ms != 0) { 5911 struct spdk_nvme_transport_opts drv_opts; 5912 5913 spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts)); 5914 if (opts->rdma_srq_size != 0) { 5915 drv_opts.rdma_srq_size = opts->rdma_srq_size; 5916 } 5917 if (opts->rdma_max_cq_size != 0) { 5918 drv_opts.rdma_max_cq_size = opts->rdma_max_cq_size; 5919 } 5920 if (opts->rdma_cm_event_timeout_ms != 0) { 5921 drv_opts.rdma_cm_event_timeout_ms = opts->rdma_cm_event_timeout_ms; 5922 } 5923 5924 ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts)); 5925 if (ret) { 5926 SPDK_ERRLOG("Failed to set NVMe transport opts.\n"); 5927 return ret; 5928 } 5929 } 5930 5931 g_opts = *opts; 5932 5933 return 0; 5934 } 5935 5936 struct set_nvme_hotplug_ctx { 5937 uint64_t period_us; 5938 bool enabled; 5939 spdk_msg_fn fn; 5940 void *fn_ctx; 5941 }; 5942 5943 static void 5944 set_nvme_hotplug_period_cb(void *_ctx) 5945 { 5946 struct set_nvme_hotplug_ctx *ctx = _ctx; 5947 5948 spdk_poller_unregister(&g_hotplug_poller); 5949 if (ctx->enabled) { 5950 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us); 5951 } else { 5952 g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL, 5953 NVME_HOTPLUG_POLL_PERIOD_DEFAULT); 5954 } 5955 5956 g_nvme_hotplug_poll_period_us = ctx->period_us; 5957 g_nvme_hotplug_enabled = ctx->enabled; 5958 if (ctx->fn) { 5959 ctx->fn(ctx->fn_ctx); 5960 } 5961 5962 free(ctx); 5963 } 5964 5965 int 5966 bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx) 5967 { 5968 struct set_nvme_hotplug_ctx *ctx; 5969 5970 if (enabled == true && !spdk_process_is_primary()) { 5971 return -EPERM; 5972 } 5973 5974 ctx = calloc(1, sizeof(*ctx)); 5975 if (ctx == NULL) { 5976 return -ENOMEM; 5977 } 5978 5979 period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us; 5980 ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX); 5981 ctx->enabled = enabled; 5982 ctx->fn = cb; 5983 ctx->fn_ctx = cb_ctx; 5984 5985 spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx); 5986 return 0; 5987 } 5988 5989 static void 5990 nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr, 5991 struct nvme_async_probe_ctx *ctx) 5992 { 5993 struct nvme_ns *nvme_ns; 5994 struct nvme_bdev *nvme_bdev; 5995 size_t j; 5996 5997 assert(nvme_ctrlr != NULL); 5998 5999 if (ctx->names == NULL) { 6000 ctx->reported_bdevs = 0; 6001 populate_namespaces_cb(ctx, 0); 6002 return; 6003 } 6004 6005 /* 6006 * Report the new bdevs that were created in this call. 6007 * There can be more than one bdev per NVMe controller. 6008 */ 6009 j = 0; 6010 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 6011 while (nvme_ns != NULL) { 6012 nvme_bdev = nvme_ns->bdev; 6013 if (j < ctx->max_bdevs) { 6014 ctx->names[j] = nvme_bdev->disk.name; 6015 j++; 6016 } else { 6017 SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n", 6018 ctx->max_bdevs); 6019 ctx->reported_bdevs = 0; 6020 populate_namespaces_cb(ctx, -ERANGE); 6021 return; 6022 } 6023 6024 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 6025 } 6026 6027 ctx->reported_bdevs = j; 6028 populate_namespaces_cb(ctx, 0); 6029 } 6030 6031 static int 6032 bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 6033 struct spdk_nvme_ctrlr *new_ctrlr, 6034 struct spdk_nvme_transport_id *trid) 6035 { 6036 struct nvme_path_id *tmp_trid; 6037 6038 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 6039 SPDK_ERRLOG("PCIe failover is not supported.\n"); 6040 return -ENOTSUP; 6041 } 6042 6043 /* Currently we only support failover to the same transport type. */ 6044 if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) { 6045 SPDK_WARNLOG("Failover from trtype: %s to a different trtype: %s is not supported currently\n", 6046 spdk_nvme_transport_id_trtype_str(nvme_ctrlr->active_path_id->trid.trtype), 6047 spdk_nvme_transport_id_trtype_str(trid->trtype)); 6048 return -EINVAL; 6049 } 6050 6051 6052 /* Currently we only support failover to the same NQN. */ 6053 if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) { 6054 SPDK_WARNLOG("Failover from subnqn: %s to a different subnqn: %s is not supported currently\n", 6055 nvme_ctrlr->active_path_id->trid.subnqn, trid->subnqn); 6056 return -EINVAL; 6057 } 6058 6059 /* Skip all the other checks if we've already registered this path. */ 6060 TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) { 6061 if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) { 6062 SPDK_WARNLOG("This path (traddr: %s subnqn: %s) is already registered\n", trid->traddr, 6063 trid->subnqn); 6064 return -EALREADY; 6065 } 6066 } 6067 6068 return 0; 6069 } 6070 6071 static int 6072 bdev_nvme_check_secondary_namespace(struct nvme_ctrlr *nvme_ctrlr, 6073 struct spdk_nvme_ctrlr *new_ctrlr) 6074 { 6075 struct nvme_ns *nvme_ns; 6076 struct spdk_nvme_ns *new_ns; 6077 6078 nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr); 6079 while (nvme_ns != NULL) { 6080 new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id); 6081 assert(new_ns != NULL); 6082 6083 if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) { 6084 return -EINVAL; 6085 } 6086 6087 nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns); 6088 } 6089 6090 return 0; 6091 } 6092 6093 static int 6094 _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 6095 struct spdk_nvme_transport_id *trid) 6096 { 6097 struct nvme_path_id *active_id, *new_trid, *tmp_trid; 6098 6099 new_trid = calloc(1, sizeof(*new_trid)); 6100 if (new_trid == NULL) { 6101 return -ENOMEM; 6102 } 6103 new_trid->trid = *trid; 6104 6105 active_id = nvme_ctrlr->active_path_id; 6106 assert(active_id != NULL); 6107 assert(active_id == TAILQ_FIRST(&nvme_ctrlr->trids)); 6108 6109 /* Skip the active trid not to replace it until it is failed. */ 6110 tmp_trid = TAILQ_NEXT(active_id, link); 6111 if (tmp_trid == NULL) { 6112 goto add_tail; 6113 } 6114 6115 /* It means the trid is faled if its last failed time is non-zero. 6116 * Insert the new alternate trid before any failed trid. 6117 */ 6118 TAILQ_FOREACH_FROM(tmp_trid, &nvme_ctrlr->trids, link) { 6119 if (tmp_trid->last_failed_tsc != 0) { 6120 TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link); 6121 return 0; 6122 } 6123 } 6124 6125 add_tail: 6126 TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link); 6127 return 0; 6128 } 6129 6130 /* This is the case that a secondary path is added to an existing 6131 * nvme_ctrlr for failover. After checking if it can access the same 6132 * namespaces as the primary path, it is disconnected until failover occurs. 6133 */ 6134 static int 6135 bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 6136 struct spdk_nvme_ctrlr *new_ctrlr, 6137 struct spdk_nvme_transport_id *trid) 6138 { 6139 int rc; 6140 6141 assert(nvme_ctrlr != NULL); 6142 6143 pthread_mutex_lock(&nvme_ctrlr->mutex); 6144 6145 rc = bdev_nvme_check_secondary_trid(nvme_ctrlr, new_ctrlr, trid); 6146 if (rc != 0) { 6147 goto exit; 6148 } 6149 6150 rc = bdev_nvme_check_secondary_namespace(nvme_ctrlr, new_ctrlr); 6151 if (rc != 0) { 6152 goto exit; 6153 } 6154 6155 rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid); 6156 6157 exit: 6158 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6159 6160 spdk_nvme_detach(new_ctrlr); 6161 6162 return rc; 6163 } 6164 6165 static void 6166 connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 6167 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 6168 { 6169 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 6170 struct nvme_async_probe_ctx *ctx; 6171 int rc; 6172 6173 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 6174 ctx->ctrlr_attached = true; 6175 6176 rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx); 6177 if (rc != 0) { 6178 ctx->reported_bdevs = 0; 6179 populate_namespaces_cb(ctx, rc); 6180 } 6181 } 6182 6183 static void 6184 connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 6185 struct spdk_nvme_ctrlr *ctrlr, 6186 const struct spdk_nvme_ctrlr_opts *opts) 6187 { 6188 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 6189 struct nvme_ctrlr *nvme_ctrlr; 6190 struct nvme_async_probe_ctx *ctx; 6191 int rc; 6192 6193 ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts); 6194 ctx->ctrlr_attached = true; 6195 6196 nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name); 6197 if (nvme_ctrlr) { 6198 rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid); 6199 } else { 6200 rc = -ENODEV; 6201 } 6202 6203 ctx->reported_bdevs = 0; 6204 populate_namespaces_cb(ctx, rc); 6205 } 6206 6207 static int 6208 bdev_nvme_async_poll(void *arg) 6209 { 6210 struct nvme_async_probe_ctx *ctx = arg; 6211 int rc; 6212 6213 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 6214 if (spdk_unlikely(rc != -EAGAIN)) { 6215 ctx->probe_done = true; 6216 spdk_poller_unregister(&ctx->poller); 6217 if (!ctx->ctrlr_attached) { 6218 /* The probe is done, but no controller was attached. 6219 * That means we had a failure, so report -EIO back to 6220 * the caller (usually the RPC). populate_namespaces_cb() 6221 * will take care of freeing the nvme_async_probe_ctx. 6222 */ 6223 ctx->reported_bdevs = 0; 6224 populate_namespaces_cb(ctx, -EIO); 6225 } else if (ctx->namespaces_populated) { 6226 /* The namespaces for the attached controller were all 6227 * populated and the response was already sent to the 6228 * caller (usually the RPC). So free the context here. 6229 */ 6230 free_nvme_async_probe_ctx(ctx); 6231 } 6232 } 6233 6234 return SPDK_POLLER_BUSY; 6235 } 6236 6237 static bool 6238 bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec, 6239 uint32_t reconnect_delay_sec, 6240 uint32_t fast_io_fail_timeout_sec) 6241 { 6242 if (ctrlr_loss_timeout_sec < -1) { 6243 SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n"); 6244 return false; 6245 } else if (ctrlr_loss_timeout_sec == -1) { 6246 if (reconnect_delay_sec == 0) { 6247 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 6248 return false; 6249 } else if (fast_io_fail_timeout_sec != 0 && 6250 fast_io_fail_timeout_sec < reconnect_delay_sec) { 6251 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n"); 6252 return false; 6253 } 6254 } else if (ctrlr_loss_timeout_sec != 0) { 6255 if (reconnect_delay_sec == 0) { 6256 SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n"); 6257 return false; 6258 } else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) { 6259 SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n"); 6260 return false; 6261 } else if (fast_io_fail_timeout_sec != 0) { 6262 if (fast_io_fail_timeout_sec < reconnect_delay_sec) { 6263 SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n"); 6264 return false; 6265 } else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) { 6266 SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n"); 6267 return false; 6268 } 6269 } 6270 } else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) { 6271 SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n"); 6272 return false; 6273 } 6274 6275 return true; 6276 } 6277 6278 int 6279 spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid, 6280 const char *base_name, 6281 const char **names, 6282 uint32_t count, 6283 spdk_bdev_nvme_create_cb cb_fn, 6284 void *cb_ctx, 6285 struct spdk_nvme_ctrlr_opts *drv_opts, 6286 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, 6287 bool multipath) 6288 { 6289 struct nvme_probe_skip_entry *entry, *tmp; 6290 struct nvme_async_probe_ctx *ctx; 6291 spdk_nvme_attach_cb attach_cb; 6292 int len; 6293 6294 /* TODO expand this check to include both the host and target TRIDs. 6295 * Only if both are the same should we fail. 6296 */ 6297 if (nvme_ctrlr_get(trid, drv_opts->hostnqn) != NULL) { 6298 SPDK_ERRLOG("A controller with the provided trid (traddr: %s, hostnqn: %s) " 6299 "already exists.\n", trid->traddr, drv_opts->hostnqn); 6300 return -EEXIST; 6301 } 6302 6303 len = strnlen(base_name, SPDK_CONTROLLER_NAME_MAX); 6304 6305 if (len == 0 || len == SPDK_CONTROLLER_NAME_MAX) { 6306 SPDK_ERRLOG("controller name must be between 1 and %d characters\n", SPDK_CONTROLLER_NAME_MAX - 1); 6307 return -EINVAL; 6308 } 6309 6310 if (bdev_opts != NULL && 6311 !bdev_nvme_check_io_error_resiliency_params(bdev_opts->ctrlr_loss_timeout_sec, 6312 bdev_opts->reconnect_delay_sec, 6313 bdev_opts->fast_io_fail_timeout_sec)) { 6314 return -EINVAL; 6315 } 6316 6317 ctx = calloc(1, sizeof(*ctx)); 6318 if (!ctx) { 6319 return -ENOMEM; 6320 } 6321 ctx->base_name = base_name; 6322 ctx->names = names; 6323 ctx->max_bdevs = count; 6324 ctx->cb_fn = cb_fn; 6325 ctx->cb_ctx = cb_ctx; 6326 ctx->trid = *trid; 6327 6328 if (bdev_opts) { 6329 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 6330 } else { 6331 spdk_bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts); 6332 } 6333 6334 if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { 6335 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) { 6336 if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) { 6337 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 6338 free(entry); 6339 break; 6340 } 6341 } 6342 } 6343 6344 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 6345 ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count; 6346 ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout; 6347 ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms; 6348 ctx->drv_opts.disable_read_ana_log_page = true; 6349 ctx->drv_opts.transport_tos = g_opts.transport_tos; 6350 6351 if (ctx->bdev_opts.psk != NULL) { 6352 ctx->drv_opts.tls_psk = spdk_keyring_get_key(ctx->bdev_opts.psk); 6353 if (ctx->drv_opts.tls_psk == NULL) { 6354 SPDK_ERRLOG("Could not load PSK: %s\n", ctx->bdev_opts.psk); 6355 free_nvme_async_probe_ctx(ctx); 6356 return -ENOKEY; 6357 } 6358 } 6359 6360 if (ctx->bdev_opts.dhchap_key != NULL) { 6361 ctx->drv_opts.dhchap_key = spdk_keyring_get_key(ctx->bdev_opts.dhchap_key); 6362 if (ctx->drv_opts.dhchap_key == NULL) { 6363 SPDK_ERRLOG("Could not load DH-HMAC-CHAP key: %s\n", 6364 ctx->bdev_opts.dhchap_key); 6365 free_nvme_async_probe_ctx(ctx); 6366 return -ENOKEY; 6367 } 6368 6369 ctx->drv_opts.dhchap_digests = g_opts.dhchap_digests; 6370 ctx->drv_opts.dhchap_dhgroups = g_opts.dhchap_dhgroups; 6371 } 6372 if (ctx->bdev_opts.dhchap_ctrlr_key != NULL) { 6373 ctx->drv_opts.dhchap_ctrlr_key = 6374 spdk_keyring_get_key(ctx->bdev_opts.dhchap_ctrlr_key); 6375 if (ctx->drv_opts.dhchap_ctrlr_key == NULL) { 6376 SPDK_ERRLOG("Could not load DH-HMAC-CHAP controller key: %s\n", 6377 ctx->bdev_opts.dhchap_ctrlr_key); 6378 free_nvme_async_probe_ctx(ctx); 6379 return -ENOKEY; 6380 } 6381 } 6382 6383 if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || multipath) { 6384 attach_cb = connect_attach_cb; 6385 } else { 6386 attach_cb = connect_set_failover_cb; 6387 } 6388 6389 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb); 6390 if (ctx->probe_ctx == NULL) { 6391 SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr); 6392 free_nvme_async_probe_ctx(ctx); 6393 return -ENODEV; 6394 } 6395 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000); 6396 6397 return 0; 6398 } 6399 6400 struct bdev_nvme_delete_ctx { 6401 char *name; 6402 struct nvme_path_id path_id; 6403 bdev_nvme_delete_done_fn delete_done; 6404 void *delete_done_ctx; 6405 uint64_t timeout_ticks; 6406 struct spdk_poller *poller; 6407 }; 6408 6409 static void 6410 free_bdev_nvme_delete_ctx(struct bdev_nvme_delete_ctx *ctx) 6411 { 6412 if (ctx != NULL) { 6413 free(ctx->name); 6414 free(ctx); 6415 } 6416 } 6417 6418 static bool 6419 nvme_path_id_compare(struct nvme_path_id *p, const struct nvme_path_id *path_id) 6420 { 6421 if (path_id->trid.trtype != 0) { 6422 if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) { 6423 if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) { 6424 return false; 6425 } 6426 } else { 6427 if (path_id->trid.trtype != p->trid.trtype) { 6428 return false; 6429 } 6430 } 6431 } 6432 6433 if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) { 6434 if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) { 6435 return false; 6436 } 6437 } 6438 6439 if (path_id->trid.adrfam != 0) { 6440 if (path_id->trid.adrfam != p->trid.adrfam) { 6441 return false; 6442 } 6443 } 6444 6445 if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) { 6446 if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) { 6447 return false; 6448 } 6449 } 6450 6451 if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) { 6452 if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) { 6453 return false; 6454 } 6455 } 6456 6457 if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) { 6458 if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) { 6459 return false; 6460 } 6461 } 6462 6463 if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) { 6464 if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) { 6465 return false; 6466 } 6467 } 6468 6469 return true; 6470 } 6471 6472 static bool 6473 nvme_path_id_exists(const char *name, const struct nvme_path_id *path_id) 6474 { 6475 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6476 struct nvme_ctrlr *ctrlr; 6477 struct nvme_path_id *p; 6478 6479 pthread_mutex_lock(&g_bdev_nvme_mutex); 6480 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 6481 if (!nbdev_ctrlr) { 6482 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6483 return false; 6484 } 6485 6486 TAILQ_FOREACH(ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 6487 pthread_mutex_lock(&ctrlr->mutex); 6488 TAILQ_FOREACH(p, &ctrlr->trids, link) { 6489 if (nvme_path_id_compare(p, path_id)) { 6490 pthread_mutex_unlock(&ctrlr->mutex); 6491 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6492 return true; 6493 } 6494 } 6495 pthread_mutex_unlock(&ctrlr->mutex); 6496 } 6497 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6498 6499 return false; 6500 } 6501 6502 static int 6503 bdev_nvme_delete_complete_poll(void *arg) 6504 { 6505 struct bdev_nvme_delete_ctx *ctx = arg; 6506 int rc = 0; 6507 6508 if (nvme_path_id_exists(ctx->name, &ctx->path_id)) { 6509 if (ctx->timeout_ticks > spdk_get_ticks()) { 6510 return SPDK_POLLER_BUSY; 6511 } 6512 6513 SPDK_ERRLOG("NVMe path '%s' still exists after delete\n", ctx->name); 6514 rc = -ETIMEDOUT; 6515 } 6516 6517 spdk_poller_unregister(&ctx->poller); 6518 6519 ctx->delete_done(ctx->delete_done_ctx, rc); 6520 free_bdev_nvme_delete_ctx(ctx); 6521 6522 return SPDK_POLLER_BUSY; 6523 } 6524 6525 static int 6526 _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id) 6527 { 6528 struct nvme_path_id *p, *t; 6529 spdk_msg_fn msg_fn; 6530 int rc = -ENXIO; 6531 6532 pthread_mutex_lock(&nvme_ctrlr->mutex); 6533 6534 TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) { 6535 if (p == TAILQ_FIRST(&nvme_ctrlr->trids)) { 6536 break; 6537 } 6538 6539 if (!nvme_path_id_compare(p, path_id)) { 6540 continue; 6541 } 6542 6543 /* We are not using the specified path. */ 6544 TAILQ_REMOVE(&nvme_ctrlr->trids, p, link); 6545 free(p); 6546 rc = 0; 6547 } 6548 6549 if (p == NULL || !nvme_path_id_compare(p, path_id)) { 6550 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6551 return rc; 6552 } 6553 6554 /* If we made it here, then this path is a match! Now we need to remove it. */ 6555 6556 /* This is the active path in use right now. The active path is always the first in the list. */ 6557 assert(p == nvme_ctrlr->active_path_id); 6558 6559 if (!TAILQ_NEXT(p, link)) { 6560 /* The current path is the only path. */ 6561 msg_fn = _nvme_ctrlr_destruct; 6562 rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, false); 6563 } else { 6564 /* There is an alternative path. */ 6565 msg_fn = _bdev_nvme_reset_ctrlr; 6566 rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, true); 6567 } 6568 6569 pthread_mutex_unlock(&nvme_ctrlr->mutex); 6570 6571 if (rc == 0) { 6572 spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr); 6573 } else if (rc == -EALREADY) { 6574 rc = 0; 6575 } 6576 6577 return rc; 6578 } 6579 6580 int 6581 bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id, 6582 bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx) 6583 { 6584 struct nvme_bdev_ctrlr *nbdev_ctrlr; 6585 struct nvme_ctrlr *nvme_ctrlr, *tmp_nvme_ctrlr; 6586 struct bdev_nvme_delete_ctx *ctx = NULL; 6587 int rc = -ENXIO, _rc; 6588 6589 if (name == NULL || path_id == NULL) { 6590 rc = -EINVAL; 6591 goto exit; 6592 } 6593 6594 pthread_mutex_lock(&g_bdev_nvme_mutex); 6595 6596 nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); 6597 if (nbdev_ctrlr == NULL) { 6598 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6599 6600 SPDK_ERRLOG("Failed to find NVMe bdev controller\n"); 6601 rc = -ENODEV; 6602 goto exit; 6603 } 6604 6605 TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) { 6606 _rc = _bdev_nvme_delete(nvme_ctrlr, path_id); 6607 if (_rc < 0 && _rc != -ENXIO) { 6608 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6609 rc = _rc; 6610 goto exit; 6611 } else if (_rc == 0) { 6612 /* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr 6613 * was deleted successfully. To remember the successful deletion, 6614 * overwrite rc only if _rc is zero. 6615 */ 6616 rc = 0; 6617 } 6618 } 6619 6620 pthread_mutex_unlock(&g_bdev_nvme_mutex); 6621 6622 if (rc != 0 || delete_done == NULL) { 6623 goto exit; 6624 } 6625 6626 ctx = calloc(1, sizeof(*ctx)); 6627 if (ctx == NULL) { 6628 SPDK_ERRLOG("Failed to allocate context for bdev_nvme_delete\n"); 6629 rc = -ENOMEM; 6630 goto exit; 6631 } 6632 6633 ctx->name = strdup(name); 6634 if (ctx->name == NULL) { 6635 SPDK_ERRLOG("Failed to copy controller name for deletion\n"); 6636 rc = -ENOMEM; 6637 goto exit; 6638 } 6639 6640 ctx->delete_done = delete_done; 6641 ctx->delete_done_ctx = delete_done_ctx; 6642 ctx->path_id = *path_id; 6643 ctx->timeout_ticks = spdk_get_ticks() + 10 * spdk_get_ticks_hz(); 6644 ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_delete_complete_poll, ctx, 1000); 6645 if (ctx->poller == NULL) { 6646 SPDK_ERRLOG("Failed to register bdev_nvme_delete poller\n"); 6647 rc = -ENOMEM; 6648 goto exit; 6649 } 6650 6651 exit: 6652 if (rc != 0) { 6653 free_bdev_nvme_delete_ctx(ctx); 6654 } 6655 6656 return rc; 6657 } 6658 6659 #define DISCOVERY_INFOLOG(ctx, format, ...) \ 6660 SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 6661 6662 #define DISCOVERY_ERRLOG(ctx, format, ...) \ 6663 SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__); 6664 6665 struct discovery_entry_ctx { 6666 char name[128]; 6667 struct spdk_nvme_transport_id trid; 6668 struct spdk_nvme_ctrlr_opts drv_opts; 6669 struct spdk_nvmf_discovery_log_page_entry entry; 6670 TAILQ_ENTRY(discovery_entry_ctx) tailq; 6671 struct discovery_ctx *ctx; 6672 }; 6673 6674 struct discovery_ctx { 6675 char *name; 6676 spdk_bdev_nvme_start_discovery_fn start_cb_fn; 6677 spdk_bdev_nvme_stop_discovery_fn stop_cb_fn; 6678 void *cb_ctx; 6679 struct spdk_nvme_probe_ctx *probe_ctx; 6680 struct spdk_nvme_detach_ctx *detach_ctx; 6681 struct spdk_nvme_ctrlr *ctrlr; 6682 struct spdk_nvme_transport_id trid; 6683 struct discovery_entry_ctx *entry_ctx_in_use; 6684 struct spdk_poller *poller; 6685 struct spdk_nvme_ctrlr_opts drv_opts; 6686 struct spdk_bdev_nvme_ctrlr_opts bdev_opts; 6687 struct spdk_nvmf_discovery_log_page *log_page; 6688 TAILQ_ENTRY(discovery_ctx) tailq; 6689 TAILQ_HEAD(, discovery_entry_ctx) nvm_entry_ctxs; 6690 TAILQ_HEAD(, discovery_entry_ctx) discovery_entry_ctxs; 6691 int rc; 6692 bool wait_for_attach; 6693 uint64_t timeout_ticks; 6694 /* Denotes that the discovery service is being started. We're waiting 6695 * for the initial connection to the discovery controller to be 6696 * established and attach discovered NVM ctrlrs. 6697 */ 6698 bool initializing; 6699 /* Denotes if a discovery is currently in progress for this context. 6700 * That includes connecting to newly discovered subsystems. Used to 6701 * ensure we do not start a new discovery until an existing one is 6702 * complete. 6703 */ 6704 bool in_progress; 6705 6706 /* Denotes if another discovery is needed after the one in progress 6707 * completes. Set when we receive an AER completion while a discovery 6708 * is already in progress. 6709 */ 6710 bool pending; 6711 6712 /* Signal to the discovery context poller that it should stop the 6713 * discovery service, including detaching from the current discovery 6714 * controller. 6715 */ 6716 bool stop; 6717 6718 struct spdk_thread *calling_thread; 6719 uint32_t index; 6720 uint32_t attach_in_progress; 6721 char *hostnqn; 6722 6723 /* Denotes if the discovery service was started by the mdns discovery. 6724 */ 6725 bool from_mdns_discovery_service; 6726 }; 6727 6728 TAILQ_HEAD(discovery_ctxs, discovery_ctx); 6729 static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs); 6730 6731 static void get_discovery_log_page(struct discovery_ctx *ctx); 6732 6733 static void 6734 free_discovery_ctx(struct discovery_ctx *ctx) 6735 { 6736 free(ctx->log_page); 6737 free(ctx->hostnqn); 6738 free(ctx->name); 6739 free(ctx); 6740 } 6741 6742 static void 6743 discovery_complete(struct discovery_ctx *ctx) 6744 { 6745 ctx->initializing = false; 6746 ctx->in_progress = false; 6747 if (ctx->pending) { 6748 ctx->pending = false; 6749 get_discovery_log_page(ctx); 6750 } 6751 } 6752 6753 static void 6754 build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid, 6755 struct spdk_nvmf_discovery_log_page_entry *entry) 6756 { 6757 char *space; 6758 6759 trid->trtype = entry->trtype; 6760 trid->adrfam = entry->adrfam; 6761 memcpy(trid->traddr, entry->traddr, sizeof(entry->traddr)); 6762 memcpy(trid->trsvcid, entry->trsvcid, sizeof(entry->trsvcid)); 6763 /* Because the source buffer (entry->subnqn) is longer than trid->subnqn, and 6764 * before call to this function trid->subnqn is zeroed out, we need 6765 * to copy sizeof(trid->subnqn) minus one byte to make sure the last character 6766 * remains 0. Then we can shorten the string (replace ' ' with 0) if required 6767 */ 6768 memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn) - 1); 6769 6770 /* We want the traddr, trsvcid and subnqn fields to be NULL-terminated. 6771 * But the log page entries typically pad them with spaces, not zeroes. 6772 * So add a NULL terminator to each of these fields at the appropriate 6773 * location. 6774 */ 6775 space = strchr(trid->traddr, ' '); 6776 if (space) { 6777 *space = 0; 6778 } 6779 space = strchr(trid->trsvcid, ' '); 6780 if (space) { 6781 *space = 0; 6782 } 6783 space = strchr(trid->subnqn, ' '); 6784 if (space) { 6785 *space = 0; 6786 } 6787 } 6788 6789 static void 6790 _stop_discovery(void *_ctx) 6791 { 6792 struct discovery_ctx *ctx = _ctx; 6793 6794 if (ctx->attach_in_progress > 0) { 6795 spdk_thread_send_msg(spdk_get_thread(), _stop_discovery, ctx); 6796 return; 6797 } 6798 6799 ctx->stop = true; 6800 6801 while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) { 6802 struct discovery_entry_ctx *entry_ctx; 6803 struct nvme_path_id path = {}; 6804 6805 entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs); 6806 path.trid = entry_ctx->trid; 6807 bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL); 6808 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 6809 free(entry_ctx); 6810 } 6811 6812 while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) { 6813 struct discovery_entry_ctx *entry_ctx; 6814 6815 entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 6816 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 6817 free(entry_ctx); 6818 } 6819 6820 free(ctx->entry_ctx_in_use); 6821 ctx->entry_ctx_in_use = NULL; 6822 } 6823 6824 static void 6825 stop_discovery(struct discovery_ctx *ctx, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 6826 { 6827 ctx->stop_cb_fn = cb_fn; 6828 ctx->cb_ctx = cb_ctx; 6829 6830 if (ctx->attach_in_progress > 0) { 6831 DISCOVERY_INFOLOG(ctx, "stopping discovery with attach_in_progress: %"PRIu32"\n", 6832 ctx->attach_in_progress); 6833 } 6834 6835 _stop_discovery(ctx); 6836 } 6837 6838 static void 6839 remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr) 6840 { 6841 struct discovery_ctx *d_ctx; 6842 struct nvme_path_id *path_id; 6843 struct spdk_nvme_transport_id trid = {}; 6844 struct discovery_entry_ctx *entry_ctx, *tmp; 6845 6846 path_id = TAILQ_FIRST(&nvme_ctrlr->trids); 6847 6848 TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) { 6849 TAILQ_FOREACH_SAFE(entry_ctx, &d_ctx->nvm_entry_ctxs, tailq, tmp) { 6850 build_trid_from_log_page_entry(&trid, &entry_ctx->entry); 6851 if (spdk_nvme_transport_id_compare(&trid, &path_id->trid) != 0) { 6852 continue; 6853 } 6854 6855 TAILQ_REMOVE(&d_ctx->nvm_entry_ctxs, entry_ctx, tailq); 6856 free(entry_ctx); 6857 DISCOVERY_INFOLOG(d_ctx, "Remove discovery entry: %s:%s:%s\n", 6858 trid.subnqn, trid.traddr, trid.trsvcid); 6859 6860 /* Fail discovery ctrlr to force reattach attempt */ 6861 spdk_nvme_ctrlr_fail(d_ctx->ctrlr); 6862 } 6863 } 6864 } 6865 6866 static void 6867 discovery_remove_controllers(struct discovery_ctx *ctx) 6868 { 6869 struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page; 6870 struct discovery_entry_ctx *entry_ctx, *tmp; 6871 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 6872 struct spdk_nvme_transport_id old_trid = {}; 6873 uint64_t numrec, i; 6874 bool found; 6875 6876 numrec = from_le64(&log_page->numrec); 6877 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) { 6878 found = false; 6879 old_entry = &entry_ctx->entry; 6880 build_trid_from_log_page_entry(&old_trid, old_entry); 6881 for (i = 0; i < numrec; i++) { 6882 new_entry = &log_page->entries[i]; 6883 if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) { 6884 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n", 6885 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 6886 found = true; 6887 break; 6888 } 6889 } 6890 if (!found) { 6891 struct nvme_path_id path = {}; 6892 6893 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n", 6894 old_trid.subnqn, old_trid.traddr, old_trid.trsvcid); 6895 6896 path.trid = entry_ctx->trid; 6897 bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL); 6898 TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq); 6899 free(entry_ctx); 6900 } 6901 } 6902 free(log_page); 6903 ctx->log_page = NULL; 6904 discovery_complete(ctx); 6905 } 6906 6907 static void 6908 complete_discovery_start(struct discovery_ctx *ctx, int status) 6909 { 6910 ctx->timeout_ticks = 0; 6911 ctx->rc = status; 6912 if (ctx->start_cb_fn) { 6913 ctx->start_cb_fn(ctx->cb_ctx, status); 6914 ctx->start_cb_fn = NULL; 6915 ctx->cb_ctx = NULL; 6916 } 6917 } 6918 6919 static void 6920 discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc) 6921 { 6922 struct discovery_entry_ctx *entry_ctx = cb_ctx; 6923 struct discovery_ctx *ctx = entry_ctx->ctx; 6924 6925 DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name); 6926 ctx->attach_in_progress--; 6927 if (ctx->attach_in_progress == 0) { 6928 complete_discovery_start(ctx, ctx->rc); 6929 if (ctx->initializing && ctx->rc != 0) { 6930 DISCOVERY_ERRLOG(ctx, "stopping discovery due to errors: %d\n", ctx->rc); 6931 stop_discovery(ctx, NULL, ctx->cb_ctx); 6932 } else { 6933 discovery_remove_controllers(ctx); 6934 } 6935 } 6936 } 6937 6938 static struct discovery_entry_ctx * 6939 create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid) 6940 { 6941 struct discovery_entry_ctx *new_ctx; 6942 6943 new_ctx = calloc(1, sizeof(*new_ctx)); 6944 if (new_ctx == NULL) { 6945 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 6946 return NULL; 6947 } 6948 6949 new_ctx->ctx = ctx; 6950 memcpy(&new_ctx->trid, trid, sizeof(*trid)); 6951 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 6952 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 6953 return new_ctx; 6954 } 6955 6956 static void 6957 discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl, 6958 struct spdk_nvmf_discovery_log_page *log_page) 6959 { 6960 struct discovery_ctx *ctx = cb_arg; 6961 struct discovery_entry_ctx *entry_ctx, *tmp; 6962 struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry; 6963 uint64_t numrec, i; 6964 bool found; 6965 6966 if (rc || spdk_nvme_cpl_is_error(cpl)) { 6967 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 6968 return; 6969 } 6970 6971 ctx->log_page = log_page; 6972 assert(ctx->attach_in_progress == 0); 6973 numrec = from_le64(&log_page->numrec); 6974 TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) { 6975 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq); 6976 free(entry_ctx); 6977 } 6978 for (i = 0; i < numrec; i++) { 6979 found = false; 6980 new_entry = &log_page->entries[i]; 6981 if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY_CURRENT || 6982 new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) { 6983 struct discovery_entry_ctx *new_ctx; 6984 struct spdk_nvme_transport_id trid = {}; 6985 6986 build_trid_from_log_page_entry(&trid, new_entry); 6987 new_ctx = create_discovery_entry_ctx(ctx, &trid); 6988 if (new_ctx == NULL) { 6989 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 6990 break; 6991 } 6992 6993 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq); 6994 continue; 6995 } 6996 TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) { 6997 old_entry = &entry_ctx->entry; 6998 if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) { 6999 found = true; 7000 break; 7001 } 7002 } 7003 if (!found) { 7004 struct discovery_entry_ctx *subnqn_ctx = NULL, *new_ctx; 7005 struct discovery_ctx *d_ctx; 7006 7007 TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) { 7008 TAILQ_FOREACH(subnqn_ctx, &d_ctx->nvm_entry_ctxs, tailq) { 7009 if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn, 7010 sizeof(new_entry->subnqn))) { 7011 break; 7012 } 7013 } 7014 if (subnqn_ctx) { 7015 break; 7016 } 7017 } 7018 7019 new_ctx = calloc(1, sizeof(*new_ctx)); 7020 if (new_ctx == NULL) { 7021 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 7022 break; 7023 } 7024 7025 new_ctx->ctx = ctx; 7026 memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry)); 7027 build_trid_from_log_page_entry(&new_ctx->trid, new_entry); 7028 if (subnqn_ctx) { 7029 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name); 7030 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n", 7031 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 7032 new_ctx->name); 7033 } else { 7034 snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++); 7035 DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n", 7036 new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid, 7037 new_ctx->name); 7038 } 7039 spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts)); 7040 snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn); 7041 rc = spdk_bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0, 7042 discovery_attach_controller_done, new_ctx, 7043 &new_ctx->drv_opts, &ctx->bdev_opts, true); 7044 if (rc == 0) { 7045 TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq); 7046 ctx->attach_in_progress++; 7047 } else { 7048 DISCOVERY_ERRLOG(ctx, "spdk_bdev_nvme_create failed (%s)\n", spdk_strerror(-rc)); 7049 } 7050 } 7051 } 7052 7053 if (ctx->attach_in_progress == 0) { 7054 discovery_remove_controllers(ctx); 7055 } 7056 } 7057 7058 static void 7059 get_discovery_log_page(struct discovery_ctx *ctx) 7060 { 7061 int rc; 7062 7063 assert(ctx->in_progress == false); 7064 ctx->in_progress = true; 7065 rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx); 7066 if (rc != 0) { 7067 DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n"); 7068 } 7069 DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n"); 7070 } 7071 7072 static void 7073 discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) 7074 { 7075 struct discovery_ctx *ctx = arg; 7076 uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16; 7077 7078 if (spdk_nvme_cpl_is_error(cpl)) { 7079 DISCOVERY_ERRLOG(ctx, "aer failed\n"); 7080 return; 7081 } 7082 7083 if (log_page_id != SPDK_NVME_LOG_DISCOVERY) { 7084 DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id); 7085 return; 7086 } 7087 7088 DISCOVERY_INFOLOG(ctx, "got aer\n"); 7089 if (ctx->in_progress) { 7090 ctx->pending = true; 7091 return; 7092 } 7093 7094 get_discovery_log_page(ctx); 7095 } 7096 7097 static void 7098 discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 7099 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 7100 { 7101 struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx; 7102 struct discovery_ctx *ctx; 7103 7104 ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts); 7105 7106 DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n"); 7107 ctx->probe_ctx = NULL; 7108 ctx->ctrlr = ctrlr; 7109 7110 if (ctx->rc != 0) { 7111 DISCOVERY_ERRLOG(ctx, "encountered error while attaching discovery ctrlr: %d\n", 7112 ctx->rc); 7113 return; 7114 } 7115 7116 spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx); 7117 } 7118 7119 static int 7120 discovery_poller(void *arg) 7121 { 7122 struct discovery_ctx *ctx = arg; 7123 struct spdk_nvme_transport_id *trid; 7124 int rc; 7125 7126 if (ctx->detach_ctx) { 7127 rc = spdk_nvme_detach_poll_async(ctx->detach_ctx); 7128 if (rc != -EAGAIN) { 7129 ctx->detach_ctx = NULL; 7130 ctx->ctrlr = NULL; 7131 } 7132 } else if (ctx->stop) { 7133 if (ctx->ctrlr != NULL) { 7134 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 7135 if (rc == 0) { 7136 return SPDK_POLLER_BUSY; 7137 } 7138 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 7139 } 7140 spdk_poller_unregister(&ctx->poller); 7141 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 7142 assert(ctx->start_cb_fn == NULL); 7143 if (ctx->stop_cb_fn != NULL) { 7144 ctx->stop_cb_fn(ctx->cb_ctx); 7145 } 7146 free_discovery_ctx(ctx); 7147 } else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) { 7148 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7149 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 7150 assert(ctx->initializing); 7151 spdk_poller_unregister(&ctx->poller); 7152 TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq); 7153 complete_discovery_start(ctx, -ETIMEDOUT); 7154 stop_discovery(ctx, NULL, NULL); 7155 free_discovery_ctx(ctx); 7156 return SPDK_POLLER_BUSY; 7157 } 7158 7159 assert(ctx->entry_ctx_in_use == NULL); 7160 ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs); 7161 TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7162 trid = &ctx->entry_ctx_in_use->trid; 7163 ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb); 7164 if (ctx->probe_ctx) { 7165 spdk_poller_unregister(&ctx->poller); 7166 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000); 7167 } else { 7168 DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n"); 7169 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7170 ctx->entry_ctx_in_use = NULL; 7171 } 7172 } else if (ctx->probe_ctx) { 7173 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7174 DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n"); 7175 complete_discovery_start(ctx, -ETIMEDOUT); 7176 return SPDK_POLLER_BUSY; 7177 } 7178 7179 rc = spdk_nvme_probe_poll_async(ctx->probe_ctx); 7180 if (rc != -EAGAIN) { 7181 if (ctx->rc != 0) { 7182 assert(ctx->initializing); 7183 stop_discovery(ctx, NULL, ctx->cb_ctx); 7184 } else { 7185 assert(rc == 0); 7186 DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n"); 7187 ctx->rc = rc; 7188 get_discovery_log_page(ctx); 7189 } 7190 } 7191 } else { 7192 if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) { 7193 DISCOVERY_ERRLOG(ctx, "timed out while attaching NVM ctrlrs\n"); 7194 complete_discovery_start(ctx, -ETIMEDOUT); 7195 /* We need to wait until all NVM ctrlrs are attached before we stop the 7196 * discovery service to make sure we don't detach a ctrlr that is still 7197 * being attached. 7198 */ 7199 if (ctx->attach_in_progress == 0) { 7200 stop_discovery(ctx, NULL, ctx->cb_ctx); 7201 return SPDK_POLLER_BUSY; 7202 } 7203 } 7204 7205 rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr); 7206 if (rc < 0) { 7207 spdk_poller_unregister(&ctx->poller); 7208 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 7209 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq); 7210 ctx->entry_ctx_in_use = NULL; 7211 7212 rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx); 7213 if (rc != 0) { 7214 DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n"); 7215 ctx->ctrlr = NULL; 7216 } 7217 } 7218 } 7219 7220 return SPDK_POLLER_BUSY; 7221 } 7222 7223 static void 7224 start_discovery_poller(void *arg) 7225 { 7226 struct discovery_ctx *ctx = arg; 7227 7228 TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq); 7229 ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000); 7230 } 7231 7232 int 7233 bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid, 7234 const char *base_name, 7235 struct spdk_nvme_ctrlr_opts *drv_opts, 7236 struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, 7237 uint64_t attach_timeout, 7238 bool from_mdns, 7239 spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx) 7240 { 7241 struct discovery_ctx *ctx; 7242 struct discovery_entry_ctx *discovery_entry_ctx; 7243 7244 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 7245 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7246 if (strcmp(ctx->name, base_name) == 0) { 7247 return -EEXIST; 7248 } 7249 7250 if (ctx->entry_ctx_in_use != NULL) { 7251 if (!spdk_nvme_transport_id_compare(trid, &ctx->entry_ctx_in_use->trid)) { 7252 return -EEXIST; 7253 } 7254 } 7255 7256 TAILQ_FOREACH(discovery_entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 7257 if (!spdk_nvme_transport_id_compare(trid, &discovery_entry_ctx->trid)) { 7258 return -EEXIST; 7259 } 7260 } 7261 } 7262 7263 ctx = calloc(1, sizeof(*ctx)); 7264 if (ctx == NULL) { 7265 return -ENOMEM; 7266 } 7267 7268 ctx->name = strdup(base_name); 7269 if (ctx->name == NULL) { 7270 free_discovery_ctx(ctx); 7271 return -ENOMEM; 7272 } 7273 memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts)); 7274 memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts)); 7275 ctx->from_mdns_discovery_service = from_mdns; 7276 ctx->bdev_opts.from_discovery_service = true; 7277 ctx->calling_thread = spdk_get_thread(); 7278 ctx->start_cb_fn = cb_fn; 7279 ctx->cb_ctx = cb_ctx; 7280 ctx->initializing = true; 7281 if (ctx->start_cb_fn) { 7282 /* We can use this when dumping json to denote if this RPC parameter 7283 * was specified or not. 7284 */ 7285 ctx->wait_for_attach = true; 7286 } 7287 if (attach_timeout != 0) { 7288 ctx->timeout_ticks = spdk_get_ticks() + attach_timeout * 7289 spdk_get_ticks_hz() / 1000ull; 7290 } 7291 TAILQ_INIT(&ctx->nvm_entry_ctxs); 7292 TAILQ_INIT(&ctx->discovery_entry_ctxs); 7293 memcpy(&ctx->trid, trid, sizeof(*trid)); 7294 /* Even if user did not specify hostnqn, we can still strdup("\0"); */ 7295 ctx->hostnqn = strdup(ctx->drv_opts.hostnqn); 7296 if (ctx->hostnqn == NULL) { 7297 free_discovery_ctx(ctx); 7298 return -ENOMEM; 7299 } 7300 discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid); 7301 if (discovery_entry_ctx == NULL) { 7302 DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n"); 7303 free_discovery_ctx(ctx); 7304 return -ENOMEM; 7305 } 7306 7307 TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq); 7308 spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx); 7309 return 0; 7310 } 7311 7312 int 7313 bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx) 7314 { 7315 struct discovery_ctx *ctx; 7316 7317 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7318 if (strcmp(name, ctx->name) == 0) { 7319 if (ctx->stop) { 7320 return -EALREADY; 7321 } 7322 /* If we're still starting the discovery service and ->rc is non-zero, we're 7323 * going to stop it as soon as we can 7324 */ 7325 if (ctx->initializing && ctx->rc != 0) { 7326 return -EALREADY; 7327 } 7328 stop_discovery(ctx, cb_fn, cb_ctx); 7329 return 0; 7330 } 7331 } 7332 7333 return -ENOENT; 7334 } 7335 7336 static int 7337 bdev_nvme_library_init(void) 7338 { 7339 g_bdev_nvme_init_thread = spdk_get_thread(); 7340 7341 spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb, 7342 bdev_nvme_destroy_poll_group_cb, 7343 sizeof(struct nvme_poll_group), "nvme_poll_groups"); 7344 7345 return 0; 7346 } 7347 7348 static void 7349 bdev_nvme_fini_destruct_ctrlrs(void) 7350 { 7351 struct nvme_bdev_ctrlr *nbdev_ctrlr; 7352 struct nvme_ctrlr *nvme_ctrlr; 7353 7354 pthread_mutex_lock(&g_bdev_nvme_mutex); 7355 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 7356 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 7357 pthread_mutex_lock(&nvme_ctrlr->mutex); 7358 if (nvme_ctrlr->destruct) { 7359 /* This controller's destruction was already started 7360 * before the application started shutting down 7361 */ 7362 pthread_mutex_unlock(&nvme_ctrlr->mutex); 7363 continue; 7364 } 7365 nvme_ctrlr->destruct = true; 7366 pthread_mutex_unlock(&nvme_ctrlr->mutex); 7367 7368 spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct, 7369 nvme_ctrlr); 7370 } 7371 } 7372 7373 g_bdev_nvme_module_finish = true; 7374 if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) { 7375 pthread_mutex_unlock(&g_bdev_nvme_mutex); 7376 spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL); 7377 spdk_bdev_module_fini_done(); 7378 return; 7379 } 7380 7381 pthread_mutex_unlock(&g_bdev_nvme_mutex); 7382 } 7383 7384 static void 7385 check_discovery_fini(void *arg) 7386 { 7387 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 7388 bdev_nvme_fini_destruct_ctrlrs(); 7389 } 7390 } 7391 7392 static void 7393 bdev_nvme_library_fini(void) 7394 { 7395 struct nvme_probe_skip_entry *entry, *entry_tmp; 7396 struct discovery_ctx *ctx; 7397 7398 spdk_poller_unregister(&g_hotplug_poller); 7399 free(g_hotplug_probe_ctx); 7400 g_hotplug_probe_ctx = NULL; 7401 7402 TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) { 7403 TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq); 7404 free(entry); 7405 } 7406 7407 assert(spdk_get_thread() == g_bdev_nvme_init_thread); 7408 if (TAILQ_EMPTY(&g_discovery_ctxs)) { 7409 bdev_nvme_fini_destruct_ctrlrs(); 7410 } else { 7411 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 7412 stop_discovery(ctx, check_discovery_fini, NULL); 7413 } 7414 } 7415 } 7416 7417 static void 7418 bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio) 7419 { 7420 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7421 struct spdk_bdev *bdev = bdev_io->bdev; 7422 struct spdk_dif_ctx dif_ctx; 7423 struct spdk_dif_error err_blk = {}; 7424 int rc; 7425 struct spdk_dif_ctx_init_ext_opts dif_opts; 7426 7427 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 7428 dif_opts.dif_pi_format = bdev->dif_pi_format; 7429 rc = spdk_dif_ctx_init(&dif_ctx, 7430 bdev->blocklen, bdev->md_len, bdev->md_interleave, 7431 bdev->dif_is_head_of_md, bdev->dif_type, 7432 bdev_io->u.bdev.dif_check_flags, 7433 bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0, &dif_opts); 7434 if (rc != 0) { 7435 SPDK_ERRLOG("Initialization of DIF context failed\n"); 7436 return; 7437 } 7438 7439 if (bdev->md_interleave) { 7440 rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 7441 bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 7442 } else { 7443 struct iovec md_iov = { 7444 .iov_base = bdev_io->u.bdev.md_buf, 7445 .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len, 7446 }; 7447 7448 rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, 7449 &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk); 7450 } 7451 7452 if (rc != 0) { 7453 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", 7454 err_blk.err_type, err_blk.err_offset); 7455 } else { 7456 SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n"); 7457 } 7458 } 7459 7460 static void 7461 bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7462 { 7463 struct nvme_bdev_io *bio = ref; 7464 7465 if (spdk_nvme_cpl_is_success(cpl)) { 7466 /* Run PI verification for read data buffer. */ 7467 bdev_nvme_verify_pi_error(bio); 7468 } 7469 7470 /* Return original completion status */ 7471 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 7472 } 7473 7474 static void 7475 bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7476 { 7477 struct nvme_bdev_io *bio = ref; 7478 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7479 int ret; 7480 7481 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 7482 SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n", 7483 cpl->status.sct, cpl->status.sc); 7484 7485 /* Save completion status to use after verifying PI error. */ 7486 bio->cpl = *cpl; 7487 7488 if (spdk_likely(nvme_io_path_is_available(bio->io_path))) { 7489 /* Read without PI checking to verify PI error. */ 7490 ret = bdev_nvme_no_pi_readv(bio, 7491 bdev_io->u.bdev.iovs, 7492 bdev_io->u.bdev.iovcnt, 7493 bdev_io->u.bdev.md_buf, 7494 bdev_io->u.bdev.num_blocks, 7495 bdev_io->u.bdev.offset_blocks); 7496 if (ret == 0) { 7497 return; 7498 } 7499 } 7500 } 7501 7502 bdev_nvme_io_complete_nvme_status(bio, cpl); 7503 } 7504 7505 static void 7506 bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7507 { 7508 struct nvme_bdev_io *bio = ref; 7509 7510 if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) { 7511 SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n", 7512 cpl->status.sct, cpl->status.sc); 7513 /* Run PI verification for write data buffer if PI error is detected. */ 7514 bdev_nvme_verify_pi_error(bio); 7515 } 7516 7517 bdev_nvme_io_complete_nvme_status(bio, cpl); 7518 } 7519 7520 static void 7521 bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl) 7522 { 7523 struct nvme_bdev_io *bio = ref; 7524 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7525 7526 /* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks. 7527 * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error(). 7528 */ 7529 bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0; 7530 7531 if (spdk_nvme_cpl_is_pi_error(cpl)) { 7532 SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n", 7533 cpl->status.sct, cpl->status.sc); 7534 /* Run PI verification for zone append data buffer if PI error is detected. */ 7535 bdev_nvme_verify_pi_error(bio); 7536 } 7537 7538 bdev_nvme_io_complete_nvme_status(bio, cpl); 7539 } 7540 7541 static void 7542 bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7543 { 7544 struct nvme_bdev_io *bio = ref; 7545 7546 if (spdk_nvme_cpl_is_pi_error(cpl)) { 7547 SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n", 7548 cpl->status.sct, cpl->status.sc); 7549 /* Run PI verification for compare data buffer if PI error is detected. */ 7550 bdev_nvme_verify_pi_error(bio); 7551 } 7552 7553 bdev_nvme_io_complete_nvme_status(bio, cpl); 7554 } 7555 7556 static void 7557 bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl) 7558 { 7559 struct nvme_bdev_io *bio = ref; 7560 7561 /* Compare operation completion */ 7562 if (!bio->first_fused_completed) { 7563 /* Save compare result for write callback */ 7564 bio->cpl = *cpl; 7565 bio->first_fused_completed = true; 7566 return; 7567 } 7568 7569 /* Write operation completion */ 7570 if (spdk_nvme_cpl_is_error(&bio->cpl)) { 7571 /* If bio->cpl is already an error, it means the compare operation failed. In that case, 7572 * complete the IO with the compare operation's status. 7573 */ 7574 if (!spdk_nvme_cpl_is_error(cpl)) { 7575 SPDK_ERRLOG("Unexpected write success after compare failure.\n"); 7576 } 7577 7578 bdev_nvme_io_complete_nvme_status(bio, &bio->cpl); 7579 } else { 7580 bdev_nvme_io_complete_nvme_status(bio, cpl); 7581 } 7582 } 7583 7584 static void 7585 bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl) 7586 { 7587 struct nvme_bdev_io *bio = ref; 7588 7589 bdev_nvme_io_complete_nvme_status(bio, cpl); 7590 } 7591 7592 static int 7593 fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc) 7594 { 7595 switch (desc->zt) { 7596 case SPDK_NVME_ZONE_TYPE_SEQWR: 7597 info->type = SPDK_BDEV_ZONE_TYPE_SEQWR; 7598 break; 7599 default: 7600 SPDK_ERRLOG("Invalid zone type: %#x in zone report\n", desc->zt); 7601 return -EIO; 7602 } 7603 7604 switch (desc->zs) { 7605 case SPDK_NVME_ZONE_STATE_EMPTY: 7606 info->state = SPDK_BDEV_ZONE_STATE_EMPTY; 7607 break; 7608 case SPDK_NVME_ZONE_STATE_IOPEN: 7609 info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN; 7610 break; 7611 case SPDK_NVME_ZONE_STATE_EOPEN: 7612 info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN; 7613 break; 7614 case SPDK_NVME_ZONE_STATE_CLOSED: 7615 info->state = SPDK_BDEV_ZONE_STATE_CLOSED; 7616 break; 7617 case SPDK_NVME_ZONE_STATE_RONLY: 7618 info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY; 7619 break; 7620 case SPDK_NVME_ZONE_STATE_FULL: 7621 info->state = SPDK_BDEV_ZONE_STATE_FULL; 7622 break; 7623 case SPDK_NVME_ZONE_STATE_OFFLINE: 7624 info->state = SPDK_BDEV_ZONE_STATE_OFFLINE; 7625 break; 7626 default: 7627 SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs); 7628 return -EIO; 7629 } 7630 7631 info->zone_id = desc->zslba; 7632 info->write_pointer = desc->wp; 7633 info->capacity = desc->zcap; 7634 7635 return 0; 7636 } 7637 7638 static void 7639 bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl) 7640 { 7641 struct nvme_bdev_io *bio = ref; 7642 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7643 uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id; 7644 uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones; 7645 struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf; 7646 uint64_t max_zones_per_buf, i; 7647 uint32_t zone_report_bufsize; 7648 struct spdk_nvme_ns *ns; 7649 struct spdk_nvme_qpair *qpair; 7650 int ret; 7651 7652 if (spdk_nvme_cpl_is_error(cpl)) { 7653 goto out_complete_io_nvme_cpl; 7654 } 7655 7656 if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) { 7657 ret = -ENXIO; 7658 goto out_complete_io_ret; 7659 } 7660 7661 ns = bio->io_path->nvme_ns->ns; 7662 qpair = bio->io_path->qpair->qpair; 7663 7664 zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 7665 max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) / 7666 sizeof(bio->zone_report_buf->descs[0]); 7667 7668 if (bio->zone_report_buf->nr_zones > max_zones_per_buf) { 7669 ret = -EINVAL; 7670 goto out_complete_io_ret; 7671 } 7672 7673 if (!bio->zone_report_buf->nr_zones) { 7674 ret = -EINVAL; 7675 goto out_complete_io_ret; 7676 } 7677 7678 for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) { 7679 ret = fill_zone_from_report(&info[bio->handled_zones], 7680 &bio->zone_report_buf->descs[i]); 7681 if (ret) { 7682 goto out_complete_io_ret; 7683 } 7684 bio->handled_zones++; 7685 } 7686 7687 if (bio->handled_zones < zones_to_copy) { 7688 uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 7689 uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones); 7690 7691 memset(bio->zone_report_buf, 0, zone_report_bufsize); 7692 ret = spdk_nvme_zns_report_zones(ns, qpair, 7693 bio->zone_report_buf, zone_report_bufsize, 7694 slba, SPDK_NVME_ZRA_LIST_ALL, true, 7695 bdev_nvme_get_zone_info_done, bio); 7696 if (!ret) { 7697 return; 7698 } else { 7699 goto out_complete_io_ret; 7700 } 7701 } 7702 7703 out_complete_io_nvme_cpl: 7704 free(bio->zone_report_buf); 7705 bio->zone_report_buf = NULL; 7706 bdev_nvme_io_complete_nvme_status(bio, cpl); 7707 return; 7708 7709 out_complete_io_ret: 7710 free(bio->zone_report_buf); 7711 bio->zone_report_buf = NULL; 7712 bdev_nvme_io_complete(bio, ret); 7713 } 7714 7715 static void 7716 bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl) 7717 { 7718 struct nvme_bdev_io *bio = ref; 7719 7720 bdev_nvme_io_complete_nvme_status(bio, cpl); 7721 } 7722 7723 static void 7724 bdev_nvme_admin_passthru_complete_nvme_status(void *ctx) 7725 { 7726 struct nvme_bdev_io *bio = ctx; 7727 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7728 const struct spdk_nvme_cpl *cpl = &bio->cpl; 7729 7730 assert(bdev_nvme_io_type_is_admin(bdev_io->type)); 7731 7732 __bdev_nvme_io_complete(bdev_io, 0, cpl); 7733 } 7734 7735 static void 7736 bdev_nvme_abort_complete(void *ctx) 7737 { 7738 struct nvme_bdev_io *bio = ctx; 7739 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7740 7741 if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) { 7742 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL); 7743 } else { 7744 __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL); 7745 } 7746 } 7747 7748 static void 7749 bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl) 7750 { 7751 struct nvme_bdev_io *bio = ref; 7752 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7753 7754 bio->cpl = *cpl; 7755 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), bdev_nvme_abort_complete, bio); 7756 } 7757 7758 static void 7759 bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl) 7760 { 7761 struct nvme_bdev_io *bio = ref; 7762 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 7763 7764 bio->cpl = *cpl; 7765 spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), 7766 bdev_nvme_admin_passthru_complete_nvme_status, bio); 7767 } 7768 7769 static void 7770 bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset) 7771 { 7772 struct nvme_bdev_io *bio = ref; 7773 struct iovec *iov; 7774 7775 bio->iov_offset = sgl_offset; 7776 for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) { 7777 iov = &bio->iovs[bio->iovpos]; 7778 if (bio->iov_offset < iov->iov_len) { 7779 break; 7780 } 7781 7782 bio->iov_offset -= iov->iov_len; 7783 } 7784 } 7785 7786 static int 7787 bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length) 7788 { 7789 struct nvme_bdev_io *bio = ref; 7790 struct iovec *iov; 7791 7792 assert(bio->iovpos < bio->iovcnt); 7793 7794 iov = &bio->iovs[bio->iovpos]; 7795 7796 *address = iov->iov_base; 7797 *length = iov->iov_len; 7798 7799 if (bio->iov_offset) { 7800 assert(bio->iov_offset <= iov->iov_len); 7801 *address += bio->iov_offset; 7802 *length -= bio->iov_offset; 7803 } 7804 7805 bio->iov_offset += *length; 7806 if (bio->iov_offset == iov->iov_len) { 7807 bio->iovpos++; 7808 bio->iov_offset = 0; 7809 } 7810 7811 return 0; 7812 } 7813 7814 static void 7815 bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset) 7816 { 7817 struct nvme_bdev_io *bio = ref; 7818 struct iovec *iov; 7819 7820 bio->fused_iov_offset = sgl_offset; 7821 for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) { 7822 iov = &bio->fused_iovs[bio->fused_iovpos]; 7823 if (bio->fused_iov_offset < iov->iov_len) { 7824 break; 7825 } 7826 7827 bio->fused_iov_offset -= iov->iov_len; 7828 } 7829 } 7830 7831 static int 7832 bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length) 7833 { 7834 struct nvme_bdev_io *bio = ref; 7835 struct iovec *iov; 7836 7837 assert(bio->fused_iovpos < bio->fused_iovcnt); 7838 7839 iov = &bio->fused_iovs[bio->fused_iovpos]; 7840 7841 *address = iov->iov_base; 7842 *length = iov->iov_len; 7843 7844 if (bio->fused_iov_offset) { 7845 assert(bio->fused_iov_offset <= iov->iov_len); 7846 *address += bio->fused_iov_offset; 7847 *length -= bio->fused_iov_offset; 7848 } 7849 7850 bio->fused_iov_offset += *length; 7851 if (bio->fused_iov_offset == iov->iov_len) { 7852 bio->fused_iovpos++; 7853 bio->fused_iov_offset = 0; 7854 } 7855 7856 return 0; 7857 } 7858 7859 static int 7860 bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7861 void *md, uint64_t lba_count, uint64_t lba) 7862 { 7863 int rc; 7864 7865 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n", 7866 lba_count, lba); 7867 7868 bio->iovs = iov; 7869 bio->iovcnt = iovcnt; 7870 bio->iovpos = 0; 7871 bio->iov_offset = 0; 7872 7873 rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns, 7874 bio->io_path->qpair->qpair, 7875 lba, lba_count, 7876 bdev_nvme_no_pi_readv_done, bio, 0, 7877 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 7878 md, 0, 0); 7879 7880 if (rc != 0 && rc != -ENOMEM) { 7881 SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc); 7882 } 7883 return rc; 7884 } 7885 7886 static int 7887 bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7888 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 7889 struct spdk_memory_domain *domain, void *domain_ctx, 7890 struct spdk_accel_sequence *seq) 7891 { 7892 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7893 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7894 int rc; 7895 7896 SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n", 7897 lba_count, lba); 7898 7899 bio->iovs = iov; 7900 bio->iovcnt = iovcnt; 7901 bio->iovpos = 0; 7902 bio->iov_offset = 0; 7903 7904 if (domain != NULL || seq != NULL) { 7905 bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence); 7906 bio->ext_opts.memory_domain = domain; 7907 bio->ext_opts.memory_domain_ctx = domain_ctx; 7908 bio->ext_opts.io_flags = flags; 7909 bio->ext_opts.metadata = md; 7910 bio->ext_opts.accel_sequence = seq; 7911 7912 if (iovcnt == 1) { 7913 rc = spdk_nvme_ns_cmd_read_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_readv_done, 7914 bio, &bio->ext_opts); 7915 } else { 7916 rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count, 7917 bdev_nvme_readv_done, bio, 7918 bdev_nvme_queued_reset_sgl, 7919 bdev_nvme_queued_next_sge, 7920 &bio->ext_opts); 7921 } 7922 } else if (iovcnt == 1) { 7923 rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, iov[0].iov_base, 7924 md, lba, lba_count, bdev_nvme_readv_done, 7925 bio, flags, 0, 0); 7926 } else { 7927 rc = spdk_nvme_ns_cmd_readv_with_md(ns, qpair, lba, lba_count, 7928 bdev_nvme_readv_done, bio, flags, 7929 bdev_nvme_queued_reset_sgl, 7930 bdev_nvme_queued_next_sge, md, 0, 0); 7931 } 7932 7933 if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) { 7934 SPDK_ERRLOG("readv failed: rc = %d\n", rc); 7935 } 7936 return rc; 7937 } 7938 7939 static int 7940 bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7941 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags, 7942 struct spdk_memory_domain *domain, void *domain_ctx, 7943 struct spdk_accel_sequence *seq, 7944 union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13) 7945 { 7946 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 7947 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 7948 int rc; 7949 7950 SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 7951 lba_count, lba); 7952 7953 bio->iovs = iov; 7954 bio->iovcnt = iovcnt; 7955 bio->iovpos = 0; 7956 bio->iov_offset = 0; 7957 7958 if (domain != NULL || seq != NULL) { 7959 bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence); 7960 bio->ext_opts.memory_domain = domain; 7961 bio->ext_opts.memory_domain_ctx = domain_ctx; 7962 bio->ext_opts.io_flags = flags | SPDK_NVME_IO_FLAGS_DIRECTIVE(cdw12.write.dtype); 7963 bio->ext_opts.cdw13 = cdw13.raw; 7964 bio->ext_opts.metadata = md; 7965 bio->ext_opts.accel_sequence = seq; 7966 7967 if (iovcnt == 1) { 7968 rc = spdk_nvme_ns_cmd_write_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_writev_done, 7969 bio, &bio->ext_opts); 7970 } else { 7971 rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count, 7972 bdev_nvme_writev_done, bio, 7973 bdev_nvme_queued_reset_sgl, 7974 bdev_nvme_queued_next_sge, 7975 &bio->ext_opts); 7976 } 7977 } else if (iovcnt == 1) { 7978 rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, iov[0].iov_base, 7979 md, lba, lba_count, bdev_nvme_writev_done, 7980 bio, flags, 0, 0); 7981 } else { 7982 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 7983 bdev_nvme_writev_done, bio, flags, 7984 bdev_nvme_queued_reset_sgl, 7985 bdev_nvme_queued_next_sge, md, 0, 0); 7986 } 7987 7988 if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) { 7989 SPDK_ERRLOG("writev failed: rc = %d\n", rc); 7990 } 7991 return rc; 7992 } 7993 7994 static int 7995 bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 7996 void *md, uint64_t lba_count, uint64_t zslba, 7997 uint32_t flags) 7998 { 7999 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8000 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8001 int rc; 8002 8003 SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n", 8004 lba_count, zslba); 8005 8006 bio->iovs = iov; 8007 bio->iovcnt = iovcnt; 8008 bio->iovpos = 0; 8009 bio->iov_offset = 0; 8010 8011 if (iovcnt == 1) { 8012 rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba, 8013 lba_count, 8014 bdev_nvme_zone_appendv_done, bio, 8015 flags, 8016 0, 0); 8017 } else { 8018 rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count, 8019 bdev_nvme_zone_appendv_done, bio, flags, 8020 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 8021 md, 0, 0); 8022 } 8023 8024 if (rc != 0 && rc != -ENOMEM) { 8025 SPDK_ERRLOG("zone append failed: rc = %d\n", rc); 8026 } 8027 return rc; 8028 } 8029 8030 static int 8031 bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt, 8032 void *md, uint64_t lba_count, uint64_t lba, 8033 uint32_t flags) 8034 { 8035 int rc; 8036 8037 SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n", 8038 lba_count, lba); 8039 8040 bio->iovs = iov; 8041 bio->iovcnt = iovcnt; 8042 bio->iovpos = 0; 8043 bio->iov_offset = 0; 8044 8045 rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns, 8046 bio->io_path->qpair->qpair, 8047 lba, lba_count, 8048 bdev_nvme_comparev_done, bio, flags, 8049 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, 8050 md, 0, 0); 8051 8052 if (rc != 0 && rc != -ENOMEM) { 8053 SPDK_ERRLOG("comparev failed: rc = %d\n", rc); 8054 } 8055 return rc; 8056 } 8057 8058 static int 8059 bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt, 8060 struct iovec *write_iov, int write_iovcnt, 8061 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags) 8062 { 8063 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8064 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8065 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio); 8066 int rc; 8067 8068 SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n", 8069 lba_count, lba); 8070 8071 bio->iovs = cmp_iov; 8072 bio->iovcnt = cmp_iovcnt; 8073 bio->iovpos = 0; 8074 bio->iov_offset = 0; 8075 bio->fused_iovs = write_iov; 8076 bio->fused_iovcnt = write_iovcnt; 8077 bio->fused_iovpos = 0; 8078 bio->fused_iov_offset = 0; 8079 8080 if (bdev_io->num_retries == 0) { 8081 bio->first_fused_submitted = false; 8082 bio->first_fused_completed = false; 8083 } 8084 8085 if (!bio->first_fused_submitted) { 8086 flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST; 8087 memset(&bio->cpl, 0, sizeof(bio->cpl)); 8088 8089 rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count, 8090 bdev_nvme_comparev_and_writev_done, bio, flags, 8091 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0); 8092 if (rc == 0) { 8093 bio->first_fused_submitted = true; 8094 flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST; 8095 } else { 8096 if (rc != -ENOMEM) { 8097 SPDK_ERRLOG("compare failed: rc = %d\n", rc); 8098 } 8099 return rc; 8100 } 8101 } 8102 8103 flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND; 8104 8105 rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count, 8106 bdev_nvme_comparev_and_writev_done, bio, flags, 8107 bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0); 8108 if (rc != 0 && rc != -ENOMEM) { 8109 SPDK_ERRLOG("write failed: rc = %d\n", rc); 8110 rc = 0; 8111 } 8112 8113 return rc; 8114 } 8115 8116 static int 8117 bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 8118 { 8119 struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES]; 8120 struct spdk_nvme_dsm_range *range; 8121 uint64_t offset, remaining; 8122 uint64_t num_ranges_u64; 8123 uint16_t num_ranges; 8124 int rc; 8125 8126 num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) / 8127 SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8128 if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) { 8129 SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks); 8130 return -EINVAL; 8131 } 8132 num_ranges = (uint16_t)num_ranges_u64; 8133 8134 offset = offset_blocks; 8135 remaining = num_blocks; 8136 range = &dsm_ranges[0]; 8137 8138 /* Fill max-size ranges until the remaining blocks fit into one range */ 8139 while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) { 8140 range->attributes.raw = 0; 8141 range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8142 range->starting_lba = offset; 8143 8144 offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8145 remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS; 8146 range++; 8147 } 8148 8149 /* Final range describes the remaining blocks */ 8150 range->attributes.raw = 0; 8151 range->length = remaining; 8152 range->starting_lba = offset; 8153 8154 rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns, 8155 bio->io_path->qpair->qpair, 8156 SPDK_NVME_DSM_ATTR_DEALLOCATE, 8157 dsm_ranges, num_ranges, 8158 bdev_nvme_queued_done, bio); 8159 8160 return rc; 8161 } 8162 8163 static int 8164 bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks) 8165 { 8166 if (num_blocks > UINT16_MAX + 1) { 8167 SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n"); 8168 return -EINVAL; 8169 } 8170 8171 return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns, 8172 bio->io_path->qpair->qpair, 8173 offset_blocks, num_blocks, 8174 bdev_nvme_queued_done, bio, 8175 0); 8176 } 8177 8178 static int 8179 bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones, 8180 struct spdk_bdev_zone_info *info) 8181 { 8182 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8183 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8184 uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns); 8185 uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns); 8186 uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns); 8187 8188 if (zone_id % zone_size != 0) { 8189 return -EINVAL; 8190 } 8191 8192 if (num_zones > total_zones || !num_zones) { 8193 return -EINVAL; 8194 } 8195 8196 assert(!bio->zone_report_buf); 8197 bio->zone_report_buf = calloc(1, zone_report_bufsize); 8198 if (!bio->zone_report_buf) { 8199 return -ENOMEM; 8200 } 8201 8202 bio->handled_zones = 0; 8203 8204 return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize, 8205 zone_id, SPDK_NVME_ZRA_LIST_ALL, true, 8206 bdev_nvme_get_zone_info_done, bio); 8207 } 8208 8209 static int 8210 bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id, 8211 enum spdk_bdev_zone_action action) 8212 { 8213 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8214 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8215 8216 switch (action) { 8217 case SPDK_BDEV_ZONE_CLOSE: 8218 return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false, 8219 bdev_nvme_zone_management_done, bio); 8220 case SPDK_BDEV_ZONE_FINISH: 8221 return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false, 8222 bdev_nvme_zone_management_done, bio); 8223 case SPDK_BDEV_ZONE_OPEN: 8224 return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false, 8225 bdev_nvme_zone_management_done, bio); 8226 case SPDK_BDEV_ZONE_RESET: 8227 return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false, 8228 bdev_nvme_zone_management_done, bio); 8229 case SPDK_BDEV_ZONE_OFFLINE: 8230 return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false, 8231 bdev_nvme_zone_management_done, bio); 8232 default: 8233 return -EINVAL; 8234 } 8235 } 8236 8237 static void 8238 bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 8239 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes) 8240 { 8241 struct nvme_io_path *io_path; 8242 struct nvme_ctrlr *nvme_ctrlr; 8243 uint32_t max_xfer_size; 8244 int rc = -ENXIO; 8245 8246 /* Choose the first ctrlr which is not failed. */ 8247 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 8248 nvme_ctrlr = io_path->qpair->ctrlr; 8249 8250 /* We should skip any unavailable nvme_ctrlr rather than checking 8251 * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO. 8252 */ 8253 if (!nvme_ctrlr_is_available(nvme_ctrlr)) { 8254 continue; 8255 } 8256 8257 max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr); 8258 8259 if (nbytes > max_xfer_size) { 8260 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8261 rc = -EINVAL; 8262 goto err; 8263 } 8264 8265 rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes, 8266 bdev_nvme_admin_passthru_done, bio); 8267 if (rc == 0) { 8268 return; 8269 } 8270 } 8271 8272 err: 8273 bdev_nvme_admin_complete(bio, rc); 8274 } 8275 8276 static int 8277 bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 8278 void *buf, size_t nbytes) 8279 { 8280 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8281 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8282 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8283 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8284 8285 if (nbytes > max_xfer_size) { 8286 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8287 return -EINVAL; 8288 } 8289 8290 /* 8291 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 8292 * so fill it out automatically. 8293 */ 8294 cmd->nsid = spdk_nvme_ns_get_id(ns); 8295 8296 return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf, 8297 (uint32_t)nbytes, bdev_nvme_queued_done, bio); 8298 } 8299 8300 static int 8301 bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd, 8302 void *buf, size_t nbytes, void *md_buf, size_t md_len) 8303 { 8304 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8305 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8306 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 8307 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8308 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8309 8310 if (nbytes > max_xfer_size) { 8311 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8312 return -EINVAL; 8313 } 8314 8315 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 8316 SPDK_ERRLOG("invalid meta data buffer size\n"); 8317 return -EINVAL; 8318 } 8319 8320 /* 8321 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid, 8322 * so fill it out automatically. 8323 */ 8324 cmd->nsid = spdk_nvme_ns_get_id(ns); 8325 8326 return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf, 8327 (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio); 8328 } 8329 8330 static int 8331 bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, 8332 struct spdk_nvme_cmd *cmd, struct iovec *iov, int iovcnt, 8333 size_t nbytes, void *md_buf, size_t md_len) 8334 { 8335 struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns; 8336 struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair; 8337 size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns); 8338 uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 8339 struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns); 8340 8341 bio->iovs = iov; 8342 bio->iovcnt = iovcnt; 8343 bio->iovpos = 0; 8344 bio->iov_offset = 0; 8345 8346 if (nbytes > max_xfer_size) { 8347 SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size); 8348 return -EINVAL; 8349 } 8350 8351 if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) { 8352 SPDK_ERRLOG("invalid meta data buffer size\n"); 8353 return -EINVAL; 8354 } 8355 8356 /* 8357 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands 8358 * require a nsid, so fill it out automatically. 8359 */ 8360 cmd->nsid = spdk_nvme_ns_get_id(ns); 8361 8362 return spdk_nvme_ctrlr_cmd_iov_raw_with_md( 8363 ctrlr, qpair, cmd, (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio, 8364 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge); 8365 } 8366 8367 static void 8368 bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio, 8369 struct nvme_bdev_io *bio_to_abort) 8370 { 8371 struct nvme_io_path *io_path; 8372 int rc = 0; 8373 8374 rc = bdev_nvme_abort_retry_io(nbdev_ch, bio_to_abort); 8375 if (rc == 0) { 8376 bdev_nvme_admin_complete(bio, 0); 8377 return; 8378 } 8379 8380 io_path = bio_to_abort->io_path; 8381 if (io_path != NULL) { 8382 rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr, 8383 io_path->qpair->qpair, 8384 bio_to_abort, 8385 bdev_nvme_abort_done, bio); 8386 } else { 8387 STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { 8388 rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr, 8389 NULL, 8390 bio_to_abort, 8391 bdev_nvme_abort_done, bio); 8392 8393 if (rc != -ENOENT) { 8394 break; 8395 } 8396 } 8397 } 8398 8399 if (rc != 0) { 8400 /* If no command was found or there was any error, complete the abort 8401 * request with failure. 8402 */ 8403 bdev_nvme_admin_complete(bio, rc); 8404 } 8405 } 8406 8407 static int 8408 bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, uint64_t src_offset_blocks, 8409 uint64_t num_blocks) 8410 { 8411 struct spdk_nvme_scc_source_range range = { 8412 .slba = src_offset_blocks, 8413 .nlb = num_blocks - 1 8414 }; 8415 8416 return spdk_nvme_ns_cmd_copy(bio->io_path->nvme_ns->ns, 8417 bio->io_path->qpair->qpair, 8418 &range, 1, dst_offset_blocks, 8419 bdev_nvme_queued_done, bio); 8420 } 8421 8422 static void 8423 bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w) 8424 { 8425 const char *action; 8426 uint32_t i; 8427 8428 if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) { 8429 action = "reset"; 8430 } else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) { 8431 action = "abort"; 8432 } else { 8433 action = "none"; 8434 } 8435 8436 spdk_json_write_object_begin(w); 8437 8438 spdk_json_write_named_string(w, "method", "bdev_nvme_set_options"); 8439 8440 spdk_json_write_named_object_begin(w, "params"); 8441 spdk_json_write_named_string(w, "action_on_timeout", action); 8442 spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us); 8443 spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us); 8444 spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms); 8445 spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst); 8446 spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight); 8447 spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight); 8448 spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight); 8449 spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us); 8450 spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us); 8451 spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests); 8452 spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit); 8453 spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count); 8454 spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count); 8455 spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout); 8456 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec); 8457 spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec); 8458 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec); 8459 spdk_json_write_named_bool(w, "disable_auto_failback", g_opts.disable_auto_failback); 8460 spdk_json_write_named_bool(w, "generate_uuids", g_opts.generate_uuids); 8461 spdk_json_write_named_uint8(w, "transport_tos", g_opts.transport_tos); 8462 spdk_json_write_named_bool(w, "nvme_error_stat", g_opts.nvme_error_stat); 8463 spdk_json_write_named_uint32(w, "rdma_srq_size", g_opts.rdma_srq_size); 8464 spdk_json_write_named_bool(w, "io_path_stat", g_opts.io_path_stat); 8465 spdk_json_write_named_bool(w, "allow_accel_sequence", g_opts.allow_accel_sequence); 8466 spdk_json_write_named_uint32(w, "rdma_max_cq_size", g_opts.rdma_max_cq_size); 8467 spdk_json_write_named_uint16(w, "rdma_cm_event_timeout_ms", g_opts.rdma_cm_event_timeout_ms); 8468 spdk_json_write_named_array_begin(w, "dhchap_digests"); 8469 for (i = 0; i < 32; ++i) { 8470 if (g_opts.dhchap_digests & SPDK_BIT(i)) { 8471 spdk_json_write_string(w, spdk_nvme_dhchap_get_digest_name(i)); 8472 } 8473 } 8474 spdk_json_write_array_end(w); 8475 spdk_json_write_named_array_begin(w, "dhchap_dhgroups"); 8476 for (i = 0; i < 32; ++i) { 8477 if (g_opts.dhchap_dhgroups & SPDK_BIT(i)) { 8478 spdk_json_write_string(w, spdk_nvme_dhchap_get_dhgroup_name(i)); 8479 } 8480 } 8481 8482 spdk_json_write_array_end(w); 8483 spdk_json_write_object_end(w); 8484 8485 spdk_json_write_object_end(w); 8486 } 8487 8488 static void 8489 bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx) 8490 { 8491 struct spdk_nvme_transport_id trid; 8492 8493 spdk_json_write_object_begin(w); 8494 8495 spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery"); 8496 8497 spdk_json_write_named_object_begin(w, "params"); 8498 spdk_json_write_named_string(w, "name", ctx->name); 8499 spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn); 8500 8501 trid = ctx->trid; 8502 memset(trid.subnqn, 0, sizeof(trid.subnqn)); 8503 nvme_bdev_dump_trid_json(&trid, w); 8504 8505 spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach); 8506 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec); 8507 spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec); 8508 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 8509 ctx->bdev_opts.fast_io_fail_timeout_sec); 8510 spdk_json_write_object_end(w); 8511 8512 spdk_json_write_object_end(w); 8513 } 8514 8515 #ifdef SPDK_CONFIG_NVME_CUSE 8516 static void 8517 nvme_ctrlr_cuse_config_json(struct spdk_json_write_ctx *w, 8518 struct nvme_ctrlr *nvme_ctrlr) 8519 { 8520 size_t cuse_name_size = 128; 8521 char cuse_name[cuse_name_size]; 8522 8523 if (spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, 8524 cuse_name, &cuse_name_size) != 0) { 8525 return; 8526 } 8527 8528 spdk_json_write_object_begin(w); 8529 8530 spdk_json_write_named_string(w, "method", "bdev_nvme_cuse_register"); 8531 8532 spdk_json_write_named_object_begin(w, "params"); 8533 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 8534 spdk_json_write_object_end(w); 8535 8536 spdk_json_write_object_end(w); 8537 } 8538 #endif 8539 8540 static void 8541 nvme_ctrlr_config_json(struct spdk_json_write_ctx *w, 8542 struct nvme_ctrlr *nvme_ctrlr, 8543 struct nvme_path_id *path_id) 8544 { 8545 struct spdk_nvme_transport_id *trid; 8546 const struct spdk_nvme_ctrlr_opts *opts; 8547 8548 if (nvme_ctrlr->opts.from_discovery_service) { 8549 /* Do not emit an RPC for this - it will be implicitly 8550 * covered by a separate bdev_nvme_start_discovery or 8551 * bdev_nvme_start_mdns_discovery RPC. 8552 */ 8553 return; 8554 } 8555 8556 trid = &path_id->trid; 8557 8558 spdk_json_write_object_begin(w); 8559 8560 spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller"); 8561 8562 spdk_json_write_named_object_begin(w, "params"); 8563 spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name); 8564 nvme_bdev_dump_trid_json(trid, w); 8565 spdk_json_write_named_bool(w, "prchk_reftag", 8566 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0); 8567 spdk_json_write_named_bool(w, "prchk_guard", 8568 (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0); 8569 spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec); 8570 spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec); 8571 spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", 8572 nvme_ctrlr->opts.fast_io_fail_timeout_sec); 8573 if (nvme_ctrlr->psk != NULL) { 8574 spdk_json_write_named_string(w, "psk", spdk_key_get_name(nvme_ctrlr->psk)); 8575 } 8576 8577 opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr); 8578 spdk_json_write_named_string(w, "hostnqn", opts->hostnqn); 8579 spdk_json_write_named_bool(w, "hdgst", opts->header_digest); 8580 spdk_json_write_named_bool(w, "ddgst", opts->data_digest); 8581 if (opts->src_addr[0] != '\0') { 8582 spdk_json_write_named_string(w, "hostaddr", opts->src_addr); 8583 } 8584 if (opts->src_svcid[0] != '\0') { 8585 spdk_json_write_named_string(w, "hostsvcid", opts->src_svcid); 8586 } 8587 8588 if (nvme_ctrlr->opts.multipath) { 8589 spdk_json_write_named_string(w, "multipath", "multipath"); 8590 } 8591 spdk_json_write_object_end(w); 8592 8593 spdk_json_write_object_end(w); 8594 } 8595 8596 static void 8597 bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w) 8598 { 8599 spdk_json_write_object_begin(w); 8600 spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug"); 8601 8602 spdk_json_write_named_object_begin(w, "params"); 8603 spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us); 8604 spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled); 8605 spdk_json_write_object_end(w); 8606 8607 spdk_json_write_object_end(w); 8608 } 8609 8610 static int 8611 bdev_nvme_config_json(struct spdk_json_write_ctx *w) 8612 { 8613 struct nvme_bdev_ctrlr *nbdev_ctrlr; 8614 struct nvme_ctrlr *nvme_ctrlr; 8615 struct discovery_ctx *ctx; 8616 struct nvme_path_id *path_id; 8617 8618 bdev_nvme_opts_config_json(w); 8619 8620 pthread_mutex_lock(&g_bdev_nvme_mutex); 8621 8622 TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { 8623 TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) { 8624 path_id = nvme_ctrlr->active_path_id; 8625 assert(path_id == TAILQ_FIRST(&nvme_ctrlr->trids)); 8626 nvme_ctrlr_config_json(w, nvme_ctrlr, path_id); 8627 8628 path_id = TAILQ_NEXT(path_id, link); 8629 while (path_id != NULL) { 8630 nvme_ctrlr_config_json(w, nvme_ctrlr, path_id); 8631 path_id = TAILQ_NEXT(path_id, link); 8632 } 8633 8634 #ifdef SPDK_CONFIG_NVME_CUSE 8635 nvme_ctrlr_cuse_config_json(w, nvme_ctrlr); 8636 #endif 8637 } 8638 } 8639 8640 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 8641 if (!ctx->from_mdns_discovery_service) { 8642 bdev_nvme_discovery_config_json(w, ctx); 8643 } 8644 } 8645 8646 bdev_nvme_mdns_discovery_config_json(w); 8647 8648 /* Dump as last parameter to give all NVMe bdevs chance to be constructed 8649 * before enabling hotplug poller. 8650 */ 8651 bdev_nvme_hotplug_config_json(w); 8652 8653 pthread_mutex_unlock(&g_bdev_nvme_mutex); 8654 return 0; 8655 } 8656 8657 struct spdk_nvme_ctrlr * 8658 bdev_nvme_get_ctrlr(struct spdk_bdev *bdev) 8659 { 8660 struct nvme_bdev *nbdev; 8661 struct nvme_ns *nvme_ns; 8662 8663 if (!bdev || bdev->module != &nvme_if) { 8664 return NULL; 8665 } 8666 8667 nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk); 8668 nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list); 8669 assert(nvme_ns != NULL); 8670 8671 return nvme_ns->ctrlr->ctrlr; 8672 } 8673 8674 static bool 8675 nvme_io_path_is_current(struct nvme_io_path *io_path) 8676 { 8677 const struct nvme_bdev_channel *nbdev_ch; 8678 bool current; 8679 8680 if (!nvme_io_path_is_available(io_path)) { 8681 return false; 8682 } 8683 8684 nbdev_ch = io_path->nbdev_ch; 8685 if (nbdev_ch == NULL) { 8686 current = false; 8687 } else if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) { 8688 struct nvme_io_path *optimized_io_path = NULL; 8689 8690 STAILQ_FOREACH(optimized_io_path, &nbdev_ch->io_path_list, stailq) { 8691 if (optimized_io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) { 8692 break; 8693 } 8694 } 8695 8696 /* A non-optimized path is only current if there are no optimized paths. */ 8697 current = (io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) || 8698 (optimized_io_path == NULL); 8699 } else { 8700 if (nbdev_ch->current_io_path) { 8701 current = (io_path == nbdev_ch->current_io_path); 8702 } else { 8703 struct nvme_io_path *first_path; 8704 8705 /* We arrived here as there are no optimized paths for active-passive 8706 * mode. Check if this io_path is the first one available on the list. 8707 */ 8708 current = false; 8709 STAILQ_FOREACH(first_path, &nbdev_ch->io_path_list, stailq) { 8710 if (nvme_io_path_is_available(first_path)) { 8711 current = (io_path == first_path); 8712 break; 8713 } 8714 } 8715 } 8716 } 8717 8718 return current; 8719 } 8720 8721 void 8722 nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path) 8723 { 8724 struct nvme_ns *nvme_ns = io_path->nvme_ns; 8725 struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr; 8726 const struct spdk_nvme_ctrlr_data *cdata; 8727 const struct spdk_nvme_transport_id *trid; 8728 const char *adrfam_str; 8729 8730 spdk_json_write_object_begin(w); 8731 8732 spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name); 8733 8734 cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr); 8735 trid = spdk_nvme_ctrlr_get_transport_id(nvme_ctrlr->ctrlr); 8736 8737 spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid); 8738 spdk_json_write_named_bool(w, "current", nvme_io_path_is_current(io_path)); 8739 spdk_json_write_named_bool(w, "connected", nvme_qpair_is_connected(io_path->qpair)); 8740 spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns)); 8741 8742 spdk_json_write_named_object_begin(w, "transport"); 8743 spdk_json_write_named_string(w, "trtype", trid->trstring); 8744 spdk_json_write_named_string(w, "traddr", trid->traddr); 8745 if (trid->trsvcid[0] != '\0') { 8746 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 8747 } 8748 adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 8749 if (adrfam_str) { 8750 spdk_json_write_named_string(w, "adrfam", adrfam_str); 8751 } 8752 spdk_json_write_object_end(w); 8753 8754 spdk_json_write_object_end(w); 8755 } 8756 8757 void 8758 bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w) 8759 { 8760 struct discovery_ctx *ctx; 8761 struct discovery_entry_ctx *entry_ctx; 8762 8763 spdk_json_write_array_begin(w); 8764 TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) { 8765 spdk_json_write_object_begin(w); 8766 spdk_json_write_named_string(w, "name", ctx->name); 8767 8768 spdk_json_write_named_object_begin(w, "trid"); 8769 nvme_bdev_dump_trid_json(&ctx->trid, w); 8770 spdk_json_write_object_end(w); 8771 8772 spdk_json_write_named_array_begin(w, "referrals"); 8773 TAILQ_FOREACH(entry_ctx, &ctx->discovery_entry_ctxs, tailq) { 8774 spdk_json_write_object_begin(w); 8775 spdk_json_write_named_object_begin(w, "trid"); 8776 nvme_bdev_dump_trid_json(&entry_ctx->trid, w); 8777 spdk_json_write_object_end(w); 8778 spdk_json_write_object_end(w); 8779 } 8780 spdk_json_write_array_end(w); 8781 8782 spdk_json_write_object_end(w); 8783 } 8784 spdk_json_write_array_end(w); 8785 } 8786 8787 SPDK_LOG_REGISTER_COMPONENT(bdev_nvme) 8788 8789 SPDK_TRACE_REGISTER_FN(bdev_nvme_trace, "bdev_nvme", TRACE_GROUP_BDEV_NVME) 8790 { 8791 struct spdk_trace_tpoint_opts opts[] = { 8792 { 8793 "BDEV_NVME_IO_START", TRACE_BDEV_NVME_IO_START, 8794 OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 1, 8795 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 8796 }, 8797 { 8798 "BDEV_NVME_IO_DONE", TRACE_BDEV_NVME_IO_DONE, 8799 OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 0, 8800 {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }} 8801 } 8802 }; 8803 8804 8805 spdk_trace_register_object(OBJECT_BDEV_NVME_IO, 'N'); 8806 spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts)); 8807 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 8808 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_SUBMIT, OBJECT_BDEV_NVME_IO, 0); 8809 spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 8810 spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_COMPLETE, OBJECT_BDEV_NVME_IO, 0); 8811 } 8812