xref: /spdk/module/bdev/nvme/bdev_nvme.c (revision 2bc134eb4bfe9aaa378a9944aa7cb89ef1e4d6a4)
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation. All rights reserved.
5  *   Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
6  *   Copyright (c) 2021, 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
7  *
8  *   Redistribution and use in source and binary forms, with or without
9  *   modification, are permitted provided that the following conditions
10  *   are met:
11  *
12  *     * Redistributions of source code must retain the above copyright
13  *       notice, this list of conditions and the following disclaimer.
14  *     * Redistributions in binary form must reproduce the above copyright
15  *       notice, this list of conditions and the following disclaimer in
16  *       the documentation and/or other materials provided with the
17  *       distribution.
18  *     * Neither the name of Intel Corporation nor the names of its
19  *       contributors may be used to endorse or promote products derived
20  *       from this software without specific prior written permission.
21  *
22  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include "spdk/stdinc.h"
36 
37 #include "bdev_nvme.h"
38 
39 #include "spdk/accel_engine.h"
40 #include "spdk/config.h"
41 #include "spdk/endian.h"
42 #include "spdk/bdev.h"
43 #include "spdk/json.h"
44 #include "spdk/likely.h"
45 #include "spdk/nvme.h"
46 #include "spdk/nvme_ocssd.h"
47 #include "spdk/nvme_zns.h"
48 #include "spdk/opal.h"
49 #include "spdk/thread.h"
50 #include "spdk/string.h"
51 #include "spdk/util.h"
52 
53 #include "spdk/bdev_module.h"
54 #include "spdk/log.h"
55 
56 #include "spdk_internal/usdt.h"
57 
58 #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true
59 #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS	(10000)
60 
61 static int bdev_nvme_config_json(struct spdk_json_write_ctx *w);
62 
63 struct nvme_bdev_io {
64 	/** array of iovecs to transfer. */
65 	struct iovec *iovs;
66 
67 	/** Number of iovecs in iovs array. */
68 	int iovcnt;
69 
70 	/** Current iovec position. */
71 	int iovpos;
72 
73 	/** Offset in current iovec. */
74 	uint32_t iov_offset;
75 
76 	/** I/O path the current I/O or admin passthrough is submitted on, or the I/O path
77 	 *  being reset in a reset I/O.
78 	 */
79 	struct nvme_io_path *io_path;
80 
81 	/** array of iovecs to transfer. */
82 	struct iovec *fused_iovs;
83 
84 	/** Number of iovecs in iovs array. */
85 	int fused_iovcnt;
86 
87 	/** Current iovec position. */
88 	int fused_iovpos;
89 
90 	/** Offset in current iovec. */
91 	uint32_t fused_iov_offset;
92 
93 	/** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */
94 	struct spdk_nvme_cpl cpl;
95 
96 	/** Extended IO opts passed by the user to bdev layer and mapped to NVME format */
97 	struct spdk_nvme_ns_cmd_ext_io_opts ext_opts;
98 
99 	/** Originating thread */
100 	struct spdk_thread *orig_thread;
101 
102 	/** Keeps track if first of fused commands was submitted */
103 	bool first_fused_submitted;
104 
105 	/** Keeps track if first of fused commands was completed */
106 	bool first_fused_completed;
107 
108 	/** Temporary pointer to zone report buffer */
109 	struct spdk_nvme_zns_zone_report *zone_report_buf;
110 
111 	/** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */
112 	uint64_t handled_zones;
113 
114 	/** Expiration value in ticks to retry the current I/O. */
115 	uint64_t retry_ticks;
116 
117 	/* How many times the current I/O was retried. */
118 	int32_t retry_count;
119 };
120 
121 struct nvme_probe_skip_entry {
122 	struct spdk_nvme_transport_id		trid;
123 	TAILQ_ENTRY(nvme_probe_skip_entry)	tailq;
124 };
125 /* All the controllers deleted by users via RPC are skipped by hotplug monitor */
126 static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER(
127 			g_skipped_nvme_ctrlrs);
128 
129 static struct spdk_bdev_nvme_opts g_opts = {
130 	.action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE,
131 	.timeout_us = 0,
132 	.timeout_admin_us = 0,
133 	.keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS,
134 	.transport_retry_count = 4,
135 	.arbitration_burst = 0,
136 	.low_priority_weight = 0,
137 	.medium_priority_weight = 0,
138 	.high_priority_weight = 0,
139 	.nvme_adminq_poll_period_us = 10000ULL,
140 	.nvme_ioq_poll_period_us = 0,
141 	.io_queue_requests = 0,
142 	.delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT,
143 	.bdev_retry_count = 3,
144 	.transport_ack_timeout = 0,
145 	.ctrlr_loss_timeout_sec = 0,
146 	.reconnect_delay_sec = 0,
147 	.fast_io_fail_timeout_sec = 0,
148 };
149 
150 #define NVME_HOTPLUG_POLL_PERIOD_MAX			10000000ULL
151 #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT		100000ULL
152 
153 static int g_hot_insert_nvme_controller_index = 0;
154 static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT;
155 static bool g_nvme_hotplug_enabled = false;
156 static struct spdk_thread *g_bdev_nvme_init_thread;
157 static struct spdk_poller *g_hotplug_poller;
158 static struct spdk_poller *g_hotplug_probe_poller;
159 static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx;
160 
161 static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
162 		struct nvme_async_probe_ctx *ctx);
163 static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
164 		struct nvme_async_probe_ctx *ctx);
165 static int bdev_nvme_library_init(void);
166 static void bdev_nvme_library_fini(void);
167 static void bdev_nvme_submit_request(struct spdk_io_channel *ch,
168 				     struct spdk_bdev_io *bdev_io);
169 static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
170 			   void *md, uint64_t lba_count, uint64_t lba,
171 			   uint32_t flags, struct spdk_bdev_ext_io_opts *ext_opts);
172 static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
173 				 void *md, uint64_t lba_count, uint64_t lba);
174 static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
175 			    void *md, uint64_t lba_count, uint64_t lba,
176 			    uint32_t flags, struct spdk_bdev_ext_io_opts *ext_opts);
177 static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
178 				  void *md, uint64_t lba_count,
179 				  uint64_t zslba, uint32_t flags);
180 static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
181 			      void *md, uint64_t lba_count, uint64_t lba,
182 			      uint32_t flags);
183 static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio,
184 		struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov,
185 		int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba,
186 		uint32_t flags);
187 static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id,
188 				   uint32_t num_zones, struct spdk_bdev_zone_info *info);
189 static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id,
190 				     enum spdk_bdev_zone_action action);
191 static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch,
192 				     struct nvme_bdev_io *bio,
193 				     struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes);
194 static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
195 				 void *buf, size_t nbytes);
196 static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
197 				    void *buf, size_t nbytes, void *md_buf, size_t md_len);
198 static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch,
199 			    struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort);
200 static void bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio);
201 static int bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr);
202 static int bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove);
203 static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr);
204 static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr);
205 
206 static int
207 nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2)
208 {
209 	return ns1->id < ns2->id ? -1 : ns1->id > ns2->id;
210 }
211 
212 RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp);
213 
214 struct spdk_nvme_qpair *
215 bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch)
216 {
217 	struct nvme_ctrlr_channel *ctrlr_ch;
218 
219 	assert(ctrlr_io_ch != NULL);
220 
221 	ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch);
222 
223 	return ctrlr_ch->qpair->qpair;
224 }
225 
226 static int
227 bdev_nvme_get_ctx_size(void)
228 {
229 	return sizeof(struct nvme_bdev_io);
230 }
231 
232 static struct spdk_bdev_module nvme_if = {
233 	.name = "nvme",
234 	.async_fini = true,
235 	.module_init = bdev_nvme_library_init,
236 	.module_fini = bdev_nvme_library_fini,
237 	.config_json = bdev_nvme_config_json,
238 	.get_ctx_size = bdev_nvme_get_ctx_size,
239 
240 };
241 SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if)
242 
243 struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs);
244 pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER;
245 bool g_bdev_nvme_module_finish;
246 
247 struct nvme_bdev_ctrlr *
248 nvme_bdev_ctrlr_get_by_name(const char *name)
249 {
250 	struct nvme_bdev_ctrlr *nbdev_ctrlr;
251 
252 	TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
253 		if (strcmp(name, nbdev_ctrlr->name) == 0) {
254 			break;
255 		}
256 	}
257 
258 	return nbdev_ctrlr;
259 }
260 
261 static struct nvme_ctrlr *
262 nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr,
263 			  const struct spdk_nvme_transport_id *trid)
264 {
265 	struct nvme_ctrlr *nvme_ctrlr;
266 
267 	TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
268 		if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0) {
269 			break;
270 		}
271 	}
272 
273 	return nvme_ctrlr;
274 }
275 
276 static struct nvme_bdev *
277 nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid)
278 {
279 	struct nvme_bdev *bdev;
280 
281 	pthread_mutex_lock(&g_bdev_nvme_mutex);
282 	TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) {
283 		if (bdev->nsid == nsid) {
284 			break;
285 		}
286 	}
287 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
288 
289 	return bdev;
290 }
291 
292 struct nvme_ns *
293 nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid)
294 {
295 	struct nvme_ns ns;
296 
297 	assert(nsid > 0);
298 
299 	ns.id = nsid;
300 	return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns);
301 }
302 
303 struct nvme_ns *
304 nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr)
305 {
306 	return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces);
307 }
308 
309 struct nvme_ns *
310 nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns)
311 {
312 	if (ns == NULL) {
313 		return NULL;
314 	}
315 
316 	return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
317 }
318 
319 static struct nvme_ctrlr *
320 nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid)
321 {
322 	struct nvme_bdev_ctrlr	*nbdev_ctrlr;
323 	struct nvme_ctrlr	*nvme_ctrlr = NULL;
324 
325 	pthread_mutex_lock(&g_bdev_nvme_mutex);
326 	TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
327 		nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid);
328 		if (nvme_ctrlr != NULL) {
329 			break;
330 		}
331 	}
332 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
333 
334 	return nvme_ctrlr;
335 }
336 
337 struct nvme_ctrlr *
338 nvme_ctrlr_get_by_name(const char *name)
339 {
340 	struct nvme_bdev_ctrlr *nbdev_ctrlr;
341 	struct nvme_ctrlr *nvme_ctrlr = NULL;
342 
343 	if (name == NULL) {
344 		return NULL;
345 	}
346 
347 	pthread_mutex_lock(&g_bdev_nvme_mutex);
348 	nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
349 	if (nbdev_ctrlr != NULL) {
350 		nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
351 	}
352 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
353 
354 	return nvme_ctrlr;
355 }
356 
357 void
358 nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx)
359 {
360 	struct nvme_bdev_ctrlr *nbdev_ctrlr;
361 
362 	pthread_mutex_lock(&g_bdev_nvme_mutex);
363 	TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
364 		fn(nbdev_ctrlr, ctx);
365 	}
366 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
367 }
368 
369 void
370 nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w)
371 {
372 	const char *trtype_str;
373 	const char *adrfam_str;
374 
375 	trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype);
376 	if (trtype_str) {
377 		spdk_json_write_named_string(w, "trtype", trtype_str);
378 	}
379 
380 	adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam);
381 	if (adrfam_str) {
382 		spdk_json_write_named_string(w, "adrfam", adrfam_str);
383 	}
384 
385 	if (trid->traddr[0] != '\0') {
386 		spdk_json_write_named_string(w, "traddr", trid->traddr);
387 	}
388 
389 	if (trid->trsvcid[0] != '\0') {
390 		spdk_json_write_named_string(w, "trsvcid", trid->trsvcid);
391 	}
392 
393 	if (trid->subnqn[0] != '\0') {
394 		spdk_json_write_named_string(w, "subnqn", trid->subnqn);
395 	}
396 }
397 
398 static void
399 nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr,
400 		       struct nvme_ctrlr *nvme_ctrlr)
401 {
402 	SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name);
403 	pthread_mutex_lock(&g_bdev_nvme_mutex);
404 
405 	TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq);
406 	if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) {
407 		pthread_mutex_unlock(&g_bdev_nvme_mutex);
408 
409 		return;
410 	}
411 	TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq);
412 
413 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
414 
415 	assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs));
416 
417 	free(nbdev_ctrlr->name);
418 	free(nbdev_ctrlr);
419 }
420 
421 static void
422 _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)
423 {
424 	struct nvme_path_id *path_id, *tmp_path;
425 	struct nvme_ns *ns, *tmp_ns;
426 
427 	free(nvme_ctrlr->copied_ana_desc);
428 	spdk_free(nvme_ctrlr->ana_log_page);
429 
430 	if (nvme_ctrlr->opal_dev) {
431 		spdk_opal_dev_destruct(nvme_ctrlr->opal_dev);
432 		nvme_ctrlr->opal_dev = NULL;
433 	}
434 
435 	if (nvme_ctrlr->nbdev_ctrlr) {
436 		nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr);
437 	}
438 
439 	RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) {
440 		RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
441 		free(ns);
442 	}
443 
444 	TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) {
445 		TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link);
446 		free(path_id);
447 	}
448 
449 	pthread_mutex_destroy(&nvme_ctrlr->mutex);
450 
451 	free(nvme_ctrlr);
452 
453 	pthread_mutex_lock(&g_bdev_nvme_mutex);
454 	if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
455 		pthread_mutex_unlock(&g_bdev_nvme_mutex);
456 		spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL);
457 		spdk_bdev_module_fini_done();
458 		return;
459 	}
460 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
461 }
462 
463 static int
464 nvme_detach_poller(void *arg)
465 {
466 	struct nvme_ctrlr *nvme_ctrlr = arg;
467 	int rc;
468 
469 	rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx);
470 	if (rc != -EAGAIN) {
471 		spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
472 		_nvme_ctrlr_delete(nvme_ctrlr);
473 	}
474 
475 	return SPDK_POLLER_BUSY;
476 }
477 
478 static void
479 nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)
480 {
481 	int rc;
482 
483 	spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
484 
485 	/* First, unregister the adminq poller, as the driver will poll adminq if necessary */
486 	spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller);
487 
488 	/* If we got here, the reset/detach poller cannot be active */
489 	assert(nvme_ctrlr->reset_detach_poller == NULL);
490 	nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller,
491 					  nvme_ctrlr, 1000);
492 	if (nvme_ctrlr->reset_detach_poller == NULL) {
493 		SPDK_ERRLOG("Failed to register detach poller\n");
494 		goto error;
495 	}
496 
497 	rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx);
498 	if (rc != 0) {
499 		SPDK_ERRLOG("Failed to detach the NVMe controller\n");
500 		goto error;
501 	}
502 
503 	return;
504 error:
505 	/* We don't have a good way to handle errors here, so just do what we can and delete the
506 	 * controller without detaching the underlying NVMe device.
507 	 */
508 	spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
509 	_nvme_ctrlr_delete(nvme_ctrlr);
510 }
511 
512 static void
513 nvme_ctrlr_unregister_cb(void *io_device)
514 {
515 	struct nvme_ctrlr *nvme_ctrlr = io_device;
516 
517 	nvme_ctrlr_delete(nvme_ctrlr);
518 }
519 
520 static void
521 nvme_ctrlr_unregister(void *ctx)
522 {
523 	struct nvme_ctrlr *nvme_ctrlr = ctx;
524 
525 	spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb);
526 }
527 
528 static bool
529 nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr)
530 {
531 	if (!nvme_ctrlr->destruct) {
532 		return false;
533 	}
534 
535 	if (nvme_ctrlr->ref > 0) {
536 		return false;
537 	}
538 
539 	if (nvme_ctrlr->resetting) {
540 		return false;
541 	}
542 
543 	if (nvme_ctrlr->ana_log_page_updating) {
544 		return false;
545 	}
546 
547 	return true;
548 }
549 
550 static void
551 nvme_ctrlr_release(struct nvme_ctrlr *nvme_ctrlr)
552 {
553 	pthread_mutex_lock(&nvme_ctrlr->mutex);
554 	SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref);
555 
556 	assert(nvme_ctrlr->ref > 0);
557 	nvme_ctrlr->ref--;
558 
559 	if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
560 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
561 		return;
562 	}
563 
564 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
565 
566 	spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr);
567 }
568 
569 static struct nvme_io_path *
570 _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns)
571 {
572 	struct nvme_io_path *io_path;
573 
574 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
575 		if (io_path->nvme_ns == nvme_ns) {
576 			break;
577 		}
578 	}
579 
580 	return io_path;
581 }
582 
583 static int
584 _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns)
585 {
586 	struct nvme_io_path *io_path;
587 	struct spdk_io_channel *ch;
588 	struct nvme_ctrlr_channel *ctrlr_ch;
589 	struct nvme_qpair *nvme_qpair;
590 
591 	io_path = calloc(1, sizeof(*io_path));
592 	if (io_path == NULL) {
593 		SPDK_ERRLOG("Failed to alloc io_path.\n");
594 		return -ENOMEM;
595 	}
596 
597 	io_path->nvme_ns = nvme_ns;
598 
599 	ch = spdk_get_io_channel(nvme_ns->ctrlr);
600 	if (ch == NULL) {
601 		free(io_path);
602 		SPDK_ERRLOG("Failed to alloc io_channel.\n");
603 		return -ENOMEM;
604 	}
605 
606 	ctrlr_ch = spdk_io_channel_get_ctx(ch);
607 
608 	nvme_qpair = ctrlr_ch->qpair;
609 	assert(nvme_qpair != NULL);
610 
611 	io_path->qpair = nvme_qpair;
612 	TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq);
613 
614 	io_path->nbdev_ch = nbdev_ch;
615 	STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq);
616 
617 	nbdev_ch->current_io_path = NULL;
618 
619 	return 0;
620 }
621 
622 static void
623 _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path)
624 {
625 	struct spdk_io_channel *ch;
626 	struct nvme_qpair *nvme_qpair;
627 	struct nvme_ctrlr_channel *ctrlr_ch;
628 
629 	nbdev_ch->current_io_path = NULL;
630 
631 	STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq);
632 
633 	nvme_qpair = io_path->qpair;
634 	assert(nvme_qpair != NULL);
635 
636 	TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq);
637 
638 	ctrlr_ch = nvme_qpair->ctrlr_ch;
639 	assert(ctrlr_ch != NULL);
640 
641 	ch = spdk_io_channel_from_ctx(ctrlr_ch);
642 	spdk_put_io_channel(ch);
643 
644 	free(io_path);
645 }
646 
647 static void
648 _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch)
649 {
650 	struct nvme_io_path *io_path, *tmp_io_path;
651 
652 	STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) {
653 		_bdev_nvme_delete_io_path(nbdev_ch, io_path);
654 	}
655 }
656 
657 static int
658 bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf)
659 {
660 	struct nvme_bdev_channel *nbdev_ch = ctx_buf;
661 	struct nvme_bdev *nbdev = io_device;
662 	struct nvme_ns *nvme_ns;
663 	int rc;
664 
665 	STAILQ_INIT(&nbdev_ch->io_path_list);
666 	TAILQ_INIT(&nbdev_ch->retry_io_list);
667 
668 	pthread_mutex_lock(&nbdev->mutex);
669 	TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
670 		rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns);
671 		if (rc != 0) {
672 			pthread_mutex_unlock(&nbdev->mutex);
673 
674 			_bdev_nvme_delete_io_paths(nbdev_ch);
675 			return rc;
676 		}
677 	}
678 	pthread_mutex_unlock(&nbdev->mutex);
679 
680 	return 0;
681 }
682 
683 static void
684 bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch)
685 {
686 	struct spdk_bdev_io *bdev_io, *tmp_io;
687 
688 	TAILQ_FOREACH_SAFE(bdev_io, &nbdev_ch->retry_io_list, module_link, tmp_io) {
689 		TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io, module_link);
690 		spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_ABORTED);
691 	}
692 
693 	spdk_poller_unregister(&nbdev_ch->retry_io_poller);
694 }
695 
696 static void
697 bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf)
698 {
699 	struct nvme_bdev_channel *nbdev_ch = ctx_buf;
700 
701 	bdev_nvme_abort_retry_ios(nbdev_ch);
702 	_bdev_nvme_delete_io_paths(nbdev_ch);
703 }
704 
705 static inline bool
706 bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type)
707 {
708 	switch (io_type) {
709 	case SPDK_BDEV_IO_TYPE_RESET:
710 	case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
711 	case SPDK_BDEV_IO_TYPE_ABORT:
712 		return true;
713 	default:
714 		break;
715 	}
716 
717 	return false;
718 }
719 
720 static inline bool
721 nvme_ns_is_accessible(struct nvme_ns *nvme_ns)
722 {
723 	if (spdk_unlikely(nvme_ns->ana_state_updating)) {
724 		return false;
725 	}
726 
727 	switch (nvme_ns->ana_state) {
728 	case SPDK_NVME_ANA_OPTIMIZED_STATE:
729 	case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
730 		return true;
731 	default:
732 		break;
733 	}
734 
735 	return false;
736 }
737 
738 static inline bool
739 nvme_io_path_is_connected(struct nvme_io_path *io_path)
740 {
741 	if (spdk_unlikely(io_path->qpair->qpair == NULL)) {
742 		return false;
743 	}
744 
745 	if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(io_path->qpair->qpair) !=
746 			  SPDK_NVME_QPAIR_FAILURE_NONE)) {
747 		return false;
748 	}
749 
750 	if (spdk_unlikely(io_path->qpair->ctrlr_ch->reset_iter != NULL)) {
751 		return false;
752 	}
753 
754 	if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(io_path->qpair->ctrlr->ctrlr) !=
755 	    SPDK_NVME_QPAIR_FAILURE_NONE) {
756 		return false;
757 	}
758 
759 	return true;
760 }
761 
762 static inline bool
763 nvme_io_path_is_available(struct nvme_io_path *io_path)
764 {
765 	if (spdk_unlikely(!nvme_io_path_is_connected(io_path))) {
766 		return false;
767 	}
768 
769 	if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) {
770 		return false;
771 	}
772 
773 	return true;
774 }
775 
776 static inline bool
777 nvme_io_path_is_failed(struct nvme_io_path *io_path)
778 {
779 	struct nvme_ctrlr *nvme_ctrlr;
780 
781 	nvme_ctrlr = io_path->qpair->ctrlr;
782 
783 	if (nvme_ctrlr->destruct) {
784 		return true;
785 	}
786 
787 	if (nvme_ctrlr->fast_io_fail_timedout) {
788 		return true;
789 	}
790 
791 	if (nvme_ctrlr->resetting) {
792 		if (nvme_ctrlr->opts.reconnect_delay_sec != 0) {
793 			return false;
794 		} else {
795 			return true;
796 		}
797 	}
798 
799 	if (nvme_ctrlr->reconnect_is_delayed) {
800 		return false;
801 	}
802 
803 	if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
804 		return true;
805 	} else {
806 		return false;
807 	}
808 }
809 
810 static bool
811 nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr)
812 {
813 	if (nvme_ctrlr->destruct) {
814 		return false;
815 	}
816 
817 	if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
818 		return false;
819 	}
820 
821 	if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) {
822 		return false;
823 	}
824 
825 	return true;
826 }
827 
828 static inline struct nvme_io_path *
829 bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch)
830 {
831 	struct nvme_io_path *io_path, *non_optimized = NULL;
832 
833 	if (spdk_likely(nbdev_ch->current_io_path != NULL)) {
834 		return nbdev_ch->current_io_path;
835 	}
836 
837 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
838 		if (spdk_unlikely(!nvme_io_path_is_connected(io_path))) {
839 			/* The device is currently resetting. */
840 			continue;
841 		}
842 
843 		if (spdk_unlikely(io_path->nvme_ns->ana_state_updating)) {
844 			continue;
845 		}
846 
847 		switch (io_path->nvme_ns->ana_state) {
848 		case SPDK_NVME_ANA_OPTIMIZED_STATE:
849 			nbdev_ch->current_io_path = io_path;
850 			return io_path;
851 		case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
852 			if (non_optimized == NULL) {
853 				non_optimized = io_path;
854 			}
855 			break;
856 		default:
857 			break;
858 		}
859 	}
860 
861 	return non_optimized;
862 }
863 
864 /* Return true if there is any io_path whose qpair is active or ctrlr is not failed,
865  * or false otherwise.
866  *
867  * If any io_path has an active qpair but find_io_path() returned NULL, its namespace
868  * is likely to be non-accessible now but may become accessible.
869  *
870  * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr
871  * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed
872  * when starting to reset it but it is set to failed when the reset failed. Hence, if
873  * a ctrlr is unfailed, it is likely that it works fine or is resetting.
874  */
875 static bool
876 any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch)
877 {
878 	struct nvme_io_path *io_path;
879 
880 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
881 		if (io_path->nvme_ns->ana_transition_timedout) {
882 			continue;
883 		}
884 
885 		if (nvme_io_path_is_connected(io_path) ||
886 		    !nvme_io_path_is_failed(io_path)) {
887 			return true;
888 		}
889 	}
890 
891 	return false;
892 }
893 
894 static bool
895 any_ctrlr_may_become_available(struct nvme_bdev_channel *nbdev_ch)
896 {
897 	struct nvme_io_path *io_path;
898 
899 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
900 		if (!nvme_io_path_is_failed(io_path)) {
901 			return true;
902 		}
903 	}
904 
905 	return false;
906 }
907 
908 static int
909 bdev_nvme_retry_ios(void *arg)
910 {
911 	struct nvme_bdev_channel *nbdev_ch = arg;
912 	struct spdk_io_channel *ch = spdk_io_channel_from_ctx(nbdev_ch);
913 	struct spdk_bdev_io *bdev_io, *tmp_bdev_io;
914 	struct nvme_bdev_io *bio;
915 	uint64_t now, delay_us;
916 
917 	now = spdk_get_ticks();
918 
919 	TAILQ_FOREACH_SAFE(bdev_io, &nbdev_ch->retry_io_list, module_link, tmp_bdev_io) {
920 		bio = (struct nvme_bdev_io *)bdev_io->driver_ctx;
921 		if (bio->retry_ticks > now) {
922 			break;
923 		}
924 
925 		TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io, module_link);
926 
927 		bdev_nvme_submit_request(ch, bdev_io);
928 	}
929 
930 	spdk_poller_unregister(&nbdev_ch->retry_io_poller);
931 
932 	bdev_io = TAILQ_FIRST(&nbdev_ch->retry_io_list);
933 	if (bdev_io != NULL) {
934 		bio = (struct nvme_bdev_io *)bdev_io->driver_ctx;
935 
936 		delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz();
937 
938 		nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch,
939 					    delay_us);
940 	}
941 
942 	return SPDK_POLLER_BUSY;
943 }
944 
945 static void
946 bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch,
947 			 struct nvme_bdev_io *bio, uint64_t delay_ms)
948 {
949 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
950 	struct spdk_bdev_io *tmp_bdev_io;
951 	struct nvme_bdev_io *tmp_bio;
952 
953 	bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL;
954 
955 	TAILQ_FOREACH_REVERSE(tmp_bdev_io, &nbdev_ch->retry_io_list, retry_io_head, module_link) {
956 		tmp_bio = (struct nvme_bdev_io *)tmp_bdev_io->driver_ctx;
957 
958 		if (tmp_bio->retry_ticks <= bio->retry_ticks) {
959 			TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bdev_io, bdev_io,
960 					   module_link);
961 			return;
962 		}
963 	}
964 
965 	/* No earlier I/Os were found. This I/O must be the new head. */
966 	TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bdev_io, module_link);
967 
968 	spdk_poller_unregister(&nbdev_ch->retry_io_poller);
969 
970 	nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch,
971 				    delay_ms * 1000ULL);
972 }
973 
974 static inline void
975 bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio,
976 				  const struct spdk_nvme_cpl *cpl)
977 {
978 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
979 	struct nvme_bdev_channel *nbdev_ch;
980 	struct nvme_ctrlr *nvme_ctrlr;
981 	const struct spdk_nvme_ctrlr_data *cdata;
982 	uint64_t delay_ms;
983 
984 	assert(!bdev_nvme_io_type_is_admin(bdev_io->type));
985 
986 	if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) {
987 		goto complete;
988 	}
989 
990 	if (cpl->status.dnr != 0 || (g_opts.bdev_retry_count != -1 &&
991 				     bio->retry_count >= g_opts.bdev_retry_count)) {
992 		goto complete;
993 	}
994 
995 	nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
996 
997 	assert(bio->io_path != NULL);
998 	nvme_ctrlr = bio->io_path->qpair->ctrlr;
999 
1000 	if (spdk_nvme_cpl_is_path_error(cpl) ||
1001 	    spdk_nvme_cpl_is_aborted_sq_deletion(cpl) ||
1002 	    !nvme_io_path_is_available(bio->io_path) ||
1003 	    !nvme_ctrlr_is_available(nvme_ctrlr)) {
1004 		nbdev_ch->current_io_path = NULL;
1005 		if (spdk_nvme_cpl_is_ana_error(cpl)) {
1006 			if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) {
1007 				bio->io_path->nvme_ns->ana_state_updating = true;
1008 			}
1009 		}
1010 		delay_ms = 0;
1011 	} else if (spdk_nvme_cpl_is_aborted_by_request(cpl)) {
1012 		goto complete;
1013 	} else {
1014 		bio->retry_count++;
1015 
1016 		cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
1017 
1018 		if (cpl->status.crd != 0) {
1019 			delay_ms = cdata->crdt[cpl->status.crd] * 100;
1020 		} else {
1021 			delay_ms = 0;
1022 		}
1023 	}
1024 
1025 	if (any_io_path_may_become_available(nbdev_ch)) {
1026 		bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms);
1027 		return;
1028 	}
1029 
1030 complete:
1031 	bio->retry_count = 0;
1032 	spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc);
1033 }
1034 
1035 static inline void
1036 bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc)
1037 {
1038 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1039 	struct nvme_bdev_channel *nbdev_ch;
1040 	enum spdk_bdev_io_status io_status;
1041 
1042 	switch (rc) {
1043 	case 0:
1044 		io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1045 		break;
1046 	case -ENOMEM:
1047 		io_status = SPDK_BDEV_IO_STATUS_NOMEM;
1048 		break;
1049 	case -ENXIO:
1050 		nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
1051 
1052 		nbdev_ch->current_io_path = NULL;
1053 
1054 		if (any_io_path_may_become_available(nbdev_ch)) {
1055 			bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL);
1056 			return;
1057 		}
1058 
1059 	/* fallthrough */
1060 	default:
1061 		io_status = SPDK_BDEV_IO_STATUS_FAILED;
1062 		break;
1063 	}
1064 
1065 	bio->retry_count = 0;
1066 	spdk_bdev_io_complete(bdev_io, io_status);
1067 }
1068 
1069 static inline void
1070 bdev_nvme_admin_passthru_complete(struct nvme_bdev_io *bio, int rc)
1071 {
1072 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1073 	struct nvme_bdev_channel *nbdev_ch;
1074 	enum spdk_bdev_io_status io_status;
1075 
1076 	switch (rc) {
1077 	case 0:
1078 		io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1079 		break;
1080 	case -ENOMEM:
1081 		io_status = SPDK_BDEV_IO_STATUS_NOMEM;
1082 		break;
1083 	case -ENXIO:
1084 		nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
1085 
1086 		if (any_ctrlr_may_become_available(nbdev_ch)) {
1087 			bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL);
1088 			return;
1089 		}
1090 
1091 	/* fallthrough */
1092 	default:
1093 		io_status = SPDK_BDEV_IO_STATUS_FAILED;
1094 		break;
1095 	}
1096 
1097 	bio->retry_count = 0;
1098 	spdk_bdev_io_complete(bdev_io, io_status);
1099 }
1100 
1101 static void
1102 _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair)
1103 {
1104 	struct nvme_io_path *io_path;
1105 
1106 	TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) {
1107 		io_path->nbdev_ch->current_io_path = NULL;
1108 	}
1109 }
1110 
1111 static void
1112 bdev_nvme_clear_io_path_cache(struct spdk_io_channel_iter *i)
1113 {
1114 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
1115 	struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch);
1116 
1117 	assert(ctrlr_ch->qpair != NULL);
1118 
1119 	_bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair);
1120 
1121 	spdk_for_each_channel_continue(i, 0);
1122 }
1123 
1124 static void
1125 bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr,
1126 			       spdk_channel_for_each_cpl cpl)
1127 {
1128 	spdk_for_each_channel(nvme_ctrlr,
1129 			      bdev_nvme_clear_io_path_cache,
1130 			      NULL,
1131 			      cpl);
1132 }
1133 
1134 static struct nvme_qpair *
1135 nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
1136 {
1137 	struct nvme_qpair *nvme_qpair;
1138 
1139 	TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) {
1140 		if (nvme_qpair->qpair == qpair) {
1141 			break;
1142 		}
1143 	}
1144 
1145 	return nvme_qpair;
1146 }
1147 
1148 static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair);
1149 
1150 static void
1151 bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx)
1152 {
1153 	struct nvme_poll_group *group = poll_group_ctx;
1154 	struct nvme_qpair *nvme_qpair;
1155 	struct nvme_ctrlr_channel *ctrlr_ch;
1156 
1157 	nvme_qpair = nvme_poll_group_get_qpair(group, qpair);
1158 	if (nvme_qpair == NULL) {
1159 		return;
1160 	}
1161 
1162 	if (nvme_qpair->qpair != NULL) {
1163 		spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair);
1164 		nvme_qpair->qpair = NULL;
1165 	}
1166 
1167 	_bdev_nvme_clear_io_path_cache(nvme_qpair);
1168 
1169 	ctrlr_ch = nvme_qpair->ctrlr_ch;
1170 
1171 	if (ctrlr_ch != NULL) {
1172 		if (ctrlr_ch->reset_iter != NULL) {
1173 			/* If we are already in a full reset sequence, we do not have
1174 			 * to restart it. Just move to the next ctrlr_channel.
1175 			 */
1176 			SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed in a reset ctrlr sequence.\n",
1177 				      qpair);
1178 			spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0);
1179 			ctrlr_ch->reset_iter = NULL;
1180 		} else {
1181 			/* qpair was disconnected unexpectedly. Reset controller for recovery. */
1182 			SPDK_NOTICELOG("qpair %p was disconnected and freed. reset controller.\n", qpair);
1183 			bdev_nvme_failover(nvme_qpair->ctrlr, false);
1184 		}
1185 	} else {
1186 		/* In this case, ctrlr_channel is already deleted. */
1187 		SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed. delete nvme_qpair.\n", qpair);
1188 		nvme_qpair_delete(nvme_qpair);
1189 	}
1190 }
1191 
1192 static void
1193 bdev_nvme_check_io_qpairs(struct nvme_poll_group *group)
1194 {
1195 	struct nvme_qpair *nvme_qpair;
1196 
1197 	TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) {
1198 		if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) {
1199 			continue;
1200 		}
1201 
1202 		if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) !=
1203 		    SPDK_NVME_QPAIR_FAILURE_NONE) {
1204 			_bdev_nvme_clear_io_path_cache(nvme_qpair);
1205 		}
1206 	}
1207 }
1208 
1209 static int
1210 bdev_nvme_poll(void *arg)
1211 {
1212 	struct nvme_poll_group *group = arg;
1213 	int64_t num_completions;
1214 
1215 	if (group->collect_spin_stat && group->start_ticks == 0) {
1216 		group->start_ticks = spdk_get_ticks();
1217 	}
1218 
1219 	num_completions = spdk_nvme_poll_group_process_completions(group->group, 0,
1220 			  bdev_nvme_disconnected_qpair_cb);
1221 	if (group->collect_spin_stat) {
1222 		if (num_completions > 0) {
1223 			if (group->end_ticks != 0) {
1224 				group->spin_ticks += (group->end_ticks - group->start_ticks);
1225 				group->end_ticks = 0;
1226 			}
1227 			group->start_ticks = 0;
1228 		} else {
1229 			group->end_ticks = spdk_get_ticks();
1230 		}
1231 	}
1232 
1233 	if (spdk_unlikely(num_completions < 0)) {
1234 		bdev_nvme_check_io_qpairs(group);
1235 	}
1236 
1237 	return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
1238 }
1239 
1240 static int
1241 bdev_nvme_poll_adminq(void *arg)
1242 {
1243 	int32_t rc;
1244 	struct nvme_ctrlr *nvme_ctrlr = arg;
1245 	nvme_ctrlr_disconnected_cb disconnected_cb;
1246 
1247 	assert(nvme_ctrlr != NULL);
1248 
1249 	rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr);
1250 	if (rc < 0) {
1251 		disconnected_cb = nvme_ctrlr->disconnected_cb;
1252 		nvme_ctrlr->disconnected_cb = NULL;
1253 
1254 		if (rc == -ENXIO && disconnected_cb != NULL) {
1255 			disconnected_cb(nvme_ctrlr);
1256 		} else {
1257 			bdev_nvme_failover(nvme_ctrlr, false);
1258 		}
1259 	} else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) !=
1260 		   SPDK_NVME_QPAIR_FAILURE_NONE) {
1261 		bdev_nvme_clear_io_path_caches(nvme_ctrlr, NULL);
1262 	}
1263 
1264 	return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY;
1265 }
1266 
1267 static void
1268 _bdev_nvme_unregister_dev_cb(void *io_device)
1269 {
1270 	struct nvme_bdev *nvme_disk = io_device;
1271 
1272 	free(nvme_disk->disk.name);
1273 	free(nvme_disk);
1274 }
1275 
1276 static int
1277 bdev_nvme_destruct(void *ctx)
1278 {
1279 	struct nvme_bdev *nvme_disk = ctx;
1280 	struct nvme_ns *nvme_ns, *tmp_nvme_ns;
1281 
1282 	SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid);
1283 
1284 	TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) {
1285 		pthread_mutex_lock(&nvme_ns->ctrlr->mutex);
1286 
1287 		nvme_ns->bdev = NULL;
1288 
1289 		assert(nvme_ns->id > 0);
1290 
1291 		if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) {
1292 			pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
1293 
1294 			nvme_ctrlr_release(nvme_ns->ctrlr);
1295 			free(nvme_ns);
1296 		} else {
1297 			pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
1298 		}
1299 	}
1300 
1301 	pthread_mutex_lock(&g_bdev_nvme_mutex);
1302 	TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq);
1303 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
1304 
1305 	spdk_io_device_unregister(nvme_disk, _bdev_nvme_unregister_dev_cb);
1306 
1307 	return 0;
1308 }
1309 
1310 static int
1311 bdev_nvme_flush(struct nvme_bdev_io *bio, uint64_t offset, uint64_t nbytes)
1312 {
1313 	bdev_nvme_io_complete(bio, 0);
1314 
1315 	return 0;
1316 }
1317 
1318 static int
1319 bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair)
1320 {
1321 	struct nvme_ctrlr *nvme_ctrlr;
1322 	struct spdk_nvme_io_qpair_opts opts;
1323 	struct spdk_nvme_qpair *qpair;
1324 	int rc;
1325 
1326 	nvme_ctrlr = nvme_qpair->ctrlr;
1327 
1328 	spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts));
1329 	opts.delay_cmd_submit = g_opts.delay_cmd_submit;
1330 	opts.create_only = true;
1331 	opts.async_mode = true;
1332 	opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests);
1333 	g_opts.io_queue_requests = opts.io_queue_requests;
1334 
1335 	qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts));
1336 	if (qpair == NULL) {
1337 		return -1;
1338 	}
1339 
1340 	SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name,
1341 			   spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread));
1342 
1343 	assert(nvme_qpair->group != NULL);
1344 
1345 	rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair);
1346 	if (rc != 0) {
1347 		SPDK_ERRLOG("Unable to begin polling on NVMe Channel.\n");
1348 		goto err;
1349 	}
1350 
1351 	rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair);
1352 	if (rc != 0) {
1353 		SPDK_ERRLOG("Unable to connect I/O qpair.\n");
1354 		goto err;
1355 	}
1356 
1357 	nvme_qpair->qpair = qpair;
1358 
1359 	_bdev_nvme_clear_io_path_cache(nvme_qpair);
1360 
1361 	return 0;
1362 
1363 err:
1364 	spdk_nvme_ctrlr_free_io_qpair(qpair);
1365 
1366 	return rc;
1367 }
1368 
1369 static void
1370 bdev_nvme_complete_pending_resets(struct spdk_io_channel_iter *i)
1371 {
1372 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
1373 	struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch);
1374 	enum spdk_bdev_io_status status = SPDK_BDEV_IO_STATUS_SUCCESS;
1375 	struct spdk_bdev_io *bdev_io;
1376 
1377 	if (spdk_io_channel_iter_get_ctx(i) != NULL) {
1378 		status = SPDK_BDEV_IO_STATUS_FAILED;
1379 	}
1380 
1381 	while (!TAILQ_EMPTY(&ctrlr_ch->pending_resets)) {
1382 		bdev_io = TAILQ_FIRST(&ctrlr_ch->pending_resets);
1383 		TAILQ_REMOVE(&ctrlr_ch->pending_resets, bdev_io, module_link);
1384 		spdk_bdev_io_complete(bdev_io, status);
1385 	}
1386 
1387 	spdk_for_each_channel_continue(i, 0);
1388 }
1389 
1390 static void
1391 bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove)
1392 {
1393 	struct nvme_path_id *path_id, *next_path;
1394 	int rc __attribute__((unused));
1395 
1396 	path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
1397 	assert(path_id);
1398 	assert(path_id == nvme_ctrlr->active_path_id);
1399 	next_path = TAILQ_NEXT(path_id, link);
1400 
1401 	path_id->is_failed = true;
1402 
1403 	if (next_path) {
1404 		assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE);
1405 
1406 		SPDK_NOTICELOG("Start failover from %s:%s to %s:%s\n", path_id->trid.traddr,
1407 			       path_id->trid.trsvcid,	next_path->trid.traddr, next_path->trid.trsvcid);
1408 
1409 		spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr);
1410 		nvme_ctrlr->active_path_id = next_path;
1411 		rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid);
1412 		assert(rc == 0);
1413 		TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link);
1414 		if (!remove) {
1415 			/** Shuffle the old trid to the end of the list and use the new one.
1416 			 * Allows for round robin through multiple connections.
1417 			 */
1418 			TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link);
1419 		} else {
1420 			free(path_id);
1421 		}
1422 	}
1423 }
1424 
1425 static bool
1426 bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr)
1427 {
1428 	int32_t elapsed;
1429 
1430 	if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 ||
1431 	    nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) {
1432 		return false;
1433 	}
1434 
1435 	elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();
1436 	if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) {
1437 		return true;
1438 	} else {
1439 		return false;
1440 	}
1441 }
1442 
1443 static bool
1444 bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr)
1445 {
1446 	uint32_t elapsed;
1447 
1448 	if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) {
1449 		return false;
1450 	}
1451 
1452 	elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();
1453 	if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) {
1454 		return true;
1455 	} else {
1456 		return false;
1457 	}
1458 }
1459 
1460 enum bdev_nvme_op_after_reset {
1461 	OP_NONE,
1462 	OP_COMPLETE_PENDING_DESTRUCT,
1463 	OP_DESTRUCT,
1464 	OP_DELAYED_RECONNECT,
1465 };
1466 
1467 typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset;
1468 
1469 static _bdev_nvme_op_after_reset
1470 bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success)
1471 {
1472 	if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
1473 		/* Complete pending destruct after reset completes. */
1474 		return OP_COMPLETE_PENDING_DESTRUCT;
1475 	} else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) {
1476 		nvme_ctrlr->reset_start_tsc = 0;
1477 		return OP_NONE;
1478 	} else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
1479 		return OP_DESTRUCT;
1480 	} else {
1481 		if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) {
1482 			nvme_ctrlr->fast_io_fail_timedout = true;
1483 		}
1484 		bdev_nvme_failover_trid(nvme_ctrlr, false);
1485 		return OP_DELAYED_RECONNECT;
1486 	}
1487 }
1488 
1489 static int _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, bool hotplug);
1490 static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
1491 
1492 static int
1493 bdev_nvme_reconnect_delay_timer_expired(void *ctx)
1494 {
1495 	struct nvme_ctrlr *nvme_ctrlr = ctx;
1496 
1497 	pthread_mutex_lock(&nvme_ctrlr->mutex);
1498 
1499 	spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
1500 
1501 	assert(nvme_ctrlr->reconnect_is_delayed == true);
1502 	nvme_ctrlr->reconnect_is_delayed = false;
1503 
1504 	if (nvme_ctrlr->destruct) {
1505 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
1506 		return SPDK_POLLER_BUSY;
1507 	}
1508 
1509 	assert(nvme_ctrlr->resetting == false);
1510 	nvme_ctrlr->resetting = true;
1511 
1512 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
1513 
1514 	spdk_poller_resume(nvme_ctrlr->adminq_timer_poller);
1515 
1516 	bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
1517 	return SPDK_POLLER_BUSY;
1518 }
1519 
1520 static void
1521 bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr)
1522 {
1523 	spdk_poller_pause(nvme_ctrlr->adminq_timer_poller);
1524 
1525 	assert(nvme_ctrlr->reconnect_is_delayed == false);
1526 	nvme_ctrlr->reconnect_is_delayed = true;
1527 
1528 	assert(nvme_ctrlr->reconnect_delay_timer == NULL);
1529 	nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired,
1530 					    nvme_ctrlr,
1531 					    nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC);
1532 }
1533 
1534 static void
1535 _bdev_nvme_reset_complete(struct spdk_io_channel_iter *i, int status)
1536 {
1537 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
1538 	bool success = spdk_io_channel_iter_get_ctx(i) == NULL;
1539 	struct nvme_path_id *path_id;
1540 	bdev_nvme_reset_cb reset_cb_fn = nvme_ctrlr->reset_cb_fn;
1541 	void *reset_cb_arg = nvme_ctrlr->reset_cb_arg;
1542 	enum bdev_nvme_op_after_reset op_after_reset;
1543 
1544 	assert(nvme_ctrlr->thread == spdk_get_thread());
1545 
1546 	nvme_ctrlr->reset_cb_fn = NULL;
1547 	nvme_ctrlr->reset_cb_arg = NULL;
1548 
1549 	if (!success) {
1550 		SPDK_ERRLOG("Resetting controller failed.\n");
1551 	} else {
1552 		SPDK_NOTICELOG("Resetting controller successful.\n");
1553 	}
1554 
1555 	pthread_mutex_lock(&nvme_ctrlr->mutex);
1556 	nvme_ctrlr->resetting = false;
1557 
1558 	path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
1559 	assert(path_id != NULL);
1560 	assert(path_id == nvme_ctrlr->active_path_id);
1561 
1562 	path_id->is_failed = !success;
1563 
1564 	op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success);
1565 
1566 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
1567 
1568 	if (reset_cb_fn) {
1569 		reset_cb_fn(reset_cb_arg, success);
1570 	}
1571 
1572 	switch (op_after_reset) {
1573 	case OP_COMPLETE_PENDING_DESTRUCT:
1574 		nvme_ctrlr_unregister(nvme_ctrlr);
1575 		break;
1576 	case OP_DESTRUCT:
1577 		_bdev_nvme_delete(nvme_ctrlr, false);
1578 		break;
1579 	case OP_DELAYED_RECONNECT:
1580 		/* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq.
1581 		 * Set callback here to start reconnect delay timer after ctrlr is really disconnected.
1582 		 */
1583 		assert(nvme_ctrlr->disconnected_cb == NULL);
1584 		nvme_ctrlr->disconnected_cb = bdev_nvme_start_reconnect_delay_timer;
1585 
1586 		spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr);
1587 		break;
1588 	default:
1589 		break;
1590 	}
1591 }
1592 
1593 static void
1594 bdev_nvme_reset_complete(struct nvme_ctrlr *nvme_ctrlr, bool success)
1595 {
1596 	/* Make sure we clear any pending resets before returning. */
1597 	spdk_for_each_channel(nvme_ctrlr,
1598 			      bdev_nvme_complete_pending_resets,
1599 			      success ? NULL : (void *)0x1,
1600 			      _bdev_nvme_reset_complete);
1601 }
1602 
1603 static void
1604 bdev_nvme_reset_create_qpairs_failed(struct spdk_io_channel_iter *i, int status)
1605 {
1606 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
1607 
1608 	bdev_nvme_reset_complete(nvme_ctrlr, false);
1609 }
1610 
1611 static void
1612 bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i)
1613 {
1614 	struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
1615 	struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch);
1616 	struct nvme_qpair *nvme_qpair;
1617 
1618 	nvme_qpair = ctrlr_ch->qpair;
1619 	assert(nvme_qpair != NULL);
1620 
1621 	_bdev_nvme_clear_io_path_cache(nvme_qpair);
1622 
1623 	if (nvme_qpair->qpair != NULL) {
1624 		spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair);
1625 
1626 		/* The current full reset sequence will move to the next
1627 		 * ctrlr_channel after the qpair is actually disconnected.
1628 		 */
1629 		assert(ctrlr_ch->reset_iter == NULL);
1630 		ctrlr_ch->reset_iter = i;
1631 	} else {
1632 		spdk_for_each_channel_continue(i, 0);
1633 	}
1634 }
1635 
1636 static void
1637 bdev_nvme_reset_create_qpairs_done(struct spdk_io_channel_iter *i, int status)
1638 {
1639 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
1640 
1641 	if (status == 0) {
1642 		bdev_nvme_reset_complete(nvme_ctrlr, true);
1643 	} else {
1644 		/* Delete the added qpairs and quiesce ctrlr to make the states clean. */
1645 		spdk_for_each_channel(nvme_ctrlr,
1646 				      bdev_nvme_reset_destroy_qpair,
1647 				      NULL,
1648 				      bdev_nvme_reset_create_qpairs_failed);
1649 	}
1650 }
1651 
1652 static void
1653 bdev_nvme_reset_create_qpair(struct spdk_io_channel_iter *i)
1654 {
1655 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
1656 	struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch);
1657 	int rc;
1658 
1659 	rc = bdev_nvme_create_qpair(ctrlr_ch->qpair);
1660 
1661 	spdk_for_each_channel_continue(i, rc);
1662 }
1663 
1664 static int
1665 bdev_nvme_reconnect_ctrlr_poll(void *arg)
1666 {
1667 	struct nvme_ctrlr *nvme_ctrlr = arg;
1668 	int rc = -ETIMEDOUT;
1669 
1670 	if (!bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
1671 		rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr);
1672 		if (rc == -EAGAIN) {
1673 			return SPDK_POLLER_BUSY;
1674 		}
1675 	}
1676 
1677 	spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
1678 	if (rc == 0) {
1679 		/* Recreate all of the I/O queue pairs */
1680 		spdk_for_each_channel(nvme_ctrlr,
1681 				      bdev_nvme_reset_create_qpair,
1682 				      NULL,
1683 				      bdev_nvme_reset_create_qpairs_done);
1684 	} else {
1685 		bdev_nvme_reset_complete(nvme_ctrlr, false);
1686 	}
1687 	return SPDK_POLLER_BUSY;
1688 }
1689 
1690 static void
1691 bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
1692 {
1693 	spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr);
1694 
1695 	assert(nvme_ctrlr->reset_detach_poller == NULL);
1696 	nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll,
1697 					  nvme_ctrlr, 0);
1698 }
1699 
1700 static void
1701 bdev_nvme_reset_ctrlr(struct spdk_io_channel_iter *i, int status)
1702 {
1703 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
1704 	int rc __attribute__((unused));
1705 
1706 	assert(status == 0);
1707 
1708 	/* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq.
1709 	 * Set callback here to reconnect after ctrlr is really disconnected.
1710 	 */
1711 	assert(nvme_ctrlr->disconnected_cb == NULL);
1712 	nvme_ctrlr->disconnected_cb = bdev_nvme_reconnect_ctrlr;
1713 
1714 	/* Disconnect fails if ctrlr is already resetting or removed. Both cases are
1715 	 * not possible. Reset is controlled and the callback to hot remove is called
1716 	 * when ctrlr is hot removed.
1717 	 */
1718 	rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr);
1719 	assert(rc == 0);
1720 }
1721 
1722 static void
1723 _bdev_nvme_reset(void *ctx)
1724 {
1725 	struct nvme_ctrlr *nvme_ctrlr = ctx;
1726 
1727 	assert(nvme_ctrlr->resetting == true);
1728 	assert(nvme_ctrlr->thread == spdk_get_thread());
1729 
1730 	spdk_nvme_ctrlr_prepare_for_reset(nvme_ctrlr->ctrlr);
1731 
1732 	/* First, delete all NVMe I/O queue pairs. */
1733 	spdk_for_each_channel(nvme_ctrlr,
1734 			      bdev_nvme_reset_destroy_qpair,
1735 			      NULL,
1736 			      bdev_nvme_reset_ctrlr);
1737 }
1738 
1739 static int
1740 bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr)
1741 {
1742 	pthread_mutex_lock(&nvme_ctrlr->mutex);
1743 	if (nvme_ctrlr->destruct) {
1744 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
1745 		return -ENXIO;
1746 	}
1747 
1748 	if (nvme_ctrlr->resetting) {
1749 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
1750 		SPDK_NOTICELOG("Unable to perform reset, already in progress.\n");
1751 		return -EBUSY;
1752 	}
1753 
1754 	if (nvme_ctrlr->reconnect_is_delayed) {
1755 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
1756 		SPDK_NOTICELOG("Reconnect is already scheduled.\n");
1757 		return -EBUSY;
1758 	}
1759 
1760 	nvme_ctrlr->resetting = true;
1761 
1762 	assert(nvme_ctrlr->reset_start_tsc == 0);
1763 	nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
1764 
1765 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
1766 
1767 	spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr);
1768 	return 0;
1769 }
1770 
1771 int
1772 bdev_nvme_reset_rpc(struct nvme_ctrlr *nvme_ctrlr, bdev_nvme_reset_cb cb_fn, void *cb_arg)
1773 {
1774 	int rc;
1775 
1776 	rc = bdev_nvme_reset(nvme_ctrlr);
1777 	if (rc == 0) {
1778 		nvme_ctrlr->reset_cb_fn = cb_fn;
1779 		nvme_ctrlr->reset_cb_arg = cb_arg;
1780 	}
1781 	return rc;
1782 }
1783 
1784 static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio);
1785 
1786 static void
1787 bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio)
1788 {
1789 	enum spdk_bdev_io_status io_status;
1790 
1791 	if (bio->cpl.cdw0 == 0) {
1792 		io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1793 	} else {
1794 		io_status = SPDK_BDEV_IO_STATUS_FAILED;
1795 	}
1796 
1797 	spdk_bdev_io_complete(spdk_bdev_io_from_ctx(bio), io_status);
1798 }
1799 
1800 static void
1801 _bdev_nvme_reset_io_continue(void *ctx)
1802 {
1803 	struct nvme_bdev_io *bio = ctx;
1804 	struct nvme_io_path *prev_io_path, *next_io_path;
1805 	int rc;
1806 
1807 	prev_io_path = bio->io_path;
1808 	bio->io_path = NULL;
1809 
1810 	if (bio->cpl.cdw0 != 0) {
1811 		goto complete;
1812 	}
1813 
1814 	next_io_path = STAILQ_NEXT(prev_io_path, stailq);
1815 	if (next_io_path == NULL) {
1816 		goto complete;
1817 	}
1818 
1819 	rc = _bdev_nvme_reset_io(next_io_path, bio);
1820 	if (rc == 0) {
1821 		return;
1822 	}
1823 
1824 	bio->cpl.cdw0 = 1;
1825 
1826 complete:
1827 	bdev_nvme_reset_io_complete(bio);
1828 }
1829 
1830 static void
1831 bdev_nvme_reset_io_continue(void *cb_arg, bool success)
1832 {
1833 	struct nvme_bdev_io *bio = cb_arg;
1834 
1835 	bio->cpl.cdw0 = !success;
1836 
1837 	spdk_thread_send_msg(bio->orig_thread, _bdev_nvme_reset_io_continue, bio);
1838 }
1839 
1840 static int
1841 _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio)
1842 {
1843 	struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
1844 	struct nvme_ctrlr_channel *ctrlr_ch;
1845 	struct spdk_bdev_io *bdev_io;
1846 	int rc;
1847 
1848 	rc = bdev_nvme_reset(nvme_ctrlr);
1849 	if (rc == 0) {
1850 		assert(bio->io_path == NULL);
1851 		bio->io_path = io_path;
1852 
1853 		assert(nvme_ctrlr->reset_cb_fn == NULL);
1854 		assert(nvme_ctrlr->reset_cb_arg == NULL);
1855 		nvme_ctrlr->reset_cb_fn = bdev_nvme_reset_io_continue;
1856 		nvme_ctrlr->reset_cb_arg = bio;
1857 	} else if (rc == -EBUSY) {
1858 		ctrlr_ch = io_path->qpair->ctrlr_ch;
1859 		assert(ctrlr_ch != NULL);
1860 		/*
1861 		 * Reset call is queued only if it is from the app framework. This is on purpose so that
1862 		 * we don't interfere with the app framework reset strategy. i.e. we are deferring to the
1863 		 * upper level. If they are in the middle of a reset, we won't try to schedule another one.
1864 		 */
1865 		bdev_io = spdk_bdev_io_from_ctx(bio);
1866 		TAILQ_INSERT_TAIL(&ctrlr_ch->pending_resets, bdev_io, module_link);
1867 	} else {
1868 		return rc;
1869 	}
1870 
1871 	return 0;
1872 }
1873 
1874 static void
1875 bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio)
1876 {
1877 	struct nvme_io_path *io_path;
1878 	int rc;
1879 
1880 	bio->cpl.cdw0 = 0;
1881 	bio->orig_thread = spdk_get_thread();
1882 
1883 	/* Reset only the first nvme_ctrlr in the nvme_bdev_ctrlr for now.
1884 	 *
1885 	 * TODO: Reset all nvme_ctrlrs in the nvme_bdev_ctrlr sequentially.
1886 	 * This will be done in the following patches.
1887 	 */
1888 	io_path = STAILQ_FIRST(&nbdev_ch->io_path_list);
1889 	assert(io_path != NULL);
1890 
1891 	rc = _bdev_nvme_reset_io(io_path, bio);
1892 	if (rc != 0) {
1893 		bio->cpl.cdw0 = 1;
1894 		bdev_nvme_reset_io_complete(bio);
1895 	}
1896 }
1897 
1898 static int
1899 bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove)
1900 {
1901 	pthread_mutex_lock(&nvme_ctrlr->mutex);
1902 	if (nvme_ctrlr->destruct) {
1903 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
1904 		/* Don't bother resetting if the controller is in the process of being destructed. */
1905 		return -ENXIO;
1906 	}
1907 
1908 	if (nvme_ctrlr->resetting) {
1909 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
1910 		SPDK_NOTICELOG("Unable to perform reset, already in progress.\n");
1911 		return -EBUSY;
1912 	}
1913 
1914 	bdev_nvme_failover_trid(nvme_ctrlr, remove);
1915 
1916 	if (nvme_ctrlr->reconnect_is_delayed) {
1917 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
1918 		SPDK_NOTICELOG("Reconnect is already scheduled.\n");
1919 
1920 		/* We rely on the next reconnect for the failover. */
1921 		return 0;
1922 	}
1923 
1924 	nvme_ctrlr->resetting = true;
1925 
1926 	assert(nvme_ctrlr->reset_start_tsc == 0);
1927 	nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
1928 
1929 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
1930 
1931 	spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr);
1932 	return 0;
1933 }
1934 
1935 static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks,
1936 			   uint64_t num_blocks);
1937 
1938 static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks,
1939 				  uint64_t num_blocks);
1940 
1941 static void
1942 bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
1943 		     bool success)
1944 {
1945 	struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx;
1946 	struct spdk_bdev *bdev = bdev_io->bdev;
1947 	int ret;
1948 
1949 	if (!success) {
1950 		ret = -EINVAL;
1951 		goto exit;
1952 	}
1953 
1954 	if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) {
1955 		ret = -ENXIO;
1956 		goto exit;
1957 	}
1958 
1959 	ret = bdev_nvme_readv(bio,
1960 			      bdev_io->u.bdev.iovs,
1961 			      bdev_io->u.bdev.iovcnt,
1962 			      bdev_io->u.bdev.md_buf,
1963 			      bdev_io->u.bdev.num_blocks,
1964 			      bdev_io->u.bdev.offset_blocks,
1965 			      bdev->dif_check_flags,
1966 			      bdev_io->u.bdev.ext_opts);
1967 
1968 exit:
1969 	if (spdk_unlikely(ret != 0)) {
1970 		bdev_nvme_io_complete(bio, ret);
1971 	}
1972 }
1973 
1974 static void
1975 bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
1976 {
1977 	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
1978 	struct spdk_bdev *bdev = bdev_io->bdev;
1979 	struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
1980 	struct nvme_bdev_io *nbdev_io_to_abort;
1981 	int rc = 0;
1982 
1983 	nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch);
1984 	if (spdk_unlikely(!nbdev_io->io_path)) {
1985 		if (!bdev_nvme_io_type_is_admin(bdev_io->type)) {
1986 			rc = -ENXIO;
1987 			goto exit;
1988 		}
1989 
1990 		/* Admin commands do not use the optimal I/O path.
1991 		 * Simply fall through even if it is not found.
1992 		 */
1993 	}
1994 
1995 	switch (bdev_io->type) {
1996 	case SPDK_BDEV_IO_TYPE_READ:
1997 		if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) {
1998 			rc = bdev_nvme_readv(nbdev_io,
1999 					     bdev_io->u.bdev.iovs,
2000 					     bdev_io->u.bdev.iovcnt,
2001 					     bdev_io->u.bdev.md_buf,
2002 					     bdev_io->u.bdev.num_blocks,
2003 					     bdev_io->u.bdev.offset_blocks,
2004 					     bdev->dif_check_flags,
2005 					     bdev_io->u.bdev.ext_opts);
2006 		} else {
2007 			spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb,
2008 					     bdev_io->u.bdev.num_blocks * bdev->blocklen);
2009 			rc = 0;
2010 		}
2011 		break;
2012 	case SPDK_BDEV_IO_TYPE_WRITE:
2013 		rc = bdev_nvme_writev(nbdev_io,
2014 				      bdev_io->u.bdev.iovs,
2015 				      bdev_io->u.bdev.iovcnt,
2016 				      bdev_io->u.bdev.md_buf,
2017 				      bdev_io->u.bdev.num_blocks,
2018 				      bdev_io->u.bdev.offset_blocks,
2019 				      bdev->dif_check_flags,
2020 				      bdev_io->u.bdev.ext_opts);
2021 		break;
2022 	case SPDK_BDEV_IO_TYPE_COMPARE:
2023 		rc = bdev_nvme_comparev(nbdev_io,
2024 					bdev_io->u.bdev.iovs,
2025 					bdev_io->u.bdev.iovcnt,
2026 					bdev_io->u.bdev.md_buf,
2027 					bdev_io->u.bdev.num_blocks,
2028 					bdev_io->u.bdev.offset_blocks,
2029 					bdev->dif_check_flags);
2030 		break;
2031 	case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE:
2032 		rc = bdev_nvme_comparev_and_writev(nbdev_io,
2033 						   bdev_io->u.bdev.iovs,
2034 						   bdev_io->u.bdev.iovcnt,
2035 						   bdev_io->u.bdev.fused_iovs,
2036 						   bdev_io->u.bdev.fused_iovcnt,
2037 						   bdev_io->u.bdev.md_buf,
2038 						   bdev_io->u.bdev.num_blocks,
2039 						   bdev_io->u.bdev.offset_blocks,
2040 						   bdev->dif_check_flags);
2041 		break;
2042 	case SPDK_BDEV_IO_TYPE_UNMAP:
2043 		rc = bdev_nvme_unmap(nbdev_io,
2044 				     bdev_io->u.bdev.offset_blocks,
2045 				     bdev_io->u.bdev.num_blocks);
2046 		break;
2047 	case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
2048 		rc =  bdev_nvme_write_zeroes(nbdev_io,
2049 					     bdev_io->u.bdev.offset_blocks,
2050 					     bdev_io->u.bdev.num_blocks);
2051 		break;
2052 	case SPDK_BDEV_IO_TYPE_RESET:
2053 		nbdev_io->io_path = NULL;
2054 		bdev_nvme_reset_io(nbdev_ch, nbdev_io);
2055 		break;
2056 	case SPDK_BDEV_IO_TYPE_FLUSH:
2057 		rc = bdev_nvme_flush(nbdev_io,
2058 				     bdev_io->u.bdev.offset_blocks,
2059 				     bdev_io->u.bdev.num_blocks);
2060 		break;
2061 	case SPDK_BDEV_IO_TYPE_ZONE_APPEND:
2062 		rc = bdev_nvme_zone_appendv(nbdev_io,
2063 					    bdev_io->u.bdev.iovs,
2064 					    bdev_io->u.bdev.iovcnt,
2065 					    bdev_io->u.bdev.md_buf,
2066 					    bdev_io->u.bdev.num_blocks,
2067 					    bdev_io->u.bdev.offset_blocks,
2068 					    bdev->dif_check_flags);
2069 		break;
2070 	case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO:
2071 		rc = bdev_nvme_get_zone_info(nbdev_io,
2072 					     bdev_io->u.zone_mgmt.zone_id,
2073 					     bdev_io->u.zone_mgmt.num_zones,
2074 					     bdev_io->u.zone_mgmt.buf);
2075 		break;
2076 	case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT:
2077 		rc = bdev_nvme_zone_management(nbdev_io,
2078 					       bdev_io->u.zone_mgmt.zone_id,
2079 					       bdev_io->u.zone_mgmt.zone_action);
2080 		break;
2081 	case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
2082 		nbdev_io->io_path = NULL;
2083 		bdev_nvme_admin_passthru(nbdev_ch,
2084 					 nbdev_io,
2085 					 &bdev_io->u.nvme_passthru.cmd,
2086 					 bdev_io->u.nvme_passthru.buf,
2087 					 bdev_io->u.nvme_passthru.nbytes);
2088 		break;
2089 	case SPDK_BDEV_IO_TYPE_NVME_IO:
2090 		rc = bdev_nvme_io_passthru(nbdev_io,
2091 					   &bdev_io->u.nvme_passthru.cmd,
2092 					   bdev_io->u.nvme_passthru.buf,
2093 					   bdev_io->u.nvme_passthru.nbytes);
2094 		break;
2095 	case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
2096 		rc = bdev_nvme_io_passthru_md(nbdev_io,
2097 					      &bdev_io->u.nvme_passthru.cmd,
2098 					      bdev_io->u.nvme_passthru.buf,
2099 					      bdev_io->u.nvme_passthru.nbytes,
2100 					      bdev_io->u.nvme_passthru.md_buf,
2101 					      bdev_io->u.nvme_passthru.md_len);
2102 		break;
2103 	case SPDK_BDEV_IO_TYPE_ABORT:
2104 		nbdev_io->io_path = NULL;
2105 		nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx;
2106 		bdev_nvme_abort(nbdev_ch,
2107 				nbdev_io,
2108 				nbdev_io_to_abort);
2109 		break;
2110 	default:
2111 		rc = -EINVAL;
2112 		break;
2113 	}
2114 
2115 exit:
2116 	if (spdk_unlikely(rc != 0)) {
2117 		bdev_nvme_io_complete(nbdev_io, rc);
2118 	}
2119 }
2120 
2121 static bool
2122 bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
2123 {
2124 	struct nvme_bdev *nbdev = ctx;
2125 	struct nvme_ns *nvme_ns;
2126 	struct spdk_nvme_ns *ns;
2127 	struct spdk_nvme_ctrlr *ctrlr;
2128 	const struct spdk_nvme_ctrlr_data *cdata;
2129 
2130 	nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
2131 	assert(nvme_ns != NULL);
2132 	ns = nvme_ns->ns;
2133 	ctrlr = spdk_nvme_ns_get_ctrlr(ns);
2134 
2135 	switch (io_type) {
2136 	case SPDK_BDEV_IO_TYPE_READ:
2137 	case SPDK_BDEV_IO_TYPE_WRITE:
2138 	case SPDK_BDEV_IO_TYPE_RESET:
2139 	case SPDK_BDEV_IO_TYPE_FLUSH:
2140 	case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
2141 	case SPDK_BDEV_IO_TYPE_NVME_IO:
2142 	case SPDK_BDEV_IO_TYPE_ABORT:
2143 		return true;
2144 
2145 	case SPDK_BDEV_IO_TYPE_COMPARE:
2146 		return spdk_nvme_ns_supports_compare(ns);
2147 
2148 	case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
2149 		return spdk_nvme_ns_get_md_size(ns) ? true : false;
2150 
2151 	case SPDK_BDEV_IO_TYPE_UNMAP:
2152 		cdata = spdk_nvme_ctrlr_get_data(ctrlr);
2153 		return cdata->oncs.dsm;
2154 
2155 	case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
2156 		cdata = spdk_nvme_ctrlr_get_data(ctrlr);
2157 		return cdata->oncs.write_zeroes;
2158 
2159 	case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE:
2160 		if (spdk_nvme_ctrlr_get_flags(ctrlr) &
2161 		    SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) {
2162 			return true;
2163 		}
2164 		return false;
2165 
2166 	case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO:
2167 	case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT:
2168 		return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS;
2169 
2170 	case SPDK_BDEV_IO_TYPE_ZONE_APPEND:
2171 		return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS &&
2172 		       spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED;
2173 
2174 	default:
2175 		return false;
2176 	}
2177 }
2178 
2179 static int
2180 nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch)
2181 {
2182 	struct nvme_qpair *nvme_qpair;
2183 	struct spdk_io_channel *pg_ch;
2184 	int rc;
2185 
2186 	nvme_qpair = calloc(1, sizeof(*nvme_qpair));
2187 	if (!nvme_qpair) {
2188 		SPDK_ERRLOG("Failed to alloc nvme_qpair.\n");
2189 		return -1;
2190 	}
2191 
2192 	TAILQ_INIT(&nvme_qpair->io_path_list);
2193 
2194 	nvme_qpair->ctrlr = nvme_ctrlr;
2195 	nvme_qpair->ctrlr_ch = ctrlr_ch;
2196 
2197 	pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs);
2198 	if (!pg_ch) {
2199 		free(nvme_qpair);
2200 		return -1;
2201 	}
2202 
2203 	nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch);
2204 
2205 #ifdef SPDK_CONFIG_VTUNE
2206 	nvme_qpair->group->collect_spin_stat = true;
2207 #else
2208 	nvme_qpair->group->collect_spin_stat = false;
2209 #endif
2210 
2211 	rc = bdev_nvme_create_qpair(nvme_qpair);
2212 	if (rc != 0) {
2213 		/* nvme_ctrlr can't create IO qpair if connection is down. If nvme_ctrlr is
2214 		 * being reset or scheduled to reconnect later, ignore this failure.
2215 		 * Then IO qpair will be created later when reconnect completes.
2216 		 * If the user submits IO requests in the meantime, they will be queued and
2217 		 * resubmitted later */
2218 		if (!nvme_ctrlr->resetting && !nvme_ctrlr->reconnect_is_delayed) {
2219 			spdk_put_io_channel(pg_ch);
2220 			free(nvme_qpair);
2221 			return rc;
2222 		}
2223 	}
2224 
2225 	TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq);
2226 
2227 	ctrlr_ch->qpair = nvme_qpair;
2228 
2229 	pthread_mutex_lock(&nvme_qpair->ctrlr->mutex);
2230 	nvme_qpair->ctrlr->ref++;
2231 	pthread_mutex_unlock(&nvme_qpair->ctrlr->mutex);
2232 
2233 	return 0;
2234 }
2235 
2236 static int
2237 bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf)
2238 {
2239 	struct nvme_ctrlr *nvme_ctrlr = io_device;
2240 	struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf;
2241 
2242 	TAILQ_INIT(&ctrlr_ch->pending_resets);
2243 
2244 	return nvme_qpair_create(nvme_ctrlr, ctrlr_ch);
2245 }
2246 
2247 static void
2248 nvme_qpair_delete(struct nvme_qpair *nvme_qpair)
2249 {
2250 	assert(nvme_qpair->group != NULL);
2251 
2252 	TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq);
2253 
2254 	spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group));
2255 
2256 	nvme_ctrlr_release(nvme_qpair->ctrlr);
2257 
2258 	free(nvme_qpair);
2259 }
2260 
2261 static void
2262 bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf)
2263 {
2264 	struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf;
2265 	struct nvme_qpair *nvme_qpair;
2266 
2267 	nvme_qpair = ctrlr_ch->qpair;
2268 	assert(nvme_qpair != NULL);
2269 
2270 	_bdev_nvme_clear_io_path_cache(nvme_qpair);
2271 
2272 	if (nvme_qpair->qpair != NULL) {
2273 		if (ctrlr_ch->reset_iter == NULL) {
2274 			spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair);
2275 		} else {
2276 			/* Skip current ctrlr_channel in a full reset sequence because
2277 			 * it is being deleted now. The qpair is already being disconnected.
2278 			 * We do not have to restart disconnecting it.
2279 			 */
2280 			spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0);
2281 		}
2282 
2283 		/* We cannot release a reference to the poll group now.
2284 		 * The qpair may be disconnected asynchronously later.
2285 		 * We need to poll it until it is actually disconnected.
2286 		 * Just detach the qpair from the deleting ctrlr_channel.
2287 		 */
2288 		nvme_qpair->ctrlr_ch = NULL;
2289 	} else {
2290 		assert(ctrlr_ch->reset_iter == NULL);
2291 
2292 		nvme_qpair_delete(nvme_qpair);
2293 	}
2294 }
2295 
2296 static void
2297 bdev_nvme_submit_accel_crc32c(void *ctx, uint32_t *dst, struct iovec *iov,
2298 			      uint32_t iov_cnt, uint32_t seed,
2299 			      spdk_nvme_accel_completion_cb cb_fn, void *cb_arg)
2300 {
2301 	struct nvme_poll_group *group = ctx;
2302 	int rc;
2303 
2304 	assert(group->accel_channel != NULL);
2305 	assert(cb_fn != NULL);
2306 
2307 	rc = spdk_accel_submit_crc32cv(group->accel_channel, dst, iov, iov_cnt, seed, cb_fn, cb_arg);
2308 	if (rc) {
2309 		/* For the two cases, spdk_accel_submit_crc32cv does not call the user's cb_fn */
2310 		if (rc == -ENOMEM || rc == -EINVAL) {
2311 			cb_fn(cb_arg, rc);
2312 		}
2313 		SPDK_ERRLOG("Cannot complete the accelerated crc32c operation with iov=%p\n", iov);
2314 	}
2315 }
2316 
2317 static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = {
2318 	.table_size		= sizeof(struct spdk_nvme_accel_fn_table),
2319 	.submit_accel_crc32c	= bdev_nvme_submit_accel_crc32c,
2320 };
2321 
2322 static int
2323 bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf)
2324 {
2325 	struct nvme_poll_group *group = ctx_buf;
2326 
2327 	TAILQ_INIT(&group->qpair_list);
2328 
2329 	group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table);
2330 	if (group->group == NULL) {
2331 		return -1;
2332 	}
2333 
2334 	group->accel_channel = spdk_accel_engine_get_io_channel();
2335 	if (!group->accel_channel) {
2336 		spdk_nvme_poll_group_destroy(group->group);
2337 		SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n",
2338 			    group);
2339 		return -1;
2340 	}
2341 
2342 	group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us);
2343 
2344 	if (group->poller == NULL) {
2345 		spdk_put_io_channel(group->accel_channel);
2346 		spdk_nvme_poll_group_destroy(group->group);
2347 		return -1;
2348 	}
2349 
2350 	return 0;
2351 }
2352 
2353 static void
2354 bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf)
2355 {
2356 	struct nvme_poll_group *group = ctx_buf;
2357 
2358 	assert(TAILQ_EMPTY(&group->qpair_list));
2359 
2360 	if (group->accel_channel) {
2361 		spdk_put_io_channel(group->accel_channel);
2362 	}
2363 
2364 	spdk_poller_unregister(&group->poller);
2365 	if (spdk_nvme_poll_group_destroy(group->group)) {
2366 		SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n");
2367 		assert(false);
2368 	}
2369 }
2370 
2371 static struct spdk_io_channel *
2372 bdev_nvme_get_io_channel(void *ctx)
2373 {
2374 	struct nvme_bdev *nvme_bdev = ctx;
2375 
2376 	return spdk_get_io_channel(nvme_bdev);
2377 }
2378 
2379 static void *
2380 bdev_nvme_get_module_ctx(void *ctx)
2381 {
2382 	struct nvme_bdev *nvme_bdev = ctx;
2383 	struct nvme_ns *nvme_ns;
2384 
2385 	if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) {
2386 		return NULL;
2387 	}
2388 
2389 	nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list);
2390 	if (!nvme_ns) {
2391 		return NULL;
2392 	}
2393 
2394 	return nvme_ns->ns;
2395 }
2396 
2397 static const char *
2398 _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state)
2399 {
2400 	switch (ana_state) {
2401 	case SPDK_NVME_ANA_OPTIMIZED_STATE:
2402 		return "optimized";
2403 	case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
2404 		return "non_optimized";
2405 	case SPDK_NVME_ANA_INACCESSIBLE_STATE:
2406 		return "inaccessible";
2407 	case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE:
2408 		return "persistent_loss";
2409 	case SPDK_NVME_ANA_CHANGE_STATE:
2410 		return "change";
2411 	default:
2412 		return NULL;
2413 	}
2414 }
2415 
2416 static int
2417 bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
2418 {
2419 	struct nvme_bdev *nbdev = ctx;
2420 	struct nvme_ns *nvme_ns;
2421 
2422 	nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
2423 	assert(nvme_ns != NULL);
2424 
2425 	return spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, domains, array_size);
2426 }
2427 
2428 static const char *
2429 nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr)
2430 {
2431 	if (nvme_ctrlr->destruct) {
2432 		return "deleting";
2433 	} else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
2434 		return "failed";
2435 	} else if (nvme_ctrlr->resetting) {
2436 		return "resetting";
2437 	} else if (nvme_ctrlr->reconnect_is_delayed > 0) {
2438 		return "reconnect_is_delayed";
2439 	} else {
2440 		return "enabled";
2441 	}
2442 }
2443 
2444 void
2445 nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr)
2446 {
2447 	struct spdk_nvme_transport_id *trid;
2448 	const struct spdk_nvme_ctrlr_opts *opts;
2449 	const struct spdk_nvme_ctrlr_data *cdata;
2450 
2451 	spdk_json_write_object_begin(w);
2452 
2453 	spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr));
2454 
2455 #ifdef SPDK_CONFIG_NVME_CUSE
2456 	size_t cuse_name_size = 128;
2457 	char cuse_name[cuse_name_size];
2458 
2459 	int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size);
2460 	if (rc == 0) {
2461 		spdk_json_write_named_string(w, "cuse_device", cuse_name);
2462 	}
2463 #endif
2464 	trid = &nvme_ctrlr->active_path_id->trid;
2465 	spdk_json_write_named_object_begin(w, "trid");
2466 	nvme_bdev_dump_trid_json(trid, w);
2467 	spdk_json_write_object_end(w);
2468 
2469 	cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
2470 	spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid);
2471 
2472 	opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
2473 	spdk_json_write_named_object_begin(w, "host");
2474 	spdk_json_write_named_string(w, "nqn", opts->hostnqn);
2475 	spdk_json_write_named_string(w, "addr", opts->src_addr);
2476 	spdk_json_write_named_string(w, "svcid", opts->src_svcid);
2477 	spdk_json_write_object_end(w);
2478 
2479 	spdk_json_write_object_end(w);
2480 }
2481 
2482 static void
2483 nvme_namespace_info_json(struct spdk_json_write_ctx *w,
2484 			 struct nvme_ns *nvme_ns)
2485 {
2486 	struct spdk_nvme_ns *ns;
2487 	struct spdk_nvme_ctrlr *ctrlr;
2488 	const struct spdk_nvme_ctrlr_data *cdata;
2489 	const struct spdk_nvme_transport_id *trid;
2490 	union spdk_nvme_vs_register vs;
2491 	const struct spdk_nvme_ns_data *nsdata;
2492 	char buf[128];
2493 
2494 	ns = nvme_ns->ns;
2495 	ctrlr = spdk_nvme_ns_get_ctrlr(ns);
2496 
2497 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
2498 	trid = spdk_nvme_ctrlr_get_transport_id(ctrlr);
2499 	vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr);
2500 
2501 	spdk_json_write_object_begin(w);
2502 
2503 	if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
2504 		spdk_json_write_named_string(w, "pci_address", trid->traddr);
2505 	}
2506 
2507 	spdk_json_write_named_object_begin(w, "trid");
2508 
2509 	nvme_bdev_dump_trid_json(trid, w);
2510 
2511 	spdk_json_write_object_end(w);
2512 
2513 #ifdef SPDK_CONFIG_NVME_CUSE
2514 	size_t cuse_name_size = 128;
2515 	char cuse_name[cuse_name_size];
2516 
2517 	int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns),
2518 					    cuse_name, &cuse_name_size);
2519 	if (rc == 0) {
2520 		spdk_json_write_named_string(w, "cuse_device", cuse_name);
2521 	}
2522 #endif
2523 
2524 	spdk_json_write_named_object_begin(w, "ctrlr_data");
2525 
2526 	spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid);
2527 
2528 	spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid);
2529 
2530 	snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn);
2531 	spdk_str_trim(buf);
2532 	spdk_json_write_named_string(w, "model_number", buf);
2533 
2534 	snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn);
2535 	spdk_str_trim(buf);
2536 	spdk_json_write_named_string(w, "serial_number", buf);
2537 
2538 	snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr);
2539 	spdk_str_trim(buf);
2540 	spdk_json_write_named_string(w, "firmware_revision", buf);
2541 
2542 	if (cdata->subnqn[0] != '\0') {
2543 		spdk_json_write_named_string(w, "subnqn", cdata->subnqn);
2544 	}
2545 
2546 	spdk_json_write_named_object_begin(w, "oacs");
2547 
2548 	spdk_json_write_named_uint32(w, "security", cdata->oacs.security);
2549 	spdk_json_write_named_uint32(w, "format", cdata->oacs.format);
2550 	spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware);
2551 	spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage);
2552 
2553 	spdk_json_write_object_end(w);
2554 
2555 	spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr);
2556 	spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting);
2557 
2558 	spdk_json_write_object_end(w);
2559 
2560 	spdk_json_write_named_object_begin(w, "vs");
2561 
2562 	spdk_json_write_name(w, "nvme_version");
2563 	if (vs.bits.ter) {
2564 		spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter);
2565 	} else {
2566 		spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr);
2567 	}
2568 
2569 	spdk_json_write_object_end(w);
2570 
2571 	nsdata = spdk_nvme_ns_get_data(ns);
2572 
2573 	spdk_json_write_named_object_begin(w, "ns_data");
2574 
2575 	spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns));
2576 
2577 	if (cdata->cmic.ana_reporting) {
2578 		spdk_json_write_named_string(w, "ana_state",
2579 					     _nvme_ana_state_str(nvme_ns->ana_state));
2580 	}
2581 
2582 	spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share);
2583 
2584 	spdk_json_write_object_end(w);
2585 
2586 	if (cdata->oacs.security) {
2587 		spdk_json_write_named_object_begin(w, "security");
2588 
2589 		spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal);
2590 
2591 		spdk_json_write_object_end(w);
2592 	}
2593 
2594 	spdk_json_write_object_end(w);
2595 }
2596 
2597 static int
2598 bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
2599 {
2600 	struct nvme_bdev *nvme_bdev = ctx;
2601 	struct nvme_ns *nvme_ns;
2602 
2603 	pthread_mutex_lock(&nvme_bdev->mutex);
2604 	spdk_json_write_named_array_begin(w, "nvme");
2605 	TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) {
2606 		nvme_namespace_info_json(w, nvme_ns);
2607 	}
2608 	spdk_json_write_array_end(w);
2609 	pthread_mutex_unlock(&nvme_bdev->mutex);
2610 
2611 	return 0;
2612 }
2613 
2614 static void
2615 bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
2616 {
2617 	/* No config per bdev needed */
2618 }
2619 
2620 static uint64_t
2621 bdev_nvme_get_spin_time(struct spdk_io_channel *ch)
2622 {
2623 	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
2624 	struct nvme_io_path *io_path;
2625 	struct nvme_poll_group *group;
2626 	uint64_t spin_time = 0;
2627 
2628 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
2629 		group = io_path->qpair->group;
2630 
2631 		if (!group || !group->collect_spin_stat) {
2632 			continue;
2633 		}
2634 
2635 		if (group->end_ticks != 0) {
2636 			group->spin_ticks += (group->end_ticks - group->start_ticks);
2637 			group->end_ticks = 0;
2638 		}
2639 
2640 		spin_time += group->spin_ticks;
2641 		group->start_ticks = 0;
2642 		group->spin_ticks = 0;
2643 	}
2644 
2645 	return (spin_time * 1000000ULL) / spdk_get_ticks_hz();
2646 }
2647 
2648 static const struct spdk_bdev_fn_table nvmelib_fn_table = {
2649 	.destruct		= bdev_nvme_destruct,
2650 	.submit_request		= bdev_nvme_submit_request,
2651 	.io_type_supported	= bdev_nvme_io_type_supported,
2652 	.get_io_channel		= bdev_nvme_get_io_channel,
2653 	.dump_info_json		= bdev_nvme_dump_info_json,
2654 	.write_config_json	= bdev_nvme_write_config_json,
2655 	.get_spin_time		= bdev_nvme_get_spin_time,
2656 	.get_module_ctx		= bdev_nvme_get_module_ctx,
2657 	.get_memory_domains	= bdev_nvme_get_memory_domains,
2658 };
2659 
2660 typedef int (*bdev_nvme_parse_ana_log_page_cb)(
2661 	const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg);
2662 
2663 static int
2664 bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr,
2665 			     bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg)
2666 {
2667 	struct spdk_nvme_ana_group_descriptor *copied_desc;
2668 	uint8_t *orig_desc;
2669 	uint32_t i, desc_size, copy_len;
2670 	int rc = 0;
2671 
2672 	if (nvme_ctrlr->ana_log_page == NULL) {
2673 		return -EINVAL;
2674 	}
2675 
2676 	copied_desc = nvme_ctrlr->copied_ana_desc;
2677 
2678 	orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page);
2679 	copy_len = nvme_ctrlr->ana_log_page_size - sizeof(struct spdk_nvme_ana_page);
2680 
2681 	for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) {
2682 		memcpy(copied_desc, orig_desc, copy_len);
2683 
2684 		rc = cb_fn(copied_desc, cb_arg);
2685 		if (rc != 0) {
2686 			break;
2687 		}
2688 
2689 		desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) +
2690 			    copied_desc->num_of_nsid * sizeof(uint32_t);
2691 		orig_desc += desc_size;
2692 		copy_len -= desc_size;
2693 	}
2694 
2695 	return rc;
2696 }
2697 
2698 static int
2699 nvme_ns_ana_transition_timedout(void *ctx)
2700 {
2701 	struct nvme_ns *nvme_ns = ctx;
2702 
2703 	spdk_poller_unregister(&nvme_ns->anatt_timer);
2704 	nvme_ns->ana_transition_timedout = true;
2705 
2706 	return SPDK_POLLER_BUSY;
2707 }
2708 
2709 static void
2710 _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns,
2711 		       const struct spdk_nvme_ana_group_descriptor *desc)
2712 {
2713 	const struct spdk_nvme_ctrlr_data *cdata;
2714 
2715 	nvme_ns->ana_group_id = desc->ana_group_id;
2716 	nvme_ns->ana_state = desc->ana_state;
2717 	nvme_ns->ana_state_updating = false;
2718 
2719 	switch (nvme_ns->ana_state) {
2720 	case SPDK_NVME_ANA_OPTIMIZED_STATE:
2721 	case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
2722 		nvme_ns->ana_transition_timedout = false;
2723 		spdk_poller_unregister(&nvme_ns->anatt_timer);
2724 		break;
2725 
2726 	case SPDK_NVME_ANA_INACCESSIBLE_STATE:
2727 	case SPDK_NVME_ANA_CHANGE_STATE:
2728 		if (nvme_ns->anatt_timer != NULL) {
2729 			break;
2730 		}
2731 
2732 		cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr);
2733 		nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout,
2734 				       nvme_ns,
2735 				       cdata->anatt * SPDK_SEC_TO_USEC);
2736 		break;
2737 	default:
2738 		break;
2739 	}
2740 }
2741 
2742 static int
2743 nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg)
2744 {
2745 	struct nvme_ns *nvme_ns = cb_arg;
2746 	uint32_t i;
2747 
2748 	for (i = 0; i < desc->num_of_nsid; i++) {
2749 		if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) {
2750 			continue;
2751 		}
2752 
2753 		_nvme_ns_set_ana_state(nvme_ns, desc);
2754 		return 1;
2755 	}
2756 
2757 	return 0;
2758 }
2759 
2760 static int
2761 nvme_disk_create(struct spdk_bdev *disk, const char *base_name,
2762 		 struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns,
2763 		 uint32_t prchk_flags, void *ctx)
2764 {
2765 	const struct spdk_uuid		*uuid;
2766 	const uint8_t *nguid;
2767 	const struct spdk_nvme_ctrlr_data *cdata;
2768 	const struct spdk_nvme_ns_data	*nsdata;
2769 	enum spdk_nvme_csi		csi;
2770 	uint32_t atomic_bs, phys_bs, bs;
2771 
2772 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
2773 	csi = spdk_nvme_ns_get_csi(ns);
2774 
2775 	switch (csi) {
2776 	case SPDK_NVME_CSI_NVM:
2777 		disk->product_name = "NVMe disk";
2778 		break;
2779 	case SPDK_NVME_CSI_ZNS:
2780 		disk->product_name = "NVMe ZNS disk";
2781 		disk->zoned = true;
2782 		disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
2783 		disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) /
2784 					     spdk_nvme_ns_get_extended_sector_size(ns);
2785 		disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns);
2786 		disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns);
2787 		break;
2788 	default:
2789 		SPDK_ERRLOG("unsupported CSI: %u\n", csi);
2790 		return -ENOTSUP;
2791 	}
2792 
2793 	disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns));
2794 	if (!disk->name) {
2795 		return -ENOMEM;
2796 	}
2797 
2798 	disk->write_cache = 0;
2799 	if (cdata->vwc.present) {
2800 		/* Enable if the Volatile Write Cache exists */
2801 		disk->write_cache = 1;
2802 	}
2803 	if (cdata->oncs.write_zeroes) {
2804 		disk->max_write_zeroes = UINT16_MAX + 1;
2805 	}
2806 	disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns);
2807 	disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns);
2808 	disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns);
2809 
2810 	nguid = spdk_nvme_ns_get_nguid(ns);
2811 	if (!nguid) {
2812 		uuid = spdk_nvme_ns_get_uuid(ns);
2813 		if (uuid) {
2814 			disk->uuid = *uuid;
2815 		}
2816 	} else {
2817 		memcpy(&disk->uuid, nguid, sizeof(disk->uuid));
2818 	}
2819 
2820 	nsdata = spdk_nvme_ns_get_data(ns);
2821 	bs = spdk_nvme_ns_get_sector_size(ns);
2822 	atomic_bs = bs;
2823 	phys_bs = bs;
2824 	if (nsdata->nabo == 0) {
2825 		if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) {
2826 			atomic_bs = bs * (1 + nsdata->nawupf);
2827 		} else {
2828 			atomic_bs = bs * (1 + cdata->awupf);
2829 		}
2830 	}
2831 	if (nsdata->nsfeat.optperf) {
2832 		phys_bs = bs * (1 + nsdata->npwg);
2833 	}
2834 	disk->phys_blocklen = spdk_min(phys_bs, atomic_bs);
2835 
2836 	disk->md_len = spdk_nvme_ns_get_md_size(ns);
2837 	if (disk->md_len != 0) {
2838 		disk->md_interleave = nsdata->flbas.extended;
2839 		disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns);
2840 		if (disk->dif_type != SPDK_DIF_DISABLE) {
2841 			disk->dif_is_head_of_md = nsdata->dps.md_start;
2842 			disk->dif_check_flags = prchk_flags;
2843 		}
2844 	}
2845 
2846 	if (!(spdk_nvme_ctrlr_get_flags(ctrlr) &
2847 	      SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) {
2848 		disk->acwu = 0;
2849 	} else if (nsdata->nsfeat.ns_atomic_write_unit) {
2850 		disk->acwu = nsdata->nacwu;
2851 	} else {
2852 		disk->acwu = cdata->acwu;
2853 	}
2854 
2855 	disk->ctxt = ctx;
2856 	disk->fn_table = &nvmelib_fn_table;
2857 	disk->module = &nvme_if;
2858 
2859 	return 0;
2860 }
2861 
2862 static int
2863 nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
2864 {
2865 	struct nvme_bdev *bdev;
2866 	int rc;
2867 
2868 	bdev = calloc(1, sizeof(*bdev));
2869 	if (!bdev) {
2870 		SPDK_ERRLOG("bdev calloc() failed\n");
2871 		return -ENOMEM;
2872 	}
2873 
2874 	rc = pthread_mutex_init(&bdev->mutex, NULL);
2875 	if (rc != 0) {
2876 		free(bdev);
2877 		return rc;
2878 	}
2879 
2880 	bdev->ref = 1;
2881 	TAILQ_INIT(&bdev->nvme_ns_list);
2882 	TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq);
2883 	bdev->opal = nvme_ctrlr->opal_dev != NULL;
2884 
2885 	rc = nvme_disk_create(&bdev->disk, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ctrlr,
2886 			      nvme_ns->ns, nvme_ctrlr->opts.prchk_flags, bdev);
2887 	if (rc != 0) {
2888 		SPDK_ERRLOG("Failed to create NVMe disk\n");
2889 		pthread_mutex_destroy(&bdev->mutex);
2890 		free(bdev);
2891 		return rc;
2892 	}
2893 
2894 	spdk_io_device_register(bdev,
2895 				bdev_nvme_create_bdev_channel_cb,
2896 				bdev_nvme_destroy_bdev_channel_cb,
2897 				sizeof(struct nvme_bdev_channel),
2898 				bdev->disk.name);
2899 
2900 	rc = spdk_bdev_register(&bdev->disk);
2901 	if (rc != 0) {
2902 		SPDK_ERRLOG("spdk_bdev_register() failed\n");
2903 		spdk_io_device_unregister(bdev, NULL);
2904 		pthread_mutex_destroy(&bdev->mutex);
2905 		free(bdev->disk.name);
2906 		free(bdev);
2907 		return rc;
2908 	}
2909 
2910 	nvme_ns->bdev = bdev;
2911 	bdev->nsid = nvme_ns->id;
2912 
2913 	bdev->nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr;
2914 	TAILQ_INSERT_TAIL(&nvme_ctrlr->nbdev_ctrlr->bdevs, bdev, tailq);
2915 
2916 	return 0;
2917 }
2918 
2919 static bool
2920 bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2)
2921 {
2922 	const struct spdk_nvme_ns_data *nsdata1, *nsdata2;
2923 	const struct spdk_uuid *uuid1, *uuid2;
2924 
2925 	nsdata1 = spdk_nvme_ns_get_data(ns1);
2926 	nsdata2 = spdk_nvme_ns_get_data(ns2);
2927 	uuid1 = spdk_nvme_ns_get_uuid(ns1);
2928 	uuid2 = spdk_nvme_ns_get_uuid(ns2);
2929 
2930 	return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 &&
2931 	       nsdata1->eui64 == nsdata2->eui64 &&
2932 	       ((uuid1 == NULL && uuid2 == NULL) ||
2933 		(uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) &&
2934 	       spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2);
2935 }
2936 
2937 static bool
2938 hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
2939 		 struct spdk_nvme_ctrlr_opts *opts)
2940 {
2941 	struct nvme_probe_skip_entry *entry;
2942 
2943 	TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) {
2944 		if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) {
2945 			return false;
2946 		}
2947 	}
2948 
2949 	opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst;
2950 	opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight;
2951 	opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight;
2952 	opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight;
2953 	opts->disable_read_ana_log_page = true;
2954 
2955 	SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr);
2956 
2957 	return true;
2958 }
2959 
2960 static void
2961 nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl)
2962 {
2963 	struct nvme_ctrlr *nvme_ctrlr = ctx;
2964 
2965 	if (spdk_nvme_cpl_is_error(cpl)) {
2966 		SPDK_WARNLOG("Abort failed. Resetting controller. sc is %u, sct is %u.\n", cpl->status.sc,
2967 			     cpl->status.sct);
2968 		bdev_nvme_reset(nvme_ctrlr);
2969 	} else if (cpl->cdw0 & 0x1) {
2970 		SPDK_WARNLOG("Specified command could not be aborted.\n");
2971 		bdev_nvme_reset(nvme_ctrlr);
2972 	}
2973 }
2974 
2975 static void
2976 timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
2977 	   struct spdk_nvme_qpair *qpair, uint16_t cid)
2978 {
2979 	struct nvme_ctrlr *nvme_ctrlr = cb_arg;
2980 	union spdk_nvme_csts_register csts;
2981 	int rc;
2982 
2983 	assert(nvme_ctrlr->ctrlr == ctrlr);
2984 
2985 	SPDK_WARNLOG("Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n", ctrlr, qpair, cid);
2986 
2987 	/* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O
2988 	 * queue.  (Note: qpair == NULL when there's an admin cmd timeout.)  Otherwise we
2989 	 * would submit another fabrics cmd on the admin queue to read CSTS and check for its
2990 	 * completion recursively.
2991 	 */
2992 	if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) {
2993 		csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr);
2994 		if (csts.bits.cfs) {
2995 			SPDK_ERRLOG("Controller Fatal Status, reset required\n");
2996 			bdev_nvme_reset(nvme_ctrlr);
2997 			return;
2998 		}
2999 	}
3000 
3001 	switch (g_opts.action_on_timeout) {
3002 	case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT:
3003 		if (qpair) {
3004 			/* Don't send abort to ctrlr when ctrlr is not available. */
3005 			pthread_mutex_lock(&nvme_ctrlr->mutex);
3006 			if (!nvme_ctrlr_is_available(nvme_ctrlr)) {
3007 				pthread_mutex_unlock(&nvme_ctrlr->mutex);
3008 				SPDK_NOTICELOG("Quit abort. Ctrlr is not available.\n");
3009 				return;
3010 			}
3011 			pthread_mutex_unlock(&nvme_ctrlr->mutex);
3012 
3013 			rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid,
3014 						       nvme_abort_cpl, nvme_ctrlr);
3015 			if (rc == 0) {
3016 				return;
3017 			}
3018 
3019 			SPDK_ERRLOG("Unable to send abort. Resetting, rc is %d.\n", rc);
3020 		}
3021 
3022 	/* FALLTHROUGH */
3023 	case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET:
3024 		bdev_nvme_reset(nvme_ctrlr);
3025 		break;
3026 	case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE:
3027 		SPDK_DEBUGLOG(bdev_nvme, "No action for nvme controller timeout.\n");
3028 		break;
3029 	default:
3030 		SPDK_ERRLOG("An invalid timeout action value is found.\n");
3031 		break;
3032 	}
3033 }
3034 
3035 static void
3036 nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc)
3037 {
3038 	struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr;
3039 	struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx;
3040 
3041 	if (rc == 0) {
3042 		nvme_ns->probe_ctx = NULL;
3043 		pthread_mutex_lock(&nvme_ctrlr->mutex);
3044 		nvme_ctrlr->ref++;
3045 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
3046 	} else {
3047 		RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
3048 		free(nvme_ns);
3049 	}
3050 
3051 	if (ctx) {
3052 		ctx->populates_in_progress--;
3053 		if (ctx->populates_in_progress == 0) {
3054 			nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx);
3055 		}
3056 	}
3057 }
3058 
3059 static void
3060 bdev_nvme_add_io_path(struct spdk_io_channel_iter *i)
3061 {
3062 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
3063 	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch);
3064 	struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i);
3065 	int rc;
3066 
3067 	rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns);
3068 	if (rc != 0) {
3069 		SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n");
3070 	}
3071 
3072 	spdk_for_each_channel_continue(i, rc);
3073 }
3074 
3075 static void
3076 bdev_nvme_delete_io_path(struct spdk_io_channel_iter *i)
3077 {
3078 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
3079 	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch);
3080 	struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i);
3081 	struct nvme_io_path *io_path;
3082 
3083 	io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns);
3084 	if (io_path != NULL) {
3085 		_bdev_nvme_delete_io_path(nbdev_ch, io_path);
3086 	}
3087 
3088 	spdk_for_each_channel_continue(i, 0);
3089 }
3090 
3091 static void
3092 bdev_nvme_add_io_path_failed(struct spdk_io_channel_iter *i, int status)
3093 {
3094 	struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i);
3095 
3096 	nvme_ctrlr_populate_namespace_done(nvme_ns, -1);
3097 }
3098 
3099 static void
3100 bdev_nvme_add_io_path_done(struct spdk_io_channel_iter *i, int status)
3101 {
3102 	struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i);
3103 	struct nvme_bdev *bdev = spdk_io_channel_iter_get_io_device(i);
3104 
3105 	if (status == 0) {
3106 		nvme_ctrlr_populate_namespace_done(nvme_ns, 0);
3107 	} else {
3108 		/* Delete the added io_paths and fail populating the namespace. */
3109 		spdk_for_each_channel(bdev,
3110 				      bdev_nvme_delete_io_path,
3111 				      nvme_ns,
3112 				      bdev_nvme_add_io_path_failed);
3113 	}
3114 }
3115 
3116 static int
3117 nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns)
3118 {
3119 	struct nvme_ns *tmp_ns;
3120 	const struct spdk_nvme_ns_data *nsdata;
3121 
3122 	nsdata = spdk_nvme_ns_get_data(nvme_ns->ns);
3123 	if (!nsdata->nmic.can_share) {
3124 		SPDK_ERRLOG("Namespace cannot be shared.\n");
3125 		return -EINVAL;
3126 	}
3127 
3128 	pthread_mutex_lock(&bdev->mutex);
3129 
3130 	tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list);
3131 	assert(tmp_ns != NULL);
3132 
3133 	if (!bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) {
3134 		pthread_mutex_unlock(&bdev->mutex);
3135 		SPDK_ERRLOG("Namespaces are not identical.\n");
3136 		return -EINVAL;
3137 	}
3138 
3139 	bdev->ref++;
3140 	TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq);
3141 	nvme_ns->bdev = bdev;
3142 
3143 	pthread_mutex_unlock(&bdev->mutex);
3144 
3145 	/* Add nvme_io_path to nvme_bdev_channels dynamically. */
3146 	spdk_for_each_channel(bdev,
3147 			      bdev_nvme_add_io_path,
3148 			      nvme_ns,
3149 			      bdev_nvme_add_io_path_done);
3150 
3151 	return 0;
3152 }
3153 
3154 static void
3155 nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
3156 {
3157 	struct spdk_nvme_ns	*ns;
3158 	struct nvme_bdev	*bdev;
3159 	int			rc = 0;
3160 
3161 	ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id);
3162 	if (!ns) {
3163 		SPDK_DEBUGLOG(bdev_nvme, "Invalid NS %d\n", nvme_ns->id);
3164 		rc = -EINVAL;
3165 		goto done;
3166 	}
3167 
3168 	nvme_ns->ns = ns;
3169 	nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
3170 
3171 	if (nvme_ctrlr->ana_log_page != NULL) {
3172 		bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns);
3173 	}
3174 
3175 	bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id);
3176 	if (bdev == NULL) {
3177 		rc = nvme_bdev_create(nvme_ctrlr, nvme_ns);
3178 	} else {
3179 		rc = nvme_bdev_add_ns(bdev, nvme_ns);
3180 		if (rc == 0) {
3181 			return;
3182 		}
3183 	}
3184 done:
3185 	nvme_ctrlr_populate_namespace_done(nvme_ns, rc);
3186 }
3187 
3188 static void
3189 nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns)
3190 {
3191 	struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr;
3192 
3193 	assert(nvme_ctrlr != NULL);
3194 
3195 	pthread_mutex_lock(&nvme_ctrlr->mutex);
3196 
3197 	RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
3198 
3199 	if (nvme_ns->bdev != NULL) {
3200 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
3201 		return;
3202 	}
3203 
3204 	free(nvme_ns);
3205 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
3206 
3207 	nvme_ctrlr_release(nvme_ctrlr);
3208 }
3209 
3210 static void
3211 bdev_nvme_delete_io_path_done(struct spdk_io_channel_iter *i, int status)
3212 {
3213 	struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i);
3214 
3215 	nvme_ctrlr_depopulate_namespace_done(nvme_ns);
3216 }
3217 
3218 static void
3219 nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
3220 {
3221 	struct nvme_bdev *bdev;
3222 
3223 	spdk_poller_unregister(&nvme_ns->anatt_timer);
3224 
3225 	bdev = nvme_ns->bdev;
3226 	if (bdev != NULL) {
3227 		pthread_mutex_lock(&bdev->mutex);
3228 
3229 		assert(bdev->ref > 0);
3230 		bdev->ref--;
3231 		if (bdev->ref == 0) {
3232 			pthread_mutex_unlock(&bdev->mutex);
3233 
3234 			spdk_bdev_unregister(&bdev->disk, NULL, NULL);
3235 		} else {
3236 			/* spdk_bdev_unregister() is not called until the last nvme_ns is
3237 			 * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list
3238 			 * and clear nvme_ns->bdev here.
3239 			 */
3240 			TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq);
3241 			nvme_ns->bdev = NULL;
3242 
3243 			pthread_mutex_unlock(&bdev->mutex);
3244 
3245 			/* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that,
3246 			 * we call depopulate_namespace_done() to avoid use-after-free.
3247 			 */
3248 			spdk_for_each_channel(bdev,
3249 					      bdev_nvme_delete_io_path,
3250 					      nvme_ns,
3251 					      bdev_nvme_delete_io_path_done);
3252 			return;
3253 		}
3254 	}
3255 
3256 	nvme_ctrlr_depopulate_namespace_done(nvme_ns);
3257 }
3258 
3259 static void
3260 nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
3261 			       struct nvme_async_probe_ctx *ctx)
3262 {
3263 	struct spdk_nvme_ctrlr	*ctrlr = nvme_ctrlr->ctrlr;
3264 	struct nvme_ns	*nvme_ns, *next;
3265 	struct spdk_nvme_ns	*ns;
3266 	struct nvme_bdev	*bdev;
3267 	uint32_t		nsid;
3268 	int			rc;
3269 	uint64_t		num_sectors;
3270 
3271 	if (ctx) {
3272 		/* Initialize this count to 1 to handle the populate functions
3273 		 * calling nvme_ctrlr_populate_namespace_done() immediately.
3274 		 */
3275 		ctx->populates_in_progress = 1;
3276 	}
3277 
3278 	/* First loop over our existing namespaces and see if they have been
3279 	 * removed. */
3280 	nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
3281 	while (nvme_ns != NULL) {
3282 		next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
3283 
3284 		if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) {
3285 			/* NS is still there but attributes may have changed */
3286 			ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id);
3287 			num_sectors = spdk_nvme_ns_get_num_sectors(ns);
3288 			bdev = nvme_ns->bdev;
3289 			assert(bdev != NULL);
3290 			if (bdev->disk.blockcnt != num_sectors) {
3291 				SPDK_NOTICELOG("NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n",
3292 					       nvme_ns->id,
3293 					       bdev->disk.name,
3294 					       bdev->disk.blockcnt,
3295 					       num_sectors);
3296 				rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors);
3297 				if (rc != 0) {
3298 					SPDK_ERRLOG("Could not change num blocks for nvme bdev: name %s, errno: %d.\n",
3299 						    bdev->disk.name, rc);
3300 				}
3301 			}
3302 		} else {
3303 			/* Namespace was removed */
3304 			nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns);
3305 		}
3306 
3307 		nvme_ns = next;
3308 	}
3309 
3310 	/* Loop through all of the namespaces at the nvme level and see if any of them are new */
3311 	nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
3312 	while (nsid != 0) {
3313 		nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
3314 
3315 		if (nvme_ns == NULL) {
3316 			/* Found a new one */
3317 			nvme_ns = calloc(1, sizeof(struct nvme_ns));
3318 			if (nvme_ns == NULL) {
3319 				SPDK_ERRLOG("Failed to allocate namespace\n");
3320 				/* This just fails to attach the namespace. It may work on a future attempt. */
3321 				continue;
3322 			}
3323 
3324 			nvme_ns->id = nsid;
3325 			nvme_ns->ctrlr = nvme_ctrlr;
3326 
3327 			nvme_ns->bdev = NULL;
3328 
3329 			if (ctx) {
3330 				ctx->populates_in_progress++;
3331 			}
3332 			nvme_ns->probe_ctx = ctx;
3333 
3334 			RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
3335 
3336 			nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns);
3337 		}
3338 
3339 		nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid);
3340 	}
3341 
3342 	if (ctx) {
3343 		/* Decrement this count now that the loop is over to account
3344 		 * for the one we started with.  If the count is then 0, we
3345 		 * know any populate_namespace functions completed immediately,
3346 		 * so we'll kick the callback here.
3347 		 */
3348 		ctx->populates_in_progress--;
3349 		if (ctx->populates_in_progress == 0) {
3350 			nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx);
3351 		}
3352 	}
3353 
3354 }
3355 
3356 static void
3357 nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr)
3358 {
3359 	struct nvme_ns *nvme_ns, *tmp;
3360 
3361 	RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) {
3362 		nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns);
3363 	}
3364 }
3365 
3366 static int
3367 nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc,
3368 			  void *cb_arg)
3369 {
3370 	struct nvme_ctrlr *nvme_ctrlr = cb_arg;
3371 	struct nvme_ns *nvme_ns;
3372 	uint32_t i, nsid;
3373 
3374 	for (i = 0; i < desc->num_of_nsid; i++) {
3375 		nsid = desc->nsid[i];
3376 		if (nsid == 0) {
3377 			continue;
3378 		}
3379 
3380 		nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
3381 
3382 		assert(nvme_ns != NULL);
3383 		if (nvme_ns == NULL) {
3384 			/* Target told us that an inactive namespace had an ANA change */
3385 			continue;
3386 		}
3387 
3388 		_nvme_ns_set_ana_state(nvme_ns, desc);
3389 	}
3390 
3391 	return 0;
3392 }
3393 
3394 static void
3395 _nvme_ctrlr_read_ana_log_page_done(struct spdk_io_channel_iter *i, int status)
3396 {
3397 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
3398 
3399 	pthread_mutex_lock(&nvme_ctrlr->mutex);
3400 
3401 	assert(nvme_ctrlr->ana_log_page_updating == true);
3402 	nvme_ctrlr->ana_log_page_updating = false;
3403 
3404 	if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
3405 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
3406 		return;
3407 	}
3408 
3409 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
3410 
3411 	nvme_ctrlr_unregister(nvme_ctrlr);
3412 }
3413 
3414 static void
3415 bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr)
3416 {
3417 	struct nvme_ns *nvme_ns;
3418 
3419 	spdk_free(nvme_ctrlr->ana_log_page);
3420 	nvme_ctrlr->ana_log_page = NULL;
3421 
3422 	for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
3423 	     nvme_ns != NULL;
3424 	     nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) {
3425 		nvme_ns->ana_state_updating = false;
3426 		nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
3427 	}
3428 }
3429 
3430 static void
3431 nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl)
3432 {
3433 	struct nvme_ctrlr *nvme_ctrlr = ctx;
3434 
3435 	if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) {
3436 		bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states,
3437 					     nvme_ctrlr);
3438 	} else {
3439 		bdev_nvme_disable_read_ana_log_page(nvme_ctrlr);
3440 	}
3441 
3442 	bdev_nvme_clear_io_path_caches(nvme_ctrlr, _nvme_ctrlr_read_ana_log_page_done);
3443 }
3444 
3445 static int
3446 nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr)
3447 {
3448 	int rc;
3449 
3450 	if (nvme_ctrlr->ana_log_page == NULL) {
3451 		return -EINVAL;
3452 	}
3453 
3454 	pthread_mutex_lock(&nvme_ctrlr->mutex);
3455 	if (!nvme_ctrlr_is_available(nvme_ctrlr) ||
3456 	    nvme_ctrlr->ana_log_page_updating) {
3457 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
3458 		return -EBUSY;
3459 	}
3460 
3461 	nvme_ctrlr->ana_log_page_updating = true;
3462 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
3463 
3464 	rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr,
3465 					      SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
3466 					      SPDK_NVME_GLOBAL_NS_TAG,
3467 					      nvme_ctrlr->ana_log_page,
3468 					      nvme_ctrlr->ana_log_page_size, 0,
3469 					      nvme_ctrlr_read_ana_log_page_done,
3470 					      nvme_ctrlr);
3471 	if (rc != 0) {
3472 		nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL);
3473 	}
3474 
3475 	return rc;
3476 }
3477 
3478 static void
3479 aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
3480 {
3481 	struct nvme_ctrlr *nvme_ctrlr		= arg;
3482 	union spdk_nvme_async_event_completion	event;
3483 
3484 	if (spdk_nvme_cpl_is_error(cpl)) {
3485 		SPDK_WARNLOG("AER request execute failed");
3486 		return;
3487 	}
3488 
3489 	event.raw = cpl->cdw0;
3490 	if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
3491 	    (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) {
3492 		nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL);
3493 	} else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
3494 		   (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) {
3495 		nvme_ctrlr_read_ana_log_page(nvme_ctrlr);
3496 	}
3497 }
3498 
3499 static void
3500 populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, size_t count, int rc)
3501 {
3502 	if (ctx->cb_fn) {
3503 		ctx->cb_fn(ctx->cb_ctx, count, rc);
3504 	}
3505 
3506 	ctx->namespaces_populated = true;
3507 	if (ctx->probe_done) {
3508 		/* The probe was already completed, so we need to free the context
3509 		 * here.  This can happen for cases like OCSSD, where we need to
3510 		 * send additional commands to the SSD after attach.
3511 		 */
3512 		free(ctx);
3513 	}
3514 }
3515 
3516 static void
3517 nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr,
3518 		       struct nvme_async_probe_ctx *ctx)
3519 {
3520 	spdk_io_device_register(nvme_ctrlr,
3521 				bdev_nvme_create_ctrlr_channel_cb,
3522 				bdev_nvme_destroy_ctrlr_channel_cb,
3523 				sizeof(struct nvme_ctrlr_channel),
3524 				nvme_ctrlr->nbdev_ctrlr->name);
3525 
3526 	nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx);
3527 }
3528 
3529 static void
3530 nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl)
3531 {
3532 	struct nvme_ctrlr *nvme_ctrlr = _ctx;
3533 	struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx;
3534 
3535 	nvme_ctrlr->probe_ctx = NULL;
3536 
3537 	if (spdk_nvme_cpl_is_error(cpl)) {
3538 		nvme_ctrlr_delete(nvme_ctrlr);
3539 
3540 		if (ctx != NULL) {
3541 			populate_namespaces_cb(ctx, 0, -1);
3542 		}
3543 		return;
3544 	}
3545 
3546 	nvme_ctrlr_create_done(nvme_ctrlr, ctx);
3547 }
3548 
3549 static int
3550 nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr,
3551 			     struct nvme_async_probe_ctx *ctx)
3552 {
3553 	struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
3554 	const struct spdk_nvme_ctrlr_data *cdata;
3555 	uint32_t ana_log_page_size;
3556 
3557 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3558 
3559 	ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid *
3560 			    sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->nn *
3561 			    sizeof(uint32_t);
3562 
3563 	nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL,
3564 						SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
3565 	if (nvme_ctrlr->ana_log_page == NULL) {
3566 		SPDK_ERRLOG("could not allocate ANA log page buffer\n");
3567 		return -ENXIO;
3568 	}
3569 
3570 	/* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned.
3571 	 * Hence copy each descriptor to a temporary area when parsing it.
3572 	 *
3573 	 * Allocate a buffer whose size is as large as ANA log page buffer because
3574 	 * we do not know the size of a descriptor until actually reading it.
3575 	 */
3576 	nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size);
3577 	if (nvme_ctrlr->copied_ana_desc == NULL) {
3578 		SPDK_ERRLOG("could not allocate a buffer to parse ANA descriptor\n");
3579 		return -ENOMEM;
3580 	}
3581 
3582 	nvme_ctrlr->ana_log_page_size = ana_log_page_size;
3583 
3584 	nvme_ctrlr->probe_ctx = ctx;
3585 
3586 	return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr,
3587 						SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
3588 						SPDK_NVME_GLOBAL_NS_TAG,
3589 						nvme_ctrlr->ana_log_page,
3590 						nvme_ctrlr->ana_log_page_size, 0,
3591 						nvme_ctrlr_init_ana_log_page_done,
3592 						nvme_ctrlr);
3593 }
3594 
3595 /* hostnqn and subnqn were already verified before attaching a controller.
3596  * Hence check only the multipath capability and cntlid here.
3597  */
3598 static bool
3599 bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr)
3600 {
3601 	struct nvme_ctrlr *tmp;
3602 	const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata;
3603 
3604 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3605 
3606 	if (!cdata->cmic.multi_ctrlr) {
3607 		SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid);
3608 		return false;
3609 	}
3610 
3611 	TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) {
3612 		tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr);
3613 
3614 		if (!tmp_cdata->cmic.multi_ctrlr) {
3615 			SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid);
3616 			return false;
3617 		}
3618 		if (cdata->cntlid == tmp_cdata->cntlid) {
3619 			SPDK_ERRLOG("cntlid %u are duplicated.\n", tmp_cdata->cntlid);
3620 			return false;
3621 		}
3622 	}
3623 
3624 	return true;
3625 }
3626 
3627 static int
3628 nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr)
3629 {
3630 	struct nvme_bdev_ctrlr *nbdev_ctrlr;
3631 	struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
3632 	int rc = 0;
3633 
3634 	pthread_mutex_lock(&g_bdev_nvme_mutex);
3635 
3636 	nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
3637 	if (nbdev_ctrlr != NULL) {
3638 		if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) {
3639 			rc = -EINVAL;
3640 			goto exit;
3641 		}
3642 	} else {
3643 		nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr));
3644 		if (nbdev_ctrlr == NULL) {
3645 			SPDK_ERRLOG("Failed to allocate nvme_bdev_ctrlr.\n");
3646 			rc = -ENOMEM;
3647 			goto exit;
3648 		}
3649 		nbdev_ctrlr->name = strdup(name);
3650 		if (nbdev_ctrlr->name == NULL) {
3651 			SPDK_ERRLOG("Failed to allocate name of nvme_bdev_ctrlr.\n");
3652 			free(nbdev_ctrlr);
3653 			goto exit;
3654 		}
3655 		TAILQ_INIT(&nbdev_ctrlr->ctrlrs);
3656 		TAILQ_INIT(&nbdev_ctrlr->bdevs);
3657 		TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq);
3658 	}
3659 	nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr;
3660 	TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq);
3661 exit:
3662 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
3663 	return rc;
3664 }
3665 
3666 static int
3667 nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr,
3668 		  const char *name,
3669 		  const struct spdk_nvme_transport_id *trid,
3670 		  struct nvme_async_probe_ctx *ctx)
3671 {
3672 	struct nvme_ctrlr *nvme_ctrlr;
3673 	struct nvme_path_id *path_id;
3674 	const struct spdk_nvme_ctrlr_data *cdata;
3675 	int rc;
3676 
3677 	nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr));
3678 	if (nvme_ctrlr == NULL) {
3679 		SPDK_ERRLOG("Failed to allocate device struct\n");
3680 		return -ENOMEM;
3681 	}
3682 
3683 	rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL);
3684 	if (rc != 0) {
3685 		free(nvme_ctrlr);
3686 		return rc;
3687 	}
3688 
3689 	TAILQ_INIT(&nvme_ctrlr->trids);
3690 
3691 	RB_INIT(&nvme_ctrlr->namespaces);
3692 
3693 	path_id = calloc(1, sizeof(*path_id));
3694 	if (path_id == NULL) {
3695 		SPDK_ERRLOG("Failed to allocate trid entry pointer\n");
3696 		rc = -ENOMEM;
3697 		goto err;
3698 	}
3699 
3700 	path_id->trid = *trid;
3701 	if (ctx != NULL) {
3702 		memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr));
3703 		memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid));
3704 	}
3705 	nvme_ctrlr->active_path_id = path_id;
3706 	TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link);
3707 
3708 	nvme_ctrlr->thread = spdk_get_thread();
3709 	nvme_ctrlr->ctrlr = ctrlr;
3710 	nvme_ctrlr->ref = 1;
3711 
3712 	if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) {
3713 		SPDK_ERRLOG("OCSSDs are not supported");
3714 		rc = -ENOTSUP;
3715 		goto err;
3716 	}
3717 
3718 	if (ctx != NULL) {
3719 		memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts));
3720 	} else {
3721 		bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts);
3722 	}
3723 
3724 	nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr,
3725 					  g_opts.nvme_adminq_poll_period_us);
3726 
3727 	if (g_opts.timeout_us > 0) {
3728 		/* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */
3729 		/* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */
3730 		uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ?
3731 					  g_opts.timeout_us : g_opts.timeout_admin_us;
3732 		spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us,
3733 				adm_timeout_us, timeout_cb, nvme_ctrlr);
3734 	}
3735 
3736 	spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr);
3737 	spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr);
3738 
3739 	if (spdk_nvme_ctrlr_get_flags(ctrlr) &
3740 	    SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
3741 		nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr);
3742 	}
3743 
3744 	rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr);
3745 	if (rc != 0) {
3746 		goto err;
3747 	}
3748 
3749 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3750 
3751 	if (cdata->cmic.ana_reporting) {
3752 		rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx);
3753 		if (rc == 0) {
3754 			return 0;
3755 		}
3756 	} else {
3757 		nvme_ctrlr_create_done(nvme_ctrlr, ctx);
3758 		return 0;
3759 	}
3760 
3761 err:
3762 	nvme_ctrlr_delete(nvme_ctrlr);
3763 	return rc;
3764 }
3765 
3766 void
3767 bdev_nvme_get_default_ctrlr_opts(struct nvme_ctrlr_opts *opts)
3768 {
3769 	opts->prchk_flags = 0;
3770 	opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec;
3771 	opts->reconnect_delay_sec = g_opts.reconnect_delay_sec;
3772 	opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec;
3773 }
3774 
3775 static void
3776 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
3777 	  struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts)
3778 {
3779 	char *name;
3780 
3781 	name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++);
3782 	if (!name) {
3783 		SPDK_ERRLOG("Failed to assign name to NVMe device\n");
3784 		return;
3785 	}
3786 
3787 	SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name);
3788 
3789 	nvme_ctrlr_create(ctrlr, name, trid, NULL);
3790 
3791 	free(name);
3792 }
3793 
3794 static void
3795 _nvme_ctrlr_destruct(void *ctx)
3796 {
3797 	struct nvme_ctrlr *nvme_ctrlr = ctx;
3798 
3799 	nvme_ctrlr_depopulate_namespaces(nvme_ctrlr);
3800 	nvme_ctrlr_release(nvme_ctrlr);
3801 }
3802 
3803 static int
3804 _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, bool hotplug)
3805 {
3806 	struct nvme_probe_skip_entry *entry;
3807 
3808 	pthread_mutex_lock(&nvme_ctrlr->mutex);
3809 
3810 	/* The controller's destruction was already started */
3811 	if (nvme_ctrlr->destruct) {
3812 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
3813 		return 0;
3814 	}
3815 
3816 	if (!hotplug &&
3817 	    nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
3818 		entry = calloc(1, sizeof(*entry));
3819 		if (!entry) {
3820 			pthread_mutex_unlock(&nvme_ctrlr->mutex);
3821 			return -ENOMEM;
3822 		}
3823 		entry->trid = nvme_ctrlr->active_path_id->trid;
3824 		TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq);
3825 	}
3826 
3827 	nvme_ctrlr->destruct = true;
3828 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
3829 
3830 	_nvme_ctrlr_destruct(nvme_ctrlr);
3831 
3832 	return 0;
3833 }
3834 
3835 static void
3836 remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr)
3837 {
3838 	struct nvme_ctrlr *nvme_ctrlr = cb_ctx;
3839 
3840 	_bdev_nvme_delete(nvme_ctrlr, true);
3841 }
3842 
3843 static int
3844 bdev_nvme_hotplug_probe(void *arg)
3845 {
3846 	if (g_hotplug_probe_ctx == NULL) {
3847 		spdk_poller_unregister(&g_hotplug_probe_poller);
3848 		return SPDK_POLLER_IDLE;
3849 	}
3850 
3851 	if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) {
3852 		g_hotplug_probe_ctx = NULL;
3853 		spdk_poller_unregister(&g_hotplug_probe_poller);
3854 	}
3855 
3856 	return SPDK_POLLER_BUSY;
3857 }
3858 
3859 static int
3860 bdev_nvme_hotplug(void *arg)
3861 {
3862 	struct spdk_nvme_transport_id trid_pcie;
3863 
3864 	if (g_hotplug_probe_ctx) {
3865 		return SPDK_POLLER_BUSY;
3866 	}
3867 
3868 	memset(&trid_pcie, 0, sizeof(trid_pcie));
3869 	spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE);
3870 
3871 	g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL,
3872 			      hotplug_probe_cb, attach_cb, NULL);
3873 
3874 	if (g_hotplug_probe_ctx) {
3875 		assert(g_hotplug_probe_poller == NULL);
3876 		g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000);
3877 	}
3878 
3879 	return SPDK_POLLER_BUSY;
3880 }
3881 
3882 void
3883 bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts)
3884 {
3885 	*opts = g_opts;
3886 }
3887 
3888 static bool bdev_nvme_check_multipath_params(int32_t ctrlr_loss_timeout_sec,
3889 		uint32_t reconnect_delay_sec,
3890 		uint32_t fast_io_fail_timeout_sec);
3891 
3892 static int
3893 bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts)
3894 {
3895 	if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) {
3896 		/* Can't set timeout_admin_us without also setting timeout_us */
3897 		SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n");
3898 		return -EINVAL;
3899 	}
3900 
3901 	if (opts->bdev_retry_count < -1) {
3902 		SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n");
3903 		return -EINVAL;
3904 	}
3905 
3906 	if (!bdev_nvme_check_multipath_params(opts->ctrlr_loss_timeout_sec,
3907 					      opts->reconnect_delay_sec,
3908 					      opts->fast_io_fail_timeout_sec)) {
3909 		return -EINVAL;
3910 	}
3911 
3912 	return 0;
3913 }
3914 
3915 int
3916 bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts)
3917 {
3918 	int ret = bdev_nvme_validate_opts(opts);
3919 	if (ret) {
3920 		SPDK_WARNLOG("Failed to set nvme opts.\n");
3921 		return ret;
3922 	}
3923 
3924 	if (g_bdev_nvme_init_thread != NULL) {
3925 		if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
3926 			return -EPERM;
3927 		}
3928 	}
3929 
3930 	g_opts = *opts;
3931 
3932 	return 0;
3933 }
3934 
3935 struct set_nvme_hotplug_ctx {
3936 	uint64_t period_us;
3937 	bool enabled;
3938 	spdk_msg_fn fn;
3939 	void *fn_ctx;
3940 };
3941 
3942 static void
3943 set_nvme_hotplug_period_cb(void *_ctx)
3944 {
3945 	struct set_nvme_hotplug_ctx *ctx = _ctx;
3946 
3947 	spdk_poller_unregister(&g_hotplug_poller);
3948 	if (ctx->enabled) {
3949 		g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us);
3950 	}
3951 
3952 	g_nvme_hotplug_poll_period_us = ctx->period_us;
3953 	g_nvme_hotplug_enabled = ctx->enabled;
3954 	if (ctx->fn) {
3955 		ctx->fn(ctx->fn_ctx);
3956 	}
3957 
3958 	free(ctx);
3959 }
3960 
3961 int
3962 bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx)
3963 {
3964 	struct set_nvme_hotplug_ctx *ctx;
3965 
3966 	if (enabled == true && !spdk_process_is_primary()) {
3967 		return -EPERM;
3968 	}
3969 
3970 	ctx = calloc(1, sizeof(*ctx));
3971 	if (ctx == NULL) {
3972 		return -ENOMEM;
3973 	}
3974 
3975 	period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us;
3976 	ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX);
3977 	ctx->enabled = enabled;
3978 	ctx->fn = cb;
3979 	ctx->fn_ctx = cb_ctx;
3980 
3981 	spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx);
3982 	return 0;
3983 }
3984 
3985 static void
3986 nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
3987 				    struct nvme_async_probe_ctx *ctx)
3988 {
3989 	struct nvme_ns	*nvme_ns;
3990 	struct nvme_bdev	*nvme_bdev;
3991 	size_t			j;
3992 
3993 	assert(nvme_ctrlr != NULL);
3994 
3995 	if (ctx->names == NULL) {
3996 		populate_namespaces_cb(ctx, 0, 0);
3997 		return;
3998 	}
3999 
4000 	/*
4001 	 * Report the new bdevs that were created in this call.
4002 	 * There can be more than one bdev per NVMe controller.
4003 	 */
4004 	j = 0;
4005 	nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
4006 	while (nvme_ns != NULL) {
4007 		nvme_bdev = nvme_ns->bdev;
4008 		if (j < ctx->count) {
4009 			ctx->names[j] = nvme_bdev->disk.name;
4010 			j++;
4011 		} else {
4012 			SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n",
4013 				    ctx->count);
4014 			populate_namespaces_cb(ctx, 0, -ERANGE);
4015 			return;
4016 		}
4017 
4018 		nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
4019 	}
4020 
4021 	populate_namespaces_cb(ctx, j, 0);
4022 }
4023 
4024 static int
4025 bdev_nvme_compare_trids(struct nvme_ctrlr *nvme_ctrlr,
4026 			struct spdk_nvme_ctrlr *new_ctrlr,
4027 			struct spdk_nvme_transport_id *trid)
4028 {
4029 	struct nvme_path_id *tmp_trid;
4030 
4031 	if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
4032 		SPDK_ERRLOG("PCIe failover is not supported.\n");
4033 		return -ENOTSUP;
4034 	}
4035 
4036 	/* Currently we only support failover to the same transport type. */
4037 	if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) {
4038 		return -EINVAL;
4039 	}
4040 
4041 	/* Currently we only support failover to the same NQN. */
4042 	if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) {
4043 		return -EINVAL;
4044 	}
4045 
4046 	/* Skip all the other checks if we've already registered this path. */
4047 	TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) {
4048 		if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) {
4049 			return -EEXIST;
4050 		}
4051 	}
4052 
4053 	return 0;
4054 }
4055 
4056 static int
4057 bdev_nvme_compare_namespaces(struct nvme_ctrlr *nvme_ctrlr,
4058 			     struct spdk_nvme_ctrlr *new_ctrlr)
4059 {
4060 	struct nvme_ns *nvme_ns;
4061 	struct spdk_nvme_ns *new_ns;
4062 
4063 	nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
4064 	while (nvme_ns != NULL) {
4065 		new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id);
4066 		assert(new_ns != NULL);
4067 
4068 		if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) {
4069 			return -EINVAL;
4070 		}
4071 
4072 		nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
4073 	}
4074 
4075 	return 0;
4076 }
4077 
4078 static int
4079 _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
4080 			      struct spdk_nvme_transport_id *trid)
4081 {
4082 	struct nvme_path_id *new_trid, *tmp_trid;
4083 
4084 	new_trid = calloc(1, sizeof(*new_trid));
4085 	if (new_trid == NULL) {
4086 		return -ENOMEM;
4087 	}
4088 	new_trid->trid = *trid;
4089 	new_trid->is_failed = false;
4090 
4091 	TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) {
4092 		if (tmp_trid->is_failed && tmp_trid != nvme_ctrlr->active_path_id) {
4093 			TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link);
4094 			return 0;
4095 		}
4096 	}
4097 
4098 	TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link);
4099 	return 0;
4100 }
4101 
4102 /* This is the case that a secondary path is added to an existing
4103  * nvme_ctrlr for failover. After checking if it can access the same
4104  * namespaces as the primary path, it is disconnected until failover occurs.
4105  */
4106 static int
4107 bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
4108 			     struct spdk_nvme_ctrlr *new_ctrlr,
4109 			     struct spdk_nvme_transport_id *trid)
4110 {
4111 	int rc;
4112 
4113 	assert(nvme_ctrlr != NULL);
4114 
4115 	pthread_mutex_lock(&nvme_ctrlr->mutex);
4116 
4117 	rc = bdev_nvme_compare_trids(nvme_ctrlr, new_ctrlr, trid);
4118 	if (rc != 0) {
4119 		goto exit;
4120 	}
4121 
4122 	rc = bdev_nvme_compare_namespaces(nvme_ctrlr, new_ctrlr);
4123 	if (rc != 0) {
4124 		goto exit;
4125 	}
4126 
4127 	rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid);
4128 
4129 exit:
4130 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
4131 
4132 	spdk_nvme_detach(new_ctrlr);
4133 
4134 	return rc;
4135 }
4136 
4137 static void
4138 connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
4139 		  struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
4140 {
4141 	struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
4142 	struct nvme_async_probe_ctx *ctx;
4143 	int rc;
4144 
4145 	ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts);
4146 	ctx->ctrlr_attached = true;
4147 
4148 	rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx);
4149 	if (rc != 0) {
4150 		populate_namespaces_cb(ctx, 0, rc);
4151 	}
4152 }
4153 
4154 static void
4155 connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
4156 			struct spdk_nvme_ctrlr *ctrlr,
4157 			const struct spdk_nvme_ctrlr_opts *opts)
4158 {
4159 	struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
4160 	struct nvme_ctrlr *nvme_ctrlr;
4161 	struct nvme_async_probe_ctx *ctx;
4162 	int rc;
4163 
4164 	ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts);
4165 	ctx->ctrlr_attached = true;
4166 
4167 	nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name);
4168 	if (nvme_ctrlr) {
4169 		rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid);
4170 	} else {
4171 		rc = -ENODEV;
4172 	}
4173 
4174 	populate_namespaces_cb(ctx, 0, rc);
4175 }
4176 
4177 static int
4178 bdev_nvme_async_poll(void *arg)
4179 {
4180 	struct nvme_async_probe_ctx	*ctx = arg;
4181 	int				rc;
4182 
4183 	rc = spdk_nvme_probe_poll_async(ctx->probe_ctx);
4184 	if (spdk_unlikely(rc != -EAGAIN)) {
4185 		ctx->probe_done = true;
4186 		spdk_poller_unregister(&ctx->poller);
4187 		if (!ctx->ctrlr_attached) {
4188 			/* The probe is done, but no controller was attached.
4189 			 * That means we had a failure, so report -EIO back to
4190 			 * the caller (usually the RPC). populate_namespaces_cb()
4191 			 * will take care of freeing the nvme_async_probe_ctx.
4192 			 */
4193 			populate_namespaces_cb(ctx, 0, -EIO);
4194 		} else if (ctx->namespaces_populated) {
4195 			/* The namespaces for the attached controller were all
4196 			 * populated and the response was already sent to the
4197 			 * caller (usually the RPC).  So free the context here.
4198 			 */
4199 			free(ctx);
4200 		}
4201 	}
4202 
4203 	return SPDK_POLLER_BUSY;
4204 }
4205 
4206 static bool
4207 bdev_nvme_check_multipath_params(int32_t ctrlr_loss_timeout_sec,
4208 				 uint32_t reconnect_delay_sec,
4209 				 uint32_t fast_io_fail_timeout_sec)
4210 {
4211 	if (ctrlr_loss_timeout_sec < -1) {
4212 		SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n");
4213 		return false;
4214 	} else if (ctrlr_loss_timeout_sec == -1) {
4215 		if (reconnect_delay_sec == 0) {
4216 			SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n");
4217 			return false;
4218 		} else if (fast_io_fail_timeout_sec != 0 &&
4219 			   fast_io_fail_timeout_sec < reconnect_delay_sec) {
4220 			SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n");
4221 			return false;
4222 		}
4223 	} else if (ctrlr_loss_timeout_sec != 0) {
4224 		if (reconnect_delay_sec == 0) {
4225 			SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n");
4226 			return false;
4227 		} else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) {
4228 			SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n");
4229 			return false;
4230 		} else if (fast_io_fail_timeout_sec != 0) {
4231 			if (fast_io_fail_timeout_sec < reconnect_delay_sec) {
4232 				SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n");
4233 				return false;
4234 			} else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) {
4235 				SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n");
4236 				return false;
4237 			}
4238 		}
4239 	} else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) {
4240 		SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n");
4241 		return false;
4242 	}
4243 
4244 	return true;
4245 }
4246 
4247 int
4248 bdev_nvme_create(struct spdk_nvme_transport_id *trid,
4249 		 const char *base_name,
4250 		 const char **names,
4251 		 uint32_t count,
4252 		 spdk_bdev_create_nvme_fn cb_fn,
4253 		 void *cb_ctx,
4254 		 struct spdk_nvme_ctrlr_opts *drv_opts,
4255 		 struct nvme_ctrlr_opts *bdev_opts,
4256 		 bool multipath)
4257 {
4258 	struct nvme_probe_skip_entry	*entry, *tmp;
4259 	struct nvme_async_probe_ctx	*ctx;
4260 	spdk_nvme_attach_cb attach_cb;
4261 
4262 	/* TODO expand this check to include both the host and target TRIDs.
4263 	 * Only if both are the same should we fail.
4264 	 */
4265 	if (nvme_ctrlr_get(trid) != NULL) {
4266 		SPDK_ERRLOG("A controller with the provided trid (traddr: %s) already exists.\n", trid->traddr);
4267 		return -EEXIST;
4268 	}
4269 
4270 	if (bdev_opts != NULL &&
4271 	    !bdev_nvme_check_multipath_params(bdev_opts->ctrlr_loss_timeout_sec,
4272 					      bdev_opts->reconnect_delay_sec,
4273 					      bdev_opts->fast_io_fail_timeout_sec)) {
4274 		return -EINVAL;
4275 	}
4276 
4277 	ctx = calloc(1, sizeof(*ctx));
4278 	if (!ctx) {
4279 		return -ENOMEM;
4280 	}
4281 	ctx->base_name = base_name;
4282 	ctx->names = names;
4283 	ctx->count = count;
4284 	ctx->cb_fn = cb_fn;
4285 	ctx->cb_ctx = cb_ctx;
4286 	ctx->trid = *trid;
4287 
4288 	if (bdev_opts) {
4289 		memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));
4290 	} else {
4291 		bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts);
4292 	}
4293 
4294 	if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
4295 		TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) {
4296 			if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) {
4297 				TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq);
4298 				free(entry);
4299 				break;
4300 			}
4301 		}
4302 	}
4303 
4304 	if (drv_opts) {
4305 		memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts));
4306 	} else {
4307 		spdk_nvme_ctrlr_get_default_ctrlr_opts(&ctx->drv_opts, sizeof(ctx->drv_opts));
4308 	}
4309 
4310 	ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count;
4311 	ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout;
4312 	ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms;
4313 	ctx->drv_opts.disable_read_ana_log_page = true;
4314 
4315 	if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || multipath) {
4316 		attach_cb = connect_attach_cb;
4317 	} else {
4318 		attach_cb = connect_set_failover_cb;
4319 	}
4320 
4321 	ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb);
4322 	if (ctx->probe_ctx == NULL) {
4323 		SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr);
4324 		free(ctx);
4325 		return -ENODEV;
4326 	}
4327 	ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000);
4328 
4329 	return 0;
4330 }
4331 
4332 int
4333 bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id)
4334 {
4335 	struct nvme_bdev_ctrlr	*nbdev_ctrlr;
4336 	struct nvme_ctrlr	*nvme_ctrlr, *tmp_nvme_ctrlr;
4337 	struct nvme_path_id	*p, *t;
4338 	int			rc = -ENXIO;
4339 
4340 	if (name == NULL || path_id == NULL) {
4341 		return -EINVAL;
4342 	}
4343 
4344 	nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
4345 	if (nbdev_ctrlr == NULL) {
4346 		SPDK_ERRLOG("Failed to find NVMe bdev controller\n");
4347 		return -ENODEV;
4348 	}
4349 
4350 	TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) {
4351 		TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) {
4352 			if (path_id->trid.trtype != 0) {
4353 				if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) {
4354 					if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) {
4355 						continue;
4356 					}
4357 				} else {
4358 					if (path_id->trid.trtype != p->trid.trtype) {
4359 						continue;
4360 					}
4361 				}
4362 			}
4363 
4364 			if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) {
4365 				if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) {
4366 					continue;
4367 				}
4368 			}
4369 
4370 			if (path_id->trid.adrfam != 0) {
4371 				if (path_id->trid.adrfam != p->trid.adrfam) {
4372 					continue;
4373 				}
4374 			}
4375 
4376 			if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) {
4377 				if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) {
4378 					continue;
4379 				}
4380 			}
4381 
4382 			if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) {
4383 				if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) {
4384 					continue;
4385 				}
4386 			}
4387 
4388 			if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) {
4389 				if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) {
4390 					continue;
4391 				}
4392 			}
4393 
4394 			if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) {
4395 				if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) {
4396 					continue;
4397 				}
4398 			}
4399 
4400 			/* If we made it here, then this path is a match! Now we need to remove it. */
4401 			if (p == nvme_ctrlr->active_path_id) {
4402 				/* This is the active path in use right now. The active path is always the first in the list. */
4403 
4404 				if (!TAILQ_NEXT(p, link)) {
4405 					/* The current path is the only path. */
4406 					rc = _bdev_nvme_delete(nvme_ctrlr, false);
4407 				} else {
4408 					/* There is an alternative path. */
4409 					rc = bdev_nvme_failover(nvme_ctrlr, true);
4410 				}
4411 			} else {
4412 				/* We are not using the specified path. */
4413 				TAILQ_REMOVE(&nvme_ctrlr->trids, p, link);
4414 				free(p);
4415 				rc = 0;
4416 			}
4417 
4418 			if (rc < 0 && rc != -ENXIO) {
4419 				return rc;
4420 			}
4421 
4422 
4423 		}
4424 	}
4425 
4426 	/* All nvme_ctrlrs were deleted or no nvme_ctrlr which had the trid was found. */
4427 	return rc;
4428 }
4429 
4430 #define DISCOVERY_INFOLOG(ctx, format, ...) \
4431 	SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__);
4432 
4433 #define DISCOVERY_ERRLOG(ctx, format, ...) \
4434 	SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__);
4435 
4436 struct discovery_entry_ctx {
4437 	char						name[128];
4438 	struct spdk_nvme_transport_id			trid;
4439 	struct spdk_nvme_ctrlr_opts			drv_opts;
4440 	struct spdk_nvmf_discovery_log_page_entry	entry;
4441 	TAILQ_ENTRY(discovery_entry_ctx)		tailq;
4442 	struct discovery_ctx				*ctx;
4443 };
4444 
4445 struct discovery_ctx {
4446 	char					*name;
4447 	spdk_bdev_nvme_start_discovery_fn	start_cb_fn;
4448 	spdk_bdev_nvme_stop_discovery_fn	stop_cb_fn;
4449 	void					*cb_ctx;
4450 	struct spdk_nvme_probe_ctx		*probe_ctx;
4451 	struct spdk_nvme_detach_ctx		*detach_ctx;
4452 	struct spdk_nvme_ctrlr			*ctrlr;
4453 	struct spdk_nvme_transport_id		trid;
4454 	struct discovery_entry_ctx		*entry_ctx_in_use;
4455 	struct spdk_poller			*poller;
4456 	struct spdk_nvme_ctrlr_opts		drv_opts;
4457 	struct nvme_ctrlr_opts			bdev_opts;
4458 	struct spdk_nvmf_discovery_log_page	*log_page;
4459 	TAILQ_ENTRY(discovery_ctx)		tailq;
4460 	TAILQ_HEAD(, discovery_entry_ctx)	nvm_entry_ctxs;
4461 	TAILQ_HEAD(, discovery_entry_ctx)	discovery_entry_ctxs;
4462 	int					rc;
4463 	bool					wait_for_attach;
4464 	/* Denotes if a discovery is currently in progress for this context.
4465 	 * That includes connecting to newly discovered subsystems.  Used to
4466 	 * ensure we do not start a new discovery until an existing one is
4467 	 * complete.
4468 	 */
4469 	bool					in_progress;
4470 
4471 	/* Denotes if another discovery is needed after the one in progress
4472 	 * completes.  Set when we receive an AER completion while a discovery
4473 	 * is already in progress.
4474 	 */
4475 	bool					pending;
4476 
4477 	/* Signal to the discovery context poller that it should stop the
4478 	 * discovery service, including detaching from the current discovery
4479 	 * controller.
4480 	 */
4481 	bool					stop;
4482 
4483 	struct spdk_thread			*calling_thread;
4484 	uint32_t				index;
4485 	uint32_t				attach_in_progress;
4486 	char					*hostnqn;
4487 };
4488 
4489 TAILQ_HEAD(discovery_ctxs, discovery_ctx);
4490 static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs);
4491 
4492 static void get_discovery_log_page(struct discovery_ctx *ctx);
4493 
4494 static void
4495 free_discovery_ctx(struct discovery_ctx *ctx)
4496 {
4497 	free(ctx->hostnqn);
4498 	free(ctx->name);
4499 	free(ctx);
4500 }
4501 
4502 static void
4503 discovery_complete(struct discovery_ctx *ctx)
4504 {
4505 	ctx->in_progress = false;
4506 	if (ctx->pending) {
4507 		ctx->pending = false;
4508 		get_discovery_log_page(ctx);
4509 	}
4510 }
4511 
4512 static void
4513 build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid,
4514 			       struct spdk_nvmf_discovery_log_page_entry *entry)
4515 {
4516 	char *space;
4517 
4518 	trid->trtype = entry->trtype;
4519 	trid->adrfam = entry->adrfam;
4520 	memcpy(trid->traddr, entry->traddr, sizeof(trid->traddr));
4521 	memcpy(trid->trsvcid, entry->trsvcid, sizeof(trid->trsvcid));
4522 	memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn));
4523 
4524 	/* We want the traddr, trsvcid and subnqn fields to be NULL-terminated.
4525 	 * But the log page entries typically pad them with spaces, not zeroes.
4526 	 * So add a NULL terminator to each of these fields at the appropriate
4527 	 * location.
4528 	 */
4529 	space = strchr(trid->traddr, ' ');
4530 	if (space) {
4531 		*space = 0;
4532 	}
4533 	space = strchr(trid->trsvcid, ' ');
4534 	if (space) {
4535 		*space = 0;
4536 	}
4537 	space = strchr(trid->subnqn, ' ');
4538 	if (space) {
4539 		*space = 0;
4540 	}
4541 }
4542 
4543 static void
4544 discovery_remove_controllers(struct discovery_ctx *ctx)
4545 {
4546 	struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page;
4547 	struct discovery_entry_ctx *entry_ctx, *tmp;
4548 	struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry;
4549 	struct spdk_nvme_transport_id old_trid;
4550 	uint64_t numrec, i;
4551 	bool found;
4552 
4553 	numrec = from_le64(&log_page->numrec);
4554 	TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) {
4555 		found = false;
4556 		old_entry = &entry_ctx->entry;
4557 		build_trid_from_log_page_entry(&old_trid, old_entry);
4558 		for (i = 0; i < numrec; i++) {
4559 			new_entry = &log_page->entries[i];
4560 			if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) {
4561 				DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n",
4562 						  old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);
4563 				found = true;
4564 				break;
4565 			}
4566 		}
4567 		if (!found) {
4568 			struct nvme_path_id path = {};
4569 
4570 			DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n",
4571 					  old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);
4572 
4573 			path.trid = entry_ctx->trid;
4574 			bdev_nvme_delete(entry_ctx->name, &path);
4575 			TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);
4576 			free(entry_ctx);
4577 		}
4578 	}
4579 	free(log_page);
4580 	ctx->log_page = NULL;
4581 	discovery_complete(ctx);
4582 }
4583 
4584 static void
4585 discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc)
4586 {
4587 	struct discovery_entry_ctx *entry_ctx = cb_ctx;
4588 	struct discovery_ctx *ctx = entry_ctx->ctx;;
4589 
4590 	DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name);
4591 	ctx->attach_in_progress--;
4592 	if (ctx->attach_in_progress == 0) {
4593 		if (ctx->start_cb_fn) {
4594 			ctx->start_cb_fn(ctx->cb_ctx);
4595 			ctx->start_cb_fn = NULL;
4596 			ctx->cb_ctx = NULL;
4597 		}
4598 		discovery_remove_controllers(ctx);
4599 	}
4600 }
4601 
4602 static struct discovery_entry_ctx *
4603 create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid)
4604 {
4605 	struct discovery_entry_ctx *new_ctx;
4606 
4607 	new_ctx = calloc(1, sizeof(*new_ctx));
4608 	if (new_ctx == NULL) {
4609 		DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
4610 		return NULL;
4611 	}
4612 
4613 	new_ctx->ctx = ctx;
4614 	memcpy(&new_ctx->trid, trid, sizeof(*trid));
4615 	spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));
4616 	snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);
4617 	return new_ctx;
4618 }
4619 
4620 static void
4621 discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl,
4622 		      struct spdk_nvmf_discovery_log_page *log_page)
4623 {
4624 	struct discovery_ctx *ctx = cb_arg;
4625 	struct discovery_entry_ctx *entry_ctx, *tmp;
4626 	struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry;
4627 	uint64_t numrec, i;
4628 	bool found;
4629 
4630 	if (rc || spdk_nvme_cpl_is_error(cpl)) {
4631 		DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n");
4632 		return;
4633 	}
4634 
4635 	ctx->log_page = log_page;
4636 	assert(ctx->attach_in_progress == 0);
4637 	numrec = from_le64(&log_page->numrec);
4638 	TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) {
4639 		TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq);
4640 		free(entry_ctx);
4641 	}
4642 	for (i = 0; i < numrec; i++) {
4643 		found = false;
4644 		new_entry = &log_page->entries[i];
4645 		if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) {
4646 			struct discovery_entry_ctx *new_ctx;
4647 			struct spdk_nvme_transport_id trid;
4648 
4649 			build_trid_from_log_page_entry(&trid, new_entry);
4650 			new_ctx = create_discovery_entry_ctx(ctx, &trid);
4651 			if (new_ctx == NULL) {
4652 				DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
4653 				break;
4654 			}
4655 
4656 			TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq);
4657 			continue;
4658 		}
4659 		TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) {
4660 			old_entry = &entry_ctx->entry;
4661 			if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) {
4662 				found = true;
4663 				break;
4664 			}
4665 		}
4666 		if (!found) {
4667 			struct discovery_entry_ctx *subnqn_ctx, *new_ctx;
4668 
4669 			TAILQ_FOREACH(subnqn_ctx, &ctx->nvm_entry_ctxs, tailq) {
4670 				if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn,
4671 					    sizeof(new_entry->subnqn))) {
4672 					break;
4673 				}
4674 			}
4675 
4676 			new_ctx = calloc(1, sizeof(*new_ctx));
4677 			if (new_ctx == NULL) {
4678 				DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
4679 				break;
4680 			}
4681 
4682 			new_ctx->ctx = ctx;
4683 			memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry));
4684 			build_trid_from_log_page_entry(&new_ctx->trid, new_entry);
4685 			if (subnqn_ctx) {
4686 				snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name);
4687 				DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n",
4688 						  new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid,
4689 						  new_ctx->name);
4690 			} else {
4691 				snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++);
4692 				DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n",
4693 						  new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid,
4694 						  new_ctx->name);
4695 			}
4696 			spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));
4697 			snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);
4698 			rc = bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0,
4699 					      discovery_attach_controller_done, new_ctx,
4700 					      &new_ctx->drv_opts, &ctx->bdev_opts, true);
4701 			if (rc == 0) {
4702 				TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq);
4703 				ctx->attach_in_progress++;
4704 			} else {
4705 				DISCOVERY_ERRLOG(ctx, "bdev_nvme_create failed (%s)\n", spdk_strerror(-rc));
4706 			}
4707 		}
4708 	}
4709 
4710 	if (ctx->attach_in_progress == 0) {
4711 		discovery_remove_controllers(ctx);
4712 	}
4713 }
4714 
4715 static void
4716 get_discovery_log_page(struct discovery_ctx *ctx)
4717 {
4718 	int rc;
4719 
4720 	assert(ctx->in_progress == false);
4721 	ctx->in_progress = true;
4722 	rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx);
4723 	if (rc != 0) {
4724 		DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n");
4725 	}
4726 	DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n");
4727 }
4728 
4729 static void
4730 discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
4731 {
4732 	struct discovery_ctx *ctx = arg;
4733 	uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
4734 
4735 	if (spdk_nvme_cpl_is_error(cpl)) {
4736 		DISCOVERY_ERRLOG(ctx, "aer failed\n");
4737 		return;
4738 	}
4739 
4740 	if (log_page_id != SPDK_NVME_LOG_DISCOVERY) {
4741 		DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id);
4742 		return;
4743 	}
4744 
4745 	DISCOVERY_INFOLOG(ctx, "got aer\n");
4746 	if (ctx->in_progress) {
4747 		ctx->pending = true;
4748 		return;
4749 	}
4750 
4751 	get_discovery_log_page(ctx);
4752 }
4753 
4754 static void
4755 discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
4756 		    struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
4757 {
4758 	struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
4759 	struct discovery_ctx *ctx;
4760 
4761 	ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts);
4762 
4763 	DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n");
4764 	ctx->probe_ctx = NULL;
4765 	ctx->ctrlr = ctrlr;
4766 	spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx);
4767 }
4768 
4769 static int
4770 discovery_poller(void *arg)
4771 {
4772 	struct discovery_ctx *ctx = arg;
4773 	struct spdk_nvme_transport_id *trid;
4774 	int rc;
4775 
4776 	if (ctx->detach_ctx) {
4777 		rc = spdk_nvme_detach_poll_async(ctx->detach_ctx);
4778 		if (rc != -EAGAIN) {
4779 			ctx->detach_ctx = NULL;
4780 			ctx->ctrlr = NULL;
4781 		}
4782 	} else if (ctx->stop) {
4783 		if (ctx->ctrlr != NULL) {
4784 			rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx);
4785 			if (rc == 0) {
4786 				return SPDK_POLLER_BUSY;
4787 			}
4788 			DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n");
4789 		}
4790 		spdk_poller_unregister(&ctx->poller);
4791 		TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq);
4792 		ctx->stop_cb_fn(ctx->cb_ctx);
4793 		free_discovery_ctx(ctx);
4794 	} else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) {
4795 		assert(ctx->entry_ctx_in_use == NULL);
4796 		ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs);
4797 		TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
4798 		trid = &ctx->entry_ctx_in_use->trid;
4799 		ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb);
4800 		if (ctx->probe_ctx) {
4801 			spdk_poller_unregister(&ctx->poller);
4802 			ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000);
4803 		} else {
4804 			DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n");
4805 			TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
4806 			ctx->entry_ctx_in_use = NULL;
4807 		}
4808 	} else if (ctx->probe_ctx) {
4809 		rc = spdk_nvme_probe_poll_async(ctx->probe_ctx);
4810 		if (rc != -EAGAIN) {
4811 			DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n");
4812 			ctx->rc = rc;
4813 			if (rc == 0) {
4814 				get_discovery_log_page(ctx);
4815 			}
4816 		}
4817 	} else {
4818 		rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr);
4819 		if (rc < 0) {
4820 			spdk_poller_unregister(&ctx->poller);
4821 			ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000);
4822 			TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
4823 			ctx->entry_ctx_in_use = NULL;
4824 
4825 			rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx);
4826 			if (rc != 0) {
4827 				DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n");
4828 				ctx->ctrlr = NULL;
4829 			}
4830 		}
4831 	}
4832 
4833 	return SPDK_POLLER_BUSY;
4834 }
4835 
4836 static void
4837 start_discovery_poller(void *arg)
4838 {
4839 	struct discovery_ctx *ctx = arg;
4840 
4841 	TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq);
4842 	ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000);
4843 }
4844 
4845 int
4846 bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid,
4847 			  const char *base_name,
4848 			  struct spdk_nvme_ctrlr_opts *drv_opts,
4849 			  struct nvme_ctrlr_opts *bdev_opts,
4850 			  spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx)
4851 {
4852 	struct discovery_ctx *ctx;
4853 	struct discovery_entry_ctx *discovery_entry_ctx;
4854 
4855 	ctx = calloc(1, sizeof(*ctx));
4856 	if (ctx == NULL) {
4857 		return -ENOMEM;
4858 	}
4859 
4860 	ctx->name = strdup(base_name);
4861 	if (ctx->name == NULL) {
4862 		free_discovery_ctx(ctx);
4863 		return -ENOMEM;
4864 	}
4865 	memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts));
4866 	memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));
4867 	ctx->bdev_opts.from_discovery_service = true;
4868 	ctx->calling_thread = spdk_get_thread();
4869 	if (ctx->start_cb_fn) {
4870 		/* We can use this when dumping json to denote if this RPC parameter
4871 		 * was specified or not.
4872 		 */
4873 		ctx->wait_for_attach = true;
4874 	}
4875 	ctx->start_cb_fn = cb_fn;
4876 	ctx->cb_ctx = cb_ctx;
4877 	TAILQ_INIT(&ctx->nvm_entry_ctxs);
4878 	TAILQ_INIT(&ctx->discovery_entry_ctxs);
4879 	snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
4880 	memcpy(&ctx->trid, trid, sizeof(*trid));
4881 	/* Even if user did not specify hostnqn, we can still strdup("\0"); */
4882 	ctx->hostnqn = strdup(ctx->drv_opts.hostnqn);
4883 	if (ctx->hostnqn == NULL) {
4884 		free_discovery_ctx(ctx);
4885 		return -ENOMEM;
4886 	}
4887 	discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid);
4888 	if (discovery_entry_ctx == NULL) {
4889 		DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
4890 		free_discovery_ctx(ctx);
4891 		return -ENOMEM;
4892 	}
4893 
4894 	TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq);
4895 	spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx);
4896 	return 0;
4897 }
4898 
4899 int
4900 bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx)
4901 {
4902 	struct discovery_ctx *ctx;
4903 
4904 	TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
4905 		if (strcmp(name, ctx->name) == 0) {
4906 			if (ctx->stop) {
4907 				return -EALREADY;
4908 			}
4909 			ctx->stop = true;
4910 			ctx->stop_cb_fn = cb_fn;
4911 			ctx->cb_ctx = cb_ctx;
4912 			while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) {
4913 				struct discovery_entry_ctx *entry_ctx;
4914 				struct nvme_path_id path = {};
4915 
4916 				entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs);
4917 				path.trid = entry_ctx->trid;
4918 				bdev_nvme_delete(entry_ctx->name, &path);
4919 				TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);
4920 				free(entry_ctx);
4921 			}
4922 			while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) {
4923 				struct discovery_entry_ctx *entry_ctx;
4924 
4925 				entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs);
4926 				TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq);
4927 				free(entry_ctx);
4928 			}
4929 			return 0;
4930 		}
4931 	}
4932 
4933 	return -ENOENT;
4934 }
4935 
4936 static int
4937 bdev_nvme_library_init(void)
4938 {
4939 	g_bdev_nvme_init_thread = spdk_get_thread();
4940 
4941 	spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb,
4942 				bdev_nvme_destroy_poll_group_cb,
4943 				sizeof(struct nvme_poll_group),  "nvme_poll_groups");
4944 
4945 	return 0;
4946 }
4947 
4948 static void
4949 bdev_nvme_fini_destruct_ctrlrs(void)
4950 {
4951 	struct nvme_bdev_ctrlr *nbdev_ctrlr;
4952 	struct nvme_ctrlr *nvme_ctrlr;
4953 
4954 	pthread_mutex_lock(&g_bdev_nvme_mutex);
4955 	TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
4956 		TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
4957 			pthread_mutex_lock(&nvme_ctrlr->mutex);
4958 			if (nvme_ctrlr->destruct) {
4959 				/* This controller's destruction was already started
4960 				 * before the application started shutting down
4961 				 */
4962 				pthread_mutex_unlock(&nvme_ctrlr->mutex);
4963 				continue;
4964 			}
4965 			nvme_ctrlr->destruct = true;
4966 			pthread_mutex_unlock(&nvme_ctrlr->mutex);
4967 
4968 			spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct,
4969 					     nvme_ctrlr);
4970 		}
4971 	}
4972 
4973 	g_bdev_nvme_module_finish = true;
4974 	if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
4975 		pthread_mutex_unlock(&g_bdev_nvme_mutex);
4976 		spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL);
4977 		spdk_bdev_module_fini_done();
4978 		return;
4979 	}
4980 
4981 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
4982 }
4983 
4984 static void
4985 check_discovery_fini(void *arg)
4986 {
4987 	if (TAILQ_EMPTY(&g_discovery_ctxs)) {
4988 		bdev_nvme_fini_destruct_ctrlrs();
4989 	}
4990 }
4991 
4992 static void
4993 bdev_nvme_library_fini(void)
4994 {
4995 	struct nvme_probe_skip_entry *entry, *entry_tmp;
4996 	struct discovery_ctx *ctx;
4997 
4998 	spdk_poller_unregister(&g_hotplug_poller);
4999 	free(g_hotplug_probe_ctx);
5000 	g_hotplug_probe_ctx = NULL;
5001 
5002 	TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) {
5003 		TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq);
5004 		free(entry);
5005 	}
5006 
5007 	assert(spdk_get_thread() == g_bdev_nvme_init_thread);
5008 	if (TAILQ_EMPTY(&g_discovery_ctxs)) {
5009 		bdev_nvme_fini_destruct_ctrlrs();
5010 	} else {
5011 		TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
5012 			ctx->stop = true;
5013 			ctx->stop_cb_fn = check_discovery_fini;
5014 		}
5015 	}
5016 }
5017 
5018 static void
5019 bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio)
5020 {
5021 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
5022 	struct spdk_bdev *bdev = bdev_io->bdev;
5023 	struct spdk_dif_ctx dif_ctx;
5024 	struct spdk_dif_error err_blk = {};
5025 	int rc;
5026 
5027 	rc = spdk_dif_ctx_init(&dif_ctx,
5028 			       bdev->blocklen, bdev->md_len, bdev->md_interleave,
5029 			       bdev->dif_is_head_of_md, bdev->dif_type, bdev->dif_check_flags,
5030 			       bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0);
5031 	if (rc != 0) {
5032 		SPDK_ERRLOG("Initialization of DIF context failed\n");
5033 		return;
5034 	}
5035 
5036 	if (bdev->md_interleave) {
5037 		rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
5038 				     bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk);
5039 	} else {
5040 		struct iovec md_iov = {
5041 			.iov_base	= bdev_io->u.bdev.md_buf,
5042 			.iov_len	= bdev_io->u.bdev.num_blocks * bdev->md_len,
5043 		};
5044 
5045 		rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
5046 				     &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk);
5047 	}
5048 
5049 	if (rc != 0) {
5050 		SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n",
5051 			    err_blk.err_type, err_blk.err_offset);
5052 	} else {
5053 		SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n");
5054 	}
5055 }
5056 
5057 static void
5058 bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl)
5059 {
5060 	struct nvme_bdev_io *bio = ref;
5061 
5062 	if (spdk_nvme_cpl_is_success(cpl)) {
5063 		/* Run PI verification for read data buffer. */
5064 		bdev_nvme_verify_pi_error(bio);
5065 	}
5066 
5067 	/* Return original completion status */
5068 	bdev_nvme_io_complete_nvme_status(bio, &bio->cpl);
5069 }
5070 
5071 static void
5072 bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl)
5073 {
5074 	struct nvme_bdev_io *bio = ref;
5075 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
5076 	int ret;
5077 
5078 	if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) {
5079 		SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n",
5080 			    cpl->status.sct, cpl->status.sc);
5081 
5082 		/* Save completion status to use after verifying PI error. */
5083 		bio->cpl = *cpl;
5084 
5085 		if (spdk_likely(nvme_io_path_is_available(bio->io_path))) {
5086 			/* Read without PI checking to verify PI error. */
5087 			ret = bdev_nvme_no_pi_readv(bio,
5088 						    bdev_io->u.bdev.iovs,
5089 						    bdev_io->u.bdev.iovcnt,
5090 						    bdev_io->u.bdev.md_buf,
5091 						    bdev_io->u.bdev.num_blocks,
5092 						    bdev_io->u.bdev.offset_blocks);
5093 			if (ret == 0) {
5094 				return;
5095 			}
5096 		}
5097 	}
5098 
5099 	bdev_nvme_io_complete_nvme_status(bio, cpl);
5100 }
5101 
5102 static void
5103 bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl)
5104 {
5105 	struct nvme_bdev_io *bio = ref;
5106 
5107 	if (spdk_nvme_cpl_is_pi_error(cpl)) {
5108 		SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n",
5109 			    cpl->status.sct, cpl->status.sc);
5110 		/* Run PI verification for write data buffer if PI error is detected. */
5111 		bdev_nvme_verify_pi_error(bio);
5112 	}
5113 
5114 	bdev_nvme_io_complete_nvme_status(bio, cpl);
5115 }
5116 
5117 static void
5118 bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl)
5119 {
5120 	struct nvme_bdev_io *bio = ref;
5121 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
5122 
5123 	/* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks.
5124 	 * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error().
5125 	 */
5126 	bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0;
5127 
5128 	if (spdk_nvme_cpl_is_pi_error(cpl)) {
5129 		SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n",
5130 			    cpl->status.sct, cpl->status.sc);
5131 		/* Run PI verification for zone append data buffer if PI error is detected. */
5132 		bdev_nvme_verify_pi_error(bio);
5133 	}
5134 
5135 	bdev_nvme_io_complete_nvme_status(bio, cpl);
5136 }
5137 
5138 static void
5139 bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl)
5140 {
5141 	struct nvme_bdev_io *bio = ref;
5142 
5143 	if (spdk_nvme_cpl_is_pi_error(cpl)) {
5144 		SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n",
5145 			    cpl->status.sct, cpl->status.sc);
5146 		/* Run PI verification for compare data buffer if PI error is detected. */
5147 		bdev_nvme_verify_pi_error(bio);
5148 	}
5149 
5150 	bdev_nvme_io_complete_nvme_status(bio, cpl);
5151 }
5152 
5153 static void
5154 bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl)
5155 {
5156 	struct nvme_bdev_io *bio = ref;
5157 
5158 	/* Compare operation completion */
5159 	if (!bio->first_fused_completed) {
5160 		/* Save compare result for write callback */
5161 		bio->cpl = *cpl;
5162 		bio->first_fused_completed = true;
5163 		return;
5164 	}
5165 
5166 	/* Write operation completion */
5167 	if (spdk_nvme_cpl_is_error(&bio->cpl)) {
5168 		/* If bio->cpl is already an error, it means the compare operation failed.  In that case,
5169 		 * complete the IO with the compare operation's status.
5170 		 */
5171 		if (!spdk_nvme_cpl_is_error(cpl)) {
5172 			SPDK_ERRLOG("Unexpected write success after compare failure.\n");
5173 		}
5174 
5175 		bdev_nvme_io_complete_nvme_status(bio, &bio->cpl);
5176 	} else {
5177 		bdev_nvme_io_complete_nvme_status(bio, cpl);
5178 	}
5179 }
5180 
5181 static void
5182 bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl)
5183 {
5184 	struct nvme_bdev_io *bio = ref;
5185 
5186 	bdev_nvme_io_complete_nvme_status(bio, cpl);
5187 }
5188 
5189 static int
5190 fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc)
5191 {
5192 	switch (desc->zs) {
5193 	case SPDK_NVME_ZONE_STATE_EMPTY:
5194 		info->state = SPDK_BDEV_ZONE_STATE_EMPTY;
5195 		break;
5196 	case SPDK_NVME_ZONE_STATE_IOPEN:
5197 		info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN;
5198 		break;
5199 	case SPDK_NVME_ZONE_STATE_EOPEN:
5200 		info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN;
5201 		break;
5202 	case SPDK_NVME_ZONE_STATE_CLOSED:
5203 		info->state = SPDK_BDEV_ZONE_STATE_CLOSED;
5204 		break;
5205 	case SPDK_NVME_ZONE_STATE_RONLY:
5206 		info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY;
5207 		break;
5208 	case SPDK_NVME_ZONE_STATE_FULL:
5209 		info->state = SPDK_BDEV_ZONE_STATE_FULL;
5210 		break;
5211 	case SPDK_NVME_ZONE_STATE_OFFLINE:
5212 		info->state = SPDK_BDEV_ZONE_STATE_OFFLINE;
5213 		break;
5214 	default:
5215 		SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs);
5216 		return -EIO;
5217 	}
5218 
5219 	info->zone_id = desc->zslba;
5220 	info->write_pointer = desc->wp;
5221 	info->capacity = desc->zcap;
5222 
5223 	return 0;
5224 }
5225 
5226 static void
5227 bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl)
5228 {
5229 	struct nvme_bdev_io *bio = ref;
5230 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
5231 	uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id;
5232 	uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones;
5233 	struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf;
5234 	uint64_t max_zones_per_buf, i;
5235 	uint32_t zone_report_bufsize;
5236 	struct spdk_nvme_ns *ns;
5237 	struct spdk_nvme_qpair *qpair;
5238 	int ret;
5239 
5240 	if (spdk_nvme_cpl_is_error(cpl)) {
5241 		goto out_complete_io_nvme_cpl;
5242 	}
5243 
5244 	if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) {
5245 		ret = -ENXIO;
5246 		goto out_complete_io_ret;
5247 	}
5248 
5249 	ns = bio->io_path->nvme_ns->ns;
5250 	qpair = bio->io_path->qpair->qpair;
5251 
5252 	zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns);
5253 	max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) /
5254 			    sizeof(bio->zone_report_buf->descs[0]);
5255 
5256 	if (bio->zone_report_buf->nr_zones > max_zones_per_buf) {
5257 		ret = -EINVAL;
5258 		goto out_complete_io_ret;
5259 	}
5260 
5261 	if (!bio->zone_report_buf->nr_zones) {
5262 		ret = -EINVAL;
5263 		goto out_complete_io_ret;
5264 	}
5265 
5266 	for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) {
5267 		ret = fill_zone_from_report(&info[bio->handled_zones],
5268 					    &bio->zone_report_buf->descs[i]);
5269 		if (ret) {
5270 			goto out_complete_io_ret;
5271 		}
5272 		bio->handled_zones++;
5273 	}
5274 
5275 	if (bio->handled_zones < zones_to_copy) {
5276 		uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
5277 		uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones);
5278 
5279 		memset(bio->zone_report_buf, 0, zone_report_bufsize);
5280 		ret = spdk_nvme_zns_report_zones(ns, qpair,
5281 						 bio->zone_report_buf, zone_report_bufsize,
5282 						 slba, SPDK_NVME_ZRA_LIST_ALL, true,
5283 						 bdev_nvme_get_zone_info_done, bio);
5284 		if (!ret) {
5285 			return;
5286 		} else {
5287 			goto out_complete_io_ret;
5288 		}
5289 	}
5290 
5291 out_complete_io_nvme_cpl:
5292 	free(bio->zone_report_buf);
5293 	bio->zone_report_buf = NULL;
5294 	bdev_nvme_io_complete_nvme_status(bio, cpl);
5295 	return;
5296 
5297 out_complete_io_ret:
5298 	free(bio->zone_report_buf);
5299 	bio->zone_report_buf = NULL;
5300 	bdev_nvme_io_complete(bio, ret);
5301 }
5302 
5303 static void
5304 bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl)
5305 {
5306 	struct nvme_bdev_io *bio = ref;
5307 
5308 	bdev_nvme_io_complete_nvme_status(bio, cpl);
5309 }
5310 
5311 static void
5312 bdev_nvme_admin_passthru_complete_nvme_status(void *ctx)
5313 {
5314 	struct nvme_bdev_io *bio = ctx;
5315 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
5316 	const struct spdk_nvme_cpl *cpl = &bio->cpl;
5317 	struct nvme_bdev_channel *nbdev_ch;
5318 	struct nvme_ctrlr *nvme_ctrlr;
5319 	const struct spdk_nvme_ctrlr_data *cdata;
5320 	uint64_t delay_ms;
5321 
5322 	assert(bdev_nvme_io_type_is_admin(bdev_io->type));
5323 
5324 	if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) {
5325 		goto complete;
5326 	}
5327 
5328 	if (cpl->status.dnr != 0 || (g_opts.bdev_retry_count != -1 &&
5329 				     bio->retry_count >= g_opts.bdev_retry_count)) {
5330 		goto complete;
5331 	}
5332 
5333 	nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
5334 	nvme_ctrlr = bio->io_path->qpair->ctrlr;
5335 
5336 	if (spdk_nvme_cpl_is_path_error(cpl) ||
5337 	    spdk_nvme_cpl_is_aborted_sq_deletion(cpl) ||
5338 	    !nvme_ctrlr_is_available(nvme_ctrlr)) {
5339 		delay_ms = 0;
5340 	} else if (spdk_nvme_cpl_is_aborted_by_request(cpl)) {
5341 		goto complete;
5342 	} else {
5343 		bio->retry_count++;
5344 
5345 		cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
5346 
5347 		if (cpl->status.crd != 0) {
5348 			delay_ms = cdata->crdt[cpl->status.crd] * 100;
5349 		} else {
5350 			delay_ms = 0;
5351 		}
5352 	}
5353 
5354 	if (any_ctrlr_may_become_available(nbdev_ch)) {
5355 		bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms);
5356 		return;
5357 	}
5358 
5359 complete:
5360 	bio->retry_count = 0;
5361 	spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc);
5362 }
5363 
5364 static void
5365 bdev_nvme_abort_complete(void *ctx)
5366 {
5367 	struct nvme_bdev_io *bio = ctx;
5368 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
5369 
5370 	if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) {
5371 		spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS);
5372 	} else {
5373 		spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
5374 	}
5375 }
5376 
5377 static void
5378 bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl)
5379 {
5380 	struct nvme_bdev_io *bio = ref;
5381 
5382 	bio->cpl = *cpl;
5383 	spdk_thread_send_msg(bio->orig_thread, bdev_nvme_abort_complete, bio);
5384 }
5385 
5386 static void
5387 bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl)
5388 {
5389 	struct nvme_bdev_io *bio = ref;
5390 
5391 	bio->cpl = *cpl;
5392 	spdk_thread_send_msg(bio->orig_thread,
5393 			     bdev_nvme_admin_passthru_complete_nvme_status, bio);
5394 }
5395 
5396 static void
5397 bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset)
5398 {
5399 	struct nvme_bdev_io *bio = ref;
5400 	struct iovec *iov;
5401 
5402 	bio->iov_offset = sgl_offset;
5403 	for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) {
5404 		iov = &bio->iovs[bio->iovpos];
5405 		if (bio->iov_offset < iov->iov_len) {
5406 			break;
5407 		}
5408 
5409 		bio->iov_offset -= iov->iov_len;
5410 	}
5411 }
5412 
5413 static int
5414 bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length)
5415 {
5416 	struct nvme_bdev_io *bio = ref;
5417 	struct iovec *iov;
5418 
5419 	assert(bio->iovpos < bio->iovcnt);
5420 
5421 	iov = &bio->iovs[bio->iovpos];
5422 
5423 	*address = iov->iov_base;
5424 	*length = iov->iov_len;
5425 
5426 	if (bio->iov_offset) {
5427 		assert(bio->iov_offset <= iov->iov_len);
5428 		*address += bio->iov_offset;
5429 		*length -= bio->iov_offset;
5430 	}
5431 
5432 	bio->iov_offset += *length;
5433 	if (bio->iov_offset == iov->iov_len) {
5434 		bio->iovpos++;
5435 		bio->iov_offset = 0;
5436 	}
5437 
5438 	return 0;
5439 }
5440 
5441 static void
5442 bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset)
5443 {
5444 	struct nvme_bdev_io *bio = ref;
5445 	struct iovec *iov;
5446 
5447 	bio->fused_iov_offset = sgl_offset;
5448 	for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) {
5449 		iov = &bio->fused_iovs[bio->fused_iovpos];
5450 		if (bio->fused_iov_offset < iov->iov_len) {
5451 			break;
5452 		}
5453 
5454 		bio->fused_iov_offset -= iov->iov_len;
5455 	}
5456 }
5457 
5458 static int
5459 bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length)
5460 {
5461 	struct nvme_bdev_io *bio = ref;
5462 	struct iovec *iov;
5463 
5464 	assert(bio->fused_iovpos < bio->fused_iovcnt);
5465 
5466 	iov = &bio->fused_iovs[bio->fused_iovpos];
5467 
5468 	*address = iov->iov_base;
5469 	*length = iov->iov_len;
5470 
5471 	if (bio->fused_iov_offset) {
5472 		assert(bio->fused_iov_offset <= iov->iov_len);
5473 		*address += bio->fused_iov_offset;
5474 		*length -= bio->fused_iov_offset;
5475 	}
5476 
5477 	bio->fused_iov_offset += *length;
5478 	if (bio->fused_iov_offset == iov->iov_len) {
5479 		bio->fused_iovpos++;
5480 		bio->fused_iov_offset = 0;
5481 	}
5482 
5483 	return 0;
5484 }
5485 
5486 static int
5487 bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
5488 		      void *md, uint64_t lba_count, uint64_t lba)
5489 {
5490 	int rc;
5491 
5492 	SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n",
5493 		      lba_count, lba);
5494 
5495 	bio->iovs = iov;
5496 	bio->iovcnt = iovcnt;
5497 	bio->iovpos = 0;
5498 	bio->iov_offset = 0;
5499 
5500 	rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns,
5501 					    bio->io_path->qpair->qpair,
5502 					    lba, lba_count,
5503 					    bdev_nvme_no_pi_readv_done, bio, 0,
5504 					    bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
5505 					    md, 0, 0);
5506 
5507 	if (rc != 0 && rc != -ENOMEM) {
5508 		SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc);
5509 	}
5510 	return rc;
5511 }
5512 
5513 static int
5514 bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
5515 		void *md, uint64_t lba_count, uint64_t lba, uint32_t flags,
5516 		struct spdk_bdev_ext_io_opts *ext_opts)
5517 {
5518 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
5519 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
5520 	int rc;
5521 
5522 	SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n",
5523 		      lba_count, lba);
5524 
5525 	bio->iovs = iov;
5526 	bio->iovcnt = iovcnt;
5527 	bio->iovpos = 0;
5528 	bio->iov_offset = 0;
5529 
5530 	if (ext_opts) {
5531 		bio->ext_opts.size = sizeof(struct spdk_nvme_ns_cmd_ext_io_opts);
5532 		bio->ext_opts.memory_domain = ext_opts->memory_domain;
5533 		bio->ext_opts.memory_domain_ctx = ext_opts->memory_domain_ctx;
5534 		bio->ext_opts.io_flags = flags;
5535 		bio->ext_opts.metadata = md;
5536 
5537 		rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count,
5538 						bdev_nvme_readv_done, bio,
5539 						bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
5540 						&bio->ext_opts);
5541 	} else if (iovcnt == 1) {
5542 		rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, iov[0].iov_base, md, lba,
5543 						   lba_count,
5544 						   bdev_nvme_readv_done, bio,
5545 						   flags,
5546 						   0, 0);
5547 	} else {
5548 		rc = spdk_nvme_ns_cmd_readv_with_md(ns, qpair, lba, lba_count,
5549 						    bdev_nvme_readv_done, bio, flags,
5550 						    bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
5551 						    md, 0, 0);
5552 	}
5553 
5554 	if (rc != 0 && rc != -ENOMEM) {
5555 		SPDK_ERRLOG("readv failed: rc = %d\n", rc);
5556 	}
5557 	return rc;
5558 }
5559 
5560 static int
5561 bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
5562 		 void *md, uint64_t lba_count, uint64_t lba,
5563 		 uint32_t flags, struct spdk_bdev_ext_io_opts *ext_opts)
5564 {
5565 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
5566 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
5567 	int rc;
5568 
5569 	SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n",
5570 		      lba_count, lba);
5571 
5572 	bio->iovs = iov;
5573 	bio->iovcnt = iovcnt;
5574 	bio->iovpos = 0;
5575 	bio->iov_offset = 0;
5576 
5577 	if (ext_opts) {
5578 		bio->ext_opts.size = sizeof(struct spdk_nvme_ns_cmd_ext_io_opts);
5579 		bio->ext_opts.memory_domain = ext_opts->memory_domain;
5580 		bio->ext_opts.memory_domain_ctx = ext_opts->memory_domain_ctx;
5581 		bio->ext_opts.io_flags = flags;
5582 		bio->ext_opts.metadata = md;
5583 
5584 		rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count,
5585 						 bdev_nvme_writev_done, bio,
5586 						 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
5587 						 &bio->ext_opts);
5588 	} else if (iovcnt == 1) {
5589 		rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, iov[0].iov_base, md, lba,
5590 						    lba_count,
5591 						    bdev_nvme_writev_done, bio,
5592 						    flags,
5593 						    0, 0);
5594 	} else {
5595 		rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count,
5596 						     bdev_nvme_writev_done, bio, flags,
5597 						     bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
5598 						     md, 0, 0);
5599 	}
5600 
5601 	if (rc != 0 && rc != -ENOMEM) {
5602 		SPDK_ERRLOG("writev failed: rc = %d\n", rc);
5603 	}
5604 	return rc;
5605 }
5606 
5607 static int
5608 bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
5609 		       void *md, uint64_t lba_count, uint64_t zslba,
5610 		       uint32_t flags)
5611 {
5612 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
5613 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
5614 	int rc;
5615 
5616 	SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n",
5617 		      lba_count, zslba);
5618 
5619 	bio->iovs = iov;
5620 	bio->iovcnt = iovcnt;
5621 	bio->iovpos = 0;
5622 	bio->iov_offset = 0;
5623 
5624 	if (iovcnt == 1) {
5625 		rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba,
5626 						       lba_count,
5627 						       bdev_nvme_zone_appendv_done, bio,
5628 						       flags,
5629 						       0, 0);
5630 	} else {
5631 		rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count,
5632 							bdev_nvme_zone_appendv_done, bio, flags,
5633 							bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
5634 							md, 0, 0);
5635 	}
5636 
5637 	if (rc != 0 && rc != -ENOMEM) {
5638 		SPDK_ERRLOG("zone append failed: rc = %d\n", rc);
5639 	}
5640 	return rc;
5641 }
5642 
5643 static int
5644 bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
5645 		   void *md, uint64_t lba_count, uint64_t lba,
5646 		   uint32_t flags)
5647 {
5648 	int rc;
5649 
5650 	SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n",
5651 		      lba_count, lba);
5652 
5653 	bio->iovs = iov;
5654 	bio->iovcnt = iovcnt;
5655 	bio->iovpos = 0;
5656 	bio->iov_offset = 0;
5657 
5658 	rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns,
5659 					       bio->io_path->qpair->qpair,
5660 					       lba, lba_count,
5661 					       bdev_nvme_comparev_done, bio, flags,
5662 					       bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
5663 					       md, 0, 0);
5664 
5665 	if (rc != 0 && rc != -ENOMEM) {
5666 		SPDK_ERRLOG("comparev failed: rc = %d\n", rc);
5667 	}
5668 	return rc;
5669 }
5670 
5671 static int
5672 bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt,
5673 			      struct iovec *write_iov, int write_iovcnt,
5674 			      void *md, uint64_t lba_count, uint64_t lba, uint32_t flags)
5675 {
5676 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
5677 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
5678 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
5679 	int rc;
5680 
5681 	SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n",
5682 		      lba_count, lba);
5683 
5684 	bio->iovs = cmp_iov;
5685 	bio->iovcnt = cmp_iovcnt;
5686 	bio->iovpos = 0;
5687 	bio->iov_offset = 0;
5688 	bio->fused_iovs = write_iov;
5689 	bio->fused_iovcnt = write_iovcnt;
5690 	bio->fused_iovpos = 0;
5691 	bio->fused_iov_offset = 0;
5692 
5693 	if (bdev_io->num_retries == 0) {
5694 		bio->first_fused_submitted = false;
5695 		bio->first_fused_completed = false;
5696 	}
5697 
5698 	if (!bio->first_fused_submitted) {
5699 		flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST;
5700 		memset(&bio->cpl, 0, sizeof(bio->cpl));
5701 
5702 		rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count,
5703 						       bdev_nvme_comparev_and_writev_done, bio, flags,
5704 						       bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0);
5705 		if (rc == 0) {
5706 			bio->first_fused_submitted = true;
5707 			flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST;
5708 		} else {
5709 			if (rc != -ENOMEM) {
5710 				SPDK_ERRLOG("compare failed: rc = %d\n", rc);
5711 			}
5712 			return rc;
5713 		}
5714 	}
5715 
5716 	flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND;
5717 
5718 	rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count,
5719 					     bdev_nvme_comparev_and_writev_done, bio, flags,
5720 					     bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0);
5721 	if (rc != 0 && rc != -ENOMEM) {
5722 		SPDK_ERRLOG("write failed: rc = %d\n", rc);
5723 		rc = 0;
5724 	}
5725 
5726 	return rc;
5727 }
5728 
5729 static int
5730 bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks)
5731 {
5732 	struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES];
5733 	struct spdk_nvme_dsm_range *range;
5734 	uint64_t offset, remaining;
5735 	uint64_t num_ranges_u64;
5736 	uint16_t num_ranges;
5737 	int rc;
5738 
5739 	num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) /
5740 			 SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
5741 	if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) {
5742 		SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks);
5743 		return -EINVAL;
5744 	}
5745 	num_ranges = (uint16_t)num_ranges_u64;
5746 
5747 	offset = offset_blocks;
5748 	remaining = num_blocks;
5749 	range = &dsm_ranges[0];
5750 
5751 	/* Fill max-size ranges until the remaining blocks fit into one range */
5752 	while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) {
5753 		range->attributes.raw = 0;
5754 		range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
5755 		range->starting_lba = offset;
5756 
5757 		offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
5758 		remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
5759 		range++;
5760 	}
5761 
5762 	/* Final range describes the remaining blocks */
5763 	range->attributes.raw = 0;
5764 	range->length = remaining;
5765 	range->starting_lba = offset;
5766 
5767 	rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns,
5768 			bio->io_path->qpair->qpair,
5769 			SPDK_NVME_DSM_ATTR_DEALLOCATE,
5770 			dsm_ranges, num_ranges,
5771 			bdev_nvme_queued_done, bio);
5772 
5773 	return rc;
5774 }
5775 
5776 static int
5777 bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks)
5778 {
5779 	if (num_blocks > UINT16_MAX + 1) {
5780 		SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n");
5781 		return -EINVAL;
5782 	}
5783 
5784 	return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns,
5785 					     bio->io_path->qpair->qpair,
5786 					     offset_blocks, num_blocks,
5787 					     bdev_nvme_queued_done, bio,
5788 					     0);
5789 }
5790 
5791 static int
5792 bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones,
5793 			struct spdk_bdev_zone_info *info)
5794 {
5795 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
5796 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
5797 	uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns);
5798 	uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
5799 	uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns);
5800 
5801 	if (zone_id % zone_size != 0) {
5802 		return -EINVAL;
5803 	}
5804 
5805 	if (num_zones > total_zones || !num_zones) {
5806 		return -EINVAL;
5807 	}
5808 
5809 	assert(!bio->zone_report_buf);
5810 	bio->zone_report_buf = calloc(1, zone_report_bufsize);
5811 	if (!bio->zone_report_buf) {
5812 		return -ENOMEM;
5813 	}
5814 
5815 	bio->handled_zones = 0;
5816 
5817 	return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize,
5818 					  zone_id, SPDK_NVME_ZRA_LIST_ALL, true,
5819 					  bdev_nvme_get_zone_info_done, bio);
5820 }
5821 
5822 static int
5823 bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id,
5824 			  enum spdk_bdev_zone_action action)
5825 {
5826 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
5827 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
5828 
5829 	switch (action) {
5830 	case SPDK_BDEV_ZONE_CLOSE:
5831 		return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false,
5832 						bdev_nvme_zone_management_done, bio);
5833 	case SPDK_BDEV_ZONE_FINISH:
5834 		return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false,
5835 						 bdev_nvme_zone_management_done, bio);
5836 	case SPDK_BDEV_ZONE_OPEN:
5837 		return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false,
5838 					       bdev_nvme_zone_management_done, bio);
5839 	case SPDK_BDEV_ZONE_RESET:
5840 		return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false,
5841 						bdev_nvme_zone_management_done, bio);
5842 	case SPDK_BDEV_ZONE_OFFLINE:
5843 		return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false,
5844 						  bdev_nvme_zone_management_done, bio);
5845 	default:
5846 		return -EINVAL;
5847 	}
5848 }
5849 
5850 static void
5851 bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio,
5852 			 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes)
5853 {
5854 	struct nvme_io_path *io_path;
5855 	struct nvme_ctrlr *nvme_ctrlr;
5856 	uint32_t max_xfer_size;
5857 	int rc = -ENXIO;
5858 
5859 	/* Choose the first ctrlr which is not failed. */
5860 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
5861 		nvme_ctrlr = io_path->qpair->ctrlr;
5862 
5863 		/* We should skip any unavailable nvme_ctrlr rather than checking
5864 		 * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO.
5865 		 */
5866 		if (!nvme_ctrlr_is_available(nvme_ctrlr)) {
5867 			continue;
5868 		}
5869 
5870 		max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr);
5871 
5872 		if (nbytes > max_xfer_size) {
5873 			SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
5874 			rc = -EINVAL;
5875 			goto err;
5876 		}
5877 
5878 		bio->io_path = io_path;
5879 		bio->orig_thread = spdk_get_thread();
5880 
5881 		rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes,
5882 						   bdev_nvme_admin_passthru_done, bio);
5883 		if (rc == 0) {
5884 			return;
5885 		}
5886 	}
5887 
5888 err:
5889 	bdev_nvme_admin_passthru_complete(bio, rc);
5890 }
5891 
5892 static int
5893 bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
5894 		      void *buf, size_t nbytes)
5895 {
5896 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
5897 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
5898 	uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
5899 	struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
5900 
5901 	if (nbytes > max_xfer_size) {
5902 		SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
5903 		return -EINVAL;
5904 	}
5905 
5906 	/*
5907 	 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid,
5908 	 * so fill it out automatically.
5909 	 */
5910 	cmd->nsid = spdk_nvme_ns_get_id(ns);
5911 
5912 	return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf,
5913 					  (uint32_t)nbytes, bdev_nvme_queued_done, bio);
5914 }
5915 
5916 static int
5917 bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
5918 			 void *buf, size_t nbytes, void *md_buf, size_t md_len)
5919 {
5920 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
5921 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
5922 	size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns);
5923 	uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
5924 	struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
5925 
5926 	if (nbytes > max_xfer_size) {
5927 		SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
5928 		return -EINVAL;
5929 	}
5930 
5931 	if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) {
5932 		SPDK_ERRLOG("invalid meta data buffer size\n");
5933 		return -EINVAL;
5934 	}
5935 
5936 	/*
5937 	 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid,
5938 	 * so fill it out automatically.
5939 	 */
5940 	cmd->nsid = spdk_nvme_ns_get_id(ns);
5941 
5942 	return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf,
5943 			(uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio);
5944 }
5945 
5946 static void
5947 bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio,
5948 		struct nvme_bdev_io *bio_to_abort)
5949 {
5950 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
5951 	struct spdk_bdev_io *bdev_io_to_abort;
5952 	struct nvme_io_path *io_path;
5953 	struct nvme_ctrlr *nvme_ctrlr;
5954 	int rc = 0;
5955 
5956 	bio->orig_thread = spdk_get_thread();
5957 
5958 	/* Traverse the retry_io_list first. */
5959 	TAILQ_FOREACH(bdev_io_to_abort, &nbdev_ch->retry_io_list, module_link) {
5960 		if ((struct nvme_bdev_io *)bdev_io_to_abort->driver_ctx == bio_to_abort) {
5961 			TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io_to_abort, module_link);
5962 			spdk_bdev_io_complete(bdev_io_to_abort, SPDK_BDEV_IO_STATUS_ABORTED);
5963 
5964 			spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS);
5965 			return;
5966 		}
5967 	}
5968 
5969 	/* Even admin commands, they were submitted to only nvme_ctrlrs which were
5970 	 * on any io_path. So traverse the io_path list for not only I/O commands
5971 	 * but also admin commands.
5972 	 */
5973 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
5974 		nvme_ctrlr = io_path->qpair->ctrlr;
5975 
5976 		rc = spdk_nvme_ctrlr_cmd_abort_ext(nvme_ctrlr->ctrlr,
5977 						   io_path->qpair->qpair,
5978 						   bio_to_abort,
5979 						   bdev_nvme_abort_done, bio);
5980 		if (rc == -ENOENT) {
5981 			/* If no command was found in I/O qpair, the target command may be
5982 			 * admin command.
5983 			 */
5984 			rc = spdk_nvme_ctrlr_cmd_abort_ext(nvme_ctrlr->ctrlr,
5985 							   NULL,
5986 							   bio_to_abort,
5987 							   bdev_nvme_abort_done, bio);
5988 		}
5989 
5990 		if (rc != -ENOENT) {
5991 			break;
5992 		}
5993 	}
5994 
5995 	if (rc != 0) {
5996 		/* If no command was found or there was any error, complete the abort
5997 		 * request with failure.
5998 		 */
5999 		spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
6000 	}
6001 }
6002 
6003 static void
6004 bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w)
6005 {
6006 	const char	*action;
6007 
6008 	if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) {
6009 		action = "reset";
6010 	} else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) {
6011 		action = "abort";
6012 	} else {
6013 		action = "none";
6014 	}
6015 
6016 	spdk_json_write_object_begin(w);
6017 
6018 	spdk_json_write_named_string(w, "method", "bdev_nvme_set_options");
6019 
6020 	spdk_json_write_named_object_begin(w, "params");
6021 	spdk_json_write_named_string(w, "action_on_timeout", action);
6022 	spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us);
6023 	spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us);
6024 	spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms);
6025 	spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count);
6026 	spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst);
6027 	spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight);
6028 	spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight);
6029 	spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight);
6030 	spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us);
6031 	spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us);
6032 	spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests);
6033 	spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit);
6034 	spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count);
6035 	spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout);
6036 	spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec);
6037 	spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec);
6038 	spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec);
6039 	spdk_json_write_object_end(w);
6040 
6041 	spdk_json_write_object_end(w);
6042 }
6043 
6044 static void
6045 bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx)
6046 {
6047 	struct spdk_nvme_transport_id trid;
6048 
6049 	spdk_json_write_object_begin(w);
6050 
6051 	spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery");
6052 
6053 	spdk_json_write_named_object_begin(w, "params");
6054 	spdk_json_write_named_string(w, "name", ctx->name);
6055 	spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn);
6056 
6057 	trid = ctx->trid;
6058 	memset(trid.subnqn, 0, sizeof(trid.subnqn));
6059 	nvme_bdev_dump_trid_json(&trid, w);
6060 
6061 	spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach);
6062 	spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec);
6063 	spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec);
6064 	spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",
6065 				     ctx->bdev_opts.fast_io_fail_timeout_sec);
6066 	spdk_json_write_object_end(w);
6067 
6068 	spdk_json_write_object_end(w);
6069 }
6070 
6071 static void
6072 nvme_ctrlr_config_json(struct spdk_json_write_ctx *w,
6073 		       struct nvme_ctrlr *nvme_ctrlr)
6074 {
6075 	struct spdk_nvme_transport_id	*trid;
6076 
6077 	if (nvme_ctrlr->opts.from_discovery_service) {
6078 		/* Do not emit an RPC for this - it will be implicitly
6079 		 * covered by a separate bdev_nvme_start_discovery RPC.
6080 		 */
6081 		return;
6082 	}
6083 
6084 	trid = &nvme_ctrlr->active_path_id->trid;
6085 
6086 	spdk_json_write_object_begin(w);
6087 
6088 	spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller");
6089 
6090 	spdk_json_write_named_object_begin(w, "params");
6091 	spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name);
6092 	nvme_bdev_dump_trid_json(trid, w);
6093 	spdk_json_write_named_bool(w, "prchk_reftag",
6094 				   (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0);
6095 	spdk_json_write_named_bool(w, "prchk_guard",
6096 				   (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0);
6097 	spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec);
6098 	spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec);
6099 	spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",
6100 				     nvme_ctrlr->opts.fast_io_fail_timeout_sec);
6101 
6102 	spdk_json_write_object_end(w);
6103 
6104 	spdk_json_write_object_end(w);
6105 }
6106 
6107 static void
6108 bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w)
6109 {
6110 	spdk_json_write_object_begin(w);
6111 	spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug");
6112 
6113 	spdk_json_write_named_object_begin(w, "params");
6114 	spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us);
6115 	spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled);
6116 	spdk_json_write_object_end(w);
6117 
6118 	spdk_json_write_object_end(w);
6119 }
6120 
6121 static int
6122 bdev_nvme_config_json(struct spdk_json_write_ctx *w)
6123 {
6124 	struct nvme_bdev_ctrlr	*nbdev_ctrlr;
6125 	struct nvme_ctrlr	*nvme_ctrlr;
6126 	struct discovery_ctx	*ctx;
6127 
6128 	bdev_nvme_opts_config_json(w);
6129 
6130 	pthread_mutex_lock(&g_bdev_nvme_mutex);
6131 
6132 	TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
6133 		TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
6134 			nvme_ctrlr_config_json(w, nvme_ctrlr);
6135 		}
6136 	}
6137 
6138 	TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
6139 		bdev_nvme_discovery_config_json(w, ctx);
6140 	}
6141 
6142 	/* Dump as last parameter to give all NVMe bdevs chance to be constructed
6143 	 * before enabling hotplug poller.
6144 	 */
6145 	bdev_nvme_hotplug_config_json(w);
6146 
6147 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
6148 	return 0;
6149 }
6150 
6151 struct spdk_nvme_ctrlr *
6152 bdev_nvme_get_ctrlr(struct spdk_bdev *bdev)
6153 {
6154 	struct nvme_bdev *nbdev;
6155 	struct nvme_ns *nvme_ns;
6156 
6157 	if (!bdev || bdev->module != &nvme_if) {
6158 		return NULL;
6159 	}
6160 
6161 	nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
6162 	nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
6163 	assert(nvme_ns != NULL);
6164 
6165 	return nvme_ns->ctrlr->ctrlr;
6166 }
6167 
6168 void
6169 nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path)
6170 {
6171 	struct nvme_ns *nvme_ns = io_path->nvme_ns;
6172 	struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
6173 	const struct spdk_nvme_ctrlr_data *cdata;
6174 
6175 	spdk_json_write_object_begin(w);
6176 
6177 	spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name);
6178 
6179 	cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
6180 
6181 	spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid);
6182 
6183 	spdk_json_write_named_bool(w, "current", io_path == io_path->nbdev_ch->current_io_path);
6184 
6185 	spdk_json_write_named_bool(w, "connected", nvme_io_path_is_connected(io_path));
6186 
6187 	spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns));
6188 
6189 	spdk_json_write_object_end(w);
6190 }
6191 
6192 SPDK_LOG_REGISTER_COMPONENT(bdev_nvme)
6193