xref: /spdk/module/bdev/nvme/bdev_nvme.c (revision 2d590e74b469760b9a6ec4d06e1f559c96e3a101)
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (C) 2016 Intel Corporation. All rights reserved.
3  *   Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
4  *   Copyright (c) 2021, 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5  *   Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved.
6  */
7 
8 #include "spdk/stdinc.h"
9 
10 #include "bdev_nvme.h"
11 
12 #include "spdk/accel.h"
13 #include "spdk/config.h"
14 #include "spdk/endian.h"
15 #include "spdk/bdev.h"
16 #include "spdk/json.h"
17 #include "spdk/likely.h"
18 #include "spdk/nvme.h"
19 #include "spdk/nvme_ocssd.h"
20 #include "spdk/nvme_zns.h"
21 #include "spdk/opal.h"
22 #include "spdk/thread.h"
23 #include "spdk/trace.h"
24 #include "spdk/string.h"
25 #include "spdk/util.h"
26 #include "spdk/uuid.h"
27 
28 #include "spdk/bdev_module.h"
29 #include "spdk/log.h"
30 
31 #include "spdk_internal/usdt.h"
32 #include "spdk_internal/trace_defs.h"
33 
34 #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true
35 #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS	(10000)
36 
37 #define NSID_STR_LEN 10
38 
39 static int bdev_nvme_config_json(struct spdk_json_write_ctx *w);
40 
41 struct nvme_bdev_io {
42 	/** array of iovecs to transfer. */
43 	struct iovec *iovs;
44 
45 	/** Number of iovecs in iovs array. */
46 	int iovcnt;
47 
48 	/** Current iovec position. */
49 	int iovpos;
50 
51 	/** Offset in current iovec. */
52 	uint32_t iov_offset;
53 
54 	/** I/O path the current I/O or admin passthrough is submitted on, or the I/O path
55 	 *  being reset in a reset I/O.
56 	 */
57 	struct nvme_io_path *io_path;
58 
59 	/** array of iovecs to transfer. */
60 	struct iovec *fused_iovs;
61 
62 	/** Number of iovecs in iovs array. */
63 	int fused_iovcnt;
64 
65 	/** Current iovec position. */
66 	int fused_iovpos;
67 
68 	/** Offset in current iovec. */
69 	uint32_t fused_iov_offset;
70 
71 	/** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */
72 	struct spdk_nvme_cpl cpl;
73 
74 	/** Extended IO opts passed by the user to bdev layer and mapped to NVME format */
75 	struct spdk_nvme_ns_cmd_ext_io_opts ext_opts;
76 
77 	/** Originating thread */
78 	struct spdk_thread *orig_thread;
79 
80 	/** Keeps track if first of fused commands was submitted */
81 	bool first_fused_submitted;
82 
83 	/** Keeps track if first of fused commands was completed */
84 	bool first_fused_completed;
85 
86 	/** Temporary pointer to zone report buffer */
87 	struct spdk_nvme_zns_zone_report *zone_report_buf;
88 
89 	/** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */
90 	uint64_t handled_zones;
91 
92 	/** Expiration value in ticks to retry the current I/O. */
93 	uint64_t retry_ticks;
94 
95 	/* How many times the current I/O was retried. */
96 	int32_t retry_count;
97 
98 	/* Current tsc at submit time. */
99 	uint64_t submit_tsc;
100 };
101 
102 struct nvme_probe_skip_entry {
103 	struct spdk_nvme_transport_id		trid;
104 	TAILQ_ENTRY(nvme_probe_skip_entry)	tailq;
105 };
106 /* All the controllers deleted by users via RPC are skipped by hotplug monitor */
107 static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER(
108 			g_skipped_nvme_ctrlrs);
109 
110 static struct spdk_bdev_nvme_opts g_opts = {
111 	.action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE,
112 	.timeout_us = 0,
113 	.timeout_admin_us = 0,
114 	.keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS,
115 	.transport_retry_count = 4,
116 	.arbitration_burst = 0,
117 	.low_priority_weight = 0,
118 	.medium_priority_weight = 0,
119 	.high_priority_weight = 0,
120 	.nvme_adminq_poll_period_us = 10000ULL,
121 	.nvme_ioq_poll_period_us = 0,
122 	.io_queue_requests = 0,
123 	.delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT,
124 	.bdev_retry_count = 3,
125 	.transport_ack_timeout = 0,
126 	.ctrlr_loss_timeout_sec = 0,
127 	.reconnect_delay_sec = 0,
128 	.fast_io_fail_timeout_sec = 0,
129 	.disable_auto_failback = false,
130 	.generate_uuids = false,
131 	.transport_tos = 0,
132 	.nvme_error_stat = false,
133 	.io_path_stat = false,
134 };
135 
136 #define NVME_HOTPLUG_POLL_PERIOD_MAX			10000000ULL
137 #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT		100000ULL
138 
139 static int g_hot_insert_nvme_controller_index = 0;
140 static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT;
141 static bool g_nvme_hotplug_enabled = false;
142 struct spdk_thread *g_bdev_nvme_init_thread;
143 static struct spdk_poller *g_hotplug_poller;
144 static struct spdk_poller *g_hotplug_probe_poller;
145 static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx;
146 
147 static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
148 		struct nvme_async_probe_ctx *ctx);
149 static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
150 		struct nvme_async_probe_ctx *ctx);
151 static int bdev_nvme_library_init(void);
152 static void bdev_nvme_library_fini(void);
153 static void _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch,
154 				      struct spdk_bdev_io *bdev_io);
155 static void bdev_nvme_submit_request(struct spdk_io_channel *ch,
156 				     struct spdk_bdev_io *bdev_io);
157 static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
158 			   void *md, uint64_t lba_count, uint64_t lba,
159 			   uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx);
160 static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
161 				 void *md, uint64_t lba_count, uint64_t lba);
162 static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
163 			    void *md, uint64_t lba_count, uint64_t lba,
164 			    uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx);
165 static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
166 				  void *md, uint64_t lba_count,
167 				  uint64_t zslba, uint32_t flags);
168 static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
169 			      void *md, uint64_t lba_count, uint64_t lba,
170 			      uint32_t flags);
171 static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio,
172 		struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov,
173 		int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba,
174 		uint32_t flags);
175 static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id,
176 				   uint32_t num_zones, struct spdk_bdev_zone_info *info);
177 static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id,
178 				     enum spdk_bdev_zone_action action);
179 static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch,
180 				     struct nvme_bdev_io *bio,
181 				     struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes);
182 static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
183 				 void *buf, size_t nbytes);
184 static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
185 				    void *buf, size_t nbytes, void *md_buf, size_t md_len);
186 static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch,
187 			    struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort);
188 static void bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio);
189 static int bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr);
190 static int bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove);
191 static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr);
192 static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr);
193 
194 static struct nvme_ns *nvme_ns_alloc(void);
195 static void nvme_ns_free(struct nvme_ns *ns);
196 
197 static int
198 nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2)
199 {
200 	return ns1->id < ns2->id ? -1 : ns1->id > ns2->id;
201 }
202 
203 RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp);
204 
205 struct spdk_nvme_qpair *
206 bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch)
207 {
208 	struct nvme_ctrlr_channel *ctrlr_ch;
209 
210 	assert(ctrlr_io_ch != NULL);
211 
212 	ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch);
213 
214 	return ctrlr_ch->qpair->qpair;
215 }
216 
217 static int
218 bdev_nvme_get_ctx_size(void)
219 {
220 	return sizeof(struct nvme_bdev_io);
221 }
222 
223 static struct spdk_bdev_module nvme_if = {
224 	.name = "nvme",
225 	.async_fini = true,
226 	.module_init = bdev_nvme_library_init,
227 	.module_fini = bdev_nvme_library_fini,
228 	.config_json = bdev_nvme_config_json,
229 	.get_ctx_size = bdev_nvme_get_ctx_size,
230 
231 };
232 SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if)
233 
234 struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs);
235 pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER;
236 bool g_bdev_nvme_module_finish;
237 
238 struct nvme_bdev_ctrlr *
239 nvme_bdev_ctrlr_get_by_name(const char *name)
240 {
241 	struct nvme_bdev_ctrlr *nbdev_ctrlr;
242 
243 	TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
244 		if (strcmp(name, nbdev_ctrlr->name) == 0) {
245 			break;
246 		}
247 	}
248 
249 	return nbdev_ctrlr;
250 }
251 
252 static struct nvme_ctrlr *
253 nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr,
254 			  const struct spdk_nvme_transport_id *trid)
255 {
256 	struct nvme_ctrlr *nvme_ctrlr;
257 
258 	TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
259 		if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0) {
260 			break;
261 		}
262 	}
263 
264 	return nvme_ctrlr;
265 }
266 
267 static struct nvme_bdev *
268 nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid)
269 {
270 	struct nvme_bdev *bdev;
271 
272 	pthread_mutex_lock(&g_bdev_nvme_mutex);
273 	TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) {
274 		if (bdev->nsid == nsid) {
275 			break;
276 		}
277 	}
278 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
279 
280 	return bdev;
281 }
282 
283 struct nvme_ns *
284 nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid)
285 {
286 	struct nvme_ns ns;
287 
288 	assert(nsid > 0);
289 
290 	ns.id = nsid;
291 	return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns);
292 }
293 
294 struct nvme_ns *
295 nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr)
296 {
297 	return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces);
298 }
299 
300 struct nvme_ns *
301 nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns)
302 {
303 	if (ns == NULL) {
304 		return NULL;
305 	}
306 
307 	return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
308 }
309 
310 static struct nvme_ctrlr *
311 nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid)
312 {
313 	struct nvme_bdev_ctrlr	*nbdev_ctrlr;
314 	struct nvme_ctrlr	*nvme_ctrlr = NULL;
315 
316 	pthread_mutex_lock(&g_bdev_nvme_mutex);
317 	TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
318 		nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid);
319 		if (nvme_ctrlr != NULL) {
320 			break;
321 		}
322 	}
323 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
324 
325 	return nvme_ctrlr;
326 }
327 
328 struct nvme_ctrlr *
329 nvme_ctrlr_get_by_name(const char *name)
330 {
331 	struct nvme_bdev_ctrlr *nbdev_ctrlr;
332 	struct nvme_ctrlr *nvme_ctrlr = NULL;
333 
334 	if (name == NULL) {
335 		return NULL;
336 	}
337 
338 	pthread_mutex_lock(&g_bdev_nvme_mutex);
339 	nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
340 	if (nbdev_ctrlr != NULL) {
341 		nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
342 	}
343 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
344 
345 	return nvme_ctrlr;
346 }
347 
348 void
349 nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx)
350 {
351 	struct nvme_bdev_ctrlr *nbdev_ctrlr;
352 
353 	pthread_mutex_lock(&g_bdev_nvme_mutex);
354 	TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
355 		fn(nbdev_ctrlr, ctx);
356 	}
357 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
358 }
359 
360 void
361 nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w)
362 {
363 	const char *trtype_str;
364 	const char *adrfam_str;
365 
366 	trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype);
367 	if (trtype_str) {
368 		spdk_json_write_named_string(w, "trtype", trtype_str);
369 	}
370 
371 	adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam);
372 	if (adrfam_str) {
373 		spdk_json_write_named_string(w, "adrfam", adrfam_str);
374 	}
375 
376 	if (trid->traddr[0] != '\0') {
377 		spdk_json_write_named_string(w, "traddr", trid->traddr);
378 	}
379 
380 	if (trid->trsvcid[0] != '\0') {
381 		spdk_json_write_named_string(w, "trsvcid", trid->trsvcid);
382 	}
383 
384 	if (trid->subnqn[0] != '\0') {
385 		spdk_json_write_named_string(w, "subnqn", trid->subnqn);
386 	}
387 }
388 
389 static void
390 nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr,
391 		       struct nvme_ctrlr *nvme_ctrlr)
392 {
393 	SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name);
394 	pthread_mutex_lock(&g_bdev_nvme_mutex);
395 
396 	TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq);
397 	if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) {
398 		pthread_mutex_unlock(&g_bdev_nvme_mutex);
399 
400 		return;
401 	}
402 	TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq);
403 
404 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
405 
406 	assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs));
407 
408 	free(nbdev_ctrlr->name);
409 	free(nbdev_ctrlr);
410 }
411 
412 static void
413 _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)
414 {
415 	struct nvme_path_id *path_id, *tmp_path;
416 	struct nvme_ns *ns, *tmp_ns;
417 
418 	free(nvme_ctrlr->copied_ana_desc);
419 	spdk_free(nvme_ctrlr->ana_log_page);
420 
421 	if (nvme_ctrlr->opal_dev) {
422 		spdk_opal_dev_destruct(nvme_ctrlr->opal_dev);
423 		nvme_ctrlr->opal_dev = NULL;
424 	}
425 
426 	if (nvme_ctrlr->nbdev_ctrlr) {
427 		nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr);
428 	}
429 
430 	RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) {
431 		RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
432 		nvme_ns_free(ns);
433 	}
434 
435 	TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) {
436 		TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link);
437 		free(path_id);
438 	}
439 
440 	pthread_mutex_destroy(&nvme_ctrlr->mutex);
441 
442 	free(nvme_ctrlr);
443 
444 	pthread_mutex_lock(&g_bdev_nvme_mutex);
445 	if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
446 		pthread_mutex_unlock(&g_bdev_nvme_mutex);
447 		spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL);
448 		spdk_bdev_module_fini_done();
449 		return;
450 	}
451 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
452 }
453 
454 static int
455 nvme_detach_poller(void *arg)
456 {
457 	struct nvme_ctrlr *nvme_ctrlr = arg;
458 	int rc;
459 
460 	rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx);
461 	if (rc != -EAGAIN) {
462 		spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
463 		_nvme_ctrlr_delete(nvme_ctrlr);
464 	}
465 
466 	return SPDK_POLLER_BUSY;
467 }
468 
469 static void
470 nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)
471 {
472 	int rc;
473 
474 	spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
475 
476 	/* First, unregister the adminq poller, as the driver will poll adminq if necessary */
477 	spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller);
478 
479 	/* If we got here, the reset/detach poller cannot be active */
480 	assert(nvme_ctrlr->reset_detach_poller == NULL);
481 	nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller,
482 					  nvme_ctrlr, 1000);
483 	if (nvme_ctrlr->reset_detach_poller == NULL) {
484 		SPDK_ERRLOG("Failed to register detach poller\n");
485 		goto error;
486 	}
487 
488 	rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx);
489 	if (rc != 0) {
490 		SPDK_ERRLOG("Failed to detach the NVMe controller\n");
491 		goto error;
492 	}
493 
494 	return;
495 error:
496 	/* We don't have a good way to handle errors here, so just do what we can and delete the
497 	 * controller without detaching the underlying NVMe device.
498 	 */
499 	spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
500 	_nvme_ctrlr_delete(nvme_ctrlr);
501 }
502 
503 static void
504 nvme_ctrlr_unregister_cb(void *io_device)
505 {
506 	struct nvme_ctrlr *nvme_ctrlr = io_device;
507 
508 	nvme_ctrlr_delete(nvme_ctrlr);
509 }
510 
511 static void
512 nvme_ctrlr_unregister(void *ctx)
513 {
514 	struct nvme_ctrlr *nvme_ctrlr = ctx;
515 
516 	spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb);
517 }
518 
519 static bool
520 nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr)
521 {
522 	if (!nvme_ctrlr->destruct) {
523 		return false;
524 	}
525 
526 	if (nvme_ctrlr->ref > 0) {
527 		return false;
528 	}
529 
530 	if (nvme_ctrlr->resetting) {
531 		return false;
532 	}
533 
534 	if (nvme_ctrlr->ana_log_page_updating) {
535 		return false;
536 	}
537 
538 	if (nvme_ctrlr->io_path_cache_clearing) {
539 		return false;
540 	}
541 
542 	return true;
543 }
544 
545 static void
546 nvme_ctrlr_release(struct nvme_ctrlr *nvme_ctrlr)
547 {
548 	pthread_mutex_lock(&nvme_ctrlr->mutex);
549 	SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref);
550 
551 	assert(nvme_ctrlr->ref > 0);
552 	nvme_ctrlr->ref--;
553 
554 	if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
555 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
556 		return;
557 	}
558 
559 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
560 
561 	spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr);
562 }
563 
564 static void
565 bdev_nvme_clear_current_io_path(struct nvme_bdev_channel *nbdev_ch)
566 {
567 	nbdev_ch->current_io_path = NULL;
568 	nbdev_ch->rr_counter = 0;
569 }
570 
571 static struct nvme_io_path *
572 _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns)
573 {
574 	struct nvme_io_path *io_path;
575 
576 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
577 		if (io_path->nvme_ns == nvme_ns) {
578 			break;
579 		}
580 	}
581 
582 	return io_path;
583 }
584 
585 static int
586 _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns)
587 {
588 	struct nvme_io_path *io_path;
589 	struct spdk_io_channel *ch;
590 	struct nvme_ctrlr_channel *ctrlr_ch;
591 	struct nvme_qpair *nvme_qpair;
592 
593 	io_path = calloc(1, sizeof(*io_path));
594 	if (io_path == NULL) {
595 		SPDK_ERRLOG("Failed to alloc io_path.\n");
596 		return -ENOMEM;
597 	}
598 
599 	if (g_opts.io_path_stat) {
600 		io_path->stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
601 		if (io_path->stat == NULL) {
602 			free(io_path);
603 			SPDK_ERRLOG("Failed to alloc io_path stat.\n");
604 			return -ENOMEM;
605 		}
606 		spdk_bdev_reset_io_stat(io_path->stat, SPDK_BDEV_RESET_STAT_MAXMIN);
607 	}
608 
609 	io_path->nvme_ns = nvme_ns;
610 
611 	ch = spdk_get_io_channel(nvme_ns->ctrlr);
612 	if (ch == NULL) {
613 		free(io_path->stat);
614 		free(io_path);
615 		SPDK_ERRLOG("Failed to alloc io_channel.\n");
616 		return -ENOMEM;
617 	}
618 
619 	ctrlr_ch = spdk_io_channel_get_ctx(ch);
620 
621 	nvme_qpair = ctrlr_ch->qpair;
622 	assert(nvme_qpair != NULL);
623 
624 	io_path->qpair = nvme_qpair;
625 	TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq);
626 
627 	io_path->nbdev_ch = nbdev_ch;
628 	STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq);
629 
630 	bdev_nvme_clear_current_io_path(nbdev_ch);
631 
632 	return 0;
633 }
634 
635 static void
636 _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path)
637 {
638 	struct spdk_io_channel *ch;
639 	struct nvme_qpair *nvme_qpair;
640 	struct nvme_ctrlr_channel *ctrlr_ch;
641 	struct nvme_bdev *nbdev;
642 
643 	nbdev = spdk_io_channel_get_io_device(spdk_io_channel_from_ctx(nbdev_ch));
644 
645 	/* Add the statistics to nvme_ns before this path is destroyed. */
646 	pthread_mutex_lock(&nbdev->mutex);
647 	if (nbdev->ref != 0 && io_path->nvme_ns->stat != NULL && io_path->stat != NULL) {
648 		spdk_bdev_add_io_stat(io_path->nvme_ns->stat, io_path->stat);
649 	}
650 	pthread_mutex_unlock(&nbdev->mutex);
651 
652 	bdev_nvme_clear_current_io_path(nbdev_ch);
653 
654 	STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq);
655 
656 	nvme_qpair = io_path->qpair;
657 	assert(nvme_qpair != NULL);
658 
659 	TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq);
660 
661 	ctrlr_ch = nvme_qpair->ctrlr_ch;
662 	assert(ctrlr_ch != NULL);
663 
664 	ch = spdk_io_channel_from_ctx(ctrlr_ch);
665 	spdk_put_io_channel(ch);
666 
667 	free(io_path->stat);
668 	free(io_path);
669 }
670 
671 static void
672 _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch)
673 {
674 	struct nvme_io_path *io_path, *tmp_io_path;
675 
676 	STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) {
677 		_bdev_nvme_delete_io_path(nbdev_ch, io_path);
678 	}
679 }
680 
681 static int
682 bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf)
683 {
684 	struct nvme_bdev_channel *nbdev_ch = ctx_buf;
685 	struct nvme_bdev *nbdev = io_device;
686 	struct nvme_ns *nvme_ns;
687 	int rc;
688 
689 	STAILQ_INIT(&nbdev_ch->io_path_list);
690 	TAILQ_INIT(&nbdev_ch->retry_io_list);
691 
692 	pthread_mutex_lock(&nbdev->mutex);
693 
694 	nbdev_ch->mp_policy = nbdev->mp_policy;
695 	nbdev_ch->mp_selector = nbdev->mp_selector;
696 	nbdev_ch->rr_min_io = nbdev->rr_min_io;
697 
698 	TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
699 		rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns);
700 		if (rc != 0) {
701 			pthread_mutex_unlock(&nbdev->mutex);
702 
703 			_bdev_nvme_delete_io_paths(nbdev_ch);
704 			return rc;
705 		}
706 	}
707 	pthread_mutex_unlock(&nbdev->mutex);
708 
709 	return 0;
710 }
711 
712 /* If cpl != NULL, complete the bdev_io with nvme status based on 'cpl'.
713  * If cpl == NULL, complete the bdev_io with bdev status based on 'status'.
714  */
715 static inline void
716 __bdev_nvme_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status,
717 			const struct spdk_nvme_cpl *cpl)
718 {
719 	spdk_trace_record(TRACE_BDEV_NVME_IO_DONE, 0, 0, (uintptr_t)bdev_io->driver_ctx,
720 			  (uintptr_t)bdev_io);
721 	if (cpl) {
722 		spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc);
723 	} else {
724 		spdk_bdev_io_complete(bdev_io, status);
725 	}
726 }
727 
728 static void bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch);
729 
730 static void
731 bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf)
732 {
733 	struct nvme_bdev_channel *nbdev_ch = ctx_buf;
734 
735 	bdev_nvme_abort_retry_ios(nbdev_ch);
736 	_bdev_nvme_delete_io_paths(nbdev_ch);
737 }
738 
739 static inline bool
740 bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type)
741 {
742 	switch (io_type) {
743 	case SPDK_BDEV_IO_TYPE_RESET:
744 	case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
745 	case SPDK_BDEV_IO_TYPE_ABORT:
746 		return true;
747 	default:
748 		break;
749 	}
750 
751 	return false;
752 }
753 
754 static inline bool
755 nvme_ns_is_accessible(struct nvme_ns *nvme_ns)
756 {
757 	if (spdk_unlikely(nvme_ns->ana_state_updating)) {
758 		return false;
759 	}
760 
761 	switch (nvme_ns->ana_state) {
762 	case SPDK_NVME_ANA_OPTIMIZED_STATE:
763 	case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
764 		return true;
765 	default:
766 		break;
767 	}
768 
769 	return false;
770 }
771 
772 static inline bool
773 nvme_io_path_is_connected(struct nvme_io_path *io_path)
774 {
775 	if (spdk_unlikely(io_path->qpair->qpair == NULL)) {
776 		return false;
777 	}
778 
779 	if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(io_path->qpair->qpair) !=
780 			  SPDK_NVME_QPAIR_FAILURE_NONE)) {
781 		return false;
782 	}
783 
784 	if (spdk_unlikely(io_path->qpair->ctrlr_ch->reset_iter != NULL)) {
785 		return false;
786 	}
787 
788 	if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(io_path->qpair->ctrlr->ctrlr) !=
789 	    SPDK_NVME_QPAIR_FAILURE_NONE) {
790 		return false;
791 	}
792 
793 	return true;
794 }
795 
796 static inline bool
797 nvme_io_path_is_available(struct nvme_io_path *io_path)
798 {
799 	if (spdk_unlikely(!nvme_io_path_is_connected(io_path))) {
800 		return false;
801 	}
802 
803 	if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) {
804 		return false;
805 	}
806 
807 	return true;
808 }
809 
810 static inline bool
811 nvme_io_path_is_failed(struct nvme_io_path *io_path)
812 {
813 	struct nvme_ctrlr *nvme_ctrlr;
814 
815 	nvme_ctrlr = io_path->qpair->ctrlr;
816 
817 	if (nvme_ctrlr->destruct) {
818 		return true;
819 	}
820 
821 	if (nvme_ctrlr->fast_io_fail_timedout) {
822 		return true;
823 	}
824 
825 	if (nvme_ctrlr->resetting) {
826 		if (nvme_ctrlr->opts.reconnect_delay_sec != 0) {
827 			return false;
828 		} else {
829 			return true;
830 		}
831 	}
832 
833 	if (nvme_ctrlr->reconnect_is_delayed) {
834 		return false;
835 	}
836 
837 	if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
838 		return true;
839 	} else {
840 		return false;
841 	}
842 }
843 
844 static bool
845 nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr)
846 {
847 	if (nvme_ctrlr->destruct) {
848 		return false;
849 	}
850 
851 	if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
852 		return false;
853 	}
854 
855 	if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) {
856 		return false;
857 	}
858 
859 	return true;
860 }
861 
862 /* Simulate circular linked list. */
863 static inline struct nvme_io_path *
864 nvme_io_path_get_next(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *prev_path)
865 {
866 	struct nvme_io_path *next_path;
867 
868 	if (prev_path != NULL) {
869 		next_path = STAILQ_NEXT(prev_path, stailq);
870 		if (next_path != NULL) {
871 			return next_path;
872 		}
873 	}
874 
875 	return STAILQ_FIRST(&nbdev_ch->io_path_list);
876 }
877 
878 static struct nvme_io_path *
879 _bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch)
880 {
881 	struct nvme_io_path *io_path, *start, *non_optimized = NULL;
882 
883 	start = nvme_io_path_get_next(nbdev_ch, nbdev_ch->current_io_path);
884 
885 	io_path = start;
886 	do {
887 		if (spdk_likely(nvme_io_path_is_connected(io_path) &&
888 				!io_path->nvme_ns->ana_state_updating)) {
889 			switch (io_path->nvme_ns->ana_state) {
890 			case SPDK_NVME_ANA_OPTIMIZED_STATE:
891 				nbdev_ch->current_io_path = io_path;
892 				return io_path;
893 			case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
894 				if (non_optimized == NULL) {
895 					non_optimized = io_path;
896 				}
897 				break;
898 			default:
899 				break;
900 			}
901 		}
902 		io_path = nvme_io_path_get_next(nbdev_ch, io_path);
903 	} while (io_path != start);
904 
905 	if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) {
906 		/* We come here only if there is no optimized path. Cache even non_optimized
907 		 * path for load balance across multiple non_optimized paths.
908 		 */
909 		nbdev_ch->current_io_path = non_optimized;
910 	}
911 
912 	return non_optimized;
913 }
914 
915 static struct nvme_io_path *
916 _bdev_nvme_find_io_path_min_qd(struct nvme_bdev_channel *nbdev_ch)
917 {
918 	struct nvme_io_path *io_path;
919 	struct nvme_io_path *optimized = NULL, *non_optimized = NULL;
920 	uint32_t opt_min_qd = UINT32_MAX, non_opt_min_qd = UINT32_MAX;
921 	uint32_t num_outstanding_reqs;
922 
923 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
924 		if (spdk_unlikely(!nvme_io_path_is_connected(io_path))) {
925 			/* The device is currently resetting. */
926 			continue;
927 		}
928 
929 		if (spdk_unlikely(io_path->nvme_ns->ana_state_updating)) {
930 			continue;
931 		}
932 
933 		num_outstanding_reqs = spdk_nvme_qpair_get_num_outstanding_reqs(io_path->qpair->qpair);
934 		switch (io_path->nvme_ns->ana_state) {
935 		case SPDK_NVME_ANA_OPTIMIZED_STATE:
936 			if (num_outstanding_reqs < opt_min_qd) {
937 				opt_min_qd = num_outstanding_reqs;
938 				optimized = io_path;
939 			}
940 			break;
941 		case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
942 			if (num_outstanding_reqs < non_opt_min_qd) {
943 				non_opt_min_qd = num_outstanding_reqs;
944 				non_optimized = io_path;
945 			}
946 			break;
947 		default:
948 			break;
949 		}
950 	}
951 
952 	/* don't cache io path for BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH selector */
953 	if (optimized != NULL) {
954 		return optimized;
955 	}
956 
957 	return non_optimized;
958 }
959 
960 static inline struct nvme_io_path *
961 bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch)
962 {
963 	if (spdk_likely(nbdev_ch->current_io_path != NULL)) {
964 		if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE) {
965 			return nbdev_ch->current_io_path;
966 		} else if (nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
967 			if (++nbdev_ch->rr_counter < nbdev_ch->rr_min_io) {
968 				return nbdev_ch->current_io_path;
969 			}
970 			nbdev_ch->rr_counter = 0;
971 		}
972 	}
973 
974 	if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE ||
975 	    nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
976 		return _bdev_nvme_find_io_path(nbdev_ch);
977 	} else {
978 		return _bdev_nvme_find_io_path_min_qd(nbdev_ch);
979 	}
980 }
981 
982 /* Return true if there is any io_path whose qpair is active or ctrlr is not failed,
983  * or false otherwise.
984  *
985  * If any io_path has an active qpair but find_io_path() returned NULL, its namespace
986  * is likely to be non-accessible now but may become accessible.
987  *
988  * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr
989  * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed
990  * when starting to reset it but it is set to failed when the reset failed. Hence, if
991  * a ctrlr is unfailed, it is likely that it works fine or is resetting.
992  */
993 static bool
994 any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch)
995 {
996 	struct nvme_io_path *io_path;
997 
998 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
999 		if (io_path->nvme_ns->ana_transition_timedout) {
1000 			continue;
1001 		}
1002 
1003 		if (nvme_io_path_is_connected(io_path) ||
1004 		    !nvme_io_path_is_failed(io_path)) {
1005 			return true;
1006 		}
1007 	}
1008 
1009 	return false;
1010 }
1011 
1012 static void
1013 bdev_nvme_retry_io(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io)
1014 {
1015 	struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
1016 	struct spdk_io_channel *ch;
1017 
1018 	if (nbdev_io->io_path != NULL && nvme_io_path_is_available(nbdev_io->io_path)) {
1019 		_bdev_nvme_submit_request(nbdev_ch, bdev_io);
1020 	} else {
1021 		ch = spdk_io_channel_from_ctx(nbdev_ch);
1022 		bdev_nvme_submit_request(ch, bdev_io);
1023 	}
1024 }
1025 
1026 static int
1027 bdev_nvme_retry_ios(void *arg)
1028 {
1029 	struct nvme_bdev_channel *nbdev_ch = arg;
1030 	struct spdk_bdev_io *bdev_io, *tmp_bdev_io;
1031 	struct nvme_bdev_io *bio;
1032 	uint64_t now, delay_us;
1033 
1034 	now = spdk_get_ticks();
1035 
1036 	TAILQ_FOREACH_SAFE(bdev_io, &nbdev_ch->retry_io_list, module_link, tmp_bdev_io) {
1037 		bio = (struct nvme_bdev_io *)bdev_io->driver_ctx;
1038 		if (bio->retry_ticks > now) {
1039 			break;
1040 		}
1041 
1042 		TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io, module_link);
1043 
1044 		bdev_nvme_retry_io(nbdev_ch, bdev_io);
1045 	}
1046 
1047 	spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1048 
1049 	bdev_io = TAILQ_FIRST(&nbdev_ch->retry_io_list);
1050 	if (bdev_io != NULL) {
1051 		bio = (struct nvme_bdev_io *)bdev_io->driver_ctx;
1052 
1053 		delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz();
1054 
1055 		nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch,
1056 					    delay_us);
1057 	}
1058 
1059 	return SPDK_POLLER_BUSY;
1060 }
1061 
1062 static void
1063 bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch,
1064 			 struct nvme_bdev_io *bio, uint64_t delay_ms)
1065 {
1066 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1067 	struct spdk_bdev_io *tmp_bdev_io;
1068 	struct nvme_bdev_io *tmp_bio;
1069 
1070 	bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL;
1071 
1072 	TAILQ_FOREACH_REVERSE(tmp_bdev_io, &nbdev_ch->retry_io_list, retry_io_head, module_link) {
1073 		tmp_bio = (struct nvme_bdev_io *)tmp_bdev_io->driver_ctx;
1074 
1075 		if (tmp_bio->retry_ticks <= bio->retry_ticks) {
1076 			TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bdev_io, bdev_io,
1077 					   module_link);
1078 			return;
1079 		}
1080 	}
1081 
1082 	/* No earlier I/Os were found. This I/O must be the new head. */
1083 	TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bdev_io, module_link);
1084 
1085 	spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1086 
1087 	nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch,
1088 				    delay_ms * 1000ULL);
1089 }
1090 
1091 static void
1092 bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch)
1093 {
1094 	struct spdk_bdev_io *bdev_io, *tmp_io;
1095 
1096 	TAILQ_FOREACH_SAFE(bdev_io, &nbdev_ch->retry_io_list, module_link, tmp_io) {
1097 		TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io, module_link);
1098 		__bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_ABORTED, NULL);
1099 	}
1100 
1101 	spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1102 }
1103 
1104 static int
1105 bdev_nvme_abort_retry_io(struct nvme_bdev_channel *nbdev_ch,
1106 			 struct nvme_bdev_io *bio_to_abort)
1107 {
1108 	struct spdk_bdev_io *bdev_io_to_abort;
1109 
1110 	TAILQ_FOREACH(bdev_io_to_abort, &nbdev_ch->retry_io_list, module_link) {
1111 		if ((struct nvme_bdev_io *)bdev_io_to_abort->driver_ctx == bio_to_abort) {
1112 			TAILQ_REMOVE(&nbdev_ch->retry_io_list, bdev_io_to_abort, module_link);
1113 			__bdev_nvme_io_complete(bdev_io_to_abort, SPDK_BDEV_IO_STATUS_ABORTED, NULL);
1114 			return 0;
1115 		}
1116 	}
1117 
1118 	return -ENOENT;
1119 }
1120 
1121 static void
1122 bdev_nvme_update_nvme_error_stat(struct spdk_bdev_io *bdev_io, const struct spdk_nvme_cpl *cpl)
1123 {
1124 	struct nvme_bdev *nbdev;
1125 	uint16_t sct, sc;
1126 
1127 	assert(spdk_nvme_cpl_is_error(cpl));
1128 
1129 	nbdev = bdev_io->bdev->ctxt;
1130 
1131 	if (nbdev->err_stat == NULL) {
1132 		return;
1133 	}
1134 
1135 	sct = cpl->status.sct;
1136 	sc = cpl->status.sc;
1137 
1138 	pthread_mutex_lock(&nbdev->mutex);
1139 
1140 	nbdev->err_stat->status_type[sct]++;
1141 	switch (sct) {
1142 	case SPDK_NVME_SCT_GENERIC:
1143 	case SPDK_NVME_SCT_COMMAND_SPECIFIC:
1144 	case SPDK_NVME_SCT_MEDIA_ERROR:
1145 	case SPDK_NVME_SCT_PATH:
1146 		nbdev->err_stat->status[sct][sc]++;
1147 		break;
1148 	default:
1149 		break;
1150 	}
1151 
1152 	pthread_mutex_unlock(&nbdev->mutex);
1153 }
1154 
1155 static inline void
1156 bdev_nvme_update_io_path_stat(struct nvme_bdev_io *bio)
1157 {
1158 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1159 	uint64_t num_blocks = bdev_io->u.bdev.num_blocks;
1160 	uint32_t blocklen = bdev_io->bdev->blocklen;
1161 	struct spdk_bdev_io_stat *stat;
1162 	uint64_t tsc_diff;
1163 
1164 	if (bio->io_path->stat == NULL) {
1165 		return;
1166 	}
1167 
1168 	tsc_diff = spdk_get_ticks() - bio->submit_tsc;
1169 	stat = bio->io_path->stat;
1170 
1171 	switch (bdev_io->type) {
1172 	case SPDK_BDEV_IO_TYPE_READ:
1173 		stat->bytes_read += num_blocks * blocklen;
1174 		stat->num_read_ops++;
1175 		stat->read_latency_ticks += tsc_diff;
1176 		if (stat->max_read_latency_ticks < tsc_diff) {
1177 			stat->max_read_latency_ticks = tsc_diff;
1178 		}
1179 		if (stat->min_read_latency_ticks > tsc_diff) {
1180 			stat->min_read_latency_ticks = tsc_diff;
1181 		}
1182 		break;
1183 	case SPDK_BDEV_IO_TYPE_WRITE:
1184 		stat->bytes_written += num_blocks * blocklen;
1185 		stat->num_write_ops++;
1186 		stat->write_latency_ticks += tsc_diff;
1187 		if (stat->max_write_latency_ticks < tsc_diff) {
1188 			stat->max_write_latency_ticks = tsc_diff;
1189 		}
1190 		if (stat->min_write_latency_ticks > tsc_diff) {
1191 			stat->min_write_latency_ticks = tsc_diff;
1192 		}
1193 		break;
1194 	case SPDK_BDEV_IO_TYPE_UNMAP:
1195 		stat->bytes_unmapped += num_blocks * blocklen;
1196 		stat->num_unmap_ops++;
1197 		stat->unmap_latency_ticks += tsc_diff;
1198 		if (stat->max_unmap_latency_ticks < tsc_diff) {
1199 			stat->max_unmap_latency_ticks = tsc_diff;
1200 		}
1201 		if (stat->min_unmap_latency_ticks > tsc_diff) {
1202 			stat->min_unmap_latency_ticks = tsc_diff;
1203 		}
1204 		break;
1205 	case SPDK_BDEV_IO_TYPE_ZCOPY:
1206 		/* Track the data in the start phase only */
1207 		if (!bdev_io->u.bdev.zcopy.start) {
1208 			break;
1209 		}
1210 		if (bdev_io->u.bdev.zcopy.populate) {
1211 			stat->bytes_read += num_blocks * blocklen;
1212 			stat->num_read_ops++;
1213 			stat->read_latency_ticks += tsc_diff;
1214 			if (stat->max_read_latency_ticks < tsc_diff) {
1215 				stat->max_read_latency_ticks = tsc_diff;
1216 			}
1217 			if (stat->min_read_latency_ticks > tsc_diff) {
1218 				stat->min_read_latency_ticks = tsc_diff;
1219 			}
1220 		} else {
1221 			stat->bytes_written += num_blocks * blocklen;
1222 			stat->num_write_ops++;
1223 			stat->write_latency_ticks += tsc_diff;
1224 			if (stat->max_write_latency_ticks < tsc_diff) {
1225 				stat->max_write_latency_ticks = tsc_diff;
1226 			}
1227 			if (stat->min_write_latency_ticks > tsc_diff) {
1228 				stat->min_write_latency_ticks = tsc_diff;
1229 			}
1230 		}
1231 		break;
1232 	case SPDK_BDEV_IO_TYPE_COPY:
1233 		stat->bytes_copied += num_blocks * blocklen;
1234 		stat->num_copy_ops++;
1235 		stat->copy_latency_ticks += tsc_diff;
1236 		if (stat->max_copy_latency_ticks < tsc_diff) {
1237 			stat->max_copy_latency_ticks = tsc_diff;
1238 		}
1239 		if (stat->min_copy_latency_ticks > tsc_diff) {
1240 			stat->min_copy_latency_ticks = tsc_diff;
1241 		}
1242 		break;
1243 	default:
1244 		break;
1245 	}
1246 }
1247 
1248 static inline void
1249 bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio,
1250 				  const struct spdk_nvme_cpl *cpl)
1251 {
1252 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1253 	struct nvme_bdev_channel *nbdev_ch;
1254 	struct nvme_io_path *io_path;
1255 	struct nvme_ctrlr *nvme_ctrlr;
1256 	const struct spdk_nvme_ctrlr_data *cdata;
1257 	uint64_t delay_ms;
1258 
1259 	assert(!bdev_nvme_io_type_is_admin(bdev_io->type));
1260 
1261 	if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) {
1262 		bdev_nvme_update_io_path_stat(bio);
1263 		goto complete;
1264 	}
1265 
1266 	/* Update error counts before deciding if retry is needed.
1267 	 * Hence, error counts may be more than the number of I/O errors.
1268 	 */
1269 	bdev_nvme_update_nvme_error_stat(bdev_io, cpl);
1270 
1271 	if (cpl->status.dnr != 0 || spdk_nvme_cpl_is_aborted_by_request(cpl) ||
1272 	    (g_opts.bdev_retry_count != -1 && bio->retry_count >= g_opts.bdev_retry_count)) {
1273 		goto complete;
1274 	}
1275 
1276 	nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
1277 
1278 	assert(bio->io_path != NULL);
1279 	io_path = bio->io_path;
1280 
1281 	nvme_ctrlr = io_path->qpair->ctrlr;
1282 
1283 	if (spdk_nvme_cpl_is_path_error(cpl) ||
1284 	    spdk_nvme_cpl_is_aborted_sq_deletion(cpl) ||
1285 	    !nvme_io_path_is_available(io_path) ||
1286 	    !nvme_ctrlr_is_available(nvme_ctrlr)) {
1287 		bdev_nvme_clear_current_io_path(nbdev_ch);
1288 		bio->io_path = NULL;
1289 		if (spdk_nvme_cpl_is_ana_error(cpl)) {
1290 			if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) {
1291 				io_path->nvme_ns->ana_state_updating = true;
1292 			}
1293 		}
1294 		if (!any_io_path_may_become_available(nbdev_ch)) {
1295 			goto complete;
1296 		}
1297 		delay_ms = 0;
1298 	} else {
1299 		bio->retry_count++;
1300 
1301 		cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
1302 
1303 		if (cpl->status.crd != 0) {
1304 			delay_ms = cdata->crdt[cpl->status.crd] * 100;
1305 		} else {
1306 			delay_ms = 0;
1307 		}
1308 	}
1309 
1310 	bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms);
1311 	return;
1312 
1313 complete:
1314 	bio->retry_count = 0;
1315 	bio->submit_tsc = 0;
1316 	__bdev_nvme_io_complete(bdev_io, 0, cpl);
1317 }
1318 
1319 static inline void
1320 bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc)
1321 {
1322 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1323 	struct nvme_bdev_channel *nbdev_ch;
1324 	enum spdk_bdev_io_status io_status;
1325 
1326 	switch (rc) {
1327 	case 0:
1328 		io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1329 		break;
1330 	case -ENOMEM:
1331 		io_status = SPDK_BDEV_IO_STATUS_NOMEM;
1332 		break;
1333 	case -ENXIO:
1334 		nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
1335 
1336 		bdev_nvme_clear_current_io_path(nbdev_ch);
1337 		bio->io_path = NULL;
1338 
1339 		if (any_io_path_may_become_available(nbdev_ch)) {
1340 			bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL);
1341 			return;
1342 		}
1343 
1344 	/* fallthrough */
1345 	default:
1346 		io_status = SPDK_BDEV_IO_STATUS_FAILED;
1347 		break;
1348 	}
1349 
1350 	bio->retry_count = 0;
1351 	bio->submit_tsc = 0;
1352 	__bdev_nvme_io_complete(bdev_io, io_status, NULL);
1353 }
1354 
1355 static inline void
1356 bdev_nvme_admin_passthru_complete(struct nvme_bdev_io *bio, int rc)
1357 {
1358 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1359 	enum spdk_bdev_io_status io_status;
1360 
1361 	switch (rc) {
1362 	case 0:
1363 		io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1364 		break;
1365 	case -ENOMEM:
1366 		io_status = SPDK_BDEV_IO_STATUS_NOMEM;
1367 		break;
1368 	case -ENXIO:
1369 	/* fallthrough */
1370 	default:
1371 		io_status = SPDK_BDEV_IO_STATUS_FAILED;
1372 		break;
1373 	}
1374 
1375 	__bdev_nvme_io_complete(bdev_io, io_status, NULL);
1376 }
1377 
1378 static void
1379 bdev_nvme_clear_io_path_caches_done(struct spdk_io_channel_iter *i, int status)
1380 {
1381 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
1382 
1383 	pthread_mutex_lock(&nvme_ctrlr->mutex);
1384 
1385 	assert(nvme_ctrlr->io_path_cache_clearing == true);
1386 	nvme_ctrlr->io_path_cache_clearing = false;
1387 
1388 	if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
1389 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
1390 		return;
1391 	}
1392 
1393 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
1394 
1395 	nvme_ctrlr_unregister(nvme_ctrlr);
1396 }
1397 
1398 static void
1399 _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair)
1400 {
1401 	struct nvme_io_path *io_path;
1402 
1403 	TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) {
1404 		bdev_nvme_clear_current_io_path(io_path->nbdev_ch);
1405 	}
1406 }
1407 
1408 static void
1409 bdev_nvme_clear_io_path_cache(struct spdk_io_channel_iter *i)
1410 {
1411 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
1412 	struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch);
1413 
1414 	assert(ctrlr_ch->qpair != NULL);
1415 
1416 	_bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair);
1417 
1418 	spdk_for_each_channel_continue(i, 0);
1419 }
1420 
1421 static void
1422 bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr)
1423 {
1424 	pthread_mutex_lock(&nvme_ctrlr->mutex);
1425 	if (!nvme_ctrlr_is_available(nvme_ctrlr) ||
1426 	    nvme_ctrlr->io_path_cache_clearing) {
1427 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
1428 		return;
1429 	}
1430 
1431 	nvme_ctrlr->io_path_cache_clearing = true;
1432 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
1433 
1434 	spdk_for_each_channel(nvme_ctrlr,
1435 			      bdev_nvme_clear_io_path_cache,
1436 			      NULL,
1437 			      bdev_nvme_clear_io_path_caches_done);
1438 }
1439 
1440 static struct nvme_qpair *
1441 nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
1442 {
1443 	struct nvme_qpair *nvme_qpair;
1444 
1445 	TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) {
1446 		if (nvme_qpair->qpair == qpair) {
1447 			break;
1448 		}
1449 	}
1450 
1451 	return nvme_qpair;
1452 }
1453 
1454 static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair);
1455 
1456 static void
1457 bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx)
1458 {
1459 	struct nvme_poll_group *group = poll_group_ctx;
1460 	struct nvme_qpair *nvme_qpair;
1461 	struct nvme_ctrlr_channel *ctrlr_ch;
1462 
1463 	nvme_qpair = nvme_poll_group_get_qpair(group, qpair);
1464 	if (nvme_qpair == NULL) {
1465 		return;
1466 	}
1467 
1468 	if (nvme_qpair->qpair != NULL) {
1469 		spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair);
1470 		nvme_qpair->qpair = NULL;
1471 	}
1472 
1473 	_bdev_nvme_clear_io_path_cache(nvme_qpair);
1474 
1475 	ctrlr_ch = nvme_qpair->ctrlr_ch;
1476 
1477 	if (ctrlr_ch != NULL) {
1478 		if (ctrlr_ch->reset_iter != NULL) {
1479 			/* If we are already in a full reset sequence, we do not have
1480 			 * to restart it. Just move to the next ctrlr_channel.
1481 			 */
1482 			SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed in a reset ctrlr sequence.\n",
1483 				      qpair);
1484 			spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0);
1485 			ctrlr_ch->reset_iter = NULL;
1486 		} else {
1487 			/* qpair was disconnected unexpectedly. Reset controller for recovery. */
1488 			SPDK_NOTICELOG("qpair %p was disconnected and freed. reset controller.\n", qpair);
1489 			bdev_nvme_failover(nvme_qpair->ctrlr, false);
1490 		}
1491 	} else {
1492 		/* In this case, ctrlr_channel is already deleted. */
1493 		SPDK_DEBUGLOG(bdev_nvme, "qpair %p was disconnected and freed. delete nvme_qpair.\n", qpair);
1494 		nvme_qpair_delete(nvme_qpair);
1495 	}
1496 }
1497 
1498 static void
1499 bdev_nvme_check_io_qpairs(struct nvme_poll_group *group)
1500 {
1501 	struct nvme_qpair *nvme_qpair;
1502 
1503 	TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) {
1504 		if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) {
1505 			continue;
1506 		}
1507 
1508 		if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) !=
1509 		    SPDK_NVME_QPAIR_FAILURE_NONE) {
1510 			_bdev_nvme_clear_io_path_cache(nvme_qpair);
1511 		}
1512 	}
1513 }
1514 
1515 static int
1516 bdev_nvme_poll(void *arg)
1517 {
1518 	struct nvme_poll_group *group = arg;
1519 	int64_t num_completions;
1520 
1521 	if (group->collect_spin_stat && group->start_ticks == 0) {
1522 		group->start_ticks = spdk_get_ticks();
1523 	}
1524 
1525 	num_completions = spdk_nvme_poll_group_process_completions(group->group, 0,
1526 			  bdev_nvme_disconnected_qpair_cb);
1527 	if (group->collect_spin_stat) {
1528 		if (num_completions > 0) {
1529 			if (group->end_ticks != 0) {
1530 				group->spin_ticks += (group->end_ticks - group->start_ticks);
1531 				group->end_ticks = 0;
1532 			}
1533 			group->start_ticks = 0;
1534 		} else {
1535 			group->end_ticks = spdk_get_ticks();
1536 		}
1537 	}
1538 
1539 	if (spdk_unlikely(num_completions < 0)) {
1540 		bdev_nvme_check_io_qpairs(group);
1541 	}
1542 
1543 	return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
1544 }
1545 
1546 static int bdev_nvme_poll_adminq(void *arg);
1547 
1548 static void
1549 bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_period_us)
1550 {
1551 	spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller);
1552 
1553 	nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq,
1554 					  nvme_ctrlr, new_period_us);
1555 }
1556 
1557 static int
1558 bdev_nvme_poll_adminq(void *arg)
1559 {
1560 	int32_t rc;
1561 	struct nvme_ctrlr *nvme_ctrlr = arg;
1562 	nvme_ctrlr_disconnected_cb disconnected_cb;
1563 
1564 	assert(nvme_ctrlr != NULL);
1565 
1566 	rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr);
1567 	if (rc < 0) {
1568 		disconnected_cb = nvme_ctrlr->disconnected_cb;
1569 		nvme_ctrlr->disconnected_cb = NULL;
1570 
1571 		if (rc == -ENXIO && disconnected_cb != NULL) {
1572 			bdev_nvme_change_adminq_poll_period(nvme_ctrlr,
1573 							    g_opts.nvme_adminq_poll_period_us);
1574 			disconnected_cb(nvme_ctrlr);
1575 		} else {
1576 			bdev_nvme_failover(nvme_ctrlr, false);
1577 		}
1578 	} else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) !=
1579 		   SPDK_NVME_QPAIR_FAILURE_NONE) {
1580 		bdev_nvme_clear_io_path_caches(nvme_ctrlr);
1581 	}
1582 
1583 	return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY;
1584 }
1585 
1586 static void
1587 _bdev_nvme_unregister_dev_cb(void *io_device)
1588 {
1589 	struct nvme_bdev *nvme_disk = io_device;
1590 
1591 	free(nvme_disk->disk.name);
1592 	free(nvme_disk->err_stat);
1593 	free(nvme_disk);
1594 }
1595 
1596 static int
1597 bdev_nvme_destruct(void *ctx)
1598 {
1599 	struct nvme_bdev *nvme_disk = ctx;
1600 	struct nvme_ns *nvme_ns, *tmp_nvme_ns;
1601 
1602 	SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid);
1603 
1604 	TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) {
1605 		pthread_mutex_lock(&nvme_ns->ctrlr->mutex);
1606 
1607 		nvme_ns->bdev = NULL;
1608 
1609 		assert(nvme_ns->id > 0);
1610 
1611 		if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) {
1612 			pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
1613 
1614 			nvme_ctrlr_release(nvme_ns->ctrlr);
1615 			nvme_ns_free(nvme_ns);
1616 		} else {
1617 			pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
1618 		}
1619 	}
1620 
1621 	pthread_mutex_lock(&g_bdev_nvme_mutex);
1622 	TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq);
1623 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
1624 
1625 	spdk_io_device_unregister(nvme_disk, _bdev_nvme_unregister_dev_cb);
1626 
1627 	return 0;
1628 }
1629 
1630 static int
1631 bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair)
1632 {
1633 	struct nvme_ctrlr *nvme_ctrlr;
1634 	struct spdk_nvme_io_qpair_opts opts;
1635 	struct spdk_nvme_qpair *qpair;
1636 	int rc;
1637 
1638 	nvme_ctrlr = nvme_qpair->ctrlr;
1639 
1640 	spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts));
1641 	opts.delay_cmd_submit = g_opts.delay_cmd_submit;
1642 	opts.create_only = true;
1643 	opts.async_mode = true;
1644 	opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests);
1645 	g_opts.io_queue_requests = opts.io_queue_requests;
1646 
1647 	qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts));
1648 	if (qpair == NULL) {
1649 		return -1;
1650 	}
1651 
1652 	SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name,
1653 			   spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread));
1654 
1655 	assert(nvme_qpair->group != NULL);
1656 
1657 	rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair);
1658 	if (rc != 0) {
1659 		SPDK_ERRLOG("Unable to begin polling on NVMe Channel.\n");
1660 		goto err;
1661 	}
1662 
1663 	rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair);
1664 	if (rc != 0) {
1665 		SPDK_ERRLOG("Unable to connect I/O qpair.\n");
1666 		goto err;
1667 	}
1668 
1669 	nvme_qpair->qpair = qpair;
1670 
1671 	if (!g_opts.disable_auto_failback) {
1672 		_bdev_nvme_clear_io_path_cache(nvme_qpair);
1673 	}
1674 
1675 	return 0;
1676 
1677 err:
1678 	spdk_nvme_ctrlr_free_io_qpair(qpair);
1679 
1680 	return rc;
1681 }
1682 
1683 static void
1684 bdev_nvme_complete_pending_resets(struct spdk_io_channel_iter *i)
1685 {
1686 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
1687 	struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch);
1688 	enum spdk_bdev_io_status status = SPDK_BDEV_IO_STATUS_SUCCESS;
1689 	struct spdk_bdev_io *bdev_io;
1690 
1691 	if (spdk_io_channel_iter_get_ctx(i) != NULL) {
1692 		status = SPDK_BDEV_IO_STATUS_FAILED;
1693 	}
1694 
1695 	while (!TAILQ_EMPTY(&ctrlr_ch->pending_resets)) {
1696 		bdev_io = TAILQ_FIRST(&ctrlr_ch->pending_resets);
1697 		TAILQ_REMOVE(&ctrlr_ch->pending_resets, bdev_io, module_link);
1698 		__bdev_nvme_io_complete(bdev_io, status, NULL);
1699 	}
1700 
1701 	spdk_for_each_channel_continue(i, 0);
1702 }
1703 
1704 static void
1705 bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove)
1706 {
1707 	struct nvme_path_id *path_id, *next_path;
1708 	int rc __attribute__((unused));
1709 
1710 	path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
1711 	assert(path_id);
1712 	assert(path_id == nvme_ctrlr->active_path_id);
1713 	next_path = TAILQ_NEXT(path_id, link);
1714 
1715 	path_id->is_failed = true;
1716 
1717 	if (next_path) {
1718 		assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE);
1719 
1720 		SPDK_NOTICELOG("Start failover from %s:%s to %s:%s\n", path_id->trid.traddr,
1721 			       path_id->trid.trsvcid,	next_path->trid.traddr, next_path->trid.trsvcid);
1722 
1723 		spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr);
1724 		nvme_ctrlr->active_path_id = next_path;
1725 		rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid);
1726 		assert(rc == 0);
1727 		TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link);
1728 		if (!remove) {
1729 			/** Shuffle the old trid to the end of the list and use the new one.
1730 			 * Allows for round robin through multiple connections.
1731 			 */
1732 			TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link);
1733 		} else {
1734 			free(path_id);
1735 		}
1736 	}
1737 }
1738 
1739 static bool
1740 bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr)
1741 {
1742 	int32_t elapsed;
1743 
1744 	if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 ||
1745 	    nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) {
1746 		return false;
1747 	}
1748 
1749 	elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();
1750 	if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) {
1751 		return true;
1752 	} else {
1753 		return false;
1754 	}
1755 }
1756 
1757 static bool
1758 bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr)
1759 {
1760 	uint32_t elapsed;
1761 
1762 	if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) {
1763 		return false;
1764 	}
1765 
1766 	elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();
1767 	if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) {
1768 		return true;
1769 	} else {
1770 		return false;
1771 	}
1772 }
1773 
1774 static void bdev_nvme_reset_complete(struct nvme_ctrlr *nvme_ctrlr, bool success);
1775 
1776 static void
1777 nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb cb_fn)
1778 {
1779 	int rc;
1780 
1781 	rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr);
1782 	if (rc != 0) {
1783 		/* Disconnect fails if ctrlr is already resetting or removed. In this case,
1784 		 * fail the reset sequence immediately.
1785 		 */
1786 		bdev_nvme_reset_complete(nvme_ctrlr, false);
1787 		return;
1788 	}
1789 
1790 	/* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq.
1791 	 * Set callback here to execute the specified operation after ctrlr is really disconnected.
1792 	 */
1793 	assert(nvme_ctrlr->disconnected_cb == NULL);
1794 	nvme_ctrlr->disconnected_cb = cb_fn;
1795 
1796 	/* During disconnection, reduce the period to poll adminq more often. */
1797 	bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0);
1798 }
1799 
1800 enum bdev_nvme_op_after_reset {
1801 	OP_NONE,
1802 	OP_COMPLETE_PENDING_DESTRUCT,
1803 	OP_DESTRUCT,
1804 	OP_DELAYED_RECONNECT,
1805 };
1806 
1807 typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset;
1808 
1809 static _bdev_nvme_op_after_reset
1810 bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success)
1811 {
1812 	if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
1813 		/* Complete pending destruct after reset completes. */
1814 		return OP_COMPLETE_PENDING_DESTRUCT;
1815 	} else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) {
1816 		nvme_ctrlr->reset_start_tsc = 0;
1817 		return OP_NONE;
1818 	} else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
1819 		return OP_DESTRUCT;
1820 	} else {
1821 		if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) {
1822 			nvme_ctrlr->fast_io_fail_timedout = true;
1823 		}
1824 		bdev_nvme_failover_trid(nvme_ctrlr, false);
1825 		return OP_DELAYED_RECONNECT;
1826 	}
1827 }
1828 
1829 static int bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug);
1830 static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
1831 
1832 static int
1833 bdev_nvme_reconnect_delay_timer_expired(void *ctx)
1834 {
1835 	struct nvme_ctrlr *nvme_ctrlr = ctx;
1836 
1837 	SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect_delay, nvme_ctrlr->nbdev_ctrlr->name);
1838 	pthread_mutex_lock(&nvme_ctrlr->mutex);
1839 
1840 	spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
1841 
1842 	assert(nvme_ctrlr->reconnect_is_delayed == true);
1843 	nvme_ctrlr->reconnect_is_delayed = false;
1844 
1845 	if (nvme_ctrlr->destruct) {
1846 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
1847 		return SPDK_POLLER_BUSY;
1848 	}
1849 
1850 	assert(nvme_ctrlr->resetting == false);
1851 	nvme_ctrlr->resetting = true;
1852 
1853 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
1854 
1855 	spdk_poller_resume(nvme_ctrlr->adminq_timer_poller);
1856 
1857 	bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
1858 	return SPDK_POLLER_BUSY;
1859 }
1860 
1861 static void
1862 bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr)
1863 {
1864 	spdk_poller_pause(nvme_ctrlr->adminq_timer_poller);
1865 
1866 	assert(nvme_ctrlr->reconnect_is_delayed == false);
1867 	nvme_ctrlr->reconnect_is_delayed = true;
1868 
1869 	assert(nvme_ctrlr->reconnect_delay_timer == NULL);
1870 	nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired,
1871 					    nvme_ctrlr,
1872 					    nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC);
1873 }
1874 
1875 static void
1876 _bdev_nvme_reset_complete(struct spdk_io_channel_iter *i, int status)
1877 {
1878 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
1879 	bool success = spdk_io_channel_iter_get_ctx(i) == NULL;
1880 	struct nvme_path_id *path_id;
1881 	bdev_nvme_reset_cb reset_cb_fn = nvme_ctrlr->reset_cb_fn;
1882 	void *reset_cb_arg = nvme_ctrlr->reset_cb_arg;
1883 	enum bdev_nvme_op_after_reset op_after_reset;
1884 
1885 	assert(nvme_ctrlr->thread == spdk_get_thread());
1886 
1887 	nvme_ctrlr->reset_cb_fn = NULL;
1888 	nvme_ctrlr->reset_cb_arg = NULL;
1889 
1890 	if (!success) {
1891 		SPDK_ERRLOG("Resetting controller failed.\n");
1892 	} else {
1893 		SPDK_NOTICELOG("Resetting controller successful.\n");
1894 	}
1895 
1896 	pthread_mutex_lock(&nvme_ctrlr->mutex);
1897 	nvme_ctrlr->resetting = false;
1898 
1899 	path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
1900 	assert(path_id != NULL);
1901 	assert(path_id == nvme_ctrlr->active_path_id);
1902 
1903 	path_id->is_failed = !success;
1904 
1905 	op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success);
1906 
1907 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
1908 
1909 	if (reset_cb_fn) {
1910 		reset_cb_fn(reset_cb_arg, success);
1911 	}
1912 
1913 	switch (op_after_reset) {
1914 	case OP_COMPLETE_PENDING_DESTRUCT:
1915 		nvme_ctrlr_unregister(nvme_ctrlr);
1916 		break;
1917 	case OP_DESTRUCT:
1918 		bdev_nvme_delete_ctrlr(nvme_ctrlr, false);
1919 		break;
1920 	case OP_DELAYED_RECONNECT:
1921 		nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_start_reconnect_delay_timer);
1922 		break;
1923 	default:
1924 		break;
1925 	}
1926 }
1927 
1928 static void
1929 bdev_nvme_reset_complete(struct nvme_ctrlr *nvme_ctrlr, bool success)
1930 {
1931 	/* Make sure we clear any pending resets before returning. */
1932 	spdk_for_each_channel(nvme_ctrlr,
1933 			      bdev_nvme_complete_pending_resets,
1934 			      success ? NULL : (void *)0x1,
1935 			      _bdev_nvme_reset_complete);
1936 }
1937 
1938 static void
1939 bdev_nvme_reset_create_qpairs_failed(struct spdk_io_channel_iter *i, int status)
1940 {
1941 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
1942 
1943 	bdev_nvme_reset_complete(nvme_ctrlr, false);
1944 }
1945 
1946 static void
1947 bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i)
1948 {
1949 	struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
1950 	struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch);
1951 	struct nvme_qpair *nvme_qpair;
1952 
1953 	nvme_qpair = ctrlr_ch->qpair;
1954 	assert(nvme_qpair != NULL);
1955 
1956 	_bdev_nvme_clear_io_path_cache(nvme_qpair);
1957 
1958 	if (nvme_qpair->qpair != NULL) {
1959 		spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair);
1960 
1961 		/* The current full reset sequence will move to the next
1962 		 * ctrlr_channel after the qpair is actually disconnected.
1963 		 */
1964 		assert(ctrlr_ch->reset_iter == NULL);
1965 		ctrlr_ch->reset_iter = i;
1966 	} else {
1967 		spdk_for_each_channel_continue(i, 0);
1968 	}
1969 }
1970 
1971 static void
1972 bdev_nvme_reset_create_qpairs_done(struct spdk_io_channel_iter *i, int status)
1973 {
1974 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
1975 
1976 	if (status == 0) {
1977 		bdev_nvme_reset_complete(nvme_ctrlr, true);
1978 	} else {
1979 		/* Delete the added qpairs and quiesce ctrlr to make the states clean. */
1980 		spdk_for_each_channel(nvme_ctrlr,
1981 				      bdev_nvme_reset_destroy_qpair,
1982 				      NULL,
1983 				      bdev_nvme_reset_create_qpairs_failed);
1984 	}
1985 }
1986 
1987 static void
1988 bdev_nvme_reset_create_qpair(struct spdk_io_channel_iter *i)
1989 {
1990 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
1991 	struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(_ch);
1992 	int rc;
1993 
1994 	rc = bdev_nvme_create_qpair(ctrlr_ch->qpair);
1995 
1996 	spdk_for_each_channel_continue(i, rc);
1997 }
1998 
1999 static int
2000 bdev_nvme_reconnect_ctrlr_poll(void *arg)
2001 {
2002 	struct nvme_ctrlr *nvme_ctrlr = arg;
2003 	int rc = -ETIMEDOUT;
2004 
2005 	if (!bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
2006 		rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr);
2007 		if (rc == -EAGAIN) {
2008 			return SPDK_POLLER_BUSY;
2009 		}
2010 	}
2011 
2012 	spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
2013 	if (rc == 0) {
2014 		/* Recreate all of the I/O queue pairs */
2015 		spdk_for_each_channel(nvme_ctrlr,
2016 				      bdev_nvme_reset_create_qpair,
2017 				      NULL,
2018 				      bdev_nvme_reset_create_qpairs_done);
2019 	} else {
2020 		bdev_nvme_reset_complete(nvme_ctrlr, false);
2021 	}
2022 	return SPDK_POLLER_BUSY;
2023 }
2024 
2025 static void
2026 bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2027 {
2028 	spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr);
2029 
2030 	SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect, nvme_ctrlr->nbdev_ctrlr->name);
2031 	assert(nvme_ctrlr->reset_detach_poller == NULL);
2032 	nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll,
2033 					  nvme_ctrlr, 0);
2034 }
2035 
2036 static void
2037 bdev_nvme_reset_ctrlr(struct spdk_io_channel_iter *i, int status)
2038 {
2039 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
2040 
2041 	SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reset, nvme_ctrlr->nbdev_ctrlr->name);
2042 	assert(status == 0);
2043 
2044 	if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
2045 		bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
2046 	} else {
2047 		nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr);
2048 	}
2049 }
2050 
2051 static void
2052 bdev_nvme_reset_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr)
2053 {
2054 	spdk_for_each_channel(nvme_ctrlr,
2055 			      bdev_nvme_reset_destroy_qpair,
2056 			      NULL,
2057 			      bdev_nvme_reset_ctrlr);
2058 }
2059 
2060 static void
2061 _bdev_nvme_reset(void *ctx)
2062 {
2063 	struct nvme_ctrlr *nvme_ctrlr = ctx;
2064 
2065 	assert(nvme_ctrlr->resetting == true);
2066 	assert(nvme_ctrlr->thread == spdk_get_thread());
2067 
2068 	if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
2069 		nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs);
2070 	} else {
2071 		bdev_nvme_reset_destroy_qpairs(nvme_ctrlr);
2072 	}
2073 }
2074 
2075 static int
2076 bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr)
2077 {
2078 	pthread_mutex_lock(&nvme_ctrlr->mutex);
2079 	if (nvme_ctrlr->destruct) {
2080 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
2081 		return -ENXIO;
2082 	}
2083 
2084 	if (nvme_ctrlr->resetting) {
2085 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
2086 		SPDK_NOTICELOG("Unable to perform reset, already in progress.\n");
2087 		return -EBUSY;
2088 	}
2089 
2090 	if (nvme_ctrlr->reconnect_is_delayed) {
2091 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
2092 		SPDK_NOTICELOG("Reconnect is already scheduled.\n");
2093 		return -EBUSY;
2094 	}
2095 
2096 	nvme_ctrlr->resetting = true;
2097 
2098 	assert(nvme_ctrlr->reset_start_tsc == 0);
2099 	nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2100 
2101 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
2102 
2103 	spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr);
2104 	return 0;
2105 }
2106 
2107 int
2108 bdev_nvme_reset_rpc(struct nvme_ctrlr *nvme_ctrlr, bdev_nvme_reset_cb cb_fn, void *cb_arg)
2109 {
2110 	int rc;
2111 
2112 	rc = bdev_nvme_reset(nvme_ctrlr);
2113 	if (rc == 0) {
2114 		nvme_ctrlr->reset_cb_fn = cb_fn;
2115 		nvme_ctrlr->reset_cb_arg = cb_arg;
2116 	}
2117 	return rc;
2118 }
2119 
2120 static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio);
2121 
2122 static void
2123 bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio)
2124 {
2125 	enum spdk_bdev_io_status io_status;
2126 
2127 	if (bio->cpl.cdw0 == 0) {
2128 		io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
2129 	} else {
2130 		io_status = SPDK_BDEV_IO_STATUS_FAILED;
2131 	}
2132 
2133 	__bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL);
2134 }
2135 
2136 static void
2137 _bdev_nvme_reset_io_continue(void *ctx)
2138 {
2139 	struct nvme_bdev_io *bio = ctx;
2140 	struct nvme_io_path *prev_io_path, *next_io_path;
2141 	int rc;
2142 
2143 	prev_io_path = bio->io_path;
2144 	bio->io_path = NULL;
2145 
2146 	if (bio->cpl.cdw0 != 0) {
2147 		goto complete;
2148 	}
2149 
2150 	next_io_path = STAILQ_NEXT(prev_io_path, stailq);
2151 	if (next_io_path == NULL) {
2152 		goto complete;
2153 	}
2154 
2155 	rc = _bdev_nvme_reset_io(next_io_path, bio);
2156 	if (rc == 0) {
2157 		return;
2158 	}
2159 
2160 	bio->cpl.cdw0 = 1;
2161 
2162 complete:
2163 	bdev_nvme_reset_io_complete(bio);
2164 }
2165 
2166 static void
2167 bdev_nvme_reset_io_continue(void *cb_arg, bool success)
2168 {
2169 	struct nvme_bdev_io *bio = cb_arg;
2170 
2171 	bio->cpl.cdw0 = !success;
2172 
2173 	spdk_thread_send_msg(bio->orig_thread, _bdev_nvme_reset_io_continue, bio);
2174 }
2175 
2176 static int
2177 _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio)
2178 {
2179 	struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
2180 	struct nvme_ctrlr_channel *ctrlr_ch;
2181 	struct spdk_bdev_io *bdev_io;
2182 	int rc;
2183 
2184 	rc = bdev_nvme_reset(nvme_ctrlr);
2185 	if (rc == 0) {
2186 		assert(bio->io_path == NULL);
2187 		bio->io_path = io_path;
2188 
2189 		assert(nvme_ctrlr->reset_cb_fn == NULL);
2190 		assert(nvme_ctrlr->reset_cb_arg == NULL);
2191 		nvme_ctrlr->reset_cb_fn = bdev_nvme_reset_io_continue;
2192 		nvme_ctrlr->reset_cb_arg = bio;
2193 	} else if (rc == -EBUSY) {
2194 		ctrlr_ch = io_path->qpair->ctrlr_ch;
2195 		assert(ctrlr_ch != NULL);
2196 		/*
2197 		 * Reset call is queued only if it is from the app framework. This is on purpose so that
2198 		 * we don't interfere with the app framework reset strategy. i.e. we are deferring to the
2199 		 * upper level. If they are in the middle of a reset, we won't try to schedule another one.
2200 		 */
2201 		bdev_io = spdk_bdev_io_from_ctx(bio);
2202 		TAILQ_INSERT_TAIL(&ctrlr_ch->pending_resets, bdev_io, module_link);
2203 		rc = 0;
2204 	}
2205 
2206 	return rc;
2207 }
2208 
2209 static void
2210 bdev_nvme_reset_io(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio)
2211 {
2212 	struct nvme_io_path *io_path;
2213 	int rc;
2214 
2215 	bio->cpl.cdw0 = 0;
2216 	bio->orig_thread = spdk_get_thread();
2217 
2218 	/* Reset all nvme_ctrlrs of a bdev controller sequentially. */
2219 	io_path = STAILQ_FIRST(&nbdev_ch->io_path_list);
2220 	assert(io_path != NULL);
2221 
2222 	rc = _bdev_nvme_reset_io(io_path, bio);
2223 	if (rc != 0) {
2224 		bio->cpl.cdw0 = 1;
2225 		bdev_nvme_reset_io_complete(bio);
2226 	}
2227 }
2228 
2229 static int
2230 bdev_nvme_failover_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool remove)
2231 {
2232 	if (nvme_ctrlr->destruct) {
2233 		/* Don't bother resetting if the controller is in the process of being destructed. */
2234 		return -ENXIO;
2235 	}
2236 
2237 	if (nvme_ctrlr->resetting) {
2238 		SPDK_NOTICELOG("Unable to perform reset, already in progress.\n");
2239 		return -EBUSY;
2240 	}
2241 
2242 	bdev_nvme_failover_trid(nvme_ctrlr, remove);
2243 
2244 	if (nvme_ctrlr->reconnect_is_delayed) {
2245 		SPDK_NOTICELOG("Reconnect is already scheduled.\n");
2246 
2247 		/* We rely on the next reconnect for the failover. */
2248 		return -EALREADY;
2249 	}
2250 
2251 	nvme_ctrlr->resetting = true;
2252 
2253 	assert(nvme_ctrlr->reset_start_tsc == 0);
2254 	nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2255 
2256 	return 0;
2257 }
2258 
2259 static int
2260 bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove)
2261 {
2262 	int rc;
2263 
2264 	pthread_mutex_lock(&nvme_ctrlr->mutex);
2265 	rc = bdev_nvme_failover_unsafe(nvme_ctrlr, remove);
2266 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
2267 
2268 	if (rc == 0) {
2269 		spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr);
2270 	} else if (rc == -EALREADY) {
2271 		rc = 0;
2272 	}
2273 
2274 	return rc;
2275 }
2276 
2277 static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks,
2278 			   uint64_t num_blocks);
2279 
2280 static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks,
2281 				  uint64_t num_blocks);
2282 
2283 static int bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks,
2284 			  uint64_t src_offset_blocks,
2285 			  uint64_t num_blocks);
2286 
2287 static void
2288 bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
2289 		     bool success)
2290 {
2291 	struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx;
2292 	struct spdk_bdev *bdev = bdev_io->bdev;
2293 	int ret;
2294 
2295 	if (!success) {
2296 		ret = -EINVAL;
2297 		goto exit;
2298 	}
2299 
2300 	if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) {
2301 		ret = -ENXIO;
2302 		goto exit;
2303 	}
2304 
2305 	ret = bdev_nvme_readv(bio,
2306 			      bdev_io->u.bdev.iovs,
2307 			      bdev_io->u.bdev.iovcnt,
2308 			      bdev_io->u.bdev.md_buf,
2309 			      bdev_io->u.bdev.num_blocks,
2310 			      bdev_io->u.bdev.offset_blocks,
2311 			      bdev->dif_check_flags,
2312 			      bdev_io->u.bdev.memory_domain,
2313 			      bdev_io->u.bdev.memory_domain_ctx);
2314 
2315 exit:
2316 	if (spdk_unlikely(ret != 0)) {
2317 		bdev_nvme_io_complete(bio, ret);
2318 	}
2319 }
2320 
2321 static inline void
2322 _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io)
2323 {
2324 	struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
2325 	struct spdk_bdev *bdev = bdev_io->bdev;
2326 	struct nvme_bdev_io *nbdev_io_to_abort;
2327 	int rc = 0;
2328 
2329 	switch (bdev_io->type) {
2330 	case SPDK_BDEV_IO_TYPE_READ:
2331 		if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) {
2332 			rc = bdev_nvme_readv(nbdev_io,
2333 					     bdev_io->u.bdev.iovs,
2334 					     bdev_io->u.bdev.iovcnt,
2335 					     bdev_io->u.bdev.md_buf,
2336 					     bdev_io->u.bdev.num_blocks,
2337 					     bdev_io->u.bdev.offset_blocks,
2338 					     bdev->dif_check_flags,
2339 					     bdev_io->u.bdev.memory_domain,
2340 					     bdev_io->u.bdev.memory_domain_ctx);
2341 		} else {
2342 			spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb,
2343 					     bdev_io->u.bdev.num_blocks * bdev->blocklen);
2344 			rc = 0;
2345 		}
2346 		break;
2347 	case SPDK_BDEV_IO_TYPE_WRITE:
2348 		rc = bdev_nvme_writev(nbdev_io,
2349 				      bdev_io->u.bdev.iovs,
2350 				      bdev_io->u.bdev.iovcnt,
2351 				      bdev_io->u.bdev.md_buf,
2352 				      bdev_io->u.bdev.num_blocks,
2353 				      bdev_io->u.bdev.offset_blocks,
2354 				      bdev->dif_check_flags,
2355 				      bdev_io->u.bdev.memory_domain,
2356 				      bdev_io->u.bdev.memory_domain_ctx);
2357 		break;
2358 	case SPDK_BDEV_IO_TYPE_COMPARE:
2359 		rc = bdev_nvme_comparev(nbdev_io,
2360 					bdev_io->u.bdev.iovs,
2361 					bdev_io->u.bdev.iovcnt,
2362 					bdev_io->u.bdev.md_buf,
2363 					bdev_io->u.bdev.num_blocks,
2364 					bdev_io->u.bdev.offset_blocks,
2365 					bdev->dif_check_flags);
2366 		break;
2367 	case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE:
2368 		rc = bdev_nvme_comparev_and_writev(nbdev_io,
2369 						   bdev_io->u.bdev.iovs,
2370 						   bdev_io->u.bdev.iovcnt,
2371 						   bdev_io->u.bdev.fused_iovs,
2372 						   bdev_io->u.bdev.fused_iovcnt,
2373 						   bdev_io->u.bdev.md_buf,
2374 						   bdev_io->u.bdev.num_blocks,
2375 						   bdev_io->u.bdev.offset_blocks,
2376 						   bdev->dif_check_flags);
2377 		break;
2378 	case SPDK_BDEV_IO_TYPE_UNMAP:
2379 		rc = bdev_nvme_unmap(nbdev_io,
2380 				     bdev_io->u.bdev.offset_blocks,
2381 				     bdev_io->u.bdev.num_blocks);
2382 		break;
2383 	case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
2384 		rc =  bdev_nvme_write_zeroes(nbdev_io,
2385 					     bdev_io->u.bdev.offset_blocks,
2386 					     bdev_io->u.bdev.num_blocks);
2387 		break;
2388 	case SPDK_BDEV_IO_TYPE_RESET:
2389 		nbdev_io->io_path = NULL;
2390 		bdev_nvme_reset_io(nbdev_ch, nbdev_io);
2391 		break;
2392 	case SPDK_BDEV_IO_TYPE_FLUSH:
2393 		bdev_nvme_io_complete(nbdev_io, 0);
2394 		break;
2395 	case SPDK_BDEV_IO_TYPE_ZONE_APPEND:
2396 		rc = bdev_nvme_zone_appendv(nbdev_io,
2397 					    bdev_io->u.bdev.iovs,
2398 					    bdev_io->u.bdev.iovcnt,
2399 					    bdev_io->u.bdev.md_buf,
2400 					    bdev_io->u.bdev.num_blocks,
2401 					    bdev_io->u.bdev.offset_blocks,
2402 					    bdev->dif_check_flags);
2403 		break;
2404 	case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO:
2405 		rc = bdev_nvme_get_zone_info(nbdev_io,
2406 					     bdev_io->u.zone_mgmt.zone_id,
2407 					     bdev_io->u.zone_mgmt.num_zones,
2408 					     bdev_io->u.zone_mgmt.buf);
2409 		break;
2410 	case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT:
2411 		rc = bdev_nvme_zone_management(nbdev_io,
2412 					       bdev_io->u.zone_mgmt.zone_id,
2413 					       bdev_io->u.zone_mgmt.zone_action);
2414 		break;
2415 	case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
2416 		nbdev_io->io_path = NULL;
2417 		bdev_nvme_admin_passthru(nbdev_ch,
2418 					 nbdev_io,
2419 					 &bdev_io->u.nvme_passthru.cmd,
2420 					 bdev_io->u.nvme_passthru.buf,
2421 					 bdev_io->u.nvme_passthru.nbytes);
2422 		break;
2423 	case SPDK_BDEV_IO_TYPE_NVME_IO:
2424 		rc = bdev_nvme_io_passthru(nbdev_io,
2425 					   &bdev_io->u.nvme_passthru.cmd,
2426 					   bdev_io->u.nvme_passthru.buf,
2427 					   bdev_io->u.nvme_passthru.nbytes);
2428 		break;
2429 	case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
2430 		rc = bdev_nvme_io_passthru_md(nbdev_io,
2431 					      &bdev_io->u.nvme_passthru.cmd,
2432 					      bdev_io->u.nvme_passthru.buf,
2433 					      bdev_io->u.nvme_passthru.nbytes,
2434 					      bdev_io->u.nvme_passthru.md_buf,
2435 					      bdev_io->u.nvme_passthru.md_len);
2436 		break;
2437 	case SPDK_BDEV_IO_TYPE_ABORT:
2438 		nbdev_io->io_path = NULL;
2439 		nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx;
2440 		bdev_nvme_abort(nbdev_ch,
2441 				nbdev_io,
2442 				nbdev_io_to_abort);
2443 		break;
2444 	case SPDK_BDEV_IO_TYPE_COPY:
2445 		rc = bdev_nvme_copy(nbdev_io,
2446 				    bdev_io->u.bdev.offset_blocks,
2447 				    bdev_io->u.bdev.copy.src_offset_blocks,
2448 				    bdev_io->u.bdev.num_blocks);
2449 		break;
2450 	default:
2451 		rc = -EINVAL;
2452 		break;
2453 	}
2454 
2455 	if (spdk_unlikely(rc != 0)) {
2456 		bdev_nvme_io_complete(nbdev_io, rc);
2457 	}
2458 }
2459 
2460 static void
2461 bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
2462 {
2463 	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
2464 	struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
2465 
2466 	if (spdk_likely(nbdev_io->submit_tsc == 0)) {
2467 		nbdev_io->submit_tsc = spdk_bdev_io_get_submit_tsc(bdev_io);
2468 	} else {
2469 		/* There are cases where submit_tsc != 0, i.e. retry I/O.
2470 		 * We need to update submit_tsc here.
2471 		 */
2472 		nbdev_io->submit_tsc = spdk_get_ticks();
2473 	}
2474 
2475 	spdk_trace_record(TRACE_BDEV_NVME_IO_START, 0, 0, (uintptr_t)nbdev_io, (uintptr_t)bdev_io);
2476 	nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch);
2477 	if (spdk_unlikely(!nbdev_io->io_path)) {
2478 		if (!bdev_nvme_io_type_is_admin(bdev_io->type)) {
2479 			bdev_nvme_io_complete(nbdev_io, -ENXIO);
2480 			return;
2481 		}
2482 
2483 		/* Admin commands do not use the optimal I/O path.
2484 		 * Simply fall through even if it is not found.
2485 		 */
2486 	}
2487 
2488 	_bdev_nvme_submit_request(nbdev_ch, bdev_io);
2489 }
2490 
2491 static bool
2492 bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
2493 {
2494 	struct nvme_bdev *nbdev = ctx;
2495 	struct nvme_ns *nvme_ns;
2496 	struct spdk_nvme_ns *ns;
2497 	struct spdk_nvme_ctrlr *ctrlr;
2498 	const struct spdk_nvme_ctrlr_data *cdata;
2499 
2500 	nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
2501 	assert(nvme_ns != NULL);
2502 	ns = nvme_ns->ns;
2503 	ctrlr = spdk_nvme_ns_get_ctrlr(ns);
2504 
2505 	switch (io_type) {
2506 	case SPDK_BDEV_IO_TYPE_READ:
2507 	case SPDK_BDEV_IO_TYPE_WRITE:
2508 	case SPDK_BDEV_IO_TYPE_RESET:
2509 	case SPDK_BDEV_IO_TYPE_FLUSH:
2510 	case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
2511 	case SPDK_BDEV_IO_TYPE_NVME_IO:
2512 	case SPDK_BDEV_IO_TYPE_ABORT:
2513 		return true;
2514 
2515 	case SPDK_BDEV_IO_TYPE_COMPARE:
2516 		return spdk_nvme_ns_supports_compare(ns);
2517 
2518 	case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
2519 		return spdk_nvme_ns_get_md_size(ns) ? true : false;
2520 
2521 	case SPDK_BDEV_IO_TYPE_UNMAP:
2522 		cdata = spdk_nvme_ctrlr_get_data(ctrlr);
2523 		return cdata->oncs.dsm;
2524 
2525 	case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
2526 		cdata = spdk_nvme_ctrlr_get_data(ctrlr);
2527 		return cdata->oncs.write_zeroes;
2528 
2529 	case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE:
2530 		if (spdk_nvme_ctrlr_get_flags(ctrlr) &
2531 		    SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) {
2532 			return true;
2533 		}
2534 		return false;
2535 
2536 	case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO:
2537 	case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT:
2538 		return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS;
2539 
2540 	case SPDK_BDEV_IO_TYPE_ZONE_APPEND:
2541 		return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS &&
2542 		       spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED;
2543 
2544 	case SPDK_BDEV_IO_TYPE_COPY:
2545 		cdata = spdk_nvme_ctrlr_get_data(ctrlr);
2546 		return cdata->oncs.copy;
2547 
2548 	default:
2549 		return false;
2550 	}
2551 }
2552 
2553 static int
2554 nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch)
2555 {
2556 	struct nvme_qpair *nvme_qpair;
2557 	struct spdk_io_channel *pg_ch;
2558 	int rc;
2559 
2560 	nvme_qpair = calloc(1, sizeof(*nvme_qpair));
2561 	if (!nvme_qpair) {
2562 		SPDK_ERRLOG("Failed to alloc nvme_qpair.\n");
2563 		return -1;
2564 	}
2565 
2566 	TAILQ_INIT(&nvme_qpair->io_path_list);
2567 
2568 	nvme_qpair->ctrlr = nvme_ctrlr;
2569 	nvme_qpair->ctrlr_ch = ctrlr_ch;
2570 
2571 	pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs);
2572 	if (!pg_ch) {
2573 		free(nvme_qpair);
2574 		return -1;
2575 	}
2576 
2577 	nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch);
2578 
2579 #ifdef SPDK_CONFIG_VTUNE
2580 	nvme_qpair->group->collect_spin_stat = true;
2581 #else
2582 	nvme_qpair->group->collect_spin_stat = false;
2583 #endif
2584 
2585 	rc = bdev_nvme_create_qpair(nvme_qpair);
2586 	if (rc != 0) {
2587 		/* nvme_ctrlr can't create IO qpair if connection is down.
2588 		 *
2589 		 * If reconnect_delay_sec is non-zero, creating IO qpair is retried
2590 		 * after reconnect_delay_sec seconds. If bdev_retry_count is non-zero,
2591 		 * submitted IO will be queued until IO qpair is successfully created.
2592 		 *
2593 		 * Hence, if both are satisfied, ignore the failure.
2594 		 */
2595 		if (nvme_ctrlr->opts.reconnect_delay_sec == 0 || g_opts.bdev_retry_count == 0) {
2596 			spdk_put_io_channel(pg_ch);
2597 			free(nvme_qpair);
2598 			return rc;
2599 		}
2600 	}
2601 
2602 	TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq);
2603 
2604 	ctrlr_ch->qpair = nvme_qpair;
2605 
2606 	pthread_mutex_lock(&nvme_qpair->ctrlr->mutex);
2607 	nvme_qpair->ctrlr->ref++;
2608 	pthread_mutex_unlock(&nvme_qpair->ctrlr->mutex);
2609 
2610 	return 0;
2611 }
2612 
2613 static int
2614 bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf)
2615 {
2616 	struct nvme_ctrlr *nvme_ctrlr = io_device;
2617 	struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf;
2618 
2619 	TAILQ_INIT(&ctrlr_ch->pending_resets);
2620 
2621 	return nvme_qpair_create(nvme_ctrlr, ctrlr_ch);
2622 }
2623 
2624 static void
2625 nvme_qpair_delete(struct nvme_qpair *nvme_qpair)
2626 {
2627 	assert(nvme_qpair->group != NULL);
2628 
2629 	TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq);
2630 
2631 	spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group));
2632 
2633 	nvme_ctrlr_release(nvme_qpair->ctrlr);
2634 
2635 	free(nvme_qpair);
2636 }
2637 
2638 static void
2639 bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf)
2640 {
2641 	struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf;
2642 	struct nvme_qpair *nvme_qpair;
2643 
2644 	nvme_qpair = ctrlr_ch->qpair;
2645 	assert(nvme_qpair != NULL);
2646 
2647 	_bdev_nvme_clear_io_path_cache(nvme_qpair);
2648 
2649 	if (nvme_qpair->qpair != NULL) {
2650 		if (ctrlr_ch->reset_iter == NULL) {
2651 			spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair);
2652 		} else {
2653 			/* Skip current ctrlr_channel in a full reset sequence because
2654 			 * it is being deleted now. The qpair is already being disconnected.
2655 			 * We do not have to restart disconnecting it.
2656 			 */
2657 			spdk_for_each_channel_continue(ctrlr_ch->reset_iter, 0);
2658 		}
2659 
2660 		/* We cannot release a reference to the poll group now.
2661 		 * The qpair may be disconnected asynchronously later.
2662 		 * We need to poll it until it is actually disconnected.
2663 		 * Just detach the qpair from the deleting ctrlr_channel.
2664 		 */
2665 		nvme_qpair->ctrlr_ch = NULL;
2666 	} else {
2667 		assert(ctrlr_ch->reset_iter == NULL);
2668 
2669 		nvme_qpair_delete(nvme_qpair);
2670 	}
2671 }
2672 
2673 static void
2674 bdev_nvme_submit_accel_crc32c(void *ctx, uint32_t *dst, struct iovec *iov,
2675 			      uint32_t iov_cnt, uint32_t seed,
2676 			      spdk_nvme_accel_completion_cb cb_fn, void *cb_arg)
2677 {
2678 	struct nvme_poll_group *group = ctx;
2679 	int rc;
2680 
2681 	assert(group->accel_channel != NULL);
2682 	assert(cb_fn != NULL);
2683 
2684 	rc = spdk_accel_submit_crc32cv(group->accel_channel, dst, iov, iov_cnt, seed, cb_fn, cb_arg);
2685 	if (rc) {
2686 		/* For the two cases, spdk_accel_submit_crc32cv does not call the user's cb_fn */
2687 		if (rc == -ENOMEM || rc == -EINVAL) {
2688 			cb_fn(cb_arg, rc);
2689 		}
2690 		SPDK_ERRLOG("Cannot complete the accelerated crc32c operation with iov=%p\n", iov);
2691 	}
2692 }
2693 
2694 static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = {
2695 	.table_size		= sizeof(struct spdk_nvme_accel_fn_table),
2696 	.submit_accel_crc32c	= bdev_nvme_submit_accel_crc32c,
2697 };
2698 
2699 static int
2700 bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf)
2701 {
2702 	struct nvme_poll_group *group = ctx_buf;
2703 
2704 	TAILQ_INIT(&group->qpair_list);
2705 
2706 	group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table);
2707 	if (group->group == NULL) {
2708 		return -1;
2709 	}
2710 
2711 	group->accel_channel = spdk_accel_get_io_channel();
2712 	if (!group->accel_channel) {
2713 		spdk_nvme_poll_group_destroy(group->group);
2714 		SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n",
2715 			    group);
2716 		return -1;
2717 	}
2718 
2719 	group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us);
2720 
2721 	if (group->poller == NULL) {
2722 		spdk_put_io_channel(group->accel_channel);
2723 		spdk_nvme_poll_group_destroy(group->group);
2724 		return -1;
2725 	}
2726 
2727 	return 0;
2728 }
2729 
2730 static void
2731 bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf)
2732 {
2733 	struct nvme_poll_group *group = ctx_buf;
2734 
2735 	assert(TAILQ_EMPTY(&group->qpair_list));
2736 
2737 	if (group->accel_channel) {
2738 		spdk_put_io_channel(group->accel_channel);
2739 	}
2740 
2741 	spdk_poller_unregister(&group->poller);
2742 	if (spdk_nvme_poll_group_destroy(group->group)) {
2743 		SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n");
2744 		assert(false);
2745 	}
2746 }
2747 
2748 static struct spdk_io_channel *
2749 bdev_nvme_get_io_channel(void *ctx)
2750 {
2751 	struct nvme_bdev *nvme_bdev = ctx;
2752 
2753 	return spdk_get_io_channel(nvme_bdev);
2754 }
2755 
2756 static void *
2757 bdev_nvme_get_module_ctx(void *ctx)
2758 {
2759 	struct nvme_bdev *nvme_bdev = ctx;
2760 	struct nvme_ns *nvme_ns;
2761 
2762 	if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) {
2763 		return NULL;
2764 	}
2765 
2766 	nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list);
2767 	if (!nvme_ns) {
2768 		return NULL;
2769 	}
2770 
2771 	return nvme_ns->ns;
2772 }
2773 
2774 static const char *
2775 _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state)
2776 {
2777 	switch (ana_state) {
2778 	case SPDK_NVME_ANA_OPTIMIZED_STATE:
2779 		return "optimized";
2780 	case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
2781 		return "non_optimized";
2782 	case SPDK_NVME_ANA_INACCESSIBLE_STATE:
2783 		return "inaccessible";
2784 	case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE:
2785 		return "persistent_loss";
2786 	case SPDK_NVME_ANA_CHANGE_STATE:
2787 		return "change";
2788 	default:
2789 		return NULL;
2790 	}
2791 }
2792 
2793 static int
2794 bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
2795 {
2796 	struct spdk_memory_domain **_domains = NULL;
2797 	struct nvme_bdev *nbdev = ctx;
2798 	struct nvme_ns *nvme_ns;
2799 	int i = 0, _array_size = array_size;
2800 	int rc = 0;
2801 
2802 	TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
2803 		if (domains && array_size >= i) {
2804 			_domains = &domains[i];
2805 		} else {
2806 			_domains = NULL;
2807 		}
2808 		rc = spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, _domains, _array_size);
2809 		if (rc > 0) {
2810 			i += rc;
2811 			if (_array_size >= rc) {
2812 				_array_size -= rc;
2813 			} else {
2814 				_array_size = 0;
2815 			}
2816 		} else if (rc < 0) {
2817 			return rc;
2818 		}
2819 	}
2820 
2821 	return i;
2822 }
2823 
2824 static const char *
2825 nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr)
2826 {
2827 	if (nvme_ctrlr->destruct) {
2828 		return "deleting";
2829 	} else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
2830 		return "failed";
2831 	} else if (nvme_ctrlr->resetting) {
2832 		return "resetting";
2833 	} else if (nvme_ctrlr->reconnect_is_delayed > 0) {
2834 		return "reconnect_is_delayed";
2835 	} else {
2836 		return "enabled";
2837 	}
2838 }
2839 
2840 void
2841 nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr)
2842 {
2843 	struct spdk_nvme_transport_id *trid;
2844 	const struct spdk_nvme_ctrlr_opts *opts;
2845 	const struct spdk_nvme_ctrlr_data *cdata;
2846 
2847 	spdk_json_write_object_begin(w);
2848 
2849 	spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr));
2850 
2851 #ifdef SPDK_CONFIG_NVME_CUSE
2852 	size_t cuse_name_size = 128;
2853 	char cuse_name[cuse_name_size];
2854 
2855 	int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size);
2856 	if (rc == 0) {
2857 		spdk_json_write_named_string(w, "cuse_device", cuse_name);
2858 	}
2859 #endif
2860 	trid = &nvme_ctrlr->active_path_id->trid;
2861 	spdk_json_write_named_object_begin(w, "trid");
2862 	nvme_bdev_dump_trid_json(trid, w);
2863 	spdk_json_write_object_end(w);
2864 
2865 	cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
2866 	spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid);
2867 
2868 	opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
2869 	spdk_json_write_named_object_begin(w, "host");
2870 	spdk_json_write_named_string(w, "nqn", opts->hostnqn);
2871 	spdk_json_write_named_string(w, "addr", opts->src_addr);
2872 	spdk_json_write_named_string(w, "svcid", opts->src_svcid);
2873 	spdk_json_write_object_end(w);
2874 
2875 	spdk_json_write_object_end(w);
2876 }
2877 
2878 static void
2879 nvme_namespace_info_json(struct spdk_json_write_ctx *w,
2880 			 struct nvme_ns *nvme_ns)
2881 {
2882 	struct spdk_nvme_ns *ns;
2883 	struct spdk_nvme_ctrlr *ctrlr;
2884 	const struct spdk_nvme_ctrlr_data *cdata;
2885 	const struct spdk_nvme_transport_id *trid;
2886 	union spdk_nvme_vs_register vs;
2887 	const struct spdk_nvme_ns_data *nsdata;
2888 	char buf[128];
2889 
2890 	ns = nvme_ns->ns;
2891 	ctrlr = spdk_nvme_ns_get_ctrlr(ns);
2892 
2893 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
2894 	trid = spdk_nvme_ctrlr_get_transport_id(ctrlr);
2895 	vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr);
2896 
2897 	spdk_json_write_object_begin(w);
2898 
2899 	if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
2900 		spdk_json_write_named_string(w, "pci_address", trid->traddr);
2901 	}
2902 
2903 	spdk_json_write_named_object_begin(w, "trid");
2904 
2905 	nvme_bdev_dump_trid_json(trid, w);
2906 
2907 	spdk_json_write_object_end(w);
2908 
2909 #ifdef SPDK_CONFIG_NVME_CUSE
2910 	size_t cuse_name_size = 128;
2911 	char cuse_name[cuse_name_size];
2912 
2913 	int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns),
2914 					    cuse_name, &cuse_name_size);
2915 	if (rc == 0) {
2916 		spdk_json_write_named_string(w, "cuse_device", cuse_name);
2917 	}
2918 #endif
2919 
2920 	spdk_json_write_named_object_begin(w, "ctrlr_data");
2921 
2922 	spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid);
2923 
2924 	spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid);
2925 
2926 	snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn);
2927 	spdk_str_trim(buf);
2928 	spdk_json_write_named_string(w, "model_number", buf);
2929 
2930 	snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn);
2931 	spdk_str_trim(buf);
2932 	spdk_json_write_named_string(w, "serial_number", buf);
2933 
2934 	snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr);
2935 	spdk_str_trim(buf);
2936 	spdk_json_write_named_string(w, "firmware_revision", buf);
2937 
2938 	if (cdata->subnqn[0] != '\0') {
2939 		spdk_json_write_named_string(w, "subnqn", cdata->subnqn);
2940 	}
2941 
2942 	spdk_json_write_named_object_begin(w, "oacs");
2943 
2944 	spdk_json_write_named_uint32(w, "security", cdata->oacs.security);
2945 	spdk_json_write_named_uint32(w, "format", cdata->oacs.format);
2946 	spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware);
2947 	spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage);
2948 
2949 	spdk_json_write_object_end(w);
2950 
2951 	spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr);
2952 	spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting);
2953 
2954 	spdk_json_write_object_end(w);
2955 
2956 	spdk_json_write_named_object_begin(w, "vs");
2957 
2958 	spdk_json_write_name(w, "nvme_version");
2959 	if (vs.bits.ter) {
2960 		spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter);
2961 	} else {
2962 		spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr);
2963 	}
2964 
2965 	spdk_json_write_object_end(w);
2966 
2967 	nsdata = spdk_nvme_ns_get_data(ns);
2968 
2969 	spdk_json_write_named_object_begin(w, "ns_data");
2970 
2971 	spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns));
2972 
2973 	if (cdata->cmic.ana_reporting) {
2974 		spdk_json_write_named_string(w, "ana_state",
2975 					     _nvme_ana_state_str(nvme_ns->ana_state));
2976 	}
2977 
2978 	spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share);
2979 
2980 	spdk_json_write_object_end(w);
2981 
2982 	if (cdata->oacs.security) {
2983 		spdk_json_write_named_object_begin(w, "security");
2984 
2985 		spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal);
2986 
2987 		spdk_json_write_object_end(w);
2988 	}
2989 
2990 	spdk_json_write_object_end(w);
2991 }
2992 
2993 static const char *
2994 nvme_bdev_get_mp_policy_str(struct nvme_bdev *nbdev)
2995 {
2996 	switch (nbdev->mp_policy) {
2997 	case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE:
2998 		return "active_passive";
2999 	case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE:
3000 		return "active_active";
3001 	default:
3002 		assert(false);
3003 		return "invalid";
3004 	}
3005 }
3006 
3007 static int
3008 bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
3009 {
3010 	struct nvme_bdev *nvme_bdev = ctx;
3011 	struct nvme_ns *nvme_ns;
3012 
3013 	pthread_mutex_lock(&nvme_bdev->mutex);
3014 	spdk_json_write_named_array_begin(w, "nvme");
3015 	TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) {
3016 		nvme_namespace_info_json(w, nvme_ns);
3017 	}
3018 	spdk_json_write_array_end(w);
3019 	spdk_json_write_named_string(w, "mp_policy", nvme_bdev_get_mp_policy_str(nvme_bdev));
3020 	pthread_mutex_unlock(&nvme_bdev->mutex);
3021 
3022 	return 0;
3023 }
3024 
3025 static void
3026 bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
3027 {
3028 	/* No config per bdev needed */
3029 }
3030 
3031 static uint64_t
3032 bdev_nvme_get_spin_time(struct spdk_io_channel *ch)
3033 {
3034 	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
3035 	struct nvme_io_path *io_path;
3036 	struct nvme_poll_group *group;
3037 	uint64_t spin_time = 0;
3038 
3039 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
3040 		group = io_path->qpair->group;
3041 
3042 		if (!group || !group->collect_spin_stat) {
3043 			continue;
3044 		}
3045 
3046 		if (group->end_ticks != 0) {
3047 			group->spin_ticks += (group->end_ticks - group->start_ticks);
3048 			group->end_ticks = 0;
3049 		}
3050 
3051 		spin_time += group->spin_ticks;
3052 		group->start_ticks = 0;
3053 		group->spin_ticks = 0;
3054 	}
3055 
3056 	return (spin_time * 1000000ULL) / spdk_get_ticks_hz();
3057 }
3058 
3059 static void
3060 bdev_nvme_reset_device_stat(void *ctx)
3061 {
3062 	struct nvme_bdev *nbdev = ctx;
3063 
3064 	if (nbdev->err_stat != NULL) {
3065 		memset(nbdev->err_stat, 0, sizeof(struct nvme_error_stat));
3066 	}
3067 }
3068 
3069 /* JSON string should be lowercases and underscore delimited string. */
3070 static void
3071 bdev_nvme_format_nvme_status(char *dst, const char *src)
3072 {
3073 	char tmp[256];
3074 
3075 	spdk_strcpy_replace(dst, 256, src, " - ", "_");
3076 	spdk_strcpy_replace(tmp, 256, dst, "-", "_");
3077 	spdk_strcpy_replace(dst, 256, tmp, " ", "_");
3078 	spdk_strlwr(dst);
3079 }
3080 
3081 static void
3082 bdev_nvme_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w)
3083 {
3084 	struct nvme_bdev *nbdev = ctx;
3085 	struct spdk_nvme_status status = {};
3086 	uint16_t sct, sc;
3087 	char status_json[256];
3088 	const char *status_str;
3089 
3090 	if (nbdev->err_stat == NULL) {
3091 		return;
3092 	}
3093 
3094 	spdk_json_write_named_object_begin(w, "nvme_error");
3095 
3096 	spdk_json_write_named_object_begin(w, "status_type");
3097 	for (sct = 0; sct < 8; sct++) {
3098 		if (nbdev->err_stat->status_type[sct] == 0) {
3099 			continue;
3100 		}
3101 		status.sct = sct;
3102 
3103 		status_str = spdk_nvme_cpl_get_status_type_string(&status);
3104 		assert(status_str != NULL);
3105 		bdev_nvme_format_nvme_status(status_json, status_str);
3106 
3107 		spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status_type[sct]);
3108 	}
3109 	spdk_json_write_object_end(w);
3110 
3111 	spdk_json_write_named_object_begin(w, "status_code");
3112 	for (sct = 0; sct < 4; sct++) {
3113 		status.sct = sct;
3114 		for (sc = 0; sc < 256; sc++) {
3115 			if (nbdev->err_stat->status[sct][sc] == 0) {
3116 				continue;
3117 			}
3118 			status.sc = sc;
3119 
3120 			status_str = spdk_nvme_cpl_get_status_string(&status);
3121 			assert(status_str != NULL);
3122 			bdev_nvme_format_nvme_status(status_json, status_str);
3123 
3124 			spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status[sct][sc]);
3125 		}
3126 	}
3127 	spdk_json_write_object_end(w);
3128 
3129 	spdk_json_write_object_end(w);
3130 }
3131 
3132 static const struct spdk_bdev_fn_table nvmelib_fn_table = {
3133 	.destruct		= bdev_nvme_destruct,
3134 	.submit_request		= bdev_nvme_submit_request,
3135 	.io_type_supported	= bdev_nvme_io_type_supported,
3136 	.get_io_channel		= bdev_nvme_get_io_channel,
3137 	.dump_info_json		= bdev_nvme_dump_info_json,
3138 	.write_config_json	= bdev_nvme_write_config_json,
3139 	.get_spin_time		= bdev_nvme_get_spin_time,
3140 	.get_module_ctx		= bdev_nvme_get_module_ctx,
3141 	.get_memory_domains	= bdev_nvme_get_memory_domains,
3142 	.reset_device_stat	= bdev_nvme_reset_device_stat,
3143 	.dump_device_stat_json	= bdev_nvme_dump_device_stat_json,
3144 };
3145 
3146 typedef int (*bdev_nvme_parse_ana_log_page_cb)(
3147 	const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg);
3148 
3149 static int
3150 bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr,
3151 			     bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg)
3152 {
3153 	struct spdk_nvme_ana_group_descriptor *copied_desc;
3154 	uint8_t *orig_desc;
3155 	uint32_t i, desc_size, copy_len;
3156 	int rc = 0;
3157 
3158 	if (nvme_ctrlr->ana_log_page == NULL) {
3159 		return -EINVAL;
3160 	}
3161 
3162 	copied_desc = nvme_ctrlr->copied_ana_desc;
3163 
3164 	orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page);
3165 	copy_len = nvme_ctrlr->max_ana_log_page_size - sizeof(struct spdk_nvme_ana_page);
3166 
3167 	for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) {
3168 		memcpy(copied_desc, orig_desc, copy_len);
3169 
3170 		rc = cb_fn(copied_desc, cb_arg);
3171 		if (rc != 0) {
3172 			break;
3173 		}
3174 
3175 		desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) +
3176 			    copied_desc->num_of_nsid * sizeof(uint32_t);
3177 		orig_desc += desc_size;
3178 		copy_len -= desc_size;
3179 	}
3180 
3181 	return rc;
3182 }
3183 
3184 static int
3185 nvme_ns_ana_transition_timedout(void *ctx)
3186 {
3187 	struct nvme_ns *nvme_ns = ctx;
3188 
3189 	spdk_poller_unregister(&nvme_ns->anatt_timer);
3190 	nvme_ns->ana_transition_timedout = true;
3191 
3192 	return SPDK_POLLER_BUSY;
3193 }
3194 
3195 static void
3196 _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns,
3197 		       const struct spdk_nvme_ana_group_descriptor *desc)
3198 {
3199 	const struct spdk_nvme_ctrlr_data *cdata;
3200 
3201 	nvme_ns->ana_group_id = desc->ana_group_id;
3202 	nvme_ns->ana_state = desc->ana_state;
3203 	nvme_ns->ana_state_updating = false;
3204 
3205 	switch (nvme_ns->ana_state) {
3206 	case SPDK_NVME_ANA_OPTIMIZED_STATE:
3207 	case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
3208 		nvme_ns->ana_transition_timedout = false;
3209 		spdk_poller_unregister(&nvme_ns->anatt_timer);
3210 		break;
3211 
3212 	case SPDK_NVME_ANA_INACCESSIBLE_STATE:
3213 	case SPDK_NVME_ANA_CHANGE_STATE:
3214 		if (nvme_ns->anatt_timer != NULL) {
3215 			break;
3216 		}
3217 
3218 		cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr);
3219 		nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout,
3220 				       nvme_ns,
3221 				       cdata->anatt * SPDK_SEC_TO_USEC);
3222 		break;
3223 	default:
3224 		break;
3225 	}
3226 }
3227 
3228 static int
3229 nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg)
3230 {
3231 	struct nvme_ns *nvme_ns = cb_arg;
3232 	uint32_t i;
3233 
3234 	for (i = 0; i < desc->num_of_nsid; i++) {
3235 		if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) {
3236 			continue;
3237 		}
3238 
3239 		_nvme_ns_set_ana_state(nvme_ns, desc);
3240 		return 1;
3241 	}
3242 
3243 	return 0;
3244 }
3245 
3246 static struct spdk_uuid
3247 nvme_generate_uuid(const char *sn, uint32_t nsid)
3248 {
3249 	struct spdk_uuid new_uuid, namespace_uuid;
3250 	char merged_str[SPDK_NVME_CTRLR_SN_LEN + NSID_STR_LEN + 1] = {'\0'};
3251 	/* This namespace UUID was generated using uuid_generate() method. */
3252 	const char *namespace_str = {"edaed2de-24bc-4b07-b559-f47ecbe730fd"};
3253 	int size;
3254 
3255 	assert(strlen(sn) <= SPDK_NVME_CTRLR_SN_LEN);
3256 
3257 	memset(&new_uuid, 0, sizeof(new_uuid));
3258 	memset(&namespace_uuid, 0, sizeof(namespace_uuid));
3259 
3260 	size = snprintf(merged_str, sizeof(merged_str), "%s%"PRIu32, sn, nsid);
3261 	assert(size > 0 && (unsigned long)size < sizeof(merged_str));
3262 
3263 	spdk_uuid_parse(&namespace_uuid, namespace_str);
3264 
3265 	spdk_uuid_generate_sha1(&new_uuid, &namespace_uuid, merged_str, size);
3266 
3267 	return new_uuid;
3268 }
3269 
3270 static int
3271 nvme_disk_create(struct spdk_bdev *disk, const char *base_name,
3272 		 struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns,
3273 		 uint32_t prchk_flags, void *ctx)
3274 {
3275 	const struct spdk_uuid		*uuid;
3276 	const uint8_t *nguid;
3277 	const struct spdk_nvme_ctrlr_data *cdata;
3278 	const struct spdk_nvme_ns_data	*nsdata;
3279 	const struct spdk_nvme_ctrlr_opts *opts;
3280 	enum spdk_nvme_csi		csi;
3281 	uint32_t atomic_bs, phys_bs, bs;
3282 	char sn_tmp[SPDK_NVME_CTRLR_SN_LEN + 1] = {'\0'};
3283 
3284 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3285 	csi = spdk_nvme_ns_get_csi(ns);
3286 	opts = spdk_nvme_ctrlr_get_opts(ctrlr);
3287 
3288 	switch (csi) {
3289 	case SPDK_NVME_CSI_NVM:
3290 		disk->product_name = "NVMe disk";
3291 		break;
3292 	case SPDK_NVME_CSI_ZNS:
3293 		disk->product_name = "NVMe ZNS disk";
3294 		disk->zoned = true;
3295 		disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
3296 		disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) /
3297 					     spdk_nvme_ns_get_extended_sector_size(ns);
3298 		disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns);
3299 		disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns);
3300 		break;
3301 	default:
3302 		SPDK_ERRLOG("unsupported CSI: %u\n", csi);
3303 		return -ENOTSUP;
3304 	}
3305 
3306 	disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns));
3307 	if (!disk->name) {
3308 		return -ENOMEM;
3309 	}
3310 
3311 	disk->write_cache = 0;
3312 	if (cdata->vwc.present) {
3313 		/* Enable if the Volatile Write Cache exists */
3314 		disk->write_cache = 1;
3315 	}
3316 	if (cdata->oncs.write_zeroes) {
3317 		disk->max_write_zeroes = UINT16_MAX + 1;
3318 	}
3319 	disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns);
3320 	disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns);
3321 	disk->max_segment_size = spdk_nvme_ctrlr_get_max_xfer_size(ctrlr);
3322 	/* NVMe driver will split one request into multiple requests
3323 	 * based on MDTS and stripe boundary, the bdev layer will use
3324 	 * max_segment_size and max_num_segments to split one big IO
3325 	 * into multiple requests, then small request can't run out
3326 	 * of NVMe internal requests data structure.
3327 	 */
3328 	if (opts && opts->io_queue_requests) {
3329 		disk->max_num_segments = opts->io_queue_requests / 2;
3330 	}
3331 	disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns);
3332 
3333 	nguid = spdk_nvme_ns_get_nguid(ns);
3334 	if (!nguid) {
3335 		uuid = spdk_nvme_ns_get_uuid(ns);
3336 		if (uuid) {
3337 			disk->uuid = *uuid;
3338 		} else if (g_opts.generate_uuids) {
3339 			spdk_strcpy_pad(sn_tmp, cdata->sn, SPDK_NVME_CTRLR_SN_LEN, '\0');
3340 			disk->uuid = nvme_generate_uuid(sn_tmp, spdk_nvme_ns_get_id(ns));
3341 		}
3342 	} else {
3343 		memcpy(&disk->uuid, nguid, sizeof(disk->uuid));
3344 	}
3345 
3346 	nsdata = spdk_nvme_ns_get_data(ns);
3347 	bs = spdk_nvme_ns_get_sector_size(ns);
3348 	atomic_bs = bs;
3349 	phys_bs = bs;
3350 	if (nsdata->nabo == 0) {
3351 		if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) {
3352 			atomic_bs = bs * (1 + nsdata->nawupf);
3353 		} else {
3354 			atomic_bs = bs * (1 + cdata->awupf);
3355 		}
3356 	}
3357 	if (nsdata->nsfeat.optperf) {
3358 		phys_bs = bs * (1 + nsdata->npwg);
3359 	}
3360 	disk->phys_blocklen = spdk_min(phys_bs, atomic_bs);
3361 
3362 	disk->md_len = spdk_nvme_ns_get_md_size(ns);
3363 	if (disk->md_len != 0) {
3364 		disk->md_interleave = nsdata->flbas.extended;
3365 		disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns);
3366 		if (disk->dif_type != SPDK_DIF_DISABLE) {
3367 			disk->dif_is_head_of_md = nsdata->dps.md_start;
3368 			disk->dif_check_flags = prchk_flags;
3369 		}
3370 	}
3371 
3372 	if (!(spdk_nvme_ctrlr_get_flags(ctrlr) &
3373 	      SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) {
3374 		disk->acwu = 0;
3375 	} else if (nsdata->nsfeat.ns_atomic_write_unit) {
3376 		disk->acwu = nsdata->nacwu + 1; /* 0-based */
3377 	} else {
3378 		disk->acwu = cdata->acwu + 1; /* 0-based */
3379 	}
3380 
3381 	if (cdata->oncs.copy) {
3382 		/* For now bdev interface allows only single segment copy */
3383 		disk->max_copy = nsdata->mssrl;
3384 	}
3385 
3386 	disk->ctxt = ctx;
3387 	disk->fn_table = &nvmelib_fn_table;
3388 	disk->module = &nvme_if;
3389 
3390 	return 0;
3391 }
3392 
3393 static int
3394 nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
3395 {
3396 	struct nvme_bdev *bdev;
3397 	int rc;
3398 
3399 	bdev = calloc(1, sizeof(*bdev));
3400 	if (!bdev) {
3401 		SPDK_ERRLOG("bdev calloc() failed\n");
3402 		return -ENOMEM;
3403 	}
3404 
3405 	if (g_opts.nvme_error_stat) {
3406 		bdev->err_stat = calloc(1, sizeof(struct nvme_error_stat));
3407 		if (!bdev->err_stat) {
3408 			SPDK_ERRLOG("err_stat calloc() failed\n");
3409 			free(bdev);
3410 			return -ENOMEM;
3411 		}
3412 	}
3413 
3414 	rc = pthread_mutex_init(&bdev->mutex, NULL);
3415 	if (rc != 0) {
3416 		free(bdev->err_stat);
3417 		free(bdev);
3418 		return rc;
3419 	}
3420 
3421 	bdev->ref = 1;
3422 	bdev->mp_policy = BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE;
3423 	bdev->mp_selector = BDEV_NVME_MP_SELECTOR_ROUND_ROBIN;
3424 	bdev->rr_min_io = UINT32_MAX;
3425 	TAILQ_INIT(&bdev->nvme_ns_list);
3426 	TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq);
3427 	bdev->opal = nvme_ctrlr->opal_dev != NULL;
3428 
3429 	rc = nvme_disk_create(&bdev->disk, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ctrlr,
3430 			      nvme_ns->ns, nvme_ctrlr->opts.prchk_flags, bdev);
3431 	if (rc != 0) {
3432 		SPDK_ERRLOG("Failed to create NVMe disk\n");
3433 		pthread_mutex_destroy(&bdev->mutex);
3434 		free(bdev->err_stat);
3435 		free(bdev);
3436 		return rc;
3437 	}
3438 
3439 	spdk_io_device_register(bdev,
3440 				bdev_nvme_create_bdev_channel_cb,
3441 				bdev_nvme_destroy_bdev_channel_cb,
3442 				sizeof(struct nvme_bdev_channel),
3443 				bdev->disk.name);
3444 
3445 	rc = spdk_bdev_register(&bdev->disk);
3446 	if (rc != 0) {
3447 		SPDK_ERRLOG("spdk_bdev_register() failed\n");
3448 		spdk_io_device_unregister(bdev, NULL);
3449 		pthread_mutex_destroy(&bdev->mutex);
3450 		free(bdev->disk.name);
3451 		free(bdev->err_stat);
3452 		free(bdev);
3453 		return rc;
3454 	}
3455 
3456 	nvme_ns->bdev = bdev;
3457 	bdev->nsid = nvme_ns->id;
3458 
3459 	bdev->nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr;
3460 	TAILQ_INSERT_TAIL(&nvme_ctrlr->nbdev_ctrlr->bdevs, bdev, tailq);
3461 
3462 	return 0;
3463 }
3464 
3465 static bool
3466 bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2)
3467 {
3468 	const struct spdk_nvme_ns_data *nsdata1, *nsdata2;
3469 	const struct spdk_uuid *uuid1, *uuid2;
3470 
3471 	nsdata1 = spdk_nvme_ns_get_data(ns1);
3472 	nsdata2 = spdk_nvme_ns_get_data(ns2);
3473 	uuid1 = spdk_nvme_ns_get_uuid(ns1);
3474 	uuid2 = spdk_nvme_ns_get_uuid(ns2);
3475 
3476 	return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 &&
3477 	       nsdata1->eui64 == nsdata2->eui64 &&
3478 	       ((uuid1 == NULL && uuid2 == NULL) ||
3479 		(uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) &&
3480 	       spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2);
3481 }
3482 
3483 static bool
3484 hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
3485 		 struct spdk_nvme_ctrlr_opts *opts)
3486 {
3487 	struct nvme_probe_skip_entry *entry;
3488 
3489 	TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) {
3490 		if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) {
3491 			return false;
3492 		}
3493 	}
3494 
3495 	opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst;
3496 	opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight;
3497 	opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight;
3498 	opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight;
3499 	opts->disable_read_ana_log_page = true;
3500 
3501 	SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr);
3502 
3503 	return true;
3504 }
3505 
3506 static void
3507 nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl)
3508 {
3509 	struct nvme_ctrlr *nvme_ctrlr = ctx;
3510 
3511 	if (spdk_nvme_cpl_is_error(cpl)) {
3512 		SPDK_WARNLOG("Abort failed. Resetting controller. sc is %u, sct is %u.\n", cpl->status.sc,
3513 			     cpl->status.sct);
3514 		bdev_nvme_reset(nvme_ctrlr);
3515 	} else if (cpl->cdw0 & 0x1) {
3516 		SPDK_WARNLOG("Specified command could not be aborted.\n");
3517 		bdev_nvme_reset(nvme_ctrlr);
3518 	}
3519 }
3520 
3521 static void
3522 timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
3523 	   struct spdk_nvme_qpair *qpair, uint16_t cid)
3524 {
3525 	struct nvme_ctrlr *nvme_ctrlr = cb_arg;
3526 	union spdk_nvme_csts_register csts;
3527 	int rc;
3528 
3529 	assert(nvme_ctrlr->ctrlr == ctrlr);
3530 
3531 	SPDK_WARNLOG("Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n", ctrlr, qpair, cid);
3532 
3533 	/* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O
3534 	 * queue.  (Note: qpair == NULL when there's an admin cmd timeout.)  Otherwise we
3535 	 * would submit another fabrics cmd on the admin queue to read CSTS and check for its
3536 	 * completion recursively.
3537 	 */
3538 	if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) {
3539 		csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr);
3540 		if (csts.bits.cfs) {
3541 			SPDK_ERRLOG("Controller Fatal Status, reset required\n");
3542 			bdev_nvme_reset(nvme_ctrlr);
3543 			return;
3544 		}
3545 	}
3546 
3547 	switch (g_opts.action_on_timeout) {
3548 	case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT:
3549 		if (qpair) {
3550 			/* Don't send abort to ctrlr when ctrlr is not available. */
3551 			pthread_mutex_lock(&nvme_ctrlr->mutex);
3552 			if (!nvme_ctrlr_is_available(nvme_ctrlr)) {
3553 				pthread_mutex_unlock(&nvme_ctrlr->mutex);
3554 				SPDK_NOTICELOG("Quit abort. Ctrlr is not available.\n");
3555 				return;
3556 			}
3557 			pthread_mutex_unlock(&nvme_ctrlr->mutex);
3558 
3559 			rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid,
3560 						       nvme_abort_cpl, nvme_ctrlr);
3561 			if (rc == 0) {
3562 				return;
3563 			}
3564 
3565 			SPDK_ERRLOG("Unable to send abort. Resetting, rc is %d.\n", rc);
3566 		}
3567 
3568 	/* FALLTHROUGH */
3569 	case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET:
3570 		bdev_nvme_reset(nvme_ctrlr);
3571 		break;
3572 	case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE:
3573 		SPDK_DEBUGLOG(bdev_nvme, "No action for nvme controller timeout.\n");
3574 		break;
3575 	default:
3576 		SPDK_ERRLOG("An invalid timeout action value is found.\n");
3577 		break;
3578 	}
3579 }
3580 
3581 static struct nvme_ns *
3582 nvme_ns_alloc(void)
3583 {
3584 	struct nvme_ns *nvme_ns;
3585 
3586 	nvme_ns = calloc(1, sizeof(struct nvme_ns));
3587 	if (nvme_ns == NULL) {
3588 		return NULL;
3589 	}
3590 
3591 	if (g_opts.io_path_stat) {
3592 		nvme_ns->stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
3593 		if (nvme_ns->stat == NULL) {
3594 			free(nvme_ns);
3595 			return NULL;
3596 		}
3597 		spdk_bdev_reset_io_stat(nvme_ns->stat, SPDK_BDEV_RESET_STAT_MAXMIN);
3598 	}
3599 
3600 	return nvme_ns;
3601 }
3602 
3603 static void
3604 nvme_ns_free(struct nvme_ns *nvme_ns)
3605 {
3606 	free(nvme_ns->stat);
3607 	free(nvme_ns);
3608 }
3609 
3610 static void
3611 nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc)
3612 {
3613 	struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr;
3614 	struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx;
3615 
3616 	if (rc == 0) {
3617 		nvme_ns->probe_ctx = NULL;
3618 		pthread_mutex_lock(&nvme_ctrlr->mutex);
3619 		nvme_ctrlr->ref++;
3620 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
3621 	} else {
3622 		RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
3623 		nvme_ns_free(nvme_ns);
3624 	}
3625 
3626 	if (ctx) {
3627 		ctx->populates_in_progress--;
3628 		if (ctx->populates_in_progress == 0) {
3629 			nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx);
3630 		}
3631 	}
3632 }
3633 
3634 static void
3635 bdev_nvme_add_io_path(struct spdk_io_channel_iter *i)
3636 {
3637 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
3638 	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch);
3639 	struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i);
3640 	int rc;
3641 
3642 	rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns);
3643 	if (rc != 0) {
3644 		SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n");
3645 	}
3646 
3647 	spdk_for_each_channel_continue(i, rc);
3648 }
3649 
3650 static void
3651 bdev_nvme_delete_io_path(struct spdk_io_channel_iter *i)
3652 {
3653 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
3654 	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch);
3655 	struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i);
3656 	struct nvme_io_path *io_path;
3657 
3658 	io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns);
3659 	if (io_path != NULL) {
3660 		_bdev_nvme_delete_io_path(nbdev_ch, io_path);
3661 	}
3662 
3663 	spdk_for_each_channel_continue(i, 0);
3664 }
3665 
3666 static void
3667 bdev_nvme_add_io_path_failed(struct spdk_io_channel_iter *i, int status)
3668 {
3669 	struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i);
3670 
3671 	nvme_ctrlr_populate_namespace_done(nvme_ns, -1);
3672 }
3673 
3674 static void
3675 bdev_nvme_add_io_path_done(struct spdk_io_channel_iter *i, int status)
3676 {
3677 	struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i);
3678 	struct nvme_bdev *bdev = spdk_io_channel_iter_get_io_device(i);
3679 
3680 	if (status == 0) {
3681 		nvme_ctrlr_populate_namespace_done(nvme_ns, 0);
3682 	} else {
3683 		/* Delete the added io_paths and fail populating the namespace. */
3684 		spdk_for_each_channel(bdev,
3685 				      bdev_nvme_delete_io_path,
3686 				      nvme_ns,
3687 				      bdev_nvme_add_io_path_failed);
3688 	}
3689 }
3690 
3691 static int
3692 nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns)
3693 {
3694 	struct nvme_ns *tmp_ns;
3695 	const struct spdk_nvme_ns_data *nsdata;
3696 
3697 	nsdata = spdk_nvme_ns_get_data(nvme_ns->ns);
3698 	if (!nsdata->nmic.can_share) {
3699 		SPDK_ERRLOG("Namespace cannot be shared.\n");
3700 		return -EINVAL;
3701 	}
3702 
3703 	pthread_mutex_lock(&bdev->mutex);
3704 
3705 	tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list);
3706 	assert(tmp_ns != NULL);
3707 
3708 	if (!bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) {
3709 		pthread_mutex_unlock(&bdev->mutex);
3710 		SPDK_ERRLOG("Namespaces are not identical.\n");
3711 		return -EINVAL;
3712 	}
3713 
3714 	bdev->ref++;
3715 	TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq);
3716 	nvme_ns->bdev = bdev;
3717 
3718 	pthread_mutex_unlock(&bdev->mutex);
3719 
3720 	/* Add nvme_io_path to nvme_bdev_channels dynamically. */
3721 	spdk_for_each_channel(bdev,
3722 			      bdev_nvme_add_io_path,
3723 			      nvme_ns,
3724 			      bdev_nvme_add_io_path_done);
3725 
3726 	return 0;
3727 }
3728 
3729 static void
3730 nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
3731 {
3732 	struct spdk_nvme_ns	*ns;
3733 	struct nvme_bdev	*bdev;
3734 	int			rc = 0;
3735 
3736 	ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id);
3737 	if (!ns) {
3738 		SPDK_DEBUGLOG(bdev_nvme, "Invalid NS %d\n", nvme_ns->id);
3739 		rc = -EINVAL;
3740 		goto done;
3741 	}
3742 
3743 	nvme_ns->ns = ns;
3744 	nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
3745 
3746 	if (nvme_ctrlr->ana_log_page != NULL) {
3747 		bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns);
3748 	}
3749 
3750 	bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id);
3751 	if (bdev == NULL) {
3752 		rc = nvme_bdev_create(nvme_ctrlr, nvme_ns);
3753 	} else {
3754 		rc = nvme_bdev_add_ns(bdev, nvme_ns);
3755 		if (rc == 0) {
3756 			return;
3757 		}
3758 	}
3759 done:
3760 	nvme_ctrlr_populate_namespace_done(nvme_ns, rc);
3761 }
3762 
3763 static void
3764 nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns)
3765 {
3766 	struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr;
3767 
3768 	assert(nvme_ctrlr != NULL);
3769 
3770 	pthread_mutex_lock(&nvme_ctrlr->mutex);
3771 
3772 	RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
3773 
3774 	if (nvme_ns->bdev != NULL) {
3775 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
3776 		return;
3777 	}
3778 
3779 	nvme_ns_free(nvme_ns);
3780 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
3781 
3782 	nvme_ctrlr_release(nvme_ctrlr);
3783 }
3784 
3785 static void
3786 bdev_nvme_delete_io_path_done(struct spdk_io_channel_iter *i, int status)
3787 {
3788 	struct nvme_ns *nvme_ns = spdk_io_channel_iter_get_ctx(i);
3789 
3790 	nvme_ctrlr_depopulate_namespace_done(nvme_ns);
3791 }
3792 
3793 static void
3794 nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
3795 {
3796 	struct nvme_bdev *bdev;
3797 
3798 	spdk_poller_unregister(&nvme_ns->anatt_timer);
3799 
3800 	bdev = nvme_ns->bdev;
3801 	if (bdev != NULL) {
3802 		pthread_mutex_lock(&bdev->mutex);
3803 
3804 		assert(bdev->ref > 0);
3805 		bdev->ref--;
3806 		if (bdev->ref == 0) {
3807 			pthread_mutex_unlock(&bdev->mutex);
3808 
3809 			spdk_bdev_unregister(&bdev->disk, NULL, NULL);
3810 		} else {
3811 			/* spdk_bdev_unregister() is not called until the last nvme_ns is
3812 			 * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list
3813 			 * and clear nvme_ns->bdev here.
3814 			 */
3815 			TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq);
3816 			nvme_ns->bdev = NULL;
3817 
3818 			pthread_mutex_unlock(&bdev->mutex);
3819 
3820 			/* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that,
3821 			 * we call depopulate_namespace_done() to avoid use-after-free.
3822 			 */
3823 			spdk_for_each_channel(bdev,
3824 					      bdev_nvme_delete_io_path,
3825 					      nvme_ns,
3826 					      bdev_nvme_delete_io_path_done);
3827 			return;
3828 		}
3829 	}
3830 
3831 	nvme_ctrlr_depopulate_namespace_done(nvme_ns);
3832 }
3833 
3834 static void
3835 nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
3836 			       struct nvme_async_probe_ctx *ctx)
3837 {
3838 	struct spdk_nvme_ctrlr	*ctrlr = nvme_ctrlr->ctrlr;
3839 	struct nvme_ns	*nvme_ns, *next;
3840 	struct spdk_nvme_ns	*ns;
3841 	struct nvme_bdev	*bdev;
3842 	uint32_t		nsid;
3843 	int			rc;
3844 	uint64_t		num_sectors;
3845 
3846 	if (ctx) {
3847 		/* Initialize this count to 1 to handle the populate functions
3848 		 * calling nvme_ctrlr_populate_namespace_done() immediately.
3849 		 */
3850 		ctx->populates_in_progress = 1;
3851 	}
3852 
3853 	/* First loop over our existing namespaces and see if they have been
3854 	 * removed. */
3855 	nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
3856 	while (nvme_ns != NULL) {
3857 		next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
3858 
3859 		if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) {
3860 			/* NS is still there but attributes may have changed */
3861 			ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id);
3862 			num_sectors = spdk_nvme_ns_get_num_sectors(ns);
3863 			bdev = nvme_ns->bdev;
3864 			assert(bdev != NULL);
3865 			if (bdev->disk.blockcnt != num_sectors) {
3866 				SPDK_NOTICELOG("NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n",
3867 					       nvme_ns->id,
3868 					       bdev->disk.name,
3869 					       bdev->disk.blockcnt,
3870 					       num_sectors);
3871 				rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors);
3872 				if (rc != 0) {
3873 					SPDK_ERRLOG("Could not change num blocks for nvme bdev: name %s, errno: %d.\n",
3874 						    bdev->disk.name, rc);
3875 				}
3876 			}
3877 		} else {
3878 			/* Namespace was removed */
3879 			nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns);
3880 		}
3881 
3882 		nvme_ns = next;
3883 	}
3884 
3885 	/* Loop through all of the namespaces at the nvme level and see if any of them are new */
3886 	nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
3887 	while (nsid != 0) {
3888 		nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
3889 
3890 		if (nvme_ns == NULL) {
3891 			/* Found a new one */
3892 			nvme_ns = nvme_ns_alloc();
3893 			if (nvme_ns == NULL) {
3894 				SPDK_ERRLOG("Failed to allocate namespace\n");
3895 				/* This just fails to attach the namespace. It may work on a future attempt. */
3896 				continue;
3897 			}
3898 
3899 			nvme_ns->id = nsid;
3900 			nvme_ns->ctrlr = nvme_ctrlr;
3901 
3902 			nvme_ns->bdev = NULL;
3903 
3904 			if (ctx) {
3905 				ctx->populates_in_progress++;
3906 			}
3907 			nvme_ns->probe_ctx = ctx;
3908 
3909 			RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
3910 
3911 			nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns);
3912 		}
3913 
3914 		nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid);
3915 	}
3916 
3917 	if (ctx) {
3918 		/* Decrement this count now that the loop is over to account
3919 		 * for the one we started with.  If the count is then 0, we
3920 		 * know any populate_namespace functions completed immediately,
3921 		 * so we'll kick the callback here.
3922 		 */
3923 		ctx->populates_in_progress--;
3924 		if (ctx->populates_in_progress == 0) {
3925 			nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx);
3926 		}
3927 	}
3928 
3929 }
3930 
3931 static void
3932 nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr)
3933 {
3934 	struct nvme_ns *nvme_ns, *tmp;
3935 
3936 	RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) {
3937 		nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns);
3938 	}
3939 }
3940 
3941 static uint32_t
3942 nvme_ctrlr_get_ana_log_page_size(struct nvme_ctrlr *nvme_ctrlr)
3943 {
3944 	struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
3945 	const struct spdk_nvme_ctrlr_data *cdata;
3946 	uint32_t nsid, ns_count = 0;
3947 
3948 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3949 
3950 	for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
3951 	     nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
3952 		ns_count++;
3953 	}
3954 
3955 	return sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid *
3956 	       sizeof(struct spdk_nvme_ana_group_descriptor) + ns_count *
3957 	       sizeof(uint32_t);
3958 }
3959 
3960 static int
3961 nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc,
3962 			  void *cb_arg)
3963 {
3964 	struct nvme_ctrlr *nvme_ctrlr = cb_arg;
3965 	struct nvme_ns *nvme_ns;
3966 	uint32_t i, nsid;
3967 
3968 	for (i = 0; i < desc->num_of_nsid; i++) {
3969 		nsid = desc->nsid[i];
3970 		if (nsid == 0) {
3971 			continue;
3972 		}
3973 
3974 		nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
3975 
3976 		assert(nvme_ns != NULL);
3977 		if (nvme_ns == NULL) {
3978 			/* Target told us that an inactive namespace had an ANA change */
3979 			continue;
3980 		}
3981 
3982 		_nvme_ns_set_ana_state(nvme_ns, desc);
3983 	}
3984 
3985 	return 0;
3986 }
3987 
3988 static void
3989 bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr)
3990 {
3991 	struct nvme_ns *nvme_ns;
3992 
3993 	spdk_free(nvme_ctrlr->ana_log_page);
3994 	nvme_ctrlr->ana_log_page = NULL;
3995 
3996 	for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
3997 	     nvme_ns != NULL;
3998 	     nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) {
3999 		nvme_ns->ana_state_updating = false;
4000 		nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
4001 	}
4002 }
4003 
4004 static void
4005 nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl)
4006 {
4007 	struct nvme_ctrlr *nvme_ctrlr = ctx;
4008 
4009 	if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) {
4010 		bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states,
4011 					     nvme_ctrlr);
4012 	} else {
4013 		bdev_nvme_disable_read_ana_log_page(nvme_ctrlr);
4014 	}
4015 
4016 	pthread_mutex_lock(&nvme_ctrlr->mutex);
4017 
4018 	assert(nvme_ctrlr->ana_log_page_updating == true);
4019 	nvme_ctrlr->ana_log_page_updating = false;
4020 
4021 	if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
4022 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
4023 
4024 		nvme_ctrlr_unregister(nvme_ctrlr);
4025 	} else {
4026 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
4027 
4028 		bdev_nvme_clear_io_path_caches(nvme_ctrlr);
4029 	}
4030 }
4031 
4032 static int
4033 nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr)
4034 {
4035 	uint32_t ana_log_page_size;
4036 	int rc;
4037 
4038 	if (nvme_ctrlr->ana_log_page == NULL) {
4039 		return -EINVAL;
4040 	}
4041 
4042 	ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr);
4043 
4044 	if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) {
4045 		SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n",
4046 			    ana_log_page_size, nvme_ctrlr->max_ana_log_page_size);
4047 		return -EINVAL;
4048 	}
4049 
4050 	pthread_mutex_lock(&nvme_ctrlr->mutex);
4051 	if (!nvme_ctrlr_is_available(nvme_ctrlr) ||
4052 	    nvme_ctrlr->ana_log_page_updating) {
4053 		pthread_mutex_unlock(&nvme_ctrlr->mutex);
4054 		return -EBUSY;
4055 	}
4056 
4057 	nvme_ctrlr->ana_log_page_updating = true;
4058 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
4059 
4060 	rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr,
4061 					      SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
4062 					      SPDK_NVME_GLOBAL_NS_TAG,
4063 					      nvme_ctrlr->ana_log_page,
4064 					      ana_log_page_size, 0,
4065 					      nvme_ctrlr_read_ana_log_page_done,
4066 					      nvme_ctrlr);
4067 	if (rc != 0) {
4068 		nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL);
4069 	}
4070 
4071 	return rc;
4072 }
4073 
4074 static void
4075 dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx)
4076 {
4077 }
4078 
4079 struct bdev_nvme_set_preferred_path_ctx {
4080 	struct spdk_bdev_desc *desc;
4081 	struct nvme_ns *nvme_ns;
4082 	bdev_nvme_set_preferred_path_cb cb_fn;
4083 	void *cb_arg;
4084 };
4085 
4086 static void
4087 bdev_nvme_set_preferred_path_done(struct spdk_io_channel_iter *i, int status)
4088 {
4089 	struct bdev_nvme_set_preferred_path_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
4090 
4091 	assert(ctx != NULL);
4092 	assert(ctx->desc != NULL);
4093 	assert(ctx->cb_fn != NULL);
4094 
4095 	spdk_bdev_close(ctx->desc);
4096 
4097 	ctx->cb_fn(ctx->cb_arg, status);
4098 
4099 	free(ctx);
4100 }
4101 
4102 static void
4103 _bdev_nvme_set_preferred_path(struct spdk_io_channel_iter *i)
4104 {
4105 	struct bdev_nvme_set_preferred_path_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
4106 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
4107 	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch);
4108 	struct nvme_io_path *io_path, *prev;
4109 
4110 	prev = NULL;
4111 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
4112 		if (io_path->nvme_ns == ctx->nvme_ns) {
4113 			break;
4114 		}
4115 		prev = io_path;
4116 	}
4117 
4118 	if (io_path != NULL) {
4119 		if (prev != NULL) {
4120 			STAILQ_REMOVE_AFTER(&nbdev_ch->io_path_list, prev, stailq);
4121 			STAILQ_INSERT_HEAD(&nbdev_ch->io_path_list, io_path, stailq);
4122 		}
4123 
4124 		/* We can set io_path to nbdev_ch->current_io_path directly here.
4125 		 * However, it needs to be conditional. To simplify the code,
4126 		 * just clear nbdev_ch->current_io_path and let find_io_path()
4127 		 * fill it.
4128 		 *
4129 		 * Automatic failback may be disabled. Hence even if the io_path is
4130 		 * already at the head, clear nbdev_ch->current_io_path.
4131 		 */
4132 		bdev_nvme_clear_current_io_path(nbdev_ch);
4133 	}
4134 
4135 	spdk_for_each_channel_continue(i, 0);
4136 }
4137 
4138 static struct nvme_ns *
4139 bdev_nvme_set_preferred_ns(struct nvme_bdev *nbdev, uint16_t cntlid)
4140 {
4141 	struct nvme_ns *nvme_ns, *prev;
4142 	const struct spdk_nvme_ctrlr_data *cdata;
4143 
4144 	prev = NULL;
4145 	TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
4146 		cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr);
4147 
4148 		if (cdata->cntlid == cntlid) {
4149 			break;
4150 		}
4151 		prev = nvme_ns;
4152 	}
4153 
4154 	if (nvme_ns != NULL && prev != NULL) {
4155 		TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq);
4156 		TAILQ_INSERT_HEAD(&nbdev->nvme_ns_list, nvme_ns, tailq);
4157 	}
4158 
4159 	return nvme_ns;
4160 }
4161 
4162 /* This function supports only multipath mode. There is only a single I/O path
4163  * for each NVMe-oF controller. Hence, just move the matched I/O path to the
4164  * head of the I/O path list for each NVMe bdev channel.
4165  *
4166  * NVMe bdev channel may be acquired after completing this function. move the
4167  * matched namespace to the head of the namespace list for the NVMe bdev too.
4168  */
4169 void
4170 bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid,
4171 			     bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg)
4172 {
4173 	struct bdev_nvme_set_preferred_path_ctx *ctx;
4174 	struct spdk_bdev *bdev;
4175 	struct nvme_bdev *nbdev;
4176 	int rc = 0;
4177 
4178 	assert(cb_fn != NULL);
4179 
4180 	ctx = calloc(1, sizeof(*ctx));
4181 	if (ctx == NULL) {
4182 		SPDK_ERRLOG("Failed to alloc context.\n");
4183 		rc = -ENOMEM;
4184 		goto err_alloc;
4185 	}
4186 
4187 	ctx->cb_fn = cb_fn;
4188 	ctx->cb_arg = cb_arg;
4189 
4190 	rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc);
4191 	if (rc != 0) {
4192 		SPDK_ERRLOG("Failed to open bdev %s.\n", name);
4193 		goto err_open;
4194 	}
4195 
4196 	bdev = spdk_bdev_desc_get_bdev(ctx->desc);
4197 
4198 	if (bdev->module != &nvme_if) {
4199 		SPDK_ERRLOG("bdev %s is not registered in this module.\n", name);
4200 		rc = -ENODEV;
4201 		goto err_bdev;
4202 	}
4203 
4204 	nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
4205 
4206 	pthread_mutex_lock(&nbdev->mutex);
4207 
4208 	ctx->nvme_ns = bdev_nvme_set_preferred_ns(nbdev, cntlid);
4209 	if (ctx->nvme_ns == NULL) {
4210 		pthread_mutex_unlock(&nbdev->mutex);
4211 
4212 		SPDK_ERRLOG("bdev %s does not have namespace to controller %u.\n", name, cntlid);
4213 		rc = -ENODEV;
4214 		goto err_bdev;
4215 	}
4216 
4217 	pthread_mutex_unlock(&nbdev->mutex);
4218 
4219 	spdk_for_each_channel(nbdev,
4220 			      _bdev_nvme_set_preferred_path,
4221 			      ctx,
4222 			      bdev_nvme_set_preferred_path_done);
4223 	return;
4224 
4225 err_bdev:
4226 	spdk_bdev_close(ctx->desc);
4227 err_open:
4228 	free(ctx);
4229 err_alloc:
4230 	cb_fn(cb_arg, rc);
4231 }
4232 
4233 struct bdev_nvme_set_multipath_policy_ctx {
4234 	struct spdk_bdev_desc *desc;
4235 	bdev_nvme_set_multipath_policy_cb cb_fn;
4236 	void *cb_arg;
4237 };
4238 
4239 static void
4240 bdev_nvme_set_multipath_policy_done(struct spdk_io_channel_iter *i, int status)
4241 {
4242 	struct bdev_nvme_set_multipath_policy_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
4243 
4244 	assert(ctx != NULL);
4245 	assert(ctx->desc != NULL);
4246 	assert(ctx->cb_fn != NULL);
4247 
4248 	spdk_bdev_close(ctx->desc);
4249 
4250 	ctx->cb_fn(ctx->cb_arg, status);
4251 
4252 	free(ctx);
4253 }
4254 
4255 static void
4256 _bdev_nvme_set_multipath_policy(struct spdk_io_channel_iter *i)
4257 {
4258 	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
4259 	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(_ch);
4260 	struct nvme_bdev *nbdev = spdk_io_channel_get_io_device(_ch);
4261 
4262 	nbdev_ch->mp_policy = nbdev->mp_policy;
4263 	nbdev_ch->mp_selector = nbdev->mp_selector;
4264 	nbdev_ch->rr_min_io = nbdev->rr_min_io;
4265 	bdev_nvme_clear_current_io_path(nbdev_ch);
4266 
4267 	spdk_for_each_channel_continue(i, 0);
4268 }
4269 
4270 void
4271 bdev_nvme_set_multipath_policy(const char *name, enum bdev_nvme_multipath_policy policy,
4272 			       enum bdev_nvme_multipath_selector selector, uint32_t rr_min_io,
4273 			       bdev_nvme_set_multipath_policy_cb cb_fn, void *cb_arg)
4274 {
4275 	struct bdev_nvme_set_multipath_policy_ctx *ctx;
4276 	struct spdk_bdev *bdev;
4277 	struct nvme_bdev *nbdev;
4278 	int rc;
4279 
4280 	assert(cb_fn != NULL);
4281 
4282 	if (policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE && selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
4283 		if (rr_min_io == UINT32_MAX) {
4284 			rr_min_io = 1;
4285 		} else if (rr_min_io == 0) {
4286 			rc = -EINVAL;
4287 			goto exit;
4288 		}
4289 	} else if (rr_min_io != UINT32_MAX) {
4290 		rc = -EINVAL;
4291 		goto exit;
4292 	}
4293 
4294 	ctx = calloc(1, sizeof(*ctx));
4295 	if (ctx == NULL) {
4296 		SPDK_ERRLOG("Failed to alloc context.\n");
4297 		rc = -ENOMEM;
4298 		goto exit;
4299 	}
4300 
4301 	ctx->cb_fn = cb_fn;
4302 	ctx->cb_arg = cb_arg;
4303 
4304 	rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc);
4305 	if (rc != 0) {
4306 		SPDK_ERRLOG("Failed to open bdev %s.\n", name);
4307 		rc = -ENODEV;
4308 		goto err_open;
4309 	}
4310 
4311 	bdev = spdk_bdev_desc_get_bdev(ctx->desc);
4312 	if (bdev->module != &nvme_if) {
4313 		SPDK_ERRLOG("bdev %s is not registered in this module.\n", name);
4314 		rc = -ENODEV;
4315 		goto err_module;
4316 	}
4317 	nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
4318 
4319 	pthread_mutex_lock(&nbdev->mutex);
4320 	nbdev->mp_policy = policy;
4321 	nbdev->mp_selector = selector;
4322 	nbdev->rr_min_io = rr_min_io;
4323 	pthread_mutex_unlock(&nbdev->mutex);
4324 
4325 	spdk_for_each_channel(nbdev,
4326 			      _bdev_nvme_set_multipath_policy,
4327 			      ctx,
4328 			      bdev_nvme_set_multipath_policy_done);
4329 	return;
4330 
4331 err_module:
4332 	spdk_bdev_close(ctx->desc);
4333 err_open:
4334 	free(ctx);
4335 exit:
4336 	cb_fn(cb_arg, rc);
4337 }
4338 
4339 static void
4340 aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
4341 {
4342 	struct nvme_ctrlr *nvme_ctrlr		= arg;
4343 	union spdk_nvme_async_event_completion	event;
4344 
4345 	if (spdk_nvme_cpl_is_error(cpl)) {
4346 		SPDK_WARNLOG("AER request execute failed\n");
4347 		return;
4348 	}
4349 
4350 	event.raw = cpl->cdw0;
4351 	if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
4352 	    (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) {
4353 		nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL);
4354 	} else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
4355 		   (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) {
4356 		nvme_ctrlr_read_ana_log_page(nvme_ctrlr);
4357 	}
4358 }
4359 
4360 static void
4361 populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, size_t count, int rc)
4362 {
4363 	if (ctx->cb_fn) {
4364 		ctx->cb_fn(ctx->cb_ctx, count, rc);
4365 	}
4366 
4367 	ctx->namespaces_populated = true;
4368 	if (ctx->probe_done) {
4369 		/* The probe was already completed, so we need to free the context
4370 		 * here.  This can happen for cases like OCSSD, where we need to
4371 		 * send additional commands to the SSD after attach.
4372 		 */
4373 		free(ctx);
4374 	}
4375 }
4376 
4377 static void
4378 nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr,
4379 		       struct nvme_async_probe_ctx *ctx)
4380 {
4381 	spdk_io_device_register(nvme_ctrlr,
4382 				bdev_nvme_create_ctrlr_channel_cb,
4383 				bdev_nvme_destroy_ctrlr_channel_cb,
4384 				sizeof(struct nvme_ctrlr_channel),
4385 				nvme_ctrlr->nbdev_ctrlr->name);
4386 
4387 	nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx);
4388 }
4389 
4390 static void
4391 nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl)
4392 {
4393 	struct nvme_ctrlr *nvme_ctrlr = _ctx;
4394 	struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx;
4395 
4396 	nvme_ctrlr->probe_ctx = NULL;
4397 
4398 	if (spdk_nvme_cpl_is_error(cpl)) {
4399 		nvme_ctrlr_delete(nvme_ctrlr);
4400 
4401 		if (ctx != NULL) {
4402 			populate_namespaces_cb(ctx, 0, -1);
4403 		}
4404 		return;
4405 	}
4406 
4407 	nvme_ctrlr_create_done(nvme_ctrlr, ctx);
4408 }
4409 
4410 static int
4411 nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr,
4412 			     struct nvme_async_probe_ctx *ctx)
4413 {
4414 	struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
4415 	const struct spdk_nvme_ctrlr_data *cdata;
4416 	uint32_t ana_log_page_size;
4417 
4418 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
4419 
4420 	/* Set buffer size enough to include maximum number of allowed namespaces. */
4421 	ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid *
4422 			    sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->mnan *
4423 			    sizeof(uint32_t);
4424 
4425 	nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL,
4426 						SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
4427 	if (nvme_ctrlr->ana_log_page == NULL) {
4428 		SPDK_ERRLOG("could not allocate ANA log page buffer\n");
4429 		return -ENXIO;
4430 	}
4431 
4432 	/* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned.
4433 	 * Hence copy each descriptor to a temporary area when parsing it.
4434 	 *
4435 	 * Allocate a buffer whose size is as large as ANA log page buffer because
4436 	 * we do not know the size of a descriptor until actually reading it.
4437 	 */
4438 	nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size);
4439 	if (nvme_ctrlr->copied_ana_desc == NULL) {
4440 		SPDK_ERRLOG("could not allocate a buffer to parse ANA descriptor\n");
4441 		return -ENOMEM;
4442 	}
4443 
4444 	nvme_ctrlr->max_ana_log_page_size = ana_log_page_size;
4445 
4446 	nvme_ctrlr->probe_ctx = ctx;
4447 
4448 	/* Then, set the read size only to include the current active namespaces. */
4449 	ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr);
4450 
4451 	if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) {
4452 		SPDK_ERRLOG("ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n",
4453 			    ana_log_page_size, nvme_ctrlr->max_ana_log_page_size);
4454 		return -EINVAL;
4455 	}
4456 
4457 	return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr,
4458 						SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
4459 						SPDK_NVME_GLOBAL_NS_TAG,
4460 						nvme_ctrlr->ana_log_page,
4461 						ana_log_page_size, 0,
4462 						nvme_ctrlr_init_ana_log_page_done,
4463 						nvme_ctrlr);
4464 }
4465 
4466 /* hostnqn and subnqn were already verified before attaching a controller.
4467  * Hence check only the multipath capability and cntlid here.
4468  */
4469 static bool
4470 bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr)
4471 {
4472 	struct nvme_ctrlr *tmp;
4473 	const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata;
4474 
4475 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
4476 
4477 	if (!cdata->cmic.multi_ctrlr) {
4478 		SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid);
4479 		return false;
4480 	}
4481 
4482 	TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) {
4483 		tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr);
4484 
4485 		if (!tmp_cdata->cmic.multi_ctrlr) {
4486 			SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid);
4487 			return false;
4488 		}
4489 		if (cdata->cntlid == tmp_cdata->cntlid) {
4490 			SPDK_ERRLOG("cntlid %u are duplicated.\n", tmp_cdata->cntlid);
4491 			return false;
4492 		}
4493 	}
4494 
4495 	return true;
4496 }
4497 
4498 static int
4499 nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr)
4500 {
4501 	struct nvme_bdev_ctrlr *nbdev_ctrlr;
4502 	struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
4503 	int rc = 0;
4504 
4505 	pthread_mutex_lock(&g_bdev_nvme_mutex);
4506 
4507 	nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
4508 	if (nbdev_ctrlr != NULL) {
4509 		if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) {
4510 			rc = -EINVAL;
4511 			goto exit;
4512 		}
4513 	} else {
4514 		nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr));
4515 		if (nbdev_ctrlr == NULL) {
4516 			SPDK_ERRLOG("Failed to allocate nvme_bdev_ctrlr.\n");
4517 			rc = -ENOMEM;
4518 			goto exit;
4519 		}
4520 		nbdev_ctrlr->name = strdup(name);
4521 		if (nbdev_ctrlr->name == NULL) {
4522 			SPDK_ERRLOG("Failed to allocate name of nvme_bdev_ctrlr.\n");
4523 			free(nbdev_ctrlr);
4524 			goto exit;
4525 		}
4526 		TAILQ_INIT(&nbdev_ctrlr->ctrlrs);
4527 		TAILQ_INIT(&nbdev_ctrlr->bdevs);
4528 		TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq);
4529 	}
4530 	nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr;
4531 	TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq);
4532 exit:
4533 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
4534 	return rc;
4535 }
4536 
4537 static int
4538 nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr,
4539 		  const char *name,
4540 		  const struct spdk_nvme_transport_id *trid,
4541 		  struct nvme_async_probe_ctx *ctx)
4542 {
4543 	struct nvme_ctrlr *nvme_ctrlr;
4544 	struct nvme_path_id *path_id;
4545 	const struct spdk_nvme_ctrlr_data *cdata;
4546 	int rc;
4547 
4548 	nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr));
4549 	if (nvme_ctrlr == NULL) {
4550 		SPDK_ERRLOG("Failed to allocate device struct\n");
4551 		return -ENOMEM;
4552 	}
4553 
4554 	rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL);
4555 	if (rc != 0) {
4556 		free(nvme_ctrlr);
4557 		return rc;
4558 	}
4559 
4560 	TAILQ_INIT(&nvme_ctrlr->trids);
4561 
4562 	RB_INIT(&nvme_ctrlr->namespaces);
4563 
4564 	path_id = calloc(1, sizeof(*path_id));
4565 	if (path_id == NULL) {
4566 		SPDK_ERRLOG("Failed to allocate trid entry pointer\n");
4567 		rc = -ENOMEM;
4568 		goto err;
4569 	}
4570 
4571 	path_id->trid = *trid;
4572 	if (ctx != NULL) {
4573 		memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr));
4574 		memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid));
4575 	}
4576 	nvme_ctrlr->active_path_id = path_id;
4577 	TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link);
4578 
4579 	nvme_ctrlr->thread = spdk_get_thread();
4580 	nvme_ctrlr->ctrlr = ctrlr;
4581 	nvme_ctrlr->ref = 1;
4582 
4583 	if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) {
4584 		SPDK_ERRLOG("OCSSDs are not supported");
4585 		rc = -ENOTSUP;
4586 		goto err;
4587 	}
4588 
4589 	if (ctx != NULL) {
4590 		memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts));
4591 	} else {
4592 		bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts);
4593 	}
4594 
4595 	nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr,
4596 					  g_opts.nvme_adminq_poll_period_us);
4597 
4598 	if (g_opts.timeout_us > 0) {
4599 		/* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */
4600 		/* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */
4601 		uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ?
4602 					  g_opts.timeout_us : g_opts.timeout_admin_us;
4603 		spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us,
4604 				adm_timeout_us, timeout_cb, nvme_ctrlr);
4605 	}
4606 
4607 	spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr);
4608 	spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr);
4609 
4610 	if (spdk_nvme_ctrlr_get_flags(ctrlr) &
4611 	    SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
4612 		nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr);
4613 	}
4614 
4615 	rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr);
4616 	if (rc != 0) {
4617 		goto err;
4618 	}
4619 
4620 	cdata = spdk_nvme_ctrlr_get_data(ctrlr);
4621 
4622 	if (cdata->cmic.ana_reporting) {
4623 		rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx);
4624 		if (rc == 0) {
4625 			return 0;
4626 		}
4627 	} else {
4628 		nvme_ctrlr_create_done(nvme_ctrlr, ctx);
4629 		return 0;
4630 	}
4631 
4632 err:
4633 	nvme_ctrlr_delete(nvme_ctrlr);
4634 	return rc;
4635 }
4636 
4637 void
4638 bdev_nvme_get_default_ctrlr_opts(struct nvme_ctrlr_opts *opts)
4639 {
4640 	opts->prchk_flags = 0;
4641 	opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec;
4642 	opts->reconnect_delay_sec = g_opts.reconnect_delay_sec;
4643 	opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec;
4644 }
4645 
4646 static void
4647 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
4648 	  struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts)
4649 {
4650 	char *name;
4651 
4652 	name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++);
4653 	if (!name) {
4654 		SPDK_ERRLOG("Failed to assign name to NVMe device\n");
4655 		return;
4656 	}
4657 
4658 	if (nvme_ctrlr_create(ctrlr, name, trid, NULL) == 0) {
4659 		SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name);
4660 	} else {
4661 		SPDK_ERRLOG("Failed to attach to %s (%s)\n", trid->traddr, name);
4662 	}
4663 
4664 	free(name);
4665 }
4666 
4667 static void
4668 _nvme_ctrlr_destruct(void *ctx)
4669 {
4670 	struct nvme_ctrlr *nvme_ctrlr = ctx;
4671 
4672 	nvme_ctrlr_depopulate_namespaces(nvme_ctrlr);
4673 	nvme_ctrlr_release(nvme_ctrlr);
4674 }
4675 
4676 static int
4677 bdev_nvme_delete_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool hotplug)
4678 {
4679 	struct nvme_probe_skip_entry *entry;
4680 
4681 	/* The controller's destruction was already started */
4682 	if (nvme_ctrlr->destruct) {
4683 		return -EALREADY;
4684 	}
4685 
4686 	if (!hotplug &&
4687 	    nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
4688 		entry = calloc(1, sizeof(*entry));
4689 		if (!entry) {
4690 			return -ENOMEM;
4691 		}
4692 		entry->trid = nvme_ctrlr->active_path_id->trid;
4693 		TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq);
4694 	}
4695 
4696 	nvme_ctrlr->destruct = true;
4697 	return 0;
4698 }
4699 
4700 static int
4701 bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug)
4702 {
4703 	int rc;
4704 
4705 	pthread_mutex_lock(&nvme_ctrlr->mutex);
4706 	rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, hotplug);
4707 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
4708 
4709 	if (rc == 0) {
4710 		_nvme_ctrlr_destruct(nvme_ctrlr);
4711 	} else if (rc == -EALREADY) {
4712 		rc = 0;
4713 	}
4714 
4715 	return rc;
4716 }
4717 
4718 static void
4719 remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr)
4720 {
4721 	struct nvme_ctrlr *nvme_ctrlr = cb_ctx;
4722 
4723 	bdev_nvme_delete_ctrlr(nvme_ctrlr, true);
4724 }
4725 
4726 static int
4727 bdev_nvme_hotplug_probe(void *arg)
4728 {
4729 	if (g_hotplug_probe_ctx == NULL) {
4730 		spdk_poller_unregister(&g_hotplug_probe_poller);
4731 		return SPDK_POLLER_IDLE;
4732 	}
4733 
4734 	if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) {
4735 		g_hotplug_probe_ctx = NULL;
4736 		spdk_poller_unregister(&g_hotplug_probe_poller);
4737 	}
4738 
4739 	return SPDK_POLLER_BUSY;
4740 }
4741 
4742 static int
4743 bdev_nvme_hotplug(void *arg)
4744 {
4745 	struct spdk_nvme_transport_id trid_pcie;
4746 
4747 	if (g_hotplug_probe_ctx) {
4748 		return SPDK_POLLER_BUSY;
4749 	}
4750 
4751 	memset(&trid_pcie, 0, sizeof(trid_pcie));
4752 	spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE);
4753 
4754 	g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL,
4755 			      hotplug_probe_cb, attach_cb, NULL);
4756 
4757 	if (g_hotplug_probe_ctx) {
4758 		assert(g_hotplug_probe_poller == NULL);
4759 		g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000);
4760 	}
4761 
4762 	return SPDK_POLLER_BUSY;
4763 }
4764 
4765 void
4766 bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts)
4767 {
4768 	*opts = g_opts;
4769 }
4770 
4771 static bool bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec,
4772 		uint32_t reconnect_delay_sec,
4773 		uint32_t fast_io_fail_timeout_sec);
4774 
4775 static int
4776 bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts)
4777 {
4778 	if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) {
4779 		/* Can't set timeout_admin_us without also setting timeout_us */
4780 		SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n");
4781 		return -EINVAL;
4782 	}
4783 
4784 	if (opts->bdev_retry_count < -1) {
4785 		SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n");
4786 		return -EINVAL;
4787 	}
4788 
4789 	if (!bdev_nvme_check_io_error_resiliency_params(opts->ctrlr_loss_timeout_sec,
4790 			opts->reconnect_delay_sec,
4791 			opts->fast_io_fail_timeout_sec)) {
4792 		return -EINVAL;
4793 	}
4794 
4795 	return 0;
4796 }
4797 
4798 int
4799 bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts)
4800 {
4801 	int ret;
4802 
4803 	ret = bdev_nvme_validate_opts(opts);
4804 	if (ret) {
4805 		SPDK_WARNLOG("Failed to set nvme opts.\n");
4806 		return ret;
4807 	}
4808 
4809 	if (g_bdev_nvme_init_thread != NULL) {
4810 		if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
4811 			return -EPERM;
4812 		}
4813 	}
4814 
4815 	if (opts->rdma_srq_size != 0) {
4816 		struct spdk_nvme_transport_opts drv_opts;
4817 
4818 		spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts));
4819 		drv_opts.rdma_srq_size = opts->rdma_srq_size;
4820 
4821 		ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts));
4822 		if (ret) {
4823 			SPDK_ERRLOG("Failed to set NVMe transport opts.\n");
4824 			return ret;
4825 		}
4826 	}
4827 
4828 	g_opts = *opts;
4829 
4830 	return 0;
4831 }
4832 
4833 struct set_nvme_hotplug_ctx {
4834 	uint64_t period_us;
4835 	bool enabled;
4836 	spdk_msg_fn fn;
4837 	void *fn_ctx;
4838 };
4839 
4840 static void
4841 set_nvme_hotplug_period_cb(void *_ctx)
4842 {
4843 	struct set_nvme_hotplug_ctx *ctx = _ctx;
4844 
4845 	spdk_poller_unregister(&g_hotplug_poller);
4846 	if (ctx->enabled) {
4847 		g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us);
4848 	}
4849 
4850 	g_nvme_hotplug_poll_period_us = ctx->period_us;
4851 	g_nvme_hotplug_enabled = ctx->enabled;
4852 	if (ctx->fn) {
4853 		ctx->fn(ctx->fn_ctx);
4854 	}
4855 
4856 	free(ctx);
4857 }
4858 
4859 int
4860 bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx)
4861 {
4862 	struct set_nvme_hotplug_ctx *ctx;
4863 
4864 	if (enabled == true && !spdk_process_is_primary()) {
4865 		return -EPERM;
4866 	}
4867 
4868 	ctx = calloc(1, sizeof(*ctx));
4869 	if (ctx == NULL) {
4870 		return -ENOMEM;
4871 	}
4872 
4873 	period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us;
4874 	ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX);
4875 	ctx->enabled = enabled;
4876 	ctx->fn = cb;
4877 	ctx->fn_ctx = cb_ctx;
4878 
4879 	spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx);
4880 	return 0;
4881 }
4882 
4883 static void
4884 nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
4885 				    struct nvme_async_probe_ctx *ctx)
4886 {
4887 	struct nvme_ns	*nvme_ns;
4888 	struct nvme_bdev	*nvme_bdev;
4889 	size_t			j;
4890 
4891 	assert(nvme_ctrlr != NULL);
4892 
4893 	if (ctx->names == NULL) {
4894 		populate_namespaces_cb(ctx, 0, 0);
4895 		return;
4896 	}
4897 
4898 	/*
4899 	 * Report the new bdevs that were created in this call.
4900 	 * There can be more than one bdev per NVMe controller.
4901 	 */
4902 	j = 0;
4903 	nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
4904 	while (nvme_ns != NULL) {
4905 		nvme_bdev = nvme_ns->bdev;
4906 		if (j < ctx->count) {
4907 			ctx->names[j] = nvme_bdev->disk.name;
4908 			j++;
4909 		} else {
4910 			SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n",
4911 				    ctx->count);
4912 			populate_namespaces_cb(ctx, 0, -ERANGE);
4913 			return;
4914 		}
4915 
4916 		nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
4917 	}
4918 
4919 	populate_namespaces_cb(ctx, j, 0);
4920 }
4921 
4922 static int
4923 bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
4924 			       struct spdk_nvme_ctrlr *new_ctrlr,
4925 			       struct spdk_nvme_transport_id *trid)
4926 {
4927 	struct nvme_path_id *tmp_trid;
4928 
4929 	if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
4930 		SPDK_ERRLOG("PCIe failover is not supported.\n");
4931 		return -ENOTSUP;
4932 	}
4933 
4934 	/* Currently we only support failover to the same transport type. */
4935 	if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) {
4936 		SPDK_WARNLOG("Failover from trtype: %s to a different trtype: %s is not supported currently\n",
4937 			     spdk_nvme_transport_id_trtype_str(nvme_ctrlr->active_path_id->trid.trtype),
4938 			     spdk_nvme_transport_id_trtype_str(trid->trtype));
4939 		return -EINVAL;
4940 	}
4941 
4942 
4943 	/* Currently we only support failover to the same NQN. */
4944 	if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) {
4945 		SPDK_WARNLOG("Failover from subnqn: %s to a different subnqn: %s is not supported currently\n",
4946 			     nvme_ctrlr->active_path_id->trid.subnqn, trid->subnqn);
4947 		return -EINVAL;
4948 	}
4949 
4950 	/* Skip all the other checks if we've already registered this path. */
4951 	TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) {
4952 		if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) {
4953 			SPDK_WARNLOG("This path (traddr: %s subnqn: %s) is already registered\n", trid->traddr,
4954 				     trid->subnqn);
4955 			return -EEXIST;
4956 		}
4957 	}
4958 
4959 	return 0;
4960 }
4961 
4962 static int
4963 bdev_nvme_check_secondary_namespace(struct nvme_ctrlr *nvme_ctrlr,
4964 				    struct spdk_nvme_ctrlr *new_ctrlr)
4965 {
4966 	struct nvme_ns *nvme_ns;
4967 	struct spdk_nvme_ns *new_ns;
4968 
4969 	nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
4970 	while (nvme_ns != NULL) {
4971 		new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id);
4972 		assert(new_ns != NULL);
4973 
4974 		if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) {
4975 			return -EINVAL;
4976 		}
4977 
4978 		nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
4979 	}
4980 
4981 	return 0;
4982 }
4983 
4984 static int
4985 _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
4986 			      struct spdk_nvme_transport_id *trid)
4987 {
4988 	struct nvme_path_id *new_trid, *tmp_trid;
4989 
4990 	new_trid = calloc(1, sizeof(*new_trid));
4991 	if (new_trid == NULL) {
4992 		return -ENOMEM;
4993 	}
4994 	new_trid->trid = *trid;
4995 	new_trid->is_failed = false;
4996 
4997 	TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) {
4998 		if (tmp_trid->is_failed && tmp_trid != nvme_ctrlr->active_path_id) {
4999 			TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link);
5000 			return 0;
5001 		}
5002 	}
5003 
5004 	TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link);
5005 	return 0;
5006 }
5007 
5008 /* This is the case that a secondary path is added to an existing
5009  * nvme_ctrlr for failover. After checking if it can access the same
5010  * namespaces as the primary path, it is disconnected until failover occurs.
5011  */
5012 static int
5013 bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
5014 			     struct spdk_nvme_ctrlr *new_ctrlr,
5015 			     struct spdk_nvme_transport_id *trid)
5016 {
5017 	int rc;
5018 
5019 	assert(nvme_ctrlr != NULL);
5020 
5021 	pthread_mutex_lock(&nvme_ctrlr->mutex);
5022 
5023 	rc = bdev_nvme_check_secondary_trid(nvme_ctrlr, new_ctrlr, trid);
5024 	if (rc != 0) {
5025 		goto exit;
5026 	}
5027 
5028 	rc = bdev_nvme_check_secondary_namespace(nvme_ctrlr, new_ctrlr);
5029 	if (rc != 0) {
5030 		goto exit;
5031 	}
5032 
5033 	rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid);
5034 
5035 exit:
5036 	pthread_mutex_unlock(&nvme_ctrlr->mutex);
5037 
5038 	spdk_nvme_detach(new_ctrlr);
5039 
5040 	return rc;
5041 }
5042 
5043 static void
5044 connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
5045 		  struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
5046 {
5047 	struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
5048 	struct nvme_async_probe_ctx *ctx;
5049 	int rc;
5050 
5051 	ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts);
5052 	ctx->ctrlr_attached = true;
5053 
5054 	rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx);
5055 	if (rc != 0) {
5056 		populate_namespaces_cb(ctx, 0, rc);
5057 	}
5058 }
5059 
5060 static void
5061 connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
5062 			struct spdk_nvme_ctrlr *ctrlr,
5063 			const struct spdk_nvme_ctrlr_opts *opts)
5064 {
5065 	struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
5066 	struct nvme_ctrlr *nvme_ctrlr;
5067 	struct nvme_async_probe_ctx *ctx;
5068 	int rc;
5069 
5070 	ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts);
5071 	ctx->ctrlr_attached = true;
5072 
5073 	nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name);
5074 	if (nvme_ctrlr) {
5075 		rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid);
5076 	} else {
5077 		rc = -ENODEV;
5078 	}
5079 
5080 	populate_namespaces_cb(ctx, 0, rc);
5081 }
5082 
5083 static int
5084 bdev_nvme_async_poll(void *arg)
5085 {
5086 	struct nvme_async_probe_ctx	*ctx = arg;
5087 	int				rc;
5088 
5089 	rc = spdk_nvme_probe_poll_async(ctx->probe_ctx);
5090 	if (spdk_unlikely(rc != -EAGAIN)) {
5091 		ctx->probe_done = true;
5092 		spdk_poller_unregister(&ctx->poller);
5093 		if (!ctx->ctrlr_attached) {
5094 			/* The probe is done, but no controller was attached.
5095 			 * That means we had a failure, so report -EIO back to
5096 			 * the caller (usually the RPC). populate_namespaces_cb()
5097 			 * will take care of freeing the nvme_async_probe_ctx.
5098 			 */
5099 			populate_namespaces_cb(ctx, 0, -EIO);
5100 		} else if (ctx->namespaces_populated) {
5101 			/* The namespaces for the attached controller were all
5102 			 * populated and the response was already sent to the
5103 			 * caller (usually the RPC).  So free the context here.
5104 			 */
5105 			free(ctx);
5106 		}
5107 	}
5108 
5109 	return SPDK_POLLER_BUSY;
5110 }
5111 
5112 static bool
5113 bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec,
5114 		uint32_t reconnect_delay_sec,
5115 		uint32_t fast_io_fail_timeout_sec)
5116 {
5117 	if (ctrlr_loss_timeout_sec < -1) {
5118 		SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n");
5119 		return false;
5120 	} else if (ctrlr_loss_timeout_sec == -1) {
5121 		if (reconnect_delay_sec == 0) {
5122 			SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n");
5123 			return false;
5124 		} else if (fast_io_fail_timeout_sec != 0 &&
5125 			   fast_io_fail_timeout_sec < reconnect_delay_sec) {
5126 			SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n");
5127 			return false;
5128 		}
5129 	} else if (ctrlr_loss_timeout_sec != 0) {
5130 		if (reconnect_delay_sec == 0) {
5131 			SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n");
5132 			return false;
5133 		} else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) {
5134 			SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n");
5135 			return false;
5136 		} else if (fast_io_fail_timeout_sec != 0) {
5137 			if (fast_io_fail_timeout_sec < reconnect_delay_sec) {
5138 				SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n");
5139 				return false;
5140 			} else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) {
5141 				SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n");
5142 				return false;
5143 			}
5144 		}
5145 	} else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) {
5146 		SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n");
5147 		return false;
5148 	}
5149 
5150 	return true;
5151 }
5152 
5153 int
5154 bdev_nvme_create(struct spdk_nvme_transport_id *trid,
5155 		 const char *base_name,
5156 		 const char **names,
5157 		 uint32_t count,
5158 		 spdk_bdev_create_nvme_fn cb_fn,
5159 		 void *cb_ctx,
5160 		 struct spdk_nvme_ctrlr_opts *drv_opts,
5161 		 struct nvme_ctrlr_opts *bdev_opts,
5162 		 bool multipath)
5163 {
5164 	struct nvme_probe_skip_entry	*entry, *tmp;
5165 	struct nvme_async_probe_ctx	*ctx;
5166 	spdk_nvme_attach_cb attach_cb;
5167 
5168 	/* TODO expand this check to include both the host and target TRIDs.
5169 	 * Only if both are the same should we fail.
5170 	 */
5171 	if (nvme_ctrlr_get(trid) != NULL) {
5172 		SPDK_ERRLOG("A controller with the provided trid (traddr: %s) already exists.\n", trid->traddr);
5173 		return -EEXIST;
5174 	}
5175 
5176 	if (bdev_opts != NULL &&
5177 	    !bdev_nvme_check_io_error_resiliency_params(bdev_opts->ctrlr_loss_timeout_sec,
5178 			    bdev_opts->reconnect_delay_sec,
5179 			    bdev_opts->fast_io_fail_timeout_sec)) {
5180 		return -EINVAL;
5181 	}
5182 
5183 	ctx = calloc(1, sizeof(*ctx));
5184 	if (!ctx) {
5185 		return -ENOMEM;
5186 	}
5187 	ctx->base_name = base_name;
5188 	ctx->names = names;
5189 	ctx->count = count;
5190 	ctx->cb_fn = cb_fn;
5191 	ctx->cb_ctx = cb_ctx;
5192 	ctx->trid = *trid;
5193 
5194 	if (bdev_opts) {
5195 		memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));
5196 	} else {
5197 		bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts);
5198 	}
5199 
5200 	if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
5201 		TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) {
5202 			if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) {
5203 				TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq);
5204 				free(entry);
5205 				break;
5206 			}
5207 		}
5208 	}
5209 
5210 	if (drv_opts) {
5211 		memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts));
5212 	} else {
5213 		spdk_nvme_ctrlr_get_default_ctrlr_opts(&ctx->drv_opts, sizeof(ctx->drv_opts));
5214 	}
5215 
5216 	ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count;
5217 	ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout;
5218 	ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms;
5219 	ctx->drv_opts.disable_read_ana_log_page = true;
5220 	ctx->drv_opts.transport_tos = g_opts.transport_tos;
5221 
5222 	if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || multipath) {
5223 		attach_cb = connect_attach_cb;
5224 	} else {
5225 		attach_cb = connect_set_failover_cb;
5226 	}
5227 
5228 	ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb);
5229 	if (ctx->probe_ctx == NULL) {
5230 		SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr);
5231 		free(ctx);
5232 		return -ENODEV;
5233 	}
5234 	ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000);
5235 
5236 	return 0;
5237 }
5238 
5239 static bool
5240 nvme_path_should_delete(struct nvme_path_id *p, const struct nvme_path_id *path_id)
5241 {
5242 	if (path_id->trid.trtype != 0) {
5243 		if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) {
5244 			if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) {
5245 				return false;
5246 			}
5247 		} else {
5248 			if (path_id->trid.trtype != p->trid.trtype) {
5249 				return false;
5250 			}
5251 		}
5252 	}
5253 
5254 	if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) {
5255 		if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) {
5256 			return false;
5257 		}
5258 	}
5259 
5260 	if (path_id->trid.adrfam != 0) {
5261 		if (path_id->trid.adrfam != p->trid.adrfam) {
5262 			return false;
5263 		}
5264 	}
5265 
5266 	if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) {
5267 		if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) {
5268 			return false;
5269 		}
5270 	}
5271 
5272 	if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) {
5273 		if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) {
5274 			return false;
5275 		}
5276 	}
5277 
5278 	if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) {
5279 		if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) {
5280 			return false;
5281 		}
5282 	}
5283 
5284 	if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) {
5285 		if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) {
5286 			return false;
5287 		}
5288 	}
5289 
5290 	return true;
5291 }
5292 
5293 static int
5294 _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id)
5295 {
5296 	struct nvme_path_id	*p, *t;
5297 	int			rc = -ENXIO;
5298 
5299 	TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) {
5300 		if (p == TAILQ_FIRST(&nvme_ctrlr->trids)) {
5301 			break;
5302 		}
5303 
5304 		if (!nvme_path_should_delete(p, path_id)) {
5305 			continue;
5306 		}
5307 
5308 		/* We are not using the specified path. */
5309 		TAILQ_REMOVE(&nvme_ctrlr->trids, p, link);
5310 		free(p);
5311 		rc = 0;
5312 	}
5313 
5314 	if (p == NULL || !nvme_path_should_delete(p, path_id)) {
5315 		return rc;
5316 	}
5317 
5318 	/* If we made it here, then this path is a match! Now we need to remove it. */
5319 
5320 	/* This is the active path in use right now. The active path is always the first in the list. */
5321 	assert(p == nvme_ctrlr->active_path_id);
5322 
5323 	if (!TAILQ_NEXT(p, link)) {
5324 		/* The current path is the only path. */
5325 		rc = bdev_nvme_delete_ctrlr(nvme_ctrlr, false);
5326 	} else {
5327 		/* There is an alternative path. */
5328 		rc = bdev_nvme_failover(nvme_ctrlr, true);
5329 	}
5330 
5331 	return rc;
5332 }
5333 
5334 int
5335 bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id)
5336 {
5337 	struct nvme_bdev_ctrlr	*nbdev_ctrlr;
5338 	struct nvme_ctrlr	*nvme_ctrlr, *tmp_nvme_ctrlr;
5339 	int			rc = -ENXIO, _rc;
5340 
5341 	if (name == NULL || path_id == NULL) {
5342 		return -EINVAL;
5343 	}
5344 
5345 	nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
5346 	if (nbdev_ctrlr == NULL) {
5347 		SPDK_ERRLOG("Failed to find NVMe bdev controller\n");
5348 		return -ENODEV;
5349 	}
5350 
5351 	TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) {
5352 		_rc = _bdev_nvme_delete(nvme_ctrlr, path_id);
5353 		if (_rc < 0 && _rc != -ENXIO) {
5354 			return _rc;
5355 		} else if (_rc == 0) {
5356 			/* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr
5357 			 * was deleted successfully. To remember the successful deletion,
5358 			 * overwrite rc only if _rc is zero.
5359 			 */
5360 			rc = 0;
5361 		}
5362 	}
5363 
5364 	/* All nvme_ctrlrs were deleted or no nvme_ctrlr which had the trid was found. */
5365 	return rc;
5366 }
5367 
5368 #define DISCOVERY_INFOLOG(ctx, format, ...) \
5369 	SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__);
5370 
5371 #define DISCOVERY_ERRLOG(ctx, format, ...) \
5372 	SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__);
5373 
5374 struct discovery_entry_ctx {
5375 	char						name[128];
5376 	struct spdk_nvme_transport_id			trid;
5377 	struct spdk_nvme_ctrlr_opts			drv_opts;
5378 	struct spdk_nvmf_discovery_log_page_entry	entry;
5379 	TAILQ_ENTRY(discovery_entry_ctx)		tailq;
5380 	struct discovery_ctx				*ctx;
5381 };
5382 
5383 struct discovery_ctx {
5384 	char					*name;
5385 	spdk_bdev_nvme_start_discovery_fn	start_cb_fn;
5386 	spdk_bdev_nvme_stop_discovery_fn	stop_cb_fn;
5387 	void					*cb_ctx;
5388 	struct spdk_nvme_probe_ctx		*probe_ctx;
5389 	struct spdk_nvme_detach_ctx		*detach_ctx;
5390 	struct spdk_nvme_ctrlr			*ctrlr;
5391 	struct spdk_nvme_transport_id		trid;
5392 	struct discovery_entry_ctx		*entry_ctx_in_use;
5393 	struct spdk_poller			*poller;
5394 	struct spdk_nvme_ctrlr_opts		drv_opts;
5395 	struct nvme_ctrlr_opts			bdev_opts;
5396 	struct spdk_nvmf_discovery_log_page	*log_page;
5397 	TAILQ_ENTRY(discovery_ctx)		tailq;
5398 	TAILQ_HEAD(, discovery_entry_ctx)	nvm_entry_ctxs;
5399 	TAILQ_HEAD(, discovery_entry_ctx)	discovery_entry_ctxs;
5400 	int					rc;
5401 	bool					wait_for_attach;
5402 	uint64_t				timeout_ticks;
5403 	/* Denotes that the discovery service is being started. We're waiting
5404 	 * for the initial connection to the discovery controller to be
5405 	 * established and attach discovered NVM ctrlrs.
5406 	 */
5407 	bool					initializing;
5408 	/* Denotes if a discovery is currently in progress for this context.
5409 	 * That includes connecting to newly discovered subsystems.  Used to
5410 	 * ensure we do not start a new discovery until an existing one is
5411 	 * complete.
5412 	 */
5413 	bool					in_progress;
5414 
5415 	/* Denotes if another discovery is needed after the one in progress
5416 	 * completes.  Set when we receive an AER completion while a discovery
5417 	 * is already in progress.
5418 	 */
5419 	bool					pending;
5420 
5421 	/* Signal to the discovery context poller that it should stop the
5422 	 * discovery service, including detaching from the current discovery
5423 	 * controller.
5424 	 */
5425 	bool					stop;
5426 
5427 	struct spdk_thread			*calling_thread;
5428 	uint32_t				index;
5429 	uint32_t				attach_in_progress;
5430 	char					*hostnqn;
5431 
5432 	/* Denotes if the discovery service was started by the mdns discovery.
5433 	 */
5434 	bool					from_mdns_discovery_service;
5435 };
5436 
5437 TAILQ_HEAD(discovery_ctxs, discovery_ctx);
5438 static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs);
5439 
5440 static void get_discovery_log_page(struct discovery_ctx *ctx);
5441 
5442 static void
5443 free_discovery_ctx(struct discovery_ctx *ctx)
5444 {
5445 	free(ctx->log_page);
5446 	free(ctx->hostnqn);
5447 	free(ctx->name);
5448 	free(ctx);
5449 }
5450 
5451 static void
5452 discovery_complete(struct discovery_ctx *ctx)
5453 {
5454 	ctx->initializing = false;
5455 	ctx->in_progress = false;
5456 	if (ctx->pending) {
5457 		ctx->pending = false;
5458 		get_discovery_log_page(ctx);
5459 	}
5460 }
5461 
5462 static void
5463 build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid,
5464 			       struct spdk_nvmf_discovery_log_page_entry *entry)
5465 {
5466 	char *space;
5467 
5468 	trid->trtype = entry->trtype;
5469 	trid->adrfam = entry->adrfam;
5470 	memcpy(trid->traddr, entry->traddr, sizeof(entry->traddr));
5471 	memcpy(trid->trsvcid, entry->trsvcid, sizeof(entry->trsvcid));
5472 	memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn));
5473 
5474 	/* We want the traddr, trsvcid and subnqn fields to be NULL-terminated.
5475 	 * But the log page entries typically pad them with spaces, not zeroes.
5476 	 * So add a NULL terminator to each of these fields at the appropriate
5477 	 * location.
5478 	 */
5479 	space = strchr(trid->traddr, ' ');
5480 	if (space) {
5481 		*space = 0;
5482 	}
5483 	space = strchr(trid->trsvcid, ' ');
5484 	if (space) {
5485 		*space = 0;
5486 	}
5487 	space = strchr(trid->subnqn, ' ');
5488 	if (space) {
5489 		*space = 0;
5490 	}
5491 }
5492 
5493 static void
5494 stop_discovery(struct discovery_ctx *ctx, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx)
5495 {
5496 	ctx->stop = true;
5497 	ctx->stop_cb_fn = cb_fn;
5498 	ctx->cb_ctx = cb_ctx;
5499 
5500 	while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) {
5501 		struct discovery_entry_ctx *entry_ctx;
5502 		struct nvme_path_id path = {};
5503 
5504 		entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs);
5505 		path.trid = entry_ctx->trid;
5506 		bdev_nvme_delete(entry_ctx->name, &path);
5507 		TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);
5508 		free(entry_ctx);
5509 	}
5510 
5511 	while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) {
5512 		struct discovery_entry_ctx *entry_ctx;
5513 
5514 		entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs);
5515 		TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq);
5516 		free(entry_ctx);
5517 	}
5518 
5519 	free(ctx->entry_ctx_in_use);
5520 	ctx->entry_ctx_in_use = NULL;
5521 }
5522 
5523 static void
5524 discovery_remove_controllers(struct discovery_ctx *ctx)
5525 {
5526 	struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page;
5527 	struct discovery_entry_ctx *entry_ctx, *tmp;
5528 	struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry;
5529 	struct spdk_nvme_transport_id old_trid;
5530 	uint64_t numrec, i;
5531 	bool found;
5532 
5533 	numrec = from_le64(&log_page->numrec);
5534 	TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) {
5535 		found = false;
5536 		old_entry = &entry_ctx->entry;
5537 		build_trid_from_log_page_entry(&old_trid, old_entry);
5538 		for (i = 0; i < numrec; i++) {
5539 			new_entry = &log_page->entries[i];
5540 			if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) {
5541 				DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n",
5542 						  old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);
5543 				found = true;
5544 				break;
5545 			}
5546 		}
5547 		if (!found) {
5548 			struct nvme_path_id path = {};
5549 
5550 			DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n",
5551 					  old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);
5552 
5553 			path.trid = entry_ctx->trid;
5554 			bdev_nvme_delete(entry_ctx->name, &path);
5555 			TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);
5556 			free(entry_ctx);
5557 		}
5558 	}
5559 	free(log_page);
5560 	ctx->log_page = NULL;
5561 	discovery_complete(ctx);
5562 }
5563 
5564 static void
5565 complete_discovery_start(struct discovery_ctx *ctx, int status)
5566 {
5567 	ctx->timeout_ticks = 0;
5568 	ctx->rc = status;
5569 	if (ctx->start_cb_fn) {
5570 		ctx->start_cb_fn(ctx->cb_ctx, status);
5571 		ctx->start_cb_fn = NULL;
5572 		ctx->cb_ctx = NULL;
5573 	}
5574 }
5575 
5576 static void
5577 discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc)
5578 {
5579 	struct discovery_entry_ctx *entry_ctx = cb_ctx;
5580 	struct discovery_ctx *ctx = entry_ctx->ctx;
5581 
5582 	DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name);
5583 	ctx->attach_in_progress--;
5584 	if (ctx->attach_in_progress == 0) {
5585 		complete_discovery_start(ctx, ctx->rc);
5586 		if (ctx->initializing && ctx->rc != 0) {
5587 			DISCOVERY_ERRLOG(ctx, "stopping discovery due to errors: %d\n", ctx->rc);
5588 			stop_discovery(ctx, NULL, ctx->cb_ctx);
5589 		} else {
5590 			discovery_remove_controllers(ctx);
5591 		}
5592 	}
5593 }
5594 
5595 static struct discovery_entry_ctx *
5596 create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid)
5597 {
5598 	struct discovery_entry_ctx *new_ctx;
5599 
5600 	new_ctx = calloc(1, sizeof(*new_ctx));
5601 	if (new_ctx == NULL) {
5602 		DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
5603 		return NULL;
5604 	}
5605 
5606 	new_ctx->ctx = ctx;
5607 	memcpy(&new_ctx->trid, trid, sizeof(*trid));
5608 	spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));
5609 	snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);
5610 	return new_ctx;
5611 }
5612 
5613 static void
5614 discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl,
5615 		      struct spdk_nvmf_discovery_log_page *log_page)
5616 {
5617 	struct discovery_ctx *ctx = cb_arg;
5618 	struct discovery_entry_ctx *entry_ctx, *tmp;
5619 	struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry;
5620 	uint64_t numrec, i;
5621 	bool found;
5622 
5623 	if (rc || spdk_nvme_cpl_is_error(cpl)) {
5624 		DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n");
5625 		return;
5626 	}
5627 
5628 	ctx->log_page = log_page;
5629 	assert(ctx->attach_in_progress == 0);
5630 	numrec = from_le64(&log_page->numrec);
5631 	TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) {
5632 		TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq);
5633 		free(entry_ctx);
5634 	}
5635 	for (i = 0; i < numrec; i++) {
5636 		found = false;
5637 		new_entry = &log_page->entries[i];
5638 		if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) {
5639 			struct discovery_entry_ctx *new_ctx;
5640 			struct spdk_nvme_transport_id trid = {};
5641 
5642 			build_trid_from_log_page_entry(&trid, new_entry);
5643 			new_ctx = create_discovery_entry_ctx(ctx, &trid);
5644 			if (new_ctx == NULL) {
5645 				DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
5646 				break;
5647 			}
5648 
5649 			TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq);
5650 			continue;
5651 		}
5652 		TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) {
5653 			old_entry = &entry_ctx->entry;
5654 			if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) {
5655 				found = true;
5656 				break;
5657 			}
5658 		}
5659 		if (!found) {
5660 			struct discovery_entry_ctx *subnqn_ctx = NULL, *new_ctx;
5661 			struct discovery_ctx *d_ctx;
5662 
5663 			TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) {
5664 				TAILQ_FOREACH(subnqn_ctx, &d_ctx->nvm_entry_ctxs, tailq) {
5665 					if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn,
5666 						    sizeof(new_entry->subnqn))) {
5667 						break;
5668 					}
5669 				}
5670 				if (subnqn_ctx) {
5671 					break;
5672 				}
5673 			}
5674 
5675 			new_ctx = calloc(1, sizeof(*new_ctx));
5676 			if (new_ctx == NULL) {
5677 				DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
5678 				break;
5679 			}
5680 
5681 			new_ctx->ctx = ctx;
5682 			memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry));
5683 			build_trid_from_log_page_entry(&new_ctx->trid, new_entry);
5684 			if (subnqn_ctx) {
5685 				snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name);
5686 				DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n",
5687 						  new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid,
5688 						  new_ctx->name);
5689 			} else {
5690 				snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++);
5691 				DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n",
5692 						  new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid,
5693 						  new_ctx->name);
5694 			}
5695 			spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));
5696 			snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);
5697 			rc = bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0,
5698 					      discovery_attach_controller_done, new_ctx,
5699 					      &new_ctx->drv_opts, &ctx->bdev_opts, true);
5700 			if (rc == 0) {
5701 				TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq);
5702 				ctx->attach_in_progress++;
5703 			} else {
5704 				DISCOVERY_ERRLOG(ctx, "bdev_nvme_create failed (%s)\n", spdk_strerror(-rc));
5705 			}
5706 		}
5707 	}
5708 
5709 	if (ctx->attach_in_progress == 0) {
5710 		discovery_remove_controllers(ctx);
5711 	}
5712 }
5713 
5714 static void
5715 get_discovery_log_page(struct discovery_ctx *ctx)
5716 {
5717 	int rc;
5718 
5719 	assert(ctx->in_progress == false);
5720 	ctx->in_progress = true;
5721 	rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx);
5722 	if (rc != 0) {
5723 		DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n");
5724 	}
5725 	DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n");
5726 }
5727 
5728 static void
5729 discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
5730 {
5731 	struct discovery_ctx *ctx = arg;
5732 	uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
5733 
5734 	if (spdk_nvme_cpl_is_error(cpl)) {
5735 		DISCOVERY_ERRLOG(ctx, "aer failed\n");
5736 		return;
5737 	}
5738 
5739 	if (log_page_id != SPDK_NVME_LOG_DISCOVERY) {
5740 		DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id);
5741 		return;
5742 	}
5743 
5744 	DISCOVERY_INFOLOG(ctx, "got aer\n");
5745 	if (ctx->in_progress) {
5746 		ctx->pending = true;
5747 		return;
5748 	}
5749 
5750 	get_discovery_log_page(ctx);
5751 }
5752 
5753 static void
5754 discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
5755 		    struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
5756 {
5757 	struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
5758 	struct discovery_ctx *ctx;
5759 
5760 	ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts);
5761 
5762 	DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n");
5763 	ctx->probe_ctx = NULL;
5764 	ctx->ctrlr = ctrlr;
5765 
5766 	if (ctx->rc != 0) {
5767 		DISCOVERY_ERRLOG(ctx, "encountered error while attaching discovery ctrlr: %d\n",
5768 				 ctx->rc);
5769 		return;
5770 	}
5771 
5772 	spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx);
5773 }
5774 
5775 static int
5776 discovery_poller(void *arg)
5777 {
5778 	struct discovery_ctx *ctx = arg;
5779 	struct spdk_nvme_transport_id *trid;
5780 	int rc;
5781 
5782 	if (ctx->detach_ctx) {
5783 		rc = spdk_nvme_detach_poll_async(ctx->detach_ctx);
5784 		if (rc != -EAGAIN) {
5785 			ctx->detach_ctx = NULL;
5786 			ctx->ctrlr = NULL;
5787 		}
5788 	} else if (ctx->stop) {
5789 		if (ctx->ctrlr != NULL) {
5790 			rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx);
5791 			if (rc == 0) {
5792 				return SPDK_POLLER_BUSY;
5793 			}
5794 			DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n");
5795 		}
5796 		spdk_poller_unregister(&ctx->poller);
5797 		TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq);
5798 		assert(ctx->start_cb_fn == NULL);
5799 		if (ctx->stop_cb_fn != NULL) {
5800 			ctx->stop_cb_fn(ctx->cb_ctx);
5801 		}
5802 		free_discovery_ctx(ctx);
5803 	} else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) {
5804 		if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
5805 			DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n");
5806 			assert(ctx->initializing);
5807 			spdk_poller_unregister(&ctx->poller);
5808 			TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq);
5809 			complete_discovery_start(ctx, -ETIMEDOUT);
5810 			stop_discovery(ctx, NULL, NULL);
5811 			free_discovery_ctx(ctx);
5812 			return SPDK_POLLER_BUSY;
5813 		}
5814 
5815 		assert(ctx->entry_ctx_in_use == NULL);
5816 		ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs);
5817 		TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
5818 		trid = &ctx->entry_ctx_in_use->trid;
5819 		ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb);
5820 		if (ctx->probe_ctx) {
5821 			spdk_poller_unregister(&ctx->poller);
5822 			ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000);
5823 		} else {
5824 			DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n");
5825 			TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
5826 			ctx->entry_ctx_in_use = NULL;
5827 		}
5828 	} else if (ctx->probe_ctx) {
5829 		if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
5830 			DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n");
5831 			complete_discovery_start(ctx, -ETIMEDOUT);
5832 			return SPDK_POLLER_BUSY;
5833 		}
5834 
5835 		rc = spdk_nvme_probe_poll_async(ctx->probe_ctx);
5836 		if (rc != -EAGAIN) {
5837 			if (ctx->rc != 0) {
5838 				assert(ctx->initializing);
5839 				stop_discovery(ctx, NULL, ctx->cb_ctx);
5840 			} else {
5841 				assert(rc == 0);
5842 				DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n");
5843 				ctx->rc = rc;
5844 				get_discovery_log_page(ctx);
5845 			}
5846 		}
5847 	} else {
5848 		if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
5849 			DISCOVERY_ERRLOG(ctx, "timed out while attaching NVM ctrlrs\n");
5850 			complete_discovery_start(ctx, -ETIMEDOUT);
5851 			/* We need to wait until all NVM ctrlrs are attached before we stop the
5852 			 * discovery service to make sure we don't detach a ctrlr that is still
5853 			 * being attached.
5854 			 */
5855 			if (ctx->attach_in_progress == 0) {
5856 				stop_discovery(ctx, NULL, ctx->cb_ctx);
5857 				return SPDK_POLLER_BUSY;
5858 			}
5859 		}
5860 
5861 		rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr);
5862 		if (rc < 0) {
5863 			spdk_poller_unregister(&ctx->poller);
5864 			ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000);
5865 			TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
5866 			ctx->entry_ctx_in_use = NULL;
5867 
5868 			rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx);
5869 			if (rc != 0) {
5870 				DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n");
5871 				ctx->ctrlr = NULL;
5872 			}
5873 		}
5874 	}
5875 
5876 	return SPDK_POLLER_BUSY;
5877 }
5878 
5879 static void
5880 start_discovery_poller(void *arg)
5881 {
5882 	struct discovery_ctx *ctx = arg;
5883 
5884 	TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq);
5885 	ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000);
5886 }
5887 
5888 int
5889 bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid,
5890 			  const char *base_name,
5891 			  struct spdk_nvme_ctrlr_opts *drv_opts,
5892 			  struct nvme_ctrlr_opts *bdev_opts,
5893 			  uint64_t attach_timeout,
5894 			  bool from_mdns,
5895 			  spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx)
5896 {
5897 	struct discovery_ctx *ctx;
5898 	struct discovery_entry_ctx *discovery_entry_ctx;
5899 
5900 	snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
5901 	TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
5902 		if (strcmp(ctx->name, base_name) == 0) {
5903 			return -EEXIST;
5904 		}
5905 
5906 		if (ctx->entry_ctx_in_use != NULL) {
5907 			if (!spdk_nvme_transport_id_compare(trid, &ctx->entry_ctx_in_use->trid)) {
5908 				return -EEXIST;
5909 			}
5910 		}
5911 
5912 		TAILQ_FOREACH(discovery_entry_ctx, &ctx->discovery_entry_ctxs, tailq) {
5913 			if (!spdk_nvme_transport_id_compare(trid, &discovery_entry_ctx->trid)) {
5914 				return -EEXIST;
5915 			}
5916 		}
5917 	}
5918 
5919 	ctx = calloc(1, sizeof(*ctx));
5920 	if (ctx == NULL) {
5921 		return -ENOMEM;
5922 	}
5923 
5924 	ctx->name = strdup(base_name);
5925 	if (ctx->name == NULL) {
5926 		free_discovery_ctx(ctx);
5927 		return -ENOMEM;
5928 	}
5929 	memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts));
5930 	memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));
5931 	ctx->from_mdns_discovery_service = from_mdns;
5932 	ctx->bdev_opts.from_discovery_service = true;
5933 	ctx->calling_thread = spdk_get_thread();
5934 	ctx->start_cb_fn = cb_fn;
5935 	ctx->cb_ctx = cb_ctx;
5936 	ctx->initializing = true;
5937 	if (ctx->start_cb_fn) {
5938 		/* We can use this when dumping json to denote if this RPC parameter
5939 		 * was specified or not.
5940 		 */
5941 		ctx->wait_for_attach = true;
5942 	}
5943 	if (attach_timeout != 0) {
5944 		ctx->timeout_ticks = spdk_get_ticks() + attach_timeout *
5945 				     spdk_get_ticks_hz() / 1000ull;
5946 	}
5947 	TAILQ_INIT(&ctx->nvm_entry_ctxs);
5948 	TAILQ_INIT(&ctx->discovery_entry_ctxs);
5949 	memcpy(&ctx->trid, trid, sizeof(*trid));
5950 	/* Even if user did not specify hostnqn, we can still strdup("\0"); */
5951 	ctx->hostnqn = strdup(ctx->drv_opts.hostnqn);
5952 	if (ctx->hostnqn == NULL) {
5953 		free_discovery_ctx(ctx);
5954 		return -ENOMEM;
5955 	}
5956 	discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid);
5957 	if (discovery_entry_ctx == NULL) {
5958 		DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
5959 		free_discovery_ctx(ctx);
5960 		return -ENOMEM;
5961 	}
5962 
5963 	TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq);
5964 	spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx);
5965 	return 0;
5966 }
5967 
5968 int
5969 bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx)
5970 {
5971 	struct discovery_ctx *ctx;
5972 
5973 	TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
5974 		if (strcmp(name, ctx->name) == 0) {
5975 			if (ctx->stop) {
5976 				return -EALREADY;
5977 			}
5978 			/* If we're still starting the discovery service and ->rc is non-zero, we're
5979 			 * going to stop it as soon as we can
5980 			 */
5981 			if (ctx->initializing && ctx->rc != 0) {
5982 				return -EALREADY;
5983 			}
5984 			stop_discovery(ctx, cb_fn, cb_ctx);
5985 			return 0;
5986 		}
5987 	}
5988 
5989 	return -ENOENT;
5990 }
5991 
5992 static int
5993 bdev_nvme_library_init(void)
5994 {
5995 	g_bdev_nvme_init_thread = spdk_get_thread();
5996 
5997 	spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb,
5998 				bdev_nvme_destroy_poll_group_cb,
5999 				sizeof(struct nvme_poll_group),  "nvme_poll_groups");
6000 
6001 	return 0;
6002 }
6003 
6004 static void
6005 bdev_nvme_fini_destruct_ctrlrs(void)
6006 {
6007 	struct nvme_bdev_ctrlr *nbdev_ctrlr;
6008 	struct nvme_ctrlr *nvme_ctrlr;
6009 
6010 	pthread_mutex_lock(&g_bdev_nvme_mutex);
6011 	TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
6012 		TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
6013 			pthread_mutex_lock(&nvme_ctrlr->mutex);
6014 			if (nvme_ctrlr->destruct) {
6015 				/* This controller's destruction was already started
6016 				 * before the application started shutting down
6017 				 */
6018 				pthread_mutex_unlock(&nvme_ctrlr->mutex);
6019 				continue;
6020 			}
6021 			nvme_ctrlr->destruct = true;
6022 			pthread_mutex_unlock(&nvme_ctrlr->mutex);
6023 
6024 			spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct,
6025 					     nvme_ctrlr);
6026 		}
6027 	}
6028 
6029 	g_bdev_nvme_module_finish = true;
6030 	if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
6031 		pthread_mutex_unlock(&g_bdev_nvme_mutex);
6032 		spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL);
6033 		spdk_bdev_module_fini_done();
6034 		return;
6035 	}
6036 
6037 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
6038 }
6039 
6040 static void
6041 check_discovery_fini(void *arg)
6042 {
6043 	if (TAILQ_EMPTY(&g_discovery_ctxs)) {
6044 		bdev_nvme_fini_destruct_ctrlrs();
6045 	}
6046 }
6047 
6048 static void
6049 bdev_nvme_library_fini(void)
6050 {
6051 	struct nvme_probe_skip_entry *entry, *entry_tmp;
6052 	struct discovery_ctx *ctx;
6053 
6054 	spdk_poller_unregister(&g_hotplug_poller);
6055 	free(g_hotplug_probe_ctx);
6056 	g_hotplug_probe_ctx = NULL;
6057 
6058 	TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) {
6059 		TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq);
6060 		free(entry);
6061 	}
6062 
6063 	assert(spdk_get_thread() == g_bdev_nvme_init_thread);
6064 	if (TAILQ_EMPTY(&g_discovery_ctxs)) {
6065 		bdev_nvme_fini_destruct_ctrlrs();
6066 	} else {
6067 		TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
6068 			stop_discovery(ctx, check_discovery_fini, NULL);
6069 		}
6070 	}
6071 }
6072 
6073 static void
6074 bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio)
6075 {
6076 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
6077 	struct spdk_bdev *bdev = bdev_io->bdev;
6078 	struct spdk_dif_ctx dif_ctx;
6079 	struct spdk_dif_error err_blk = {};
6080 	int rc;
6081 
6082 	rc = spdk_dif_ctx_init(&dif_ctx,
6083 			       bdev->blocklen, bdev->md_len, bdev->md_interleave,
6084 			       bdev->dif_is_head_of_md, bdev->dif_type, bdev->dif_check_flags,
6085 			       bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0);
6086 	if (rc != 0) {
6087 		SPDK_ERRLOG("Initialization of DIF context failed\n");
6088 		return;
6089 	}
6090 
6091 	if (bdev->md_interleave) {
6092 		rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
6093 				     bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk);
6094 	} else {
6095 		struct iovec md_iov = {
6096 			.iov_base	= bdev_io->u.bdev.md_buf,
6097 			.iov_len	= bdev_io->u.bdev.num_blocks * bdev->md_len,
6098 		};
6099 
6100 		rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
6101 				     &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk);
6102 	}
6103 
6104 	if (rc != 0) {
6105 		SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n",
6106 			    err_blk.err_type, err_blk.err_offset);
6107 	} else {
6108 		SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n");
6109 	}
6110 }
6111 
6112 static void
6113 bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl)
6114 {
6115 	struct nvme_bdev_io *bio = ref;
6116 
6117 	if (spdk_nvme_cpl_is_success(cpl)) {
6118 		/* Run PI verification for read data buffer. */
6119 		bdev_nvme_verify_pi_error(bio);
6120 	}
6121 
6122 	/* Return original completion status */
6123 	bdev_nvme_io_complete_nvme_status(bio, &bio->cpl);
6124 }
6125 
6126 static void
6127 bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl)
6128 {
6129 	struct nvme_bdev_io *bio = ref;
6130 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
6131 	int ret;
6132 
6133 	if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) {
6134 		SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n",
6135 			    cpl->status.sct, cpl->status.sc);
6136 
6137 		/* Save completion status to use after verifying PI error. */
6138 		bio->cpl = *cpl;
6139 
6140 		if (spdk_likely(nvme_io_path_is_available(bio->io_path))) {
6141 			/* Read without PI checking to verify PI error. */
6142 			ret = bdev_nvme_no_pi_readv(bio,
6143 						    bdev_io->u.bdev.iovs,
6144 						    bdev_io->u.bdev.iovcnt,
6145 						    bdev_io->u.bdev.md_buf,
6146 						    bdev_io->u.bdev.num_blocks,
6147 						    bdev_io->u.bdev.offset_blocks);
6148 			if (ret == 0) {
6149 				return;
6150 			}
6151 		}
6152 	}
6153 
6154 	bdev_nvme_io_complete_nvme_status(bio, cpl);
6155 }
6156 
6157 static void
6158 bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl)
6159 {
6160 	struct nvme_bdev_io *bio = ref;
6161 
6162 	if (spdk_nvme_cpl_is_pi_error(cpl)) {
6163 		SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n",
6164 			    cpl->status.sct, cpl->status.sc);
6165 		/* Run PI verification for write data buffer if PI error is detected. */
6166 		bdev_nvme_verify_pi_error(bio);
6167 	}
6168 
6169 	bdev_nvme_io_complete_nvme_status(bio, cpl);
6170 }
6171 
6172 static void
6173 bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl)
6174 {
6175 	struct nvme_bdev_io *bio = ref;
6176 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
6177 
6178 	/* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks.
6179 	 * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error().
6180 	 */
6181 	bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0;
6182 
6183 	if (spdk_nvme_cpl_is_pi_error(cpl)) {
6184 		SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n",
6185 			    cpl->status.sct, cpl->status.sc);
6186 		/* Run PI verification for zone append data buffer if PI error is detected. */
6187 		bdev_nvme_verify_pi_error(bio);
6188 	}
6189 
6190 	bdev_nvme_io_complete_nvme_status(bio, cpl);
6191 }
6192 
6193 static void
6194 bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl)
6195 {
6196 	struct nvme_bdev_io *bio = ref;
6197 
6198 	if (spdk_nvme_cpl_is_pi_error(cpl)) {
6199 		SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n",
6200 			    cpl->status.sct, cpl->status.sc);
6201 		/* Run PI verification for compare data buffer if PI error is detected. */
6202 		bdev_nvme_verify_pi_error(bio);
6203 	}
6204 
6205 	bdev_nvme_io_complete_nvme_status(bio, cpl);
6206 }
6207 
6208 static void
6209 bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl)
6210 {
6211 	struct nvme_bdev_io *bio = ref;
6212 
6213 	/* Compare operation completion */
6214 	if (!bio->first_fused_completed) {
6215 		/* Save compare result for write callback */
6216 		bio->cpl = *cpl;
6217 		bio->first_fused_completed = true;
6218 		return;
6219 	}
6220 
6221 	/* Write operation completion */
6222 	if (spdk_nvme_cpl_is_error(&bio->cpl)) {
6223 		/* If bio->cpl is already an error, it means the compare operation failed.  In that case,
6224 		 * complete the IO with the compare operation's status.
6225 		 */
6226 		if (!spdk_nvme_cpl_is_error(cpl)) {
6227 			SPDK_ERRLOG("Unexpected write success after compare failure.\n");
6228 		}
6229 
6230 		bdev_nvme_io_complete_nvme_status(bio, &bio->cpl);
6231 	} else {
6232 		bdev_nvme_io_complete_nvme_status(bio, cpl);
6233 	}
6234 }
6235 
6236 static void
6237 bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl)
6238 {
6239 	struct nvme_bdev_io *bio = ref;
6240 
6241 	bdev_nvme_io_complete_nvme_status(bio, cpl);
6242 }
6243 
6244 static int
6245 fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc)
6246 {
6247 	switch (desc->zt) {
6248 	case SPDK_NVME_ZONE_TYPE_SEQWR:
6249 		info->type = SPDK_BDEV_ZONE_TYPE_SEQWR;
6250 		break;
6251 	default:
6252 		SPDK_ERRLOG("Invalid zone type: %#x in zone report\n", desc->zt);
6253 		return -EIO;
6254 	}
6255 
6256 	switch (desc->zs) {
6257 	case SPDK_NVME_ZONE_STATE_EMPTY:
6258 		info->state = SPDK_BDEV_ZONE_STATE_EMPTY;
6259 		break;
6260 	case SPDK_NVME_ZONE_STATE_IOPEN:
6261 		info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN;
6262 		break;
6263 	case SPDK_NVME_ZONE_STATE_EOPEN:
6264 		info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN;
6265 		break;
6266 	case SPDK_NVME_ZONE_STATE_CLOSED:
6267 		info->state = SPDK_BDEV_ZONE_STATE_CLOSED;
6268 		break;
6269 	case SPDK_NVME_ZONE_STATE_RONLY:
6270 		info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY;
6271 		break;
6272 	case SPDK_NVME_ZONE_STATE_FULL:
6273 		info->state = SPDK_BDEV_ZONE_STATE_FULL;
6274 		break;
6275 	case SPDK_NVME_ZONE_STATE_OFFLINE:
6276 		info->state = SPDK_BDEV_ZONE_STATE_OFFLINE;
6277 		break;
6278 	default:
6279 		SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs);
6280 		return -EIO;
6281 	}
6282 
6283 	info->zone_id = desc->zslba;
6284 	info->write_pointer = desc->wp;
6285 	info->capacity = desc->zcap;
6286 
6287 	return 0;
6288 }
6289 
6290 static void
6291 bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl)
6292 {
6293 	struct nvme_bdev_io *bio = ref;
6294 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
6295 	uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id;
6296 	uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones;
6297 	struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf;
6298 	uint64_t max_zones_per_buf, i;
6299 	uint32_t zone_report_bufsize;
6300 	struct spdk_nvme_ns *ns;
6301 	struct spdk_nvme_qpair *qpair;
6302 	int ret;
6303 
6304 	if (spdk_nvme_cpl_is_error(cpl)) {
6305 		goto out_complete_io_nvme_cpl;
6306 	}
6307 
6308 	if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) {
6309 		ret = -ENXIO;
6310 		goto out_complete_io_ret;
6311 	}
6312 
6313 	ns = bio->io_path->nvme_ns->ns;
6314 	qpair = bio->io_path->qpair->qpair;
6315 
6316 	zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns);
6317 	max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) /
6318 			    sizeof(bio->zone_report_buf->descs[0]);
6319 
6320 	if (bio->zone_report_buf->nr_zones > max_zones_per_buf) {
6321 		ret = -EINVAL;
6322 		goto out_complete_io_ret;
6323 	}
6324 
6325 	if (!bio->zone_report_buf->nr_zones) {
6326 		ret = -EINVAL;
6327 		goto out_complete_io_ret;
6328 	}
6329 
6330 	for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) {
6331 		ret = fill_zone_from_report(&info[bio->handled_zones],
6332 					    &bio->zone_report_buf->descs[i]);
6333 		if (ret) {
6334 			goto out_complete_io_ret;
6335 		}
6336 		bio->handled_zones++;
6337 	}
6338 
6339 	if (bio->handled_zones < zones_to_copy) {
6340 		uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
6341 		uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones);
6342 
6343 		memset(bio->zone_report_buf, 0, zone_report_bufsize);
6344 		ret = spdk_nvme_zns_report_zones(ns, qpair,
6345 						 bio->zone_report_buf, zone_report_bufsize,
6346 						 slba, SPDK_NVME_ZRA_LIST_ALL, true,
6347 						 bdev_nvme_get_zone_info_done, bio);
6348 		if (!ret) {
6349 			return;
6350 		} else {
6351 			goto out_complete_io_ret;
6352 		}
6353 	}
6354 
6355 out_complete_io_nvme_cpl:
6356 	free(bio->zone_report_buf);
6357 	bio->zone_report_buf = NULL;
6358 	bdev_nvme_io_complete_nvme_status(bio, cpl);
6359 	return;
6360 
6361 out_complete_io_ret:
6362 	free(bio->zone_report_buf);
6363 	bio->zone_report_buf = NULL;
6364 	bdev_nvme_io_complete(bio, ret);
6365 }
6366 
6367 static void
6368 bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl)
6369 {
6370 	struct nvme_bdev_io *bio = ref;
6371 
6372 	bdev_nvme_io_complete_nvme_status(bio, cpl);
6373 }
6374 
6375 static void
6376 bdev_nvme_admin_passthru_complete_nvme_status(void *ctx)
6377 {
6378 	struct nvme_bdev_io *bio = ctx;
6379 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
6380 	const struct spdk_nvme_cpl *cpl = &bio->cpl;
6381 
6382 	assert(bdev_nvme_io_type_is_admin(bdev_io->type));
6383 
6384 	__bdev_nvme_io_complete(bdev_io, 0, cpl);
6385 }
6386 
6387 static void
6388 bdev_nvme_abort_complete(void *ctx)
6389 {
6390 	struct nvme_bdev_io *bio = ctx;
6391 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
6392 
6393 	if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) {
6394 		__bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL);
6395 	} else {
6396 		__bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL);
6397 	}
6398 }
6399 
6400 static void
6401 bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl)
6402 {
6403 	struct nvme_bdev_io *bio = ref;
6404 
6405 	bio->cpl = *cpl;
6406 	spdk_thread_send_msg(bio->orig_thread, bdev_nvme_abort_complete, bio);
6407 }
6408 
6409 static void
6410 bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl)
6411 {
6412 	struct nvme_bdev_io *bio = ref;
6413 
6414 	bio->cpl = *cpl;
6415 	spdk_thread_send_msg(bio->orig_thread,
6416 			     bdev_nvme_admin_passthru_complete_nvme_status, bio);
6417 }
6418 
6419 static void
6420 bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset)
6421 {
6422 	struct nvme_bdev_io *bio = ref;
6423 	struct iovec *iov;
6424 
6425 	bio->iov_offset = sgl_offset;
6426 	for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) {
6427 		iov = &bio->iovs[bio->iovpos];
6428 		if (bio->iov_offset < iov->iov_len) {
6429 			break;
6430 		}
6431 
6432 		bio->iov_offset -= iov->iov_len;
6433 	}
6434 }
6435 
6436 static int
6437 bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length)
6438 {
6439 	struct nvme_bdev_io *bio = ref;
6440 	struct iovec *iov;
6441 
6442 	assert(bio->iovpos < bio->iovcnt);
6443 
6444 	iov = &bio->iovs[bio->iovpos];
6445 
6446 	*address = iov->iov_base;
6447 	*length = iov->iov_len;
6448 
6449 	if (bio->iov_offset) {
6450 		assert(bio->iov_offset <= iov->iov_len);
6451 		*address += bio->iov_offset;
6452 		*length -= bio->iov_offset;
6453 	}
6454 
6455 	bio->iov_offset += *length;
6456 	if (bio->iov_offset == iov->iov_len) {
6457 		bio->iovpos++;
6458 		bio->iov_offset = 0;
6459 	}
6460 
6461 	return 0;
6462 }
6463 
6464 static void
6465 bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset)
6466 {
6467 	struct nvme_bdev_io *bio = ref;
6468 	struct iovec *iov;
6469 
6470 	bio->fused_iov_offset = sgl_offset;
6471 	for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) {
6472 		iov = &bio->fused_iovs[bio->fused_iovpos];
6473 		if (bio->fused_iov_offset < iov->iov_len) {
6474 			break;
6475 		}
6476 
6477 		bio->fused_iov_offset -= iov->iov_len;
6478 	}
6479 }
6480 
6481 static int
6482 bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length)
6483 {
6484 	struct nvme_bdev_io *bio = ref;
6485 	struct iovec *iov;
6486 
6487 	assert(bio->fused_iovpos < bio->fused_iovcnt);
6488 
6489 	iov = &bio->fused_iovs[bio->fused_iovpos];
6490 
6491 	*address = iov->iov_base;
6492 	*length = iov->iov_len;
6493 
6494 	if (bio->fused_iov_offset) {
6495 		assert(bio->fused_iov_offset <= iov->iov_len);
6496 		*address += bio->fused_iov_offset;
6497 		*length -= bio->fused_iov_offset;
6498 	}
6499 
6500 	bio->fused_iov_offset += *length;
6501 	if (bio->fused_iov_offset == iov->iov_len) {
6502 		bio->fused_iovpos++;
6503 		bio->fused_iov_offset = 0;
6504 	}
6505 
6506 	return 0;
6507 }
6508 
6509 static int
6510 bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
6511 		      void *md, uint64_t lba_count, uint64_t lba)
6512 {
6513 	int rc;
6514 
6515 	SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n",
6516 		      lba_count, lba);
6517 
6518 	bio->iovs = iov;
6519 	bio->iovcnt = iovcnt;
6520 	bio->iovpos = 0;
6521 	bio->iov_offset = 0;
6522 
6523 	rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns,
6524 					    bio->io_path->qpair->qpair,
6525 					    lba, lba_count,
6526 					    bdev_nvme_no_pi_readv_done, bio, 0,
6527 					    bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
6528 					    md, 0, 0);
6529 
6530 	if (rc != 0 && rc != -ENOMEM) {
6531 		SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc);
6532 	}
6533 	return rc;
6534 }
6535 
6536 static int
6537 bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
6538 		void *md, uint64_t lba_count, uint64_t lba, uint32_t flags,
6539 		struct spdk_memory_domain *domain, void *domain_ctx)
6540 {
6541 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
6542 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
6543 	int rc;
6544 
6545 	SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n",
6546 		      lba_count, lba);
6547 
6548 	bio->iovs = iov;
6549 	bio->iovcnt = iovcnt;
6550 	bio->iovpos = 0;
6551 	bio->iov_offset = 0;
6552 
6553 	bio->ext_opts.size = sizeof(struct spdk_nvme_ns_cmd_ext_io_opts);
6554 	bio->ext_opts.memory_domain = domain;
6555 	bio->ext_opts.memory_domain_ctx = domain_ctx;
6556 	bio->ext_opts.io_flags = flags;
6557 	bio->ext_opts.metadata = md;
6558 
6559 	rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count,
6560 					bdev_nvme_readv_done, bio,
6561 					bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
6562 					&bio->ext_opts);
6563 	if (rc != 0 && rc != -ENOMEM) {
6564 		SPDK_ERRLOG("readv failed: rc = %d\n", rc);
6565 	}
6566 	return rc;
6567 }
6568 
6569 static int
6570 bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
6571 		 void *md, uint64_t lba_count, uint64_t lba, uint32_t flags,
6572 		 struct spdk_memory_domain *domain, void *domain_ctx)
6573 {
6574 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
6575 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
6576 	int rc;
6577 
6578 	SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n",
6579 		      lba_count, lba);
6580 
6581 	bio->iovs = iov;
6582 	bio->iovcnt = iovcnt;
6583 	bio->iovpos = 0;
6584 	bio->iov_offset = 0;
6585 
6586 	bio->ext_opts.size = sizeof(struct spdk_nvme_ns_cmd_ext_io_opts);
6587 	bio->ext_opts.memory_domain = domain;
6588 	bio->ext_opts.memory_domain_ctx = domain_ctx;
6589 	bio->ext_opts.io_flags = flags;
6590 	bio->ext_opts.metadata = md;
6591 
6592 	rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count,
6593 					 bdev_nvme_writev_done, bio,
6594 					 bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
6595 					 &bio->ext_opts);
6596 	if (rc != 0 && rc != -ENOMEM) {
6597 		SPDK_ERRLOG("writev failed: rc = %d\n", rc);
6598 	}
6599 	return rc;
6600 }
6601 
6602 static int
6603 bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
6604 		       void *md, uint64_t lba_count, uint64_t zslba,
6605 		       uint32_t flags)
6606 {
6607 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
6608 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
6609 	int rc;
6610 
6611 	SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n",
6612 		      lba_count, zslba);
6613 
6614 	bio->iovs = iov;
6615 	bio->iovcnt = iovcnt;
6616 	bio->iovpos = 0;
6617 	bio->iov_offset = 0;
6618 
6619 	if (iovcnt == 1) {
6620 		rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba,
6621 						       lba_count,
6622 						       bdev_nvme_zone_appendv_done, bio,
6623 						       flags,
6624 						       0, 0);
6625 	} else {
6626 		rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count,
6627 							bdev_nvme_zone_appendv_done, bio, flags,
6628 							bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
6629 							md, 0, 0);
6630 	}
6631 
6632 	if (rc != 0 && rc != -ENOMEM) {
6633 		SPDK_ERRLOG("zone append failed: rc = %d\n", rc);
6634 	}
6635 	return rc;
6636 }
6637 
6638 static int
6639 bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
6640 		   void *md, uint64_t lba_count, uint64_t lba,
6641 		   uint32_t flags)
6642 {
6643 	int rc;
6644 
6645 	SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n",
6646 		      lba_count, lba);
6647 
6648 	bio->iovs = iov;
6649 	bio->iovcnt = iovcnt;
6650 	bio->iovpos = 0;
6651 	bio->iov_offset = 0;
6652 
6653 	rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns,
6654 					       bio->io_path->qpair->qpair,
6655 					       lba, lba_count,
6656 					       bdev_nvme_comparev_done, bio, flags,
6657 					       bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
6658 					       md, 0, 0);
6659 
6660 	if (rc != 0 && rc != -ENOMEM) {
6661 		SPDK_ERRLOG("comparev failed: rc = %d\n", rc);
6662 	}
6663 	return rc;
6664 }
6665 
6666 static int
6667 bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt,
6668 			      struct iovec *write_iov, int write_iovcnt,
6669 			      void *md, uint64_t lba_count, uint64_t lba, uint32_t flags)
6670 {
6671 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
6672 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
6673 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
6674 	int rc;
6675 
6676 	SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n",
6677 		      lba_count, lba);
6678 
6679 	bio->iovs = cmp_iov;
6680 	bio->iovcnt = cmp_iovcnt;
6681 	bio->iovpos = 0;
6682 	bio->iov_offset = 0;
6683 	bio->fused_iovs = write_iov;
6684 	bio->fused_iovcnt = write_iovcnt;
6685 	bio->fused_iovpos = 0;
6686 	bio->fused_iov_offset = 0;
6687 
6688 	if (bdev_io->num_retries == 0) {
6689 		bio->first_fused_submitted = false;
6690 		bio->first_fused_completed = false;
6691 	}
6692 
6693 	if (!bio->first_fused_submitted) {
6694 		flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST;
6695 		memset(&bio->cpl, 0, sizeof(bio->cpl));
6696 
6697 		rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count,
6698 						       bdev_nvme_comparev_and_writev_done, bio, flags,
6699 						       bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0);
6700 		if (rc == 0) {
6701 			bio->first_fused_submitted = true;
6702 			flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST;
6703 		} else {
6704 			if (rc != -ENOMEM) {
6705 				SPDK_ERRLOG("compare failed: rc = %d\n", rc);
6706 			}
6707 			return rc;
6708 		}
6709 	}
6710 
6711 	flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND;
6712 
6713 	rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count,
6714 					     bdev_nvme_comparev_and_writev_done, bio, flags,
6715 					     bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0);
6716 	if (rc != 0 && rc != -ENOMEM) {
6717 		SPDK_ERRLOG("write failed: rc = %d\n", rc);
6718 		rc = 0;
6719 	}
6720 
6721 	return rc;
6722 }
6723 
6724 static int
6725 bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks)
6726 {
6727 	struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES];
6728 	struct spdk_nvme_dsm_range *range;
6729 	uint64_t offset, remaining;
6730 	uint64_t num_ranges_u64;
6731 	uint16_t num_ranges;
6732 	int rc;
6733 
6734 	num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) /
6735 			 SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
6736 	if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) {
6737 		SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks);
6738 		return -EINVAL;
6739 	}
6740 	num_ranges = (uint16_t)num_ranges_u64;
6741 
6742 	offset = offset_blocks;
6743 	remaining = num_blocks;
6744 	range = &dsm_ranges[0];
6745 
6746 	/* Fill max-size ranges until the remaining blocks fit into one range */
6747 	while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) {
6748 		range->attributes.raw = 0;
6749 		range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
6750 		range->starting_lba = offset;
6751 
6752 		offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
6753 		remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
6754 		range++;
6755 	}
6756 
6757 	/* Final range describes the remaining blocks */
6758 	range->attributes.raw = 0;
6759 	range->length = remaining;
6760 	range->starting_lba = offset;
6761 
6762 	rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns,
6763 			bio->io_path->qpair->qpair,
6764 			SPDK_NVME_DSM_ATTR_DEALLOCATE,
6765 			dsm_ranges, num_ranges,
6766 			bdev_nvme_queued_done, bio);
6767 
6768 	return rc;
6769 }
6770 
6771 static int
6772 bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks)
6773 {
6774 	if (num_blocks > UINT16_MAX + 1) {
6775 		SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n");
6776 		return -EINVAL;
6777 	}
6778 
6779 	return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns,
6780 					     bio->io_path->qpair->qpair,
6781 					     offset_blocks, num_blocks,
6782 					     bdev_nvme_queued_done, bio,
6783 					     0);
6784 }
6785 
6786 static int
6787 bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones,
6788 			struct spdk_bdev_zone_info *info)
6789 {
6790 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
6791 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
6792 	uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns);
6793 	uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
6794 	uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns);
6795 
6796 	if (zone_id % zone_size != 0) {
6797 		return -EINVAL;
6798 	}
6799 
6800 	if (num_zones > total_zones || !num_zones) {
6801 		return -EINVAL;
6802 	}
6803 
6804 	assert(!bio->zone_report_buf);
6805 	bio->zone_report_buf = calloc(1, zone_report_bufsize);
6806 	if (!bio->zone_report_buf) {
6807 		return -ENOMEM;
6808 	}
6809 
6810 	bio->handled_zones = 0;
6811 
6812 	return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize,
6813 					  zone_id, SPDK_NVME_ZRA_LIST_ALL, true,
6814 					  bdev_nvme_get_zone_info_done, bio);
6815 }
6816 
6817 static int
6818 bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id,
6819 			  enum spdk_bdev_zone_action action)
6820 {
6821 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
6822 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
6823 
6824 	switch (action) {
6825 	case SPDK_BDEV_ZONE_CLOSE:
6826 		return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false,
6827 						bdev_nvme_zone_management_done, bio);
6828 	case SPDK_BDEV_ZONE_FINISH:
6829 		return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false,
6830 						 bdev_nvme_zone_management_done, bio);
6831 	case SPDK_BDEV_ZONE_OPEN:
6832 		return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false,
6833 					       bdev_nvme_zone_management_done, bio);
6834 	case SPDK_BDEV_ZONE_RESET:
6835 		return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false,
6836 						bdev_nvme_zone_management_done, bio);
6837 	case SPDK_BDEV_ZONE_OFFLINE:
6838 		return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false,
6839 						  bdev_nvme_zone_management_done, bio);
6840 	default:
6841 		return -EINVAL;
6842 	}
6843 }
6844 
6845 static void
6846 bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio,
6847 			 struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes)
6848 {
6849 	struct nvme_io_path *io_path;
6850 	struct nvme_ctrlr *nvme_ctrlr;
6851 	uint32_t max_xfer_size;
6852 	int rc = -ENXIO;
6853 
6854 	/* Choose the first ctrlr which is not failed. */
6855 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
6856 		nvme_ctrlr = io_path->qpair->ctrlr;
6857 
6858 		/* We should skip any unavailable nvme_ctrlr rather than checking
6859 		 * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO.
6860 		 */
6861 		if (!nvme_ctrlr_is_available(nvme_ctrlr)) {
6862 			continue;
6863 		}
6864 
6865 		max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr);
6866 
6867 		if (nbytes > max_xfer_size) {
6868 			SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
6869 			rc = -EINVAL;
6870 			goto err;
6871 		}
6872 
6873 		bio->io_path = io_path;
6874 		bio->orig_thread = spdk_get_thread();
6875 
6876 		rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes,
6877 						   bdev_nvme_admin_passthru_done, bio);
6878 		if (rc == 0) {
6879 			return;
6880 		}
6881 	}
6882 
6883 err:
6884 	bdev_nvme_admin_passthru_complete(bio, rc);
6885 }
6886 
6887 static int
6888 bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
6889 		      void *buf, size_t nbytes)
6890 {
6891 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
6892 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
6893 	uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
6894 	struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
6895 
6896 	if (nbytes > max_xfer_size) {
6897 		SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
6898 		return -EINVAL;
6899 	}
6900 
6901 	/*
6902 	 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid,
6903 	 * so fill it out automatically.
6904 	 */
6905 	cmd->nsid = spdk_nvme_ns_get_id(ns);
6906 
6907 	return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf,
6908 					  (uint32_t)nbytes, bdev_nvme_queued_done, bio);
6909 }
6910 
6911 static int
6912 bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
6913 			 void *buf, size_t nbytes, void *md_buf, size_t md_len)
6914 {
6915 	struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
6916 	struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
6917 	size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns);
6918 	uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
6919 	struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
6920 
6921 	if (nbytes > max_xfer_size) {
6922 		SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
6923 		return -EINVAL;
6924 	}
6925 
6926 	if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) {
6927 		SPDK_ERRLOG("invalid meta data buffer size\n");
6928 		return -EINVAL;
6929 	}
6930 
6931 	/*
6932 	 * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid,
6933 	 * so fill it out automatically.
6934 	 */
6935 	cmd->nsid = spdk_nvme_ns_get_id(ns);
6936 
6937 	return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf,
6938 			(uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio);
6939 }
6940 
6941 static void
6942 bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio,
6943 		struct nvme_bdev_io *bio_to_abort)
6944 {
6945 	struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
6946 	struct nvme_io_path *io_path;
6947 	struct nvme_ctrlr *nvme_ctrlr;
6948 	int rc = 0;
6949 
6950 	bio->orig_thread = spdk_get_thread();
6951 
6952 	rc = bdev_nvme_abort_retry_io(nbdev_ch, bio_to_abort);
6953 	if (rc == 0) {
6954 		__bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL);
6955 		return;
6956 	}
6957 
6958 	rc = 0;
6959 
6960 	/* Even admin commands, they were submitted to only nvme_ctrlrs which were
6961 	 * on any io_path. So traverse the io_path list for not only I/O commands
6962 	 * but also admin commands.
6963 	 */
6964 	STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
6965 		nvme_ctrlr = io_path->qpair->ctrlr;
6966 
6967 		rc = spdk_nvme_ctrlr_cmd_abort_ext(nvme_ctrlr->ctrlr,
6968 						   io_path->qpair->qpair,
6969 						   bio_to_abort,
6970 						   bdev_nvme_abort_done, bio);
6971 		if (rc == -ENOENT) {
6972 			/* If no command was found in I/O qpair, the target command may be
6973 			 * admin command.
6974 			 */
6975 			rc = spdk_nvme_ctrlr_cmd_abort_ext(nvme_ctrlr->ctrlr,
6976 							   NULL,
6977 							   bio_to_abort,
6978 							   bdev_nvme_abort_done, bio);
6979 		}
6980 
6981 		if (rc != -ENOENT) {
6982 			break;
6983 		}
6984 	}
6985 
6986 	if (rc != 0) {
6987 		/* If no command was found or there was any error, complete the abort
6988 		 * request with failure.
6989 		 */
6990 		__bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL);
6991 	}
6992 }
6993 
6994 static int
6995 bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, uint64_t src_offset_blocks,
6996 	       uint64_t num_blocks)
6997 {
6998 	struct spdk_nvme_scc_source_range range = {
6999 		.slba = src_offset_blocks,
7000 		.nlb = num_blocks - 1
7001 	};
7002 
7003 	return spdk_nvme_ns_cmd_copy(bio->io_path->nvme_ns->ns,
7004 				     bio->io_path->qpair->qpair,
7005 				     &range, 1, dst_offset_blocks,
7006 				     bdev_nvme_queued_done, bio);
7007 }
7008 
7009 static void
7010 bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w)
7011 {
7012 	const char	*action;
7013 
7014 	if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) {
7015 		action = "reset";
7016 	} else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) {
7017 		action = "abort";
7018 	} else {
7019 		action = "none";
7020 	}
7021 
7022 	spdk_json_write_object_begin(w);
7023 
7024 	spdk_json_write_named_string(w, "method", "bdev_nvme_set_options");
7025 
7026 	spdk_json_write_named_object_begin(w, "params");
7027 	spdk_json_write_named_string(w, "action_on_timeout", action);
7028 	spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us);
7029 	spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us);
7030 	spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms);
7031 	spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count);
7032 	spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst);
7033 	spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight);
7034 	spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight);
7035 	spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight);
7036 	spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us);
7037 	spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us);
7038 	spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests);
7039 	spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit);
7040 	spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count);
7041 	spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout);
7042 	spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec);
7043 	spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec);
7044 	spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec);
7045 	spdk_json_write_named_bool(w, "generate_uuids", g_opts.generate_uuids);
7046 	spdk_json_write_named_uint8(w, "transport_tos", g_opts.transport_tos);
7047 	spdk_json_write_named_bool(w, "io_path_stat", g_opts.io_path_stat);
7048 	spdk_json_write_object_end(w);
7049 
7050 	spdk_json_write_object_end(w);
7051 }
7052 
7053 static void
7054 bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx)
7055 {
7056 	struct spdk_nvme_transport_id trid;
7057 
7058 	spdk_json_write_object_begin(w);
7059 
7060 	spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery");
7061 
7062 	spdk_json_write_named_object_begin(w, "params");
7063 	spdk_json_write_named_string(w, "name", ctx->name);
7064 	spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn);
7065 
7066 	trid = ctx->trid;
7067 	memset(trid.subnqn, 0, sizeof(trid.subnqn));
7068 	nvme_bdev_dump_trid_json(&trid, w);
7069 
7070 	spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach);
7071 	spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec);
7072 	spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec);
7073 	spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",
7074 				     ctx->bdev_opts.fast_io_fail_timeout_sec);
7075 	spdk_json_write_object_end(w);
7076 
7077 	spdk_json_write_object_end(w);
7078 }
7079 
7080 static void
7081 nvme_ctrlr_config_json(struct spdk_json_write_ctx *w,
7082 		       struct nvme_ctrlr *nvme_ctrlr)
7083 {
7084 	struct spdk_nvme_transport_id	*trid;
7085 
7086 	if (nvme_ctrlr->opts.from_discovery_service) {
7087 		/* Do not emit an RPC for this - it will be implicitly
7088 		 * covered by a separate bdev_nvme_start_discovery or
7089 		 * bdev_nvme_start_mdns_discovery RPC.
7090 		 */
7091 		return;
7092 	}
7093 
7094 	trid = &nvme_ctrlr->active_path_id->trid;
7095 
7096 	spdk_json_write_object_begin(w);
7097 
7098 	spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller");
7099 
7100 	spdk_json_write_named_object_begin(w, "params");
7101 	spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name);
7102 	nvme_bdev_dump_trid_json(trid, w);
7103 	spdk_json_write_named_bool(w, "prchk_reftag",
7104 				   (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0);
7105 	spdk_json_write_named_bool(w, "prchk_guard",
7106 				   (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0);
7107 	spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec);
7108 	spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec);
7109 	spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",
7110 				     nvme_ctrlr->opts.fast_io_fail_timeout_sec);
7111 
7112 	spdk_json_write_object_end(w);
7113 
7114 	spdk_json_write_object_end(w);
7115 }
7116 
7117 static void
7118 bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w)
7119 {
7120 	spdk_json_write_object_begin(w);
7121 	spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug");
7122 
7123 	spdk_json_write_named_object_begin(w, "params");
7124 	spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us);
7125 	spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled);
7126 	spdk_json_write_object_end(w);
7127 
7128 	spdk_json_write_object_end(w);
7129 }
7130 
7131 static int
7132 bdev_nvme_config_json(struct spdk_json_write_ctx *w)
7133 {
7134 	struct nvme_bdev_ctrlr	*nbdev_ctrlr;
7135 	struct nvme_ctrlr	*nvme_ctrlr;
7136 	struct discovery_ctx	*ctx;
7137 
7138 	bdev_nvme_opts_config_json(w);
7139 
7140 	pthread_mutex_lock(&g_bdev_nvme_mutex);
7141 
7142 	TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
7143 		TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
7144 			nvme_ctrlr_config_json(w, nvme_ctrlr);
7145 		}
7146 	}
7147 
7148 	TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
7149 		if (!ctx->from_mdns_discovery_service) {
7150 			bdev_nvme_discovery_config_json(w, ctx);
7151 		}
7152 	}
7153 
7154 	bdev_nvme_mdns_discovery_config_json(w);
7155 
7156 	/* Dump as last parameter to give all NVMe bdevs chance to be constructed
7157 	 * before enabling hotplug poller.
7158 	 */
7159 	bdev_nvme_hotplug_config_json(w);
7160 
7161 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
7162 	return 0;
7163 }
7164 
7165 struct spdk_nvme_ctrlr *
7166 bdev_nvme_get_ctrlr(struct spdk_bdev *bdev)
7167 {
7168 	struct nvme_bdev *nbdev;
7169 	struct nvme_ns *nvme_ns;
7170 
7171 	if (!bdev || bdev->module != &nvme_if) {
7172 		return NULL;
7173 	}
7174 
7175 	nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
7176 	nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
7177 	assert(nvme_ns != NULL);
7178 
7179 	return nvme_ns->ctrlr->ctrlr;
7180 }
7181 
7182 void
7183 nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path)
7184 {
7185 	struct nvme_ns *nvme_ns = io_path->nvme_ns;
7186 	struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
7187 	const struct spdk_nvme_ctrlr_data *cdata;
7188 	const struct spdk_nvme_transport_id *trid;
7189 	const char *adrfam_str;
7190 
7191 	spdk_json_write_object_begin(w);
7192 
7193 	spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name);
7194 
7195 	cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
7196 	trid = spdk_nvme_ctrlr_get_transport_id(nvme_ctrlr->ctrlr);
7197 
7198 	spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid);
7199 	spdk_json_write_named_bool(w, "current", io_path == io_path->nbdev_ch->current_io_path);
7200 	spdk_json_write_named_bool(w, "connected", nvme_io_path_is_connected(io_path));
7201 	spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns));
7202 
7203 	spdk_json_write_named_object_begin(w, "transport");
7204 	spdk_json_write_named_string(w, "trtype", trid->trstring);
7205 	spdk_json_write_named_string(w, "traddr", trid->traddr);
7206 	if (trid->trsvcid[0] != '\0') {
7207 		spdk_json_write_named_string(w, "trsvcid", trid->trsvcid);
7208 	}
7209 	adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam);
7210 	if (adrfam_str) {
7211 		spdk_json_write_named_string(w, "adrfam", adrfam_str);
7212 	}
7213 	spdk_json_write_object_end(w);
7214 
7215 	spdk_json_write_object_end(w);
7216 }
7217 
7218 void
7219 bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w)
7220 {
7221 	struct discovery_ctx *ctx;
7222 	struct discovery_entry_ctx *entry_ctx;
7223 
7224 	spdk_json_write_array_begin(w);
7225 	TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
7226 		spdk_json_write_object_begin(w);
7227 		spdk_json_write_named_string(w, "name", ctx->name);
7228 
7229 		spdk_json_write_named_object_begin(w, "trid");
7230 		nvme_bdev_dump_trid_json(&ctx->trid, w);
7231 		spdk_json_write_object_end(w);
7232 
7233 		spdk_json_write_named_array_begin(w, "referrals");
7234 		TAILQ_FOREACH(entry_ctx, &ctx->discovery_entry_ctxs, tailq) {
7235 			spdk_json_write_object_begin(w);
7236 			spdk_json_write_named_object_begin(w, "trid");
7237 			nvme_bdev_dump_trid_json(&entry_ctx->trid, w);
7238 			spdk_json_write_object_end(w);
7239 			spdk_json_write_object_end(w);
7240 		}
7241 		spdk_json_write_array_end(w);
7242 
7243 		spdk_json_write_object_end(w);
7244 	}
7245 	spdk_json_write_array_end(w);
7246 }
7247 
7248 SPDK_LOG_REGISTER_COMPONENT(bdev_nvme)
7249 
7250 SPDK_TRACE_REGISTER_FN(bdev_nvme_trace, "bdev_nvme", TRACE_GROUP_BDEV_NVME)
7251 {
7252 	struct spdk_trace_tpoint_opts opts[] = {
7253 		{
7254 			"BDEV_NVME_IO_START", TRACE_BDEV_NVME_IO_START,
7255 			OWNER_NONE, OBJECT_BDEV_NVME_IO, 1,
7256 			{{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }}
7257 		},
7258 		{
7259 			"BDEV_NVME_IO_DONE", TRACE_BDEV_NVME_IO_DONE,
7260 			OWNER_NONE, OBJECT_BDEV_NVME_IO, 0,
7261 			{{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }}
7262 		}
7263 	};
7264 
7265 
7266 	spdk_trace_register_object(OBJECT_BDEV_NVME_IO, 'N');
7267 	spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts));
7268 	spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_SUBMIT, OBJECT_BDEV_NVME_IO, 0);
7269 	spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_SUBMIT, OBJECT_BDEV_NVME_IO, 0);
7270 	spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_COMPLETE, OBJECT_BDEV_NVME_IO, 0);
7271 	spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_COMPLETE, OBJECT_BDEV_NVME_IO, 0);
7272 }
7273