1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/bdev.h" 10 #include "spdk/bit_array.h" 11 #include "spdk/thread.h" 12 #include "spdk/nvmf.h" 13 #include "spdk/endian.h" 14 #include "spdk/string.h" 15 #include "spdk/log.h" 16 #include "spdk_internal/usdt.h" 17 18 #include "nvmf_internal.h" 19 #include "transport.h" 20 21 SPDK_LOG_REGISTER_COMPONENT(nvmf) 22 23 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 24 25 static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts); 26 27 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 28 29 /* supplied to a single call to nvmf_qpair_disconnect */ 30 struct nvmf_qpair_disconnect_ctx { 31 struct spdk_nvmf_qpair *qpair; 32 struct spdk_nvmf_ctrlr *ctrlr; 33 nvmf_qpair_disconnect_cb cb_fn; 34 struct spdk_thread *thread; 35 void *ctx; 36 uint16_t qid; 37 }; 38 39 /* 40 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 41 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 42 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 43 */ 44 struct nvmf_qpair_disconnect_many_ctx { 45 struct spdk_nvmf_subsystem *subsystem; 46 struct spdk_nvmf_poll_group *group; 47 spdk_nvmf_poll_group_mod_done cpl_fn; 48 void *cpl_ctx; 49 uint32_t count; 50 }; 51 52 static void 53 nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 54 enum spdk_nvmf_qpair_state state) 55 { 56 assert(qpair != NULL); 57 assert(qpair->group->thread == spdk_get_thread()); 58 59 qpair->state = state; 60 } 61 62 static int 63 nvmf_poll_group_poll(void *ctx) 64 { 65 struct spdk_nvmf_poll_group *group = ctx; 66 int rc; 67 int count = 0; 68 struct spdk_nvmf_transport_poll_group *tgroup; 69 70 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 71 rc = nvmf_transport_poll_group_poll(tgroup); 72 if (rc < 0) { 73 return SPDK_POLLER_BUSY; 74 } 75 count += rc; 76 } 77 78 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 79 } 80 81 /* 82 * Reset and clean up the poll group (I/O channel code will actually free the 83 * group). 84 */ 85 static void 86 nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group) 87 { 88 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 89 struct spdk_nvmf_subsystem_poll_group *sgroup; 90 uint32_t sid, nsid; 91 92 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 93 TAILQ_REMOVE(&group->tgroups, tgroup, link); 94 nvmf_transport_poll_group_destroy(tgroup); 95 } 96 97 for (sid = 0; sid < group->num_sgroups; sid++) { 98 sgroup = &group->sgroups[sid]; 99 100 assert(sgroup != NULL); 101 102 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 103 if (sgroup->ns_info[nsid].channel) { 104 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 105 sgroup->ns_info[nsid].channel = NULL; 106 } 107 } 108 109 free(sgroup->ns_info); 110 } 111 112 free(group->sgroups); 113 114 spdk_poller_unregister(&group->poller); 115 116 if (group->destroy_cb_fn) { 117 group->destroy_cb_fn(group->destroy_cb_arg, 0); 118 } 119 } 120 121 /* 122 * Callback to unregister a poll group from the target, and clean up its state. 123 */ 124 static void 125 nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 126 { 127 struct spdk_nvmf_tgt *tgt = io_device; 128 struct spdk_nvmf_poll_group *group = ctx_buf; 129 130 SPDK_DTRACE_PROBE1(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread)); 131 132 pthread_mutex_lock(&tgt->mutex); 133 TAILQ_REMOVE(&tgt->poll_groups, group, link); 134 tgt->num_poll_groups--; 135 pthread_mutex_unlock(&tgt->mutex); 136 137 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 138 nvmf_tgt_cleanup_poll_group(group); 139 } 140 141 static int 142 nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 143 struct spdk_nvmf_transport *transport) 144 { 145 struct spdk_nvmf_transport_poll_group *tgroup; 146 147 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 148 if (tgroup->transport == transport) { 149 /* Transport already in the poll group */ 150 return 0; 151 } 152 } 153 154 tgroup = nvmf_transport_poll_group_create(transport, group); 155 if (!tgroup) { 156 SPDK_ERRLOG("Unable to create poll group for transport\n"); 157 return -1; 158 } 159 SPDK_DTRACE_PROBE2(nvmf_transport_poll_group_create, transport, spdk_thread_get_id(group->thread)); 160 161 tgroup->group = group; 162 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 163 164 return 0; 165 } 166 167 static int 168 nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 169 { 170 struct spdk_nvmf_tgt *tgt = io_device; 171 struct spdk_nvmf_poll_group *group = ctx_buf; 172 struct spdk_nvmf_transport *transport; 173 struct spdk_thread *thread = spdk_get_thread(); 174 uint32_t sid; 175 int rc; 176 177 group->tgt = tgt; 178 TAILQ_INIT(&group->tgroups); 179 TAILQ_INIT(&group->qpairs); 180 group->thread = thread; 181 pthread_mutex_init(&group->mutex, NULL); 182 183 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 184 185 SPDK_DTRACE_PROBE1(nvmf_create_poll_group, spdk_thread_get_id(thread)); 186 187 TAILQ_FOREACH(transport, &tgt->transports, link) { 188 rc = nvmf_poll_group_add_transport(group, transport); 189 if (rc != 0) { 190 nvmf_tgt_cleanup_poll_group(group); 191 return rc; 192 } 193 } 194 195 group->num_sgroups = tgt->max_subsystems; 196 group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 197 if (!group->sgroups) { 198 nvmf_tgt_cleanup_poll_group(group); 199 return -ENOMEM; 200 } 201 202 for (sid = 0; sid < tgt->max_subsystems; sid++) { 203 struct spdk_nvmf_subsystem *subsystem; 204 205 subsystem = tgt->subsystems[sid]; 206 if (!subsystem) { 207 continue; 208 } 209 210 if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { 211 nvmf_tgt_cleanup_poll_group(group); 212 return -1; 213 } 214 } 215 216 pthread_mutex_lock(&tgt->mutex); 217 tgt->num_poll_groups++; 218 TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link); 219 pthread_mutex_unlock(&tgt->mutex); 220 221 return 0; 222 } 223 224 static void 225 _nvmf_tgt_disconnect_next_qpair(void *ctx) 226 { 227 struct spdk_nvmf_qpair *qpair; 228 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 229 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 230 struct spdk_io_channel *ch; 231 int rc = 0; 232 233 qpair = TAILQ_FIRST(&group->qpairs); 234 235 if (qpair) { 236 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_tgt_disconnect_next_qpair, ctx); 237 } 238 239 if (!qpair || rc != 0) { 240 /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */ 241 ch = spdk_io_channel_from_ctx(group); 242 spdk_put_io_channel(ch); 243 free(qpair_ctx); 244 } 245 } 246 247 static void 248 nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 249 { 250 struct nvmf_qpair_disconnect_many_ctx *ctx; 251 252 SPDK_DTRACE_PROBE1(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread)); 253 254 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 255 if (!ctx) { 256 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 257 return; 258 } 259 260 ctx->group = group; 261 _nvmf_tgt_disconnect_next_qpair(ctx); 262 } 263 264 struct spdk_nvmf_tgt * 265 spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *opts) 266 { 267 struct spdk_nvmf_tgt *tgt, *tmp_tgt; 268 269 if (strnlen(opts->name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) { 270 SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH); 271 return NULL; 272 } 273 274 TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) { 275 if (!strncmp(opts->name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) { 276 SPDK_ERRLOG("Provided target name must be unique.\n"); 277 return NULL; 278 } 279 } 280 281 tgt = calloc(1, sizeof(*tgt)); 282 if (!tgt) { 283 return NULL; 284 } 285 286 snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts->name); 287 288 if (!opts || !opts->max_subsystems) { 289 tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 290 } else { 291 tgt->max_subsystems = opts->max_subsystems; 292 } 293 294 if (!opts) { 295 tgt->crdt[0] = 0; 296 tgt->crdt[1] = 0; 297 tgt->crdt[2] = 0; 298 } else { 299 tgt->crdt[0] = opts->crdt[0]; 300 tgt->crdt[1] = opts->crdt[1]; 301 tgt->crdt[2] = opts->crdt[2]; 302 } 303 304 if (!opts) { 305 tgt->discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY; 306 } else { 307 tgt->discovery_filter = opts->discovery_filter; 308 } 309 310 tgt->discovery_genctr = 0; 311 TAILQ_INIT(&tgt->transports); 312 TAILQ_INIT(&tgt->poll_groups); 313 tgt->num_poll_groups = 0; 314 315 tgt->subsystems = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem *)); 316 if (!tgt->subsystems) { 317 free(tgt); 318 return NULL; 319 } 320 321 pthread_mutex_init(&tgt->mutex, NULL); 322 323 spdk_io_device_register(tgt, 324 nvmf_tgt_create_poll_group, 325 nvmf_tgt_destroy_poll_group, 326 sizeof(struct spdk_nvmf_poll_group), 327 tgt->name); 328 329 tgt->state = NVMF_TGT_RUNNING; 330 331 TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link); 332 333 return tgt; 334 } 335 336 static void 337 _nvmf_tgt_destroy_next_transport(void *ctx) 338 { 339 struct spdk_nvmf_tgt *tgt = ctx; 340 struct spdk_nvmf_transport *transport; 341 342 if (!TAILQ_EMPTY(&tgt->transports)) { 343 transport = TAILQ_FIRST(&tgt->transports); 344 TAILQ_REMOVE(&tgt->transports, transport, link); 345 spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt); 346 } else { 347 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn; 348 void *destroy_cb_arg = tgt->destroy_cb_arg; 349 350 pthread_mutex_destroy(&tgt->mutex); 351 free(tgt); 352 353 if (destroy_cb_fn) { 354 destroy_cb_fn(destroy_cb_arg, 0); 355 } 356 } 357 } 358 359 static void 360 nvmf_tgt_destroy_cb(void *io_device) 361 { 362 struct spdk_nvmf_tgt *tgt = io_device; 363 uint32_t i; 364 int rc; 365 366 if (tgt->subsystems) { 367 for (i = 0; i < tgt->max_subsystems; i++) { 368 if (tgt->subsystems[i]) { 369 nvmf_subsystem_remove_all_listeners(tgt->subsystems[i], true); 370 371 rc = spdk_nvmf_subsystem_destroy(tgt->subsystems[i], nvmf_tgt_destroy_cb, tgt); 372 if (rc) { 373 if (rc == -EINPROGRESS) { 374 /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i 375 * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */ 376 return; 377 } else { 378 SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", tgt->subsystems[i]->subnqn, rc); 379 } 380 } 381 } 382 } 383 free(tgt->subsystems); 384 } 385 386 _nvmf_tgt_destroy_next_transport(tgt); 387 } 388 389 void 390 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 391 spdk_nvmf_tgt_destroy_done_fn cb_fn, 392 void *cb_arg) 393 { 394 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 395 396 tgt->destroy_cb_fn = cb_fn; 397 tgt->destroy_cb_arg = cb_arg; 398 399 TAILQ_REMOVE(&g_nvmf_tgts, tgt, link); 400 401 spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb); 402 } 403 404 const char * 405 spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt) 406 { 407 return tgt->name; 408 } 409 410 struct spdk_nvmf_tgt * 411 spdk_nvmf_get_tgt(const char *name) 412 { 413 struct spdk_nvmf_tgt *tgt; 414 uint32_t num_targets = 0; 415 416 TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) { 417 if (name) { 418 if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) { 419 return tgt; 420 } 421 } 422 num_targets++; 423 } 424 425 /* 426 * special case. If there is only one target and 427 * no name was specified, return the only available 428 * target. If there is more than one target, name must 429 * be specified. 430 */ 431 if (!name && num_targets == 1) { 432 return TAILQ_FIRST(&g_nvmf_tgts); 433 } 434 435 return NULL; 436 } 437 438 struct spdk_nvmf_tgt * 439 spdk_nvmf_get_first_tgt(void) 440 { 441 return TAILQ_FIRST(&g_nvmf_tgts); 442 } 443 444 struct spdk_nvmf_tgt * 445 spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev) 446 { 447 return TAILQ_NEXT(prev, link); 448 } 449 450 static void 451 nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 452 struct spdk_nvmf_subsystem *subsystem) 453 { 454 struct spdk_nvmf_host *host; 455 struct spdk_nvmf_subsystem_listener *listener; 456 const struct spdk_nvme_transport_id *trid; 457 struct spdk_nvmf_ns *ns; 458 struct spdk_nvmf_ns_opts ns_opts; 459 uint32_t max_namespaces; 460 char uuid_str[SPDK_UUID_STRING_LEN]; 461 462 if (spdk_nvmf_subsystem_get_type(subsystem) != SPDK_NVMF_SUBTYPE_NVME) { 463 return; 464 } 465 466 /* { */ 467 spdk_json_write_object_begin(w); 468 spdk_json_write_named_string(w, "method", "nvmf_create_subsystem"); 469 470 /* "params" : { */ 471 spdk_json_write_named_object_begin(w, "params"); 472 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 473 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 474 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 475 spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem)); 476 477 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 478 if (max_namespaces != 0) { 479 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 480 } 481 482 spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem)); 483 spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem)); 484 spdk_json_write_named_bool(w, "ana_reporting", nvmf_subsystem_get_ana_reporting(subsystem)); 485 486 /* } "params" */ 487 spdk_json_write_object_end(w); 488 489 /* } */ 490 spdk_json_write_object_end(w); 491 492 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 493 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 494 495 spdk_json_write_object_begin(w); 496 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 497 498 /* "params" : { */ 499 spdk_json_write_named_object_begin(w, "params"); 500 501 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 502 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 503 504 /* } "params" */ 505 spdk_json_write_object_end(w); 506 507 /* } */ 508 spdk_json_write_object_end(w); 509 } 510 511 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 512 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 513 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 514 515 spdk_json_write_object_begin(w); 516 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 517 518 /* "params" : { */ 519 spdk_json_write_named_object_begin(w, "params"); 520 521 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 522 523 /* "namespace" : { */ 524 spdk_json_write_named_object_begin(w, "namespace"); 525 526 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 527 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 528 529 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 530 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 531 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 532 from_be64(&ns_opts.nguid[8])); 533 } 534 535 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 536 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 537 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 538 } 539 540 if (!spdk_mem_all_zero(&ns_opts.uuid, sizeof(ns_opts.uuid))) { 541 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &ns_opts.uuid); 542 spdk_json_write_named_string(w, "uuid", uuid_str); 543 } 544 545 if (nvmf_subsystem_get_ana_reporting(subsystem)) { 546 spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid); 547 } 548 549 /* "namespace" */ 550 spdk_json_write_object_end(w); 551 552 /* } "params" */ 553 spdk_json_write_object_end(w); 554 555 /* } */ 556 spdk_json_write_object_end(w); 557 } 558 559 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 560 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 561 trid = spdk_nvmf_subsystem_listener_get_trid(listener); 562 563 spdk_json_write_object_begin(w); 564 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 565 566 /* "params" : { */ 567 spdk_json_write_named_object_begin(w, "params"); 568 569 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 570 nvmf_transport_listen_dump_opts(listener->transport, trid, w); 571 572 /* } "params" */ 573 spdk_json_write_object_end(w); 574 575 /* } */ 576 spdk_json_write_object_end(w); 577 } 578 579 } 580 581 void 582 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 583 { 584 struct spdk_nvmf_subsystem *subsystem; 585 struct spdk_nvmf_transport *transport; 586 587 spdk_json_write_object_begin(w); 588 spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems"); 589 590 spdk_json_write_named_object_begin(w, "params"); 591 spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems); 592 spdk_json_write_object_end(w); 593 594 spdk_json_write_object_end(w); 595 596 spdk_json_write_object_begin(w); 597 spdk_json_write_named_string(w, "method", "nvmf_set_crdt"); 598 spdk_json_write_named_object_begin(w, "params"); 599 spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]); 600 spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]); 601 spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]); 602 spdk_json_write_object_end(w); 603 spdk_json_write_object_end(w); 604 605 /* write transports */ 606 TAILQ_FOREACH(transport, &tgt->transports, link) { 607 spdk_json_write_object_begin(w); 608 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 609 nvmf_transport_dump_opts(transport, w, true); 610 spdk_json_write_object_end(w); 611 } 612 613 subsystem = spdk_nvmf_subsystem_get_first(tgt); 614 while (subsystem) { 615 nvmf_write_subsystem_config_json(w, subsystem); 616 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 617 } 618 } 619 620 static void 621 nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts, 622 const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size) 623 { 624 assert(opts); 625 assert(opts_src); 626 627 opts->opts_size = opts_size; 628 629 #define SET_FIELD(field) \ 630 if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \ 631 opts->field = opts_src->field; \ 632 } \ 633 634 SET_FIELD(transport_specific); 635 #undef SET_FIELD 636 637 /* Do not remove this statement, you should always update this statement when you adding a new field, 638 * and do not forget to add the SET_FIELD statement for your added field. */ 639 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 16, "Incorrect size"); 640 } 641 642 void 643 spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size) 644 { 645 struct spdk_nvmf_listen_opts opts_local = {}; 646 647 /* local version of opts should have defaults set here */ 648 649 nvmf_listen_opts_copy(opts, &opts_local, opts_size); 650 } 651 652 int 653 spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid, 654 struct spdk_nvmf_listen_opts *opts) 655 { 656 struct spdk_nvmf_transport *transport; 657 int rc; 658 struct spdk_nvmf_listen_opts opts_local = {}; 659 660 if (!opts) { 661 SPDK_ERRLOG("opts should not be NULL\n"); 662 return -EINVAL; 663 } 664 665 if (!opts->opts_size) { 666 SPDK_ERRLOG("The opts_size in opts structure should not be zero\n"); 667 return -EINVAL; 668 } 669 670 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 671 if (!transport) { 672 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 673 trid->trstring); 674 return -EINVAL; 675 } 676 677 nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size); 678 rc = spdk_nvmf_transport_listen(transport, trid, &opts_local); 679 if (rc < 0) { 680 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 681 } 682 683 return rc; 684 } 685 686 int 687 spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt, 688 struct spdk_nvme_transport_id *trid) 689 { 690 struct spdk_nvmf_transport *transport; 691 int rc; 692 693 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 694 if (!transport) { 695 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 696 trid->trstring); 697 return -EINVAL; 698 } 699 700 rc = spdk_nvmf_transport_stop_listen(transport, trid); 701 if (rc < 0) { 702 SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr); 703 return rc; 704 } 705 return 0; 706 } 707 708 struct spdk_nvmf_tgt_add_transport_ctx { 709 struct spdk_nvmf_tgt *tgt; 710 struct spdk_nvmf_transport *transport; 711 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 712 void *cb_arg; 713 int status; 714 }; 715 716 static void 717 _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status) 718 { 719 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 720 721 ctx->cb_fn(ctx->cb_arg, ctx->status); 722 free(ctx); 723 } 724 725 static void 726 _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i) 727 { 728 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 729 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 730 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 731 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 732 733 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 734 if (tgroup->transport == ctx->transport) { 735 TAILQ_REMOVE(&group->tgroups, tgroup, link); 736 nvmf_transport_poll_group_destroy(tgroup); 737 } 738 } 739 740 spdk_for_each_channel_continue(i, 0); 741 } 742 743 static void 744 _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 745 { 746 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 747 748 if (status) { 749 ctx->status = status; 750 spdk_for_each_channel(ctx->tgt, 751 _nvmf_tgt_remove_transport, 752 ctx, 753 _nvmf_tgt_remove_transport_done); 754 return; 755 } 756 757 ctx->transport->tgt = ctx->tgt; 758 TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link); 759 ctx->cb_fn(ctx->cb_arg, status); 760 free(ctx); 761 } 762 763 static void 764 _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 765 { 766 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 767 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 768 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 769 int rc; 770 771 rc = nvmf_poll_group_add_transport(group, ctx->transport); 772 spdk_for_each_channel_continue(i, rc); 773 } 774 775 void 776 spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 777 struct spdk_nvmf_transport *transport, 778 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 779 void *cb_arg) 780 { 781 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 782 783 SPDK_DTRACE_PROBE2(nvmf_tgt_add_transport, transport, tgt->name); 784 785 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) { 786 cb_fn(cb_arg, -EEXIST); 787 return; /* transport already created */ 788 } 789 790 ctx = calloc(1, sizeof(*ctx)); 791 if (!ctx) { 792 cb_fn(cb_arg, -ENOMEM); 793 return; 794 } 795 796 ctx->tgt = tgt; 797 ctx->transport = transport; 798 ctx->cb_fn = cb_fn; 799 ctx->cb_arg = cb_arg; 800 801 spdk_for_each_channel(tgt, 802 _nvmf_tgt_add_transport, 803 ctx, 804 _nvmf_tgt_add_transport_done); 805 } 806 807 struct nvmf_tgt_pause_ctx { 808 struct spdk_nvmf_tgt *tgt; 809 spdk_nvmf_tgt_pause_polling_cb_fn cb_fn; 810 void *cb_arg; 811 }; 812 813 static void 814 _nvmf_tgt_pause_polling_done(struct spdk_io_channel_iter *i, int status) 815 { 816 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 817 818 ctx->tgt->state = NVMF_TGT_PAUSED; 819 820 ctx->cb_fn(ctx->cb_arg, status); 821 free(ctx); 822 } 823 824 static void 825 _nvmf_tgt_pause_polling(struct spdk_io_channel_iter *i) 826 { 827 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 828 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 829 830 spdk_poller_unregister(&group->poller); 831 832 spdk_for_each_channel_continue(i, 0); 833 } 834 835 int 836 spdk_nvmf_tgt_pause_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_pause_polling_cb_fn cb_fn, 837 void *cb_arg) 838 { 839 struct nvmf_tgt_pause_ctx *ctx; 840 841 SPDK_DTRACE_PROBE2(nvmf_tgt_pause_polling, tgt, tgt->name); 842 843 switch (tgt->state) { 844 case NVMF_TGT_PAUSING: 845 case NVMF_TGT_RESUMING: 846 return -EBUSY; 847 case NVMF_TGT_RUNNING: 848 break; 849 default: 850 return -EINVAL; 851 } 852 853 ctx = calloc(1, sizeof(*ctx)); 854 if (!ctx) { 855 return -ENOMEM; 856 } 857 858 859 tgt->state = NVMF_TGT_PAUSING; 860 861 ctx->tgt = tgt; 862 ctx->cb_fn = cb_fn; 863 ctx->cb_arg = cb_arg; 864 865 spdk_for_each_channel(tgt, 866 _nvmf_tgt_pause_polling, 867 ctx, 868 _nvmf_tgt_pause_polling_done); 869 return 0; 870 } 871 872 static void 873 _nvmf_tgt_resume_polling_done(struct spdk_io_channel_iter *i, int status) 874 { 875 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 876 877 ctx->tgt->state = NVMF_TGT_RUNNING; 878 879 ctx->cb_fn(ctx->cb_arg, status); 880 free(ctx); 881 } 882 883 static void 884 _nvmf_tgt_resume_polling(struct spdk_io_channel_iter *i) 885 { 886 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 887 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 888 889 assert(group->poller == NULL); 890 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 891 892 spdk_for_each_channel_continue(i, 0); 893 } 894 895 int 896 spdk_nvmf_tgt_resume_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_resume_polling_cb_fn cb_fn, 897 void *cb_arg) 898 { 899 struct nvmf_tgt_pause_ctx *ctx; 900 901 SPDK_DTRACE_PROBE2(nvmf_tgt_resume_polling, tgt, tgt->name); 902 903 switch (tgt->state) { 904 case NVMF_TGT_PAUSING: 905 case NVMF_TGT_RESUMING: 906 return -EBUSY; 907 case NVMF_TGT_PAUSED: 908 break; 909 default: 910 return -EINVAL; 911 } 912 913 ctx = calloc(1, sizeof(*ctx)); 914 if (!ctx) { 915 return -ENOMEM; 916 } 917 918 tgt->state = NVMF_TGT_RESUMING; 919 920 ctx->tgt = tgt; 921 ctx->cb_fn = cb_fn; 922 ctx->cb_arg = cb_arg; 923 924 spdk_for_each_channel(tgt, 925 _nvmf_tgt_resume_polling, 926 ctx, 927 _nvmf_tgt_resume_polling_done); 928 return 0; 929 } 930 931 struct spdk_nvmf_subsystem * 932 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 933 { 934 struct spdk_nvmf_subsystem *subsystem; 935 uint32_t sid; 936 937 if (!subnqn) { 938 return NULL; 939 } 940 941 /* Ensure that subnqn is null terminated */ 942 if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 943 SPDK_ERRLOG("Connect SUBNQN is not null terminated\n"); 944 return NULL; 945 } 946 947 for (sid = 0; sid < tgt->max_subsystems; sid++) { 948 subsystem = tgt->subsystems[sid]; 949 if (subsystem == NULL) { 950 continue; 951 } 952 953 if (strcmp(subnqn, subsystem->subnqn) == 0) { 954 return subsystem; 955 } 956 } 957 958 return NULL; 959 } 960 961 struct spdk_nvmf_transport * 962 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name) 963 { 964 struct spdk_nvmf_transport *transport; 965 966 TAILQ_FOREACH(transport, &tgt->transports, link) { 967 if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) { 968 return transport; 969 } 970 } 971 return NULL; 972 } 973 974 struct nvmf_new_qpair_ctx { 975 struct spdk_nvmf_qpair *qpair; 976 struct spdk_nvmf_poll_group *group; 977 }; 978 979 static void 980 _nvmf_poll_group_add(void *_ctx) 981 { 982 struct nvmf_new_qpair_ctx *ctx = _ctx; 983 struct spdk_nvmf_qpair *qpair = ctx->qpair; 984 struct spdk_nvmf_poll_group *group = ctx->group; 985 986 free(_ctx); 987 988 if (spdk_nvmf_poll_group_add(group, qpair) != 0) { 989 SPDK_ERRLOG("Unable to add the qpair to a poll group.\n"); 990 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 991 } 992 } 993 994 void 995 spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair) 996 { 997 struct spdk_nvmf_poll_group *group; 998 struct nvmf_new_qpair_ctx *ctx; 999 1000 group = spdk_nvmf_get_optimal_poll_group(qpair); 1001 if (group == NULL) { 1002 if (tgt->next_poll_group == NULL) { 1003 tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups); 1004 if (tgt->next_poll_group == NULL) { 1005 SPDK_ERRLOG("No poll groups exist.\n"); 1006 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1007 return; 1008 } 1009 } 1010 group = tgt->next_poll_group; 1011 tgt->next_poll_group = TAILQ_NEXT(group, link); 1012 } 1013 1014 ctx = calloc(1, sizeof(*ctx)); 1015 if (!ctx) { 1016 SPDK_ERRLOG("Unable to send message to poll group.\n"); 1017 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1018 return; 1019 } 1020 1021 ctx->qpair = qpair; 1022 ctx->group = group; 1023 1024 pthread_mutex_lock(&group->mutex); 1025 group->current_unassociated_qpairs++; 1026 pthread_mutex_unlock(&group->mutex); 1027 1028 spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx); 1029 } 1030 1031 struct spdk_nvmf_poll_group * 1032 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 1033 { 1034 struct spdk_io_channel *ch; 1035 1036 ch = spdk_get_io_channel(tgt); 1037 if (!ch) { 1038 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 1039 return NULL; 1040 } 1041 1042 return spdk_io_channel_get_ctx(ch); 1043 } 1044 1045 void 1046 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, 1047 spdk_nvmf_poll_group_destroy_done_fn cb_fn, 1048 void *cb_arg) 1049 { 1050 assert(group->destroy_cb_fn == NULL); 1051 group->destroy_cb_fn = cb_fn; 1052 group->destroy_cb_arg = cb_arg; 1053 1054 /* This function will put the io_channel associated with this poll group */ 1055 nvmf_tgt_destroy_poll_group_qpairs(group); 1056 } 1057 1058 int 1059 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 1060 struct spdk_nvmf_qpair *qpair) 1061 { 1062 int rc = -1; 1063 struct spdk_nvmf_transport_poll_group *tgroup; 1064 1065 TAILQ_INIT(&qpair->outstanding); 1066 qpair->group = group; 1067 qpair->ctrlr = NULL; 1068 qpair->disconnect_started = false; 1069 1070 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1071 if (tgroup->transport == qpair->transport) { 1072 rc = nvmf_transport_poll_group_add(tgroup, qpair); 1073 break; 1074 } 1075 } 1076 1077 /* We add the qpair to the group only it is successfully added into the tgroup */ 1078 if (rc == 0) { 1079 SPDK_DTRACE_PROBE2(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread)); 1080 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 1081 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVE); 1082 } 1083 1084 return rc; 1085 } 1086 1087 static void 1088 _nvmf_ctrlr_destruct(void *ctx) 1089 { 1090 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1091 1092 nvmf_ctrlr_destruct(ctrlr); 1093 } 1094 1095 static void 1096 _nvmf_ctrlr_free_from_qpair(void *ctx) 1097 { 1098 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1099 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 1100 uint32_t count; 1101 1102 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 1103 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1104 if (count == 0) { 1105 assert(!ctrlr->in_destruct); 1106 SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid); 1107 ctrlr->in_destruct = true; 1108 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 1109 } 1110 free(qpair_ctx); 1111 } 1112 1113 static void 1114 _nvmf_transport_qpair_fini_complete(void *cb_ctx) 1115 { 1116 struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx; 1117 struct spdk_nvmf_ctrlr *ctrlr; 1118 /* Store cb args since cb_ctx can be freed in _nvmf_ctrlr_free_from_qpair */ 1119 nvmf_qpair_disconnect_cb cb_fn = qpair_ctx->cb_fn; 1120 void *cb_arg = qpair_ctx->ctx; 1121 struct spdk_thread *cb_thread = qpair_ctx->thread; 1122 1123 ctrlr = qpair_ctx->ctrlr; 1124 SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid); 1125 1126 if (ctrlr) { 1127 if (qpair_ctx->qid == 0) { 1128 /* Admin qpair is removed, so set the pointer to NULL. 1129 * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same 1130 * as controller's thread */ 1131 assert(ctrlr->thread == spdk_get_thread()); 1132 ctrlr->admin_qpair = NULL; 1133 } 1134 /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */ 1135 if (ctrlr->thread) { 1136 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx); 1137 } else { 1138 _nvmf_ctrlr_free_from_qpair(qpair_ctx); 1139 } 1140 } else { 1141 free(qpair_ctx); 1142 } 1143 1144 if (cb_fn) { 1145 spdk_thread_send_msg(cb_thread, cb_fn, cb_arg); 1146 } 1147 } 1148 1149 void 1150 spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair) 1151 { 1152 struct spdk_nvmf_transport_poll_group *tgroup; 1153 int rc; 1154 1155 SPDK_DTRACE_PROBE2(nvmf_poll_group_remove_qpair, qpair, 1156 spdk_thread_get_id(qpair->group->thread)); 1157 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR); 1158 1159 /* Find the tgroup and remove the qpair from the tgroup */ 1160 TAILQ_FOREACH(tgroup, &qpair->group->tgroups, link) { 1161 if (tgroup->transport == qpair->transport) { 1162 rc = nvmf_transport_poll_group_remove(tgroup, qpair); 1163 if (rc && (rc != ENOTSUP)) { 1164 SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n", 1165 qpair, tgroup); 1166 } 1167 break; 1168 } 1169 } 1170 1171 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 1172 qpair->group = NULL; 1173 } 1174 1175 static void 1176 _nvmf_qpair_sgroup_req_clean(struct spdk_nvmf_subsystem_poll_group *sgroup, 1177 const struct spdk_nvmf_qpair *qpair) 1178 { 1179 struct spdk_nvmf_request *req, *tmp; 1180 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1181 if (req->qpair == qpair) { 1182 TAILQ_REMOVE(&sgroup->queued, req, link); 1183 if (nvmf_transport_req_free(req)) { 1184 SPDK_ERRLOG("Transport request free error!\n"); 1185 } 1186 } 1187 } 1188 } 1189 1190 static void 1191 _nvmf_qpair_destroy(void *ctx, int status) 1192 { 1193 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1194 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 1195 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 1196 struct spdk_nvmf_subsystem_poll_group *sgroup; 1197 uint32_t sid; 1198 1199 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 1200 qpair_ctx->qid = qpair->qid; 1201 1202 if (qpair->connect_received) { 1203 if (0 == qpair->qid) { 1204 assert(qpair->group->stat.current_admin_qpairs > 0); 1205 qpair->group->stat.current_admin_qpairs--; 1206 } else { 1207 assert(qpair->group->stat.current_io_qpairs > 0); 1208 qpair->group->stat.current_io_qpairs--; 1209 } 1210 } else { 1211 pthread_mutex_lock(&qpair->group->mutex); 1212 qpair->group->current_unassociated_qpairs--; 1213 pthread_mutex_unlock(&qpair->group->mutex); 1214 } 1215 1216 if (ctrlr) { 1217 sgroup = &qpair->group->sgroups[ctrlr->subsys->id]; 1218 _nvmf_qpair_sgroup_req_clean(sgroup, qpair); 1219 } else { 1220 for (sid = 0; sid < qpair->group->num_sgroups; sid++) { 1221 sgroup = &qpair->group->sgroups[sid]; 1222 assert(sgroup != NULL); 1223 _nvmf_qpair_sgroup_req_clean(sgroup, qpair); 1224 } 1225 } 1226 1227 qpair_ctx->ctrlr = ctrlr; 1228 spdk_nvmf_poll_group_remove(qpair); 1229 nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); 1230 } 1231 1232 static void 1233 _nvmf_qpair_disconnect_msg(void *ctx) 1234 { 1235 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1236 1237 spdk_nvmf_qpair_disconnect(qpair_ctx->qpair, qpair_ctx->cb_fn, qpair_ctx->ctx); 1238 free(ctx); 1239 } 1240 1241 int 1242 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) 1243 { 1244 struct spdk_nvmf_poll_group *group = qpair->group; 1245 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 1246 1247 if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) { 1248 if (cb_fn) { 1249 cb_fn(ctx); 1250 } 1251 return 0; 1252 } 1253 1254 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 1255 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 1256 nvmf_transport_qpair_fini(qpair, NULL, NULL); 1257 if (cb_fn) { 1258 cb_fn(ctx); 1259 } 1260 return 0; 1261 } 1262 1263 assert(group != NULL); 1264 if (spdk_get_thread() != group->thread) { 1265 /* clear the atomic so we can set it on the next call on the proper thread. */ 1266 __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED); 1267 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1268 if (!qpair_ctx) { 1269 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1270 return -ENOMEM; 1271 } 1272 qpair_ctx->qpair = qpair; 1273 qpair_ctx->cb_fn = cb_fn; 1274 qpair_ctx->thread = group->thread; 1275 qpair_ctx->ctx = ctx; 1276 spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx); 1277 return 0; 1278 } 1279 1280 SPDK_DTRACE_PROBE2(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread)); 1281 assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); 1282 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 1283 1284 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1285 if (!qpair_ctx) { 1286 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1287 return -ENOMEM; 1288 } 1289 1290 qpair_ctx->qpair = qpair; 1291 qpair_ctx->cb_fn = cb_fn; 1292 qpair_ctx->thread = group->thread; 1293 qpair_ctx->ctx = ctx; 1294 1295 /* Check for outstanding I/O */ 1296 if (!TAILQ_EMPTY(&qpair->outstanding)) { 1297 SPDK_DTRACE_PROBE2(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread)); 1298 qpair->state_cb = _nvmf_qpair_destroy; 1299 qpair->state_cb_arg = qpair_ctx; 1300 nvmf_qpair_abort_pending_zcopy_reqs(qpair); 1301 nvmf_qpair_free_aer(qpair); 1302 return 0; 1303 } 1304 1305 _nvmf_qpair_destroy(qpair_ctx, 0); 1306 1307 return 0; 1308 } 1309 1310 int 1311 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 1312 struct spdk_nvme_transport_id *trid) 1313 { 1314 return nvmf_transport_qpair_get_peer_trid(qpair, trid); 1315 } 1316 1317 int 1318 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 1319 struct spdk_nvme_transport_id *trid) 1320 { 1321 return nvmf_transport_qpair_get_local_trid(qpair, trid); 1322 } 1323 1324 int 1325 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 1326 struct spdk_nvme_transport_id *trid) 1327 { 1328 return nvmf_transport_qpair_get_listen_trid(qpair, trid); 1329 } 1330 1331 static int 1332 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1333 struct spdk_nvmf_subsystem *subsystem) 1334 { 1335 struct spdk_nvmf_subsystem_poll_group *sgroup; 1336 uint32_t new_num_ns, old_num_ns; 1337 uint32_t i, j; 1338 struct spdk_nvmf_ns *ns; 1339 struct spdk_nvmf_registrant *reg, *tmp; 1340 struct spdk_io_channel *ch; 1341 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1342 struct spdk_nvmf_ctrlr *ctrlr; 1343 bool ns_changed; 1344 1345 /* Make sure our poll group has memory for this subsystem allocated */ 1346 if (subsystem->id >= group->num_sgroups) { 1347 return -ENOMEM; 1348 } 1349 1350 sgroup = &group->sgroups[subsystem->id]; 1351 1352 /* Make sure the array of namespace information is the correct size */ 1353 new_num_ns = subsystem->max_nsid; 1354 old_num_ns = sgroup->num_ns; 1355 1356 ns_changed = false; 1357 1358 if (old_num_ns == 0) { 1359 if (new_num_ns > 0) { 1360 /* First allocation */ 1361 sgroup->ns_info = calloc(new_num_ns, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1362 if (!sgroup->ns_info) { 1363 return -ENOMEM; 1364 } 1365 } 1366 } else if (new_num_ns > old_num_ns) { 1367 void *buf; 1368 1369 /* Make the array larger */ 1370 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1371 if (!buf) { 1372 return -ENOMEM; 1373 } 1374 1375 sgroup->ns_info = buf; 1376 1377 /* Null out the new namespace information slots */ 1378 for (i = old_num_ns; i < new_num_ns; i++) { 1379 memset(&sgroup->ns_info[i], 0, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1380 } 1381 } else if (new_num_ns < old_num_ns) { 1382 void *buf; 1383 1384 /* Free the extra I/O channels */ 1385 for (i = new_num_ns; i < old_num_ns; i++) { 1386 ns_info = &sgroup->ns_info[i]; 1387 1388 if (ns_info->channel) { 1389 spdk_put_io_channel(ns_info->channel); 1390 ns_info->channel = NULL; 1391 } 1392 } 1393 1394 /* Make the array smaller */ 1395 if (new_num_ns > 0) { 1396 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1397 if (!buf) { 1398 return -ENOMEM; 1399 } 1400 sgroup->ns_info = buf; 1401 } else { 1402 free(sgroup->ns_info); 1403 sgroup->ns_info = NULL; 1404 } 1405 } 1406 1407 sgroup->num_ns = new_num_ns; 1408 1409 /* Detect bdevs that were added or removed */ 1410 for (i = 0; i < sgroup->num_ns; i++) { 1411 ns = subsystem->ns[i]; 1412 ns_info = &sgroup->ns_info[i]; 1413 ch = ns_info->channel; 1414 1415 if (ns == NULL && ch == NULL) { 1416 /* Both NULL. Leave empty */ 1417 } else if (ns == NULL && ch != NULL) { 1418 /* There was a channel here, but the namespace is gone. */ 1419 ns_changed = true; 1420 spdk_put_io_channel(ch); 1421 ns_info->channel = NULL; 1422 } else if (ns != NULL && ch == NULL) { 1423 /* A namespace appeared but there is no channel yet */ 1424 ns_changed = true; 1425 ch = spdk_bdev_get_io_channel(ns->desc); 1426 if (ch == NULL) { 1427 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1428 return -ENOMEM; 1429 } 1430 ns_info->channel = ch; 1431 } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) { 1432 /* A namespace was here before, but was replaced by a new one. */ 1433 ns_changed = true; 1434 spdk_put_io_channel(ns_info->channel); 1435 memset(ns_info, 0, sizeof(*ns_info)); 1436 1437 ch = spdk_bdev_get_io_channel(ns->desc); 1438 if (ch == NULL) { 1439 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1440 return -ENOMEM; 1441 } 1442 ns_info->channel = ch; 1443 } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) { 1444 /* Namespace is still there but size has changed */ 1445 SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u," 1446 " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n", 1447 subsystem->id, 1448 ns->nsid, 1449 group, 1450 ns_info->num_blocks, 1451 spdk_bdev_get_num_blocks(ns->bdev)); 1452 ns_changed = true; 1453 } 1454 1455 if (ns == NULL) { 1456 memset(ns_info, 0, sizeof(*ns_info)); 1457 } else { 1458 ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev); 1459 ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev); 1460 ns_info->crkey = ns->crkey; 1461 ns_info->rtype = ns->rtype; 1462 if (ns->holder) { 1463 ns_info->holder_id = ns->holder->hostid; 1464 } 1465 1466 memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid)); 1467 j = 0; 1468 TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { 1469 if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) { 1470 SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS); 1471 return -EINVAL; 1472 } 1473 ns_info->reg_hostid[j++] = reg->hostid; 1474 } 1475 } 1476 } 1477 1478 if (ns_changed) { 1479 TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) { 1480 /* It is possible that a ctrlr was added but the admin_qpair hasn't been 1481 * assigned yet. 1482 */ 1483 if (!ctrlr->admin_qpair) { 1484 continue; 1485 } 1486 if (ctrlr->admin_qpair->group == group) { 1487 nvmf_ctrlr_async_event_ns_notice(ctrlr); 1488 nvmf_ctrlr_async_event_ana_change_notice(ctrlr); 1489 } 1490 } 1491 } 1492 1493 return 0; 1494 } 1495 1496 int 1497 nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1498 struct spdk_nvmf_subsystem *subsystem) 1499 { 1500 return poll_group_update_subsystem(group, subsystem); 1501 } 1502 1503 int 1504 nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 1505 struct spdk_nvmf_subsystem *subsystem, 1506 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1507 { 1508 int rc = 0; 1509 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 1510 uint32_t i; 1511 1512 TAILQ_INIT(&sgroup->queued); 1513 1514 rc = poll_group_update_subsystem(group, subsystem); 1515 if (rc) { 1516 nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); 1517 goto fini; 1518 } 1519 1520 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1521 1522 for (i = 0; i < sgroup->num_ns; i++) { 1523 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1524 } 1525 1526 fini: 1527 if (cb_fn) { 1528 cb_fn(cb_arg, rc); 1529 } 1530 1531 SPDK_DTRACE_PROBE2(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread), 1532 subsystem->subnqn); 1533 1534 return rc; 1535 } 1536 1537 static void 1538 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 1539 { 1540 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1541 struct spdk_nvmf_subsystem *subsystem; 1542 struct spdk_nvmf_poll_group *group; 1543 struct spdk_nvmf_subsystem_poll_group *sgroup; 1544 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 1545 void *cpl_ctx = NULL; 1546 uint32_t nsid; 1547 1548 group = qpair_ctx->group; 1549 subsystem = qpair_ctx->subsystem; 1550 cpl_fn = qpair_ctx->cpl_fn; 1551 cpl_ctx = qpair_ctx->cpl_ctx; 1552 sgroup = &group->sgroups[subsystem->id]; 1553 1554 if (status) { 1555 goto fini; 1556 } 1557 1558 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 1559 if (sgroup->ns_info[nsid].channel) { 1560 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 1561 sgroup->ns_info[nsid].channel = NULL; 1562 } 1563 } 1564 1565 sgroup->num_ns = 0; 1566 free(sgroup->ns_info); 1567 sgroup->ns_info = NULL; 1568 fini: 1569 free(qpair_ctx); 1570 if (cpl_fn) { 1571 cpl_fn(cpl_ctx, status); 1572 } 1573 } 1574 1575 static void nvmf_poll_group_remove_subsystem_msg(void *ctx); 1576 1577 static void 1578 remove_subsystem_qpair_cb(void *ctx) 1579 { 1580 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1581 1582 assert(qpair_ctx->count > 0); 1583 qpair_ctx->count--; 1584 if (qpair_ctx->count == 0) { 1585 /* All of the asynchronous callbacks for this context have been 1586 * completed. Call nvmf_poll_group_remove_subsystem_msg() again 1587 * to check if all associated qpairs for this subsystem have 1588 * been removed from the poll group. 1589 */ 1590 nvmf_poll_group_remove_subsystem_msg(ctx); 1591 } 1592 } 1593 1594 static void 1595 nvmf_poll_group_remove_subsystem_msg(void *ctx) 1596 { 1597 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 1598 struct spdk_nvmf_subsystem *subsystem; 1599 struct spdk_nvmf_poll_group *group; 1600 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1601 bool qpairs_found = false; 1602 int rc = 0; 1603 1604 group = qpair_ctx->group; 1605 subsystem = qpair_ctx->subsystem; 1606 1607 /* Initialize count to 1. This acts like a ref count, to ensure that if spdk_nvmf_qpair_disconnect 1608 * immediately invokes the callback (i.e. the qpairs is already in process of being disconnected) 1609 * that we don't recursively call nvmf_poll_group_remove_subsystem_msg before we've iterated the 1610 * full list of qpairs. 1611 */ 1612 qpair_ctx->count = 1; 1613 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 1614 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1615 qpairs_found = true; 1616 qpair_ctx->count++; 1617 rc = spdk_nvmf_qpair_disconnect(qpair, remove_subsystem_qpair_cb, ctx); 1618 if (rc) { 1619 break; 1620 } 1621 } 1622 } 1623 qpair_ctx->count--; 1624 1625 if (!qpairs_found) { 1626 _nvmf_poll_group_remove_subsystem_cb(ctx, 0); 1627 return; 1628 } 1629 1630 if (qpair_ctx->count == 0 || rc) { 1631 /* If count == 0, it means there were some qpairs in the poll group but they 1632 * were already in process of being disconnected. So we send a message to this 1633 * same thread so that this function executes again later. We won't actually 1634 * invoke the remove_subsystem_cb until all of the qpairs are actually removed 1635 * from the poll group. 1636 */ 1637 spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx); 1638 } 1639 } 1640 1641 void 1642 nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1643 struct spdk_nvmf_subsystem *subsystem, 1644 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1645 { 1646 struct spdk_nvmf_subsystem_poll_group *sgroup; 1647 struct nvmf_qpair_disconnect_many_ctx *ctx; 1648 uint32_t i; 1649 1650 SPDK_DTRACE_PROBE3(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread), 1651 subsystem->subnqn); 1652 1653 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1654 if (!ctx) { 1655 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1656 if (cb_fn) { 1657 cb_fn(cb_arg, -1); 1658 } 1659 return; 1660 } 1661 1662 ctx->group = group; 1663 ctx->subsystem = subsystem; 1664 ctx->cpl_fn = cb_fn; 1665 ctx->cpl_ctx = cb_arg; 1666 1667 sgroup = &group->sgroups[subsystem->id]; 1668 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1669 1670 for (i = 0; i < sgroup->num_ns; i++) { 1671 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1672 } 1673 1674 nvmf_poll_group_remove_subsystem_msg(ctx); 1675 } 1676 1677 void 1678 nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1679 struct spdk_nvmf_subsystem *subsystem, 1680 uint32_t nsid, 1681 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1682 { 1683 struct spdk_nvmf_subsystem_poll_group *sgroup; 1684 struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL; 1685 int rc = 0; 1686 uint32_t i; 1687 1688 if (subsystem->id >= group->num_sgroups) { 1689 rc = -1; 1690 goto fini; 1691 } 1692 1693 sgroup = &group->sgroups[subsystem->id]; 1694 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) { 1695 goto fini; 1696 } 1697 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1698 1699 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1700 for (i = 0; i < sgroup->num_ns; i++) { 1701 ns_info = &sgroup->ns_info[i]; 1702 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1703 } 1704 } else { 1705 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 1706 if (nsid - 1 < sgroup->num_ns) { 1707 ns_info = &sgroup->ns_info[nsid - 1]; 1708 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1709 } 1710 } 1711 1712 if (sgroup->mgmt_io_outstanding > 0) { 1713 assert(sgroup->cb_fn == NULL); 1714 sgroup->cb_fn = cb_fn; 1715 assert(sgroup->cb_arg == NULL); 1716 sgroup->cb_arg = cb_arg; 1717 return; 1718 } 1719 1720 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1721 for (i = 0; i < sgroup->num_ns; i++) { 1722 ns_info = &sgroup->ns_info[i]; 1723 1724 if (ns_info->io_outstanding > 0) { 1725 assert(sgroup->cb_fn == NULL); 1726 sgroup->cb_fn = cb_fn; 1727 assert(sgroup->cb_arg == NULL); 1728 sgroup->cb_arg = cb_arg; 1729 return; 1730 } 1731 } 1732 } else { 1733 if (ns_info != NULL && ns_info->io_outstanding > 0) { 1734 assert(sgroup->cb_fn == NULL); 1735 sgroup->cb_fn = cb_fn; 1736 assert(sgroup->cb_arg == NULL); 1737 sgroup->cb_arg = cb_arg; 1738 return; 1739 } 1740 } 1741 1742 assert(sgroup->mgmt_io_outstanding == 0); 1743 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1744 fini: 1745 if (cb_fn) { 1746 cb_fn(cb_arg, rc); 1747 } 1748 } 1749 1750 void 1751 nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1752 struct spdk_nvmf_subsystem *subsystem, 1753 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1754 { 1755 struct spdk_nvmf_request *req, *tmp; 1756 struct spdk_nvmf_subsystem_poll_group *sgroup; 1757 int rc = 0; 1758 uint32_t i; 1759 1760 if (subsystem->id >= group->num_sgroups) { 1761 rc = -1; 1762 goto fini; 1763 } 1764 1765 sgroup = &group->sgroups[subsystem->id]; 1766 1767 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) { 1768 goto fini; 1769 } 1770 1771 rc = poll_group_update_subsystem(group, subsystem); 1772 if (rc) { 1773 goto fini; 1774 } 1775 1776 for (i = 0; i < sgroup->num_ns; i++) { 1777 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1778 } 1779 1780 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1781 1782 /* Release all queued requests */ 1783 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1784 TAILQ_REMOVE(&sgroup->queued, req, link); 1785 if (spdk_nvmf_request_using_zcopy(req)) { 1786 spdk_nvmf_request_zcopy_start(req); 1787 } else { 1788 spdk_nvmf_request_exec(req); 1789 } 1790 1791 } 1792 fini: 1793 if (cb_fn) { 1794 cb_fn(cb_arg, rc); 1795 } 1796 } 1797 1798 1799 struct spdk_nvmf_poll_group * 1800 spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 1801 { 1802 struct spdk_nvmf_transport_poll_group *tgroup; 1803 1804 tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair); 1805 1806 if (tgroup == NULL) { 1807 return NULL; 1808 } 1809 1810 return tgroup->group; 1811 } 1812 1813 void 1814 spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w) 1815 { 1816 struct spdk_nvmf_transport_poll_group *tgroup; 1817 1818 spdk_json_write_object_begin(w); 1819 1820 spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread())); 1821 spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs); 1822 spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs); 1823 spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs); 1824 spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs); 1825 spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io); 1826 spdk_json_write_named_uint64(w, "completed_nvme_io", group->stat.completed_nvme_io); 1827 1828 spdk_json_write_named_array_begin(w, "transports"); 1829 1830 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1831 spdk_json_write_object_begin(w); 1832 /* 1833 * The trtype field intentionally contains a transport name as this is more informative. 1834 * The field has not been renamed for backward compatibility. 1835 */ 1836 spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport)); 1837 1838 if (tgroup->transport->ops->poll_group_dump_stat) { 1839 tgroup->transport->ops->poll_group_dump_stat(tgroup, w); 1840 } 1841 1842 spdk_json_write_object_end(w); 1843 } 1844 1845 spdk_json_write_array_end(w); 1846 spdk_json_write_object_end(w); 1847 } 1848