1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/bdev.h" 10 #include "spdk/bit_array.h" 11 #include "spdk/thread.h" 12 #include "spdk/nvmf.h" 13 #include "spdk/endian.h" 14 #include "spdk/string.h" 15 #include "spdk/log.h" 16 #include "spdk_internal/usdt.h" 17 18 #include "nvmf_internal.h" 19 #include "transport.h" 20 21 SPDK_LOG_REGISTER_COMPONENT(nvmf) 22 23 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 24 25 static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts); 26 27 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 28 29 /* supplied to a single call to nvmf_qpair_disconnect */ 30 struct nvmf_qpair_disconnect_ctx { 31 struct spdk_nvmf_qpair *qpair; 32 struct spdk_nvmf_ctrlr *ctrlr; 33 nvmf_qpair_disconnect_cb cb_fn; 34 struct spdk_thread *thread; 35 void *ctx; 36 uint16_t qid; 37 }; 38 39 /* 40 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 41 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 42 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 43 */ 44 struct nvmf_qpair_disconnect_many_ctx { 45 struct spdk_nvmf_subsystem *subsystem; 46 struct spdk_nvmf_poll_group *group; 47 spdk_nvmf_poll_group_mod_done cpl_fn; 48 void *cpl_ctx; 49 uint32_t count; 50 }; 51 52 static void 53 nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 54 enum spdk_nvmf_qpair_state state) 55 { 56 assert(qpair != NULL); 57 assert(qpair->group->thread == spdk_get_thread()); 58 59 qpair->state = state; 60 } 61 62 static int 63 nvmf_poll_group_poll(void *ctx) 64 { 65 struct spdk_nvmf_poll_group *group = ctx; 66 int rc; 67 int count = 0; 68 struct spdk_nvmf_transport_poll_group *tgroup; 69 70 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 71 rc = nvmf_transport_poll_group_poll(tgroup); 72 if (rc < 0) { 73 return SPDK_POLLER_BUSY; 74 } 75 count += rc; 76 } 77 78 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 79 } 80 81 /* 82 * Reset and clean up the poll group (I/O channel code will actually free the 83 * group). 84 */ 85 static void 86 nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group) 87 { 88 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 89 struct spdk_nvmf_subsystem_poll_group *sgroup; 90 uint32_t sid, nsid; 91 92 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 93 TAILQ_REMOVE(&group->tgroups, tgroup, link); 94 nvmf_transport_poll_group_destroy(tgroup); 95 } 96 97 for (sid = 0; sid < group->num_sgroups; sid++) { 98 sgroup = &group->sgroups[sid]; 99 100 assert(sgroup != NULL); 101 102 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 103 if (sgroup->ns_info[nsid].channel) { 104 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 105 sgroup->ns_info[nsid].channel = NULL; 106 } 107 } 108 109 free(sgroup->ns_info); 110 } 111 112 free(group->sgroups); 113 114 spdk_poller_unregister(&group->poller); 115 116 if (group->destroy_cb_fn) { 117 group->destroy_cb_fn(group->destroy_cb_arg, 0); 118 } 119 } 120 121 /* 122 * Callback to unregister a poll group from the target, and clean up its state. 123 */ 124 static void 125 nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 126 { 127 struct spdk_nvmf_tgt *tgt = io_device; 128 struct spdk_nvmf_poll_group *group = ctx_buf; 129 130 SPDK_DTRACE_PROBE1(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread)); 131 132 pthread_mutex_lock(&tgt->mutex); 133 TAILQ_REMOVE(&tgt->poll_groups, group, link); 134 pthread_mutex_unlock(&tgt->mutex); 135 136 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 137 nvmf_tgt_cleanup_poll_group(group); 138 } 139 140 static int 141 nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 142 struct spdk_nvmf_transport *transport) 143 { 144 struct spdk_nvmf_transport_poll_group *tgroup; 145 146 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 147 if (tgroup->transport == transport) { 148 /* Transport already in the poll group */ 149 return 0; 150 } 151 } 152 153 tgroup = nvmf_transport_poll_group_create(transport, group); 154 if (!tgroup) { 155 SPDK_ERRLOG("Unable to create poll group for transport\n"); 156 return -1; 157 } 158 SPDK_DTRACE_PROBE2(nvmf_transport_poll_group_create, transport, spdk_thread_get_id(group->thread)); 159 160 tgroup->group = group; 161 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 162 163 return 0; 164 } 165 166 static int 167 nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 168 { 169 struct spdk_nvmf_tgt *tgt = io_device; 170 struct spdk_nvmf_poll_group *group = ctx_buf; 171 struct spdk_nvmf_transport *transport; 172 struct spdk_thread *thread = spdk_get_thread(); 173 uint32_t sid; 174 int rc; 175 176 TAILQ_INIT(&group->tgroups); 177 TAILQ_INIT(&group->qpairs); 178 group->thread = thread; 179 pthread_mutex_init(&group->mutex, NULL); 180 181 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 182 183 SPDK_DTRACE_PROBE1(nvmf_create_poll_group, spdk_thread_get_id(thread)); 184 185 TAILQ_FOREACH(transport, &tgt->transports, link) { 186 rc = nvmf_poll_group_add_transport(group, transport); 187 if (rc != 0) { 188 nvmf_tgt_cleanup_poll_group(group); 189 return rc; 190 } 191 } 192 193 group->num_sgroups = tgt->max_subsystems; 194 group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 195 if (!group->sgroups) { 196 nvmf_tgt_cleanup_poll_group(group); 197 return -ENOMEM; 198 } 199 200 for (sid = 0; sid < tgt->max_subsystems; sid++) { 201 struct spdk_nvmf_subsystem *subsystem; 202 203 subsystem = tgt->subsystems[sid]; 204 if (!subsystem) { 205 continue; 206 } 207 208 if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { 209 nvmf_tgt_cleanup_poll_group(group); 210 return -1; 211 } 212 } 213 214 pthread_mutex_lock(&tgt->mutex); 215 TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link); 216 pthread_mutex_unlock(&tgt->mutex); 217 218 return 0; 219 } 220 221 static void 222 _nvmf_tgt_disconnect_next_qpair(void *ctx) 223 { 224 struct spdk_nvmf_qpair *qpair; 225 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 226 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 227 struct spdk_io_channel *ch; 228 int rc = 0; 229 230 qpair = TAILQ_FIRST(&group->qpairs); 231 232 if (qpair) { 233 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_tgt_disconnect_next_qpair, ctx); 234 } 235 236 if (!qpair || rc != 0) { 237 /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */ 238 ch = spdk_io_channel_from_ctx(group); 239 spdk_put_io_channel(ch); 240 free(qpair_ctx); 241 } 242 } 243 244 static void 245 nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 246 { 247 struct nvmf_qpair_disconnect_many_ctx *ctx; 248 249 SPDK_DTRACE_PROBE1(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread)); 250 251 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 252 if (!ctx) { 253 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 254 return; 255 } 256 257 ctx->group = group; 258 _nvmf_tgt_disconnect_next_qpair(ctx); 259 } 260 261 struct spdk_nvmf_tgt * 262 spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *opts) 263 { 264 struct spdk_nvmf_tgt *tgt, *tmp_tgt; 265 266 if (strnlen(opts->name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) { 267 SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH); 268 return NULL; 269 } 270 271 TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) { 272 if (!strncmp(opts->name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) { 273 SPDK_ERRLOG("Provided target name must be unique.\n"); 274 return NULL; 275 } 276 } 277 278 tgt = calloc(1, sizeof(*tgt)); 279 if (!tgt) { 280 return NULL; 281 } 282 283 snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts->name); 284 285 if (!opts || !opts->max_subsystems) { 286 tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 287 } else { 288 tgt->max_subsystems = opts->max_subsystems; 289 } 290 291 if (!opts) { 292 tgt->crdt[0] = 0; 293 tgt->crdt[1] = 0; 294 tgt->crdt[2] = 0; 295 } else { 296 tgt->crdt[0] = opts->crdt[0]; 297 tgt->crdt[1] = opts->crdt[1]; 298 tgt->crdt[2] = opts->crdt[2]; 299 } 300 301 if (!opts) { 302 tgt->discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY; 303 } else { 304 tgt->discovery_filter = opts->discovery_filter; 305 } 306 307 tgt->discovery_genctr = 0; 308 TAILQ_INIT(&tgt->transports); 309 TAILQ_INIT(&tgt->poll_groups); 310 311 tgt->subsystems = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem *)); 312 if (!tgt->subsystems) { 313 free(tgt); 314 return NULL; 315 } 316 317 pthread_mutex_init(&tgt->mutex, NULL); 318 319 spdk_io_device_register(tgt, 320 nvmf_tgt_create_poll_group, 321 nvmf_tgt_destroy_poll_group, 322 sizeof(struct spdk_nvmf_poll_group), 323 tgt->name); 324 325 tgt->state = NVMF_TGT_RUNNING; 326 327 TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link); 328 329 return tgt; 330 } 331 332 static void 333 _nvmf_tgt_destroy_next_transport(void *ctx) 334 { 335 struct spdk_nvmf_tgt *tgt = ctx; 336 struct spdk_nvmf_transport *transport; 337 338 if (!TAILQ_EMPTY(&tgt->transports)) { 339 transport = TAILQ_FIRST(&tgt->transports); 340 TAILQ_REMOVE(&tgt->transports, transport, link); 341 spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt); 342 } else { 343 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn; 344 void *destroy_cb_arg = tgt->destroy_cb_arg; 345 346 pthread_mutex_destroy(&tgt->mutex); 347 free(tgt); 348 349 if (destroy_cb_fn) { 350 destroy_cb_fn(destroy_cb_arg, 0); 351 } 352 } 353 } 354 355 static void 356 nvmf_tgt_destroy_cb(void *io_device) 357 { 358 struct spdk_nvmf_tgt *tgt = io_device; 359 uint32_t i; 360 int rc; 361 362 if (tgt->subsystems) { 363 for (i = 0; i < tgt->max_subsystems; i++) { 364 if (tgt->subsystems[i]) { 365 nvmf_subsystem_remove_all_listeners(tgt->subsystems[i], true); 366 367 rc = spdk_nvmf_subsystem_destroy(tgt->subsystems[i], nvmf_tgt_destroy_cb, tgt); 368 if (rc) { 369 if (rc == -EINPROGRESS) { 370 /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i 371 * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */ 372 return; 373 } else { 374 SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", tgt->subsystems[i]->subnqn, rc); 375 } 376 } 377 } 378 } 379 free(tgt->subsystems); 380 } 381 382 _nvmf_tgt_destroy_next_transport(tgt); 383 } 384 385 void 386 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 387 spdk_nvmf_tgt_destroy_done_fn cb_fn, 388 void *cb_arg) 389 { 390 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 391 392 tgt->destroy_cb_fn = cb_fn; 393 tgt->destroy_cb_arg = cb_arg; 394 395 TAILQ_REMOVE(&g_nvmf_tgts, tgt, link); 396 397 spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb); 398 } 399 400 const char * 401 spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt) 402 { 403 return tgt->name; 404 } 405 406 struct spdk_nvmf_tgt * 407 spdk_nvmf_get_tgt(const char *name) 408 { 409 struct spdk_nvmf_tgt *tgt; 410 uint32_t num_targets = 0; 411 412 TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) { 413 if (name) { 414 if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) { 415 return tgt; 416 } 417 } 418 num_targets++; 419 } 420 421 /* 422 * special case. If there is only one target and 423 * no name was specified, return the only available 424 * target. If there is more than one target, name must 425 * be specified. 426 */ 427 if (!name && num_targets == 1) { 428 return TAILQ_FIRST(&g_nvmf_tgts); 429 } 430 431 return NULL; 432 } 433 434 struct spdk_nvmf_tgt * 435 spdk_nvmf_get_first_tgt(void) 436 { 437 return TAILQ_FIRST(&g_nvmf_tgts); 438 } 439 440 struct spdk_nvmf_tgt * 441 spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev) 442 { 443 return TAILQ_NEXT(prev, link); 444 } 445 446 static void 447 nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 448 struct spdk_nvmf_subsystem *subsystem) 449 { 450 struct spdk_nvmf_host *host; 451 struct spdk_nvmf_subsystem_listener *listener; 452 const struct spdk_nvme_transport_id *trid; 453 struct spdk_nvmf_ns *ns; 454 struct spdk_nvmf_ns_opts ns_opts; 455 uint32_t max_namespaces; 456 char uuid_str[SPDK_UUID_STRING_LEN]; 457 458 if (spdk_nvmf_subsystem_get_type(subsystem) != SPDK_NVMF_SUBTYPE_NVME) { 459 return; 460 } 461 462 /* { */ 463 spdk_json_write_object_begin(w); 464 spdk_json_write_named_string(w, "method", "nvmf_create_subsystem"); 465 466 /* "params" : { */ 467 spdk_json_write_named_object_begin(w, "params"); 468 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 469 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 470 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 471 spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem)); 472 473 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 474 if (max_namespaces != 0) { 475 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 476 } 477 478 spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem)); 479 spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem)); 480 spdk_json_write_named_bool(w, "ana_reporting", nvmf_subsystem_get_ana_reporting(subsystem)); 481 482 /* } "params" */ 483 spdk_json_write_object_end(w); 484 485 /* } */ 486 spdk_json_write_object_end(w); 487 488 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 489 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 490 491 spdk_json_write_object_begin(w); 492 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 493 494 /* "params" : { */ 495 spdk_json_write_named_object_begin(w, "params"); 496 497 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 498 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 499 500 /* } "params" */ 501 spdk_json_write_object_end(w); 502 503 /* } */ 504 spdk_json_write_object_end(w); 505 } 506 507 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 508 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 509 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 510 511 spdk_json_write_object_begin(w); 512 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 513 514 /* "params" : { */ 515 spdk_json_write_named_object_begin(w, "params"); 516 517 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 518 519 /* "namespace" : { */ 520 spdk_json_write_named_object_begin(w, "namespace"); 521 522 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 523 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 524 525 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 526 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 527 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 528 from_be64(&ns_opts.nguid[8])); 529 } 530 531 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 532 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 533 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 534 } 535 536 if (!spdk_mem_all_zero(&ns_opts.uuid, sizeof(ns_opts.uuid))) { 537 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &ns_opts.uuid); 538 spdk_json_write_named_string(w, "uuid", uuid_str); 539 } 540 541 if (nvmf_subsystem_get_ana_reporting(subsystem)) { 542 spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid); 543 } 544 545 /* "namespace" */ 546 spdk_json_write_object_end(w); 547 548 /* } "params" */ 549 spdk_json_write_object_end(w); 550 551 /* } */ 552 spdk_json_write_object_end(w); 553 } 554 555 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 556 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 557 trid = spdk_nvmf_subsystem_listener_get_trid(listener); 558 559 spdk_json_write_object_begin(w); 560 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 561 562 /* "params" : { */ 563 spdk_json_write_named_object_begin(w, "params"); 564 565 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 566 nvmf_transport_listen_dump_opts(listener->transport, trid, w); 567 568 /* } "params" */ 569 spdk_json_write_object_end(w); 570 571 /* } */ 572 spdk_json_write_object_end(w); 573 } 574 575 } 576 577 void 578 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 579 { 580 struct spdk_nvmf_subsystem *subsystem; 581 struct spdk_nvmf_transport *transport; 582 583 spdk_json_write_object_begin(w); 584 spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems"); 585 586 spdk_json_write_named_object_begin(w, "params"); 587 spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems); 588 spdk_json_write_object_end(w); 589 590 spdk_json_write_object_end(w); 591 592 spdk_json_write_object_begin(w); 593 spdk_json_write_named_string(w, "method", "nvmf_set_crdt"); 594 spdk_json_write_named_object_begin(w, "params"); 595 spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]); 596 spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]); 597 spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]); 598 spdk_json_write_object_end(w); 599 spdk_json_write_object_end(w); 600 601 /* write transports */ 602 TAILQ_FOREACH(transport, &tgt->transports, link) { 603 spdk_json_write_object_begin(w); 604 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 605 nvmf_transport_dump_opts(transport, w, true); 606 spdk_json_write_object_end(w); 607 } 608 609 subsystem = spdk_nvmf_subsystem_get_first(tgt); 610 while (subsystem) { 611 nvmf_write_subsystem_config_json(w, subsystem); 612 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 613 } 614 } 615 616 static void 617 nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts, 618 const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size) 619 { 620 assert(opts); 621 assert(opts_src); 622 623 opts->opts_size = opts_size; 624 625 #define SET_FIELD(field) \ 626 if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \ 627 opts->field = opts_src->field; \ 628 } \ 629 630 SET_FIELD(transport_specific); 631 #undef SET_FIELD 632 633 /* Do not remove this statement, you should always update this statement when you adding a new field, 634 * and do not forget to add the SET_FIELD statement for your added field. */ 635 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 16, "Incorrect size"); 636 } 637 638 void 639 spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size) 640 { 641 struct spdk_nvmf_listen_opts opts_local = {}; 642 643 /* local version of opts should have defaults set here */ 644 645 nvmf_listen_opts_copy(opts, &opts_local, opts_size); 646 } 647 648 int 649 spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid, 650 struct spdk_nvmf_listen_opts *opts) 651 { 652 struct spdk_nvmf_transport *transport; 653 int rc; 654 struct spdk_nvmf_listen_opts opts_local = {}; 655 656 if (!opts) { 657 SPDK_ERRLOG("opts should not be NULL\n"); 658 return -EINVAL; 659 } 660 661 if (!opts->opts_size) { 662 SPDK_ERRLOG("The opts_size in opts structure should not be zero\n"); 663 return -EINVAL; 664 } 665 666 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 667 if (!transport) { 668 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 669 trid->trstring); 670 return -EINVAL; 671 } 672 673 nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size); 674 rc = spdk_nvmf_transport_listen(transport, trid, &opts_local); 675 if (rc < 0) { 676 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 677 } 678 679 return rc; 680 } 681 682 int 683 spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt, 684 struct spdk_nvme_transport_id *trid) 685 { 686 struct spdk_nvmf_transport *transport; 687 int rc; 688 689 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 690 if (!transport) { 691 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 692 trid->trstring); 693 return -EINVAL; 694 } 695 696 rc = spdk_nvmf_transport_stop_listen(transport, trid); 697 if (rc < 0) { 698 SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr); 699 return rc; 700 } 701 return 0; 702 } 703 704 struct spdk_nvmf_tgt_add_transport_ctx { 705 struct spdk_nvmf_tgt *tgt; 706 struct spdk_nvmf_transport *transport; 707 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 708 void *cb_arg; 709 int status; 710 }; 711 712 static void 713 _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status) 714 { 715 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 716 717 ctx->cb_fn(ctx->cb_arg, ctx->status); 718 free(ctx); 719 } 720 721 static void 722 _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i) 723 { 724 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 725 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 726 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 727 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 728 729 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 730 if (tgroup->transport == ctx->transport) { 731 TAILQ_REMOVE(&group->tgroups, tgroup, link); 732 nvmf_transport_poll_group_destroy(tgroup); 733 } 734 } 735 736 spdk_for_each_channel_continue(i, 0); 737 } 738 739 static void 740 _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 741 { 742 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 743 744 if (status) { 745 ctx->status = status; 746 spdk_for_each_channel(ctx->tgt, 747 _nvmf_tgt_remove_transport, 748 ctx, 749 _nvmf_tgt_remove_transport_done); 750 return; 751 } 752 753 ctx->transport->tgt = ctx->tgt; 754 TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link); 755 ctx->cb_fn(ctx->cb_arg, status); 756 free(ctx); 757 } 758 759 static void 760 _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 761 { 762 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 763 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 764 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 765 int rc; 766 767 rc = nvmf_poll_group_add_transport(group, ctx->transport); 768 spdk_for_each_channel_continue(i, rc); 769 } 770 771 void 772 spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 773 struct spdk_nvmf_transport *transport, 774 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 775 void *cb_arg) 776 { 777 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 778 779 SPDK_DTRACE_PROBE2(nvmf_tgt_add_transport, transport, tgt->name); 780 781 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) { 782 cb_fn(cb_arg, -EEXIST); 783 return; /* transport already created */ 784 } 785 786 ctx = calloc(1, sizeof(*ctx)); 787 if (!ctx) { 788 cb_fn(cb_arg, -ENOMEM); 789 return; 790 } 791 792 ctx->tgt = tgt; 793 ctx->transport = transport; 794 ctx->cb_fn = cb_fn; 795 ctx->cb_arg = cb_arg; 796 797 spdk_for_each_channel(tgt, 798 _nvmf_tgt_add_transport, 799 ctx, 800 _nvmf_tgt_add_transport_done); 801 } 802 803 struct nvmf_tgt_pause_ctx { 804 struct spdk_nvmf_tgt *tgt; 805 spdk_nvmf_tgt_pause_polling_cb_fn cb_fn; 806 void *cb_arg; 807 }; 808 809 static void 810 _nvmf_tgt_pause_polling_done(struct spdk_io_channel_iter *i, int status) 811 { 812 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 813 814 ctx->tgt->state = NVMF_TGT_PAUSED; 815 816 ctx->cb_fn(ctx->cb_arg, status); 817 free(ctx); 818 } 819 820 static void 821 _nvmf_tgt_pause_polling(struct spdk_io_channel_iter *i) 822 { 823 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 824 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 825 826 spdk_poller_unregister(&group->poller); 827 828 spdk_for_each_channel_continue(i, 0); 829 } 830 831 int 832 spdk_nvmf_tgt_pause_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_pause_polling_cb_fn cb_fn, 833 void *cb_arg) 834 { 835 struct nvmf_tgt_pause_ctx *ctx; 836 837 SPDK_DTRACE_PROBE2(nvmf_tgt_pause_polling, tgt, tgt->name); 838 839 switch (tgt->state) { 840 case NVMF_TGT_PAUSING: 841 case NVMF_TGT_RESUMING: 842 return -EBUSY; 843 case NVMF_TGT_RUNNING: 844 break; 845 default: 846 return -EINVAL; 847 } 848 849 ctx = calloc(1, sizeof(*ctx)); 850 if (!ctx) { 851 return -ENOMEM; 852 } 853 854 855 tgt->state = NVMF_TGT_PAUSING; 856 857 ctx->tgt = tgt; 858 ctx->cb_fn = cb_fn; 859 ctx->cb_arg = cb_arg; 860 861 spdk_for_each_channel(tgt, 862 _nvmf_tgt_pause_polling, 863 ctx, 864 _nvmf_tgt_pause_polling_done); 865 return 0; 866 } 867 868 static void 869 _nvmf_tgt_resume_polling_done(struct spdk_io_channel_iter *i, int status) 870 { 871 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 872 873 ctx->tgt->state = NVMF_TGT_RUNNING; 874 875 ctx->cb_fn(ctx->cb_arg, status); 876 free(ctx); 877 } 878 879 static void 880 _nvmf_tgt_resume_polling(struct spdk_io_channel_iter *i) 881 { 882 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 883 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 884 885 assert(group->poller == NULL); 886 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 887 888 spdk_for_each_channel_continue(i, 0); 889 } 890 891 int 892 spdk_nvmf_tgt_resume_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_resume_polling_cb_fn cb_fn, 893 void *cb_arg) 894 { 895 struct nvmf_tgt_pause_ctx *ctx; 896 897 SPDK_DTRACE_PROBE2(nvmf_tgt_resume_polling, tgt, tgt->name); 898 899 switch (tgt->state) { 900 case NVMF_TGT_PAUSING: 901 case NVMF_TGT_RESUMING: 902 return -EBUSY; 903 case NVMF_TGT_PAUSED: 904 break; 905 default: 906 return -EINVAL; 907 } 908 909 ctx = calloc(1, sizeof(*ctx)); 910 if (!ctx) { 911 return -ENOMEM; 912 } 913 914 tgt->state = NVMF_TGT_RESUMING; 915 916 ctx->tgt = tgt; 917 ctx->cb_fn = cb_fn; 918 ctx->cb_arg = cb_arg; 919 920 spdk_for_each_channel(tgt, 921 _nvmf_tgt_resume_polling, 922 ctx, 923 _nvmf_tgt_resume_polling_done); 924 return 0; 925 } 926 927 struct spdk_nvmf_subsystem * 928 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 929 { 930 struct spdk_nvmf_subsystem *subsystem; 931 uint32_t sid; 932 933 if (!subnqn) { 934 return NULL; 935 } 936 937 /* Ensure that subnqn is null terminated */ 938 if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 939 SPDK_ERRLOG("Connect SUBNQN is not null terminated\n"); 940 return NULL; 941 } 942 943 for (sid = 0; sid < tgt->max_subsystems; sid++) { 944 subsystem = tgt->subsystems[sid]; 945 if (subsystem == NULL) { 946 continue; 947 } 948 949 if (strcmp(subnqn, subsystem->subnqn) == 0) { 950 return subsystem; 951 } 952 } 953 954 return NULL; 955 } 956 957 struct spdk_nvmf_transport * 958 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name) 959 { 960 struct spdk_nvmf_transport *transport; 961 962 TAILQ_FOREACH(transport, &tgt->transports, link) { 963 if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) { 964 return transport; 965 } 966 } 967 return NULL; 968 } 969 970 struct nvmf_new_qpair_ctx { 971 struct spdk_nvmf_qpair *qpair; 972 struct spdk_nvmf_poll_group *group; 973 }; 974 975 static void 976 _nvmf_poll_group_add(void *_ctx) 977 { 978 struct nvmf_new_qpair_ctx *ctx = _ctx; 979 struct spdk_nvmf_qpair *qpair = ctx->qpair; 980 struct spdk_nvmf_poll_group *group = ctx->group; 981 982 free(_ctx); 983 984 if (spdk_nvmf_poll_group_add(group, qpair) != 0) { 985 SPDK_ERRLOG("Unable to add the qpair to a poll group.\n"); 986 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 987 } 988 } 989 990 void 991 spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair) 992 { 993 struct spdk_nvmf_poll_group *group; 994 struct nvmf_new_qpair_ctx *ctx; 995 996 group = spdk_nvmf_get_optimal_poll_group(qpair); 997 if (group == NULL) { 998 if (tgt->next_poll_group == NULL) { 999 tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups); 1000 if (tgt->next_poll_group == NULL) { 1001 SPDK_ERRLOG("No poll groups exist.\n"); 1002 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1003 return; 1004 } 1005 } 1006 group = tgt->next_poll_group; 1007 tgt->next_poll_group = TAILQ_NEXT(group, link); 1008 } 1009 1010 ctx = calloc(1, sizeof(*ctx)); 1011 if (!ctx) { 1012 SPDK_ERRLOG("Unable to send message to poll group.\n"); 1013 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1014 return; 1015 } 1016 1017 ctx->qpair = qpair; 1018 ctx->group = group; 1019 1020 pthread_mutex_lock(&group->mutex); 1021 group->current_unassociated_qpairs++; 1022 pthread_mutex_unlock(&group->mutex); 1023 1024 spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx); 1025 } 1026 1027 struct spdk_nvmf_poll_group * 1028 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 1029 { 1030 struct spdk_io_channel *ch; 1031 1032 ch = spdk_get_io_channel(tgt); 1033 if (!ch) { 1034 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 1035 return NULL; 1036 } 1037 1038 return spdk_io_channel_get_ctx(ch); 1039 } 1040 1041 void 1042 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, 1043 spdk_nvmf_poll_group_destroy_done_fn cb_fn, 1044 void *cb_arg) 1045 { 1046 assert(group->destroy_cb_fn == NULL); 1047 group->destroy_cb_fn = cb_fn; 1048 group->destroy_cb_arg = cb_arg; 1049 1050 /* This function will put the io_channel associated with this poll group */ 1051 nvmf_tgt_destroy_poll_group_qpairs(group); 1052 } 1053 1054 int 1055 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 1056 struct spdk_nvmf_qpair *qpair) 1057 { 1058 int rc = -1; 1059 struct spdk_nvmf_transport_poll_group *tgroup; 1060 1061 TAILQ_INIT(&qpair->outstanding); 1062 qpair->group = group; 1063 qpair->ctrlr = NULL; 1064 qpair->disconnect_started = false; 1065 1066 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1067 if (tgroup->transport == qpair->transport) { 1068 rc = nvmf_transport_poll_group_add(tgroup, qpair); 1069 break; 1070 } 1071 } 1072 1073 /* We add the qpair to the group only it is successfully added into the tgroup */ 1074 if (rc == 0) { 1075 SPDK_DTRACE_PROBE2(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread)); 1076 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 1077 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVE); 1078 } 1079 1080 return rc; 1081 } 1082 1083 static void 1084 _nvmf_ctrlr_destruct(void *ctx) 1085 { 1086 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1087 1088 nvmf_ctrlr_destruct(ctrlr); 1089 } 1090 1091 static void 1092 _nvmf_ctrlr_free_from_qpair(void *ctx) 1093 { 1094 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1095 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 1096 uint32_t count; 1097 1098 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 1099 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1100 if (count == 0) { 1101 assert(!ctrlr->in_destruct); 1102 SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid); 1103 ctrlr->in_destruct = true; 1104 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 1105 } 1106 free(qpair_ctx); 1107 } 1108 1109 static void 1110 _nvmf_transport_qpair_fini_complete(void *cb_ctx) 1111 { 1112 struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx; 1113 struct spdk_nvmf_ctrlr *ctrlr; 1114 /* Store cb args since cb_ctx can be freed in _nvmf_ctrlr_free_from_qpair */ 1115 nvmf_qpair_disconnect_cb cb_fn = qpair_ctx->cb_fn; 1116 void *cb_arg = qpair_ctx->ctx; 1117 struct spdk_thread *cb_thread = qpair_ctx->thread; 1118 1119 ctrlr = qpair_ctx->ctrlr; 1120 SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid); 1121 1122 if (ctrlr) { 1123 if (qpair_ctx->qid == 0) { 1124 /* Admin qpair is removed, so set the pointer to NULL. 1125 * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same 1126 * as controller's thread */ 1127 assert(ctrlr->thread == spdk_get_thread()); 1128 ctrlr->admin_qpair = NULL; 1129 } 1130 /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */ 1131 if (ctrlr->thread) { 1132 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx); 1133 } else { 1134 _nvmf_ctrlr_free_from_qpair(qpair_ctx); 1135 } 1136 } else { 1137 free(qpair_ctx); 1138 } 1139 1140 if (cb_fn) { 1141 spdk_thread_send_msg(cb_thread, cb_fn, cb_arg); 1142 } 1143 } 1144 1145 void 1146 spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair) 1147 { 1148 struct spdk_nvmf_transport_poll_group *tgroup; 1149 int rc; 1150 1151 SPDK_DTRACE_PROBE2(nvmf_poll_group_remove_qpair, qpair, 1152 spdk_thread_get_id(qpair->group->thread)); 1153 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR); 1154 1155 /* Find the tgroup and remove the qpair from the tgroup */ 1156 TAILQ_FOREACH(tgroup, &qpair->group->tgroups, link) { 1157 if (tgroup->transport == qpair->transport) { 1158 rc = nvmf_transport_poll_group_remove(tgroup, qpair); 1159 if (rc && (rc != ENOTSUP)) { 1160 SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n", 1161 qpair, tgroup); 1162 } 1163 break; 1164 } 1165 } 1166 1167 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 1168 qpair->group = NULL; 1169 } 1170 1171 static void 1172 _nvmf_qpair_destroy(void *ctx, int status) 1173 { 1174 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1175 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 1176 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 1177 struct spdk_nvmf_request *req, *tmp; 1178 struct spdk_nvmf_subsystem_poll_group *sgroup; 1179 1180 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 1181 qpair_ctx->qid = qpair->qid; 1182 1183 if (qpair->connect_received) { 1184 if (0 == qpair->qid) { 1185 assert(qpair->group->stat.current_admin_qpairs > 0); 1186 qpair->group->stat.current_admin_qpairs--; 1187 } else { 1188 assert(qpair->group->stat.current_io_qpairs > 0); 1189 qpair->group->stat.current_io_qpairs--; 1190 } 1191 } else { 1192 pthread_mutex_lock(&qpair->group->mutex); 1193 qpair->group->current_unassociated_qpairs--; 1194 pthread_mutex_unlock(&qpair->group->mutex); 1195 } 1196 1197 if (ctrlr) { 1198 sgroup = &qpair->group->sgroups[ctrlr->subsys->id]; 1199 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1200 if (req->qpair == qpair) { 1201 TAILQ_REMOVE(&sgroup->queued, req, link); 1202 if (nvmf_transport_req_free(req)) { 1203 SPDK_ERRLOG("Transport request free error!/n"); 1204 } 1205 } 1206 } 1207 } 1208 1209 qpair_ctx->ctrlr = ctrlr; 1210 spdk_nvmf_poll_group_remove(qpair); 1211 nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); 1212 } 1213 1214 static void 1215 _nvmf_qpair_disconnect_msg(void *ctx) 1216 { 1217 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1218 1219 spdk_nvmf_qpair_disconnect(qpair_ctx->qpair, qpair_ctx->cb_fn, qpair_ctx->ctx); 1220 free(ctx); 1221 } 1222 1223 int 1224 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) 1225 { 1226 struct spdk_nvmf_poll_group *group = qpair->group; 1227 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 1228 1229 if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) { 1230 if (cb_fn) { 1231 cb_fn(ctx); 1232 } 1233 return 0; 1234 } 1235 1236 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 1237 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 1238 nvmf_transport_qpair_fini(qpair, NULL, NULL); 1239 if (cb_fn) { 1240 cb_fn(ctx); 1241 } 1242 return 0; 1243 } 1244 1245 assert(group != NULL); 1246 if (spdk_get_thread() != group->thread) { 1247 /* clear the atomic so we can set it on the next call on the proper thread. */ 1248 __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED); 1249 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1250 if (!qpair_ctx) { 1251 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1252 return -ENOMEM; 1253 } 1254 qpair_ctx->qpair = qpair; 1255 qpair_ctx->cb_fn = cb_fn; 1256 qpair_ctx->thread = group->thread; 1257 qpair_ctx->ctx = ctx; 1258 spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx); 1259 return 0; 1260 } 1261 1262 SPDK_DTRACE_PROBE2(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread)); 1263 assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); 1264 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 1265 1266 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1267 if (!qpair_ctx) { 1268 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1269 return -ENOMEM; 1270 } 1271 1272 qpair_ctx->qpair = qpair; 1273 qpair_ctx->cb_fn = cb_fn; 1274 qpair_ctx->thread = group->thread; 1275 qpair_ctx->ctx = ctx; 1276 1277 /* Check for outstanding I/O */ 1278 if (!TAILQ_EMPTY(&qpair->outstanding)) { 1279 SPDK_DTRACE_PROBE2(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread)); 1280 qpair->state_cb = _nvmf_qpair_destroy; 1281 qpair->state_cb_arg = qpair_ctx; 1282 nvmf_qpair_abort_pending_zcopy_reqs(qpair); 1283 nvmf_qpair_free_aer(qpair); 1284 return 0; 1285 } 1286 1287 _nvmf_qpair_destroy(qpair_ctx, 0); 1288 1289 return 0; 1290 } 1291 1292 int 1293 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 1294 struct spdk_nvme_transport_id *trid) 1295 { 1296 return nvmf_transport_qpair_get_peer_trid(qpair, trid); 1297 } 1298 1299 int 1300 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 1301 struct spdk_nvme_transport_id *trid) 1302 { 1303 return nvmf_transport_qpair_get_local_trid(qpair, trid); 1304 } 1305 1306 int 1307 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 1308 struct spdk_nvme_transport_id *trid) 1309 { 1310 return nvmf_transport_qpair_get_listen_trid(qpair, trid); 1311 } 1312 1313 static int 1314 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1315 struct spdk_nvmf_subsystem *subsystem) 1316 { 1317 struct spdk_nvmf_subsystem_poll_group *sgroup; 1318 uint32_t new_num_ns, old_num_ns; 1319 uint32_t i, j; 1320 struct spdk_nvmf_ns *ns; 1321 struct spdk_nvmf_registrant *reg, *tmp; 1322 struct spdk_io_channel *ch; 1323 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1324 struct spdk_nvmf_ctrlr *ctrlr; 1325 bool ns_changed; 1326 1327 /* Make sure our poll group has memory for this subsystem allocated */ 1328 if (subsystem->id >= group->num_sgroups) { 1329 return -ENOMEM; 1330 } 1331 1332 sgroup = &group->sgroups[subsystem->id]; 1333 1334 /* Make sure the array of namespace information is the correct size */ 1335 new_num_ns = subsystem->max_nsid; 1336 old_num_ns = sgroup->num_ns; 1337 1338 ns_changed = false; 1339 1340 if (old_num_ns == 0) { 1341 if (new_num_ns > 0) { 1342 /* First allocation */ 1343 sgroup->ns_info = calloc(new_num_ns, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1344 if (!sgroup->ns_info) { 1345 return -ENOMEM; 1346 } 1347 } 1348 } else if (new_num_ns > old_num_ns) { 1349 void *buf; 1350 1351 /* Make the array larger */ 1352 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1353 if (!buf) { 1354 return -ENOMEM; 1355 } 1356 1357 sgroup->ns_info = buf; 1358 1359 /* Null out the new namespace information slots */ 1360 for (i = old_num_ns; i < new_num_ns; i++) { 1361 memset(&sgroup->ns_info[i], 0, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1362 } 1363 } else if (new_num_ns < old_num_ns) { 1364 void *buf; 1365 1366 /* Free the extra I/O channels */ 1367 for (i = new_num_ns; i < old_num_ns; i++) { 1368 ns_info = &sgroup->ns_info[i]; 1369 1370 if (ns_info->channel) { 1371 spdk_put_io_channel(ns_info->channel); 1372 ns_info->channel = NULL; 1373 } 1374 } 1375 1376 /* Make the array smaller */ 1377 if (new_num_ns > 0) { 1378 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1379 if (!buf) { 1380 return -ENOMEM; 1381 } 1382 sgroup->ns_info = buf; 1383 } else { 1384 free(sgroup->ns_info); 1385 sgroup->ns_info = NULL; 1386 } 1387 } 1388 1389 sgroup->num_ns = new_num_ns; 1390 1391 /* Detect bdevs that were added or removed */ 1392 for (i = 0; i < sgroup->num_ns; i++) { 1393 ns = subsystem->ns[i]; 1394 ns_info = &sgroup->ns_info[i]; 1395 ch = ns_info->channel; 1396 1397 if (ns == NULL && ch == NULL) { 1398 /* Both NULL. Leave empty */ 1399 } else if (ns == NULL && ch != NULL) { 1400 /* There was a channel here, but the namespace is gone. */ 1401 ns_changed = true; 1402 spdk_put_io_channel(ch); 1403 ns_info->channel = NULL; 1404 } else if (ns != NULL && ch == NULL) { 1405 /* A namespace appeared but there is no channel yet */ 1406 ns_changed = true; 1407 ch = spdk_bdev_get_io_channel(ns->desc); 1408 if (ch == NULL) { 1409 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1410 return -ENOMEM; 1411 } 1412 ns_info->channel = ch; 1413 } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) { 1414 /* A namespace was here before, but was replaced by a new one. */ 1415 ns_changed = true; 1416 spdk_put_io_channel(ns_info->channel); 1417 memset(ns_info, 0, sizeof(*ns_info)); 1418 1419 ch = spdk_bdev_get_io_channel(ns->desc); 1420 if (ch == NULL) { 1421 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1422 return -ENOMEM; 1423 } 1424 ns_info->channel = ch; 1425 } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) { 1426 /* Namespace is still there but size has changed */ 1427 SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u," 1428 " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n", 1429 subsystem->id, 1430 ns->nsid, 1431 group, 1432 ns_info->num_blocks, 1433 spdk_bdev_get_num_blocks(ns->bdev)); 1434 ns_changed = true; 1435 } 1436 1437 if (ns == NULL) { 1438 memset(ns_info, 0, sizeof(*ns_info)); 1439 } else { 1440 ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev); 1441 ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev); 1442 ns_info->crkey = ns->crkey; 1443 ns_info->rtype = ns->rtype; 1444 if (ns->holder) { 1445 ns_info->holder_id = ns->holder->hostid; 1446 } 1447 1448 memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid)); 1449 j = 0; 1450 TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { 1451 if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) { 1452 SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS); 1453 return -EINVAL; 1454 } 1455 ns_info->reg_hostid[j++] = reg->hostid; 1456 } 1457 } 1458 } 1459 1460 if (ns_changed) { 1461 TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) { 1462 /* It is possible that a ctrlr was added but the admin_qpair hasn't been 1463 * assigned yet. 1464 */ 1465 if (!ctrlr->admin_qpair) { 1466 continue; 1467 } 1468 if (ctrlr->admin_qpair->group == group) { 1469 nvmf_ctrlr_async_event_ns_notice(ctrlr); 1470 nvmf_ctrlr_async_event_ana_change_notice(ctrlr); 1471 } 1472 } 1473 } 1474 1475 return 0; 1476 } 1477 1478 int 1479 nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1480 struct spdk_nvmf_subsystem *subsystem) 1481 { 1482 return poll_group_update_subsystem(group, subsystem); 1483 } 1484 1485 int 1486 nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 1487 struct spdk_nvmf_subsystem *subsystem, 1488 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1489 { 1490 int rc = 0; 1491 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 1492 uint32_t i; 1493 1494 TAILQ_INIT(&sgroup->queued); 1495 1496 rc = poll_group_update_subsystem(group, subsystem); 1497 if (rc) { 1498 nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); 1499 goto fini; 1500 } 1501 1502 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1503 1504 for (i = 0; i < sgroup->num_ns; i++) { 1505 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1506 } 1507 1508 fini: 1509 if (cb_fn) { 1510 cb_fn(cb_arg, rc); 1511 } 1512 1513 SPDK_DTRACE_PROBE2(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread), 1514 subsystem->subnqn); 1515 1516 return rc; 1517 } 1518 1519 static void 1520 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 1521 { 1522 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1523 struct spdk_nvmf_subsystem *subsystem; 1524 struct spdk_nvmf_poll_group *group; 1525 struct spdk_nvmf_subsystem_poll_group *sgroup; 1526 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 1527 void *cpl_ctx = NULL; 1528 uint32_t nsid; 1529 1530 group = qpair_ctx->group; 1531 subsystem = qpair_ctx->subsystem; 1532 cpl_fn = qpair_ctx->cpl_fn; 1533 cpl_ctx = qpair_ctx->cpl_ctx; 1534 sgroup = &group->sgroups[subsystem->id]; 1535 1536 if (status) { 1537 goto fini; 1538 } 1539 1540 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 1541 if (sgroup->ns_info[nsid].channel) { 1542 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 1543 sgroup->ns_info[nsid].channel = NULL; 1544 } 1545 } 1546 1547 sgroup->num_ns = 0; 1548 free(sgroup->ns_info); 1549 sgroup->ns_info = NULL; 1550 fini: 1551 free(qpair_ctx); 1552 if (cpl_fn) { 1553 cpl_fn(cpl_ctx, status); 1554 } 1555 } 1556 1557 static void nvmf_poll_group_remove_subsystem_msg(void *ctx); 1558 1559 static void 1560 remove_subsystem_qpair_cb(void *ctx) 1561 { 1562 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1563 1564 assert(qpair_ctx->count > 0); 1565 qpair_ctx->count--; 1566 if (qpair_ctx->count == 0) { 1567 /* All of the asynchronous callbacks for this context have been 1568 * completed. Call nvmf_poll_group_remove_subsystem_msg() again 1569 * to check if all associated qpairs for this subsystem have 1570 * been removed from the poll group. 1571 */ 1572 nvmf_poll_group_remove_subsystem_msg(ctx); 1573 } 1574 } 1575 1576 static void 1577 nvmf_poll_group_remove_subsystem_msg(void *ctx) 1578 { 1579 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 1580 struct spdk_nvmf_subsystem *subsystem; 1581 struct spdk_nvmf_poll_group *group; 1582 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1583 bool qpairs_found = false; 1584 int rc = 0; 1585 1586 group = qpair_ctx->group; 1587 subsystem = qpair_ctx->subsystem; 1588 1589 /* Initialize count to 1. This acts like a ref count, to ensure that if spdk_nvmf_qpair_disconnect 1590 * immediately invokes the callback (i.e. the qpairs is already in process of being disconnected) 1591 * that we don't recursively call nvmf_poll_group_remove_subsystem_msg before we've iterated the 1592 * full list of qpairs. 1593 */ 1594 qpair_ctx->count = 1; 1595 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 1596 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1597 qpairs_found = true; 1598 qpair_ctx->count++; 1599 rc = spdk_nvmf_qpair_disconnect(qpair, remove_subsystem_qpair_cb, ctx); 1600 if (rc) { 1601 break; 1602 } 1603 } 1604 } 1605 qpair_ctx->count--; 1606 1607 if (!qpairs_found) { 1608 _nvmf_poll_group_remove_subsystem_cb(ctx, 0); 1609 return; 1610 } 1611 1612 if (qpair_ctx->count == 0 || rc) { 1613 /* If count == 0, it means there were some qpairs in the poll group but they 1614 * were already in process of being disconnected. So we send a message to this 1615 * same thread so that this function executes again later. We won't actually 1616 * invoke the remove_subsystem_cb until all of the qpairs are actually removed 1617 * from the poll group. 1618 */ 1619 spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx); 1620 } 1621 } 1622 1623 void 1624 nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1625 struct spdk_nvmf_subsystem *subsystem, 1626 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1627 { 1628 struct spdk_nvmf_subsystem_poll_group *sgroup; 1629 struct nvmf_qpair_disconnect_many_ctx *ctx; 1630 uint32_t i; 1631 1632 SPDK_DTRACE_PROBE3(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread), 1633 subsystem->subnqn); 1634 1635 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1636 if (!ctx) { 1637 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1638 if (cb_fn) { 1639 cb_fn(cb_arg, -1); 1640 } 1641 return; 1642 } 1643 1644 ctx->group = group; 1645 ctx->subsystem = subsystem; 1646 ctx->cpl_fn = cb_fn; 1647 ctx->cpl_ctx = cb_arg; 1648 1649 sgroup = &group->sgroups[subsystem->id]; 1650 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1651 1652 for (i = 0; i < sgroup->num_ns; i++) { 1653 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1654 } 1655 1656 nvmf_poll_group_remove_subsystem_msg(ctx); 1657 } 1658 1659 void 1660 nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1661 struct spdk_nvmf_subsystem *subsystem, 1662 uint32_t nsid, 1663 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1664 { 1665 struct spdk_nvmf_subsystem_poll_group *sgroup; 1666 struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL; 1667 int rc = 0; 1668 uint32_t i; 1669 1670 if (subsystem->id >= group->num_sgroups) { 1671 rc = -1; 1672 goto fini; 1673 } 1674 1675 sgroup = &group->sgroups[subsystem->id]; 1676 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) { 1677 goto fini; 1678 } 1679 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1680 1681 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1682 for (i = 0; i < sgroup->num_ns; i++) { 1683 ns_info = &sgroup->ns_info[i]; 1684 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1685 } 1686 } else { 1687 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 1688 if (nsid - 1 < sgroup->num_ns) { 1689 ns_info = &sgroup->ns_info[nsid - 1]; 1690 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1691 } 1692 } 1693 1694 if (sgroup->mgmt_io_outstanding > 0) { 1695 assert(sgroup->cb_fn == NULL); 1696 sgroup->cb_fn = cb_fn; 1697 assert(sgroup->cb_arg == NULL); 1698 sgroup->cb_arg = cb_arg; 1699 return; 1700 } 1701 1702 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1703 for (i = 0; i < sgroup->num_ns; i++) { 1704 ns_info = &sgroup->ns_info[i]; 1705 1706 if (ns_info->io_outstanding > 0) { 1707 assert(sgroup->cb_fn == NULL); 1708 sgroup->cb_fn = cb_fn; 1709 assert(sgroup->cb_arg == NULL); 1710 sgroup->cb_arg = cb_arg; 1711 return; 1712 } 1713 } 1714 } else { 1715 if (ns_info != NULL && ns_info->io_outstanding > 0) { 1716 assert(sgroup->cb_fn == NULL); 1717 sgroup->cb_fn = cb_fn; 1718 assert(sgroup->cb_arg == NULL); 1719 sgroup->cb_arg = cb_arg; 1720 return; 1721 } 1722 } 1723 1724 assert(sgroup->mgmt_io_outstanding == 0); 1725 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1726 fini: 1727 if (cb_fn) { 1728 cb_fn(cb_arg, rc); 1729 } 1730 } 1731 1732 void 1733 nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1734 struct spdk_nvmf_subsystem *subsystem, 1735 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1736 { 1737 struct spdk_nvmf_request *req, *tmp; 1738 struct spdk_nvmf_subsystem_poll_group *sgroup; 1739 int rc = 0; 1740 uint32_t i; 1741 1742 if (subsystem->id >= group->num_sgroups) { 1743 rc = -1; 1744 goto fini; 1745 } 1746 1747 sgroup = &group->sgroups[subsystem->id]; 1748 1749 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) { 1750 goto fini; 1751 } 1752 1753 rc = poll_group_update_subsystem(group, subsystem); 1754 if (rc) { 1755 goto fini; 1756 } 1757 1758 for (i = 0; i < sgroup->num_ns; i++) { 1759 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1760 } 1761 1762 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1763 1764 /* Release all queued requests */ 1765 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1766 TAILQ_REMOVE(&sgroup->queued, req, link); 1767 if (spdk_nvmf_request_using_zcopy(req)) { 1768 spdk_nvmf_request_zcopy_start(req); 1769 } else { 1770 spdk_nvmf_request_exec(req); 1771 } 1772 1773 } 1774 fini: 1775 if (cb_fn) { 1776 cb_fn(cb_arg, rc); 1777 } 1778 } 1779 1780 1781 struct spdk_nvmf_poll_group * 1782 spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 1783 { 1784 struct spdk_nvmf_transport_poll_group *tgroup; 1785 1786 tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair); 1787 1788 if (tgroup == NULL) { 1789 return NULL; 1790 } 1791 1792 return tgroup->group; 1793 } 1794 1795 void 1796 spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w) 1797 { 1798 struct spdk_nvmf_transport_poll_group *tgroup; 1799 1800 spdk_json_write_object_begin(w); 1801 1802 spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread())); 1803 spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs); 1804 spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs); 1805 spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs); 1806 spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs); 1807 spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io); 1808 spdk_json_write_named_uint64(w, "completed_nvme_io", group->stat.completed_nvme_io); 1809 1810 spdk_json_write_named_array_begin(w, "transports"); 1811 1812 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1813 spdk_json_write_object_begin(w); 1814 /* 1815 * The trtype field intentionally contains a transport name as this is more informative. 1816 * The field has not been renamed for backward compatibility. 1817 */ 1818 spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport)); 1819 1820 if (tgroup->transport->ops->poll_group_dump_stat) { 1821 tgroup->transport->ops->poll_group_dump_stat(tgroup, w); 1822 } 1823 1824 spdk_json_write_object_end(w); 1825 } 1826 1827 spdk_json_write_array_end(w); 1828 spdk_json_write_object_end(w); 1829 } 1830