1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/bdev.h" 10 #include "spdk/bit_array.h" 11 #include "spdk/thread.h" 12 #include "spdk/nvmf.h" 13 #include "spdk/endian.h" 14 #include "spdk/string.h" 15 #include "spdk/log.h" 16 #include "spdk_internal/usdt.h" 17 18 #include "nvmf_internal.h" 19 #include "transport.h" 20 21 SPDK_LOG_REGISTER_COMPONENT(nvmf) 22 23 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 24 25 static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts); 26 27 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 28 29 /* supplied to a single call to nvmf_qpair_disconnect */ 30 struct nvmf_qpair_disconnect_ctx { 31 struct spdk_nvmf_qpair *qpair; 32 struct spdk_nvmf_ctrlr *ctrlr; 33 nvmf_qpair_disconnect_cb cb_fn; 34 struct spdk_thread *thread; 35 void *ctx; 36 uint16_t qid; 37 }; 38 39 /* 40 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 41 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 42 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 43 */ 44 struct nvmf_qpair_disconnect_many_ctx { 45 struct spdk_nvmf_subsystem *subsystem; 46 struct spdk_nvmf_poll_group *group; 47 spdk_nvmf_poll_group_mod_done cpl_fn; 48 void *cpl_ctx; 49 }; 50 51 static void 52 nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 53 enum spdk_nvmf_qpair_state state) 54 { 55 assert(qpair != NULL); 56 assert(qpair->group->thread == spdk_get_thread()); 57 58 qpair->state = state; 59 } 60 61 static int 62 nvmf_poll_group_poll(void *ctx) 63 { 64 struct spdk_nvmf_poll_group *group = ctx; 65 int rc; 66 int count = 0; 67 struct spdk_nvmf_transport_poll_group *tgroup; 68 69 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 70 rc = nvmf_transport_poll_group_poll(tgroup); 71 if (rc < 0) { 72 return SPDK_POLLER_BUSY; 73 } 74 count += rc; 75 } 76 77 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 78 } 79 80 /* 81 * Reset and clean up the poll group (I/O channel code will actually free the 82 * group). 83 */ 84 static void 85 nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group) 86 { 87 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 88 struct spdk_nvmf_subsystem_poll_group *sgroup; 89 uint32_t sid, nsid; 90 91 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 92 TAILQ_REMOVE(&group->tgroups, tgroup, link); 93 nvmf_transport_poll_group_destroy(tgroup); 94 } 95 96 for (sid = 0; sid < group->num_sgroups; sid++) { 97 sgroup = &group->sgroups[sid]; 98 99 assert(sgroup != NULL); 100 101 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 102 if (sgroup->ns_info[nsid].channel) { 103 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 104 sgroup->ns_info[nsid].channel = NULL; 105 } 106 } 107 108 free(sgroup->ns_info); 109 } 110 111 free(group->sgroups); 112 113 spdk_poller_unregister(&group->poller); 114 115 if (group->destroy_cb_fn) { 116 group->destroy_cb_fn(group->destroy_cb_arg, 0); 117 } 118 } 119 120 /* 121 * Callback to unregister a poll group from the target, and clean up its state. 122 */ 123 static void 124 nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 125 { 126 struct spdk_nvmf_tgt *tgt = io_device; 127 struct spdk_nvmf_poll_group *group = ctx_buf; 128 129 SPDK_DTRACE_PROBE1(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread)); 130 131 pthread_mutex_lock(&tgt->mutex); 132 TAILQ_REMOVE(&tgt->poll_groups, group, link); 133 tgt->num_poll_groups--; 134 pthread_mutex_unlock(&tgt->mutex); 135 136 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 137 nvmf_tgt_cleanup_poll_group(group); 138 } 139 140 static int 141 nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 142 struct spdk_nvmf_transport *transport) 143 { 144 struct spdk_nvmf_transport_poll_group *tgroup; 145 146 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 147 if (tgroup->transport == transport) { 148 /* Transport already in the poll group */ 149 return 0; 150 } 151 } 152 153 tgroup = nvmf_transport_poll_group_create(transport, group); 154 if (!tgroup) { 155 SPDK_ERRLOG("Unable to create poll group for transport\n"); 156 return -1; 157 } 158 SPDK_DTRACE_PROBE2(nvmf_transport_poll_group_create, transport, spdk_thread_get_id(group->thread)); 159 160 tgroup->group = group; 161 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 162 163 return 0; 164 } 165 166 static int 167 nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 168 { 169 struct spdk_nvmf_tgt *tgt = io_device; 170 struct spdk_nvmf_poll_group *group = ctx_buf; 171 struct spdk_nvmf_transport *transport; 172 struct spdk_thread *thread = spdk_get_thread(); 173 uint32_t sid; 174 int rc; 175 176 group->tgt = tgt; 177 TAILQ_INIT(&group->tgroups); 178 TAILQ_INIT(&group->qpairs); 179 group->thread = thread; 180 pthread_mutex_init(&group->mutex, NULL); 181 182 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 183 184 SPDK_DTRACE_PROBE1(nvmf_create_poll_group, spdk_thread_get_id(thread)); 185 186 TAILQ_FOREACH(transport, &tgt->transports, link) { 187 rc = nvmf_poll_group_add_transport(group, transport); 188 if (rc != 0) { 189 nvmf_tgt_cleanup_poll_group(group); 190 return rc; 191 } 192 } 193 194 group->num_sgroups = tgt->max_subsystems; 195 group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 196 if (!group->sgroups) { 197 nvmf_tgt_cleanup_poll_group(group); 198 return -ENOMEM; 199 } 200 201 for (sid = 0; sid < tgt->max_subsystems; sid++) { 202 struct spdk_nvmf_subsystem *subsystem; 203 204 subsystem = tgt->subsystems[sid]; 205 if (!subsystem) { 206 continue; 207 } 208 209 if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { 210 nvmf_tgt_cleanup_poll_group(group); 211 return -1; 212 } 213 } 214 215 pthread_mutex_lock(&tgt->mutex); 216 tgt->num_poll_groups++; 217 TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link); 218 pthread_mutex_unlock(&tgt->mutex); 219 220 return 0; 221 } 222 223 static void 224 _nvmf_tgt_disconnect_qpairs(void *ctx) 225 { 226 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 227 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 228 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 229 struct spdk_io_channel *ch; 230 int rc; 231 232 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 233 rc = spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 234 if (rc && rc != -EINPROGRESS) { 235 break; 236 } 237 } 238 239 if (TAILQ_EMPTY(&group->qpairs)) { 240 /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */ 241 ch = spdk_io_channel_from_ctx(group); 242 spdk_put_io_channel(ch); 243 free(qpair_ctx); 244 return; 245 } 246 247 /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */ 248 spdk_thread_send_msg(spdk_get_thread(), _nvmf_tgt_disconnect_qpairs, ctx); 249 } 250 251 static void 252 nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 253 { 254 struct nvmf_qpair_disconnect_many_ctx *ctx; 255 256 SPDK_DTRACE_PROBE1(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread)); 257 258 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 259 if (!ctx) { 260 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 261 return; 262 } 263 264 ctx->group = group; 265 _nvmf_tgt_disconnect_qpairs(ctx); 266 } 267 268 struct spdk_nvmf_tgt * 269 spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *opts) 270 { 271 struct spdk_nvmf_tgt *tgt, *tmp_tgt; 272 273 if (strnlen(opts->name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) { 274 SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH); 275 return NULL; 276 } 277 278 TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) { 279 if (!strncmp(opts->name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) { 280 SPDK_ERRLOG("Provided target name must be unique.\n"); 281 return NULL; 282 } 283 } 284 285 tgt = calloc(1, sizeof(*tgt)); 286 if (!tgt) { 287 return NULL; 288 } 289 290 snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts->name); 291 292 if (!opts || !opts->max_subsystems) { 293 tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 294 } else { 295 tgt->max_subsystems = opts->max_subsystems; 296 } 297 298 if (!opts) { 299 tgt->crdt[0] = 0; 300 tgt->crdt[1] = 0; 301 tgt->crdt[2] = 0; 302 } else { 303 tgt->crdt[0] = opts->crdt[0]; 304 tgt->crdt[1] = opts->crdt[1]; 305 tgt->crdt[2] = opts->crdt[2]; 306 } 307 308 if (!opts) { 309 tgt->discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY; 310 } else { 311 tgt->discovery_filter = opts->discovery_filter; 312 } 313 314 tgt->discovery_genctr = 0; 315 TAILQ_INIT(&tgt->transports); 316 TAILQ_INIT(&tgt->poll_groups); 317 tgt->num_poll_groups = 0; 318 319 tgt->subsystems = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem *)); 320 if (!tgt->subsystems) { 321 free(tgt); 322 return NULL; 323 } 324 325 pthread_mutex_init(&tgt->mutex, NULL); 326 327 spdk_io_device_register(tgt, 328 nvmf_tgt_create_poll_group, 329 nvmf_tgt_destroy_poll_group, 330 sizeof(struct spdk_nvmf_poll_group), 331 tgt->name); 332 333 tgt->state = NVMF_TGT_RUNNING; 334 335 TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link); 336 337 return tgt; 338 } 339 340 static void 341 _nvmf_tgt_destroy_next_transport(void *ctx) 342 { 343 struct spdk_nvmf_tgt *tgt = ctx; 344 struct spdk_nvmf_transport *transport; 345 346 if (!TAILQ_EMPTY(&tgt->transports)) { 347 transport = TAILQ_FIRST(&tgt->transports); 348 TAILQ_REMOVE(&tgt->transports, transport, link); 349 spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt); 350 } else { 351 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn; 352 void *destroy_cb_arg = tgt->destroy_cb_arg; 353 354 pthread_mutex_destroy(&tgt->mutex); 355 free(tgt); 356 357 if (destroy_cb_fn) { 358 destroy_cb_fn(destroy_cb_arg, 0); 359 } 360 } 361 } 362 363 static void 364 nvmf_tgt_destroy_cb(void *io_device) 365 { 366 struct spdk_nvmf_tgt *tgt = io_device; 367 uint32_t i; 368 int rc; 369 370 if (tgt->subsystems) { 371 for (i = 0; i < tgt->max_subsystems; i++) { 372 if (tgt->subsystems[i]) { 373 nvmf_subsystem_remove_all_listeners(tgt->subsystems[i], true); 374 375 rc = spdk_nvmf_subsystem_destroy(tgt->subsystems[i], nvmf_tgt_destroy_cb, tgt); 376 if (rc) { 377 if (rc == -EINPROGRESS) { 378 /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i 379 * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */ 380 return; 381 } else { 382 SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", tgt->subsystems[i]->subnqn, rc); 383 } 384 } 385 } 386 } 387 free(tgt->subsystems); 388 } 389 390 _nvmf_tgt_destroy_next_transport(tgt); 391 } 392 393 void 394 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 395 spdk_nvmf_tgt_destroy_done_fn cb_fn, 396 void *cb_arg) 397 { 398 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 399 400 tgt->destroy_cb_fn = cb_fn; 401 tgt->destroy_cb_arg = cb_arg; 402 403 TAILQ_REMOVE(&g_nvmf_tgts, tgt, link); 404 405 spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb); 406 } 407 408 const char * 409 spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt) 410 { 411 return tgt->name; 412 } 413 414 struct spdk_nvmf_tgt * 415 spdk_nvmf_get_tgt(const char *name) 416 { 417 struct spdk_nvmf_tgt *tgt; 418 uint32_t num_targets = 0; 419 420 TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) { 421 if (name) { 422 if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) { 423 return tgt; 424 } 425 } 426 num_targets++; 427 } 428 429 /* 430 * special case. If there is only one target and 431 * no name was specified, return the only available 432 * target. If there is more than one target, name must 433 * be specified. 434 */ 435 if (!name && num_targets == 1) { 436 return TAILQ_FIRST(&g_nvmf_tgts); 437 } 438 439 return NULL; 440 } 441 442 struct spdk_nvmf_tgt * 443 spdk_nvmf_get_first_tgt(void) 444 { 445 return TAILQ_FIRST(&g_nvmf_tgts); 446 } 447 448 struct spdk_nvmf_tgt * 449 spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev) 450 { 451 return TAILQ_NEXT(prev, link); 452 } 453 454 static void 455 nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 456 struct spdk_nvmf_subsystem *subsystem) 457 { 458 struct spdk_nvmf_host *host; 459 struct spdk_nvmf_subsystem_listener *listener; 460 const struct spdk_nvme_transport_id *trid; 461 struct spdk_nvmf_ns *ns; 462 struct spdk_nvmf_ns_opts ns_opts; 463 uint32_t max_namespaces; 464 char uuid_str[SPDK_UUID_STRING_LEN]; 465 466 if (spdk_nvmf_subsystem_get_type(subsystem) != SPDK_NVMF_SUBTYPE_NVME) { 467 return; 468 } 469 470 /* { */ 471 spdk_json_write_object_begin(w); 472 spdk_json_write_named_string(w, "method", "nvmf_create_subsystem"); 473 474 /* "params" : { */ 475 spdk_json_write_named_object_begin(w, "params"); 476 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 477 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 478 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 479 spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem)); 480 481 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 482 if (max_namespaces != 0) { 483 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 484 } 485 486 spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem)); 487 spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem)); 488 spdk_json_write_named_bool(w, "ana_reporting", nvmf_subsystem_get_ana_reporting(subsystem)); 489 490 /* } "params" */ 491 spdk_json_write_object_end(w); 492 493 /* } */ 494 spdk_json_write_object_end(w); 495 496 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 497 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 498 499 spdk_json_write_object_begin(w); 500 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 501 502 /* "params" : { */ 503 spdk_json_write_named_object_begin(w, "params"); 504 505 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 506 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 507 508 /* } "params" */ 509 spdk_json_write_object_end(w); 510 511 /* } */ 512 spdk_json_write_object_end(w); 513 } 514 515 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 516 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 517 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 518 519 spdk_json_write_object_begin(w); 520 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 521 522 /* "params" : { */ 523 spdk_json_write_named_object_begin(w, "params"); 524 525 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 526 527 /* "namespace" : { */ 528 spdk_json_write_named_object_begin(w, "namespace"); 529 530 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 531 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 532 533 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 534 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 535 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 536 from_be64(&ns_opts.nguid[8])); 537 } 538 539 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 540 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 541 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 542 } 543 544 if (!spdk_mem_all_zero(&ns_opts.uuid, sizeof(ns_opts.uuid))) { 545 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &ns_opts.uuid); 546 spdk_json_write_named_string(w, "uuid", uuid_str); 547 } 548 549 if (nvmf_subsystem_get_ana_reporting(subsystem)) { 550 spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid); 551 } 552 553 /* "namespace" */ 554 spdk_json_write_object_end(w); 555 556 /* } "params" */ 557 spdk_json_write_object_end(w); 558 559 /* } */ 560 spdk_json_write_object_end(w); 561 } 562 563 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 564 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 565 trid = spdk_nvmf_subsystem_listener_get_trid(listener); 566 567 spdk_json_write_object_begin(w); 568 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 569 570 /* "params" : { */ 571 spdk_json_write_named_object_begin(w, "params"); 572 573 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 574 nvmf_transport_listen_dump_opts(listener->transport, trid, w); 575 576 /* } "params" */ 577 spdk_json_write_object_end(w); 578 579 /* } */ 580 spdk_json_write_object_end(w); 581 } 582 583 } 584 585 void 586 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 587 { 588 struct spdk_nvmf_subsystem *subsystem; 589 struct spdk_nvmf_transport *transport; 590 591 spdk_json_write_object_begin(w); 592 spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems"); 593 594 spdk_json_write_named_object_begin(w, "params"); 595 spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems); 596 spdk_json_write_object_end(w); 597 598 spdk_json_write_object_end(w); 599 600 spdk_json_write_object_begin(w); 601 spdk_json_write_named_string(w, "method", "nvmf_set_crdt"); 602 spdk_json_write_named_object_begin(w, "params"); 603 spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]); 604 spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]); 605 spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]); 606 spdk_json_write_object_end(w); 607 spdk_json_write_object_end(w); 608 609 /* write transports */ 610 TAILQ_FOREACH(transport, &tgt->transports, link) { 611 spdk_json_write_object_begin(w); 612 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 613 nvmf_transport_dump_opts(transport, w, true); 614 spdk_json_write_object_end(w); 615 } 616 617 subsystem = spdk_nvmf_subsystem_get_first(tgt); 618 while (subsystem) { 619 nvmf_write_subsystem_config_json(w, subsystem); 620 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 621 } 622 } 623 624 static void 625 nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts, 626 const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size) 627 { 628 assert(opts); 629 assert(opts_src); 630 631 opts->opts_size = opts_size; 632 633 #define SET_FIELD(field) \ 634 if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \ 635 opts->field = opts_src->field; \ 636 } \ 637 638 SET_FIELD(transport_specific); 639 #undef SET_FIELD 640 641 /* Do not remove this statement, you should always update this statement when you adding a new field, 642 * and do not forget to add the SET_FIELD statement for your added field. */ 643 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 16, "Incorrect size"); 644 } 645 646 void 647 spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size) 648 { 649 struct spdk_nvmf_listen_opts opts_local = {}; 650 651 /* local version of opts should have defaults set here */ 652 653 nvmf_listen_opts_copy(opts, &opts_local, opts_size); 654 } 655 656 int 657 spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid, 658 struct spdk_nvmf_listen_opts *opts) 659 { 660 struct spdk_nvmf_transport *transport; 661 int rc; 662 struct spdk_nvmf_listen_opts opts_local = {}; 663 664 if (!opts) { 665 SPDK_ERRLOG("opts should not be NULL\n"); 666 return -EINVAL; 667 } 668 669 if (!opts->opts_size) { 670 SPDK_ERRLOG("The opts_size in opts structure should not be zero\n"); 671 return -EINVAL; 672 } 673 674 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 675 if (!transport) { 676 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 677 trid->trstring); 678 return -EINVAL; 679 } 680 681 nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size); 682 rc = spdk_nvmf_transport_listen(transport, trid, &opts_local); 683 if (rc < 0) { 684 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 685 } 686 687 return rc; 688 } 689 690 int 691 spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt, 692 struct spdk_nvme_transport_id *trid) 693 { 694 struct spdk_nvmf_transport *transport; 695 int rc; 696 697 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 698 if (!transport) { 699 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 700 trid->trstring); 701 return -EINVAL; 702 } 703 704 rc = spdk_nvmf_transport_stop_listen(transport, trid); 705 if (rc < 0) { 706 SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr); 707 return rc; 708 } 709 return 0; 710 } 711 712 struct spdk_nvmf_tgt_add_transport_ctx { 713 struct spdk_nvmf_tgt *tgt; 714 struct spdk_nvmf_transport *transport; 715 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 716 void *cb_arg; 717 int status; 718 }; 719 720 static void 721 _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status) 722 { 723 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 724 725 ctx->cb_fn(ctx->cb_arg, ctx->status); 726 free(ctx); 727 } 728 729 static void 730 _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i) 731 { 732 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 733 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 734 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 735 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 736 737 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 738 if (tgroup->transport == ctx->transport) { 739 TAILQ_REMOVE(&group->tgroups, tgroup, link); 740 nvmf_transport_poll_group_destroy(tgroup); 741 } 742 } 743 744 spdk_for_each_channel_continue(i, 0); 745 } 746 747 static void 748 _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 749 { 750 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 751 752 if (status) { 753 ctx->status = status; 754 spdk_for_each_channel(ctx->tgt, 755 _nvmf_tgt_remove_transport, 756 ctx, 757 _nvmf_tgt_remove_transport_done); 758 return; 759 } 760 761 ctx->transport->tgt = ctx->tgt; 762 TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link); 763 ctx->cb_fn(ctx->cb_arg, status); 764 free(ctx); 765 } 766 767 static void 768 _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 769 { 770 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 771 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 772 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 773 int rc; 774 775 rc = nvmf_poll_group_add_transport(group, ctx->transport); 776 spdk_for_each_channel_continue(i, rc); 777 } 778 779 void 780 spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 781 struct spdk_nvmf_transport *transport, 782 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 783 void *cb_arg) 784 { 785 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 786 787 SPDK_DTRACE_PROBE2(nvmf_tgt_add_transport, transport, tgt->name); 788 789 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) { 790 cb_fn(cb_arg, -EEXIST); 791 return; /* transport already created */ 792 } 793 794 ctx = calloc(1, sizeof(*ctx)); 795 if (!ctx) { 796 cb_fn(cb_arg, -ENOMEM); 797 return; 798 } 799 800 ctx->tgt = tgt; 801 ctx->transport = transport; 802 ctx->cb_fn = cb_fn; 803 ctx->cb_arg = cb_arg; 804 805 spdk_for_each_channel(tgt, 806 _nvmf_tgt_add_transport, 807 ctx, 808 _nvmf_tgt_add_transport_done); 809 } 810 811 struct nvmf_tgt_pause_ctx { 812 struct spdk_nvmf_tgt *tgt; 813 spdk_nvmf_tgt_pause_polling_cb_fn cb_fn; 814 void *cb_arg; 815 }; 816 817 static void 818 _nvmf_tgt_pause_polling_done(struct spdk_io_channel_iter *i, int status) 819 { 820 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 821 822 ctx->tgt->state = NVMF_TGT_PAUSED; 823 824 ctx->cb_fn(ctx->cb_arg, status); 825 free(ctx); 826 } 827 828 static void 829 _nvmf_tgt_pause_polling(struct spdk_io_channel_iter *i) 830 { 831 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 832 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 833 834 spdk_poller_unregister(&group->poller); 835 836 spdk_for_each_channel_continue(i, 0); 837 } 838 839 int 840 spdk_nvmf_tgt_pause_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_pause_polling_cb_fn cb_fn, 841 void *cb_arg) 842 { 843 struct nvmf_tgt_pause_ctx *ctx; 844 845 SPDK_DTRACE_PROBE2(nvmf_tgt_pause_polling, tgt, tgt->name); 846 847 switch (tgt->state) { 848 case NVMF_TGT_PAUSING: 849 case NVMF_TGT_RESUMING: 850 return -EBUSY; 851 case NVMF_TGT_RUNNING: 852 break; 853 default: 854 return -EINVAL; 855 } 856 857 ctx = calloc(1, sizeof(*ctx)); 858 if (!ctx) { 859 return -ENOMEM; 860 } 861 862 863 tgt->state = NVMF_TGT_PAUSING; 864 865 ctx->tgt = tgt; 866 ctx->cb_fn = cb_fn; 867 ctx->cb_arg = cb_arg; 868 869 spdk_for_each_channel(tgt, 870 _nvmf_tgt_pause_polling, 871 ctx, 872 _nvmf_tgt_pause_polling_done); 873 return 0; 874 } 875 876 static void 877 _nvmf_tgt_resume_polling_done(struct spdk_io_channel_iter *i, int status) 878 { 879 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 880 881 ctx->tgt->state = NVMF_TGT_RUNNING; 882 883 ctx->cb_fn(ctx->cb_arg, status); 884 free(ctx); 885 } 886 887 static void 888 _nvmf_tgt_resume_polling(struct spdk_io_channel_iter *i) 889 { 890 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 891 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 892 893 assert(group->poller == NULL); 894 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 895 896 spdk_for_each_channel_continue(i, 0); 897 } 898 899 int 900 spdk_nvmf_tgt_resume_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_resume_polling_cb_fn cb_fn, 901 void *cb_arg) 902 { 903 struct nvmf_tgt_pause_ctx *ctx; 904 905 SPDK_DTRACE_PROBE2(nvmf_tgt_resume_polling, tgt, tgt->name); 906 907 switch (tgt->state) { 908 case NVMF_TGT_PAUSING: 909 case NVMF_TGT_RESUMING: 910 return -EBUSY; 911 case NVMF_TGT_PAUSED: 912 break; 913 default: 914 return -EINVAL; 915 } 916 917 ctx = calloc(1, sizeof(*ctx)); 918 if (!ctx) { 919 return -ENOMEM; 920 } 921 922 tgt->state = NVMF_TGT_RESUMING; 923 924 ctx->tgt = tgt; 925 ctx->cb_fn = cb_fn; 926 ctx->cb_arg = cb_arg; 927 928 spdk_for_each_channel(tgt, 929 _nvmf_tgt_resume_polling, 930 ctx, 931 _nvmf_tgt_resume_polling_done); 932 return 0; 933 } 934 935 struct spdk_nvmf_subsystem * 936 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 937 { 938 struct spdk_nvmf_subsystem *subsystem; 939 uint32_t sid; 940 941 if (!subnqn) { 942 return NULL; 943 } 944 945 /* Ensure that subnqn is null terminated */ 946 if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 947 SPDK_ERRLOG("Connect SUBNQN is not null terminated\n"); 948 return NULL; 949 } 950 951 for (sid = 0; sid < tgt->max_subsystems; sid++) { 952 subsystem = tgt->subsystems[sid]; 953 if (subsystem == NULL) { 954 continue; 955 } 956 957 if (strcmp(subnqn, subsystem->subnqn) == 0) { 958 return subsystem; 959 } 960 } 961 962 return NULL; 963 } 964 965 struct spdk_nvmf_transport * 966 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name) 967 { 968 struct spdk_nvmf_transport *transport; 969 970 TAILQ_FOREACH(transport, &tgt->transports, link) { 971 if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) { 972 return transport; 973 } 974 } 975 return NULL; 976 } 977 978 struct nvmf_new_qpair_ctx { 979 struct spdk_nvmf_qpair *qpair; 980 struct spdk_nvmf_poll_group *group; 981 }; 982 983 static void 984 _nvmf_poll_group_add(void *_ctx) 985 { 986 struct nvmf_new_qpair_ctx *ctx = _ctx; 987 struct spdk_nvmf_qpair *qpair = ctx->qpair; 988 struct spdk_nvmf_poll_group *group = ctx->group; 989 990 free(_ctx); 991 992 if (spdk_nvmf_poll_group_add(group, qpair) != 0) { 993 SPDK_ERRLOG("Unable to add the qpair to a poll group.\n"); 994 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 995 } 996 } 997 998 void 999 spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair) 1000 { 1001 struct spdk_nvmf_poll_group *group; 1002 struct nvmf_new_qpair_ctx *ctx; 1003 1004 group = spdk_nvmf_get_optimal_poll_group(qpair); 1005 if (group == NULL) { 1006 if (tgt->next_poll_group == NULL) { 1007 tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups); 1008 if (tgt->next_poll_group == NULL) { 1009 SPDK_ERRLOG("No poll groups exist.\n"); 1010 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1011 return; 1012 } 1013 } 1014 group = tgt->next_poll_group; 1015 tgt->next_poll_group = TAILQ_NEXT(group, link); 1016 } 1017 1018 ctx = calloc(1, sizeof(*ctx)); 1019 if (!ctx) { 1020 SPDK_ERRLOG("Unable to send message to poll group.\n"); 1021 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1022 return; 1023 } 1024 1025 ctx->qpair = qpair; 1026 ctx->group = group; 1027 1028 pthread_mutex_lock(&group->mutex); 1029 group->current_unassociated_qpairs++; 1030 pthread_mutex_unlock(&group->mutex); 1031 1032 spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx); 1033 } 1034 1035 struct spdk_nvmf_poll_group * 1036 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 1037 { 1038 struct spdk_io_channel *ch; 1039 1040 ch = spdk_get_io_channel(tgt); 1041 if (!ch) { 1042 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 1043 return NULL; 1044 } 1045 1046 return spdk_io_channel_get_ctx(ch); 1047 } 1048 1049 void 1050 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, 1051 spdk_nvmf_poll_group_destroy_done_fn cb_fn, 1052 void *cb_arg) 1053 { 1054 assert(group->destroy_cb_fn == NULL); 1055 group->destroy_cb_fn = cb_fn; 1056 group->destroy_cb_arg = cb_arg; 1057 1058 /* This function will put the io_channel associated with this poll group */ 1059 nvmf_tgt_destroy_poll_group_qpairs(group); 1060 } 1061 1062 int 1063 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 1064 struct spdk_nvmf_qpair *qpair) 1065 { 1066 int rc = -1; 1067 struct spdk_nvmf_transport_poll_group *tgroup; 1068 1069 TAILQ_INIT(&qpair->outstanding); 1070 qpair->group = group; 1071 qpair->ctrlr = NULL; 1072 qpair->disconnect_started = false; 1073 1074 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1075 if (tgroup->transport == qpair->transport) { 1076 rc = nvmf_transport_poll_group_add(tgroup, qpair); 1077 break; 1078 } 1079 } 1080 1081 /* We add the qpair to the group only it is successfully added into the tgroup */ 1082 if (rc == 0) { 1083 SPDK_DTRACE_PROBE2(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread)); 1084 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 1085 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVE); 1086 } 1087 1088 return rc; 1089 } 1090 1091 static void 1092 _nvmf_ctrlr_destruct(void *ctx) 1093 { 1094 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1095 1096 nvmf_ctrlr_destruct(ctrlr); 1097 } 1098 1099 static void 1100 _nvmf_ctrlr_free_from_qpair(void *ctx) 1101 { 1102 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1103 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 1104 uint32_t count; 1105 1106 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 1107 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1108 if (count == 0) { 1109 assert(!ctrlr->in_destruct); 1110 SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid); 1111 ctrlr->in_destruct = true; 1112 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 1113 } 1114 free(qpair_ctx); 1115 } 1116 1117 static void 1118 _nvmf_transport_qpair_fini_complete(void *cb_ctx) 1119 { 1120 struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx; 1121 struct spdk_nvmf_ctrlr *ctrlr; 1122 /* Store cb args since cb_ctx can be freed in _nvmf_ctrlr_free_from_qpair */ 1123 nvmf_qpair_disconnect_cb cb_fn = qpair_ctx->cb_fn; 1124 void *cb_arg = qpair_ctx->ctx; 1125 struct spdk_thread *cb_thread = qpair_ctx->thread; 1126 1127 ctrlr = qpair_ctx->ctrlr; 1128 SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid); 1129 1130 if (ctrlr) { 1131 if (qpair_ctx->qid == 0) { 1132 /* Admin qpair is removed, so set the pointer to NULL. 1133 * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same 1134 * as controller's thread */ 1135 assert(ctrlr->thread == spdk_get_thread()); 1136 ctrlr->admin_qpair = NULL; 1137 } 1138 /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */ 1139 if (ctrlr->thread) { 1140 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx); 1141 } else { 1142 _nvmf_ctrlr_free_from_qpair(qpair_ctx); 1143 } 1144 } else { 1145 free(qpair_ctx); 1146 } 1147 1148 if (cb_fn) { 1149 spdk_thread_send_msg(cb_thread, cb_fn, cb_arg); 1150 } 1151 } 1152 1153 void 1154 spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair) 1155 { 1156 struct spdk_nvmf_transport_poll_group *tgroup; 1157 int rc; 1158 1159 SPDK_DTRACE_PROBE2(nvmf_poll_group_remove_qpair, qpair, 1160 spdk_thread_get_id(qpair->group->thread)); 1161 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR); 1162 1163 /* Find the tgroup and remove the qpair from the tgroup */ 1164 TAILQ_FOREACH(tgroup, &qpair->group->tgroups, link) { 1165 if (tgroup->transport == qpair->transport) { 1166 rc = nvmf_transport_poll_group_remove(tgroup, qpair); 1167 if (rc && (rc != ENOTSUP)) { 1168 SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n", 1169 qpair, tgroup); 1170 } 1171 break; 1172 } 1173 } 1174 1175 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 1176 qpair->group = NULL; 1177 } 1178 1179 static void 1180 _nvmf_qpair_sgroup_req_clean(struct spdk_nvmf_subsystem_poll_group *sgroup, 1181 const struct spdk_nvmf_qpair *qpair) 1182 { 1183 struct spdk_nvmf_request *req, *tmp; 1184 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1185 if (req->qpair == qpair) { 1186 TAILQ_REMOVE(&sgroup->queued, req, link); 1187 if (nvmf_transport_req_free(req)) { 1188 SPDK_ERRLOG("Transport request free error!\n"); 1189 } 1190 } 1191 } 1192 } 1193 1194 static void 1195 _nvmf_qpair_destroy(void *ctx, int status) 1196 { 1197 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1198 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 1199 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 1200 struct spdk_nvmf_subsystem_poll_group *sgroup; 1201 uint32_t sid; 1202 1203 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 1204 qpair_ctx->qid = qpair->qid; 1205 1206 if (qpair->connect_received) { 1207 if (0 == qpair->qid) { 1208 assert(qpair->group->stat.current_admin_qpairs > 0); 1209 qpair->group->stat.current_admin_qpairs--; 1210 } else { 1211 assert(qpair->group->stat.current_io_qpairs > 0); 1212 qpair->group->stat.current_io_qpairs--; 1213 } 1214 } else { 1215 pthread_mutex_lock(&qpair->group->mutex); 1216 qpair->group->current_unassociated_qpairs--; 1217 pthread_mutex_unlock(&qpair->group->mutex); 1218 } 1219 1220 if (ctrlr) { 1221 sgroup = &qpair->group->sgroups[ctrlr->subsys->id]; 1222 _nvmf_qpair_sgroup_req_clean(sgroup, qpair); 1223 } else { 1224 for (sid = 0; sid < qpair->group->num_sgroups; sid++) { 1225 sgroup = &qpair->group->sgroups[sid]; 1226 assert(sgroup != NULL); 1227 _nvmf_qpair_sgroup_req_clean(sgroup, qpair); 1228 } 1229 } 1230 1231 qpair_ctx->ctrlr = ctrlr; 1232 spdk_nvmf_poll_group_remove(qpair); 1233 nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); 1234 } 1235 1236 static void 1237 _nvmf_qpair_disconnect_msg(void *ctx) 1238 { 1239 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1240 1241 spdk_nvmf_qpair_disconnect(qpair_ctx->qpair, qpair_ctx->cb_fn, qpair_ctx->ctx); 1242 free(ctx); 1243 } 1244 1245 SPDK_LOG_DEPRECATION_REGISTER(spdk_nvmf_qpair_disconnect, "cb_fn and ctx are deprecated", "v23.09", 1246 0); 1247 1248 int 1249 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) 1250 { 1251 struct spdk_nvmf_poll_group *group = qpair->group; 1252 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 1253 1254 if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) { 1255 return -EINPROGRESS; 1256 } 1257 1258 if (cb_fn || ctx) { 1259 SPDK_LOG_DEPRECATED(spdk_nvmf_qpair_disconnect); 1260 } 1261 1262 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 1263 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 1264 nvmf_transport_qpair_fini(qpair, NULL, NULL); 1265 if (cb_fn) { 1266 cb_fn(ctx); 1267 } 1268 return 0; 1269 } 1270 1271 assert(group != NULL); 1272 if (spdk_get_thread() != group->thread) { 1273 /* clear the atomic so we can set it on the next call on the proper thread. */ 1274 __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED); 1275 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1276 if (!qpair_ctx) { 1277 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1278 return -ENOMEM; 1279 } 1280 qpair_ctx->qpair = qpair; 1281 qpair_ctx->cb_fn = cb_fn; 1282 qpair_ctx->thread = group->thread; 1283 qpair_ctx->ctx = ctx; 1284 spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx); 1285 return 0; 1286 } 1287 1288 SPDK_DTRACE_PROBE2(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread)); 1289 assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); 1290 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 1291 1292 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1293 if (!qpair_ctx) { 1294 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1295 return -ENOMEM; 1296 } 1297 1298 qpair_ctx->qpair = qpair; 1299 qpair_ctx->cb_fn = cb_fn; 1300 qpair_ctx->thread = group->thread; 1301 qpair_ctx->ctx = ctx; 1302 1303 /* Check for outstanding I/O */ 1304 if (!TAILQ_EMPTY(&qpair->outstanding)) { 1305 SPDK_DTRACE_PROBE2(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread)); 1306 qpair->state_cb = _nvmf_qpair_destroy; 1307 qpair->state_cb_arg = qpair_ctx; 1308 nvmf_qpair_abort_pending_zcopy_reqs(qpair); 1309 nvmf_qpair_free_aer(qpair); 1310 return 0; 1311 } 1312 1313 _nvmf_qpair_destroy(qpair_ctx, 0); 1314 1315 return 0; 1316 } 1317 1318 int 1319 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 1320 struct spdk_nvme_transport_id *trid) 1321 { 1322 return nvmf_transport_qpair_get_peer_trid(qpair, trid); 1323 } 1324 1325 int 1326 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 1327 struct spdk_nvme_transport_id *trid) 1328 { 1329 return nvmf_transport_qpair_get_local_trid(qpair, trid); 1330 } 1331 1332 int 1333 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 1334 struct spdk_nvme_transport_id *trid) 1335 { 1336 return nvmf_transport_qpair_get_listen_trid(qpair, trid); 1337 } 1338 1339 static int 1340 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1341 struct spdk_nvmf_subsystem *subsystem) 1342 { 1343 struct spdk_nvmf_subsystem_poll_group *sgroup; 1344 uint32_t new_num_ns, old_num_ns; 1345 uint32_t i, j; 1346 struct spdk_nvmf_ns *ns; 1347 struct spdk_nvmf_registrant *reg, *tmp; 1348 struct spdk_io_channel *ch; 1349 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1350 struct spdk_nvmf_ctrlr *ctrlr; 1351 bool ns_changed; 1352 1353 /* Make sure our poll group has memory for this subsystem allocated */ 1354 if (subsystem->id >= group->num_sgroups) { 1355 return -ENOMEM; 1356 } 1357 1358 sgroup = &group->sgroups[subsystem->id]; 1359 1360 /* Make sure the array of namespace information is the correct size */ 1361 new_num_ns = subsystem->max_nsid; 1362 old_num_ns = sgroup->num_ns; 1363 1364 ns_changed = false; 1365 1366 if (old_num_ns == 0) { 1367 if (new_num_ns > 0) { 1368 /* First allocation */ 1369 sgroup->ns_info = calloc(new_num_ns, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1370 if (!sgroup->ns_info) { 1371 return -ENOMEM; 1372 } 1373 } 1374 } else if (new_num_ns > old_num_ns) { 1375 void *buf; 1376 1377 /* Make the array larger */ 1378 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1379 if (!buf) { 1380 return -ENOMEM; 1381 } 1382 1383 sgroup->ns_info = buf; 1384 1385 /* Null out the new namespace information slots */ 1386 for (i = old_num_ns; i < new_num_ns; i++) { 1387 memset(&sgroup->ns_info[i], 0, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1388 } 1389 } else if (new_num_ns < old_num_ns) { 1390 void *buf; 1391 1392 /* Free the extra I/O channels */ 1393 for (i = new_num_ns; i < old_num_ns; i++) { 1394 ns_info = &sgroup->ns_info[i]; 1395 1396 if (ns_info->channel) { 1397 spdk_put_io_channel(ns_info->channel); 1398 ns_info->channel = NULL; 1399 } 1400 } 1401 1402 /* Make the array smaller */ 1403 if (new_num_ns > 0) { 1404 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1405 if (!buf) { 1406 return -ENOMEM; 1407 } 1408 sgroup->ns_info = buf; 1409 } else { 1410 free(sgroup->ns_info); 1411 sgroup->ns_info = NULL; 1412 } 1413 } 1414 1415 sgroup->num_ns = new_num_ns; 1416 1417 /* Detect bdevs that were added or removed */ 1418 for (i = 0; i < sgroup->num_ns; i++) { 1419 ns = subsystem->ns[i]; 1420 ns_info = &sgroup->ns_info[i]; 1421 ch = ns_info->channel; 1422 1423 if (ns == NULL && ch == NULL) { 1424 /* Both NULL. Leave empty */ 1425 } else if (ns == NULL && ch != NULL) { 1426 /* There was a channel here, but the namespace is gone. */ 1427 ns_changed = true; 1428 spdk_put_io_channel(ch); 1429 ns_info->channel = NULL; 1430 } else if (ns != NULL && ch == NULL) { 1431 /* A namespace appeared but there is no channel yet */ 1432 ns_changed = true; 1433 ch = spdk_bdev_get_io_channel(ns->desc); 1434 if (ch == NULL) { 1435 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1436 return -ENOMEM; 1437 } 1438 ns_info->channel = ch; 1439 } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) { 1440 /* A namespace was here before, but was replaced by a new one. */ 1441 ns_changed = true; 1442 spdk_put_io_channel(ns_info->channel); 1443 memset(ns_info, 0, sizeof(*ns_info)); 1444 1445 ch = spdk_bdev_get_io_channel(ns->desc); 1446 if (ch == NULL) { 1447 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1448 return -ENOMEM; 1449 } 1450 ns_info->channel = ch; 1451 } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) { 1452 /* Namespace is still there but size has changed */ 1453 SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u," 1454 " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n", 1455 subsystem->id, 1456 ns->nsid, 1457 group, 1458 ns_info->num_blocks, 1459 spdk_bdev_get_num_blocks(ns->bdev)); 1460 ns_changed = true; 1461 } 1462 1463 if (ns == NULL) { 1464 memset(ns_info, 0, sizeof(*ns_info)); 1465 } else { 1466 ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev); 1467 ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev); 1468 ns_info->crkey = ns->crkey; 1469 ns_info->rtype = ns->rtype; 1470 if (ns->holder) { 1471 ns_info->holder_id = ns->holder->hostid; 1472 } 1473 1474 memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid)); 1475 j = 0; 1476 TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { 1477 if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) { 1478 SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS); 1479 return -EINVAL; 1480 } 1481 ns_info->reg_hostid[j++] = reg->hostid; 1482 } 1483 } 1484 } 1485 1486 if (ns_changed) { 1487 TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) { 1488 /* It is possible that a ctrlr was added but the admin_qpair hasn't been 1489 * assigned yet. 1490 */ 1491 if (!ctrlr->admin_qpair) { 1492 continue; 1493 } 1494 if (ctrlr->admin_qpair->group == group) { 1495 nvmf_ctrlr_async_event_ns_notice(ctrlr); 1496 nvmf_ctrlr_async_event_ana_change_notice(ctrlr); 1497 } 1498 } 1499 } 1500 1501 return 0; 1502 } 1503 1504 int 1505 nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1506 struct spdk_nvmf_subsystem *subsystem) 1507 { 1508 return poll_group_update_subsystem(group, subsystem); 1509 } 1510 1511 int 1512 nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 1513 struct spdk_nvmf_subsystem *subsystem, 1514 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1515 { 1516 int rc = 0; 1517 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 1518 uint32_t i; 1519 1520 TAILQ_INIT(&sgroup->queued); 1521 1522 rc = poll_group_update_subsystem(group, subsystem); 1523 if (rc) { 1524 nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); 1525 goto fini; 1526 } 1527 1528 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1529 1530 for (i = 0; i < sgroup->num_ns; i++) { 1531 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1532 } 1533 1534 fini: 1535 if (cb_fn) { 1536 cb_fn(cb_arg, rc); 1537 } 1538 1539 SPDK_DTRACE_PROBE2(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread), 1540 subsystem->subnqn); 1541 1542 return rc; 1543 } 1544 1545 static void 1546 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 1547 { 1548 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1549 struct spdk_nvmf_subsystem *subsystem; 1550 struct spdk_nvmf_poll_group *group; 1551 struct spdk_nvmf_subsystem_poll_group *sgroup; 1552 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 1553 void *cpl_ctx = NULL; 1554 uint32_t nsid; 1555 1556 group = qpair_ctx->group; 1557 subsystem = qpair_ctx->subsystem; 1558 cpl_fn = qpair_ctx->cpl_fn; 1559 cpl_ctx = qpair_ctx->cpl_ctx; 1560 sgroup = &group->sgroups[subsystem->id]; 1561 1562 if (status) { 1563 goto fini; 1564 } 1565 1566 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 1567 if (sgroup->ns_info[nsid].channel) { 1568 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 1569 sgroup->ns_info[nsid].channel = NULL; 1570 } 1571 } 1572 1573 sgroup->num_ns = 0; 1574 free(sgroup->ns_info); 1575 sgroup->ns_info = NULL; 1576 fini: 1577 free(qpair_ctx); 1578 if (cpl_fn) { 1579 cpl_fn(cpl_ctx, status); 1580 } 1581 } 1582 1583 static void nvmf_poll_group_remove_subsystem_msg(void *ctx); 1584 1585 static void 1586 nvmf_poll_group_remove_subsystem_msg(void *ctx) 1587 { 1588 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 1589 struct spdk_nvmf_subsystem *subsystem; 1590 struct spdk_nvmf_poll_group *group; 1591 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1592 bool qpairs_found = false; 1593 int rc = 0; 1594 1595 group = qpair_ctx->group; 1596 subsystem = qpair_ctx->subsystem; 1597 1598 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 1599 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1600 qpairs_found = true; 1601 rc = spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1602 if (rc && rc != -EINPROGRESS) { 1603 break; 1604 } 1605 } 1606 } 1607 1608 if (!qpairs_found) { 1609 _nvmf_poll_group_remove_subsystem_cb(ctx, 0); 1610 return; 1611 } 1612 1613 /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */ 1614 spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx); 1615 } 1616 1617 void 1618 nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1619 struct spdk_nvmf_subsystem *subsystem, 1620 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1621 { 1622 struct spdk_nvmf_subsystem_poll_group *sgroup; 1623 struct nvmf_qpair_disconnect_many_ctx *ctx; 1624 uint32_t i; 1625 1626 SPDK_DTRACE_PROBE3(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread), 1627 subsystem->subnqn); 1628 1629 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1630 if (!ctx) { 1631 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1632 if (cb_fn) { 1633 cb_fn(cb_arg, -1); 1634 } 1635 return; 1636 } 1637 1638 ctx->group = group; 1639 ctx->subsystem = subsystem; 1640 ctx->cpl_fn = cb_fn; 1641 ctx->cpl_ctx = cb_arg; 1642 1643 sgroup = &group->sgroups[subsystem->id]; 1644 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1645 1646 for (i = 0; i < sgroup->num_ns; i++) { 1647 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1648 } 1649 1650 nvmf_poll_group_remove_subsystem_msg(ctx); 1651 } 1652 1653 void 1654 nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1655 struct spdk_nvmf_subsystem *subsystem, 1656 uint32_t nsid, 1657 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1658 { 1659 struct spdk_nvmf_subsystem_poll_group *sgroup; 1660 struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL; 1661 int rc = 0; 1662 uint32_t i; 1663 1664 if (subsystem->id >= group->num_sgroups) { 1665 rc = -1; 1666 goto fini; 1667 } 1668 1669 sgroup = &group->sgroups[subsystem->id]; 1670 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) { 1671 goto fini; 1672 } 1673 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1674 1675 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1676 for (i = 0; i < sgroup->num_ns; i++) { 1677 ns_info = &sgroup->ns_info[i]; 1678 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1679 } 1680 } else { 1681 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 1682 if (nsid - 1 < sgroup->num_ns) { 1683 ns_info = &sgroup->ns_info[nsid - 1]; 1684 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1685 } 1686 } 1687 1688 if (sgroup->mgmt_io_outstanding > 0) { 1689 assert(sgroup->cb_fn == NULL); 1690 sgroup->cb_fn = cb_fn; 1691 assert(sgroup->cb_arg == NULL); 1692 sgroup->cb_arg = cb_arg; 1693 return; 1694 } 1695 1696 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1697 for (i = 0; i < sgroup->num_ns; i++) { 1698 ns_info = &sgroup->ns_info[i]; 1699 1700 if (ns_info->io_outstanding > 0) { 1701 assert(sgroup->cb_fn == NULL); 1702 sgroup->cb_fn = cb_fn; 1703 assert(sgroup->cb_arg == NULL); 1704 sgroup->cb_arg = cb_arg; 1705 return; 1706 } 1707 } 1708 } else { 1709 if (ns_info != NULL && ns_info->io_outstanding > 0) { 1710 assert(sgroup->cb_fn == NULL); 1711 sgroup->cb_fn = cb_fn; 1712 assert(sgroup->cb_arg == NULL); 1713 sgroup->cb_arg = cb_arg; 1714 return; 1715 } 1716 } 1717 1718 assert(sgroup->mgmt_io_outstanding == 0); 1719 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1720 fini: 1721 if (cb_fn) { 1722 cb_fn(cb_arg, rc); 1723 } 1724 } 1725 1726 void 1727 nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1728 struct spdk_nvmf_subsystem *subsystem, 1729 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1730 { 1731 struct spdk_nvmf_request *req, *tmp; 1732 struct spdk_nvmf_subsystem_poll_group *sgroup; 1733 int rc = 0; 1734 uint32_t i; 1735 1736 if (subsystem->id >= group->num_sgroups) { 1737 rc = -1; 1738 goto fini; 1739 } 1740 1741 sgroup = &group->sgroups[subsystem->id]; 1742 1743 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) { 1744 goto fini; 1745 } 1746 1747 rc = poll_group_update_subsystem(group, subsystem); 1748 if (rc) { 1749 goto fini; 1750 } 1751 1752 for (i = 0; i < sgroup->num_ns; i++) { 1753 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1754 } 1755 1756 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1757 1758 /* Release all queued requests */ 1759 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1760 TAILQ_REMOVE(&sgroup->queued, req, link); 1761 if (spdk_nvmf_request_using_zcopy(req)) { 1762 spdk_nvmf_request_zcopy_start(req); 1763 } else { 1764 spdk_nvmf_request_exec(req); 1765 } 1766 1767 } 1768 fini: 1769 if (cb_fn) { 1770 cb_fn(cb_arg, rc); 1771 } 1772 } 1773 1774 1775 struct spdk_nvmf_poll_group * 1776 spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 1777 { 1778 struct spdk_nvmf_transport_poll_group *tgroup; 1779 1780 tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair); 1781 1782 if (tgroup == NULL) { 1783 return NULL; 1784 } 1785 1786 return tgroup->group; 1787 } 1788 1789 void 1790 spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w) 1791 { 1792 struct spdk_nvmf_transport_poll_group *tgroup; 1793 1794 spdk_json_write_object_begin(w); 1795 1796 spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread())); 1797 spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs); 1798 spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs); 1799 spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs); 1800 spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs); 1801 spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io); 1802 spdk_json_write_named_uint64(w, "completed_nvme_io", group->stat.completed_nvme_io); 1803 1804 spdk_json_write_named_array_begin(w, "transports"); 1805 1806 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1807 spdk_json_write_object_begin(w); 1808 /* 1809 * The trtype field intentionally contains a transport name as this is more informative. 1810 * The field has not been renamed for backward compatibility. 1811 */ 1812 spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport)); 1813 1814 if (tgroup->transport->ops->poll_group_dump_stat) { 1815 tgroup->transport->ops->poll_group_dump_stat(tgroup, w); 1816 } 1817 1818 spdk_json_write_object_end(w); 1819 } 1820 1821 spdk_json_write_array_end(w); 1822 spdk_json_write_object_end(w); 1823 } 1824