1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. All rights reserved. 3 * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/bdev.h" 10 #include "spdk/bit_array.h" 11 #include "spdk/thread.h" 12 #include "spdk/nvmf.h" 13 #include "spdk/endian.h" 14 #include "spdk/string.h" 15 #include "spdk/log.h" 16 #include "spdk_internal/usdt.h" 17 18 #include "nvmf_internal.h" 19 #include "transport.h" 20 21 SPDK_LOG_REGISTER_COMPONENT(nvmf) 22 23 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 24 25 static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts); 26 27 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 28 29 /* supplied to a single call to nvmf_qpair_disconnect */ 30 struct nvmf_qpair_disconnect_ctx { 31 struct spdk_nvmf_qpair *qpair; 32 struct spdk_nvmf_ctrlr *ctrlr; 33 nvmf_qpair_disconnect_cb cb_fn; 34 struct spdk_thread *thread; 35 void *ctx; 36 uint16_t qid; 37 }; 38 39 /* 40 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 41 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 42 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 43 */ 44 struct nvmf_qpair_disconnect_many_ctx { 45 struct spdk_nvmf_subsystem *subsystem; 46 struct spdk_nvmf_poll_group *group; 47 spdk_nvmf_poll_group_mod_done cpl_fn; 48 void *cpl_ctx; 49 uint32_t count; 50 }; 51 52 static void 53 nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 54 enum spdk_nvmf_qpair_state state) 55 { 56 assert(qpair != NULL); 57 assert(qpair->group->thread == spdk_get_thread()); 58 59 qpair->state = state; 60 } 61 62 static int 63 nvmf_poll_group_poll(void *ctx) 64 { 65 struct spdk_nvmf_poll_group *group = ctx; 66 int rc; 67 int count = 0; 68 struct spdk_nvmf_transport_poll_group *tgroup; 69 70 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 71 rc = nvmf_transport_poll_group_poll(tgroup); 72 if (rc < 0) { 73 return SPDK_POLLER_BUSY; 74 } 75 count += rc; 76 } 77 78 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 79 } 80 81 /* 82 * Reset and clean up the poll group (I/O channel code will actually free the 83 * group). 84 */ 85 static void 86 nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group) 87 { 88 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 89 struct spdk_nvmf_subsystem_poll_group *sgroup; 90 uint32_t sid, nsid; 91 92 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 93 TAILQ_REMOVE(&group->tgroups, tgroup, link); 94 nvmf_transport_poll_group_destroy(tgroup); 95 } 96 97 for (sid = 0; sid < group->num_sgroups; sid++) { 98 sgroup = &group->sgroups[sid]; 99 100 assert(sgroup != NULL); 101 102 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 103 if (sgroup->ns_info[nsid].channel) { 104 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 105 sgroup->ns_info[nsid].channel = NULL; 106 } 107 } 108 109 free(sgroup->ns_info); 110 } 111 112 free(group->sgroups); 113 114 spdk_poller_unregister(&group->poller); 115 116 if (group->destroy_cb_fn) { 117 group->destroy_cb_fn(group->destroy_cb_arg, 0); 118 } 119 } 120 121 /* 122 * Callback to unregister a poll group from the target, and clean up its state. 123 */ 124 static void 125 nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 126 { 127 struct spdk_nvmf_tgt *tgt = io_device; 128 struct spdk_nvmf_poll_group *group = ctx_buf; 129 130 SPDK_DTRACE_PROBE1(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread)); 131 132 pthread_mutex_lock(&tgt->mutex); 133 TAILQ_REMOVE(&tgt->poll_groups, group, link); 134 pthread_mutex_unlock(&tgt->mutex); 135 136 nvmf_tgt_cleanup_poll_group(group); 137 } 138 139 static int 140 nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 141 struct spdk_nvmf_transport *transport) 142 { 143 struct spdk_nvmf_transport_poll_group *tgroup; 144 145 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 146 if (tgroup->transport == transport) { 147 /* Transport already in the poll group */ 148 return 0; 149 } 150 } 151 152 tgroup = nvmf_transport_poll_group_create(transport, group); 153 if (!tgroup) { 154 SPDK_ERRLOG("Unable to create poll group for transport\n"); 155 return -1; 156 } 157 SPDK_DTRACE_PROBE2(nvmf_transport_poll_group_create, transport, spdk_thread_get_id(group->thread)); 158 159 tgroup->group = group; 160 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 161 162 return 0; 163 } 164 165 static int 166 nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 167 { 168 struct spdk_nvmf_tgt *tgt = io_device; 169 struct spdk_nvmf_poll_group *group = ctx_buf; 170 struct spdk_nvmf_transport *transport; 171 struct spdk_thread *thread = spdk_get_thread(); 172 uint32_t sid; 173 int rc; 174 175 TAILQ_INIT(&group->tgroups); 176 TAILQ_INIT(&group->qpairs); 177 group->thread = thread; 178 179 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 180 181 SPDK_DTRACE_PROBE1(nvmf_create_poll_group, spdk_thread_get_id(thread)); 182 183 TAILQ_FOREACH(transport, &tgt->transports, link) { 184 rc = nvmf_poll_group_add_transport(group, transport); 185 if (rc != 0) { 186 nvmf_tgt_cleanup_poll_group(group); 187 return rc; 188 } 189 } 190 191 group->num_sgroups = tgt->max_subsystems; 192 group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 193 if (!group->sgroups) { 194 nvmf_tgt_cleanup_poll_group(group); 195 return -ENOMEM; 196 } 197 198 for (sid = 0; sid < tgt->max_subsystems; sid++) { 199 struct spdk_nvmf_subsystem *subsystem; 200 201 subsystem = tgt->subsystems[sid]; 202 if (!subsystem) { 203 continue; 204 } 205 206 if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { 207 nvmf_tgt_cleanup_poll_group(group); 208 return -1; 209 } 210 } 211 212 pthread_mutex_lock(&tgt->mutex); 213 TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link); 214 pthread_mutex_unlock(&tgt->mutex); 215 216 return 0; 217 } 218 219 static void 220 _nvmf_tgt_disconnect_next_qpair(void *ctx) 221 { 222 struct spdk_nvmf_qpair *qpair; 223 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 224 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 225 struct spdk_io_channel *ch; 226 int rc = 0; 227 228 qpair = TAILQ_FIRST(&group->qpairs); 229 230 if (qpair) { 231 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_tgt_disconnect_next_qpair, ctx); 232 } 233 234 if (!qpair || rc != 0) { 235 /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */ 236 ch = spdk_io_channel_from_ctx(group); 237 spdk_put_io_channel(ch); 238 free(qpair_ctx); 239 } 240 } 241 242 static void 243 nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 244 { 245 struct nvmf_qpair_disconnect_many_ctx *ctx; 246 247 SPDK_DTRACE_PROBE1(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread)); 248 249 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 250 if (!ctx) { 251 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 252 return; 253 } 254 255 ctx->group = group; 256 _nvmf_tgt_disconnect_next_qpair(ctx); 257 } 258 259 struct spdk_nvmf_tgt * 260 spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *opts) 261 { 262 struct spdk_nvmf_tgt *tgt, *tmp_tgt; 263 264 if (strnlen(opts->name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) { 265 SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH); 266 return NULL; 267 } 268 269 TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) { 270 if (!strncmp(opts->name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) { 271 SPDK_ERRLOG("Provided target name must be unique.\n"); 272 return NULL; 273 } 274 } 275 276 tgt = calloc(1, sizeof(*tgt)); 277 if (!tgt) { 278 return NULL; 279 } 280 281 snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts->name); 282 283 if (!opts || !opts->max_subsystems) { 284 tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 285 } else { 286 tgt->max_subsystems = opts->max_subsystems; 287 } 288 289 if (!opts) { 290 tgt->crdt[0] = 0; 291 tgt->crdt[1] = 0; 292 tgt->crdt[2] = 0; 293 } else { 294 tgt->crdt[0] = opts->crdt[0]; 295 tgt->crdt[1] = opts->crdt[1]; 296 tgt->crdt[2] = opts->crdt[2]; 297 } 298 299 if (!opts) { 300 tgt->discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY; 301 } else { 302 tgt->discovery_filter = opts->discovery_filter; 303 } 304 305 tgt->discovery_genctr = 0; 306 TAILQ_INIT(&tgt->transports); 307 TAILQ_INIT(&tgt->poll_groups); 308 309 tgt->subsystems = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem *)); 310 if (!tgt->subsystems) { 311 free(tgt); 312 return NULL; 313 } 314 315 pthread_mutex_init(&tgt->mutex, NULL); 316 317 spdk_io_device_register(tgt, 318 nvmf_tgt_create_poll_group, 319 nvmf_tgt_destroy_poll_group, 320 sizeof(struct spdk_nvmf_poll_group), 321 tgt->name); 322 323 TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link); 324 325 return tgt; 326 } 327 328 static void 329 _nvmf_tgt_destroy_next_transport(void *ctx) 330 { 331 struct spdk_nvmf_tgt *tgt = ctx; 332 struct spdk_nvmf_transport *transport; 333 334 if (!TAILQ_EMPTY(&tgt->transports)) { 335 transport = TAILQ_FIRST(&tgt->transports); 336 TAILQ_REMOVE(&tgt->transports, transport, link); 337 spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt); 338 } else { 339 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn; 340 void *destroy_cb_arg = tgt->destroy_cb_arg; 341 342 pthread_mutex_destroy(&tgt->mutex); 343 free(tgt); 344 345 if (destroy_cb_fn) { 346 destroy_cb_fn(destroy_cb_arg, 0); 347 } 348 } 349 } 350 351 static void 352 nvmf_tgt_destroy_cb(void *io_device) 353 { 354 struct spdk_nvmf_tgt *tgt = io_device; 355 uint32_t i; 356 int rc; 357 358 if (tgt->subsystems) { 359 for (i = 0; i < tgt->max_subsystems; i++) { 360 if (tgt->subsystems[i]) { 361 nvmf_subsystem_remove_all_listeners(tgt->subsystems[i], true); 362 363 rc = spdk_nvmf_subsystem_destroy(tgt->subsystems[i], nvmf_tgt_destroy_cb, tgt); 364 if (rc) { 365 if (rc == -EINPROGRESS) { 366 /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i 367 * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */ 368 return; 369 } else { 370 SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", tgt->subsystems[i]->subnqn, rc); 371 } 372 } 373 } 374 } 375 free(tgt->subsystems); 376 } 377 378 _nvmf_tgt_destroy_next_transport(tgt); 379 } 380 381 void 382 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 383 spdk_nvmf_tgt_destroy_done_fn cb_fn, 384 void *cb_arg) 385 { 386 tgt->destroy_cb_fn = cb_fn; 387 tgt->destroy_cb_arg = cb_arg; 388 389 TAILQ_REMOVE(&g_nvmf_tgts, tgt, link); 390 391 spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb); 392 } 393 394 const char * 395 spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt) 396 { 397 return tgt->name; 398 } 399 400 struct spdk_nvmf_tgt * 401 spdk_nvmf_get_tgt(const char *name) 402 { 403 struct spdk_nvmf_tgt *tgt; 404 uint32_t num_targets = 0; 405 406 TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) { 407 if (name) { 408 if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) { 409 return tgt; 410 } 411 } 412 num_targets++; 413 } 414 415 /* 416 * special case. If there is only one target and 417 * no name was specified, return the only available 418 * target. If there is more than one target, name must 419 * be specified. 420 */ 421 if (!name && num_targets == 1) { 422 return TAILQ_FIRST(&g_nvmf_tgts); 423 } 424 425 return NULL; 426 } 427 428 struct spdk_nvmf_tgt * 429 spdk_nvmf_get_first_tgt(void) 430 { 431 return TAILQ_FIRST(&g_nvmf_tgts); 432 } 433 434 struct spdk_nvmf_tgt * 435 spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev) 436 { 437 return TAILQ_NEXT(prev, link); 438 } 439 440 static void 441 nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 442 struct spdk_nvmf_subsystem *subsystem) 443 { 444 struct spdk_nvmf_host *host; 445 struct spdk_nvmf_subsystem_listener *listener; 446 const struct spdk_nvme_transport_id *trid; 447 struct spdk_nvmf_ns *ns; 448 struct spdk_nvmf_ns_opts ns_opts; 449 uint32_t max_namespaces; 450 char uuid_str[SPDK_UUID_STRING_LEN]; 451 452 if (spdk_nvmf_subsystem_get_type(subsystem) != SPDK_NVMF_SUBTYPE_NVME) { 453 return; 454 } 455 456 /* { */ 457 spdk_json_write_object_begin(w); 458 spdk_json_write_named_string(w, "method", "nvmf_create_subsystem"); 459 460 /* "params" : { */ 461 spdk_json_write_named_object_begin(w, "params"); 462 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 463 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 464 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 465 spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem)); 466 467 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 468 if (max_namespaces != 0) { 469 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 470 } 471 472 spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem)); 473 spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem)); 474 spdk_json_write_named_bool(w, "ana_reporting", nvmf_subsystem_get_ana_reporting(subsystem)); 475 476 /* } "params" */ 477 spdk_json_write_object_end(w); 478 479 /* } */ 480 spdk_json_write_object_end(w); 481 482 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 483 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 484 trid = spdk_nvmf_subsystem_listener_get_trid(listener); 485 486 spdk_json_write_object_begin(w); 487 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 488 489 /* "params" : { */ 490 spdk_json_write_named_object_begin(w, "params"); 491 492 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 493 nvmf_transport_listen_dump_opts(listener->transport, trid, w); 494 495 /* } "params" */ 496 spdk_json_write_object_end(w); 497 498 /* } */ 499 spdk_json_write_object_end(w); 500 } 501 502 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 503 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 504 505 spdk_json_write_object_begin(w); 506 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 507 508 /* "params" : { */ 509 spdk_json_write_named_object_begin(w, "params"); 510 511 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 512 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 513 514 /* } "params" */ 515 spdk_json_write_object_end(w); 516 517 /* } */ 518 spdk_json_write_object_end(w); 519 } 520 521 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 522 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 523 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 524 525 spdk_json_write_object_begin(w); 526 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 527 528 /* "params" : { */ 529 spdk_json_write_named_object_begin(w, "params"); 530 531 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 532 533 /* "namespace" : { */ 534 spdk_json_write_named_object_begin(w, "namespace"); 535 536 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 537 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 538 539 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 540 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 541 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 542 from_be64(&ns_opts.nguid[8])); 543 } 544 545 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 546 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 547 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 548 } 549 550 if (!spdk_mem_all_zero(&ns_opts.uuid, sizeof(ns_opts.uuid))) { 551 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &ns_opts.uuid); 552 spdk_json_write_named_string(w, "uuid", uuid_str); 553 } 554 555 if (nvmf_subsystem_get_ana_reporting(subsystem)) { 556 spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid); 557 } 558 559 /* "namespace" */ 560 spdk_json_write_object_end(w); 561 562 /* } "params" */ 563 spdk_json_write_object_end(w); 564 565 /* } */ 566 spdk_json_write_object_end(w); 567 } 568 } 569 570 void 571 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 572 { 573 struct spdk_nvmf_subsystem *subsystem; 574 struct spdk_nvmf_transport *transport; 575 576 spdk_json_write_object_begin(w); 577 spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems"); 578 579 spdk_json_write_named_object_begin(w, "params"); 580 spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems); 581 spdk_json_write_object_end(w); 582 583 spdk_json_write_object_end(w); 584 585 spdk_json_write_object_begin(w); 586 spdk_json_write_named_string(w, "method", "nvmf_set_crdt"); 587 spdk_json_write_named_object_begin(w, "params"); 588 spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]); 589 spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]); 590 spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]); 591 spdk_json_write_object_end(w); 592 spdk_json_write_object_end(w); 593 594 /* write transports */ 595 TAILQ_FOREACH(transport, &tgt->transports, link) { 596 spdk_json_write_object_begin(w); 597 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 598 nvmf_transport_dump_opts(transport, w, true); 599 spdk_json_write_object_end(w); 600 } 601 602 subsystem = spdk_nvmf_subsystem_get_first(tgt); 603 while (subsystem) { 604 nvmf_write_subsystem_config_json(w, subsystem); 605 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 606 } 607 } 608 609 static void 610 nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts, 611 const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size) 612 { 613 assert(opts); 614 assert(opts_src); 615 616 opts->opts_size = opts_size; 617 618 #define SET_FIELD(field) \ 619 if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \ 620 opts->field = opts_src->field; \ 621 } \ 622 623 SET_FIELD(transport_specific); 624 #undef SET_FIELD 625 626 /* Do not remove this statement, you should always update this statement when you adding a new field, 627 * and do not forget to add the SET_FIELD statement for your added field. */ 628 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 16, "Incorrect size"); 629 } 630 631 void 632 spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size) 633 { 634 struct spdk_nvmf_listen_opts opts_local = {}; 635 636 /* local version of opts should have defaults set here */ 637 638 nvmf_listen_opts_copy(opts, &opts_local, opts_size); 639 } 640 641 int 642 spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid, 643 struct spdk_nvmf_listen_opts *opts) 644 { 645 struct spdk_nvmf_transport *transport; 646 int rc; 647 struct spdk_nvmf_listen_opts opts_local = {}; 648 649 if (!opts) { 650 SPDK_ERRLOG("opts should not be NULL\n"); 651 return -EINVAL; 652 } 653 654 if (!opts->opts_size) { 655 SPDK_ERRLOG("The opts_size in opts structure should not be zero\n"); 656 return -EINVAL; 657 } 658 659 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 660 if (!transport) { 661 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 662 trid->trstring); 663 return -EINVAL; 664 } 665 666 nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size); 667 rc = spdk_nvmf_transport_listen(transport, trid, &opts_local); 668 if (rc < 0) { 669 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 670 } 671 672 return rc; 673 } 674 675 int 676 spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt, 677 struct spdk_nvme_transport_id *trid) 678 { 679 struct spdk_nvmf_transport *transport; 680 int rc; 681 682 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 683 if (!transport) { 684 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 685 trid->trstring); 686 return -EINVAL; 687 } 688 689 rc = spdk_nvmf_transport_stop_listen(transport, trid); 690 if (rc < 0) { 691 SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr); 692 return rc; 693 } 694 return 0; 695 } 696 697 struct spdk_nvmf_tgt_add_transport_ctx { 698 struct spdk_nvmf_tgt *tgt; 699 struct spdk_nvmf_transport *transport; 700 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 701 void *cb_arg; 702 int status; 703 }; 704 705 static void 706 _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status) 707 { 708 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 709 710 ctx->cb_fn(ctx->cb_arg, ctx->status); 711 free(ctx); 712 } 713 714 static void 715 _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i) 716 { 717 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 718 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 719 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 720 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 721 722 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 723 if (tgroup->transport == ctx->transport) { 724 TAILQ_REMOVE(&group->tgroups, tgroup, link); 725 nvmf_transport_poll_group_destroy(tgroup); 726 } 727 } 728 729 spdk_for_each_channel_continue(i, 0); 730 } 731 732 static void 733 _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 734 { 735 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 736 737 if (status) { 738 ctx->status = status; 739 spdk_for_each_channel(ctx->tgt, 740 _nvmf_tgt_remove_transport, 741 ctx, 742 _nvmf_tgt_remove_transport_done); 743 return; 744 } 745 746 ctx->transport->tgt = ctx->tgt; 747 TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link); 748 ctx->cb_fn(ctx->cb_arg, status); 749 free(ctx); 750 } 751 752 static void 753 _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 754 { 755 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 756 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 757 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 758 int rc; 759 760 rc = nvmf_poll_group_add_transport(group, ctx->transport); 761 spdk_for_each_channel_continue(i, rc); 762 } 763 764 void 765 spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 766 struct spdk_nvmf_transport *transport, 767 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 768 void *cb_arg) 769 { 770 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 771 772 SPDK_DTRACE_PROBE2(nvmf_tgt_add_transport, transport, tgt->name); 773 774 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) { 775 cb_fn(cb_arg, -EEXIST); 776 return; /* transport already created */ 777 } 778 779 ctx = calloc(1, sizeof(*ctx)); 780 if (!ctx) { 781 cb_fn(cb_arg, -ENOMEM); 782 return; 783 } 784 785 ctx->tgt = tgt; 786 ctx->transport = transport; 787 ctx->cb_fn = cb_fn; 788 ctx->cb_arg = cb_arg; 789 790 spdk_for_each_channel(tgt, 791 _nvmf_tgt_add_transport, 792 ctx, 793 _nvmf_tgt_add_transport_done); 794 } 795 796 struct spdk_nvmf_subsystem * 797 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 798 { 799 struct spdk_nvmf_subsystem *subsystem; 800 uint32_t sid; 801 802 if (!subnqn) { 803 return NULL; 804 } 805 806 /* Ensure that subnqn is null terminated */ 807 if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 808 SPDK_ERRLOG("Connect SUBNQN is not null terminated\n"); 809 return NULL; 810 } 811 812 for (sid = 0; sid < tgt->max_subsystems; sid++) { 813 subsystem = tgt->subsystems[sid]; 814 if (subsystem == NULL) { 815 continue; 816 } 817 818 if (strcmp(subnqn, subsystem->subnqn) == 0) { 819 return subsystem; 820 } 821 } 822 823 return NULL; 824 } 825 826 struct spdk_nvmf_transport * 827 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name) 828 { 829 struct spdk_nvmf_transport *transport; 830 831 TAILQ_FOREACH(transport, &tgt->transports, link) { 832 if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) { 833 return transport; 834 } 835 } 836 return NULL; 837 } 838 839 struct nvmf_new_qpair_ctx { 840 struct spdk_nvmf_qpair *qpair; 841 struct spdk_nvmf_poll_group *group; 842 }; 843 844 static void 845 _nvmf_poll_group_add(void *_ctx) 846 { 847 struct nvmf_new_qpair_ctx *ctx = _ctx; 848 struct spdk_nvmf_qpair *qpair = ctx->qpair; 849 struct spdk_nvmf_poll_group *group = ctx->group; 850 851 free(_ctx); 852 853 if (spdk_nvmf_poll_group_add(group, qpair) != 0) { 854 SPDK_ERRLOG("Unable to add the qpair to a poll group.\n"); 855 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 856 } 857 } 858 859 void 860 spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair) 861 { 862 struct spdk_nvmf_poll_group *group; 863 struct nvmf_new_qpair_ctx *ctx; 864 865 group = spdk_nvmf_get_optimal_poll_group(qpair); 866 if (group == NULL) { 867 if (tgt->next_poll_group == NULL) { 868 tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups); 869 if (tgt->next_poll_group == NULL) { 870 SPDK_ERRLOG("No poll groups exist.\n"); 871 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 872 return; 873 } 874 } 875 group = tgt->next_poll_group; 876 tgt->next_poll_group = TAILQ_NEXT(group, link); 877 } 878 879 ctx = calloc(1, sizeof(*ctx)); 880 if (!ctx) { 881 SPDK_ERRLOG("Unable to send message to poll group.\n"); 882 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 883 return; 884 } 885 886 ctx->qpair = qpair; 887 ctx->group = group; 888 889 spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx); 890 } 891 892 struct spdk_nvmf_poll_group * 893 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 894 { 895 struct spdk_io_channel *ch; 896 897 ch = spdk_get_io_channel(tgt); 898 if (!ch) { 899 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 900 return NULL; 901 } 902 903 return spdk_io_channel_get_ctx(ch); 904 } 905 906 void 907 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, 908 spdk_nvmf_poll_group_destroy_done_fn cb_fn, 909 void *cb_arg) 910 { 911 assert(group->destroy_cb_fn == NULL); 912 group->destroy_cb_fn = cb_fn; 913 group->destroy_cb_arg = cb_arg; 914 915 /* This function will put the io_channel associated with this poll group */ 916 nvmf_tgt_destroy_poll_group_qpairs(group); 917 } 918 919 int 920 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 921 struct spdk_nvmf_qpair *qpair) 922 { 923 int rc = -1; 924 struct spdk_nvmf_transport_poll_group *tgroup; 925 926 TAILQ_INIT(&qpair->outstanding); 927 qpair->group = group; 928 qpair->ctrlr = NULL; 929 qpair->disconnect_started = false; 930 931 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 932 if (tgroup->transport == qpair->transport) { 933 rc = nvmf_transport_poll_group_add(tgroup, qpair); 934 break; 935 } 936 } 937 938 /* We add the qpair to the group only it is successfully added into the tgroup */ 939 if (rc == 0) { 940 SPDK_DTRACE_PROBE2(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread)); 941 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 942 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVE); 943 } 944 945 return rc; 946 } 947 948 static void 949 _nvmf_ctrlr_destruct(void *ctx) 950 { 951 struct spdk_nvmf_ctrlr *ctrlr = ctx; 952 953 nvmf_ctrlr_destruct(ctrlr); 954 } 955 956 static void 957 _nvmf_ctrlr_free_from_qpair(void *ctx) 958 { 959 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 960 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 961 uint32_t count; 962 963 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 964 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 965 if (count == 0) { 966 assert(!ctrlr->in_destruct); 967 SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid); 968 ctrlr->in_destruct = true; 969 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 970 } 971 free(qpair_ctx); 972 } 973 974 static void 975 _nvmf_transport_qpair_fini_complete(void *cb_ctx) 976 { 977 struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx; 978 struct spdk_nvmf_ctrlr *ctrlr; 979 /* Store cb args since cb_ctx can be freed in _nvmf_ctrlr_free_from_qpair */ 980 nvmf_qpair_disconnect_cb cb_fn = qpair_ctx->cb_fn; 981 void *cb_arg = qpair_ctx->ctx; 982 struct spdk_thread *cb_thread = qpair_ctx->thread; 983 984 ctrlr = qpair_ctx->ctrlr; 985 SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid); 986 987 if (ctrlr) { 988 if (qpair_ctx->qid == 0) { 989 /* Admin qpair is removed, so set the pointer to NULL. 990 * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same 991 * as controller's thread */ 992 assert(ctrlr->thread == spdk_get_thread()); 993 ctrlr->admin_qpair = NULL; 994 } 995 /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */ 996 if (ctrlr->thread) { 997 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx); 998 } else { 999 _nvmf_ctrlr_free_from_qpair(qpair_ctx); 1000 } 1001 } else { 1002 free(qpair_ctx); 1003 } 1004 1005 if (cb_fn) { 1006 spdk_thread_send_msg(cb_thread, cb_fn, cb_arg); 1007 } 1008 } 1009 1010 void 1011 spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair) 1012 { 1013 struct spdk_nvmf_transport_poll_group *tgroup; 1014 int rc; 1015 1016 SPDK_DTRACE_PROBE2(nvmf_poll_group_remove_qpair, qpair, 1017 spdk_thread_get_id(qpair->group->thread)); 1018 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR); 1019 1020 /* Find the tgroup and remove the qpair from the tgroup */ 1021 TAILQ_FOREACH(tgroup, &qpair->group->tgroups, link) { 1022 if (tgroup->transport == qpair->transport) { 1023 rc = nvmf_transport_poll_group_remove(tgroup, qpair); 1024 if (rc && (rc != ENOTSUP)) { 1025 SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n", 1026 qpair, tgroup); 1027 } 1028 break; 1029 } 1030 } 1031 1032 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 1033 qpair->group = NULL; 1034 } 1035 1036 static void 1037 _nvmf_qpair_destroy(void *ctx, int status) 1038 { 1039 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1040 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 1041 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 1042 struct spdk_nvmf_request *req, *tmp; 1043 struct spdk_nvmf_subsystem_poll_group *sgroup; 1044 1045 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 1046 qpair_ctx->qid = qpair->qid; 1047 1048 if (ctrlr) { 1049 if (0 == qpair->qid) { 1050 assert(qpair->group->stat.current_admin_qpairs > 0); 1051 qpair->group->stat.current_admin_qpairs--; 1052 } else { 1053 assert(qpair->group->stat.current_io_qpairs > 0); 1054 qpair->group->stat.current_io_qpairs--; 1055 } 1056 1057 sgroup = &qpair->group->sgroups[ctrlr->subsys->id]; 1058 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1059 if (req->qpair == qpair) { 1060 TAILQ_REMOVE(&sgroup->queued, req, link); 1061 if (nvmf_transport_req_free(req)) { 1062 SPDK_ERRLOG("Transport request free error!/n"); 1063 } 1064 } 1065 } 1066 } 1067 1068 qpair_ctx->ctrlr = ctrlr; 1069 spdk_nvmf_poll_group_remove(qpair); 1070 nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); 1071 } 1072 1073 static void 1074 _nvmf_qpair_disconnect_msg(void *ctx) 1075 { 1076 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1077 1078 spdk_nvmf_qpair_disconnect(qpair_ctx->qpair, qpair_ctx->cb_fn, qpair_ctx->ctx); 1079 free(ctx); 1080 } 1081 1082 int 1083 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) 1084 { 1085 struct spdk_nvmf_poll_group *group = qpair->group; 1086 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 1087 1088 if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) { 1089 if (cb_fn) { 1090 cb_fn(ctx); 1091 } 1092 return 0; 1093 } 1094 1095 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 1096 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 1097 nvmf_transport_qpair_fini(qpair, NULL, NULL); 1098 if (cb_fn) { 1099 cb_fn(ctx); 1100 } 1101 return 0; 1102 } 1103 1104 assert(group != NULL); 1105 if (spdk_get_thread() != group->thread) { 1106 /* clear the atomic so we can set it on the next call on the proper thread. */ 1107 __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED); 1108 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1109 if (!qpair_ctx) { 1110 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1111 return -ENOMEM; 1112 } 1113 qpair_ctx->qpair = qpair; 1114 qpair_ctx->cb_fn = cb_fn; 1115 qpair_ctx->thread = group->thread; 1116 qpair_ctx->ctx = ctx; 1117 spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx); 1118 return 0; 1119 } 1120 1121 SPDK_DTRACE_PROBE2(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread)); 1122 assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); 1123 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 1124 1125 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1126 if (!qpair_ctx) { 1127 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1128 return -ENOMEM; 1129 } 1130 1131 qpair_ctx->qpair = qpair; 1132 qpair_ctx->cb_fn = cb_fn; 1133 qpair_ctx->thread = group->thread; 1134 qpair_ctx->ctx = ctx; 1135 1136 /* Check for outstanding I/O */ 1137 if (!TAILQ_EMPTY(&qpair->outstanding)) { 1138 SPDK_DTRACE_PROBE2(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread)); 1139 qpair->state_cb = _nvmf_qpair_destroy; 1140 qpair->state_cb_arg = qpair_ctx; 1141 nvmf_qpair_abort_pending_zcopy_reqs(qpair); 1142 nvmf_qpair_free_aer(qpair); 1143 return 0; 1144 } 1145 1146 _nvmf_qpair_destroy(qpair_ctx, 0); 1147 1148 return 0; 1149 } 1150 1151 int 1152 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 1153 struct spdk_nvme_transport_id *trid) 1154 { 1155 return nvmf_transport_qpair_get_peer_trid(qpair, trid); 1156 } 1157 1158 int 1159 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 1160 struct spdk_nvme_transport_id *trid) 1161 { 1162 return nvmf_transport_qpair_get_local_trid(qpair, trid); 1163 } 1164 1165 int 1166 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 1167 struct spdk_nvme_transport_id *trid) 1168 { 1169 return nvmf_transport_qpair_get_listen_trid(qpair, trid); 1170 } 1171 1172 static int 1173 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1174 struct spdk_nvmf_subsystem *subsystem) 1175 { 1176 struct spdk_nvmf_subsystem_poll_group *sgroup; 1177 uint32_t new_num_ns, old_num_ns; 1178 uint32_t i, j; 1179 struct spdk_nvmf_ns *ns; 1180 struct spdk_nvmf_registrant *reg, *tmp; 1181 struct spdk_io_channel *ch; 1182 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1183 struct spdk_nvmf_ctrlr *ctrlr; 1184 bool ns_changed; 1185 1186 /* Make sure our poll group has memory for this subsystem allocated */ 1187 if (subsystem->id >= group->num_sgroups) { 1188 return -ENOMEM; 1189 } 1190 1191 sgroup = &group->sgroups[subsystem->id]; 1192 1193 /* Make sure the array of namespace information is the correct size */ 1194 new_num_ns = subsystem->max_nsid; 1195 old_num_ns = sgroup->num_ns; 1196 1197 ns_changed = false; 1198 1199 if (old_num_ns == 0) { 1200 if (new_num_ns > 0) { 1201 /* First allocation */ 1202 sgroup->ns_info = calloc(new_num_ns, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1203 if (!sgroup->ns_info) { 1204 return -ENOMEM; 1205 } 1206 } 1207 } else if (new_num_ns > old_num_ns) { 1208 void *buf; 1209 1210 /* Make the array larger */ 1211 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1212 if (!buf) { 1213 return -ENOMEM; 1214 } 1215 1216 sgroup->ns_info = buf; 1217 1218 /* Null out the new namespace information slots */ 1219 for (i = old_num_ns; i < new_num_ns; i++) { 1220 memset(&sgroup->ns_info[i], 0, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1221 } 1222 } else if (new_num_ns < old_num_ns) { 1223 void *buf; 1224 1225 /* Free the extra I/O channels */ 1226 for (i = new_num_ns; i < old_num_ns; i++) { 1227 ns_info = &sgroup->ns_info[i]; 1228 1229 if (ns_info->channel) { 1230 spdk_put_io_channel(ns_info->channel); 1231 ns_info->channel = NULL; 1232 } 1233 } 1234 1235 /* Make the array smaller */ 1236 if (new_num_ns > 0) { 1237 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1238 if (!buf) { 1239 return -ENOMEM; 1240 } 1241 sgroup->ns_info = buf; 1242 } else { 1243 free(sgroup->ns_info); 1244 sgroup->ns_info = NULL; 1245 } 1246 } 1247 1248 sgroup->num_ns = new_num_ns; 1249 1250 /* Detect bdevs that were added or removed */ 1251 for (i = 0; i < sgroup->num_ns; i++) { 1252 ns = subsystem->ns[i]; 1253 ns_info = &sgroup->ns_info[i]; 1254 ch = ns_info->channel; 1255 1256 if (ns == NULL && ch == NULL) { 1257 /* Both NULL. Leave empty */ 1258 } else if (ns == NULL && ch != NULL) { 1259 /* There was a channel here, but the namespace is gone. */ 1260 ns_changed = true; 1261 spdk_put_io_channel(ch); 1262 ns_info->channel = NULL; 1263 } else if (ns != NULL && ch == NULL) { 1264 /* A namespace appeared but there is no channel yet */ 1265 ns_changed = true; 1266 ch = spdk_bdev_get_io_channel(ns->desc); 1267 if (ch == NULL) { 1268 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1269 return -ENOMEM; 1270 } 1271 ns_info->channel = ch; 1272 } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) { 1273 /* A namespace was here before, but was replaced by a new one. */ 1274 ns_changed = true; 1275 spdk_put_io_channel(ns_info->channel); 1276 memset(ns_info, 0, sizeof(*ns_info)); 1277 1278 ch = spdk_bdev_get_io_channel(ns->desc); 1279 if (ch == NULL) { 1280 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1281 return -ENOMEM; 1282 } 1283 ns_info->channel = ch; 1284 } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) { 1285 /* Namespace is still there but size has changed */ 1286 SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u," 1287 " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n", 1288 subsystem->id, 1289 ns->nsid, 1290 group, 1291 ns_info->num_blocks, 1292 spdk_bdev_get_num_blocks(ns->bdev)); 1293 ns_changed = true; 1294 } 1295 1296 if (ns == NULL) { 1297 memset(ns_info, 0, sizeof(*ns_info)); 1298 } else { 1299 ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev); 1300 ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev); 1301 ns_info->crkey = ns->crkey; 1302 ns_info->rtype = ns->rtype; 1303 if (ns->holder) { 1304 ns_info->holder_id = ns->holder->hostid; 1305 } 1306 1307 memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid)); 1308 j = 0; 1309 TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { 1310 if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) { 1311 SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS); 1312 return -EINVAL; 1313 } 1314 ns_info->reg_hostid[j++] = reg->hostid; 1315 } 1316 } 1317 } 1318 1319 if (ns_changed) { 1320 TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) { 1321 /* It is possible that a ctrlr was added but the admin_qpair hasn't been 1322 * assigned yet. 1323 */ 1324 if (!ctrlr->admin_qpair) { 1325 continue; 1326 } 1327 if (ctrlr->admin_qpair->group == group) { 1328 nvmf_ctrlr_async_event_ns_notice(ctrlr); 1329 nvmf_ctrlr_async_event_ana_change_notice(ctrlr); 1330 } 1331 } 1332 } 1333 1334 return 0; 1335 } 1336 1337 int 1338 nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1339 struct spdk_nvmf_subsystem *subsystem) 1340 { 1341 return poll_group_update_subsystem(group, subsystem); 1342 } 1343 1344 int 1345 nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 1346 struct spdk_nvmf_subsystem *subsystem, 1347 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1348 { 1349 int rc = 0; 1350 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 1351 uint32_t i; 1352 1353 TAILQ_INIT(&sgroup->queued); 1354 1355 rc = poll_group_update_subsystem(group, subsystem); 1356 if (rc) { 1357 nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); 1358 goto fini; 1359 } 1360 1361 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1362 1363 for (i = 0; i < sgroup->num_ns; i++) { 1364 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1365 } 1366 1367 fini: 1368 if (cb_fn) { 1369 cb_fn(cb_arg, rc); 1370 } 1371 1372 SPDK_DTRACE_PROBE2(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread), 1373 subsystem->subnqn); 1374 1375 return rc; 1376 } 1377 1378 static void 1379 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 1380 { 1381 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1382 struct spdk_nvmf_subsystem *subsystem; 1383 struct spdk_nvmf_poll_group *group; 1384 struct spdk_nvmf_subsystem_poll_group *sgroup; 1385 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 1386 void *cpl_ctx = NULL; 1387 uint32_t nsid; 1388 1389 group = qpair_ctx->group; 1390 subsystem = qpair_ctx->subsystem; 1391 cpl_fn = qpair_ctx->cpl_fn; 1392 cpl_ctx = qpair_ctx->cpl_ctx; 1393 sgroup = &group->sgroups[subsystem->id]; 1394 1395 if (status) { 1396 goto fini; 1397 } 1398 1399 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 1400 if (sgroup->ns_info[nsid].channel) { 1401 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 1402 sgroup->ns_info[nsid].channel = NULL; 1403 } 1404 } 1405 1406 sgroup->num_ns = 0; 1407 free(sgroup->ns_info); 1408 sgroup->ns_info = NULL; 1409 fini: 1410 free(qpair_ctx); 1411 if (cpl_fn) { 1412 cpl_fn(cpl_ctx, status); 1413 } 1414 } 1415 1416 static void nvmf_poll_group_remove_subsystem_msg(void *ctx); 1417 1418 static void 1419 remove_subsystem_qpair_cb(void *ctx) 1420 { 1421 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1422 1423 assert(qpair_ctx->count > 0); 1424 qpair_ctx->count--; 1425 if (qpair_ctx->count == 0) { 1426 /* All of the asynchronous callbacks for this context have been 1427 * completed. Call nvmf_poll_group_remove_subsystem_msg() again 1428 * to check if all associated qpairs for this subsystem have 1429 * been removed from the poll group. 1430 */ 1431 nvmf_poll_group_remove_subsystem_msg(ctx); 1432 } 1433 } 1434 1435 static void 1436 nvmf_poll_group_remove_subsystem_msg(void *ctx) 1437 { 1438 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 1439 struct spdk_nvmf_subsystem *subsystem; 1440 struct spdk_nvmf_poll_group *group; 1441 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1442 bool qpairs_found = false; 1443 int rc = 0; 1444 1445 group = qpair_ctx->group; 1446 subsystem = qpair_ctx->subsystem; 1447 1448 /* Initialize count to 1. This acts like a ref count, to ensure that if spdk_nvmf_qpair_disconnect 1449 * immediately invokes the callback (i.e. the qpairs is already in process of being disconnected) 1450 * that we don't recursively call nvmf_poll_group_remove_subsystem_msg before we've iterated the 1451 * full list of qpairs. 1452 */ 1453 qpair_ctx->count = 1; 1454 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 1455 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1456 qpairs_found = true; 1457 qpair_ctx->count++; 1458 rc = spdk_nvmf_qpair_disconnect(qpair, remove_subsystem_qpair_cb, ctx); 1459 if (rc) { 1460 break; 1461 } 1462 } 1463 } 1464 qpair_ctx->count--; 1465 1466 if (!qpairs_found) { 1467 _nvmf_poll_group_remove_subsystem_cb(ctx, 0); 1468 return; 1469 } 1470 1471 if (qpair_ctx->count == 0 || rc) { 1472 /* If count == 0, it means there were some qpairs in the poll group but they 1473 * were already in process of being disconnected. So we send a message to this 1474 * same thread so that this function executes again later. We won't actually 1475 * invoke the remove_subsystem_cb until all of the qpairs are actually removed 1476 * from the poll group. 1477 */ 1478 spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx); 1479 } 1480 } 1481 1482 void 1483 nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1484 struct spdk_nvmf_subsystem *subsystem, 1485 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1486 { 1487 struct spdk_nvmf_subsystem_poll_group *sgroup; 1488 struct nvmf_qpair_disconnect_many_ctx *ctx; 1489 uint32_t i; 1490 1491 SPDK_DTRACE_PROBE3(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread), 1492 subsystem->subnqn); 1493 1494 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1495 if (!ctx) { 1496 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1497 if (cb_fn) { 1498 cb_fn(cb_arg, -1); 1499 } 1500 return; 1501 } 1502 1503 ctx->group = group; 1504 ctx->subsystem = subsystem; 1505 ctx->cpl_fn = cb_fn; 1506 ctx->cpl_ctx = cb_arg; 1507 1508 sgroup = &group->sgroups[subsystem->id]; 1509 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1510 1511 for (i = 0; i < sgroup->num_ns; i++) { 1512 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1513 } 1514 1515 nvmf_poll_group_remove_subsystem_msg(ctx); 1516 } 1517 1518 void 1519 nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1520 struct spdk_nvmf_subsystem *subsystem, 1521 uint32_t nsid, 1522 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1523 { 1524 struct spdk_nvmf_subsystem_poll_group *sgroup; 1525 struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL; 1526 int rc = 0; 1527 uint32_t i; 1528 1529 if (subsystem->id >= group->num_sgroups) { 1530 rc = -1; 1531 goto fini; 1532 } 1533 1534 sgroup = &group->sgroups[subsystem->id]; 1535 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) { 1536 goto fini; 1537 } 1538 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1539 1540 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1541 for (i = 0; i < sgroup->num_ns; i++) { 1542 ns_info = &sgroup->ns_info[i]; 1543 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1544 } 1545 } else { 1546 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 1547 if (nsid - 1 < sgroup->num_ns) { 1548 ns_info = &sgroup->ns_info[nsid - 1]; 1549 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1550 } 1551 } 1552 1553 if (sgroup->mgmt_io_outstanding > 0) { 1554 assert(sgroup->cb_fn == NULL); 1555 sgroup->cb_fn = cb_fn; 1556 assert(sgroup->cb_arg == NULL); 1557 sgroup->cb_arg = cb_arg; 1558 return; 1559 } 1560 1561 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1562 for (i = 0; i < sgroup->num_ns; i++) { 1563 ns_info = &sgroup->ns_info[i]; 1564 1565 if (ns_info->io_outstanding > 0) { 1566 assert(sgroup->cb_fn == NULL); 1567 sgroup->cb_fn = cb_fn; 1568 assert(sgroup->cb_arg == NULL); 1569 sgroup->cb_arg = cb_arg; 1570 return; 1571 } 1572 } 1573 } else { 1574 if (ns_info != NULL && ns_info->io_outstanding > 0) { 1575 assert(sgroup->cb_fn == NULL); 1576 sgroup->cb_fn = cb_fn; 1577 assert(sgroup->cb_arg == NULL); 1578 sgroup->cb_arg = cb_arg; 1579 return; 1580 } 1581 } 1582 1583 assert(sgroup->mgmt_io_outstanding == 0); 1584 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1585 fini: 1586 if (cb_fn) { 1587 cb_fn(cb_arg, rc); 1588 } 1589 } 1590 1591 void 1592 nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1593 struct spdk_nvmf_subsystem *subsystem, 1594 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1595 { 1596 struct spdk_nvmf_request *req, *tmp; 1597 struct spdk_nvmf_subsystem_poll_group *sgroup; 1598 int rc = 0; 1599 uint32_t i; 1600 1601 if (subsystem->id >= group->num_sgroups) { 1602 rc = -1; 1603 goto fini; 1604 } 1605 1606 sgroup = &group->sgroups[subsystem->id]; 1607 1608 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) { 1609 goto fini; 1610 } 1611 1612 rc = poll_group_update_subsystem(group, subsystem); 1613 if (rc) { 1614 goto fini; 1615 } 1616 1617 for (i = 0; i < sgroup->num_ns; i++) { 1618 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1619 } 1620 1621 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1622 1623 /* Release all queued requests */ 1624 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1625 TAILQ_REMOVE(&sgroup->queued, req, link); 1626 if (spdk_nvmf_request_using_zcopy(req)) { 1627 spdk_nvmf_request_zcopy_start(req); 1628 } else { 1629 spdk_nvmf_request_exec(req); 1630 } 1631 1632 } 1633 fini: 1634 if (cb_fn) { 1635 cb_fn(cb_arg, rc); 1636 } 1637 } 1638 1639 1640 struct spdk_nvmf_poll_group * 1641 spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 1642 { 1643 struct spdk_nvmf_transport_poll_group *tgroup; 1644 1645 tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair); 1646 1647 if (tgroup == NULL) { 1648 return NULL; 1649 } 1650 1651 return tgroup->group; 1652 } 1653 1654 void 1655 spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w) 1656 { 1657 struct spdk_nvmf_transport_poll_group *tgroup; 1658 1659 spdk_json_write_object_begin(w); 1660 1661 spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread())); 1662 spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs); 1663 spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs); 1664 spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs); 1665 spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs); 1666 spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io); 1667 1668 spdk_json_write_named_array_begin(w, "transports"); 1669 1670 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1671 spdk_json_write_object_begin(w); 1672 /* 1673 * The trtype field intentionally contains a transport name as this is more informative. 1674 * The field has not been renamed for backward compatibility. 1675 */ 1676 spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport)); 1677 1678 if (tgroup->transport->ops->poll_group_dump_stat) { 1679 tgroup->transport->ops->poll_group_dump_stat(tgroup, w); 1680 } 1681 1682 spdk_json_write_object_end(w); 1683 } 1684 1685 spdk_json_write_array_end(w); 1686 spdk_json_write_object_end(w); 1687 } 1688