1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/bdev.h" 10 #include "spdk/bit_array.h" 11 #include "spdk/thread.h" 12 #include "spdk/nvmf.h" 13 #include "spdk/endian.h" 14 #include "spdk/string.h" 15 #include "spdk/log.h" 16 #include "spdk_internal/usdt.h" 17 18 #include "nvmf_internal.h" 19 #include "transport.h" 20 21 SPDK_LOG_REGISTER_COMPONENT(nvmf) 22 23 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 24 25 static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts); 26 27 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 28 29 /* supplied to a single call to nvmf_qpair_disconnect */ 30 struct nvmf_qpair_disconnect_ctx { 31 struct spdk_nvmf_qpair *qpair; 32 struct spdk_nvmf_ctrlr *ctrlr; 33 nvmf_qpair_disconnect_cb cb_fn; 34 struct spdk_thread *thread; 35 void *ctx; 36 uint16_t qid; 37 }; 38 39 /* 40 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 41 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 42 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 43 */ 44 struct nvmf_qpair_disconnect_many_ctx { 45 struct spdk_nvmf_subsystem *subsystem; 46 struct spdk_nvmf_poll_group *group; 47 spdk_nvmf_poll_group_mod_done cpl_fn; 48 void *cpl_ctx; 49 uint32_t count; 50 }; 51 52 static void 53 nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 54 enum spdk_nvmf_qpair_state state) 55 { 56 assert(qpair != NULL); 57 assert(qpair->group->thread == spdk_get_thread()); 58 59 qpair->state = state; 60 } 61 62 static int 63 nvmf_poll_group_poll(void *ctx) 64 { 65 struct spdk_nvmf_poll_group *group = ctx; 66 int rc; 67 int count = 0; 68 struct spdk_nvmf_transport_poll_group *tgroup; 69 70 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 71 rc = nvmf_transport_poll_group_poll(tgroup); 72 if (rc < 0) { 73 return SPDK_POLLER_BUSY; 74 } 75 count += rc; 76 } 77 78 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 79 } 80 81 /* 82 * Reset and clean up the poll group (I/O channel code will actually free the 83 * group). 84 */ 85 static void 86 nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group) 87 { 88 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 89 struct spdk_nvmf_subsystem_poll_group *sgroup; 90 uint32_t sid, nsid; 91 92 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 93 TAILQ_REMOVE(&group->tgroups, tgroup, link); 94 nvmf_transport_poll_group_destroy(tgroup); 95 } 96 97 for (sid = 0; sid < group->num_sgroups; sid++) { 98 sgroup = &group->sgroups[sid]; 99 100 assert(sgroup != NULL); 101 102 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 103 if (sgroup->ns_info[nsid].channel) { 104 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 105 sgroup->ns_info[nsid].channel = NULL; 106 } 107 } 108 109 free(sgroup->ns_info); 110 } 111 112 free(group->sgroups); 113 114 spdk_poller_unregister(&group->poller); 115 116 if (group->destroy_cb_fn) { 117 group->destroy_cb_fn(group->destroy_cb_arg, 0); 118 } 119 } 120 121 /* 122 * Callback to unregister a poll group from the target, and clean up its state. 123 */ 124 static void 125 nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 126 { 127 struct spdk_nvmf_tgt *tgt = io_device; 128 struct spdk_nvmf_poll_group *group = ctx_buf; 129 130 SPDK_DTRACE_PROBE1(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread)); 131 132 pthread_mutex_lock(&tgt->mutex); 133 TAILQ_REMOVE(&tgt->poll_groups, group, link); 134 pthread_mutex_unlock(&tgt->mutex); 135 136 nvmf_tgt_cleanup_poll_group(group); 137 } 138 139 static int 140 nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 141 struct spdk_nvmf_transport *transport) 142 { 143 struct spdk_nvmf_transport_poll_group *tgroup; 144 145 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 146 if (tgroup->transport == transport) { 147 /* Transport already in the poll group */ 148 return 0; 149 } 150 } 151 152 tgroup = nvmf_transport_poll_group_create(transport, group); 153 if (!tgroup) { 154 SPDK_ERRLOG("Unable to create poll group for transport\n"); 155 return -1; 156 } 157 SPDK_DTRACE_PROBE2(nvmf_transport_poll_group_create, transport, spdk_thread_get_id(group->thread)); 158 159 tgroup->group = group; 160 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 161 162 return 0; 163 } 164 165 static int 166 nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 167 { 168 struct spdk_nvmf_tgt *tgt = io_device; 169 struct spdk_nvmf_poll_group *group = ctx_buf; 170 struct spdk_nvmf_transport *transport; 171 struct spdk_thread *thread = spdk_get_thread(); 172 uint32_t sid; 173 int rc; 174 175 TAILQ_INIT(&group->tgroups); 176 TAILQ_INIT(&group->qpairs); 177 group->thread = thread; 178 pthread_mutex_init(&group->mutex, NULL); 179 180 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 181 182 SPDK_DTRACE_PROBE1(nvmf_create_poll_group, spdk_thread_get_id(thread)); 183 184 TAILQ_FOREACH(transport, &tgt->transports, link) { 185 rc = nvmf_poll_group_add_transport(group, transport); 186 if (rc != 0) { 187 nvmf_tgt_cleanup_poll_group(group); 188 return rc; 189 } 190 } 191 192 group->num_sgroups = tgt->max_subsystems; 193 group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 194 if (!group->sgroups) { 195 nvmf_tgt_cleanup_poll_group(group); 196 return -ENOMEM; 197 } 198 199 for (sid = 0; sid < tgt->max_subsystems; sid++) { 200 struct spdk_nvmf_subsystem *subsystem; 201 202 subsystem = tgt->subsystems[sid]; 203 if (!subsystem) { 204 continue; 205 } 206 207 if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { 208 nvmf_tgt_cleanup_poll_group(group); 209 return -1; 210 } 211 } 212 213 pthread_mutex_lock(&tgt->mutex); 214 TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link); 215 pthread_mutex_unlock(&tgt->mutex); 216 217 return 0; 218 } 219 220 static void 221 _nvmf_tgt_disconnect_next_qpair(void *ctx) 222 { 223 struct spdk_nvmf_qpair *qpair; 224 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 225 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 226 struct spdk_io_channel *ch; 227 int rc = 0; 228 229 qpair = TAILQ_FIRST(&group->qpairs); 230 231 if (qpair) { 232 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_tgt_disconnect_next_qpair, ctx); 233 } 234 235 if (!qpair || rc != 0) { 236 /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */ 237 ch = spdk_io_channel_from_ctx(group); 238 spdk_put_io_channel(ch); 239 free(qpair_ctx); 240 } 241 } 242 243 static void 244 nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 245 { 246 struct nvmf_qpair_disconnect_many_ctx *ctx; 247 248 SPDK_DTRACE_PROBE1(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread)); 249 250 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 251 if (!ctx) { 252 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 253 return; 254 } 255 256 ctx->group = group; 257 _nvmf_tgt_disconnect_next_qpair(ctx); 258 } 259 260 struct spdk_nvmf_tgt * 261 spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *opts) 262 { 263 struct spdk_nvmf_tgt *tgt, *tmp_tgt; 264 265 if (strnlen(opts->name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) { 266 SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH); 267 return NULL; 268 } 269 270 TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) { 271 if (!strncmp(opts->name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) { 272 SPDK_ERRLOG("Provided target name must be unique.\n"); 273 return NULL; 274 } 275 } 276 277 tgt = calloc(1, sizeof(*tgt)); 278 if (!tgt) { 279 return NULL; 280 } 281 282 snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts->name); 283 284 if (!opts || !opts->max_subsystems) { 285 tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 286 } else { 287 tgt->max_subsystems = opts->max_subsystems; 288 } 289 290 if (!opts) { 291 tgt->crdt[0] = 0; 292 tgt->crdt[1] = 0; 293 tgt->crdt[2] = 0; 294 } else { 295 tgt->crdt[0] = opts->crdt[0]; 296 tgt->crdt[1] = opts->crdt[1]; 297 tgt->crdt[2] = opts->crdt[2]; 298 } 299 300 if (!opts) { 301 tgt->discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY; 302 } else { 303 tgt->discovery_filter = opts->discovery_filter; 304 } 305 306 tgt->discovery_genctr = 0; 307 TAILQ_INIT(&tgt->transports); 308 TAILQ_INIT(&tgt->poll_groups); 309 310 tgt->subsystems = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem *)); 311 if (!tgt->subsystems) { 312 free(tgt); 313 return NULL; 314 } 315 316 pthread_mutex_init(&tgt->mutex, NULL); 317 318 spdk_io_device_register(tgt, 319 nvmf_tgt_create_poll_group, 320 nvmf_tgt_destroy_poll_group, 321 sizeof(struct spdk_nvmf_poll_group), 322 tgt->name); 323 324 TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link); 325 326 return tgt; 327 } 328 329 static void 330 _nvmf_tgt_destroy_next_transport(void *ctx) 331 { 332 struct spdk_nvmf_tgt *tgt = ctx; 333 struct spdk_nvmf_transport *transport; 334 335 if (!TAILQ_EMPTY(&tgt->transports)) { 336 transport = TAILQ_FIRST(&tgt->transports); 337 TAILQ_REMOVE(&tgt->transports, transport, link); 338 spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt); 339 } else { 340 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn; 341 void *destroy_cb_arg = tgt->destroy_cb_arg; 342 343 pthread_mutex_destroy(&tgt->mutex); 344 free(tgt); 345 346 if (destroy_cb_fn) { 347 destroy_cb_fn(destroy_cb_arg, 0); 348 } 349 } 350 } 351 352 static void 353 nvmf_tgt_destroy_cb(void *io_device) 354 { 355 struct spdk_nvmf_tgt *tgt = io_device; 356 uint32_t i; 357 int rc; 358 359 if (tgt->subsystems) { 360 for (i = 0; i < tgt->max_subsystems; i++) { 361 if (tgt->subsystems[i]) { 362 nvmf_subsystem_remove_all_listeners(tgt->subsystems[i], true); 363 364 rc = spdk_nvmf_subsystem_destroy(tgt->subsystems[i], nvmf_tgt_destroy_cb, tgt); 365 if (rc) { 366 if (rc == -EINPROGRESS) { 367 /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i 368 * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */ 369 return; 370 } else { 371 SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", tgt->subsystems[i]->subnqn, rc); 372 } 373 } 374 } 375 } 376 free(tgt->subsystems); 377 } 378 379 _nvmf_tgt_destroy_next_transport(tgt); 380 } 381 382 void 383 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 384 spdk_nvmf_tgt_destroy_done_fn cb_fn, 385 void *cb_arg) 386 { 387 tgt->destroy_cb_fn = cb_fn; 388 tgt->destroy_cb_arg = cb_arg; 389 390 TAILQ_REMOVE(&g_nvmf_tgts, tgt, link); 391 392 spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb); 393 } 394 395 const char * 396 spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt) 397 { 398 return tgt->name; 399 } 400 401 struct spdk_nvmf_tgt * 402 spdk_nvmf_get_tgt(const char *name) 403 { 404 struct spdk_nvmf_tgt *tgt; 405 uint32_t num_targets = 0; 406 407 TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) { 408 if (name) { 409 if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) { 410 return tgt; 411 } 412 } 413 num_targets++; 414 } 415 416 /* 417 * special case. If there is only one target and 418 * no name was specified, return the only available 419 * target. If there is more than one target, name must 420 * be specified. 421 */ 422 if (!name && num_targets == 1) { 423 return TAILQ_FIRST(&g_nvmf_tgts); 424 } 425 426 return NULL; 427 } 428 429 struct spdk_nvmf_tgt * 430 spdk_nvmf_get_first_tgt(void) 431 { 432 return TAILQ_FIRST(&g_nvmf_tgts); 433 } 434 435 struct spdk_nvmf_tgt * 436 spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev) 437 { 438 return TAILQ_NEXT(prev, link); 439 } 440 441 static void 442 nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 443 struct spdk_nvmf_subsystem *subsystem) 444 { 445 struct spdk_nvmf_host *host; 446 struct spdk_nvmf_subsystem_listener *listener; 447 const struct spdk_nvme_transport_id *trid; 448 struct spdk_nvmf_ns *ns; 449 struct spdk_nvmf_ns_opts ns_opts; 450 uint32_t max_namespaces; 451 char uuid_str[SPDK_UUID_STRING_LEN]; 452 453 if (spdk_nvmf_subsystem_get_type(subsystem) != SPDK_NVMF_SUBTYPE_NVME) { 454 return; 455 } 456 457 /* { */ 458 spdk_json_write_object_begin(w); 459 spdk_json_write_named_string(w, "method", "nvmf_create_subsystem"); 460 461 /* "params" : { */ 462 spdk_json_write_named_object_begin(w, "params"); 463 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 464 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 465 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 466 spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem)); 467 468 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 469 if (max_namespaces != 0) { 470 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 471 } 472 473 spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem)); 474 spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem)); 475 spdk_json_write_named_bool(w, "ana_reporting", nvmf_subsystem_get_ana_reporting(subsystem)); 476 477 /* } "params" */ 478 spdk_json_write_object_end(w); 479 480 /* } */ 481 spdk_json_write_object_end(w); 482 483 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 484 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 485 486 spdk_json_write_object_begin(w); 487 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 488 489 /* "params" : { */ 490 spdk_json_write_named_object_begin(w, "params"); 491 492 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 493 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 494 495 /* } "params" */ 496 spdk_json_write_object_end(w); 497 498 /* } */ 499 spdk_json_write_object_end(w); 500 } 501 502 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 503 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 504 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 505 506 spdk_json_write_object_begin(w); 507 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 508 509 /* "params" : { */ 510 spdk_json_write_named_object_begin(w, "params"); 511 512 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 513 514 /* "namespace" : { */ 515 spdk_json_write_named_object_begin(w, "namespace"); 516 517 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 518 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 519 520 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 521 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 522 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 523 from_be64(&ns_opts.nguid[8])); 524 } 525 526 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 527 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 528 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 529 } 530 531 if (!spdk_mem_all_zero(&ns_opts.uuid, sizeof(ns_opts.uuid))) { 532 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &ns_opts.uuid); 533 spdk_json_write_named_string(w, "uuid", uuid_str); 534 } 535 536 if (nvmf_subsystem_get_ana_reporting(subsystem)) { 537 spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid); 538 } 539 540 /* "namespace" */ 541 spdk_json_write_object_end(w); 542 543 /* } "params" */ 544 spdk_json_write_object_end(w); 545 546 /* } */ 547 spdk_json_write_object_end(w); 548 } 549 550 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 551 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 552 trid = spdk_nvmf_subsystem_listener_get_trid(listener); 553 554 spdk_json_write_object_begin(w); 555 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 556 557 /* "params" : { */ 558 spdk_json_write_named_object_begin(w, "params"); 559 560 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 561 nvmf_transport_listen_dump_opts(listener->transport, trid, w); 562 563 /* } "params" */ 564 spdk_json_write_object_end(w); 565 566 /* } */ 567 spdk_json_write_object_end(w); 568 } 569 570 } 571 572 void 573 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 574 { 575 struct spdk_nvmf_subsystem *subsystem; 576 struct spdk_nvmf_transport *transport; 577 578 spdk_json_write_object_begin(w); 579 spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems"); 580 581 spdk_json_write_named_object_begin(w, "params"); 582 spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems); 583 spdk_json_write_object_end(w); 584 585 spdk_json_write_object_end(w); 586 587 spdk_json_write_object_begin(w); 588 spdk_json_write_named_string(w, "method", "nvmf_set_crdt"); 589 spdk_json_write_named_object_begin(w, "params"); 590 spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]); 591 spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]); 592 spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]); 593 spdk_json_write_object_end(w); 594 spdk_json_write_object_end(w); 595 596 /* write transports */ 597 TAILQ_FOREACH(transport, &tgt->transports, link) { 598 spdk_json_write_object_begin(w); 599 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 600 nvmf_transport_dump_opts(transport, w, true); 601 spdk_json_write_object_end(w); 602 } 603 604 subsystem = spdk_nvmf_subsystem_get_first(tgt); 605 while (subsystem) { 606 nvmf_write_subsystem_config_json(w, subsystem); 607 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 608 } 609 } 610 611 static void 612 nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts, 613 const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size) 614 { 615 assert(opts); 616 assert(opts_src); 617 618 opts->opts_size = opts_size; 619 620 #define SET_FIELD(field) \ 621 if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \ 622 opts->field = opts_src->field; \ 623 } \ 624 625 SET_FIELD(transport_specific); 626 #undef SET_FIELD 627 628 /* Do not remove this statement, you should always update this statement when you adding a new field, 629 * and do not forget to add the SET_FIELD statement for your added field. */ 630 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 16, "Incorrect size"); 631 } 632 633 void 634 spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size) 635 { 636 struct spdk_nvmf_listen_opts opts_local = {}; 637 638 /* local version of opts should have defaults set here */ 639 640 nvmf_listen_opts_copy(opts, &opts_local, opts_size); 641 } 642 643 int 644 spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid, 645 struct spdk_nvmf_listen_opts *opts) 646 { 647 struct spdk_nvmf_transport *transport; 648 int rc; 649 struct spdk_nvmf_listen_opts opts_local = {}; 650 651 if (!opts) { 652 SPDK_ERRLOG("opts should not be NULL\n"); 653 return -EINVAL; 654 } 655 656 if (!opts->opts_size) { 657 SPDK_ERRLOG("The opts_size in opts structure should not be zero\n"); 658 return -EINVAL; 659 } 660 661 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 662 if (!transport) { 663 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 664 trid->trstring); 665 return -EINVAL; 666 } 667 668 nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size); 669 rc = spdk_nvmf_transport_listen(transport, trid, &opts_local); 670 if (rc < 0) { 671 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 672 } 673 674 return rc; 675 } 676 677 int 678 spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt, 679 struct spdk_nvme_transport_id *trid) 680 { 681 struct spdk_nvmf_transport *transport; 682 int rc; 683 684 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 685 if (!transport) { 686 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 687 trid->trstring); 688 return -EINVAL; 689 } 690 691 rc = spdk_nvmf_transport_stop_listen(transport, trid); 692 if (rc < 0) { 693 SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr); 694 return rc; 695 } 696 return 0; 697 } 698 699 struct spdk_nvmf_tgt_add_transport_ctx { 700 struct spdk_nvmf_tgt *tgt; 701 struct spdk_nvmf_transport *transport; 702 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 703 void *cb_arg; 704 int status; 705 }; 706 707 static void 708 _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status) 709 { 710 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 711 712 ctx->cb_fn(ctx->cb_arg, ctx->status); 713 free(ctx); 714 } 715 716 static void 717 _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i) 718 { 719 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 720 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 721 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 722 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 723 724 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 725 if (tgroup->transport == ctx->transport) { 726 TAILQ_REMOVE(&group->tgroups, tgroup, link); 727 nvmf_transport_poll_group_destroy(tgroup); 728 } 729 } 730 731 spdk_for_each_channel_continue(i, 0); 732 } 733 734 static void 735 _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 736 { 737 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 738 739 if (status) { 740 ctx->status = status; 741 spdk_for_each_channel(ctx->tgt, 742 _nvmf_tgt_remove_transport, 743 ctx, 744 _nvmf_tgt_remove_transport_done); 745 return; 746 } 747 748 ctx->transport->tgt = ctx->tgt; 749 TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link); 750 ctx->cb_fn(ctx->cb_arg, status); 751 free(ctx); 752 } 753 754 static void 755 _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 756 { 757 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 758 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 759 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 760 int rc; 761 762 rc = nvmf_poll_group_add_transport(group, ctx->transport); 763 spdk_for_each_channel_continue(i, rc); 764 } 765 766 void 767 spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 768 struct spdk_nvmf_transport *transport, 769 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 770 void *cb_arg) 771 { 772 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 773 774 SPDK_DTRACE_PROBE2(nvmf_tgt_add_transport, transport, tgt->name); 775 776 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) { 777 cb_fn(cb_arg, -EEXIST); 778 return; /* transport already created */ 779 } 780 781 ctx = calloc(1, sizeof(*ctx)); 782 if (!ctx) { 783 cb_fn(cb_arg, -ENOMEM); 784 return; 785 } 786 787 ctx->tgt = tgt; 788 ctx->transport = transport; 789 ctx->cb_fn = cb_fn; 790 ctx->cb_arg = cb_arg; 791 792 spdk_for_each_channel(tgt, 793 _nvmf_tgt_add_transport, 794 ctx, 795 _nvmf_tgt_add_transport_done); 796 } 797 798 struct spdk_nvmf_subsystem * 799 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 800 { 801 struct spdk_nvmf_subsystem *subsystem; 802 uint32_t sid; 803 804 if (!subnqn) { 805 return NULL; 806 } 807 808 /* Ensure that subnqn is null terminated */ 809 if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 810 SPDK_ERRLOG("Connect SUBNQN is not null terminated\n"); 811 return NULL; 812 } 813 814 for (sid = 0; sid < tgt->max_subsystems; sid++) { 815 subsystem = tgt->subsystems[sid]; 816 if (subsystem == NULL) { 817 continue; 818 } 819 820 if (strcmp(subnqn, subsystem->subnqn) == 0) { 821 return subsystem; 822 } 823 } 824 825 return NULL; 826 } 827 828 struct spdk_nvmf_transport * 829 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name) 830 { 831 struct spdk_nvmf_transport *transport; 832 833 TAILQ_FOREACH(transport, &tgt->transports, link) { 834 if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) { 835 return transport; 836 } 837 } 838 return NULL; 839 } 840 841 struct nvmf_new_qpair_ctx { 842 struct spdk_nvmf_qpair *qpair; 843 struct spdk_nvmf_poll_group *group; 844 }; 845 846 static void 847 _nvmf_poll_group_add(void *_ctx) 848 { 849 struct nvmf_new_qpair_ctx *ctx = _ctx; 850 struct spdk_nvmf_qpair *qpair = ctx->qpair; 851 struct spdk_nvmf_poll_group *group = ctx->group; 852 853 free(_ctx); 854 855 if (spdk_nvmf_poll_group_add(group, qpair) != 0) { 856 SPDK_ERRLOG("Unable to add the qpair to a poll group.\n"); 857 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 858 } 859 } 860 861 void 862 spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair) 863 { 864 struct spdk_nvmf_poll_group *group; 865 struct nvmf_new_qpair_ctx *ctx; 866 867 group = spdk_nvmf_get_optimal_poll_group(qpair); 868 if (group == NULL) { 869 if (tgt->next_poll_group == NULL) { 870 tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups); 871 if (tgt->next_poll_group == NULL) { 872 SPDK_ERRLOG("No poll groups exist.\n"); 873 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 874 return; 875 } 876 } 877 group = tgt->next_poll_group; 878 tgt->next_poll_group = TAILQ_NEXT(group, link); 879 } 880 881 ctx = calloc(1, sizeof(*ctx)); 882 if (!ctx) { 883 SPDK_ERRLOG("Unable to send message to poll group.\n"); 884 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 885 return; 886 } 887 888 ctx->qpair = qpair; 889 ctx->group = group; 890 891 pthread_mutex_lock(&group->mutex); 892 group->current_unassociated_qpairs++; 893 pthread_mutex_unlock(&group->mutex); 894 895 spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx); 896 } 897 898 struct spdk_nvmf_poll_group * 899 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 900 { 901 struct spdk_io_channel *ch; 902 903 ch = spdk_get_io_channel(tgt); 904 if (!ch) { 905 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 906 return NULL; 907 } 908 909 return spdk_io_channel_get_ctx(ch); 910 } 911 912 void 913 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, 914 spdk_nvmf_poll_group_destroy_done_fn cb_fn, 915 void *cb_arg) 916 { 917 assert(group->destroy_cb_fn == NULL); 918 group->destroy_cb_fn = cb_fn; 919 group->destroy_cb_arg = cb_arg; 920 921 /* This function will put the io_channel associated with this poll group */ 922 nvmf_tgt_destroy_poll_group_qpairs(group); 923 } 924 925 int 926 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 927 struct spdk_nvmf_qpair *qpair) 928 { 929 int rc = -1; 930 struct spdk_nvmf_transport_poll_group *tgroup; 931 932 TAILQ_INIT(&qpair->outstanding); 933 qpair->group = group; 934 qpair->ctrlr = NULL; 935 qpair->disconnect_started = false; 936 937 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 938 if (tgroup->transport == qpair->transport) { 939 rc = nvmf_transport_poll_group_add(tgroup, qpair); 940 break; 941 } 942 } 943 944 /* We add the qpair to the group only it is successfully added into the tgroup */ 945 if (rc == 0) { 946 SPDK_DTRACE_PROBE2(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread)); 947 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 948 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVE); 949 } 950 951 return rc; 952 } 953 954 static void 955 _nvmf_ctrlr_destruct(void *ctx) 956 { 957 struct spdk_nvmf_ctrlr *ctrlr = ctx; 958 959 nvmf_ctrlr_destruct(ctrlr); 960 } 961 962 static void 963 _nvmf_ctrlr_free_from_qpair(void *ctx) 964 { 965 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 966 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 967 uint32_t count; 968 969 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 970 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 971 if (count == 0) { 972 assert(!ctrlr->in_destruct); 973 SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid); 974 ctrlr->in_destruct = true; 975 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 976 } 977 free(qpair_ctx); 978 } 979 980 static void 981 _nvmf_transport_qpair_fini_complete(void *cb_ctx) 982 { 983 struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx; 984 struct spdk_nvmf_ctrlr *ctrlr; 985 /* Store cb args since cb_ctx can be freed in _nvmf_ctrlr_free_from_qpair */ 986 nvmf_qpair_disconnect_cb cb_fn = qpair_ctx->cb_fn; 987 void *cb_arg = qpair_ctx->ctx; 988 struct spdk_thread *cb_thread = qpair_ctx->thread; 989 990 ctrlr = qpair_ctx->ctrlr; 991 SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid); 992 993 if (ctrlr) { 994 if (qpair_ctx->qid == 0) { 995 /* Admin qpair is removed, so set the pointer to NULL. 996 * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same 997 * as controller's thread */ 998 assert(ctrlr->thread == spdk_get_thread()); 999 ctrlr->admin_qpair = NULL; 1000 } 1001 /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */ 1002 if (ctrlr->thread) { 1003 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx); 1004 } else { 1005 _nvmf_ctrlr_free_from_qpair(qpair_ctx); 1006 } 1007 } else { 1008 free(qpair_ctx); 1009 } 1010 1011 if (cb_fn) { 1012 spdk_thread_send_msg(cb_thread, cb_fn, cb_arg); 1013 } 1014 } 1015 1016 void 1017 spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair) 1018 { 1019 struct spdk_nvmf_transport_poll_group *tgroup; 1020 int rc; 1021 1022 SPDK_DTRACE_PROBE2(nvmf_poll_group_remove_qpair, qpair, 1023 spdk_thread_get_id(qpair->group->thread)); 1024 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR); 1025 1026 /* Find the tgroup and remove the qpair from the tgroup */ 1027 TAILQ_FOREACH(tgroup, &qpair->group->tgroups, link) { 1028 if (tgroup->transport == qpair->transport) { 1029 rc = nvmf_transport_poll_group_remove(tgroup, qpair); 1030 if (rc && (rc != ENOTSUP)) { 1031 SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n", 1032 qpair, tgroup); 1033 } 1034 break; 1035 } 1036 } 1037 1038 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 1039 qpair->group = NULL; 1040 } 1041 1042 static void 1043 _nvmf_qpair_destroy(void *ctx, int status) 1044 { 1045 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1046 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 1047 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 1048 struct spdk_nvmf_request *req, *tmp; 1049 struct spdk_nvmf_subsystem_poll_group *sgroup; 1050 1051 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 1052 qpair_ctx->qid = qpair->qid; 1053 1054 if (qpair->connect_received) { 1055 if (0 == qpair->qid) { 1056 assert(qpair->group->stat.current_admin_qpairs > 0); 1057 qpair->group->stat.current_admin_qpairs--; 1058 } else { 1059 assert(qpair->group->stat.current_io_qpairs > 0); 1060 qpair->group->stat.current_io_qpairs--; 1061 } 1062 } else { 1063 pthread_mutex_lock(&qpair->group->mutex); 1064 qpair->group->current_unassociated_qpairs--; 1065 pthread_mutex_unlock(&qpair->group->mutex); 1066 } 1067 1068 if (ctrlr) { 1069 sgroup = &qpair->group->sgroups[ctrlr->subsys->id]; 1070 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1071 if (req->qpair == qpair) { 1072 TAILQ_REMOVE(&sgroup->queued, req, link); 1073 if (nvmf_transport_req_free(req)) { 1074 SPDK_ERRLOG("Transport request free error!/n"); 1075 } 1076 } 1077 } 1078 } 1079 1080 qpair_ctx->ctrlr = ctrlr; 1081 spdk_nvmf_poll_group_remove(qpair); 1082 nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); 1083 } 1084 1085 static void 1086 _nvmf_qpair_disconnect_msg(void *ctx) 1087 { 1088 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1089 1090 spdk_nvmf_qpair_disconnect(qpair_ctx->qpair, qpair_ctx->cb_fn, qpair_ctx->ctx); 1091 free(ctx); 1092 } 1093 1094 int 1095 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) 1096 { 1097 struct spdk_nvmf_poll_group *group = qpair->group; 1098 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 1099 1100 if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) { 1101 if (cb_fn) { 1102 cb_fn(ctx); 1103 } 1104 return 0; 1105 } 1106 1107 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 1108 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 1109 nvmf_transport_qpair_fini(qpair, NULL, NULL); 1110 if (cb_fn) { 1111 cb_fn(ctx); 1112 } 1113 return 0; 1114 } 1115 1116 assert(group != NULL); 1117 if (spdk_get_thread() != group->thread) { 1118 /* clear the atomic so we can set it on the next call on the proper thread. */ 1119 __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED); 1120 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1121 if (!qpair_ctx) { 1122 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1123 return -ENOMEM; 1124 } 1125 qpair_ctx->qpair = qpair; 1126 qpair_ctx->cb_fn = cb_fn; 1127 qpair_ctx->thread = group->thread; 1128 qpair_ctx->ctx = ctx; 1129 spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx); 1130 return 0; 1131 } 1132 1133 SPDK_DTRACE_PROBE2(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread)); 1134 assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); 1135 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 1136 1137 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1138 if (!qpair_ctx) { 1139 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1140 return -ENOMEM; 1141 } 1142 1143 qpair_ctx->qpair = qpair; 1144 qpair_ctx->cb_fn = cb_fn; 1145 qpair_ctx->thread = group->thread; 1146 qpair_ctx->ctx = ctx; 1147 1148 /* Check for outstanding I/O */ 1149 if (!TAILQ_EMPTY(&qpair->outstanding)) { 1150 SPDK_DTRACE_PROBE2(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread)); 1151 qpair->state_cb = _nvmf_qpair_destroy; 1152 qpair->state_cb_arg = qpair_ctx; 1153 nvmf_qpair_abort_pending_zcopy_reqs(qpair); 1154 nvmf_qpair_free_aer(qpair); 1155 return 0; 1156 } 1157 1158 _nvmf_qpair_destroy(qpair_ctx, 0); 1159 1160 return 0; 1161 } 1162 1163 int 1164 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 1165 struct spdk_nvme_transport_id *trid) 1166 { 1167 return nvmf_transport_qpair_get_peer_trid(qpair, trid); 1168 } 1169 1170 int 1171 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 1172 struct spdk_nvme_transport_id *trid) 1173 { 1174 return nvmf_transport_qpair_get_local_trid(qpair, trid); 1175 } 1176 1177 int 1178 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 1179 struct spdk_nvme_transport_id *trid) 1180 { 1181 return nvmf_transport_qpair_get_listen_trid(qpair, trid); 1182 } 1183 1184 static int 1185 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1186 struct spdk_nvmf_subsystem *subsystem) 1187 { 1188 struct spdk_nvmf_subsystem_poll_group *sgroup; 1189 uint32_t new_num_ns, old_num_ns; 1190 uint32_t i, j; 1191 struct spdk_nvmf_ns *ns; 1192 struct spdk_nvmf_registrant *reg, *tmp; 1193 struct spdk_io_channel *ch; 1194 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1195 struct spdk_nvmf_ctrlr *ctrlr; 1196 bool ns_changed; 1197 1198 /* Make sure our poll group has memory for this subsystem allocated */ 1199 if (subsystem->id >= group->num_sgroups) { 1200 return -ENOMEM; 1201 } 1202 1203 sgroup = &group->sgroups[subsystem->id]; 1204 1205 /* Make sure the array of namespace information is the correct size */ 1206 new_num_ns = subsystem->max_nsid; 1207 old_num_ns = sgroup->num_ns; 1208 1209 ns_changed = false; 1210 1211 if (old_num_ns == 0) { 1212 if (new_num_ns > 0) { 1213 /* First allocation */ 1214 sgroup->ns_info = calloc(new_num_ns, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1215 if (!sgroup->ns_info) { 1216 return -ENOMEM; 1217 } 1218 } 1219 } else if (new_num_ns > old_num_ns) { 1220 void *buf; 1221 1222 /* Make the array larger */ 1223 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1224 if (!buf) { 1225 return -ENOMEM; 1226 } 1227 1228 sgroup->ns_info = buf; 1229 1230 /* Null out the new namespace information slots */ 1231 for (i = old_num_ns; i < new_num_ns; i++) { 1232 memset(&sgroup->ns_info[i], 0, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1233 } 1234 } else if (new_num_ns < old_num_ns) { 1235 void *buf; 1236 1237 /* Free the extra I/O channels */ 1238 for (i = new_num_ns; i < old_num_ns; i++) { 1239 ns_info = &sgroup->ns_info[i]; 1240 1241 if (ns_info->channel) { 1242 spdk_put_io_channel(ns_info->channel); 1243 ns_info->channel = NULL; 1244 } 1245 } 1246 1247 /* Make the array smaller */ 1248 if (new_num_ns > 0) { 1249 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1250 if (!buf) { 1251 return -ENOMEM; 1252 } 1253 sgroup->ns_info = buf; 1254 } else { 1255 free(sgroup->ns_info); 1256 sgroup->ns_info = NULL; 1257 } 1258 } 1259 1260 sgroup->num_ns = new_num_ns; 1261 1262 /* Detect bdevs that were added or removed */ 1263 for (i = 0; i < sgroup->num_ns; i++) { 1264 ns = subsystem->ns[i]; 1265 ns_info = &sgroup->ns_info[i]; 1266 ch = ns_info->channel; 1267 1268 if (ns == NULL && ch == NULL) { 1269 /* Both NULL. Leave empty */ 1270 } else if (ns == NULL && ch != NULL) { 1271 /* There was a channel here, but the namespace is gone. */ 1272 ns_changed = true; 1273 spdk_put_io_channel(ch); 1274 ns_info->channel = NULL; 1275 } else if (ns != NULL && ch == NULL) { 1276 /* A namespace appeared but there is no channel yet */ 1277 ns_changed = true; 1278 ch = spdk_bdev_get_io_channel(ns->desc); 1279 if (ch == NULL) { 1280 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1281 return -ENOMEM; 1282 } 1283 ns_info->channel = ch; 1284 } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) { 1285 /* A namespace was here before, but was replaced by a new one. */ 1286 ns_changed = true; 1287 spdk_put_io_channel(ns_info->channel); 1288 memset(ns_info, 0, sizeof(*ns_info)); 1289 1290 ch = spdk_bdev_get_io_channel(ns->desc); 1291 if (ch == NULL) { 1292 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1293 return -ENOMEM; 1294 } 1295 ns_info->channel = ch; 1296 } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) { 1297 /* Namespace is still there but size has changed */ 1298 SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u," 1299 " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n", 1300 subsystem->id, 1301 ns->nsid, 1302 group, 1303 ns_info->num_blocks, 1304 spdk_bdev_get_num_blocks(ns->bdev)); 1305 ns_changed = true; 1306 } 1307 1308 if (ns == NULL) { 1309 memset(ns_info, 0, sizeof(*ns_info)); 1310 } else { 1311 ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev); 1312 ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev); 1313 ns_info->crkey = ns->crkey; 1314 ns_info->rtype = ns->rtype; 1315 if (ns->holder) { 1316 ns_info->holder_id = ns->holder->hostid; 1317 } 1318 1319 memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid)); 1320 j = 0; 1321 TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { 1322 if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) { 1323 SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS); 1324 return -EINVAL; 1325 } 1326 ns_info->reg_hostid[j++] = reg->hostid; 1327 } 1328 } 1329 } 1330 1331 if (ns_changed) { 1332 TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) { 1333 /* It is possible that a ctrlr was added but the admin_qpair hasn't been 1334 * assigned yet. 1335 */ 1336 if (!ctrlr->admin_qpair) { 1337 continue; 1338 } 1339 if (ctrlr->admin_qpair->group == group) { 1340 nvmf_ctrlr_async_event_ns_notice(ctrlr); 1341 nvmf_ctrlr_async_event_ana_change_notice(ctrlr); 1342 } 1343 } 1344 } 1345 1346 return 0; 1347 } 1348 1349 int 1350 nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1351 struct spdk_nvmf_subsystem *subsystem) 1352 { 1353 return poll_group_update_subsystem(group, subsystem); 1354 } 1355 1356 int 1357 nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 1358 struct spdk_nvmf_subsystem *subsystem, 1359 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1360 { 1361 int rc = 0; 1362 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 1363 uint32_t i; 1364 1365 TAILQ_INIT(&sgroup->queued); 1366 1367 rc = poll_group_update_subsystem(group, subsystem); 1368 if (rc) { 1369 nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); 1370 goto fini; 1371 } 1372 1373 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1374 1375 for (i = 0; i < sgroup->num_ns; i++) { 1376 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1377 } 1378 1379 fini: 1380 if (cb_fn) { 1381 cb_fn(cb_arg, rc); 1382 } 1383 1384 SPDK_DTRACE_PROBE2(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread), 1385 subsystem->subnqn); 1386 1387 return rc; 1388 } 1389 1390 static void 1391 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 1392 { 1393 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1394 struct spdk_nvmf_subsystem *subsystem; 1395 struct spdk_nvmf_poll_group *group; 1396 struct spdk_nvmf_subsystem_poll_group *sgroup; 1397 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 1398 void *cpl_ctx = NULL; 1399 uint32_t nsid; 1400 1401 group = qpair_ctx->group; 1402 subsystem = qpair_ctx->subsystem; 1403 cpl_fn = qpair_ctx->cpl_fn; 1404 cpl_ctx = qpair_ctx->cpl_ctx; 1405 sgroup = &group->sgroups[subsystem->id]; 1406 1407 if (status) { 1408 goto fini; 1409 } 1410 1411 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 1412 if (sgroup->ns_info[nsid].channel) { 1413 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 1414 sgroup->ns_info[nsid].channel = NULL; 1415 } 1416 } 1417 1418 sgroup->num_ns = 0; 1419 free(sgroup->ns_info); 1420 sgroup->ns_info = NULL; 1421 fini: 1422 free(qpair_ctx); 1423 if (cpl_fn) { 1424 cpl_fn(cpl_ctx, status); 1425 } 1426 } 1427 1428 static void nvmf_poll_group_remove_subsystem_msg(void *ctx); 1429 1430 static void 1431 remove_subsystem_qpair_cb(void *ctx) 1432 { 1433 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1434 1435 assert(qpair_ctx->count > 0); 1436 qpair_ctx->count--; 1437 if (qpair_ctx->count == 0) { 1438 /* All of the asynchronous callbacks for this context have been 1439 * completed. Call nvmf_poll_group_remove_subsystem_msg() again 1440 * to check if all associated qpairs for this subsystem have 1441 * been removed from the poll group. 1442 */ 1443 nvmf_poll_group_remove_subsystem_msg(ctx); 1444 } 1445 } 1446 1447 static void 1448 nvmf_poll_group_remove_subsystem_msg(void *ctx) 1449 { 1450 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 1451 struct spdk_nvmf_subsystem *subsystem; 1452 struct spdk_nvmf_poll_group *group; 1453 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1454 bool qpairs_found = false; 1455 int rc = 0; 1456 1457 group = qpair_ctx->group; 1458 subsystem = qpair_ctx->subsystem; 1459 1460 /* Initialize count to 1. This acts like a ref count, to ensure that if spdk_nvmf_qpair_disconnect 1461 * immediately invokes the callback (i.e. the qpairs is already in process of being disconnected) 1462 * that we don't recursively call nvmf_poll_group_remove_subsystem_msg before we've iterated the 1463 * full list of qpairs. 1464 */ 1465 qpair_ctx->count = 1; 1466 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 1467 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1468 qpairs_found = true; 1469 qpair_ctx->count++; 1470 rc = spdk_nvmf_qpair_disconnect(qpair, remove_subsystem_qpair_cb, ctx); 1471 if (rc) { 1472 break; 1473 } 1474 } 1475 } 1476 qpair_ctx->count--; 1477 1478 if (!qpairs_found) { 1479 _nvmf_poll_group_remove_subsystem_cb(ctx, 0); 1480 return; 1481 } 1482 1483 if (qpair_ctx->count == 0 || rc) { 1484 /* If count == 0, it means there were some qpairs in the poll group but they 1485 * were already in process of being disconnected. So we send a message to this 1486 * same thread so that this function executes again later. We won't actually 1487 * invoke the remove_subsystem_cb until all of the qpairs are actually removed 1488 * from the poll group. 1489 */ 1490 spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx); 1491 } 1492 } 1493 1494 void 1495 nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1496 struct spdk_nvmf_subsystem *subsystem, 1497 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1498 { 1499 struct spdk_nvmf_subsystem_poll_group *sgroup; 1500 struct nvmf_qpair_disconnect_many_ctx *ctx; 1501 uint32_t i; 1502 1503 SPDK_DTRACE_PROBE3(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread), 1504 subsystem->subnqn); 1505 1506 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1507 if (!ctx) { 1508 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1509 if (cb_fn) { 1510 cb_fn(cb_arg, -1); 1511 } 1512 return; 1513 } 1514 1515 ctx->group = group; 1516 ctx->subsystem = subsystem; 1517 ctx->cpl_fn = cb_fn; 1518 ctx->cpl_ctx = cb_arg; 1519 1520 sgroup = &group->sgroups[subsystem->id]; 1521 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1522 1523 for (i = 0; i < sgroup->num_ns; i++) { 1524 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1525 } 1526 1527 nvmf_poll_group_remove_subsystem_msg(ctx); 1528 } 1529 1530 void 1531 nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1532 struct spdk_nvmf_subsystem *subsystem, 1533 uint32_t nsid, 1534 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1535 { 1536 struct spdk_nvmf_subsystem_poll_group *sgroup; 1537 struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL; 1538 int rc = 0; 1539 uint32_t i; 1540 1541 if (subsystem->id >= group->num_sgroups) { 1542 rc = -1; 1543 goto fini; 1544 } 1545 1546 sgroup = &group->sgroups[subsystem->id]; 1547 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) { 1548 goto fini; 1549 } 1550 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1551 1552 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1553 for (i = 0; i < sgroup->num_ns; i++) { 1554 ns_info = &sgroup->ns_info[i]; 1555 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1556 } 1557 } else { 1558 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 1559 if (nsid - 1 < sgroup->num_ns) { 1560 ns_info = &sgroup->ns_info[nsid - 1]; 1561 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1562 } 1563 } 1564 1565 if (sgroup->mgmt_io_outstanding > 0) { 1566 assert(sgroup->cb_fn == NULL); 1567 sgroup->cb_fn = cb_fn; 1568 assert(sgroup->cb_arg == NULL); 1569 sgroup->cb_arg = cb_arg; 1570 return; 1571 } 1572 1573 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1574 for (i = 0; i < sgroup->num_ns; i++) { 1575 ns_info = &sgroup->ns_info[i]; 1576 1577 if (ns_info->io_outstanding > 0) { 1578 assert(sgroup->cb_fn == NULL); 1579 sgroup->cb_fn = cb_fn; 1580 assert(sgroup->cb_arg == NULL); 1581 sgroup->cb_arg = cb_arg; 1582 return; 1583 } 1584 } 1585 } else { 1586 if (ns_info != NULL && ns_info->io_outstanding > 0) { 1587 assert(sgroup->cb_fn == NULL); 1588 sgroup->cb_fn = cb_fn; 1589 assert(sgroup->cb_arg == NULL); 1590 sgroup->cb_arg = cb_arg; 1591 return; 1592 } 1593 } 1594 1595 assert(sgroup->mgmt_io_outstanding == 0); 1596 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1597 fini: 1598 if (cb_fn) { 1599 cb_fn(cb_arg, rc); 1600 } 1601 } 1602 1603 void 1604 nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1605 struct spdk_nvmf_subsystem *subsystem, 1606 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1607 { 1608 struct spdk_nvmf_request *req, *tmp; 1609 struct spdk_nvmf_subsystem_poll_group *sgroup; 1610 int rc = 0; 1611 uint32_t i; 1612 1613 if (subsystem->id >= group->num_sgroups) { 1614 rc = -1; 1615 goto fini; 1616 } 1617 1618 sgroup = &group->sgroups[subsystem->id]; 1619 1620 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) { 1621 goto fini; 1622 } 1623 1624 rc = poll_group_update_subsystem(group, subsystem); 1625 if (rc) { 1626 goto fini; 1627 } 1628 1629 for (i = 0; i < sgroup->num_ns; i++) { 1630 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1631 } 1632 1633 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1634 1635 /* Release all queued requests */ 1636 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1637 TAILQ_REMOVE(&sgroup->queued, req, link); 1638 if (spdk_nvmf_request_using_zcopy(req)) { 1639 spdk_nvmf_request_zcopy_start(req); 1640 } else { 1641 spdk_nvmf_request_exec(req); 1642 } 1643 1644 } 1645 fini: 1646 if (cb_fn) { 1647 cb_fn(cb_arg, rc); 1648 } 1649 } 1650 1651 1652 struct spdk_nvmf_poll_group * 1653 spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 1654 { 1655 struct spdk_nvmf_transport_poll_group *tgroup; 1656 1657 tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair); 1658 1659 if (tgroup == NULL) { 1660 return NULL; 1661 } 1662 1663 return tgroup->group; 1664 } 1665 1666 void 1667 spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w) 1668 { 1669 struct spdk_nvmf_transport_poll_group *tgroup; 1670 1671 spdk_json_write_object_begin(w); 1672 1673 spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread())); 1674 spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs); 1675 spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs); 1676 spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs); 1677 spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs); 1678 spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io); 1679 spdk_json_write_named_uint64(w, "completed_nvme_io", group->stat.completed_nvme_io); 1680 1681 spdk_json_write_named_array_begin(w, "transports"); 1682 1683 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1684 spdk_json_write_object_begin(w); 1685 /* 1686 * The trtype field intentionally contains a transport name as this is more informative. 1687 * The field has not been renamed for backward compatibility. 1688 */ 1689 spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport)); 1690 1691 if (tgroup->transport->ops->poll_group_dump_stat) { 1692 tgroup->transport->ops->poll_group_dump_stat(tgroup, w); 1693 } 1694 1695 spdk_json_write_object_end(w); 1696 } 1697 1698 spdk_json_write_array_end(w); 1699 spdk_json_write_object_end(w); 1700 } 1701