1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2018 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/bdev.h" 37 #include "spdk/bit_array.h" 38 #include "spdk/conf.h" 39 #include "spdk/thread.h" 40 #include "spdk/nvmf.h" 41 #include "spdk/trace.h" 42 #include "spdk/endian.h" 43 #include "spdk/string.h" 44 45 #include "spdk_internal/log.h" 46 47 #include "nvmf_internal.h" 48 #include "transport.h" 49 50 SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF) 51 52 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 53 54 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 55 static void spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf); 56 57 /* supplied to a single call to nvmf_qpair_disconnect */ 58 struct nvmf_qpair_disconnect_ctx { 59 struct spdk_nvmf_qpair *qpair; 60 struct spdk_nvmf_ctrlr *ctrlr; 61 nvmf_qpair_disconnect_cb cb_fn; 62 struct spdk_thread *thread; 63 void *ctx; 64 uint16_t qid; 65 }; 66 67 /* 68 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 69 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 70 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 71 */ 72 struct nvmf_qpair_disconnect_many_ctx { 73 struct spdk_nvmf_subsystem *subsystem; 74 struct spdk_nvmf_poll_group *group; 75 spdk_nvmf_poll_group_mod_done cpl_fn; 76 void *cpl_ctx; 77 }; 78 79 static void 80 spdk_nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 81 enum spdk_nvmf_qpair_state state) 82 { 83 assert(qpair != NULL); 84 assert(qpair->group->thread == spdk_get_thread()); 85 86 qpair->state = state; 87 } 88 89 static int 90 spdk_nvmf_poll_group_poll(void *ctx) 91 { 92 struct spdk_nvmf_poll_group *group = ctx; 93 int rc; 94 int count = 0; 95 struct spdk_nvmf_transport_poll_group *tgroup; 96 97 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 98 rc = spdk_nvmf_transport_poll_group_poll(tgroup); 99 if (rc < 0) { 100 return -1; 101 } 102 count += rc; 103 } 104 105 return count; 106 } 107 108 static int 109 spdk_nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 110 { 111 struct spdk_nvmf_tgt *tgt = io_device; 112 struct spdk_nvmf_poll_group *group = ctx_buf; 113 struct spdk_nvmf_transport *transport; 114 uint32_t sid; 115 116 TAILQ_INIT(&group->tgroups); 117 TAILQ_INIT(&group->qpairs); 118 119 TAILQ_FOREACH(transport, &tgt->transports, link) { 120 spdk_nvmf_poll_group_add_transport(group, transport); 121 } 122 123 group->num_sgroups = tgt->max_subsystems; 124 group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 125 if (!group->sgroups) { 126 return -ENOMEM; 127 } 128 129 for (sid = 0; sid < tgt->max_subsystems; sid++) { 130 struct spdk_nvmf_subsystem *subsystem; 131 132 subsystem = tgt->subsystems[sid]; 133 if (!subsystem) { 134 continue; 135 } 136 137 if (spdk_nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { 138 spdk_nvmf_tgt_destroy_poll_group(io_device, ctx_buf); 139 return -1; 140 } 141 } 142 143 group->poller = spdk_poller_register(spdk_nvmf_poll_group_poll, group, 0); 144 group->thread = spdk_get_thread(); 145 146 return 0; 147 } 148 149 static void 150 spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 151 { 152 struct spdk_nvmf_poll_group *group = ctx_buf; 153 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 154 struct spdk_nvmf_subsystem_poll_group *sgroup; 155 uint32_t sid, nsid; 156 157 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 158 TAILQ_REMOVE(&group->tgroups, tgroup, link); 159 spdk_nvmf_transport_poll_group_destroy(tgroup); 160 } 161 162 for (sid = 0; sid < group->num_sgroups; sid++) { 163 sgroup = &group->sgroups[sid]; 164 165 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 166 if (sgroup->ns_info[nsid].channel) { 167 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 168 sgroup->ns_info[nsid].channel = NULL; 169 } 170 } 171 172 free(sgroup->ns_info); 173 } 174 175 free(group->sgroups); 176 } 177 178 static void 179 _nvmf_tgt_disconnect_next_qpair(void *ctx) 180 { 181 struct spdk_nvmf_qpair *qpair; 182 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 183 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 184 struct spdk_io_channel *ch; 185 int rc = 0; 186 187 qpair = TAILQ_FIRST(&group->qpairs); 188 189 if (qpair) { 190 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_tgt_disconnect_next_qpair, ctx); 191 } 192 193 if (!qpair || rc != 0) { 194 /* When the refcount from the channels reaches 0, spdk_nvmf_tgt_destroy_poll_group will be called. */ 195 ch = spdk_io_channel_from_ctx(group); 196 spdk_put_io_channel(ch); 197 free(qpair_ctx); 198 } 199 } 200 201 static void 202 spdk_nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 203 { 204 struct nvmf_qpair_disconnect_many_ctx *ctx; 205 206 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 207 208 if (!ctx) { 209 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 210 return; 211 } 212 213 spdk_poller_unregister(&group->poller); 214 215 ctx->group = group; 216 _nvmf_tgt_disconnect_next_qpair(ctx); 217 } 218 219 struct spdk_nvmf_tgt * 220 spdk_nvmf_tgt_create(uint32_t max_subsystems) 221 { 222 struct spdk_nvmf_tgt *tgt; 223 224 tgt = calloc(1, sizeof(*tgt)); 225 if (!tgt) { 226 return NULL; 227 } 228 229 if (!max_subsystems) { 230 tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 231 } else { 232 tgt->max_subsystems = max_subsystems; 233 } 234 235 tgt->discovery_genctr = 0; 236 tgt->discovery_log_page = NULL; 237 tgt->discovery_log_page_size = 0; 238 TAILQ_INIT(&tgt->transports); 239 240 tgt->subsystems = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem *)); 241 if (!tgt->subsystems) { 242 free(tgt); 243 return NULL; 244 } 245 246 spdk_io_device_register(tgt, 247 spdk_nvmf_tgt_create_poll_group, 248 spdk_nvmf_tgt_destroy_poll_group, 249 sizeof(struct spdk_nvmf_poll_group), 250 "nvmf_tgt"); 251 252 return tgt; 253 } 254 255 static void 256 spdk_nvmf_tgt_destroy_cb(void *io_device) 257 { 258 struct spdk_nvmf_tgt *tgt = io_device; 259 struct spdk_nvmf_transport *transport, *transport_tmp; 260 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn; 261 void *destroy_cb_arg; 262 uint32_t i; 263 264 if (tgt->discovery_log_page) { 265 free(tgt->discovery_log_page); 266 } 267 268 if (tgt->subsystems) { 269 for (i = 0; i < tgt->max_subsystems; i++) { 270 if (tgt->subsystems[i]) { 271 spdk_nvmf_subsystem_destroy(tgt->subsystems[i]); 272 } 273 } 274 free(tgt->subsystems); 275 } 276 277 TAILQ_FOREACH_SAFE(transport, &tgt->transports, link, transport_tmp) { 278 TAILQ_REMOVE(&tgt->transports, transport, link); 279 spdk_nvmf_transport_destroy(transport); 280 } 281 282 destroy_cb_fn = tgt->destroy_cb_fn; 283 destroy_cb_arg = tgt->destroy_cb_arg; 284 285 free(tgt); 286 287 if (destroy_cb_fn) { 288 destroy_cb_fn(destroy_cb_arg, 0); 289 } 290 } 291 292 void 293 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 294 spdk_nvmf_tgt_destroy_done_fn cb_fn, 295 void *cb_arg) 296 { 297 tgt->destroy_cb_fn = cb_fn; 298 tgt->destroy_cb_arg = cb_arg; 299 300 spdk_io_device_unregister(tgt, spdk_nvmf_tgt_destroy_cb); 301 } 302 303 static void 304 spdk_nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 305 struct spdk_nvmf_subsystem *subsystem) 306 { 307 struct spdk_nvmf_host *host; 308 struct spdk_nvmf_listener *listener; 309 const struct spdk_nvme_transport_id *trid; 310 struct spdk_nvmf_ns *ns; 311 struct spdk_nvmf_ns_opts ns_opts; 312 uint32_t max_namespaces; 313 char uuid_str[SPDK_UUID_STRING_LEN]; 314 const char *trtype; 315 const char *adrfam; 316 317 if (spdk_nvmf_subsystem_get_type(subsystem) != SPDK_NVMF_SUBTYPE_NVME) { 318 return; 319 } 320 321 /* { */ 322 spdk_json_write_object_begin(w); 323 spdk_json_write_named_string(w, "method", "nvmf_subsystem_create"); 324 325 /* "params" : { */ 326 spdk_json_write_named_object_begin(w, "params"); 327 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 328 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 329 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 330 331 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 332 if (max_namespaces != 0) { 333 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 334 } 335 336 /* } "params" */ 337 spdk_json_write_object_end(w); 338 339 /* } */ 340 spdk_json_write_object_end(w); 341 342 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 343 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 344 trid = spdk_nvmf_listener_get_trid(listener); 345 346 trtype = spdk_nvme_transport_id_trtype_str(trid->trtype); 347 adrfam = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 348 349 spdk_json_write_object_begin(w); 350 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 351 352 /* "params" : { */ 353 spdk_json_write_named_object_begin(w, "params"); 354 355 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 356 357 /* "listen_address" : { */ 358 spdk_json_write_named_object_begin(w, "listen_address"); 359 360 spdk_json_write_named_string(w, "trtype", trtype); 361 if (adrfam) { 362 spdk_json_write_named_string(w, "adrfam", adrfam); 363 } 364 365 spdk_json_write_named_string(w, "traddr", trid->traddr); 366 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 367 /* } "listen_address" */ 368 spdk_json_write_object_end(w); 369 370 /* } "params" */ 371 spdk_json_write_object_end(w); 372 373 /* } */ 374 spdk_json_write_object_end(w); 375 } 376 377 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 378 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 379 380 spdk_json_write_object_begin(w); 381 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 382 383 /* "params" : { */ 384 spdk_json_write_named_object_begin(w, "params"); 385 386 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 387 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 388 389 /* } "params" */ 390 spdk_json_write_object_end(w); 391 392 /* } */ 393 spdk_json_write_object_end(w); 394 } 395 396 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 397 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 398 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 399 400 spdk_json_write_object_begin(w); 401 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 402 403 /* "params" : { */ 404 spdk_json_write_named_object_begin(w, "params"); 405 406 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 407 408 /* "namespace" : { */ 409 spdk_json_write_named_object_begin(w, "namespace"); 410 411 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 412 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 413 414 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 415 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 416 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 417 from_be64(&ns_opts.nguid[8])); 418 } 419 420 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 421 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 422 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 423 } 424 425 if (!spdk_mem_all_zero(&ns_opts.uuid, sizeof(ns_opts.uuid))) { 426 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &ns_opts.uuid); 427 spdk_json_write_named_string(w, "uuid", uuid_str); 428 } 429 430 /* "namespace" */ 431 spdk_json_write_object_end(w); 432 433 /* } "params" */ 434 spdk_json_write_object_end(w); 435 436 /* } */ 437 spdk_json_write_object_end(w); 438 } 439 } 440 441 void 442 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 443 { 444 struct spdk_nvmf_subsystem *subsystem; 445 struct spdk_nvmf_transport *transport; 446 447 spdk_json_write_object_begin(w); 448 spdk_json_write_named_string(w, "method", "set_nvmf_target_max_subsystems"); 449 450 spdk_json_write_named_object_begin(w, "params"); 451 spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems); 452 spdk_json_write_object_end(w); 453 454 spdk_json_write_object_end(w); 455 456 /* write transports */ 457 TAILQ_FOREACH(transport, &tgt->transports, link) { 458 spdk_json_write_object_begin(w); 459 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 460 461 spdk_json_write_named_object_begin(w, "params"); 462 spdk_json_write_named_string(w, "trtype", spdk_nvme_transport_id_trtype_str(transport->ops->type)); 463 spdk_json_write_named_uint32(w, "max_queue_depth", transport->opts.max_queue_depth); 464 spdk_json_write_named_uint32(w, "max_qpairs_per_ctrlr", transport->opts.max_qpairs_per_ctrlr); 465 spdk_json_write_named_uint32(w, "in_capsule_data_size", transport->opts.in_capsule_data_size); 466 spdk_json_write_named_uint32(w, "max_io_size", transport->opts.max_io_size); 467 spdk_json_write_named_uint32(w, "io_unit_size", transport->opts.io_unit_size); 468 spdk_json_write_named_uint32(w, "max_aq_depth", transport->opts.max_aq_depth); 469 if (transport->ops->type == SPDK_NVME_TRANSPORT_RDMA) { 470 spdk_json_write_named_uint32(w, "max_srq_depth", transport->opts.max_srq_depth); 471 } 472 spdk_json_write_object_end(w); 473 474 spdk_json_write_object_end(w); 475 } 476 477 subsystem = spdk_nvmf_subsystem_get_first(tgt); 478 while (subsystem) { 479 spdk_nvmf_write_subsystem_config_json(w, subsystem); 480 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 481 } 482 } 483 484 void 485 spdk_nvmf_tgt_listen(struct spdk_nvmf_tgt *tgt, 486 struct spdk_nvme_transport_id *trid, 487 spdk_nvmf_tgt_listen_done_fn cb_fn, 488 void *cb_arg) 489 { 490 struct spdk_nvmf_transport *transport; 491 const char *trtype; 492 int rc; 493 494 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trtype); 495 if (!transport) { 496 trtype = spdk_nvme_transport_id_trtype_str(trid->trtype); 497 if (trtype != NULL) { 498 SPDK_ERRLOG("Unable to listen on transport %s. The transport must be created first.\n", trtype); 499 } else { 500 SPDK_ERRLOG("The specified trtype %d is unknown. Please make sure that it is properly registered.\n", 501 trid->trtype); 502 } 503 cb_fn(cb_arg, -EINVAL); 504 return; 505 } 506 507 rc = spdk_nvmf_transport_listen(transport, trid); 508 if (rc < 0) { 509 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 510 cb_fn(cb_arg, rc); 511 return; 512 } 513 514 tgt->discovery_genctr++; 515 516 cb_fn(cb_arg, 0); 517 } 518 519 struct spdk_nvmf_tgt_add_transport_ctx { 520 struct spdk_nvmf_tgt *tgt; 521 struct spdk_nvmf_transport *transport; 522 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 523 void *cb_arg; 524 }; 525 526 static void 527 _spdk_nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 528 { 529 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 530 531 ctx->cb_fn(ctx->cb_arg, status); 532 533 free(ctx); 534 } 535 536 static void 537 _spdk_nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 538 { 539 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 540 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 541 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 542 int rc; 543 544 rc = spdk_nvmf_poll_group_add_transport(group, ctx->transport); 545 spdk_for_each_channel_continue(i, rc); 546 } 547 548 void spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 549 struct spdk_nvmf_transport *transport, 550 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 551 void *cb_arg) 552 { 553 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 554 555 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->type)) { 556 cb_fn(cb_arg, -EEXIST); 557 return; /* transport already created */ 558 } 559 560 transport->tgt = tgt; 561 TAILQ_INSERT_TAIL(&tgt->transports, transport, link); 562 563 ctx = calloc(1, sizeof(*ctx)); 564 if (!ctx) { 565 cb_fn(cb_arg, -ENOMEM); 566 return; 567 } 568 569 ctx->tgt = tgt; 570 ctx->transport = transport; 571 ctx->cb_fn = cb_fn; 572 ctx->cb_arg = cb_arg; 573 574 spdk_for_each_channel(tgt, 575 _spdk_nvmf_tgt_add_transport, 576 ctx, 577 _spdk_nvmf_tgt_add_transport_done); 578 } 579 580 struct spdk_nvmf_subsystem * 581 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 582 { 583 struct spdk_nvmf_subsystem *subsystem; 584 uint32_t sid; 585 586 if (!subnqn) { 587 return NULL; 588 } 589 590 for (sid = 0; sid < tgt->max_subsystems; sid++) { 591 subsystem = tgt->subsystems[sid]; 592 if (subsystem == NULL) { 593 continue; 594 } 595 596 if (strcmp(subnqn, subsystem->subnqn) == 0) { 597 return subsystem; 598 } 599 } 600 601 return NULL; 602 } 603 604 struct spdk_nvmf_transport * 605 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, enum spdk_nvme_transport_type type) 606 { 607 struct spdk_nvmf_transport *transport; 608 609 TAILQ_FOREACH(transport, &tgt->transports, link) { 610 if (transport->ops->type == type) { 611 return transport; 612 } 613 } 614 615 return NULL; 616 } 617 618 void 619 spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt, new_qpair_fn cb_fn) 620 { 621 struct spdk_nvmf_transport *transport, *tmp; 622 623 TAILQ_FOREACH_SAFE(transport, &tgt->transports, link, tmp) { 624 spdk_nvmf_transport_accept(transport, cb_fn); 625 } 626 } 627 628 struct spdk_nvmf_poll_group * 629 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 630 { 631 struct spdk_io_channel *ch; 632 633 ch = spdk_get_io_channel(tgt); 634 if (!ch) { 635 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 636 return NULL; 637 } 638 639 return spdk_io_channel_get_ctx(ch); 640 } 641 642 void 643 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group) 644 { 645 /* This function will put the io_channel associated with this poll group */ 646 spdk_nvmf_tgt_destroy_poll_group_qpairs(group); 647 } 648 649 int 650 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 651 struct spdk_nvmf_qpair *qpair) 652 { 653 int rc = -1; 654 struct spdk_nvmf_transport_poll_group *tgroup; 655 656 TAILQ_INIT(&qpair->outstanding); 657 qpair->group = group; 658 659 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 660 if (tgroup->transport == qpair->transport) { 661 rc = spdk_nvmf_transport_poll_group_add(tgroup, qpair); 662 break; 663 } 664 } 665 666 /* We add the qpair to the group only it is succesfully added into the tgroup */ 667 if (rc == 0) { 668 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 669 spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVE); 670 } 671 672 return rc; 673 } 674 675 static 676 void _nvmf_ctrlr_destruct(void *ctx) 677 { 678 struct spdk_nvmf_ctrlr *ctrlr = ctx; 679 680 spdk_nvmf_ctrlr_destruct(ctrlr); 681 } 682 683 static void 684 _spdk_nvmf_ctrlr_free_from_qpair(void *ctx) 685 { 686 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 687 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 688 uint32_t count; 689 690 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 691 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 692 if (count == 0) { 693 spdk_bit_array_free(&ctrlr->qpair_mask); 694 695 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 696 } 697 698 if (qpair_ctx->cb_fn) { 699 spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx); 700 } 701 free(qpair_ctx); 702 } 703 704 static void 705 _spdk_nvmf_qpair_destroy(void *ctx, int status) 706 { 707 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 708 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 709 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 710 struct spdk_nvmf_transport_poll_group *tgroup; 711 struct spdk_nvmf_request *req, *tmp; 712 struct spdk_nvmf_subsystem_poll_group *sgroup; 713 int rc; 714 715 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 716 spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR); 717 qpair_ctx->qid = qpair->qid; 718 719 /* Find the tgroup and remove the qpair from the tgroup */ 720 TAILQ_FOREACH(tgroup, &qpair->group->tgroups, link) { 721 if (tgroup->transport == qpair->transport) { 722 rc = spdk_nvmf_transport_poll_group_remove(tgroup, qpair); 723 if (rc && (rc != ENOTSUP)) { 724 SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n", 725 qpair, tgroup); 726 } 727 break; 728 } 729 } 730 731 if (ctrlr) { 732 sgroup = &qpair->group->sgroups[ctrlr->subsys->id]; 733 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 734 if (req->qpair == qpair) { 735 TAILQ_REMOVE(&sgroup->queued, req, link); 736 if (spdk_nvmf_transport_req_free(req)) { 737 SPDK_ERRLOG("Transport request free error!\n"); 738 } 739 } 740 } 741 } 742 743 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 744 745 spdk_nvmf_transport_qpair_fini(qpair); 746 747 if (!ctrlr || !ctrlr->thread) { 748 if (qpair_ctx->cb_fn) { 749 spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx); 750 } 751 free(qpair_ctx); 752 return; 753 } 754 755 qpair_ctx->ctrlr = ctrlr; 756 spdk_thread_send_msg(ctrlr->thread, _spdk_nvmf_ctrlr_free_from_qpair, qpair_ctx); 757 758 } 759 760 int 761 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) 762 { 763 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 764 765 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 766 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 767 spdk_nvmf_transport_qpair_fini(qpair); 768 if (cb_fn) { 769 cb_fn(ctx); 770 } 771 return 0; 772 } 773 774 /* The queue pair must be disconnected from the thread that owns it */ 775 assert(qpair->group->thread == spdk_get_thread()); 776 777 if (qpair->state != SPDK_NVMF_QPAIR_ACTIVE) { 778 /* This can occur if the connection is killed by the target, 779 * which results in a notification that the connection 780 * died. Send a message to defer the processing of this 781 * callback. This allows the stack to unwind in the case 782 * where a bunch of connections are disconnected in 783 * a loop. */ 784 if (cb_fn) { 785 spdk_thread_send_msg(qpair->group->thread, cb_fn, ctx); 786 } 787 return 0; 788 } 789 790 assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); 791 spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 792 793 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 794 if (!qpair_ctx) { 795 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 796 return -ENOMEM; 797 } 798 799 qpair_ctx->qpair = qpair; 800 qpair_ctx->cb_fn = cb_fn; 801 qpair_ctx->thread = qpair->group->thread; 802 qpair_ctx->ctx = ctx; 803 804 /* Check for outstanding I/O */ 805 if (!TAILQ_EMPTY(&qpair->outstanding)) { 806 qpair->state_cb = _spdk_nvmf_qpair_destroy; 807 qpair->state_cb_arg = qpair_ctx; 808 spdk_nvmf_qpair_free_aer(qpair); 809 return 0; 810 } 811 812 _spdk_nvmf_qpair_destroy(qpair_ctx, 0); 813 814 return 0; 815 } 816 817 int 818 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 819 struct spdk_nvme_transport_id *trid) 820 { 821 return spdk_nvmf_transport_qpair_get_peer_trid(qpair, trid); 822 } 823 824 int 825 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 826 struct spdk_nvme_transport_id *trid) 827 { 828 return spdk_nvmf_transport_qpair_get_local_trid(qpair, trid); 829 } 830 831 int 832 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 833 struct spdk_nvme_transport_id *trid) 834 { 835 return spdk_nvmf_transport_qpair_get_listen_trid(qpair, trid); 836 } 837 838 int 839 spdk_nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 840 struct spdk_nvmf_transport *transport) 841 { 842 struct spdk_nvmf_transport_poll_group *tgroup; 843 844 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 845 if (tgroup->transport == transport) { 846 /* Transport already in the poll group */ 847 return 0; 848 } 849 } 850 851 tgroup = spdk_nvmf_transport_poll_group_create(transport); 852 if (!tgroup) { 853 SPDK_ERRLOG("Unable to create poll group for transport\n"); 854 return -1; 855 } 856 857 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 858 859 return 0; 860 } 861 862 static int 863 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 864 struct spdk_nvmf_subsystem *subsystem) 865 { 866 struct spdk_nvmf_subsystem_poll_group *sgroup; 867 uint32_t new_num_ns, old_num_ns; 868 uint32_t i, j; 869 struct spdk_nvmf_ns *ns; 870 struct spdk_nvmf_registrant *reg, *tmp; 871 872 /* Make sure our poll group has memory for this subsystem allocated */ 873 if (subsystem->id >= group->num_sgroups) { 874 return -ENOMEM; 875 } 876 877 sgroup = &group->sgroups[subsystem->id]; 878 879 /* Make sure the array of namespace information is the correct size */ 880 new_num_ns = subsystem->max_nsid; 881 old_num_ns = sgroup->num_ns; 882 883 if (old_num_ns == 0) { 884 if (new_num_ns > 0) { 885 /* First allocation */ 886 sgroup->ns_info = calloc(new_num_ns, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 887 if (!sgroup->ns_info) { 888 return -ENOMEM; 889 } 890 } 891 } else if (new_num_ns > old_num_ns) { 892 void *buf; 893 894 /* Make the array larger */ 895 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 896 if (!buf) { 897 return -ENOMEM; 898 } 899 900 sgroup->ns_info = buf; 901 902 /* Null out the new namespace information slots */ 903 for (i = old_num_ns; i < new_num_ns; i++) { 904 memset(&sgroup->ns_info[i], 0, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 905 } 906 } else if (new_num_ns < old_num_ns) { 907 void *buf; 908 909 /* Free the extra I/O channels */ 910 for (i = new_num_ns; i < old_num_ns; i++) { 911 if (sgroup->ns_info[i].channel) { 912 spdk_put_io_channel(sgroup->ns_info[i].channel); 913 sgroup->ns_info[i].channel = NULL; 914 } 915 } 916 917 /* Make the array smaller */ 918 if (new_num_ns > 0) { 919 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 920 if (!buf) { 921 return -ENOMEM; 922 } 923 sgroup->ns_info = buf; 924 } else { 925 free(sgroup->ns_info); 926 sgroup->ns_info = NULL; 927 } 928 } 929 930 sgroup->num_ns = new_num_ns; 931 932 /* Detect bdevs that were added or removed */ 933 for (i = 0; i < sgroup->num_ns; i++) { 934 ns = subsystem->ns[i]; 935 if (ns == NULL && sgroup->ns_info[i].channel == NULL) { 936 /* Both NULL. Leave empty */ 937 } else if (ns == NULL && sgroup->ns_info[i].channel != NULL) { 938 /* There was a channel here, but the namespace is gone. */ 939 spdk_put_io_channel(sgroup->ns_info[i].channel); 940 sgroup->ns_info[i].channel = NULL; 941 } else if (ns != NULL && sgroup->ns_info[i].channel == NULL) { 942 /* A namespace appeared but there is no channel yet */ 943 sgroup->ns_info[i].channel = spdk_bdev_get_io_channel(ns->desc); 944 if (sgroup->ns_info[i].channel == NULL) { 945 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 946 return -ENOMEM; 947 } 948 } else { 949 /* A namespace was present before and didn't change. */ 950 } 951 952 if (ns == NULL) { 953 memset(&sgroup->ns_info[i], 0, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 954 } else if (ns->rtype && ns->holder) { 955 sgroup->ns_info[i].crkey = ns->crkey; 956 sgroup->ns_info[i].rtype = ns->rtype; 957 sgroup->ns_info[i].holder_id = ns->holder->hostid; 958 959 memset(&sgroup->ns_info[i].reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid)); 960 j = 0; 961 TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { 962 if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) { 963 SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS); 964 return -EINVAL; 965 } 966 sgroup->ns_info[i].reg_hostid[j++] = reg->hostid; 967 } 968 } 969 } 970 971 return 0; 972 } 973 974 int 975 spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 976 struct spdk_nvmf_subsystem *subsystem) 977 { 978 return poll_group_update_subsystem(group, subsystem); 979 } 980 981 int 982 spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 983 struct spdk_nvmf_subsystem *subsystem, 984 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 985 { 986 int rc = 0; 987 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 988 989 TAILQ_INIT(&sgroup->queued); 990 991 rc = poll_group_update_subsystem(group, subsystem); 992 if (rc) { 993 spdk_nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); 994 goto fini; 995 } 996 997 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 998 fini: 999 if (cb_fn) { 1000 cb_fn(cb_arg, rc); 1001 } 1002 1003 return rc; 1004 } 1005 1006 static void 1007 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 1008 { 1009 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1010 struct spdk_nvmf_subsystem *subsystem; 1011 struct spdk_nvmf_poll_group *group; 1012 struct spdk_nvmf_subsystem_poll_group *sgroup; 1013 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 1014 void *cpl_ctx = NULL; 1015 uint32_t nsid; 1016 1017 group = qpair_ctx->group; 1018 subsystem = qpair_ctx->subsystem; 1019 cpl_fn = qpair_ctx->cpl_fn; 1020 cpl_ctx = qpair_ctx->cpl_ctx; 1021 sgroup = &group->sgroups[subsystem->id]; 1022 1023 if (status) { 1024 goto fini; 1025 } 1026 1027 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 1028 if (sgroup->ns_info[nsid].channel) { 1029 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 1030 sgroup->ns_info[nsid].channel = NULL; 1031 } 1032 } 1033 1034 sgroup->num_ns = 0; 1035 free(sgroup->ns_info); 1036 sgroup->ns_info = NULL; 1037 fini: 1038 free(qpair_ctx); 1039 if (cpl_fn) { 1040 cpl_fn(cpl_ctx, status); 1041 } 1042 } 1043 1044 static void 1045 _nvmf_subsystem_disconnect_next_qpair(void *ctx) 1046 { 1047 struct spdk_nvmf_qpair *qpair; 1048 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1049 struct spdk_nvmf_subsystem *subsystem; 1050 struct spdk_nvmf_poll_group *group; 1051 int rc = 0; 1052 1053 group = qpair_ctx->group; 1054 subsystem = qpair_ctx->subsystem; 1055 1056 TAILQ_FOREACH(qpair, &group->qpairs, link) { 1057 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1058 break; 1059 } 1060 } 1061 1062 if (qpair) { 1063 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_subsystem_disconnect_next_qpair, qpair_ctx); 1064 } 1065 1066 if (!qpair || rc != 0) { 1067 _nvmf_poll_group_remove_subsystem_cb(ctx, rc); 1068 } 1069 return; 1070 } 1071 1072 void 1073 spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1074 struct spdk_nvmf_subsystem *subsystem, 1075 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1076 { 1077 struct spdk_nvmf_qpair *qpair; 1078 struct spdk_nvmf_subsystem_poll_group *sgroup; 1079 struct nvmf_qpair_disconnect_many_ctx *ctx; 1080 int rc = 0; 1081 1082 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1083 1084 if (!ctx) { 1085 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1086 goto fini; 1087 } 1088 1089 ctx->group = group; 1090 ctx->subsystem = subsystem; 1091 ctx->cpl_fn = cb_fn; 1092 ctx->cpl_ctx = cb_arg; 1093 1094 sgroup = &group->sgroups[subsystem->id]; 1095 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1096 1097 TAILQ_FOREACH(qpair, &group->qpairs, link) { 1098 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1099 break; 1100 } 1101 } 1102 1103 if (qpair) { 1104 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_subsystem_disconnect_next_qpair, ctx); 1105 } else { 1106 /* call the callback immediately. It will handle any channel iteration */ 1107 _nvmf_poll_group_remove_subsystem_cb(ctx, 0); 1108 } 1109 1110 if (rc != 0) { 1111 free(ctx); 1112 goto fini; 1113 } 1114 1115 return; 1116 fini: 1117 if (cb_fn) { 1118 cb_fn(cb_arg, rc); 1119 } 1120 } 1121 1122 void 1123 spdk_nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1124 struct spdk_nvmf_subsystem *subsystem, 1125 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1126 { 1127 struct spdk_nvmf_subsystem_poll_group *sgroup; 1128 int rc = 0; 1129 1130 if (subsystem->id >= group->num_sgroups) { 1131 rc = -1; 1132 goto fini; 1133 } 1134 1135 sgroup = &group->sgroups[subsystem->id]; 1136 if (sgroup == NULL) { 1137 rc = -1; 1138 goto fini; 1139 } 1140 1141 assert(sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE); 1142 /* TODO: This currently does not quiesce I/O */ 1143 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1144 fini: 1145 if (cb_fn) { 1146 cb_fn(cb_arg, rc); 1147 } 1148 } 1149 1150 void 1151 spdk_nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1152 struct spdk_nvmf_subsystem *subsystem, 1153 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1154 { 1155 struct spdk_nvmf_request *req, *tmp; 1156 struct spdk_nvmf_subsystem_poll_group *sgroup; 1157 int rc = 0; 1158 1159 if (subsystem->id >= group->num_sgroups) { 1160 rc = -1; 1161 goto fini; 1162 } 1163 1164 sgroup = &group->sgroups[subsystem->id]; 1165 1166 assert(sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED); 1167 1168 rc = poll_group_update_subsystem(group, subsystem); 1169 if (rc) { 1170 goto fini; 1171 } 1172 1173 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1174 1175 /* Release all queued requests */ 1176 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1177 TAILQ_REMOVE(&sgroup->queued, req, link); 1178 spdk_nvmf_request_exec(req); 1179 } 1180 fini: 1181 if (cb_fn) { 1182 cb_fn(cb_arg, rc); 1183 } 1184 } 1185