1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2018-2019 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "nvmf_internal.h" 37 #include "transport.h" 38 39 #include "spdk/config.h" 40 #include "spdk/log.h" 41 #include "spdk/nvmf.h" 42 #include "spdk/nvmf_transport.h" 43 #include "spdk/queue.h" 44 #include "spdk/util.h" 45 46 #define MAX_MEMPOOL_NAME_LENGTH 40 47 #define NVMF_TRANSPORT_DEFAULT_ASSOCIATION_TIMEOUT_IN_MS 120000 48 49 struct nvmf_transport_ops_list_element { 50 struct spdk_nvmf_transport_ops ops; 51 TAILQ_ENTRY(nvmf_transport_ops_list_element) link; 52 }; 53 54 TAILQ_HEAD(nvmf_transport_ops_list, nvmf_transport_ops_list_element) 55 g_spdk_nvmf_transport_ops = TAILQ_HEAD_INITIALIZER(g_spdk_nvmf_transport_ops); 56 57 static inline const struct spdk_nvmf_transport_ops * 58 nvmf_get_transport_ops(const char *transport_name) 59 { 60 struct nvmf_transport_ops_list_element *ops; 61 TAILQ_FOREACH(ops, &g_spdk_nvmf_transport_ops, link) { 62 if (strcasecmp(transport_name, ops->ops.name) == 0) { 63 return &ops->ops; 64 } 65 } 66 return NULL; 67 } 68 69 void 70 spdk_nvmf_transport_register(const struct spdk_nvmf_transport_ops *ops) 71 { 72 struct nvmf_transport_ops_list_element *new_ops; 73 74 if (nvmf_get_transport_ops(ops->name) != NULL) { 75 SPDK_ERRLOG("Double registering nvmf transport type %s.\n", ops->name); 76 assert(false); 77 return; 78 } 79 80 new_ops = calloc(1, sizeof(*new_ops)); 81 if (new_ops == NULL) { 82 SPDK_ERRLOG("Unable to allocate memory to register new transport type %s.\n", ops->name); 83 assert(false); 84 return; 85 } 86 87 new_ops->ops = *ops; 88 89 TAILQ_INSERT_TAIL(&g_spdk_nvmf_transport_ops, new_ops, link); 90 } 91 92 const struct spdk_nvmf_transport_opts * 93 spdk_nvmf_get_transport_opts(struct spdk_nvmf_transport *transport) 94 { 95 return &transport->opts; 96 } 97 98 spdk_nvme_transport_type_t 99 spdk_nvmf_get_transport_type(struct spdk_nvmf_transport *transport) 100 { 101 return transport->ops->type; 102 } 103 104 const char * 105 spdk_nvmf_get_transport_name(struct spdk_nvmf_transport *transport) 106 { 107 return transport->ops->name; 108 } 109 110 static void nvmf_transport_opts_copy(struct spdk_nvmf_transport_opts *opts, 111 struct spdk_nvmf_transport_opts *opts_src, 112 size_t opts_size) 113 { 114 assert(opts); 115 assert(opts_src); 116 117 opts->opts_size = opts_size; 118 119 #define SET_FIELD(field) \ 120 if (offsetof(struct spdk_nvmf_transport_opts, field) + sizeof(opts->field) <= opts_size) { \ 121 opts->field = opts_src->field; \ 122 } \ 123 124 SET_FIELD(max_queue_depth); 125 SET_FIELD(max_qpairs_per_ctrlr); 126 SET_FIELD(in_capsule_data_size); 127 SET_FIELD(max_io_size); 128 SET_FIELD(io_unit_size); 129 SET_FIELD(max_aq_depth); 130 SET_FIELD(buf_cache_size); 131 SET_FIELD(num_shared_buffers); 132 SET_FIELD(dif_insert_or_strip); 133 SET_FIELD(abort_timeout_sec); 134 SET_FIELD(association_timeout); 135 SET_FIELD(transport_specific); 136 137 /* Do not remove this statement, you should always update this statement when you adding a new field, 138 * and do not forget to add the SET_FIELD statement for your added field. */ 139 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_transport_opts) == 56, "Incorrect size"); 140 141 #undef SET_FIELD 142 #undef FILED_CHECK 143 } 144 145 struct spdk_nvmf_transport * 146 spdk_nvmf_transport_create(const char *transport_name, struct spdk_nvmf_transport_opts *opts) 147 { 148 const struct spdk_nvmf_transport_ops *ops = NULL; 149 struct spdk_nvmf_transport *transport; 150 char spdk_mempool_name[MAX_MEMPOOL_NAME_LENGTH]; 151 int chars_written; 152 struct spdk_nvmf_transport_opts opts_local = {}; 153 154 if (!opts) { 155 SPDK_ERRLOG("opts should not be NULL\n"); 156 return NULL; 157 } 158 159 if (!opts->opts_size) { 160 SPDK_ERRLOG("The opts_size in opts structure should not be zero\n"); 161 return NULL; 162 } 163 164 ops = nvmf_get_transport_ops(transport_name); 165 if (!ops) { 166 SPDK_ERRLOG("Transport type '%s' unavailable.\n", transport_name); 167 return NULL; 168 } 169 nvmf_transport_opts_copy(&opts_local, opts, opts->opts_size); 170 171 if (opts_local.max_io_size != 0 && (!spdk_u32_is_pow2(opts_local.max_io_size) || 172 opts_local.max_io_size < 8192)) { 173 SPDK_ERRLOG("max_io_size %u must be a power of 2 and be greater than or equal 8KB\n", 174 opts_local.max_io_size); 175 return NULL; 176 } 177 178 if (opts_local.max_aq_depth < SPDK_NVMF_MIN_ADMIN_MAX_SQ_SIZE) { 179 SPDK_ERRLOG("max_aq_depth %u is less than minimum defined by NVMf spec, use min value\n", 180 opts_local.max_aq_depth); 181 opts_local.max_aq_depth = SPDK_NVMF_MIN_ADMIN_MAX_SQ_SIZE; 182 } 183 184 transport = ops->create(&opts_local); 185 if (!transport) { 186 SPDK_ERRLOG("Unable to create new transport of type %s\n", transport_name); 187 return NULL; 188 } 189 190 TAILQ_INIT(&transport->listeners); 191 192 transport->ops = ops; 193 transport->opts = opts_local; 194 195 chars_written = snprintf(spdk_mempool_name, MAX_MEMPOOL_NAME_LENGTH, "%s_%s_%s", "spdk_nvmf", 196 transport_name, "data"); 197 if (chars_written < 0) { 198 SPDK_ERRLOG("Unable to generate transport data buffer pool name.\n"); 199 ops->destroy(transport, NULL, NULL); 200 return NULL; 201 } 202 203 transport->data_buf_pool = spdk_mempool_create(spdk_mempool_name, 204 opts_local.num_shared_buffers, 205 opts_local.io_unit_size + NVMF_DATA_BUFFER_ALIGNMENT, 206 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 207 SPDK_ENV_SOCKET_ID_ANY); 208 209 if (!transport->data_buf_pool) { 210 SPDK_ERRLOG("Unable to allocate buffer pool for poll group\n"); 211 ops->destroy(transport, NULL, NULL); 212 return NULL; 213 } 214 215 return transport; 216 } 217 218 struct spdk_nvmf_transport * 219 spdk_nvmf_transport_get_first(struct spdk_nvmf_tgt *tgt) 220 { 221 return TAILQ_FIRST(&tgt->transports); 222 } 223 224 struct spdk_nvmf_transport * 225 spdk_nvmf_transport_get_next(struct spdk_nvmf_transport *transport) 226 { 227 return TAILQ_NEXT(transport, link); 228 } 229 230 int 231 spdk_nvmf_transport_destroy(struct spdk_nvmf_transport *transport, 232 spdk_nvmf_transport_destroy_done_cb cb_fn, void *cb_arg) 233 { 234 if (transport->data_buf_pool != NULL) { 235 if (spdk_mempool_count(transport->data_buf_pool) != 236 transport->opts.num_shared_buffers) { 237 SPDK_ERRLOG("transport buffer pool count is %zu but should be %u\n", 238 spdk_mempool_count(transport->data_buf_pool), 239 transport->opts.num_shared_buffers); 240 } 241 } 242 243 spdk_mempool_free(transport->data_buf_pool); 244 245 return transport->ops->destroy(transport, cb_fn, cb_arg); 246 } 247 248 struct spdk_nvmf_listener * 249 nvmf_transport_find_listener(struct spdk_nvmf_transport *transport, 250 const struct spdk_nvme_transport_id *trid) 251 { 252 struct spdk_nvmf_listener *listener; 253 254 TAILQ_FOREACH(listener, &transport->listeners, link) { 255 if (spdk_nvme_transport_id_compare(&listener->trid, trid) == 0) { 256 return listener; 257 } 258 } 259 260 return NULL; 261 } 262 263 int 264 spdk_nvmf_transport_listen(struct spdk_nvmf_transport *transport, 265 const struct spdk_nvme_transport_id *trid, struct spdk_nvmf_listen_opts *opts) 266 { 267 struct spdk_nvmf_listener *listener; 268 int rc; 269 270 listener = nvmf_transport_find_listener(transport, trid); 271 if (!listener) { 272 listener = calloc(1, sizeof(*listener)); 273 if (!listener) { 274 return -ENOMEM; 275 } 276 277 listener->ref = 1; 278 listener->trid = *trid; 279 TAILQ_INSERT_TAIL(&transport->listeners, listener, link); 280 rc = transport->ops->listen(transport, &listener->trid, opts); 281 if (rc != 0) { 282 TAILQ_REMOVE(&transport->listeners, listener, link); 283 free(listener); 284 } 285 return rc; 286 } 287 288 ++listener->ref; 289 290 return 0; 291 } 292 293 int 294 spdk_nvmf_transport_stop_listen(struct spdk_nvmf_transport *transport, 295 const struct spdk_nvme_transport_id *trid) 296 { 297 struct spdk_nvmf_listener *listener; 298 299 listener = nvmf_transport_find_listener(transport, trid); 300 if (!listener) { 301 return -ENOENT; 302 } 303 304 if (--listener->ref == 0) { 305 TAILQ_REMOVE(&transport->listeners, listener, link); 306 transport->ops->stop_listen(transport, trid); 307 free(listener); 308 } 309 310 return 0; 311 } 312 313 struct nvmf_stop_listen_ctx { 314 struct spdk_nvmf_transport *transport; 315 struct spdk_nvme_transport_id trid; 316 spdk_nvmf_tgt_subsystem_listen_done_fn cb_fn; 317 void *cb_arg; 318 }; 319 320 static void 321 nvmf_stop_listen_fini(struct spdk_io_channel_iter *i, int status) 322 { 323 struct nvmf_stop_listen_ctx *ctx; 324 struct spdk_nvmf_transport *transport; 325 int rc = status; 326 327 ctx = spdk_io_channel_iter_get_ctx(i); 328 transport = ctx->transport; 329 assert(transport != NULL); 330 331 rc = spdk_nvmf_transport_stop_listen(transport, &ctx->trid); 332 if (rc) { 333 SPDK_ERRLOG("Failed to stop listening on address '%s'\n", ctx->trid.traddr); 334 } 335 336 if (ctx->cb_fn) { 337 ctx->cb_fn(ctx->cb_arg, rc); 338 } 339 free(ctx); 340 } 341 342 static void 343 nvmf_stop_listen_disconnect_qpairs(struct spdk_io_channel_iter *i) 344 { 345 struct nvmf_stop_listen_ctx *ctx; 346 struct spdk_nvmf_poll_group *group; 347 struct spdk_io_channel *ch; 348 struct spdk_nvmf_qpair *qpair, *tmp_qpair; 349 struct spdk_nvme_transport_id tmp_trid; 350 351 ctx = spdk_io_channel_iter_get_ctx(i); 352 ch = spdk_io_channel_iter_get_channel(i); 353 group = spdk_io_channel_get_ctx(ch); 354 355 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, tmp_qpair) { 356 /* skip qpairs that don't match the TRID. */ 357 if (spdk_nvmf_qpair_get_listen_trid(qpair, &tmp_trid)) { 358 continue; 359 } 360 361 if (!spdk_nvme_transport_id_compare(&ctx->trid, &tmp_trid)) { 362 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 363 } 364 } 365 spdk_for_each_channel_continue(i, 0); 366 } 367 368 int 369 spdk_nvmf_transport_stop_listen_async(struct spdk_nvmf_transport *transport, 370 const struct spdk_nvme_transport_id *trid, 371 spdk_nvmf_tgt_subsystem_listen_done_fn cb_fn, 372 void *cb_arg) 373 { 374 struct nvmf_stop_listen_ctx *ctx; 375 376 ctx = calloc(1, sizeof(struct nvmf_stop_listen_ctx)); 377 if (ctx == NULL) { 378 return -ENOMEM; 379 } 380 381 ctx->trid = *trid; 382 ctx->transport = transport; 383 ctx->cb_fn = cb_fn; 384 ctx->cb_arg = cb_arg; 385 386 spdk_for_each_channel(transport->tgt, nvmf_stop_listen_disconnect_qpairs, ctx, 387 nvmf_stop_listen_fini); 388 389 return 0; 390 } 391 392 uint32_t 393 nvmf_transport_accept(struct spdk_nvmf_transport *transport) 394 { 395 return transport->ops->accept(transport); 396 } 397 398 void 399 nvmf_transport_listener_discover(struct spdk_nvmf_transport *transport, 400 struct spdk_nvme_transport_id *trid, 401 struct spdk_nvmf_discovery_log_page_entry *entry) 402 { 403 transport->ops->listener_discover(transport, trid, entry); 404 } 405 406 struct spdk_nvmf_transport_poll_group * 407 nvmf_transport_poll_group_create(struct spdk_nvmf_transport *transport) 408 { 409 struct spdk_nvmf_transport_poll_group *group; 410 struct spdk_nvmf_transport_pg_cache_buf **bufs; 411 uint32_t i; 412 413 group = transport->ops->poll_group_create(transport); 414 if (!group) { 415 return NULL; 416 } 417 group->transport = transport; 418 419 STAILQ_INIT(&group->pending_buf_queue); 420 STAILQ_INIT(&group->buf_cache); 421 422 if (transport->opts.buf_cache_size) { 423 group->buf_cache_size = transport->opts.buf_cache_size; 424 bufs = calloc(group->buf_cache_size, sizeof(struct spdk_nvmf_transport_pg_cache_buf *)); 425 426 if (!bufs) { 427 SPDK_ERRLOG("Memory allocation failed, can't reserve buffers for the pg buffer cache\n"); 428 return group; 429 } 430 431 if (spdk_mempool_get_bulk(transport->data_buf_pool, (void **)bufs, group->buf_cache_size)) { 432 group->buf_cache_size = (uint32_t)spdk_mempool_count(transport->data_buf_pool); 433 SPDK_NOTICELOG("Unable to reserve the full number of buffers for the pg buffer cache. " 434 "Decrease the number of cached buffers from %u to %u\n", 435 transport->opts.buf_cache_size, group->buf_cache_size); 436 /* Sanity check */ 437 assert(group->buf_cache_size <= transport->opts.buf_cache_size); 438 /* Try again with less number of buffers */ 439 if (spdk_mempool_get_bulk(transport->data_buf_pool, (void **)bufs, group->buf_cache_size)) { 440 SPDK_NOTICELOG("Failed to reserve %u buffers\n", group->buf_cache_size); 441 group->buf_cache_size = 0; 442 } 443 } 444 445 for (i = 0; i < group->buf_cache_size; i++) { 446 STAILQ_INSERT_HEAD(&group->buf_cache, bufs[i], link); 447 } 448 group->buf_cache_count = group->buf_cache_size; 449 450 free(bufs); 451 } 452 return group; 453 } 454 455 struct spdk_nvmf_transport_poll_group * 456 nvmf_transport_get_optimal_poll_group(struct spdk_nvmf_transport *transport, 457 struct spdk_nvmf_qpair *qpair) 458 { 459 if (transport->ops->get_optimal_poll_group) { 460 return transport->ops->get_optimal_poll_group(qpair); 461 } else { 462 return NULL; 463 } 464 } 465 466 void 467 nvmf_transport_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group) 468 { 469 struct spdk_nvmf_transport_pg_cache_buf *buf, *tmp; 470 471 if (!STAILQ_EMPTY(&group->pending_buf_queue)) { 472 SPDK_ERRLOG("Pending I/O list wasn't empty on poll group destruction\n"); 473 } 474 475 STAILQ_FOREACH_SAFE(buf, &group->buf_cache, link, tmp) { 476 STAILQ_REMOVE(&group->buf_cache, buf, spdk_nvmf_transport_pg_cache_buf, link); 477 spdk_mempool_put(group->transport->data_buf_pool, buf); 478 } 479 group->transport->ops->poll_group_destroy(group); 480 } 481 482 int 483 nvmf_transport_poll_group_add(struct spdk_nvmf_transport_poll_group *group, 484 struct spdk_nvmf_qpair *qpair) 485 { 486 if (qpair->transport) { 487 assert(qpair->transport == group->transport); 488 if (qpair->transport != group->transport) { 489 return -1; 490 } 491 } else { 492 qpair->transport = group->transport; 493 } 494 495 return group->transport->ops->poll_group_add(group, qpair); 496 } 497 498 int 499 nvmf_transport_poll_group_remove(struct spdk_nvmf_transport_poll_group *group, 500 struct spdk_nvmf_qpair *qpair) 501 { 502 int rc = ENOTSUP; 503 504 assert(qpair->transport == group->transport); 505 if (group->transport->ops->poll_group_remove) { 506 rc = group->transport->ops->poll_group_remove(group, qpair); 507 } 508 509 return rc; 510 } 511 512 int 513 nvmf_transport_poll_group_poll(struct spdk_nvmf_transport_poll_group *group) 514 { 515 return group->transport->ops->poll_group_poll(group); 516 } 517 518 int 519 nvmf_transport_req_free(struct spdk_nvmf_request *req) 520 { 521 return req->qpair->transport->ops->req_free(req); 522 } 523 524 int 525 nvmf_transport_req_complete(struct spdk_nvmf_request *req) 526 { 527 return req->qpair->transport->ops->req_complete(req); 528 } 529 530 void 531 nvmf_transport_qpair_fini(struct spdk_nvmf_qpair *qpair, 532 spdk_nvmf_transport_qpair_fini_cb cb_fn, 533 void *cb_arg) 534 { 535 qpair->transport->ops->qpair_fini(qpair, cb_fn, cb_arg); 536 } 537 538 int 539 nvmf_transport_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 540 struct spdk_nvme_transport_id *trid) 541 { 542 return qpair->transport->ops->qpair_get_peer_trid(qpair, trid); 543 } 544 545 int 546 nvmf_transport_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 547 struct spdk_nvme_transport_id *trid) 548 { 549 return qpair->transport->ops->qpair_get_local_trid(qpair, trid); 550 } 551 552 int 553 nvmf_transport_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 554 struct spdk_nvme_transport_id *trid) 555 { 556 return qpair->transport->ops->qpair_get_listen_trid(qpair, trid); 557 } 558 559 void 560 nvmf_transport_qpair_abort_request(struct spdk_nvmf_qpair *qpair, 561 struct spdk_nvmf_request *req) 562 { 563 if (qpair->transport->ops->qpair_abort_request) { 564 qpair->transport->ops->qpair_abort_request(qpair, req); 565 } 566 } 567 568 bool 569 spdk_nvmf_transport_opts_init(const char *transport_name, 570 struct spdk_nvmf_transport_opts *opts, size_t opts_size) 571 { 572 const struct spdk_nvmf_transport_ops *ops; 573 struct spdk_nvmf_transport_opts opts_local = {}; 574 575 ops = nvmf_get_transport_ops(transport_name); 576 if (!ops) { 577 SPDK_ERRLOG("Transport type %s unavailable.\n", transport_name); 578 return false; 579 } 580 581 if (!opts) { 582 SPDK_ERRLOG("opts should not be NULL\n"); 583 return false; 584 } 585 586 if (!opts_size) { 587 SPDK_ERRLOG("opts_size inside opts should not be zero value\n"); 588 return false; 589 } 590 591 opts_local.association_timeout = NVMF_TRANSPORT_DEFAULT_ASSOCIATION_TIMEOUT_IN_MS; 592 ops->opts_init(&opts_local); 593 594 nvmf_transport_opts_copy(opts, &opts_local, opts_size); 595 596 return true; 597 } 598 599 int 600 spdk_nvmf_transport_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, 601 struct spdk_nvmf_transport *transport, 602 struct spdk_nvmf_transport_poll_group_stat **stat) 603 { 604 SPDK_ERRLOG("spdk_nvmf_transport_poll_group_get_stat is deprecated and will be removed\n"); 605 606 if (transport->ops->poll_group_get_stat) { 607 return transport->ops->poll_group_get_stat(tgt, stat); 608 } else { 609 return -ENOTSUP; 610 } 611 } 612 613 void 614 spdk_nvmf_transport_poll_group_free_stat(struct spdk_nvmf_transport *transport, 615 struct spdk_nvmf_transport_poll_group_stat *stat) 616 { 617 SPDK_ERRLOG("spdk_nvmf_transport_poll_group_free_stat is deprecated and will be removed\n"); 618 619 if (transport->ops->poll_group_free_stat) { 620 transport->ops->poll_group_free_stat(stat); 621 } 622 } 623 624 void 625 spdk_nvmf_request_free_buffers(struct spdk_nvmf_request *req, 626 struct spdk_nvmf_transport_poll_group *group, 627 struct spdk_nvmf_transport *transport) 628 { 629 uint32_t i; 630 631 for (i = 0; i < req->iovcnt; i++) { 632 if (group->buf_cache_count < group->buf_cache_size) { 633 STAILQ_INSERT_HEAD(&group->buf_cache, 634 (struct spdk_nvmf_transport_pg_cache_buf *)req->buffers[i], 635 link); 636 group->buf_cache_count++; 637 } else { 638 spdk_mempool_put(transport->data_buf_pool, req->buffers[i]); 639 } 640 req->iov[i].iov_base = NULL; 641 req->buffers[i] = NULL; 642 req->iov[i].iov_len = 0; 643 } 644 req->data_from_pool = false; 645 } 646 647 static inline int 648 nvmf_request_set_buffer(struct spdk_nvmf_request *req, void *buf, uint32_t length, 649 uint32_t io_unit_size) 650 { 651 req->buffers[req->iovcnt] = buf; 652 req->iov[req->iovcnt].iov_base = (void *)((uintptr_t)(buf + NVMF_DATA_BUFFER_MASK) & 653 ~NVMF_DATA_BUFFER_MASK); 654 req->iov[req->iovcnt].iov_len = spdk_min(length, io_unit_size); 655 length -= req->iov[req->iovcnt].iov_len; 656 req->iovcnt++; 657 658 return length; 659 } 660 661 static int 662 nvmf_request_get_buffers(struct spdk_nvmf_request *req, 663 struct spdk_nvmf_transport_poll_group *group, 664 struct spdk_nvmf_transport *transport, 665 uint32_t length) 666 { 667 uint32_t io_unit_size = transport->opts.io_unit_size; 668 uint32_t num_buffers; 669 uint32_t i = 0, j; 670 void *buffer, *buffers[NVMF_REQ_MAX_BUFFERS]; 671 672 /* If the number of buffers is too large, then we know the I/O is larger than allowed. 673 * Fail it. 674 */ 675 num_buffers = SPDK_CEIL_DIV(length, io_unit_size); 676 if (num_buffers + req->iovcnt > NVMF_REQ_MAX_BUFFERS) { 677 return -EINVAL; 678 } 679 680 while (i < num_buffers) { 681 if (!(STAILQ_EMPTY(&group->buf_cache))) { 682 group->buf_cache_count--; 683 buffer = STAILQ_FIRST(&group->buf_cache); 684 STAILQ_REMOVE_HEAD(&group->buf_cache, link); 685 assert(buffer != NULL); 686 687 length = nvmf_request_set_buffer(req, buffer, length, io_unit_size); 688 i++; 689 } else { 690 if (spdk_mempool_get_bulk(transport->data_buf_pool, buffers, 691 num_buffers - i)) { 692 return -ENOMEM; 693 } 694 for (j = 0; j < num_buffers - i; j++) { 695 length = nvmf_request_set_buffer(req, buffers[j], length, io_unit_size); 696 } 697 i += num_buffers - i; 698 } 699 } 700 701 assert(length == 0); 702 703 req->data_from_pool = true; 704 return 0; 705 } 706 707 int 708 spdk_nvmf_request_get_buffers(struct spdk_nvmf_request *req, 709 struct spdk_nvmf_transport_poll_group *group, 710 struct spdk_nvmf_transport *transport, 711 uint32_t length) 712 { 713 int rc; 714 715 req->iovcnt = 0; 716 717 rc = nvmf_request_get_buffers(req, group, transport, length); 718 if (rc == -ENOMEM) { 719 spdk_nvmf_request_free_buffers(req, group, transport); 720 } 721 722 return rc; 723 } 724 725 int 726 spdk_nvmf_request_get_buffers_multi(struct spdk_nvmf_request *req, 727 struct spdk_nvmf_transport_poll_group *group, 728 struct spdk_nvmf_transport *transport, 729 uint32_t *lengths, uint32_t num_lengths) 730 { 731 int rc = 0; 732 uint32_t i; 733 734 req->iovcnt = 0; 735 736 for (i = 0; i < num_lengths; i++) { 737 rc = nvmf_request_get_buffers(req, group, transport, lengths[i]); 738 if (rc != 0) { 739 goto err_exit; 740 } 741 } 742 743 return 0; 744 745 err_exit: 746 spdk_nvmf_request_free_buffers(req, group, transport); 747 return rc; 748 } 749