1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (C) 2008-2012 Daisuke Aoyama <aoyama@peach.ne.jp>. 5 * Copyright (c) Intel Corporation. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * * Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * * Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * * Neither the name of Intel Corporation nor the names of its 19 * contributors may be used to endorse or promote products derived 20 * from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #include "spdk/stdinc.h" 36 37 #include "spdk/bdev.h" 38 39 #include "spdk/env.h" 40 #include "spdk/io_channel.h" 41 #include "spdk/likely.h" 42 #include "spdk/queue.h" 43 #include "spdk/nvme_spec.h" 44 #include "spdk/scsi_spec.h" 45 46 #include "spdk_internal/bdev.h" 47 #include "spdk_internal/event.h" 48 #include "spdk_internal/log.h" 49 #include "spdk/string.h" 50 51 #ifdef SPDK_CONFIG_VTUNE 52 #include "ittnotify.h" 53 #endif 54 55 #define SPDK_BDEV_IO_POOL_SIZE (64 * 1024) 56 #define BUF_SMALL_POOL_SIZE 8192 57 #define BUF_LARGE_POOL_SIZE 1024 58 59 typedef TAILQ_HEAD(, spdk_bdev_io) need_buf_tailq_t; 60 61 struct spdk_bdev_mgr { 62 struct spdk_mempool *bdev_io_pool; 63 64 struct spdk_mempool *buf_small_pool; 65 struct spdk_mempool *buf_large_pool; 66 67 TAILQ_HEAD(, spdk_bdev_module_if) bdev_modules; 68 TAILQ_HEAD(, spdk_bdev_module_if) vbdev_modules; 69 70 TAILQ_HEAD(, spdk_bdev) bdevs; 71 72 #ifdef SPDK_CONFIG_VTUNE 73 __itt_domain *domain; 74 #endif 75 }; 76 77 static struct spdk_bdev_mgr g_bdev_mgr = { 78 .bdev_modules = TAILQ_HEAD_INITIALIZER(g_bdev_mgr.bdev_modules), 79 .vbdev_modules = TAILQ_HEAD_INITIALIZER(g_bdev_mgr.vbdev_modules), 80 .bdevs = TAILQ_HEAD_INITIALIZER(g_bdev_mgr.bdevs), 81 }; 82 83 static struct spdk_bdev_module_if *g_next_bdev_module; 84 static struct spdk_bdev_module_if *g_next_vbdev_module; 85 86 struct spdk_bdev_mgmt_channel { 87 need_buf_tailq_t need_buf_small; 88 need_buf_tailq_t need_buf_large; 89 }; 90 91 struct spdk_bdev_channel { 92 struct spdk_bdev *bdev; 93 94 /* The channel for the underlying device */ 95 struct spdk_io_channel *channel; 96 97 /* Channel for the bdev manager */ 98 struct spdk_io_channel *mgmt_channel; 99 100 struct spdk_bdev_io_stat stat; 101 102 /* 103 * Count of I/O submitted to bdev module and waiting for completion. 104 * Incremented before submit_request() is called on an spdk_bdev_io. 105 */ 106 uint64_t io_outstanding; 107 108 #ifdef SPDK_CONFIG_VTUNE 109 uint64_t start_tsc; 110 uint64_t interval_tsc; 111 __itt_string_handle *handle; 112 #endif 113 114 }; 115 116 struct spdk_bdev * 117 spdk_bdev_first(void) 118 { 119 struct spdk_bdev *bdev; 120 121 bdev = TAILQ_FIRST(&g_bdev_mgr.bdevs); 122 if (bdev) { 123 SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Starting bdev iteration at %s\n", bdev->name); 124 } 125 126 return bdev; 127 } 128 129 struct spdk_bdev * 130 spdk_bdev_next(struct spdk_bdev *prev) 131 { 132 struct spdk_bdev *bdev; 133 134 bdev = TAILQ_NEXT(prev, link); 135 if (bdev) { 136 SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Continuing bdev iteration at %s\n", bdev->name); 137 } 138 139 return bdev; 140 } 141 142 struct spdk_bdev * 143 spdk_bdev_get_by_name(const char *bdev_name) 144 { 145 struct spdk_bdev *bdev = spdk_bdev_first(); 146 147 while (bdev != NULL) { 148 if (strcmp(bdev_name, bdev->name) == 0) { 149 return bdev; 150 } 151 bdev = spdk_bdev_next(bdev); 152 } 153 154 return NULL; 155 } 156 157 static void 158 spdk_bdev_io_set_buf(struct spdk_bdev_io *bdev_io, void *buf) 159 { 160 assert(bdev_io->get_buf_cb != NULL); 161 assert(buf != NULL); 162 assert(bdev_io->u.read.iovs != NULL); 163 164 bdev_io->buf = buf; 165 bdev_io->u.read.iovs[0].iov_base = (void *)((unsigned long)((char *)buf + 512) & ~511UL); 166 bdev_io->u.read.iovs[0].iov_len = bdev_io->u.read.len; 167 bdev_io->get_buf_cb(bdev_io->ch->channel, bdev_io); 168 } 169 170 static void 171 spdk_bdev_io_put_buf(struct spdk_bdev_io *bdev_io) 172 { 173 struct spdk_mempool *pool; 174 struct spdk_bdev_io *tmp; 175 void *buf; 176 need_buf_tailq_t *tailq; 177 uint64_t length; 178 struct spdk_bdev_mgmt_channel *ch; 179 180 assert(bdev_io->u.read.iovcnt == 1); 181 182 length = bdev_io->u.read.len; 183 buf = bdev_io->buf; 184 185 ch = spdk_io_channel_get_ctx(bdev_io->ch->mgmt_channel); 186 187 if (length <= SPDK_BDEV_SMALL_BUF_MAX_SIZE) { 188 pool = g_bdev_mgr.buf_small_pool; 189 tailq = &ch->need_buf_small; 190 } else { 191 pool = g_bdev_mgr.buf_large_pool; 192 tailq = &ch->need_buf_large; 193 } 194 195 if (TAILQ_EMPTY(tailq)) { 196 spdk_mempool_put(pool, buf); 197 } else { 198 tmp = TAILQ_FIRST(tailq); 199 TAILQ_REMOVE(tailq, tmp, buf_link); 200 spdk_bdev_io_set_buf(tmp, buf); 201 } 202 } 203 204 void 205 spdk_bdev_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb) 206 { 207 uint64_t len = bdev_io->u.read.len; 208 struct spdk_mempool *pool; 209 need_buf_tailq_t *tailq; 210 void *buf = NULL; 211 struct spdk_bdev_mgmt_channel *ch; 212 213 assert(cb != NULL); 214 assert(bdev_io->u.read.iovs != NULL); 215 216 if (spdk_unlikely(bdev_io->u.read.iovs[0].iov_base != NULL)) { 217 /* Buffer already present */ 218 cb(bdev_io->ch->channel, bdev_io); 219 return; 220 } 221 222 ch = spdk_io_channel_get_ctx(bdev_io->ch->mgmt_channel); 223 224 bdev_io->get_buf_cb = cb; 225 if (len <= SPDK_BDEV_SMALL_BUF_MAX_SIZE) { 226 pool = g_bdev_mgr.buf_small_pool; 227 tailq = &ch->need_buf_small; 228 } else { 229 pool = g_bdev_mgr.buf_large_pool; 230 tailq = &ch->need_buf_large; 231 } 232 233 buf = spdk_mempool_get(pool); 234 235 if (!buf) { 236 TAILQ_INSERT_TAIL(tailq, bdev_io, buf_link); 237 } else { 238 spdk_bdev_io_set_buf(bdev_io, buf); 239 } 240 } 241 242 static int 243 spdk_bdev_module_get_max_ctx_size(void) 244 { 245 struct spdk_bdev_module_if *bdev_module; 246 int max_bdev_module_size = 0; 247 248 TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, tailq) { 249 if (bdev_module->get_ctx_size && bdev_module->get_ctx_size() > max_bdev_module_size) { 250 max_bdev_module_size = bdev_module->get_ctx_size(); 251 } 252 } 253 254 TAILQ_FOREACH(bdev_module, &g_bdev_mgr.vbdev_modules, tailq) { 255 if (bdev_module->get_ctx_size && bdev_module->get_ctx_size() > max_bdev_module_size) { 256 max_bdev_module_size = bdev_module->get_ctx_size(); 257 } 258 } 259 260 return max_bdev_module_size; 261 } 262 263 static void 264 spdk_bdev_config_text(FILE *fp) 265 { 266 struct spdk_bdev_module_if *bdev_module; 267 268 TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, tailq) { 269 if (bdev_module->config_text) { 270 bdev_module->config_text(fp); 271 } 272 } 273 TAILQ_FOREACH(bdev_module, &g_bdev_mgr.vbdev_modules, tailq) { 274 if (bdev_module->config_text) { 275 bdev_module->config_text(fp); 276 } 277 } 278 } 279 280 static int 281 spdk_bdev_mgmt_channel_create(void *io_device, void *ctx_buf) 282 { 283 struct spdk_bdev_mgmt_channel *ch = ctx_buf; 284 285 TAILQ_INIT(&ch->need_buf_small); 286 TAILQ_INIT(&ch->need_buf_large); 287 288 return 0; 289 } 290 291 static void 292 spdk_bdev_mgmt_channel_destroy(void *io_device, void *ctx_buf) 293 { 294 struct spdk_bdev_mgmt_channel *ch = ctx_buf; 295 296 if (!TAILQ_EMPTY(&ch->need_buf_small) || !TAILQ_EMPTY(&ch->need_buf_large)) { 297 SPDK_ERRLOG("Pending I/O list wasn't empty on channel destruction\n"); 298 } 299 } 300 301 void 302 spdk_bdev_module_init_next(int rc) 303 { 304 if (rc) { 305 assert(g_next_bdev_module != NULL); 306 SPDK_ERRLOG("Failed to init bdev module: %s\n", g_next_bdev_module->module_name); 307 spdk_subsystem_init_next(rc); 308 return; 309 } 310 311 if (!g_next_bdev_module) { 312 g_next_bdev_module = TAILQ_FIRST(&g_bdev_mgr.bdev_modules); 313 } else { 314 g_next_bdev_module = TAILQ_NEXT(g_next_bdev_module, tailq); 315 } 316 317 if (g_next_bdev_module) { 318 g_next_bdev_module->module_init(); 319 } else { 320 spdk_vbdev_module_init_next(0); 321 } 322 } 323 324 void 325 spdk_vbdev_module_init_next(int rc) 326 { 327 if (rc) { 328 assert(g_next_vbdev_module != NULL); 329 SPDK_ERRLOG("Failed to init vbdev module: %s\n", g_next_vbdev_module->module_name); 330 spdk_subsystem_init_next(rc); 331 return; 332 } 333 334 if (!g_next_vbdev_module) { 335 g_next_vbdev_module = TAILQ_FIRST(&g_bdev_mgr.vbdev_modules); 336 } else { 337 g_next_vbdev_module = TAILQ_NEXT(g_next_vbdev_module, tailq); 338 } 339 340 if (g_next_vbdev_module) { 341 g_next_vbdev_module->module_init(); 342 } else { 343 spdk_subsystem_init_next(0); 344 } 345 } 346 347 static void 348 spdk_bdev_initialize(void) 349 { 350 int cache_size; 351 int rc = 0; 352 353 g_bdev_mgr.bdev_io_pool = spdk_mempool_create("blockdev_io", 354 SPDK_BDEV_IO_POOL_SIZE, 355 sizeof(struct spdk_bdev_io) + 356 spdk_bdev_module_get_max_ctx_size(), 357 64, 358 SPDK_ENV_SOCKET_ID_ANY); 359 360 if (g_bdev_mgr.bdev_io_pool == NULL) { 361 SPDK_ERRLOG("could not allocate spdk_bdev_io pool"); 362 rc = -1; 363 goto end; 364 } 365 366 /** 367 * Ensure no more than half of the total buffers end up local caches, by 368 * using spdk_env_get_core_count() to determine how many local caches we need 369 * to account for. 370 */ 371 cache_size = BUF_SMALL_POOL_SIZE / (2 * spdk_env_get_core_count()); 372 g_bdev_mgr.buf_small_pool = spdk_mempool_create("buf_small_pool", 373 BUF_SMALL_POOL_SIZE, 374 SPDK_BDEV_SMALL_BUF_MAX_SIZE + 512, 375 cache_size, 376 SPDK_ENV_SOCKET_ID_ANY); 377 if (!g_bdev_mgr.buf_small_pool) { 378 SPDK_ERRLOG("create rbuf small pool failed\n"); 379 rc = -1; 380 goto end; 381 } 382 383 cache_size = BUF_LARGE_POOL_SIZE / (2 * spdk_env_get_core_count()); 384 g_bdev_mgr.buf_large_pool = spdk_mempool_create("buf_large_pool", 385 BUF_LARGE_POOL_SIZE, 386 SPDK_BDEV_LARGE_BUF_MAX_SIZE + 512, 387 cache_size, 388 SPDK_ENV_SOCKET_ID_ANY); 389 if (!g_bdev_mgr.buf_large_pool) { 390 SPDK_ERRLOG("create rbuf large pool failed\n"); 391 rc = -1; 392 goto end; 393 } 394 395 #ifdef SPDK_CONFIG_VTUNE 396 g_bdev_mgr.domain = __itt_domain_create("spdk_bdev"); 397 #endif 398 399 spdk_io_device_register(&g_bdev_mgr, spdk_bdev_mgmt_channel_create, 400 spdk_bdev_mgmt_channel_destroy, 401 sizeof(struct spdk_bdev_mgmt_channel)); 402 403 end: 404 spdk_bdev_module_init_next(rc); 405 } 406 407 static int 408 spdk_bdev_finish(void) 409 { 410 struct spdk_bdev_module_if *bdev_module; 411 412 TAILQ_FOREACH(bdev_module, &g_bdev_mgr.vbdev_modules, tailq) { 413 if (bdev_module->module_fini) { 414 bdev_module->module_fini(); 415 } 416 } 417 418 TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, tailq) { 419 if (bdev_module->module_fini) { 420 bdev_module->module_fini(); 421 } 422 } 423 424 if (spdk_mempool_count(g_bdev_mgr.bdev_io_pool) != SPDK_BDEV_IO_POOL_SIZE) { 425 SPDK_ERRLOG("bdev IO pool count is %zu but should be %u\n", 426 spdk_mempool_count(g_bdev_mgr.bdev_io_pool), 427 SPDK_BDEV_IO_POOL_SIZE); 428 } 429 430 if (spdk_mempool_count(g_bdev_mgr.buf_small_pool) != BUF_SMALL_POOL_SIZE) { 431 SPDK_ERRLOG("Small buffer pool count is %zu but should be %u\n", 432 spdk_mempool_count(g_bdev_mgr.buf_small_pool), 433 BUF_SMALL_POOL_SIZE); 434 assert(false); 435 } 436 437 if (spdk_mempool_count(g_bdev_mgr.buf_large_pool) != BUF_LARGE_POOL_SIZE) { 438 SPDK_ERRLOG("Large buffer pool count is %zu but should be %u\n", 439 spdk_mempool_count(g_bdev_mgr.buf_large_pool), 440 BUF_LARGE_POOL_SIZE); 441 assert(false); 442 } 443 444 spdk_mempool_free(g_bdev_mgr.bdev_io_pool); 445 spdk_mempool_free(g_bdev_mgr.buf_small_pool); 446 spdk_mempool_free(g_bdev_mgr.buf_large_pool); 447 448 spdk_io_device_unregister(&g_bdev_mgr); 449 450 return 0; 451 } 452 453 struct spdk_bdev_io * 454 spdk_bdev_get_io(void) 455 { 456 struct spdk_bdev_io *bdev_io; 457 458 bdev_io = spdk_mempool_get(g_bdev_mgr.bdev_io_pool); 459 if (!bdev_io) { 460 SPDK_ERRLOG("Unable to get spdk_bdev_io\n"); 461 abort(); 462 } 463 464 memset(bdev_io, 0, sizeof(*bdev_io)); 465 466 return bdev_io; 467 } 468 469 static void 470 spdk_bdev_put_io(struct spdk_bdev_io *bdev_io) 471 { 472 if (!bdev_io) { 473 return; 474 } 475 476 if (bdev_io->buf != NULL) { 477 spdk_bdev_io_put_buf(bdev_io); 478 } 479 480 spdk_mempool_put(g_bdev_mgr.bdev_io_pool, (void *)bdev_io); 481 } 482 483 static void 484 __submit_request(struct spdk_bdev *bdev, struct spdk_bdev_io *bdev_io) 485 { 486 struct spdk_io_channel *ch; 487 488 assert(bdev_io->status == SPDK_BDEV_IO_STATUS_PENDING); 489 490 ch = bdev_io->ch->channel; 491 492 bdev_io->ch->io_outstanding++; 493 bdev_io->in_submit_request = true; 494 bdev->fn_table->submit_request(ch, bdev_io); 495 bdev_io->in_submit_request = false; 496 } 497 498 static int 499 spdk_bdev_io_submit(struct spdk_bdev_io *bdev_io) 500 { 501 struct spdk_bdev *bdev = bdev_io->bdev; 502 503 __submit_request(bdev, bdev_io); 504 return 0; 505 } 506 507 void 508 spdk_bdev_io_resubmit(struct spdk_bdev_io *bdev_io, struct spdk_bdev *new_bdev) 509 { 510 assert(bdev_io->status == SPDK_BDEV_IO_STATUS_PENDING); 511 bdev_io->bdev = new_bdev; 512 513 /* 514 * These fields are normally set during spdk_bdev_io_init(), but since bdev is 515 * being switched, they need to be reinitialized. 516 */ 517 bdev_io->gencnt = new_bdev->gencnt; 518 519 /* 520 * This bdev_io was already submitted so decrement io_outstanding to ensure it 521 * does not get double-counted. 522 */ 523 assert(bdev_io->ch->io_outstanding > 0); 524 bdev_io->ch->io_outstanding--; 525 __submit_request(new_bdev, bdev_io); 526 } 527 528 static void 529 spdk_bdev_io_init(struct spdk_bdev_io *bdev_io, 530 struct spdk_bdev *bdev, void *cb_arg, 531 spdk_bdev_io_completion_cb cb) 532 { 533 bdev_io->bdev = bdev; 534 bdev_io->caller_ctx = cb_arg; 535 bdev_io->cb = cb; 536 bdev_io->gencnt = bdev->gencnt; 537 bdev_io->status = SPDK_BDEV_IO_STATUS_PENDING; 538 bdev_io->in_submit_request = false; 539 TAILQ_INIT(&bdev_io->child_io); 540 } 541 542 struct spdk_bdev_io * 543 spdk_bdev_get_child_io(struct spdk_bdev_io *parent, 544 struct spdk_bdev *bdev, 545 spdk_bdev_io_completion_cb cb, 546 void *cb_arg) 547 { 548 struct spdk_bdev_io *child; 549 550 child = spdk_bdev_get_io(); 551 if (!child) { 552 SPDK_ERRLOG("Unable to get spdk_bdev_io\n"); 553 return NULL; 554 } 555 556 if (cb_arg == NULL) { 557 cb_arg = child; 558 } 559 560 spdk_bdev_io_init(child, bdev, cb_arg, cb); 561 562 child->type = parent->type; 563 memcpy(&child->u, &parent->u, sizeof(child->u)); 564 child->buf = NULL; 565 child->get_buf_cb = NULL; 566 child->parent = parent; 567 568 TAILQ_INSERT_TAIL(&parent->child_io, child, link); 569 570 return child; 571 } 572 573 bool 574 spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type) 575 { 576 return bdev->fn_table->io_type_supported(bdev->ctxt, io_type); 577 } 578 579 int 580 spdk_bdev_dump_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 581 { 582 if (bdev->fn_table->dump_config_json) { 583 return bdev->fn_table->dump_config_json(bdev->ctxt, w); 584 } 585 586 return 0; 587 } 588 589 static int 590 spdk_bdev_channel_create(void *io_device, void *ctx_buf) 591 { 592 struct spdk_bdev *bdev = io_device; 593 struct spdk_bdev_channel *ch = ctx_buf; 594 595 ch->bdev = io_device; 596 ch->channel = bdev->fn_table->get_io_channel(bdev->ctxt); 597 ch->mgmt_channel = spdk_get_io_channel(&g_bdev_mgr); 598 memset(&ch->stat, 0, sizeof(ch->stat)); 599 ch->io_outstanding = 0; 600 601 #ifdef SPDK_CONFIG_VTUNE 602 { 603 char *name; 604 605 name = spdk_sprintf_alloc("spdk_bdev_%s_%p", ch->bdev->name, ch); 606 if (!name) { 607 return -1; 608 } 609 ch->handle = __itt_string_handle_create(name); 610 free(name); 611 ch->start_tsc = spdk_get_ticks(); 612 ch->interval_tsc = spdk_get_ticks_hz() / 100; 613 } 614 #endif 615 616 return 0; 617 } 618 619 static void 620 _spdk_bdev_abort_io(need_buf_tailq_t *queue, struct spdk_bdev_channel *ch) 621 { 622 struct spdk_bdev_io *bdev_io, *tmp; 623 624 TAILQ_FOREACH_SAFE(bdev_io, queue, buf_link, tmp) { 625 if (bdev_io->ch == ch) { 626 TAILQ_REMOVE(queue, bdev_io, buf_link); 627 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 628 } 629 } 630 } 631 632 static void 633 spdk_bdev_channel_destroy(void *io_device, void *ctx_buf) 634 { 635 struct spdk_bdev_channel *ch = ctx_buf; 636 struct spdk_bdev_mgmt_channel *mgmt_channel; 637 638 mgmt_channel = spdk_io_channel_get_ctx(ch->mgmt_channel); 639 640 _spdk_bdev_abort_io(&mgmt_channel->need_buf_small, ch); 641 _spdk_bdev_abort_io(&mgmt_channel->need_buf_large, ch); 642 643 spdk_put_io_channel(ch->channel); 644 spdk_put_io_channel(ch->mgmt_channel); 645 assert(ch->io_outstanding == 0); 646 } 647 648 struct spdk_io_channel * 649 spdk_bdev_get_io_channel(struct spdk_bdev *bdev) 650 { 651 return spdk_get_io_channel(bdev); 652 } 653 654 const char * 655 spdk_bdev_get_name(const struct spdk_bdev *bdev) 656 { 657 return bdev->name; 658 } 659 660 const char * 661 spdk_bdev_get_product_name(const struct spdk_bdev *bdev) 662 { 663 return bdev->product_name; 664 } 665 666 uint32_t 667 spdk_bdev_get_block_size(const struct spdk_bdev *bdev) 668 { 669 return bdev->blocklen; 670 } 671 672 uint64_t 673 spdk_bdev_get_num_blocks(const struct spdk_bdev *bdev) 674 { 675 return bdev->blockcnt; 676 } 677 678 uint32_t 679 spdk_bdev_get_max_unmap_descriptors(const struct spdk_bdev *bdev) 680 { 681 return bdev->max_unmap_bdesc_count; 682 } 683 684 size_t 685 spdk_bdev_get_buf_align(const struct spdk_bdev *bdev) 686 { 687 /* TODO: push this logic down to the bdev modules */ 688 if (bdev->need_aligned_buffer) { 689 return bdev->blocklen; 690 } 691 692 return 1; 693 } 694 695 bool 696 spdk_bdev_has_write_cache(const struct spdk_bdev *bdev) 697 { 698 return bdev->write_cache; 699 } 700 701 static int 702 spdk_bdev_io_valid(struct spdk_bdev *bdev, uint64_t offset, uint64_t nbytes) 703 { 704 /* Return failure if nbytes is not a multiple of bdev->blocklen */ 705 if (nbytes % bdev->blocklen) { 706 return -1; 707 } 708 709 /* Return failure if offset + nbytes is less than offset; indicates there 710 * has been an overflow and hence the offset has been wrapped around */ 711 if (offset + nbytes < offset) { 712 return -1; 713 } 714 715 /* Return failure if offset + nbytes exceeds the size of the blockdev */ 716 if (offset + nbytes > bdev->blockcnt * bdev->blocklen) { 717 return -1; 718 } 719 720 return 0; 721 } 722 723 int 724 spdk_bdev_read(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 725 void *buf, uint64_t offset, uint64_t nbytes, 726 spdk_bdev_io_completion_cb cb, void *cb_arg) 727 { 728 struct spdk_bdev_io *bdev_io; 729 struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); 730 int rc; 731 732 assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); 733 if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) { 734 return -EINVAL; 735 } 736 737 bdev_io = spdk_bdev_get_io(); 738 if (!bdev_io) { 739 SPDK_ERRLOG("spdk_bdev_io memory allocation failed duing read\n"); 740 return -ENOMEM; 741 } 742 743 bdev_io->ch = channel; 744 bdev_io->type = SPDK_BDEV_IO_TYPE_READ; 745 bdev_io->u.read.iov.iov_base = buf; 746 bdev_io->u.read.iov.iov_len = nbytes; 747 bdev_io->u.read.iovs = &bdev_io->u.read.iov; 748 bdev_io->u.read.iovcnt = 1; 749 bdev_io->u.read.len = nbytes; 750 bdev_io->u.read.offset = offset; 751 spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb); 752 753 rc = spdk_bdev_io_submit(bdev_io); 754 if (rc < 0) { 755 spdk_bdev_put_io(bdev_io); 756 return rc; 757 } 758 759 return 0; 760 } 761 762 int 763 spdk_bdev_readv(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 764 struct iovec *iov, int iovcnt, 765 uint64_t offset, uint64_t nbytes, 766 spdk_bdev_io_completion_cb cb, void *cb_arg) 767 { 768 struct spdk_bdev_io *bdev_io; 769 struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); 770 int rc; 771 772 assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); 773 if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) { 774 return -EINVAL; 775 } 776 777 bdev_io = spdk_bdev_get_io(); 778 if (!bdev_io) { 779 SPDK_ERRLOG("spdk_bdev_io memory allocation failed duing read\n"); 780 return -ENOMEM; 781 } 782 783 bdev_io->ch = channel; 784 bdev_io->type = SPDK_BDEV_IO_TYPE_READ; 785 bdev_io->u.read.iovs = iov; 786 bdev_io->u.read.iovcnt = iovcnt; 787 bdev_io->u.read.len = nbytes; 788 bdev_io->u.read.offset = offset; 789 spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb); 790 791 rc = spdk_bdev_io_submit(bdev_io); 792 if (rc < 0) { 793 spdk_bdev_put_io(bdev_io); 794 return rc; 795 } 796 797 return 0; 798 } 799 800 int 801 spdk_bdev_write(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 802 void *buf, uint64_t offset, uint64_t nbytes, 803 spdk_bdev_io_completion_cb cb, void *cb_arg) 804 { 805 struct spdk_bdev_io *bdev_io; 806 struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); 807 int rc; 808 809 assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); 810 if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) { 811 return -EINVAL; 812 } 813 814 bdev_io = spdk_bdev_get_io(); 815 if (!bdev_io) { 816 SPDK_ERRLOG("blockdev_io memory allocation failed duing write\n"); 817 return -ENOMEM; 818 } 819 820 bdev_io->ch = channel; 821 bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; 822 bdev_io->u.write.iov.iov_base = buf; 823 bdev_io->u.write.iov.iov_len = nbytes; 824 bdev_io->u.write.iovs = &bdev_io->u.write.iov; 825 bdev_io->u.write.iovcnt = 1; 826 bdev_io->u.write.len = nbytes; 827 bdev_io->u.write.offset = offset; 828 spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb); 829 830 rc = spdk_bdev_io_submit(bdev_io); 831 if (rc < 0) { 832 spdk_bdev_put_io(bdev_io); 833 return rc; 834 } 835 836 return 0; 837 } 838 839 int 840 spdk_bdev_writev(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 841 struct iovec *iov, int iovcnt, 842 uint64_t offset, uint64_t len, 843 spdk_bdev_io_completion_cb cb, void *cb_arg) 844 { 845 struct spdk_bdev_io *bdev_io; 846 struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); 847 int rc; 848 849 assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); 850 if (spdk_bdev_io_valid(bdev, offset, len) != 0) { 851 return -EINVAL; 852 } 853 854 bdev_io = spdk_bdev_get_io(); 855 if (!bdev_io) { 856 SPDK_ERRLOG("bdev_io memory allocation failed duing writev\n"); 857 return -ENOMEM; 858 } 859 860 bdev_io->ch = channel; 861 bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; 862 bdev_io->u.write.iovs = iov; 863 bdev_io->u.write.iovcnt = iovcnt; 864 bdev_io->u.write.len = len; 865 bdev_io->u.write.offset = offset; 866 spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb); 867 868 rc = spdk_bdev_io_submit(bdev_io); 869 if (rc < 0) { 870 spdk_bdev_put_io(bdev_io); 871 return rc; 872 } 873 874 return 0; 875 } 876 877 int 878 spdk_bdev_unmap(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 879 struct spdk_scsi_unmap_bdesc *unmap_d, 880 uint16_t bdesc_count, 881 spdk_bdev_io_completion_cb cb, void *cb_arg) 882 { 883 struct spdk_bdev_io *bdev_io; 884 struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); 885 int rc; 886 887 assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); 888 if (bdesc_count == 0) { 889 SPDK_ERRLOG("Invalid bdesc_count 0\n"); 890 return -EINVAL; 891 } 892 893 if (bdesc_count > bdev->max_unmap_bdesc_count) { 894 SPDK_ERRLOG("Invalid bdesc_count %u > max_unmap_bdesc_count %u\n", 895 bdesc_count, bdev->max_unmap_bdesc_count); 896 return -EINVAL; 897 } 898 899 bdev_io = spdk_bdev_get_io(); 900 if (!bdev_io) { 901 SPDK_ERRLOG("bdev_io memory allocation failed duing unmap\n"); 902 return -ENOMEM; 903 } 904 905 bdev_io->ch = channel; 906 bdev_io->type = SPDK_BDEV_IO_TYPE_UNMAP; 907 bdev_io->u.unmap.unmap_bdesc = unmap_d; 908 bdev_io->u.unmap.bdesc_count = bdesc_count; 909 spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb); 910 911 rc = spdk_bdev_io_submit(bdev_io); 912 if (rc < 0) { 913 spdk_bdev_put_io(bdev_io); 914 return rc; 915 } 916 917 return 0; 918 } 919 920 int 921 spdk_bdev_flush(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 922 uint64_t offset, uint64_t length, 923 spdk_bdev_io_completion_cb cb, void *cb_arg) 924 { 925 struct spdk_bdev_io *bdev_io; 926 struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); 927 int rc; 928 929 assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); 930 bdev_io = spdk_bdev_get_io(); 931 if (!bdev_io) { 932 SPDK_ERRLOG("bdev_io memory allocation failed duing flush\n"); 933 return -ENOMEM; 934 } 935 936 bdev_io->ch = channel; 937 bdev_io->type = SPDK_BDEV_IO_TYPE_FLUSH; 938 bdev_io->u.flush.offset = offset; 939 bdev_io->u.flush.length = length; 940 spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb); 941 942 rc = spdk_bdev_io_submit(bdev_io); 943 if (rc < 0) { 944 spdk_bdev_put_io(bdev_io); 945 return rc; 946 } 947 948 return 0; 949 } 950 951 static void 952 _spdk_bdev_reset_dev(void *io_device, void *ctx) 953 { 954 struct spdk_bdev_io *bdev_io = ctx; 955 int rc; 956 957 rc = spdk_bdev_io_submit(bdev_io); 958 if (rc < 0) { 959 spdk_bdev_put_io(bdev_io); 960 SPDK_ERRLOG("reset failed\n"); 961 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 962 } 963 } 964 965 static void 966 _spdk_bdev_reset_abort_channel(void *io_device, struct spdk_io_channel *ch, 967 void *ctx) 968 { 969 struct spdk_bdev_channel *channel; 970 struct spdk_bdev_mgmt_channel *mgmt_channel; 971 972 channel = spdk_io_channel_get_ctx(ch); 973 mgmt_channel = spdk_io_channel_get_ctx(channel->mgmt_channel); 974 975 _spdk_bdev_abort_io(&mgmt_channel->need_buf_small, channel); 976 _spdk_bdev_abort_io(&mgmt_channel->need_buf_large, channel); 977 } 978 979 int 980 spdk_bdev_reset(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 981 spdk_bdev_io_completion_cb cb, void *cb_arg) 982 { 983 struct spdk_bdev_io *bdev_io; 984 struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); 985 986 assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); 987 988 bdev_io = spdk_bdev_get_io(); 989 if (!bdev_io) { 990 SPDK_ERRLOG("bdev_io memory allocation failed duing reset\n"); 991 return -ENOMEM;; 992 } 993 994 bdev_io->ch = channel; 995 bdev_io->type = SPDK_BDEV_IO_TYPE_RESET; 996 spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb); 997 998 /* First, abort all I/O queued up waiting for buffers. */ 999 spdk_for_each_channel(bdev, 1000 _spdk_bdev_reset_abort_channel, 1001 bdev_io, 1002 _spdk_bdev_reset_dev); 1003 1004 return 0; 1005 } 1006 1007 void 1008 spdk_bdev_get_io_stat(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 1009 struct spdk_bdev_io_stat *stat) 1010 { 1011 #ifdef SPDK_CONFIG_VTUNE 1012 SPDK_ERRLOG("Calling spdk_bdev_get_io_stat is not allowed when VTune integration is enabled.\n"); 1013 memset(stat, 0, sizeof(*stat)); 1014 return; 1015 #endif 1016 1017 struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); 1018 1019 *stat = channel->stat; 1020 memset(&channel->stat, 0, sizeof(channel->stat)); 1021 } 1022 1023 int 1024 spdk_bdev_nvme_admin_passthru(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 1025 const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes, 1026 spdk_bdev_io_completion_cb cb, void *cb_arg) 1027 { 1028 struct spdk_bdev_io *bdev_io; 1029 struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); 1030 int rc; 1031 1032 bdev_io = spdk_bdev_get_io(); 1033 if (!bdev_io) { 1034 SPDK_ERRLOG("bdev_io memory allocation failed during nvme_admin_passthru\n"); 1035 return -ENOMEM; 1036 } 1037 1038 bdev_io->ch = channel; 1039 bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_ADMIN; 1040 bdev_io->u.nvme_passthru.cmd = *cmd; 1041 bdev_io->u.nvme_passthru.buf = buf; 1042 bdev_io->u.nvme_passthru.nbytes = nbytes; 1043 1044 spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb); 1045 1046 rc = spdk_bdev_io_submit(bdev_io); 1047 if (rc < 0) { 1048 spdk_bdev_put_io(bdev_io); 1049 return rc; 1050 } 1051 1052 return 0; 1053 } 1054 1055 int 1056 spdk_bdev_nvme_io_passthru(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 1057 const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes, 1058 spdk_bdev_io_completion_cb cb, void *cb_arg) 1059 { 1060 struct spdk_bdev_io *bdev_io; 1061 struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); 1062 int rc; 1063 1064 bdev_io = spdk_bdev_get_io(); 1065 if (!bdev_io) { 1066 SPDK_ERRLOG("bdev_io memory allocation failed during nvme_admin_passthru\n"); 1067 return -ENOMEM; 1068 } 1069 1070 bdev_io->ch = channel; 1071 bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_IO; 1072 bdev_io->u.nvme_passthru.cmd = *cmd; 1073 bdev_io->u.nvme_passthru.buf = buf; 1074 bdev_io->u.nvme_passthru.nbytes = nbytes; 1075 1076 spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb); 1077 1078 rc = spdk_bdev_io_submit(bdev_io); 1079 if (rc < 0) { 1080 spdk_bdev_put_io(bdev_io); 1081 return rc; 1082 } 1083 1084 return 0; 1085 } 1086 1087 int 1088 spdk_bdev_free_io(struct spdk_bdev_io *bdev_io) 1089 { 1090 struct spdk_bdev_io *child_io, *tmp; 1091 1092 if (!bdev_io) { 1093 SPDK_ERRLOG("bdev_io is NULL\n"); 1094 return -1; 1095 } 1096 1097 if (bdev_io->status == SPDK_BDEV_IO_STATUS_PENDING) { 1098 SPDK_ERRLOG("bdev_io is in pending state\n"); 1099 assert(false); 1100 return -1; 1101 } 1102 1103 TAILQ_FOREACH_SAFE(child_io, &bdev_io->child_io, link, tmp) { 1104 /* 1105 * Make sure no references to the parent I/O remain, since it is being 1106 * returned to the free pool. 1107 */ 1108 child_io->parent = NULL; 1109 TAILQ_REMOVE(&bdev_io->child_io, child_io, link); 1110 1111 /* 1112 * Child I/O may have a buf that needs to be returned to a pool 1113 * on a different core, so free it through the request submission 1114 * process rather than calling put_io directly here. 1115 */ 1116 spdk_bdev_free_io(child_io); 1117 } 1118 1119 spdk_bdev_put_io(bdev_io); 1120 1121 return 0; 1122 } 1123 1124 static void 1125 bdev_io_deferred_completion(void *arg1, void *arg2) 1126 { 1127 struct spdk_bdev_io *bdev_io = arg1; 1128 enum spdk_bdev_io_status status = (enum spdk_bdev_io_status)arg2; 1129 1130 assert(bdev_io->in_submit_request == false); 1131 1132 spdk_bdev_io_complete(bdev_io, status); 1133 } 1134 1135 void 1136 spdk_bdev_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status) 1137 { 1138 if (bdev_io->in_submit_request) { 1139 /* 1140 * Defer completion via an event to avoid potential infinite recursion if the 1141 * user's completion callback issues a new I/O. 1142 */ 1143 spdk_event_call(spdk_event_allocate(spdk_env_get_current_core(), 1144 bdev_io_deferred_completion, 1145 bdev_io, 1146 (void *)status)); 1147 return; 1148 } 1149 1150 assert(bdev_io->ch->io_outstanding > 0); 1151 bdev_io->ch->io_outstanding--; 1152 if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { 1153 /* Successful reset */ 1154 if (status == SPDK_BDEV_IO_STATUS_SUCCESS) { 1155 /* Increase the blockdev generation */ 1156 bdev_io->bdev->gencnt++; 1157 } 1158 } else { 1159 /* 1160 * Check the gencnt, to see if this I/O was issued before the most 1161 * recent reset. If the gencnt is not equal, then just free the I/O 1162 * without calling the callback, since the caller will have already 1163 * freed its context for this I/O. 1164 */ 1165 if (bdev_io->bdev->gencnt != bdev_io->gencnt) { 1166 spdk_bdev_put_io(bdev_io); 1167 return; 1168 } 1169 } 1170 1171 bdev_io->status = status; 1172 1173 if (bdev_io->status == SPDK_BDEV_IO_STATUS_SUCCESS) { 1174 switch (bdev_io->type) { 1175 case SPDK_BDEV_IO_TYPE_READ: 1176 bdev_io->ch->stat.bytes_read += bdev_io->u.read.len; 1177 bdev_io->ch->stat.num_read_ops++; 1178 break; 1179 case SPDK_BDEV_IO_TYPE_WRITE: 1180 bdev_io->ch->stat.bytes_written += bdev_io->u.write.len; 1181 bdev_io->ch->stat.num_write_ops++; 1182 break; 1183 default: 1184 break; 1185 } 1186 } 1187 1188 #ifdef SPDK_CONFIG_VTUNE 1189 uint64_t now_tsc = spdk_get_ticks(); 1190 if (now_tsc > (bdev_io->ch->start_tsc + bdev_io->ch->interval_tsc)) { 1191 uint64_t data[4]; 1192 1193 data[0] = bdev_io->ch->stat.num_read_ops; 1194 data[1] = bdev_io->ch->stat.bytes_read; 1195 data[2] = bdev_io->ch->stat.num_write_ops; 1196 data[3] = bdev_io->ch->stat.bytes_written; 1197 1198 __itt_metadata_add(g_bdev_mgr.domain, __itt_null, bdev_io->ch->handle, 1199 __itt_metadata_u64, 4, data); 1200 1201 memset(&bdev_io->ch->stat, 0, sizeof(bdev_io->ch->stat)); 1202 bdev_io->ch->start_tsc = now_tsc; 1203 } 1204 #endif 1205 1206 assert(bdev_io->cb != NULL); 1207 bdev_io->cb(bdev_io, status == SPDK_BDEV_IO_STATUS_SUCCESS, bdev_io->caller_ctx); 1208 } 1209 1210 void 1211 spdk_bdev_io_complete_scsi_status(struct spdk_bdev_io *bdev_io, enum spdk_scsi_status sc, 1212 enum spdk_scsi_sense sk, uint8_t asc, uint8_t ascq) 1213 { 1214 if (sc == SPDK_SCSI_STATUS_GOOD) { 1215 bdev_io->status = SPDK_BDEV_IO_STATUS_SUCCESS; 1216 } else { 1217 bdev_io->status = SPDK_BDEV_IO_STATUS_SCSI_ERROR; 1218 bdev_io->error.scsi.sc = sc; 1219 bdev_io->error.scsi.sk = sk; 1220 bdev_io->error.scsi.asc = asc; 1221 bdev_io->error.scsi.ascq = ascq; 1222 } 1223 1224 spdk_bdev_io_complete(bdev_io, bdev_io->status); 1225 } 1226 1227 void 1228 spdk_bdev_io_get_scsi_status(const struct spdk_bdev_io *bdev_io, 1229 int *sc, int *sk, int *asc, int *ascq) 1230 { 1231 assert(sc != NULL); 1232 assert(sk != NULL); 1233 assert(asc != NULL); 1234 assert(ascq != NULL); 1235 1236 switch (bdev_io->status) { 1237 case SPDK_BDEV_IO_STATUS_SUCCESS: 1238 *sc = SPDK_SCSI_STATUS_GOOD; 1239 *sk = SPDK_SCSI_SENSE_NO_SENSE; 1240 *asc = SPDK_SCSI_ASC_NO_ADDITIONAL_SENSE; 1241 *ascq = SPDK_SCSI_ASCQ_CAUSE_NOT_REPORTABLE; 1242 break; 1243 case SPDK_BDEV_IO_STATUS_NVME_ERROR: 1244 spdk_scsi_nvme_translate(bdev_io, sc, sk, asc, ascq); 1245 break; 1246 case SPDK_BDEV_IO_STATUS_SCSI_ERROR: 1247 *sc = bdev_io->error.scsi.sc; 1248 *sk = bdev_io->error.scsi.sk; 1249 *asc = bdev_io->error.scsi.asc; 1250 *ascq = bdev_io->error.scsi.ascq; 1251 break; 1252 default: 1253 *sc = SPDK_SCSI_STATUS_CHECK_CONDITION; 1254 *sk = SPDK_SCSI_SENSE_ABORTED_COMMAND; 1255 *asc = SPDK_SCSI_ASC_NO_ADDITIONAL_SENSE; 1256 *ascq = SPDK_SCSI_ASCQ_CAUSE_NOT_REPORTABLE; 1257 break; 1258 } 1259 } 1260 1261 void 1262 spdk_bdev_io_complete_nvme_status(struct spdk_bdev_io *bdev_io, int sct, int sc) 1263 { 1264 if (sct == SPDK_NVME_SCT_GENERIC && sc == SPDK_NVME_SC_SUCCESS) { 1265 bdev_io->status = SPDK_BDEV_IO_STATUS_SUCCESS; 1266 } else { 1267 bdev_io->error.nvme.sct = sct; 1268 bdev_io->error.nvme.sc = sc; 1269 bdev_io->status = SPDK_BDEV_IO_STATUS_NVME_ERROR; 1270 } 1271 1272 spdk_bdev_io_complete(bdev_io, bdev_io->status); 1273 } 1274 1275 void 1276 spdk_bdev_io_get_nvme_status(const struct spdk_bdev_io *bdev_io, int *sct, int *sc) 1277 { 1278 assert(sct != NULL); 1279 assert(sc != NULL); 1280 1281 if (bdev_io->status == SPDK_BDEV_IO_STATUS_NVME_ERROR) { 1282 *sct = bdev_io->error.nvme.sct; 1283 *sc = bdev_io->error.nvme.sc; 1284 } else if (bdev_io->status == SPDK_BDEV_IO_STATUS_SUCCESS) { 1285 *sct = SPDK_NVME_SCT_GENERIC; 1286 *sc = SPDK_NVME_SC_SUCCESS; 1287 } else { 1288 *sct = SPDK_NVME_SCT_GENERIC; 1289 *sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 1290 } 1291 } 1292 1293 void 1294 spdk_bdev_register(struct spdk_bdev *bdev) 1295 { 1296 struct spdk_bdev_module_if *vbdev_module; 1297 1298 /* initialize the reset generation value to zero */ 1299 bdev->gencnt = 0; 1300 1301 spdk_io_device_register(bdev, spdk_bdev_channel_create, spdk_bdev_channel_destroy, 1302 sizeof(struct spdk_bdev_channel)); 1303 1304 pthread_mutex_init(&bdev->mutex, NULL); 1305 bdev->status = SPDK_BDEV_STATUS_UNCLAIMED; 1306 SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Inserting bdev %s into list\n", bdev->name); 1307 TAILQ_INSERT_TAIL(&g_bdev_mgr.bdevs, bdev, link); 1308 1309 TAILQ_FOREACH(vbdev_module, &g_bdev_mgr.vbdev_modules, tailq) { 1310 if (vbdev_module->bdev_registered) { 1311 vbdev_module->bdev_registered(bdev); 1312 } 1313 } 1314 } 1315 1316 void 1317 spdk_bdev_unregister(struct spdk_bdev *bdev) 1318 { 1319 int rc; 1320 1321 SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Removing bdev %s from list\n", bdev->name); 1322 1323 pthread_mutex_lock(&bdev->mutex); 1324 assert(bdev->status == SPDK_BDEV_STATUS_CLAIMED || bdev->status == SPDK_BDEV_STATUS_UNCLAIMED); 1325 if (bdev->status == SPDK_BDEV_STATUS_CLAIMED) { 1326 if (bdev->remove_cb) { 1327 bdev->status = SPDK_BDEV_STATUS_REMOVING; 1328 pthread_mutex_unlock(&bdev->mutex); 1329 bdev->remove_cb(bdev->remove_ctx); 1330 return; 1331 } else { 1332 bdev->status = SPDK_BDEV_STATUS_UNCLAIMED; 1333 } 1334 } 1335 1336 TAILQ_REMOVE(&g_bdev_mgr.bdevs, bdev, link); 1337 pthread_mutex_unlock(&bdev->mutex); 1338 1339 pthread_mutex_destroy(&bdev->mutex); 1340 1341 spdk_io_device_unregister(bdev); 1342 1343 rc = bdev->fn_table->destruct(bdev->ctxt); 1344 if (rc < 0) { 1345 SPDK_ERRLOG("destruct failed\n"); 1346 } 1347 } 1348 1349 bool 1350 spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb, 1351 void *remove_ctx) 1352 { 1353 bool success; 1354 1355 pthread_mutex_lock(&bdev->mutex); 1356 1357 if (bdev->status != SPDK_BDEV_STATUS_CLAIMED) { 1358 /* Take ownership of bdev. */ 1359 bdev->remove_cb = remove_cb; 1360 bdev->remove_ctx = remove_ctx; 1361 bdev->status = SPDK_BDEV_STATUS_CLAIMED; 1362 success = true; 1363 } else { 1364 /* bdev is already claimed. */ 1365 success = false; 1366 } 1367 1368 pthread_mutex_unlock(&bdev->mutex); 1369 1370 return success; 1371 } 1372 1373 void 1374 spdk_bdev_unclaim(struct spdk_bdev *bdev) 1375 { 1376 bool do_unregister = false; 1377 1378 pthread_mutex_lock(&bdev->mutex); 1379 assert(bdev->status == SPDK_BDEV_STATUS_CLAIMED || bdev->status == SPDK_BDEV_STATUS_REMOVING); 1380 if (bdev->status == SPDK_BDEV_STATUS_REMOVING) { 1381 do_unregister = true; 1382 } 1383 bdev->remove_cb = NULL; 1384 bdev->remove_ctx = NULL; 1385 bdev->status = SPDK_BDEV_STATUS_UNCLAIMED; 1386 pthread_mutex_unlock(&bdev->mutex); 1387 1388 if (do_unregister == true) { 1389 spdk_bdev_unregister(bdev); 1390 } 1391 } 1392 1393 void 1394 spdk_bdev_io_get_iovec(struct spdk_bdev_io *bdev_io, struct iovec **iovp, int *iovcntp) 1395 { 1396 struct iovec *iovs; 1397 int iovcnt; 1398 1399 if (bdev_io == NULL) { 1400 return; 1401 } 1402 1403 switch (bdev_io->type) { 1404 case SPDK_BDEV_IO_TYPE_READ: 1405 iovs = bdev_io->u.read.iovs; 1406 iovcnt = bdev_io->u.read.iovcnt; 1407 break; 1408 case SPDK_BDEV_IO_TYPE_WRITE: 1409 iovs = bdev_io->u.write.iovs; 1410 iovcnt = bdev_io->u.write.iovcnt; 1411 break; 1412 default: 1413 iovs = NULL; 1414 iovcnt = 0; 1415 break; 1416 } 1417 1418 if (iovp) { 1419 *iovp = iovs; 1420 } 1421 if (iovcntp) { 1422 *iovcntp = iovcnt; 1423 } 1424 } 1425 1426 void 1427 spdk_bdev_module_list_add(struct spdk_bdev_module_if *bdev_module) 1428 { 1429 TAILQ_INSERT_TAIL(&g_bdev_mgr.bdev_modules, bdev_module, tailq); 1430 } 1431 1432 void 1433 spdk_vbdev_module_list_add(struct spdk_bdev_module_if *vbdev_module) 1434 { 1435 TAILQ_INSERT_TAIL(&g_bdev_mgr.vbdev_modules, vbdev_module, tailq); 1436 } 1437 SPDK_SUBSYSTEM_REGISTER(bdev, spdk_bdev_initialize, spdk_bdev_finish, spdk_bdev_config_text) 1438 SPDK_SUBSYSTEM_DEPEND(bdev, copy) 1439