1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk_cunit.h" 35 36 #include "common/lib/test_env.c" 37 #include "common/lib/ut_multithread.c" 38 #include "unit/lib/json_mock.c" 39 40 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 41 #undef SPDK_CONFIG_VTUNE 42 43 #include "bdev/bdev.c" 44 45 #define BDEV_UT_NUM_THREADS 3 46 47 DEFINE_STUB_V(spdk_scsi_nvme_translate, (const struct spdk_bdev_io *bdev_io, 48 int *sc, int *sk, int *asc, int *ascq)); 49 50 /* Return NULL to test hardcoded defaults. */ 51 struct spdk_conf_section * 52 spdk_conf_find_section(struct spdk_conf *cp, const char *name) 53 { 54 return NULL; 55 } 56 57 /* Return NULL to test hardcoded defaults. */ 58 char * 59 spdk_conf_section_get_nmval(struct spdk_conf_section *sp, const char *key, int idx1, int idx2) 60 { 61 return NULL; 62 } 63 64 struct ut_bdev { 65 struct spdk_bdev bdev; 66 void *io_target; 67 }; 68 69 struct ut_bdev_channel { 70 TAILQ_HEAD(, spdk_bdev_io) outstanding_io; 71 uint32_t outstanding_cnt; 72 uint32_t avail_cnt; 73 }; 74 75 int g_io_device; 76 struct ut_bdev g_bdev; 77 struct spdk_bdev_desc *g_desc; 78 bool g_teardown_done = false; 79 bool g_get_io_channel = true; 80 bool g_create_ch = true; 81 bool g_init_complete_called = false; 82 83 static int 84 stub_create_ch(void *io_device, void *ctx_buf) 85 { 86 struct ut_bdev_channel *ch = ctx_buf; 87 88 if (g_create_ch == false) { 89 return -1; 90 } 91 92 TAILQ_INIT(&ch->outstanding_io); 93 ch->outstanding_cnt = 0; 94 /* 95 * When avail gets to 0, the submit_request function will return ENOMEM. 96 * Most tests to not want ENOMEM to occur, so by default set this to a 97 * big value that won't get hit. The ENOMEM tests can then override this 98 * value to something much smaller to induce ENOMEM conditions. 99 */ 100 ch->avail_cnt = 2048; 101 return 0; 102 } 103 104 static void 105 stub_destroy_ch(void *io_device, void *ctx_buf) 106 { 107 } 108 109 static struct spdk_io_channel * 110 stub_get_io_channel(void *ctx) 111 { 112 struct ut_bdev *ut_bdev = ctx; 113 114 if (g_get_io_channel == true) { 115 return spdk_get_io_channel(ut_bdev->io_target); 116 } else { 117 return NULL; 118 } 119 } 120 121 static int 122 stub_destruct(void *ctx) 123 { 124 return 0; 125 } 126 127 static void 128 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 129 { 130 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 131 132 if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { 133 struct spdk_bdev_io *io; 134 135 while (!TAILQ_EMPTY(&ch->outstanding_io)) { 136 io = TAILQ_FIRST(&ch->outstanding_io); 137 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 138 ch->outstanding_cnt--; 139 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_FAILED); 140 ch->avail_cnt++; 141 } 142 } 143 144 if (ch->avail_cnt > 0) { 145 TAILQ_INSERT_TAIL(&ch->outstanding_io, bdev_io, module_link); 146 ch->outstanding_cnt++; 147 ch->avail_cnt--; 148 } else { 149 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); 150 } 151 } 152 153 static uint32_t 154 stub_complete_io(void *io_target, uint32_t num_to_complete) 155 { 156 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 157 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 158 struct spdk_bdev_io *io; 159 bool complete_all = (num_to_complete == 0); 160 uint32_t num_completed = 0; 161 162 while (complete_all || num_completed < num_to_complete) { 163 if (TAILQ_EMPTY(&ch->outstanding_io)) { 164 break; 165 } 166 io = TAILQ_FIRST(&ch->outstanding_io); 167 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 168 ch->outstanding_cnt--; 169 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_SUCCESS); 170 ch->avail_cnt++; 171 num_completed++; 172 } 173 174 spdk_put_io_channel(_ch); 175 return num_completed; 176 } 177 178 static struct spdk_bdev_fn_table fn_table = { 179 .get_io_channel = stub_get_io_channel, 180 .destruct = stub_destruct, 181 .submit_request = stub_submit_request, 182 }; 183 184 static int 185 module_init(void) 186 { 187 return 0; 188 } 189 190 static void 191 module_fini(void) 192 { 193 } 194 195 static void 196 init_complete(void) 197 { 198 g_init_complete_called = true; 199 } 200 201 struct spdk_bdev_module bdev_ut_if = { 202 .name = "bdev_ut", 203 .module_init = module_init, 204 .module_fini = module_fini, 205 .init_complete = init_complete, 206 }; 207 208 SPDK_BDEV_MODULE_REGISTER(&bdev_ut_if) 209 210 static void 211 register_bdev(struct ut_bdev *ut_bdev, char *name, void *io_target) 212 { 213 memset(ut_bdev, 0, sizeof(*ut_bdev)); 214 215 ut_bdev->io_target = io_target; 216 ut_bdev->bdev.ctxt = ut_bdev; 217 ut_bdev->bdev.name = name; 218 ut_bdev->bdev.fn_table = &fn_table; 219 ut_bdev->bdev.module = &bdev_ut_if; 220 ut_bdev->bdev.blocklen = 4096; 221 ut_bdev->bdev.blockcnt = 1024; 222 223 spdk_bdev_register(&ut_bdev->bdev); 224 } 225 226 static void 227 unregister_bdev(struct ut_bdev *ut_bdev) 228 { 229 /* Handle any deferred messages. */ 230 poll_threads(); 231 spdk_bdev_unregister(&ut_bdev->bdev, NULL, NULL); 232 } 233 234 static void 235 bdev_init_cb(void *done, int rc) 236 { 237 CU_ASSERT(rc == 0); 238 *(bool *)done = true; 239 } 240 241 static void 242 setup_test(void) 243 { 244 bool done = false; 245 246 allocate_threads(BDEV_UT_NUM_THREADS); 247 spdk_bdev_initialize(bdev_init_cb, &done); 248 spdk_io_device_register(&g_io_device, stub_create_ch, stub_destroy_ch, 249 sizeof(struct ut_bdev_channel)); 250 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 251 spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &g_desc); 252 } 253 254 static void 255 finish_cb(void *cb_arg) 256 { 257 g_teardown_done = true; 258 } 259 260 static void 261 teardown_test(void) 262 { 263 g_teardown_done = false; 264 spdk_bdev_close(g_desc); 265 g_desc = NULL; 266 unregister_bdev(&g_bdev); 267 spdk_io_device_unregister(&g_io_device, NULL); 268 spdk_bdev_finish(finish_cb, NULL); 269 poll_threads(); 270 memset(&g_bdev, 0, sizeof(g_bdev)); 271 CU_ASSERT(g_teardown_done == true); 272 g_teardown_done = false; 273 free_threads(); 274 } 275 276 static uint32_t 277 bdev_io_tailq_cnt(bdev_io_tailq_t *tailq) 278 { 279 struct spdk_bdev_io *io; 280 uint32_t cnt = 0; 281 282 TAILQ_FOREACH(io, tailq, link) { 283 cnt++; 284 } 285 286 return cnt; 287 } 288 289 static void 290 basic(void) 291 { 292 g_init_complete_called = false; 293 setup_test(); 294 CU_ASSERT(g_init_complete_called == true); 295 296 set_thread(0); 297 298 g_get_io_channel = false; 299 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 300 CU_ASSERT(g_ut_threads[0].ch == NULL); 301 302 g_get_io_channel = true; 303 g_create_ch = false; 304 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 305 CU_ASSERT(g_ut_threads[0].ch == NULL); 306 307 g_get_io_channel = true; 308 g_create_ch = true; 309 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 310 CU_ASSERT(g_ut_threads[0].ch != NULL); 311 spdk_put_io_channel(g_ut_threads[0].ch); 312 313 teardown_test(); 314 } 315 316 static int 317 poller_run_done(void *ctx) 318 { 319 bool *poller_run = ctx; 320 321 *poller_run = true; 322 323 return -1; 324 } 325 326 static int 327 poller_run_times_done(void *ctx) 328 { 329 int *poller_run_times = ctx; 330 331 (*poller_run_times)++; 332 333 return -1; 334 } 335 336 static void 337 basic_poller(void) 338 { 339 struct spdk_poller *poller = NULL; 340 bool poller_run = false; 341 int poller_run_times = 0; 342 343 setup_test(); 344 345 set_thread(0); 346 reset_time(); 347 /* Register a poller with no-wait time and test execution */ 348 poller = spdk_poller_register(poller_run_done, &poller_run, 0); 349 CU_ASSERT(poller != NULL); 350 351 poll_threads(); 352 CU_ASSERT(poller_run == true); 353 354 spdk_poller_unregister(&poller); 355 CU_ASSERT(poller == NULL); 356 357 /* Register a poller with 1000us wait time and test single execution */ 358 poller_run = false; 359 poller = spdk_poller_register(poller_run_done, &poller_run, 1000); 360 CU_ASSERT(poller != NULL); 361 362 poll_threads(); 363 CU_ASSERT(poller_run == false); 364 365 increment_time(1000); 366 poll_threads(); 367 CU_ASSERT(poller_run == true); 368 369 reset_time(); 370 poller_run = false; 371 poll_threads(); 372 CU_ASSERT(poller_run == false); 373 374 increment_time(1000); 375 poll_threads(); 376 CU_ASSERT(poller_run == true); 377 378 spdk_poller_unregister(&poller); 379 CU_ASSERT(poller == NULL); 380 381 reset_time(); 382 /* Register a poller with 1000us wait time and test multiple execution */ 383 poller = spdk_poller_register(poller_run_times_done, &poller_run_times, 1000); 384 CU_ASSERT(poller != NULL); 385 386 poll_threads(); 387 CU_ASSERT(poller_run_times == 0); 388 389 increment_time(1000); 390 poll_threads(); 391 CU_ASSERT(poller_run_times == 1); 392 393 poller_run_times = 0; 394 increment_time(2000); 395 poll_threads(); 396 CU_ASSERT(poller_run_times == 2); 397 398 spdk_poller_unregister(&poller); 399 CU_ASSERT(poller == NULL); 400 401 teardown_test(); 402 } 403 404 static void 405 reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 406 { 407 bool *done = cb_arg; 408 409 CU_ASSERT(success == true); 410 *done = true; 411 spdk_bdev_free_io(bdev_io); 412 } 413 414 static void 415 put_channel_during_reset(void) 416 { 417 struct spdk_io_channel *io_ch; 418 bool done = false; 419 420 setup_test(); 421 422 set_thread(0); 423 io_ch = spdk_bdev_get_io_channel(g_desc); 424 CU_ASSERT(io_ch != NULL); 425 426 /* 427 * Start a reset, but then put the I/O channel before 428 * the deferred messages for the reset get a chance to 429 * execute. 430 */ 431 spdk_bdev_reset(g_desc, io_ch, reset_done, &done); 432 spdk_put_io_channel(io_ch); 433 poll_threads(); 434 stub_complete_io(g_bdev.io_target, 0); 435 436 teardown_test(); 437 } 438 439 static void 440 aborted_reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 441 { 442 enum spdk_bdev_io_status *status = cb_arg; 443 444 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 445 spdk_bdev_free_io(bdev_io); 446 } 447 448 static void 449 aborted_reset(void) 450 { 451 struct spdk_io_channel *io_ch[2]; 452 enum spdk_bdev_io_status status1, status2; 453 454 setup_test(); 455 456 set_thread(0); 457 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 458 CU_ASSERT(io_ch[0] != NULL); 459 spdk_bdev_reset(g_desc, io_ch[0], aborted_reset_done, &status1); 460 poll_threads(); 461 CU_ASSERT(g_bdev.bdev.reset_in_progress != NULL); 462 463 /* 464 * First reset has been submitted on ch0. Now submit a second 465 * reset on ch1 which will get queued since there is already a 466 * reset in progress. 467 */ 468 set_thread(1); 469 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 470 CU_ASSERT(io_ch[1] != NULL); 471 spdk_bdev_reset(g_desc, io_ch[1], aborted_reset_done, &status2); 472 poll_threads(); 473 CU_ASSERT(g_bdev.bdev.reset_in_progress != NULL); 474 475 /* 476 * Now destroy ch1. This will abort the queued reset. Check that 477 * the second reset was completed with failed status. Also check 478 * that bdev->reset_in_progress != NULL, since the original reset 479 * has not been completed yet. This ensures that the bdev code is 480 * correctly noticing that the failed reset is *not* the one that 481 * had been submitted to the bdev module. 482 */ 483 set_thread(1); 484 spdk_put_io_channel(io_ch[1]); 485 poll_threads(); 486 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_FAILED); 487 CU_ASSERT(g_bdev.bdev.reset_in_progress != NULL); 488 489 /* 490 * Now complete the first reset, verify that it completed with SUCCESS 491 * status and that bdev->reset_in_progress is also set back to NULL. 492 */ 493 set_thread(0); 494 spdk_put_io_channel(io_ch[0]); 495 stub_complete_io(g_bdev.io_target, 0); 496 poll_threads(); 497 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 498 CU_ASSERT(g_bdev.bdev.reset_in_progress == NULL); 499 500 teardown_test(); 501 } 502 503 static void 504 io_during_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 505 { 506 enum spdk_bdev_io_status *status = cb_arg; 507 508 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 509 spdk_bdev_free_io(bdev_io); 510 } 511 512 static void 513 io_during_reset(void) 514 { 515 struct spdk_io_channel *io_ch[2]; 516 struct spdk_bdev_channel *bdev_ch[2]; 517 enum spdk_bdev_io_status status0, status1, status_reset; 518 int rc; 519 520 setup_test(); 521 522 /* 523 * First test normal case - submit an I/O on each of two channels (with no resets) 524 * and verify they complete successfully. 525 */ 526 set_thread(0); 527 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 528 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 529 CU_ASSERT(bdev_ch[0]->flags == 0); 530 status0 = SPDK_BDEV_IO_STATUS_PENDING; 531 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 532 CU_ASSERT(rc == 0); 533 534 set_thread(1); 535 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 536 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 537 CU_ASSERT(bdev_ch[1]->flags == 0); 538 status1 = SPDK_BDEV_IO_STATUS_PENDING; 539 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 540 CU_ASSERT(rc == 0); 541 542 poll_threads(); 543 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 544 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 545 546 set_thread(0); 547 stub_complete_io(g_bdev.io_target, 0); 548 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 549 550 set_thread(1); 551 stub_complete_io(g_bdev.io_target, 0); 552 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 553 554 /* 555 * Now submit a reset, and leave it pending while we submit I/O on two different 556 * channels. These I/O should be failed by the bdev layer since the reset is in 557 * progress. 558 */ 559 set_thread(0); 560 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 561 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &status_reset); 562 CU_ASSERT(rc == 0); 563 564 CU_ASSERT(bdev_ch[0]->flags == 0); 565 CU_ASSERT(bdev_ch[1]->flags == 0); 566 poll_threads(); 567 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_RESET_IN_PROGRESS); 568 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_RESET_IN_PROGRESS); 569 570 set_thread(0); 571 status0 = SPDK_BDEV_IO_STATUS_PENDING; 572 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 573 CU_ASSERT(rc == 0); 574 575 set_thread(1); 576 status1 = SPDK_BDEV_IO_STATUS_PENDING; 577 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 578 CU_ASSERT(rc == 0); 579 580 /* 581 * A reset is in progress so these read I/O should complete with failure. Note that we 582 * need to poll_threads() since I/O completed inline have their completion deferred. 583 */ 584 poll_threads(); 585 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 586 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 587 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 588 589 /* 590 * Complete the reset 591 */ 592 set_thread(0); 593 stub_complete_io(g_bdev.io_target, 0); 594 595 /* 596 * Only poll thread 0. We should not get a completion. 597 */ 598 poll_thread(0); 599 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 600 601 /* 602 * Poll both thread 0 and 1 so the messages can propagate and we 603 * get a completion. 604 */ 605 poll_threads(); 606 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 607 608 spdk_put_io_channel(io_ch[0]); 609 set_thread(1); 610 spdk_put_io_channel(io_ch[1]); 611 poll_threads(); 612 613 teardown_test(); 614 } 615 616 static void 617 basic_qos(void) 618 { 619 struct spdk_io_channel *io_ch[2]; 620 struct spdk_bdev_channel *bdev_ch[2]; 621 struct spdk_bdev *bdev; 622 enum spdk_bdev_io_status status; 623 int rc; 624 625 setup_test(); 626 627 /* Enable QoS */ 628 bdev = &g_bdev.bdev; 629 TAILQ_INIT(&bdev->qos.queued); 630 bdev->qos.rate_limit = 2000; /* 2 I/O per millisecond */ 631 bdev->qos.enabled = true; 632 633 g_get_io_channel = true; 634 635 set_thread(0); 636 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 637 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 638 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 639 640 set_thread(1); 641 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 642 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 643 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 644 645 /* 646 * Send an I/O on thread 0, which is where the QoS thread is running. 647 */ 648 set_thread(0); 649 status = SPDK_BDEV_IO_STATUS_PENDING; 650 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status); 651 CU_ASSERT(rc == 0); 652 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 653 poll_threads(); 654 stub_complete_io(g_bdev.io_target, 0); 655 poll_threads(); 656 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 657 658 /* Send an I/O on thread 1. The QoS thread is not running here. */ 659 status = SPDK_BDEV_IO_STATUS_PENDING; 660 set_thread(1); 661 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status); 662 CU_ASSERT(rc == 0); 663 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 664 poll_threads(); 665 /* Complete I/O on thread 1. This should not complete the I/O we submitted */ 666 stub_complete_io(g_bdev.io_target, 0); 667 poll_threads(); 668 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 669 /* Now complete I/O on thread 0 */ 670 set_thread(0); 671 poll_threads(); 672 stub_complete_io(g_bdev.io_target, 0); 673 poll_threads(); 674 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 675 676 /* Tear down the channels */ 677 set_thread(0); 678 spdk_put_io_channel(io_ch[0]); 679 set_thread(1); 680 spdk_put_io_channel(io_ch[1]); 681 poll_threads(); 682 set_thread(0); 683 684 /* Close the descriptor, which should stop the qos channel */ 685 spdk_bdev_close(g_desc); 686 CU_ASSERT(bdev->qos.ch == NULL); 687 688 spdk_bdev_open(bdev, true, NULL, NULL, &g_desc); 689 690 /* Create the channels in reverse order. */ 691 set_thread(1); 692 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 693 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 694 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 695 696 set_thread(0); 697 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 698 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 699 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 700 701 /* Confirm that the qos tracking was re-enabled */ 702 CU_ASSERT(bdev->qos.ch != NULL); 703 704 /* Tear down the channels */ 705 set_thread(0); 706 spdk_put_io_channel(io_ch[0]); 707 set_thread(1); 708 spdk_put_io_channel(io_ch[1]); 709 poll_threads(); 710 711 set_thread(0); 712 713 teardown_test(); 714 } 715 716 static void 717 io_during_qos_queue(void) 718 { 719 struct spdk_io_channel *io_ch[2]; 720 struct spdk_bdev_channel *bdev_ch[2]; 721 struct spdk_bdev *bdev; 722 enum spdk_bdev_io_status status0, status1; 723 int rc; 724 725 setup_test(); 726 reset_time(); 727 728 /* Enable QoS */ 729 bdev = &g_bdev.bdev; 730 TAILQ_INIT(&bdev->qos.queued); 731 bdev->qos.rate_limit = 1000; /* 1000 I/O per second, or 1 per millisecond */ 732 bdev->qos.enabled = true; 733 734 g_get_io_channel = true; 735 736 /* Create channels */ 737 set_thread(0); 738 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 739 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 740 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 741 742 set_thread(1); 743 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 744 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 745 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 746 747 /* Send two I/O */ 748 status1 = SPDK_BDEV_IO_STATUS_PENDING; 749 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 750 CU_ASSERT(rc == 0); 751 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 752 set_thread(0); 753 status0 = SPDK_BDEV_IO_STATUS_PENDING; 754 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 755 CU_ASSERT(rc == 0); 756 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 757 758 /* Complete any I/O that arrived at the disk */ 759 poll_threads(); 760 set_thread(1); 761 stub_complete_io(g_bdev.io_target, 0); 762 set_thread(0); 763 stub_complete_io(g_bdev.io_target, 0); 764 poll_threads(); 765 766 /* Only one of the I/O should complete. (logical XOR) */ 767 if (status0 == SPDK_BDEV_IO_STATUS_SUCCESS) { 768 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 769 } else { 770 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 771 } 772 773 /* Advance in time by a millisecond */ 774 increment_time(1000); 775 776 /* Complete more I/O */ 777 poll_threads(); 778 set_thread(1); 779 stub_complete_io(g_bdev.io_target, 0); 780 set_thread(0); 781 stub_complete_io(g_bdev.io_target, 0); 782 poll_threads(); 783 784 /* Now the second I/O should be done */ 785 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 786 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 787 788 /* Tear down the channels */ 789 set_thread(1); 790 spdk_put_io_channel(io_ch[1]); 791 set_thread(0); 792 spdk_put_io_channel(io_ch[0]); 793 poll_threads(); 794 795 teardown_test(); 796 } 797 798 static void 799 io_during_qos_reset(void) 800 { 801 struct spdk_io_channel *io_ch[2]; 802 struct spdk_bdev_channel *bdev_ch[2]; 803 struct spdk_bdev *bdev; 804 enum spdk_bdev_io_status status0, status1, reset_status; 805 int rc; 806 807 setup_test(); 808 reset_time(); 809 810 /* Enable QoS */ 811 bdev = &g_bdev.bdev; 812 TAILQ_INIT(&bdev->qos.queued); 813 bdev->qos.rate_limit = 1000; /* 1000 I/O per second, or 1 per millisecond */ 814 bdev->qos.enabled = true; 815 816 g_get_io_channel = true; 817 818 /* Create channels */ 819 set_thread(0); 820 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 821 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 822 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 823 824 set_thread(1); 825 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 826 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 827 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 828 829 /* Send two I/O. One of these gets queued by QoS. The other is sitting at the disk. */ 830 status1 = SPDK_BDEV_IO_STATUS_PENDING; 831 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 832 CU_ASSERT(rc == 0); 833 set_thread(0); 834 status0 = SPDK_BDEV_IO_STATUS_PENDING; 835 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 836 CU_ASSERT(rc == 0); 837 838 poll_threads(); 839 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 840 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 841 842 /* Reset the bdev. */ 843 reset_status = SPDK_BDEV_IO_STATUS_PENDING; 844 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &reset_status); 845 CU_ASSERT(rc == 0); 846 847 /* Complete any I/O that arrived at the disk */ 848 poll_threads(); 849 set_thread(1); 850 stub_complete_io(g_bdev.io_target, 0); 851 set_thread(0); 852 stub_complete_io(g_bdev.io_target, 0); 853 poll_threads(); 854 855 CU_ASSERT(reset_status == SPDK_BDEV_IO_STATUS_SUCCESS); 856 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 857 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 858 859 /* Tear down the channels */ 860 set_thread(1); 861 spdk_put_io_channel(io_ch[1]); 862 set_thread(0); 863 spdk_put_io_channel(io_ch[0]); 864 poll_threads(); 865 866 teardown_test(); 867 } 868 869 static void 870 enomem_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 871 { 872 enum spdk_bdev_io_status *status = cb_arg; 873 874 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 875 spdk_bdev_free_io(bdev_io); 876 } 877 878 static void 879 enomem(void) 880 { 881 struct spdk_io_channel *io_ch; 882 struct spdk_bdev_channel *bdev_ch; 883 struct spdk_bdev_module_channel *module_ch; 884 struct ut_bdev_channel *ut_ch; 885 const uint32_t IO_ARRAY_SIZE = 64; 886 const uint32_t AVAIL = 20; 887 enum spdk_bdev_io_status status[IO_ARRAY_SIZE], status_reset; 888 uint32_t nomem_cnt, i; 889 struct spdk_bdev_io *first_io; 890 int rc; 891 892 setup_test(); 893 894 set_thread(0); 895 io_ch = spdk_bdev_get_io_channel(g_desc); 896 bdev_ch = spdk_io_channel_get_ctx(io_ch); 897 module_ch = bdev_ch->module_ch; 898 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 899 ut_ch->avail_cnt = AVAIL; 900 901 /* First submit a number of IOs equal to what the channel can support. */ 902 for (i = 0; i < AVAIL; i++) { 903 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 904 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 905 CU_ASSERT(rc == 0); 906 } 907 CU_ASSERT(TAILQ_EMPTY(&module_ch->nomem_io)); 908 909 /* 910 * Next, submit one additional I/O. This one should fail with ENOMEM and then go onto 911 * the enomem_io list. 912 */ 913 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 914 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 915 CU_ASSERT(rc == 0); 916 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&module_ch->nomem_io)); 917 first_io = TAILQ_FIRST(&module_ch->nomem_io); 918 919 /* 920 * Now submit a bunch more I/O. These should all fail with ENOMEM and get queued behind 921 * the first_io above. 922 */ 923 for (i = AVAIL + 1; i < IO_ARRAY_SIZE; i++) { 924 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 925 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 926 CU_ASSERT(rc == 0); 927 } 928 929 /* Assert that first_io is still at the head of the list. */ 930 CU_ASSERT(TAILQ_FIRST(&module_ch->nomem_io) == first_io); 931 CU_ASSERT(bdev_io_tailq_cnt(&module_ch->nomem_io) == (IO_ARRAY_SIZE - AVAIL)); 932 nomem_cnt = bdev_io_tailq_cnt(&module_ch->nomem_io); 933 CU_ASSERT(module_ch->nomem_threshold == (AVAIL - NOMEM_THRESHOLD_COUNT)); 934 935 /* 936 * Complete 1 I/O only. The key check here is bdev_io_tailq_cnt - this should not have 937 * changed since completing just 1 I/O should not trigger retrying the queued nomem_io 938 * list. 939 */ 940 stub_complete_io(g_bdev.io_target, 1); 941 CU_ASSERT(bdev_io_tailq_cnt(&module_ch->nomem_io) == nomem_cnt); 942 943 /* 944 * Complete enough I/O to hit the nomem_theshold. This should trigger retrying nomem_io, 945 * and we should see I/O get resubmitted to the test bdev module. 946 */ 947 stub_complete_io(g_bdev.io_target, NOMEM_THRESHOLD_COUNT - 1); 948 CU_ASSERT(bdev_io_tailq_cnt(&module_ch->nomem_io) < nomem_cnt); 949 nomem_cnt = bdev_io_tailq_cnt(&module_ch->nomem_io); 950 951 /* Complete 1 I/O only. This should not trigger retrying the queued nomem_io. */ 952 stub_complete_io(g_bdev.io_target, 1); 953 CU_ASSERT(bdev_io_tailq_cnt(&module_ch->nomem_io) == nomem_cnt); 954 955 /* 956 * Send a reset and confirm that all I/O are completed, including the ones that 957 * were queued on the nomem_io list. 958 */ 959 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 960 rc = spdk_bdev_reset(g_desc, io_ch, enomem_done, &status_reset); 961 poll_threads(); 962 CU_ASSERT(rc == 0); 963 /* This will complete the reset. */ 964 stub_complete_io(g_bdev.io_target, 0); 965 966 CU_ASSERT(bdev_io_tailq_cnt(&module_ch->nomem_io) == 0); 967 CU_ASSERT(module_ch->io_outstanding == 0); 968 969 spdk_put_io_channel(io_ch); 970 poll_threads(); 971 teardown_test(); 972 } 973 974 static void 975 enomem_multi_bdev(void) 976 { 977 struct spdk_io_channel *io_ch; 978 struct spdk_bdev_channel *bdev_ch; 979 struct spdk_bdev_module_channel *module_ch; 980 struct ut_bdev_channel *ut_ch; 981 const uint32_t IO_ARRAY_SIZE = 64; 982 const uint32_t AVAIL = 20; 983 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 984 uint32_t i; 985 struct ut_bdev *second_bdev; 986 struct spdk_bdev_desc *second_desc; 987 struct spdk_bdev_channel *second_bdev_ch; 988 struct spdk_io_channel *second_ch; 989 int rc; 990 991 setup_test(); 992 993 /* Register second bdev with the same io_target */ 994 second_bdev = calloc(1, sizeof(*second_bdev)); 995 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 996 register_bdev(second_bdev, "ut_bdev2", g_bdev.io_target); 997 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 998 999 set_thread(0); 1000 io_ch = spdk_bdev_get_io_channel(g_desc); 1001 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1002 module_ch = bdev_ch->module_ch; 1003 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1004 ut_ch->avail_cnt = AVAIL; 1005 1006 second_ch = spdk_bdev_get_io_channel(second_desc); 1007 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1008 SPDK_CU_ASSERT_FATAL(module_ch == second_bdev_ch->module_ch); 1009 1010 /* Saturate io_target through bdev A. */ 1011 for (i = 0; i < AVAIL; i++) { 1012 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1013 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1014 CU_ASSERT(rc == 0); 1015 } 1016 CU_ASSERT(TAILQ_EMPTY(&module_ch->nomem_io)); 1017 1018 /* 1019 * Now submit I/O through the second bdev. This should fail with ENOMEM 1020 * and then go onto the nomem_io list. 1021 */ 1022 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1023 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1024 CU_ASSERT(rc == 0); 1025 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&module_ch->nomem_io)); 1026 1027 /* Complete first bdev's I/O. This should retry sending second bdev's nomem_io */ 1028 stub_complete_io(g_bdev.io_target, AVAIL); 1029 1030 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&module_ch->nomem_io)); 1031 CU_ASSERT(module_ch->io_outstanding == 1); 1032 1033 /* Now complete our retried I/O */ 1034 stub_complete_io(g_bdev.io_target, 1); 1035 SPDK_CU_ASSERT_FATAL(module_ch->io_outstanding == 0); 1036 1037 spdk_put_io_channel(io_ch); 1038 spdk_put_io_channel(second_ch); 1039 spdk_bdev_close(second_desc); 1040 unregister_bdev(second_bdev); 1041 poll_threads(); 1042 free(second_bdev); 1043 teardown_test(); 1044 } 1045 1046 static void 1047 qos_dynamic_enable_done(void *cb_arg, int status) 1048 { 1049 int *rc = cb_arg; 1050 *rc = status; 1051 } 1052 1053 static void 1054 qos_dynamic_enable(void) 1055 { 1056 struct spdk_io_channel *io_ch[2]; 1057 struct spdk_bdev_channel *bdev_ch[2]; 1058 struct spdk_bdev *bdev; 1059 int status, second_status; 1060 1061 setup_test(); 1062 reset_time(); 1063 1064 bdev = &g_bdev.bdev; 1065 1066 g_get_io_channel = true; 1067 1068 /* Create channels */ 1069 set_thread(0); 1070 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1071 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1072 CU_ASSERT(bdev_ch[0]->flags == 0); 1073 1074 set_thread(1); 1075 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1076 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1077 CU_ASSERT(bdev_ch[1]->flags == 0); 1078 1079 set_thread(0); 1080 1081 /* Enable QoS */ 1082 status = -1; 1083 spdk_bdev_set_qos_limit_iops(bdev, 10000, qos_dynamic_enable_done, &status); 1084 poll_threads(); 1085 CU_ASSERT(status == 0); 1086 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1087 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1088 1089 /* Disable QoS */ 1090 status = -1; 1091 spdk_bdev_set_qos_limit_iops(bdev, 0, qos_dynamic_enable_done, &status); 1092 poll_threads(); 1093 CU_ASSERT(status == 0); 1094 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1095 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1096 1097 /* Disable QoS again */ 1098 status = -1; 1099 spdk_bdev_set_qos_limit_iops(bdev, 0, qos_dynamic_enable_done, &status); 1100 poll_threads(); 1101 CU_ASSERT(status == 0); /* This should succeed */ 1102 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1103 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1104 1105 /* Enable QoS on thread 0 */ 1106 status = -1; 1107 spdk_bdev_set_qos_limit_iops(bdev, 10000, qos_dynamic_enable_done, &status); 1108 poll_threads(); 1109 CU_ASSERT(status == 0); 1110 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1111 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1112 1113 /* Disable QoS on thread 1 */ 1114 set_thread(1); 1115 status = -1; 1116 spdk_bdev_set_qos_limit_iops(bdev, 0, qos_dynamic_enable_done, &status); 1117 /* Don't poll yet. This should leave the channels with QoS enabled */ 1118 CU_ASSERT(status == -1); 1119 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1120 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1121 1122 /* Enable QoS. This should immediately fail because the previous disable QoS hasn't completed. */ 1123 second_status = 0; 1124 spdk_bdev_set_qos_limit_iops(bdev, 10000, qos_dynamic_enable_done, &second_status); 1125 poll_threads(); 1126 CU_ASSERT(status == 0); /* The disable should succeed */ 1127 CU_ASSERT(second_status < 0); /* The enable should fail */ 1128 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1129 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1130 1131 /* Enable QoS on thread 1. This should succeed now that the disable has completed. */ 1132 status = -1; 1133 spdk_bdev_set_qos_limit_iops(bdev, 10000, qos_dynamic_enable_done, &status); 1134 poll_threads(); 1135 CU_ASSERT(status == 0); 1136 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1137 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1138 1139 /* Tear down the channels */ 1140 set_thread(0); 1141 spdk_put_io_channel(io_ch[0]); 1142 set_thread(1); 1143 spdk_put_io_channel(io_ch[1]); 1144 poll_threads(); 1145 1146 set_thread(0); 1147 teardown_test(); 1148 } 1149 1150 int 1151 main(int argc, char **argv) 1152 { 1153 CU_pSuite suite = NULL; 1154 unsigned int num_failures; 1155 1156 if (CU_initialize_registry() != CUE_SUCCESS) { 1157 return CU_get_error(); 1158 } 1159 1160 suite = CU_add_suite("bdev", NULL, NULL); 1161 if (suite == NULL) { 1162 CU_cleanup_registry(); 1163 return CU_get_error(); 1164 } 1165 1166 if ( 1167 CU_add_test(suite, "basic", basic) == NULL || 1168 CU_add_test(suite, "basic_poller", basic_poller) == NULL || 1169 CU_add_test(suite, "basic_qos", basic_qos) == NULL || 1170 CU_add_test(suite, "put_channel_during_reset", put_channel_during_reset) == NULL || 1171 CU_add_test(suite, "aborted_reset", aborted_reset) == NULL || 1172 CU_add_test(suite, "io_during_reset", io_during_reset) == NULL || 1173 CU_add_test(suite, "io_during_qos_queue", io_during_qos_queue) == NULL || 1174 CU_add_test(suite, "io_during_qos_reset", io_during_qos_reset) == NULL || 1175 CU_add_test(suite, "enomem", enomem) == NULL || 1176 CU_add_test(suite, "enomem_multi_bdev", enomem_multi_bdev) == NULL || 1177 CU_add_test(suite, "qos_dynamic_enable", qos_dynamic_enable) == NULL 1178 ) { 1179 CU_cleanup_registry(); 1180 return CU_get_error(); 1181 } 1182 1183 CU_basic_set_mode(CU_BRM_VERBOSE); 1184 CU_basic_run_tests(); 1185 num_failures = CU_get_number_of_failures(); 1186 CU_cleanup_registry(); 1187 return num_failures; 1188 } 1189