1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk_cunit.h" 35 36 #include "common/lib/ut_multithread.c" 37 #include "unit/lib/json_mock.c" 38 39 #include "spdk/config.h" 40 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 41 #undef SPDK_CONFIG_VTUNE 42 43 #include "bdev/bdev.c" 44 45 #define BDEV_UT_NUM_THREADS 3 46 47 DEFINE_STUB(spdk_conf_find_section, struct spdk_conf_section *, (struct spdk_conf *cp, 48 const char *name), NULL); 49 DEFINE_STUB(spdk_conf_section_get_nmval, char *, 50 (struct spdk_conf_section *sp, const char *key, int idx1, int idx2), NULL); 51 DEFINE_STUB(spdk_conf_section_get_intval, int, (struct spdk_conf_section *sp, const char *key), -1); 52 53 struct spdk_trace_histories *g_trace_histories; 54 DEFINE_STUB_V(spdk_trace_add_register_fn, (struct spdk_trace_register_fn *reg_fn)); 55 DEFINE_STUB_V(spdk_trace_register_owner, (uint8_t type, char id_prefix)); 56 DEFINE_STUB_V(spdk_trace_register_object, (uint8_t type, char id_prefix)); 57 DEFINE_STUB_V(spdk_trace_register_description, (const char *name, 58 uint16_t tpoint_id, uint8_t owner_type, 59 uint8_t object_type, uint8_t new_object, 60 uint8_t arg1_type, const char *arg1_name)); 61 DEFINE_STUB_V(_spdk_trace_record, (uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, 62 uint32_t size, uint64_t object_id, uint64_t arg1)); 63 DEFINE_STUB(spdk_notify_send, uint64_t, (const char *type, const char *ctx), 0); 64 DEFINE_STUB(spdk_notify_type_register, struct spdk_notify_type *, (const char *type), NULL); 65 66 struct ut_bdev { 67 struct spdk_bdev bdev; 68 void *io_target; 69 }; 70 71 struct ut_bdev_channel { 72 TAILQ_HEAD(, spdk_bdev_io) outstanding_io; 73 uint32_t outstanding_cnt; 74 uint32_t avail_cnt; 75 }; 76 77 int g_io_device; 78 struct ut_bdev g_bdev; 79 struct spdk_bdev_desc *g_desc; 80 bool g_teardown_done = false; 81 bool g_get_io_channel = true; 82 bool g_create_ch = true; 83 bool g_init_complete_called = false; 84 bool g_fini_start_called = true; 85 int g_status = 0; 86 int g_count = 0; 87 struct spdk_histogram_data *g_histogram = NULL; 88 89 static int 90 stub_create_ch(void *io_device, void *ctx_buf) 91 { 92 struct ut_bdev_channel *ch = ctx_buf; 93 94 if (g_create_ch == false) { 95 return -1; 96 } 97 98 TAILQ_INIT(&ch->outstanding_io); 99 ch->outstanding_cnt = 0; 100 /* 101 * When avail gets to 0, the submit_request function will return ENOMEM. 102 * Most tests to not want ENOMEM to occur, so by default set this to a 103 * big value that won't get hit. The ENOMEM tests can then override this 104 * value to something much smaller to induce ENOMEM conditions. 105 */ 106 ch->avail_cnt = 2048; 107 return 0; 108 } 109 110 static void 111 stub_destroy_ch(void *io_device, void *ctx_buf) 112 { 113 } 114 115 static struct spdk_io_channel * 116 stub_get_io_channel(void *ctx) 117 { 118 struct ut_bdev *ut_bdev = ctx; 119 120 if (g_get_io_channel == true) { 121 return spdk_get_io_channel(ut_bdev->io_target); 122 } else { 123 return NULL; 124 } 125 } 126 127 static int 128 stub_destruct(void *ctx) 129 { 130 return 0; 131 } 132 133 static void 134 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 135 { 136 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 137 138 if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { 139 struct spdk_bdev_io *io; 140 141 while (!TAILQ_EMPTY(&ch->outstanding_io)) { 142 io = TAILQ_FIRST(&ch->outstanding_io); 143 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 144 ch->outstanding_cnt--; 145 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_FAILED); 146 ch->avail_cnt++; 147 } 148 } 149 150 if (ch->avail_cnt > 0) { 151 TAILQ_INSERT_TAIL(&ch->outstanding_io, bdev_io, module_link); 152 ch->outstanding_cnt++; 153 ch->avail_cnt--; 154 } else { 155 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); 156 } 157 } 158 159 static uint32_t 160 stub_complete_io(void *io_target, uint32_t num_to_complete) 161 { 162 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 163 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 164 struct spdk_bdev_io *io; 165 bool complete_all = (num_to_complete == 0); 166 uint32_t num_completed = 0; 167 168 while (complete_all || num_completed < num_to_complete) { 169 if (TAILQ_EMPTY(&ch->outstanding_io)) { 170 break; 171 } 172 io = TAILQ_FIRST(&ch->outstanding_io); 173 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 174 ch->outstanding_cnt--; 175 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_SUCCESS); 176 ch->avail_cnt++; 177 num_completed++; 178 } 179 180 spdk_put_io_channel(_ch); 181 return num_completed; 182 } 183 184 static struct spdk_bdev_fn_table fn_table = { 185 .get_io_channel = stub_get_io_channel, 186 .destruct = stub_destruct, 187 .submit_request = stub_submit_request, 188 }; 189 190 struct spdk_bdev_module bdev_ut_if; 191 192 static int 193 module_init(void) 194 { 195 spdk_bdev_module_init_done(&bdev_ut_if); 196 return 0; 197 } 198 199 static void 200 module_fini(void) 201 { 202 } 203 204 static void 205 init_complete(void) 206 { 207 g_init_complete_called = true; 208 } 209 210 static void 211 fini_start(void) 212 { 213 g_fini_start_called = true; 214 } 215 216 struct spdk_bdev_module bdev_ut_if = { 217 .name = "bdev_ut", 218 .module_init = module_init, 219 .module_fini = module_fini, 220 .async_init = true, 221 .init_complete = init_complete, 222 .fini_start = fini_start, 223 }; 224 225 SPDK_BDEV_MODULE_REGISTER(bdev_ut, &bdev_ut_if) 226 227 static void 228 register_bdev(struct ut_bdev *ut_bdev, char *name, void *io_target) 229 { 230 memset(ut_bdev, 0, sizeof(*ut_bdev)); 231 232 ut_bdev->io_target = io_target; 233 ut_bdev->bdev.ctxt = ut_bdev; 234 ut_bdev->bdev.name = name; 235 ut_bdev->bdev.fn_table = &fn_table; 236 ut_bdev->bdev.module = &bdev_ut_if; 237 ut_bdev->bdev.blocklen = 4096; 238 ut_bdev->bdev.blockcnt = 1024; 239 240 spdk_bdev_register(&ut_bdev->bdev); 241 } 242 243 static void 244 unregister_bdev(struct ut_bdev *ut_bdev) 245 { 246 /* Handle any deferred messages. */ 247 poll_threads(); 248 spdk_bdev_unregister(&ut_bdev->bdev, NULL, NULL); 249 } 250 251 static void 252 bdev_init_cb(void *done, int rc) 253 { 254 CU_ASSERT(rc == 0); 255 *(bool *)done = true; 256 } 257 258 static void 259 setup_test(void) 260 { 261 bool done = false; 262 263 allocate_threads(BDEV_UT_NUM_THREADS); 264 set_thread(0); 265 spdk_bdev_initialize(bdev_init_cb, &done); 266 spdk_io_device_register(&g_io_device, stub_create_ch, stub_destroy_ch, 267 sizeof(struct ut_bdev_channel), NULL); 268 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 269 spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &g_desc); 270 } 271 272 static void 273 finish_cb(void *cb_arg) 274 { 275 g_teardown_done = true; 276 } 277 278 static void 279 teardown_test(void) 280 { 281 set_thread(0); 282 g_teardown_done = false; 283 spdk_bdev_close(g_desc); 284 g_desc = NULL; 285 unregister_bdev(&g_bdev); 286 spdk_io_device_unregister(&g_io_device, NULL); 287 spdk_bdev_finish(finish_cb, NULL); 288 poll_threads(); 289 memset(&g_bdev, 0, sizeof(g_bdev)); 290 CU_ASSERT(g_teardown_done == true); 291 g_teardown_done = false; 292 free_threads(); 293 } 294 295 static uint32_t 296 bdev_io_tailq_cnt(bdev_io_tailq_t *tailq) 297 { 298 struct spdk_bdev_io *io; 299 uint32_t cnt = 0; 300 301 TAILQ_FOREACH(io, tailq, internal.link) { 302 cnt++; 303 } 304 305 return cnt; 306 } 307 308 static void 309 basic(void) 310 { 311 g_init_complete_called = false; 312 setup_test(); 313 CU_ASSERT(g_init_complete_called == true); 314 315 set_thread(0); 316 317 g_get_io_channel = false; 318 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 319 CU_ASSERT(g_ut_threads[0].ch == NULL); 320 321 g_get_io_channel = true; 322 g_create_ch = false; 323 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 324 CU_ASSERT(g_ut_threads[0].ch == NULL); 325 326 g_get_io_channel = true; 327 g_create_ch = true; 328 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 329 CU_ASSERT(g_ut_threads[0].ch != NULL); 330 spdk_put_io_channel(g_ut_threads[0].ch); 331 332 g_fini_start_called = false; 333 teardown_test(); 334 CU_ASSERT(g_fini_start_called == true); 335 } 336 337 static void 338 _bdev_removed(void *done) 339 { 340 *(bool *)done = true; 341 } 342 343 static void 344 _bdev_unregistered(void *done, int rc) 345 { 346 CU_ASSERT(rc == 0); 347 *(bool *)done = true; 348 } 349 350 static void 351 unregister_and_close(void) 352 { 353 bool done, remove_notify; 354 struct spdk_bdev_desc *desc = NULL; 355 356 setup_test(); 357 set_thread(0); 358 359 /* setup_test() automatically opens the bdev, 360 * but this test needs to do that in a different 361 * way. */ 362 spdk_bdev_close(g_desc); 363 poll_threads(); 364 365 /* Try hotremoving a bdev with descriptors which don't provide 366 * the notification callback */ 367 spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &desc); 368 SPDK_CU_ASSERT_FATAL(desc != NULL); 369 370 /* There is an open descriptor on the device. Unregister it, 371 * which can't proceed until the descriptor is closed. */ 372 done = false; 373 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 374 375 /* Poll the threads to allow all events to be processed */ 376 poll_threads(); 377 378 /* Make sure the bdev was not unregistered. We still have a 379 * descriptor open */ 380 CU_ASSERT(done == false); 381 382 spdk_bdev_close(desc); 383 poll_threads(); 384 desc = NULL; 385 386 /* The unregister should have completed */ 387 CU_ASSERT(done == true); 388 389 390 /* Register the bdev again */ 391 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 392 393 remove_notify = false; 394 spdk_bdev_open(&g_bdev.bdev, true, _bdev_removed, &remove_notify, &desc); 395 SPDK_CU_ASSERT_FATAL(desc != NULL); 396 CU_ASSERT(remove_notify == false); 397 398 /* There is an open descriptor on the device. Unregister it, 399 * which can't proceed until the descriptor is closed. */ 400 done = false; 401 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 402 /* No polling has occurred, so neither of these should execute */ 403 CU_ASSERT(remove_notify == false); 404 CU_ASSERT(done == false); 405 406 /* Prior to the unregister completing, close the descriptor */ 407 spdk_bdev_close(desc); 408 409 /* Poll the threads to allow all events to be processed */ 410 poll_threads(); 411 412 /* Remove notify should not have been called because the 413 * descriptor is already closed. */ 414 CU_ASSERT(remove_notify == false); 415 416 /* The unregister should have completed */ 417 CU_ASSERT(done == true); 418 419 /* Restore the original g_bdev so that we can use teardown_test(). */ 420 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 421 spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &g_desc); 422 teardown_test(); 423 } 424 425 static void 426 reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 427 { 428 bool *done = cb_arg; 429 430 CU_ASSERT(success == true); 431 *done = true; 432 spdk_bdev_free_io(bdev_io); 433 } 434 435 static void 436 put_channel_during_reset(void) 437 { 438 struct spdk_io_channel *io_ch; 439 bool done = false; 440 441 setup_test(); 442 443 set_thread(0); 444 io_ch = spdk_bdev_get_io_channel(g_desc); 445 CU_ASSERT(io_ch != NULL); 446 447 /* 448 * Start a reset, but then put the I/O channel before 449 * the deferred messages for the reset get a chance to 450 * execute. 451 */ 452 spdk_bdev_reset(g_desc, io_ch, reset_done, &done); 453 spdk_put_io_channel(io_ch); 454 poll_threads(); 455 stub_complete_io(g_bdev.io_target, 0); 456 457 teardown_test(); 458 } 459 460 static void 461 aborted_reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 462 { 463 enum spdk_bdev_io_status *status = cb_arg; 464 465 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 466 spdk_bdev_free_io(bdev_io); 467 } 468 469 static void 470 aborted_reset(void) 471 { 472 struct spdk_io_channel *io_ch[2]; 473 enum spdk_bdev_io_status status1 = SPDK_BDEV_IO_STATUS_PENDING, 474 status2 = SPDK_BDEV_IO_STATUS_PENDING; 475 476 setup_test(); 477 478 set_thread(0); 479 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 480 CU_ASSERT(io_ch[0] != NULL); 481 spdk_bdev_reset(g_desc, io_ch[0], aborted_reset_done, &status1); 482 poll_threads(); 483 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 484 485 /* 486 * First reset has been submitted on ch0. Now submit a second 487 * reset on ch1 which will get queued since there is already a 488 * reset in progress. 489 */ 490 set_thread(1); 491 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 492 CU_ASSERT(io_ch[1] != NULL); 493 spdk_bdev_reset(g_desc, io_ch[1], aborted_reset_done, &status2); 494 poll_threads(); 495 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 496 497 /* 498 * Now destroy ch1. This will abort the queued reset. Check that 499 * the second reset was completed with failed status. Also check 500 * that bdev->internal.reset_in_progress != NULL, since the 501 * original reset has not been completed yet. This ensures that 502 * the bdev code is correctly noticing that the failed reset is 503 * *not* the one that had been submitted to the bdev module. 504 */ 505 set_thread(1); 506 spdk_put_io_channel(io_ch[1]); 507 poll_threads(); 508 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_FAILED); 509 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 510 511 /* 512 * Now complete the first reset, verify that it completed with SUCCESS 513 * status and that bdev->internal.reset_in_progress is also set back to NULL. 514 */ 515 set_thread(0); 516 spdk_put_io_channel(io_ch[0]); 517 stub_complete_io(g_bdev.io_target, 0); 518 poll_threads(); 519 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 520 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 521 522 teardown_test(); 523 } 524 525 static void 526 io_during_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 527 { 528 enum spdk_bdev_io_status *status = cb_arg; 529 530 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 531 spdk_bdev_free_io(bdev_io); 532 } 533 534 static void 535 io_during_reset(void) 536 { 537 struct spdk_io_channel *io_ch[2]; 538 struct spdk_bdev_channel *bdev_ch[2]; 539 enum spdk_bdev_io_status status0, status1, status_reset; 540 int rc; 541 542 setup_test(); 543 544 /* 545 * First test normal case - submit an I/O on each of two channels (with no resets) 546 * and verify they complete successfully. 547 */ 548 set_thread(0); 549 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 550 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 551 CU_ASSERT(bdev_ch[0]->flags == 0); 552 status0 = SPDK_BDEV_IO_STATUS_PENDING; 553 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 554 CU_ASSERT(rc == 0); 555 556 set_thread(1); 557 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 558 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 559 CU_ASSERT(bdev_ch[1]->flags == 0); 560 status1 = SPDK_BDEV_IO_STATUS_PENDING; 561 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 562 CU_ASSERT(rc == 0); 563 564 poll_threads(); 565 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 566 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 567 568 set_thread(0); 569 stub_complete_io(g_bdev.io_target, 0); 570 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 571 572 set_thread(1); 573 stub_complete_io(g_bdev.io_target, 0); 574 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 575 576 /* 577 * Now submit a reset, and leave it pending while we submit I/O on two different 578 * channels. These I/O should be failed by the bdev layer since the reset is in 579 * progress. 580 */ 581 set_thread(0); 582 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 583 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &status_reset); 584 CU_ASSERT(rc == 0); 585 586 CU_ASSERT(bdev_ch[0]->flags == 0); 587 CU_ASSERT(bdev_ch[1]->flags == 0); 588 poll_threads(); 589 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_RESET_IN_PROGRESS); 590 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_RESET_IN_PROGRESS); 591 592 set_thread(0); 593 status0 = SPDK_BDEV_IO_STATUS_PENDING; 594 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 595 CU_ASSERT(rc == 0); 596 597 set_thread(1); 598 status1 = SPDK_BDEV_IO_STATUS_PENDING; 599 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 600 CU_ASSERT(rc == 0); 601 602 /* 603 * A reset is in progress so these read I/O should complete with failure. Note that we 604 * need to poll_threads() since I/O completed inline have their completion deferred. 605 */ 606 poll_threads(); 607 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 608 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 609 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 610 611 /* 612 * Complete the reset 613 */ 614 set_thread(0); 615 stub_complete_io(g_bdev.io_target, 0); 616 617 /* 618 * Only poll thread 0. We should not get a completion. 619 */ 620 poll_thread(0); 621 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 622 623 /* 624 * Poll both thread 0 and 1 so the messages can propagate and we 625 * get a completion. 626 */ 627 poll_threads(); 628 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 629 630 spdk_put_io_channel(io_ch[0]); 631 set_thread(1); 632 spdk_put_io_channel(io_ch[1]); 633 poll_threads(); 634 635 teardown_test(); 636 } 637 638 static void 639 basic_qos(void) 640 { 641 struct spdk_io_channel *io_ch[2]; 642 struct spdk_bdev_channel *bdev_ch[2]; 643 struct spdk_bdev *bdev; 644 enum spdk_bdev_io_status status; 645 int rc; 646 647 setup_test(); 648 649 /* Enable QoS */ 650 bdev = &g_bdev.bdev; 651 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 652 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 653 TAILQ_INIT(&bdev->internal.qos->queued); 654 /* 655 * Enable read/write IOPS, read only byte per second and 656 * read/write byte per second rate limits. 657 * In this case, all rate limits will take equal effect. 658 */ 659 /* 2000 read/write I/O per second, or 2 per millisecond */ 660 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 661 /* 8K read/write byte per millisecond with 4K block size */ 662 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 663 /* 8K read only byte per millisecond with 4K block size */ 664 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 8192000; 665 666 g_get_io_channel = true; 667 668 set_thread(0); 669 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 670 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 671 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 672 673 set_thread(1); 674 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 675 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 676 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 677 678 /* 679 * Send an I/O on thread 0, which is where the QoS thread is running. 680 */ 681 set_thread(0); 682 status = SPDK_BDEV_IO_STATUS_PENDING; 683 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status); 684 CU_ASSERT(rc == 0); 685 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 686 poll_threads(); 687 stub_complete_io(g_bdev.io_target, 0); 688 poll_threads(); 689 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 690 691 /* Send an I/O on thread 1. The QoS thread is not running here. */ 692 status = SPDK_BDEV_IO_STATUS_PENDING; 693 set_thread(1); 694 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status); 695 CU_ASSERT(rc == 0); 696 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 697 poll_threads(); 698 /* Complete I/O on thread 1. This should not complete the I/O we submitted */ 699 stub_complete_io(g_bdev.io_target, 0); 700 poll_threads(); 701 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 702 /* Now complete I/O on thread 0 */ 703 set_thread(0); 704 poll_threads(); 705 stub_complete_io(g_bdev.io_target, 0); 706 poll_threads(); 707 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 708 709 /* 710 * Close the descriptor only, which should stop the qos channel as 711 * the last descriptor removed. 712 */ 713 spdk_bdev_close(g_desc); 714 poll_threads(); 715 CU_ASSERT(bdev->internal.qos->ch == NULL); 716 717 /* 718 * Open the bdev again which shall setup the qos channel as the 719 * channels are valid. 720 */ 721 spdk_bdev_open(bdev, true, NULL, NULL, &g_desc); 722 poll_threads(); 723 CU_ASSERT(bdev->internal.qos->ch != NULL); 724 725 /* Tear down the channels */ 726 set_thread(0); 727 spdk_put_io_channel(io_ch[0]); 728 set_thread(1); 729 spdk_put_io_channel(io_ch[1]); 730 poll_threads(); 731 set_thread(0); 732 733 /* Close the descriptor, which should stop the qos channel */ 734 spdk_bdev_close(g_desc); 735 poll_threads(); 736 CU_ASSERT(bdev->internal.qos->ch == NULL); 737 738 /* Open the bdev again, no qos channel setup without valid channels. */ 739 spdk_bdev_open(bdev, true, NULL, NULL, &g_desc); 740 poll_threads(); 741 CU_ASSERT(bdev->internal.qos->ch == NULL); 742 743 /* Create the channels in reverse order. */ 744 set_thread(1); 745 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 746 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 747 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 748 749 set_thread(0); 750 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 751 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 752 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 753 754 /* Confirm that the qos thread is now thread 1 */ 755 CU_ASSERT(bdev->internal.qos->ch == bdev_ch[1]); 756 757 /* Tear down the channels */ 758 set_thread(0); 759 spdk_put_io_channel(io_ch[0]); 760 set_thread(1); 761 spdk_put_io_channel(io_ch[1]); 762 poll_threads(); 763 764 set_thread(0); 765 766 teardown_test(); 767 } 768 769 static void 770 io_during_qos_queue(void) 771 { 772 struct spdk_io_channel *io_ch[2]; 773 struct spdk_bdev_channel *bdev_ch[2]; 774 struct spdk_bdev *bdev; 775 enum spdk_bdev_io_status status0, status1, status2; 776 int rc; 777 778 setup_test(); 779 MOCK_SET(spdk_get_ticks, 0); 780 781 /* Enable QoS */ 782 bdev = &g_bdev.bdev; 783 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 784 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 785 TAILQ_INIT(&bdev->internal.qos->queued); 786 /* 787 * Enable read/write IOPS, read only byte per sec, write only 788 * byte per sec and read/write byte per sec rate limits. 789 * In this case, both read only and write only byte per sec 790 * rate limit will take effect. 791 */ 792 /* 4000 read/write I/O per second, or 4 per millisecond */ 793 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 4000; 794 /* 8K byte per millisecond with 4K block size */ 795 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 796 /* 4K byte per millisecond with 4K block size */ 797 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 4096000; 798 /* 4K byte per millisecond with 4K block size */ 799 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 4096000; 800 801 g_get_io_channel = true; 802 803 /* Create channels */ 804 set_thread(0); 805 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 806 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 807 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 808 809 set_thread(1); 810 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 811 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 812 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 813 814 /* Send two read I/Os */ 815 status1 = SPDK_BDEV_IO_STATUS_PENDING; 816 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 817 CU_ASSERT(rc == 0); 818 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 819 set_thread(0); 820 status0 = SPDK_BDEV_IO_STATUS_PENDING; 821 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 822 CU_ASSERT(rc == 0); 823 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 824 /* Send one write I/O */ 825 status2 = SPDK_BDEV_IO_STATUS_PENDING; 826 rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status2); 827 CU_ASSERT(rc == 0); 828 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_PENDING); 829 830 /* Complete any I/O that arrived at the disk */ 831 poll_threads(); 832 set_thread(1); 833 stub_complete_io(g_bdev.io_target, 0); 834 set_thread(0); 835 stub_complete_io(g_bdev.io_target, 0); 836 poll_threads(); 837 838 /* Only one of the two read I/Os should complete. (logical XOR) */ 839 if (status0 == SPDK_BDEV_IO_STATUS_SUCCESS) { 840 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 841 } else { 842 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 843 } 844 /* The write I/O should complete. */ 845 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_SUCCESS); 846 847 /* Advance in time by a millisecond */ 848 spdk_delay_us(1000); 849 850 /* Complete more I/O */ 851 poll_threads(); 852 set_thread(1); 853 stub_complete_io(g_bdev.io_target, 0); 854 set_thread(0); 855 stub_complete_io(g_bdev.io_target, 0); 856 poll_threads(); 857 858 /* Now the second read I/O should be done */ 859 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 860 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 861 862 /* Tear down the channels */ 863 set_thread(1); 864 spdk_put_io_channel(io_ch[1]); 865 set_thread(0); 866 spdk_put_io_channel(io_ch[0]); 867 poll_threads(); 868 869 teardown_test(); 870 } 871 872 static void 873 io_during_qos_reset(void) 874 { 875 struct spdk_io_channel *io_ch[2]; 876 struct spdk_bdev_channel *bdev_ch[2]; 877 struct spdk_bdev *bdev; 878 enum spdk_bdev_io_status status0, status1, reset_status; 879 int rc; 880 881 setup_test(); 882 MOCK_SET(spdk_get_ticks, 0); 883 884 /* Enable QoS */ 885 bdev = &g_bdev.bdev; 886 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 887 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 888 TAILQ_INIT(&bdev->internal.qos->queued); 889 /* 890 * Enable read/write IOPS, write only byte per sec and 891 * read/write byte per second rate limits. 892 * In this case, read/write byte per second rate limit will 893 * take effect first. 894 */ 895 /* 2000 read/write I/O per second, or 2 per millisecond */ 896 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 897 /* 4K byte per millisecond with 4K block size */ 898 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 4096000; 899 /* 8K byte per millisecond with 4K block size */ 900 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 8192000; 901 902 g_get_io_channel = true; 903 904 /* Create channels */ 905 set_thread(0); 906 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 907 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 908 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 909 910 set_thread(1); 911 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 912 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 913 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 914 915 /* Send two I/O. One of these gets queued by QoS. The other is sitting at the disk. */ 916 status1 = SPDK_BDEV_IO_STATUS_PENDING; 917 rc = spdk_bdev_write_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 918 CU_ASSERT(rc == 0); 919 set_thread(0); 920 status0 = SPDK_BDEV_IO_STATUS_PENDING; 921 rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 922 CU_ASSERT(rc == 0); 923 924 poll_threads(); 925 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 926 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 927 928 /* Reset the bdev. */ 929 reset_status = SPDK_BDEV_IO_STATUS_PENDING; 930 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &reset_status); 931 CU_ASSERT(rc == 0); 932 933 /* Complete any I/O that arrived at the disk */ 934 poll_threads(); 935 set_thread(1); 936 stub_complete_io(g_bdev.io_target, 0); 937 set_thread(0); 938 stub_complete_io(g_bdev.io_target, 0); 939 poll_threads(); 940 941 CU_ASSERT(reset_status == SPDK_BDEV_IO_STATUS_SUCCESS); 942 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 943 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 944 945 /* Tear down the channels */ 946 set_thread(1); 947 spdk_put_io_channel(io_ch[1]); 948 set_thread(0); 949 spdk_put_io_channel(io_ch[0]); 950 poll_threads(); 951 952 teardown_test(); 953 } 954 955 static void 956 enomem_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 957 { 958 enum spdk_bdev_io_status *status = cb_arg; 959 960 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 961 spdk_bdev_free_io(bdev_io); 962 } 963 964 static void 965 enomem(void) 966 { 967 struct spdk_io_channel *io_ch; 968 struct spdk_bdev_channel *bdev_ch; 969 struct spdk_bdev_shared_resource *shared_resource; 970 struct ut_bdev_channel *ut_ch; 971 const uint32_t IO_ARRAY_SIZE = 64; 972 const uint32_t AVAIL = 20; 973 enum spdk_bdev_io_status status[IO_ARRAY_SIZE], status_reset; 974 uint32_t nomem_cnt, i; 975 struct spdk_bdev_io *first_io; 976 int rc; 977 978 setup_test(); 979 980 set_thread(0); 981 io_ch = spdk_bdev_get_io_channel(g_desc); 982 bdev_ch = spdk_io_channel_get_ctx(io_ch); 983 shared_resource = bdev_ch->shared_resource; 984 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 985 ut_ch->avail_cnt = AVAIL; 986 987 /* First submit a number of IOs equal to what the channel can support. */ 988 for (i = 0; i < AVAIL; i++) { 989 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 990 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 991 CU_ASSERT(rc == 0); 992 } 993 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 994 995 /* 996 * Next, submit one additional I/O. This one should fail with ENOMEM and then go onto 997 * the enomem_io list. 998 */ 999 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1000 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1001 CU_ASSERT(rc == 0); 1002 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1003 first_io = TAILQ_FIRST(&shared_resource->nomem_io); 1004 1005 /* 1006 * Now submit a bunch more I/O. These should all fail with ENOMEM and get queued behind 1007 * the first_io above. 1008 */ 1009 for (i = AVAIL + 1; i < IO_ARRAY_SIZE; i++) { 1010 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1011 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1012 CU_ASSERT(rc == 0); 1013 } 1014 1015 /* Assert that first_io is still at the head of the list. */ 1016 CU_ASSERT(TAILQ_FIRST(&shared_resource->nomem_io) == first_io); 1017 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == (IO_ARRAY_SIZE - AVAIL)); 1018 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 1019 CU_ASSERT(shared_resource->nomem_threshold == (AVAIL - NOMEM_THRESHOLD_COUNT)); 1020 1021 /* 1022 * Complete 1 I/O only. The key check here is bdev_io_tailq_cnt - this should not have 1023 * changed since completing just 1 I/O should not trigger retrying the queued nomem_io 1024 * list. 1025 */ 1026 stub_complete_io(g_bdev.io_target, 1); 1027 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 1028 1029 /* 1030 * Complete enough I/O to hit the nomem_theshold. This should trigger retrying nomem_io, 1031 * and we should see I/O get resubmitted to the test bdev module. 1032 */ 1033 stub_complete_io(g_bdev.io_target, NOMEM_THRESHOLD_COUNT - 1); 1034 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) < nomem_cnt); 1035 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 1036 1037 /* Complete 1 I/O only. This should not trigger retrying the queued nomem_io. */ 1038 stub_complete_io(g_bdev.io_target, 1); 1039 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 1040 1041 /* 1042 * Send a reset and confirm that all I/O are completed, including the ones that 1043 * were queued on the nomem_io list. 1044 */ 1045 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 1046 rc = spdk_bdev_reset(g_desc, io_ch, enomem_done, &status_reset); 1047 poll_threads(); 1048 CU_ASSERT(rc == 0); 1049 /* This will complete the reset. */ 1050 stub_complete_io(g_bdev.io_target, 0); 1051 1052 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == 0); 1053 CU_ASSERT(shared_resource->io_outstanding == 0); 1054 1055 spdk_put_io_channel(io_ch); 1056 poll_threads(); 1057 teardown_test(); 1058 } 1059 1060 static void 1061 enomem_multi_bdev(void) 1062 { 1063 struct spdk_io_channel *io_ch; 1064 struct spdk_bdev_channel *bdev_ch; 1065 struct spdk_bdev_shared_resource *shared_resource; 1066 struct ut_bdev_channel *ut_ch; 1067 const uint32_t IO_ARRAY_SIZE = 64; 1068 const uint32_t AVAIL = 20; 1069 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1070 uint32_t i; 1071 struct ut_bdev *second_bdev; 1072 struct spdk_bdev_desc *second_desc = NULL; 1073 struct spdk_bdev_channel *second_bdev_ch; 1074 struct spdk_io_channel *second_ch; 1075 int rc; 1076 1077 setup_test(); 1078 1079 /* Register second bdev with the same io_target */ 1080 second_bdev = calloc(1, sizeof(*second_bdev)); 1081 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1082 register_bdev(second_bdev, "ut_bdev2", g_bdev.io_target); 1083 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 1084 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1085 1086 set_thread(0); 1087 io_ch = spdk_bdev_get_io_channel(g_desc); 1088 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1089 shared_resource = bdev_ch->shared_resource; 1090 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1091 ut_ch->avail_cnt = AVAIL; 1092 1093 second_ch = spdk_bdev_get_io_channel(second_desc); 1094 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1095 SPDK_CU_ASSERT_FATAL(shared_resource == second_bdev_ch->shared_resource); 1096 1097 /* Saturate io_target through bdev A. */ 1098 for (i = 0; i < AVAIL; i++) { 1099 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1100 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1101 CU_ASSERT(rc == 0); 1102 } 1103 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1104 1105 /* 1106 * Now submit I/O through the second bdev. This should fail with ENOMEM 1107 * and then go onto the nomem_io list. 1108 */ 1109 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1110 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1111 CU_ASSERT(rc == 0); 1112 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1113 1114 /* Complete first bdev's I/O. This should retry sending second bdev's nomem_io */ 1115 stub_complete_io(g_bdev.io_target, AVAIL); 1116 1117 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&shared_resource->nomem_io)); 1118 CU_ASSERT(shared_resource->io_outstanding == 1); 1119 1120 /* Now complete our retried I/O */ 1121 stub_complete_io(g_bdev.io_target, 1); 1122 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == 0); 1123 1124 spdk_put_io_channel(io_ch); 1125 spdk_put_io_channel(second_ch); 1126 spdk_bdev_close(second_desc); 1127 unregister_bdev(second_bdev); 1128 poll_threads(); 1129 free(second_bdev); 1130 teardown_test(); 1131 } 1132 1133 1134 static void 1135 enomem_multi_io_target(void) 1136 { 1137 struct spdk_io_channel *io_ch; 1138 struct spdk_bdev_channel *bdev_ch; 1139 struct ut_bdev_channel *ut_ch; 1140 const uint32_t IO_ARRAY_SIZE = 64; 1141 const uint32_t AVAIL = 20; 1142 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1143 uint32_t i; 1144 int new_io_device; 1145 struct ut_bdev *second_bdev; 1146 struct spdk_bdev_desc *second_desc = NULL; 1147 struct spdk_bdev_channel *second_bdev_ch; 1148 struct spdk_io_channel *second_ch; 1149 int rc; 1150 1151 setup_test(); 1152 1153 /* Create new io_target and a second bdev using it */ 1154 spdk_io_device_register(&new_io_device, stub_create_ch, stub_destroy_ch, 1155 sizeof(struct ut_bdev_channel), NULL); 1156 second_bdev = calloc(1, sizeof(*second_bdev)); 1157 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1158 register_bdev(second_bdev, "ut_bdev2", &new_io_device); 1159 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 1160 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1161 1162 set_thread(0); 1163 io_ch = spdk_bdev_get_io_channel(g_desc); 1164 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1165 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1166 ut_ch->avail_cnt = AVAIL; 1167 1168 /* Different io_target should imply a different shared_resource */ 1169 second_ch = spdk_bdev_get_io_channel(second_desc); 1170 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1171 SPDK_CU_ASSERT_FATAL(bdev_ch->shared_resource != second_bdev_ch->shared_resource); 1172 1173 /* Saturate io_target through bdev A. */ 1174 for (i = 0; i < AVAIL; i++) { 1175 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1176 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1177 CU_ASSERT(rc == 0); 1178 } 1179 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1180 1181 /* Issue one more I/O to fill ENOMEM list. */ 1182 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1183 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1184 CU_ASSERT(rc == 0); 1185 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1186 1187 /* 1188 * Now submit I/O through the second bdev. This should go through and complete 1189 * successfully because we're using a different io_device underneath. 1190 */ 1191 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1192 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1193 CU_ASSERT(rc == 0); 1194 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&second_bdev_ch->shared_resource->nomem_io)); 1195 stub_complete_io(second_bdev->io_target, 1); 1196 1197 /* Cleanup; Complete outstanding I/O. */ 1198 stub_complete_io(g_bdev.io_target, AVAIL); 1199 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1200 /* Complete the ENOMEM I/O */ 1201 stub_complete_io(g_bdev.io_target, 1); 1202 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1203 1204 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1205 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1206 spdk_put_io_channel(io_ch); 1207 spdk_put_io_channel(second_ch); 1208 spdk_bdev_close(second_desc); 1209 unregister_bdev(second_bdev); 1210 spdk_io_device_unregister(&new_io_device, NULL); 1211 poll_threads(); 1212 free(second_bdev); 1213 teardown_test(); 1214 } 1215 1216 static void 1217 qos_dynamic_enable_done(void *cb_arg, int status) 1218 { 1219 int *rc = cb_arg; 1220 *rc = status; 1221 } 1222 1223 static void 1224 qos_dynamic_enable(void) 1225 { 1226 struct spdk_io_channel *io_ch[2]; 1227 struct spdk_bdev_channel *bdev_ch[2]; 1228 struct spdk_bdev *bdev; 1229 enum spdk_bdev_io_status bdev_io_status[2]; 1230 uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {}; 1231 int status, second_status, rc, i; 1232 1233 setup_test(); 1234 MOCK_SET(spdk_get_ticks, 0); 1235 1236 for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) { 1237 limits[i] = UINT64_MAX; 1238 } 1239 1240 bdev = &g_bdev.bdev; 1241 1242 g_get_io_channel = true; 1243 1244 /* Create channels */ 1245 set_thread(0); 1246 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1247 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1248 CU_ASSERT(bdev_ch[0]->flags == 0); 1249 1250 set_thread(1); 1251 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1252 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1253 CU_ASSERT(bdev_ch[1]->flags == 0); 1254 1255 set_thread(0); 1256 1257 /* 1258 * Enable QoS: Read/Write IOPS, Read/Write byte, 1259 * Read only byte and Write only byte per second 1260 * rate limits. 1261 * More than 10 I/Os allowed per timeslice. 1262 */ 1263 status = -1; 1264 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1265 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 100; 1266 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 100; 1267 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 10; 1268 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1269 poll_threads(); 1270 CU_ASSERT(status == 0); 1271 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1272 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1273 1274 /* 1275 * Submit and complete 10 I/O to fill the QoS allotment for this timeslice. 1276 * Additional I/O will then be queued. 1277 */ 1278 set_thread(0); 1279 for (i = 0; i < 10; i++) { 1280 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1281 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1282 CU_ASSERT(rc == 0); 1283 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1284 poll_thread(0); 1285 stub_complete_io(g_bdev.io_target, 0); 1286 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1287 } 1288 1289 /* 1290 * Send two more I/O. These I/O will be queued since the current timeslice allotment has been 1291 * filled already. We want to test that when QoS is disabled that these two I/O: 1292 * 1) are not aborted 1293 * 2) are sent back to their original thread for resubmission 1294 */ 1295 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1296 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1297 CU_ASSERT(rc == 0); 1298 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1299 set_thread(1); 1300 bdev_io_status[1] = SPDK_BDEV_IO_STATUS_PENDING; 1301 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &bdev_io_status[1]); 1302 CU_ASSERT(rc == 0); 1303 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1304 poll_threads(); 1305 1306 /* 1307 * Disable QoS: Read/Write IOPS, Read/Write byte, 1308 * Read only byte rate limits 1309 */ 1310 status = -1; 1311 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1312 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 0; 1313 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 0; 1314 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1315 poll_threads(); 1316 CU_ASSERT(status == 0); 1317 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1318 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1319 1320 /* Disable QoS: Write only Byte per second rate limit */ 1321 status = -1; 1322 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 0; 1323 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1324 poll_threads(); 1325 CU_ASSERT(status == 0); 1326 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1327 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1328 1329 /* 1330 * All I/O should have been resubmitted back on their original thread. Complete 1331 * all I/O on thread 0, and ensure that only the thread 0 I/O was completed. 1332 */ 1333 set_thread(0); 1334 stub_complete_io(g_bdev.io_target, 0); 1335 poll_threads(); 1336 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1337 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1338 1339 /* Now complete all I/O on thread 1 and ensure the thread 1 I/O was completed. */ 1340 set_thread(1); 1341 stub_complete_io(g_bdev.io_target, 0); 1342 poll_threads(); 1343 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_SUCCESS); 1344 1345 /* Disable QoS again */ 1346 status = -1; 1347 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1348 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1349 poll_threads(); 1350 CU_ASSERT(status == 0); /* This should succeed */ 1351 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1352 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1353 1354 /* Enable QoS on thread 0 */ 1355 status = -1; 1356 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1357 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1358 poll_threads(); 1359 CU_ASSERT(status == 0); 1360 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1361 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1362 1363 /* Disable QoS on thread 1 */ 1364 set_thread(1); 1365 status = -1; 1366 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1367 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1368 /* Don't poll yet. This should leave the channels with QoS enabled */ 1369 CU_ASSERT(status == -1); 1370 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1371 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1372 1373 /* Enable QoS. This should immediately fail because the previous disable QoS hasn't completed. */ 1374 second_status = 0; 1375 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 10; 1376 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &second_status); 1377 poll_threads(); 1378 CU_ASSERT(status == 0); /* The disable should succeed */ 1379 CU_ASSERT(second_status < 0); /* The enable should fail */ 1380 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1381 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1382 1383 /* Enable QoS on thread 1. This should succeed now that the disable has completed. */ 1384 status = -1; 1385 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1386 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1387 poll_threads(); 1388 CU_ASSERT(status == 0); 1389 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1390 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1391 1392 /* Tear down the channels */ 1393 set_thread(0); 1394 spdk_put_io_channel(io_ch[0]); 1395 set_thread(1); 1396 spdk_put_io_channel(io_ch[1]); 1397 poll_threads(); 1398 1399 set_thread(0); 1400 teardown_test(); 1401 } 1402 1403 static void 1404 histogram_status_cb(void *cb_arg, int status) 1405 { 1406 g_status = status; 1407 } 1408 1409 static void 1410 histogram_data_cb(void *cb_arg, int status, struct spdk_histogram_data *histogram) 1411 { 1412 g_status = status; 1413 g_histogram = histogram; 1414 } 1415 1416 static void 1417 histogram_io_count(void *ctx, uint64_t start, uint64_t end, uint64_t count, 1418 uint64_t total, uint64_t so_far) 1419 { 1420 g_count += count; 1421 } 1422 1423 static void 1424 bdev_histograms_mt(void) 1425 { 1426 struct spdk_io_channel *ch[2]; 1427 struct spdk_histogram_data *histogram; 1428 uint8_t buf[4096]; 1429 int status = false; 1430 int rc; 1431 1432 1433 setup_test(); 1434 1435 set_thread(0); 1436 ch[0] = spdk_bdev_get_io_channel(g_desc); 1437 CU_ASSERT(ch[0] != NULL); 1438 1439 set_thread(1); 1440 ch[1] = spdk_bdev_get_io_channel(g_desc); 1441 CU_ASSERT(ch[1] != NULL); 1442 1443 1444 /* Enable histogram */ 1445 spdk_bdev_histogram_enable(&g_bdev.bdev, histogram_status_cb, NULL, true); 1446 poll_threads(); 1447 CU_ASSERT(g_status == 0); 1448 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == true); 1449 1450 /* Allocate histogram */ 1451 histogram = spdk_histogram_data_alloc(); 1452 1453 /* Check if histogram is zeroed */ 1454 spdk_bdev_histogram_get(&g_bdev.bdev, histogram, histogram_data_cb, NULL); 1455 poll_threads(); 1456 CU_ASSERT(g_status == 0); 1457 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 1458 1459 g_count = 0; 1460 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 1461 1462 CU_ASSERT(g_count == 0); 1463 1464 set_thread(0); 1465 rc = spdk_bdev_write_blocks(g_desc, ch[0], &buf, 0, 1, io_during_io_done, &status); 1466 CU_ASSERT(rc == 0); 1467 1468 spdk_delay_us(10); 1469 stub_complete_io(g_bdev.io_target, 1); 1470 poll_threads(); 1471 CU_ASSERT(status == true); 1472 1473 1474 set_thread(1); 1475 rc = spdk_bdev_read_blocks(g_desc, ch[1], &buf, 0, 1, io_during_io_done, &status); 1476 CU_ASSERT(rc == 0); 1477 1478 spdk_delay_us(10); 1479 stub_complete_io(g_bdev.io_target, 1); 1480 poll_threads(); 1481 CU_ASSERT(status == true); 1482 1483 set_thread(0); 1484 1485 /* Check if histogram gathered data from all I/O channels */ 1486 spdk_bdev_histogram_get(&g_bdev.bdev, histogram, histogram_data_cb, NULL); 1487 poll_threads(); 1488 CU_ASSERT(g_status == 0); 1489 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == true); 1490 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 1491 1492 g_count = 0; 1493 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 1494 CU_ASSERT(g_count == 2); 1495 1496 /* Disable histogram */ 1497 spdk_bdev_histogram_enable(&g_bdev.bdev, histogram_status_cb, NULL, false); 1498 poll_threads(); 1499 CU_ASSERT(g_status == 0); 1500 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == false); 1501 1502 spdk_histogram_data_free(histogram); 1503 1504 /* Tear down the channels */ 1505 set_thread(0); 1506 spdk_put_io_channel(ch[0]); 1507 set_thread(1); 1508 spdk_put_io_channel(ch[1]); 1509 poll_threads(); 1510 set_thread(0); 1511 teardown_test(); 1512 1513 } 1514 1515 int 1516 main(int argc, char **argv) 1517 { 1518 CU_pSuite suite = NULL; 1519 unsigned int num_failures; 1520 1521 if (CU_initialize_registry() != CUE_SUCCESS) { 1522 return CU_get_error(); 1523 } 1524 1525 suite = CU_add_suite("bdev", NULL, NULL); 1526 if (suite == NULL) { 1527 CU_cleanup_registry(); 1528 return CU_get_error(); 1529 } 1530 1531 if ( 1532 CU_add_test(suite, "basic", basic) == NULL || 1533 CU_add_test(suite, "unregister_and_close", unregister_and_close) == NULL || 1534 CU_add_test(suite, "basic_qos", basic_qos) == NULL || 1535 CU_add_test(suite, "put_channel_during_reset", put_channel_during_reset) == NULL || 1536 CU_add_test(suite, "aborted_reset", aborted_reset) == NULL || 1537 CU_add_test(suite, "io_during_reset", io_during_reset) == NULL || 1538 CU_add_test(suite, "io_during_qos_queue", io_during_qos_queue) == NULL || 1539 CU_add_test(suite, "io_during_qos_reset", io_during_qos_reset) == NULL || 1540 CU_add_test(suite, "enomem", enomem) == NULL || 1541 CU_add_test(suite, "enomem_multi_bdev", enomem_multi_bdev) == NULL || 1542 CU_add_test(suite, "enomem_multi_io_target", enomem_multi_io_target) == NULL || 1543 CU_add_test(suite, "qos_dynamic_enable", qos_dynamic_enable) == NULL || 1544 CU_add_test(suite, "bdev_histograms_mt", bdev_histograms_mt) == NULL 1545 ) { 1546 CU_cleanup_registry(); 1547 return CU_get_error(); 1548 } 1549 1550 CU_basic_set_mode(CU_BRM_VERBOSE); 1551 CU_basic_run_tests(); 1552 num_failures = CU_get_number_of_failures(); 1553 CU_cleanup_registry(); 1554 return num_failures; 1555 } 1556