1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk_internal/cunit.h" 8 9 #include "common/lib/ut_multithread.c" 10 #include "unit/lib/json_mock.c" 11 12 #include "spdk/config.h" 13 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 14 #undef SPDK_CONFIG_VTUNE 15 16 #include "bdev/bdev.c" 17 18 #define BDEV_UT_NUM_THREADS 3 19 20 DEFINE_STUB(spdk_notify_send, uint64_t, (const char *type, const char *ctx), 0); 21 DEFINE_STUB(spdk_notify_type_register, struct spdk_notify_type *, (const char *type), NULL); 22 DEFINE_STUB_V(spdk_scsi_nvme_translate, (const struct spdk_bdev_io *bdev_io, int *sc, int *sk, 23 int *asc, int *ascq)); 24 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, (struct spdk_memory_domain *domain), 25 "test_domain"); 26 DEFINE_STUB(spdk_memory_domain_get_dma_device_type, enum spdk_dma_device_type, 27 (struct spdk_memory_domain *domain), 0); 28 DEFINE_STUB_V(spdk_accel_sequence_finish, 29 (struct spdk_accel_sequence *seq, spdk_accel_completion_cb cb_fn, void *cb_arg)); 30 DEFINE_STUB_V(spdk_accel_sequence_abort, (struct spdk_accel_sequence *seq)); 31 DEFINE_STUB_V(spdk_accel_sequence_reverse, (struct spdk_accel_sequence *seq)); 32 DEFINE_STUB(spdk_accel_append_copy, int, 33 (struct spdk_accel_sequence **seq, struct spdk_io_channel *ch, struct iovec *dst_iovs, 34 uint32_t dst_iovcnt, struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 35 struct iovec *src_iovs, uint32_t src_iovcnt, struct spdk_memory_domain *src_domain, 36 void *src_domain_ctx, spdk_accel_step_cb cb_fn, void *cb_arg), 0); 37 DEFINE_STUB(spdk_accel_get_memory_domain, struct spdk_memory_domain *, (void), NULL); 38 39 DEFINE_RETURN_MOCK(spdk_memory_domain_pull_data, int); 40 int 41 spdk_memory_domain_pull_data(struct spdk_memory_domain *src_domain, void *src_domain_ctx, 42 struct iovec *src_iov, uint32_t src_iov_cnt, struct iovec *dst_iov, uint32_t dst_iov_cnt, 43 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) 44 { 45 HANDLE_RETURN_MOCK(spdk_memory_domain_pull_data); 46 47 cpl_cb(cpl_cb_arg, 0); 48 return 0; 49 } 50 51 DEFINE_RETURN_MOCK(spdk_memory_domain_push_data, int); 52 int 53 spdk_memory_domain_push_data(struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 54 struct iovec *dst_iov, uint32_t dst_iovcnt, struct iovec *src_iov, uint32_t src_iovcnt, 55 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) 56 { 57 HANDLE_RETURN_MOCK(spdk_memory_domain_push_data); 58 59 cpl_cb(cpl_cb_arg, 0); 60 return 0; 61 } 62 63 static int g_accel_io_device; 64 65 struct spdk_io_channel * 66 spdk_accel_get_io_channel(void) 67 { 68 return spdk_get_io_channel(&g_accel_io_device); 69 } 70 71 struct ut_bdev { 72 struct spdk_bdev bdev; 73 void *io_target; 74 }; 75 76 struct ut_bdev_io { 77 TAILQ_ENTRY(ut_bdev_io) link; 78 }; 79 80 struct ut_bdev_channel { 81 TAILQ_HEAD(, ut_bdev_io) outstanding_io; 82 uint32_t outstanding_cnt; 83 uint32_t avail_cnt; 84 struct spdk_thread *thread; 85 TAILQ_ENTRY(ut_bdev_channel) link; 86 }; 87 88 int g_io_device; 89 struct ut_bdev g_bdev; 90 struct spdk_bdev_desc *g_desc; 91 bool g_teardown_done = false; 92 bool g_get_io_channel = true; 93 bool g_create_ch = true; 94 bool g_init_complete_called = false; 95 bool g_fini_start_called = true; 96 int g_status = 0; 97 int g_count = 0; 98 struct spdk_histogram_data *g_histogram = NULL; 99 TAILQ_HEAD(, ut_bdev_channel) g_ut_channels; 100 101 static int 102 ut_accel_ch_create_cb(void *io_device, void *ctx) 103 { 104 return 0; 105 } 106 107 static void 108 ut_accel_ch_destroy_cb(void *io_device, void *ctx) 109 { 110 } 111 112 static int 113 stub_create_ch(void *io_device, void *ctx_buf) 114 { 115 struct ut_bdev_channel *ch = ctx_buf; 116 117 if (g_create_ch == false) { 118 return -1; 119 } 120 121 TAILQ_INIT(&ch->outstanding_io); 122 ch->outstanding_cnt = 0; 123 /* 124 * When avail gets to 0, the submit_request function will return ENOMEM. 125 * Most tests to not want ENOMEM to occur, so by default set this to a 126 * big value that won't get hit. The ENOMEM tests can then override this 127 * value to something much smaller to induce ENOMEM conditions. 128 */ 129 ch->avail_cnt = 2048; 130 ch->thread = spdk_get_thread(); 131 132 TAILQ_INSERT_TAIL(&g_ut_channels, ch, link); 133 134 return 0; 135 } 136 137 static void 138 stub_destroy_ch(void *io_device, void *ctx_buf) 139 { 140 struct ut_bdev_channel *ch = ctx_buf; 141 142 TAILQ_REMOVE(&g_ut_channels, ch, link); 143 } 144 145 static struct spdk_io_channel * 146 stub_get_io_channel(void *ctx) 147 { 148 struct ut_bdev *ut_bdev = ctx; 149 150 if (g_get_io_channel == true) { 151 return spdk_get_io_channel(ut_bdev->io_target); 152 } else { 153 return NULL; 154 } 155 } 156 157 static int 158 stub_destruct(void *ctx) 159 { 160 return 0; 161 } 162 163 static void 164 stub_reset_channel(void *ctx) 165 { 166 struct ut_bdev_channel *ch = ctx; 167 struct ut_bdev_io *bio; 168 169 while (!TAILQ_EMPTY(&ch->outstanding_io)) { 170 bio = TAILQ_FIRST(&ch->outstanding_io); 171 TAILQ_REMOVE(&ch->outstanding_io, bio, link); 172 ch->outstanding_cnt--; 173 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED); 174 ch->avail_cnt++; 175 } 176 } 177 178 static void 179 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 180 { 181 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch), *tmp_ch; 182 struct spdk_bdev_io *io; 183 struct ut_bdev_io *bio; 184 185 if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { 186 TAILQ_FOREACH(tmp_ch, &g_ut_channels, link) { 187 if (spdk_get_thread() == tmp_ch->thread) { 188 stub_reset_channel(tmp_ch); 189 } else { 190 spdk_thread_send_msg(tmp_ch->thread, stub_reset_channel, tmp_ch); 191 } 192 } 193 } else if (bdev_io->type == SPDK_BDEV_IO_TYPE_ABORT) { 194 TAILQ_FOREACH(bio, &ch->outstanding_io, link) { 195 io = spdk_bdev_io_from_ctx(bio); 196 if (io == bdev_io->u.abort.bio_to_abort) { 197 TAILQ_REMOVE(&ch->outstanding_io, bio, link); 198 ch->outstanding_cnt--; 199 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_ABORTED); 200 ch->avail_cnt++; 201 202 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); 203 return; 204 } 205 } 206 207 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 208 return; 209 } 210 211 if (ch->avail_cnt > 0) { 212 TAILQ_INSERT_TAIL(&ch->outstanding_io, (struct ut_bdev_io *)bdev_io->driver_ctx, link); 213 ch->outstanding_cnt++; 214 ch->avail_cnt--; 215 } else { 216 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); 217 } 218 } 219 220 static uint32_t 221 stub_complete_io(void *io_target, uint32_t num_to_complete) 222 { 223 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 224 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 225 struct ut_bdev_io *bio; 226 struct spdk_bdev_io *io; 227 bool complete_all = (num_to_complete == 0); 228 uint32_t num_completed = 0; 229 230 while (complete_all || num_completed < num_to_complete) { 231 if (TAILQ_EMPTY(&ch->outstanding_io)) { 232 break; 233 } 234 bio = TAILQ_FIRST(&ch->outstanding_io); 235 TAILQ_REMOVE(&ch->outstanding_io, bio, link); 236 io = spdk_bdev_io_from_ctx(bio); 237 ch->outstanding_cnt--; 238 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_SUCCESS); 239 ch->avail_cnt++; 240 num_completed++; 241 } 242 spdk_put_io_channel(_ch); 243 return num_completed; 244 } 245 246 static bool 247 stub_io_type_supported(void *ctx, enum spdk_bdev_io_type type) 248 { 249 return true; 250 } 251 252 static struct spdk_bdev_fn_table fn_table = { 253 .get_io_channel = stub_get_io_channel, 254 .destruct = stub_destruct, 255 .submit_request = stub_submit_request, 256 .io_type_supported = stub_io_type_supported, 257 }; 258 259 struct spdk_bdev_module bdev_ut_if; 260 261 static int 262 module_init(void) 263 { 264 spdk_bdev_module_init_done(&bdev_ut_if); 265 return 0; 266 } 267 268 static void 269 module_fini(void) 270 { 271 } 272 273 static void 274 init_complete(void) 275 { 276 g_init_complete_called = true; 277 } 278 279 static void 280 fini_start(void) 281 { 282 g_fini_start_called = true; 283 } 284 285 static int 286 get_ctx_size(void) 287 { 288 return sizeof(struct ut_bdev_io); 289 } 290 291 struct spdk_bdev_module bdev_ut_if = { 292 .name = "bdev_ut", 293 .module_init = module_init, 294 .module_fini = module_fini, 295 .async_init = true, 296 .init_complete = init_complete, 297 .fini_start = fini_start, 298 .get_ctx_size = get_ctx_size, 299 }; 300 301 SPDK_BDEV_MODULE_REGISTER(bdev_ut, &bdev_ut_if) 302 303 static void 304 register_bdev(struct ut_bdev *ut_bdev, char *name, void *io_target) 305 { 306 memset(ut_bdev, 0, sizeof(*ut_bdev)); 307 308 ut_bdev->io_target = io_target; 309 ut_bdev->bdev.ctxt = ut_bdev; 310 ut_bdev->bdev.name = name; 311 ut_bdev->bdev.fn_table = &fn_table; 312 ut_bdev->bdev.module = &bdev_ut_if; 313 ut_bdev->bdev.blocklen = 4096; 314 ut_bdev->bdev.blockcnt = 1024; 315 316 spdk_bdev_register(&ut_bdev->bdev); 317 } 318 319 static void 320 unregister_bdev(struct ut_bdev *ut_bdev) 321 { 322 /* Handle any deferred messages. */ 323 poll_threads(); 324 spdk_bdev_unregister(&ut_bdev->bdev, NULL, NULL); 325 /* Handle the async bdev unregister. */ 326 poll_threads(); 327 } 328 329 static void 330 bdev_init_cb(void *done, int rc) 331 { 332 CU_ASSERT(rc == 0); 333 *(bool *)done = true; 334 } 335 336 static void 337 _bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, 338 void *event_ctx) 339 { 340 switch (type) { 341 case SPDK_BDEV_EVENT_REMOVE: 342 if (event_ctx != NULL) { 343 *(bool *)event_ctx = true; 344 } 345 break; 346 case SPDK_BDEV_EVENT_RESIZE: 347 if (event_ctx != NULL) { 348 *(int *)event_ctx += 1; 349 } 350 break; 351 default: 352 CU_ASSERT(false); 353 break; 354 } 355 } 356 357 static void 358 setup_test(void) 359 { 360 bool done = false; 361 int rc; 362 363 TAILQ_INIT(&g_ut_channels); 364 365 allocate_cores(BDEV_UT_NUM_THREADS); 366 allocate_threads(BDEV_UT_NUM_THREADS); 367 set_thread(0); 368 369 rc = spdk_iobuf_initialize(); 370 CU_ASSERT(rc == 0); 371 spdk_bdev_initialize(bdev_init_cb, &done); 372 spdk_io_device_register(&g_io_device, stub_create_ch, stub_destroy_ch, 373 sizeof(struct ut_bdev_channel), NULL); 374 spdk_io_device_register(&g_accel_io_device, ut_accel_ch_create_cb, 375 ut_accel_ch_destroy_cb, 0, NULL); 376 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 377 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 378 } 379 380 static void 381 finish_cb(void *cb_arg) 382 { 383 g_teardown_done = true; 384 } 385 386 static void 387 teardown_test(void) 388 { 389 set_thread(0); 390 g_teardown_done = false; 391 spdk_bdev_close(g_desc); 392 g_desc = NULL; 393 unregister_bdev(&g_bdev); 394 spdk_io_device_unregister(&g_io_device, NULL); 395 spdk_bdev_finish(finish_cb, NULL); 396 spdk_io_device_unregister(&g_accel_io_device, NULL); 397 spdk_iobuf_finish(finish_cb, NULL); 398 poll_threads(); 399 memset(&g_bdev, 0, sizeof(g_bdev)); 400 CU_ASSERT(g_teardown_done == true); 401 g_teardown_done = false; 402 free_threads(); 403 free_cores(); 404 CU_ASSERT(TAILQ_EMPTY(&g_ut_channels)) 405 } 406 407 static uint32_t 408 bdev_io_tailq_cnt(bdev_io_tailq_t *tailq) 409 { 410 struct spdk_bdev_io *io; 411 uint32_t cnt = 0; 412 413 TAILQ_FOREACH(io, tailq, internal.link) { 414 cnt++; 415 } 416 417 return cnt; 418 } 419 420 static void 421 basic(void) 422 { 423 g_init_complete_called = false; 424 setup_test(); 425 CU_ASSERT(g_init_complete_called == true); 426 427 set_thread(0); 428 429 g_get_io_channel = false; 430 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 431 CU_ASSERT(g_ut_threads[0].ch == NULL); 432 433 g_get_io_channel = true; 434 g_create_ch = false; 435 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 436 CU_ASSERT(g_ut_threads[0].ch == NULL); 437 438 g_get_io_channel = true; 439 g_create_ch = true; 440 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 441 CU_ASSERT(g_ut_threads[0].ch != NULL); 442 spdk_put_io_channel(g_ut_threads[0].ch); 443 444 g_fini_start_called = false; 445 teardown_test(); 446 CU_ASSERT(g_fini_start_called == true); 447 } 448 449 static void 450 _bdev_unregistered(void *done, int rc) 451 { 452 CU_ASSERT(rc == 0); 453 *(bool *)done = true; 454 } 455 456 static void 457 unregister_and_close(void) 458 { 459 bool done, remove_notify; 460 struct spdk_bdev_desc *desc = NULL; 461 462 setup_test(); 463 set_thread(0); 464 465 /* setup_test() automatically opens the bdev, 466 * but this test needs to do that in a different 467 * way. */ 468 spdk_bdev_close(g_desc); 469 poll_threads(); 470 471 /* Try hotremoving a bdev with descriptors which don't provide 472 * any context to the notification callback */ 473 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &desc); 474 SPDK_CU_ASSERT_FATAL(desc != NULL); 475 476 /* There is an open descriptor on the device. Unregister it, 477 * which can't proceed until the descriptor is closed. */ 478 done = false; 479 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 480 481 /* Poll the threads to allow all events to be processed */ 482 poll_threads(); 483 484 /* Make sure the bdev was not unregistered. We still have a 485 * descriptor open */ 486 CU_ASSERT(done == false); 487 488 spdk_bdev_close(desc); 489 poll_threads(); 490 desc = NULL; 491 492 /* The unregister should have completed */ 493 CU_ASSERT(done == true); 494 495 496 /* Register the bdev again */ 497 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 498 499 remove_notify = false; 500 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, &remove_notify, &desc); 501 SPDK_CU_ASSERT_FATAL(desc != NULL); 502 CU_ASSERT(remove_notify == false); 503 504 /* There is an open descriptor on the device. Unregister it, 505 * which can't proceed until the descriptor is closed. */ 506 done = false; 507 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 508 /* No polling has occurred, so neither of these should execute */ 509 CU_ASSERT(remove_notify == false); 510 CU_ASSERT(done == false); 511 512 /* Prior to the unregister completing, close the descriptor */ 513 spdk_bdev_close(desc); 514 515 /* Poll the threads to allow all events to be processed */ 516 poll_threads(); 517 518 /* Remove notify should not have been called because the 519 * descriptor is already closed. */ 520 CU_ASSERT(remove_notify == false); 521 522 /* The unregister should have completed */ 523 CU_ASSERT(done == true); 524 525 /* Restore the original g_bdev so that we can use teardown_test(). */ 526 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 527 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 528 teardown_test(); 529 } 530 531 static void 532 unregister_and_close_different_threads(void) 533 { 534 bool done; 535 struct spdk_bdev_desc *desc = NULL; 536 537 setup_test(); 538 set_thread(0); 539 540 /* setup_test() automatically opens the bdev, 541 * but this test needs to do that in a different 542 * way. */ 543 spdk_bdev_close(g_desc); 544 poll_threads(); 545 546 set_thread(1); 547 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &desc); 548 SPDK_CU_ASSERT_FATAL(desc != NULL); 549 done = false; 550 551 set_thread(0); 552 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 553 554 /* Poll the threads to allow all events to be processed */ 555 poll_threads(); 556 557 /* Make sure the bdev was not unregistered. We still have a 558 * descriptor open */ 559 CU_ASSERT(done == false); 560 561 /* Close the descriptor on thread 1. Poll the thread and confirm the 562 * unregister did not complete, since it was unregistered on thread 0. 563 */ 564 set_thread(1); 565 spdk_bdev_close(desc); 566 poll_thread(1); 567 CU_ASSERT(done == false); 568 569 /* Now poll thread 0 and confirm the unregister completed. */ 570 set_thread(0); 571 poll_thread(0); 572 CU_ASSERT(done == true); 573 574 /* Restore the original g_bdev so that we can use teardown_test(). */ 575 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 576 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 577 teardown_test(); 578 } 579 580 static void 581 reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 582 { 583 bool *done = cb_arg; 584 585 CU_ASSERT(success == true); 586 *done = true; 587 spdk_bdev_free_io(bdev_io); 588 } 589 590 static void 591 put_channel_during_reset(void) 592 { 593 struct spdk_io_channel *io_ch; 594 bool done = false; 595 596 setup_test(); 597 598 set_thread(0); 599 io_ch = spdk_bdev_get_io_channel(g_desc); 600 CU_ASSERT(io_ch != NULL); 601 602 /* 603 * Start a reset, but then put the I/O channel before 604 * the deferred messages for the reset get a chance to 605 * execute. 606 */ 607 spdk_bdev_reset(g_desc, io_ch, reset_done, &done); 608 spdk_put_io_channel(io_ch); 609 poll_threads(); 610 stub_complete_io(g_bdev.io_target, 0); 611 612 teardown_test(); 613 } 614 615 static void 616 aborted_reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 617 { 618 enum spdk_bdev_io_status *status = cb_arg; 619 620 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 621 spdk_bdev_free_io(bdev_io); 622 } 623 624 static void io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); 625 626 static void 627 aborted_reset(void) 628 { 629 struct spdk_io_channel *io_ch[2]; 630 enum spdk_bdev_io_status status1 = SPDK_BDEV_IO_STATUS_PENDING, 631 status2 = SPDK_BDEV_IO_STATUS_PENDING; 632 633 setup_test(); 634 635 set_thread(0); 636 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 637 CU_ASSERT(io_ch[0] != NULL); 638 spdk_bdev_reset(g_desc, io_ch[0], aborted_reset_done, &status1); 639 poll_threads(); 640 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 641 642 /* 643 * First reset has been submitted on ch0. Now submit a second 644 * reset on ch1 which will get queued since there is already a 645 * reset in progress. 646 */ 647 set_thread(1); 648 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 649 CU_ASSERT(io_ch[1] != NULL); 650 spdk_bdev_reset(g_desc, io_ch[1], aborted_reset_done, &status2); 651 poll_threads(); 652 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 653 654 /* 655 * Now destroy ch1. This will abort the queued reset. Check that 656 * the second reset was completed with failed status. Also check 657 * that bdev->internal.reset_in_progress != NULL, since the 658 * original reset has not been completed yet. This ensures that 659 * the bdev code is correctly noticing that the failed reset is 660 * *not* the one that had been submitted to the bdev module. 661 */ 662 set_thread(1); 663 spdk_put_io_channel(io_ch[1]); 664 poll_threads(); 665 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_FAILED); 666 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 667 668 /* 669 * Now complete the first reset, verify that it completed with SUCCESS 670 * status and that bdev->internal.reset_in_progress is also set back to NULL. 671 */ 672 set_thread(0); 673 spdk_put_io_channel(io_ch[0]); 674 stub_complete_io(g_bdev.io_target, 0); 675 poll_threads(); 676 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 677 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 678 679 teardown_test(); 680 } 681 682 static void 683 aborted_reset_no_outstanding_io(void) 684 { 685 struct spdk_io_channel *io_ch[2]; 686 struct spdk_bdev_channel *bdev_ch[2]; 687 struct spdk_bdev *bdev[2]; 688 enum spdk_bdev_io_status status1 = SPDK_BDEV_IO_STATUS_PENDING, 689 status2 = SPDK_BDEV_IO_STATUS_PENDING; 690 691 setup_test(); 692 693 /* 694 * This time we test the reset without any outstanding IO 695 * present on the bdev channel, so both resets should finish 696 * immediately. 697 */ 698 699 set_thread(0); 700 /* Set reset_io_drain_timeout to allow bdev 701 * reset to stay pending until we call abort. */ 702 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 703 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 704 bdev[0] = bdev_ch[0]->bdev; 705 bdev[0]->reset_io_drain_timeout = SPDK_BDEV_RESET_IO_DRAIN_RECOMMENDED_VALUE; 706 CU_ASSERT(io_ch[0] != NULL); 707 spdk_bdev_reset(g_desc, io_ch[0], aborted_reset_done, &status1); 708 poll_threads(); 709 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 710 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 711 spdk_put_io_channel(io_ch[0]); 712 713 set_thread(1); 714 /* Set reset_io_drain_timeout to allow bdev 715 * reset to stay pending until we call abort. */ 716 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 717 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 718 bdev[1] = bdev_ch[1]->bdev; 719 bdev[1]->reset_io_drain_timeout = SPDK_BDEV_RESET_IO_DRAIN_RECOMMENDED_VALUE; 720 CU_ASSERT(io_ch[1] != NULL); 721 spdk_bdev_reset(g_desc, io_ch[1], aborted_reset_done, &status2); 722 poll_threads(); 723 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 724 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_SUCCESS); 725 spdk_put_io_channel(io_ch[1]); 726 727 stub_complete_io(g_bdev.io_target, 0); 728 poll_threads(); 729 730 teardown_test(); 731 } 732 733 734 static void 735 io_during_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 736 { 737 enum spdk_bdev_io_status *status = cb_arg; 738 739 *status = bdev_io->internal.status; 740 spdk_bdev_free_io(bdev_io); 741 } 742 743 static void 744 io_during_reset(void) 745 { 746 struct spdk_io_channel *io_ch[2]; 747 struct spdk_bdev_channel *bdev_ch[2]; 748 enum spdk_bdev_io_status status0, status1, status_reset; 749 int rc; 750 751 setup_test(); 752 753 /* 754 * First test normal case - submit an I/O on each of two channels (with no resets) 755 * and verify they complete successfully. 756 */ 757 set_thread(0); 758 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 759 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 760 CU_ASSERT(bdev_ch[0]->flags == 0); 761 status0 = SPDK_BDEV_IO_STATUS_PENDING; 762 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 763 CU_ASSERT(rc == 0); 764 765 set_thread(1); 766 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 767 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 768 CU_ASSERT(bdev_ch[1]->flags == 0); 769 status1 = SPDK_BDEV_IO_STATUS_PENDING; 770 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 771 CU_ASSERT(rc == 0); 772 773 poll_threads(); 774 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 775 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 776 777 set_thread(0); 778 stub_complete_io(g_bdev.io_target, 0); 779 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 780 781 set_thread(1); 782 stub_complete_io(g_bdev.io_target, 0); 783 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 784 785 /* 786 * Now submit a reset, and leave it pending while we submit I/O on two different 787 * channels. These I/O should be failed by the bdev layer since the reset is in 788 * progress. 789 */ 790 set_thread(0); 791 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 792 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &status_reset); 793 CU_ASSERT(rc == 0); 794 795 CU_ASSERT(bdev_ch[0]->flags == 0); 796 CU_ASSERT(bdev_ch[1]->flags == 0); 797 poll_threads(); 798 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_RESET_IN_PROGRESS); 799 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_RESET_IN_PROGRESS); 800 801 set_thread(0); 802 status0 = SPDK_BDEV_IO_STATUS_PENDING; 803 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 804 CU_ASSERT(rc == 0); 805 806 set_thread(1); 807 status1 = SPDK_BDEV_IO_STATUS_PENDING; 808 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 809 CU_ASSERT(rc == 0); 810 811 /* 812 * A reset is in progress so these read I/O should complete with aborted. Note that we 813 * need to poll_threads() since I/O completed inline have their completion deferred. 814 */ 815 poll_threads(); 816 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 817 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_ABORTED); 818 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_ABORTED); 819 820 /* 821 * Complete the reset 822 */ 823 set_thread(0); 824 stub_complete_io(g_bdev.io_target, 0); 825 826 /* 827 * Only poll thread 0. We should not get a completion. 828 */ 829 poll_thread(0); 830 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 831 832 /* 833 * Poll both thread 0 and 1 so the messages can propagate and we 834 * get a completion. 835 */ 836 poll_threads(); 837 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 838 839 spdk_put_io_channel(io_ch[0]); 840 set_thread(1); 841 spdk_put_io_channel(io_ch[1]); 842 poll_threads(); 843 844 teardown_test(); 845 } 846 847 static uint32_t 848 count_queued_resets(void *io_target) 849 { 850 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 851 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 852 struct ut_bdev_io *bio; 853 struct spdk_bdev_io *io; 854 uint32_t submitted_resets = 0; 855 856 TAILQ_FOREACH(bio, &ch->outstanding_io, link) { 857 io = spdk_bdev_io_from_ctx(bio); 858 if (io->type == SPDK_BDEV_IO_TYPE_RESET) { 859 submitted_resets++; 860 } 861 } 862 863 spdk_put_io_channel(_ch); 864 865 return submitted_resets; 866 } 867 868 static void 869 reset_completions(void) 870 { 871 struct spdk_io_channel *io_ch; 872 struct spdk_bdev_channel *bdev_ch; 873 struct spdk_bdev *bdev; 874 enum spdk_bdev_io_status status0, status_reset; 875 int rc, iter; 876 877 setup_test(); 878 879 /* This test covers four test cases: 880 * 1) reset_io_drain_timeout of a bdev is greater than 0 881 * 2) No outstandind IO are present on any bdev channel 882 * 3) Outstanding IO finish during bdev reset 883 * 4) Outstanding IO do not finish before reset is done waiting 884 * for them. 885 * 886 * Above conditions mainly affect the timing of bdev reset completion 887 * and whether a reset should be skipped via spdk_bdev_io_complete() 888 * or sent down to the underlying bdev module via bdev_io_submit_reset(). */ 889 890 /* Test preparation */ 891 set_thread(0); 892 io_ch = spdk_bdev_get_io_channel(g_desc); 893 bdev_ch = spdk_io_channel_get_ctx(io_ch); 894 CU_ASSERT(bdev_ch->flags == 0); 895 896 897 /* Test case 1) reset_io_drain_timeout set to 0. Reset should be sent down immediately. */ 898 bdev = &g_bdev.bdev; 899 bdev->reset_io_drain_timeout = 0; 900 901 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 902 rc = spdk_bdev_reset(g_desc, io_ch, io_during_io_done, &status_reset); 903 CU_ASSERT(rc == 0); 904 poll_threads(); 905 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 1); 906 907 /* Call reset completion inside bdev module. */ 908 stub_complete_io(g_bdev.io_target, 0); 909 poll_threads(); 910 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 911 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 912 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 913 914 915 /* Test case 2) no outstanding IO are present. Reset should perform one iteration over 916 * channels and then be skipped. */ 917 bdev->reset_io_drain_timeout = SPDK_BDEV_RESET_IO_DRAIN_RECOMMENDED_VALUE; 918 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 919 920 rc = spdk_bdev_reset(g_desc, io_ch, io_during_io_done, &status_reset); 921 CU_ASSERT(rc == 0); 922 poll_threads(); 923 /* Reset was never submitted to the bdev module. */ 924 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 925 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 926 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 927 928 929 /* Test case 3) outstanding IO finish during bdev reset procedure. Reset should initiate 930 * wait poller to check for IO completions every second, until reset_io_drain_timeout is 931 * reached, but finish earlier than this threshold. */ 932 status0 = SPDK_BDEV_IO_STATUS_PENDING; 933 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 934 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, io_during_io_done, &status0); 935 CU_ASSERT(rc == 0); 936 937 rc = spdk_bdev_reset(g_desc, io_ch, io_during_io_done, &status_reset); 938 CU_ASSERT(rc == 0); 939 poll_threads(); 940 /* The reset just started and should not have been submitted yet. */ 941 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 942 943 poll_threads(); 944 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 945 /* Let the poller wait for about half the time then complete outstanding IO. */ 946 for (iter = 0; iter < 2; iter++) { 947 /* Reset is still processing and not submitted at this point. */ 948 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 949 spdk_delay_us(1000 * 1000); 950 poll_threads(); 951 poll_threads(); 952 } 953 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 954 stub_complete_io(g_bdev.io_target, 0); 955 poll_threads(); 956 spdk_delay_us(BDEV_RESET_CHECK_OUTSTANDING_IO_PERIOD); 957 poll_threads(); 958 poll_threads(); 959 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 960 /* Sending reset to the bdev module has been skipped. */ 961 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 962 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 963 964 965 /* Test case 4) outstanding IO are still present after reset_io_drain_timeout 966 * seconds have passed. */ 967 status0 = SPDK_BDEV_IO_STATUS_PENDING; 968 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 969 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, io_during_io_done, &status0); 970 CU_ASSERT(rc == 0); 971 972 rc = spdk_bdev_reset(g_desc, io_ch, io_during_io_done, &status_reset); 973 CU_ASSERT(rc == 0); 974 poll_threads(); 975 /* The reset just started and should not have been submitted yet. */ 976 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 977 978 poll_threads(); 979 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 980 /* Let the poller wait for reset_io_drain_timeout seconds. */ 981 for (iter = 0; iter < bdev->reset_io_drain_timeout; iter++) { 982 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 983 spdk_delay_us(BDEV_RESET_CHECK_OUTSTANDING_IO_PERIOD); 984 poll_threads(); 985 poll_threads(); 986 } 987 988 /* After timing out, the reset should have been sent to the module. */ 989 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 1); 990 /* Complete reset submitted to the module and the read IO. */ 991 stub_complete_io(g_bdev.io_target, 0); 992 poll_threads(); 993 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 994 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 995 996 997 /* Destroy the channel and end the test. */ 998 spdk_put_io_channel(io_ch); 999 poll_threads(); 1000 1001 teardown_test(); 1002 } 1003 1004 1005 static void 1006 basic_qos(void) 1007 { 1008 struct spdk_io_channel *io_ch[2]; 1009 struct spdk_bdev_channel *bdev_ch[2]; 1010 struct spdk_bdev *bdev; 1011 enum spdk_bdev_io_status status, abort_status; 1012 int rc; 1013 1014 setup_test(); 1015 1016 /* Enable QoS */ 1017 bdev = &g_bdev.bdev; 1018 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 1019 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 1020 /* 1021 * Enable read/write IOPS, read only byte per second and 1022 * read/write byte per second rate limits. 1023 * In this case, all rate limits will take equal effect. 1024 */ 1025 /* 2000 read/write I/O per second, or 2 per millisecond */ 1026 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 1027 /* 8K read/write byte per millisecond with 4K block size */ 1028 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 1029 /* 8K read only byte per millisecond with 4K block size */ 1030 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 8192000; 1031 1032 g_get_io_channel = true; 1033 1034 set_thread(0); 1035 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1036 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1037 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 1038 1039 set_thread(1); 1040 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1041 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1042 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 1043 1044 /* 1045 * Send an I/O on thread 0, which is where the QoS thread is running. 1046 */ 1047 set_thread(0); 1048 status = SPDK_BDEV_IO_STATUS_PENDING; 1049 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status); 1050 CU_ASSERT(rc == 0); 1051 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 1052 poll_threads(); 1053 stub_complete_io(g_bdev.io_target, 0); 1054 poll_threads(); 1055 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 1056 1057 /* Send an I/O on thread 1. The QoS thread is not running here. */ 1058 status = SPDK_BDEV_IO_STATUS_PENDING; 1059 set_thread(1); 1060 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status); 1061 CU_ASSERT(rc == 0); 1062 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 1063 poll_threads(); 1064 /* Complete I/O on thread 0. This should not complete the I/O we submitted. */ 1065 set_thread(0); 1066 stub_complete_io(g_bdev.io_target, 0); 1067 poll_threads(); 1068 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 1069 /* Now complete I/O on original thread 1. */ 1070 set_thread(1); 1071 poll_threads(); 1072 stub_complete_io(g_bdev.io_target, 0); 1073 poll_threads(); 1074 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 1075 1076 /* Reset rate limit for the next test cases. */ 1077 spdk_delay_us(SPDK_BDEV_QOS_TIMESLICE_IN_USEC); 1078 poll_threads(); 1079 1080 /* 1081 * Test abort request when QoS is enabled. 1082 */ 1083 1084 /* Send an I/O on thread 0, which is where the QoS thread is running. */ 1085 set_thread(0); 1086 status = SPDK_BDEV_IO_STATUS_PENDING; 1087 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status); 1088 CU_ASSERT(rc == 0); 1089 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 1090 /* Send an abort to the I/O on the same thread. */ 1091 abort_status = SPDK_BDEV_IO_STATUS_PENDING; 1092 rc = spdk_bdev_abort(g_desc, io_ch[0], &status, io_during_io_done, &abort_status); 1093 CU_ASSERT(rc == 0); 1094 CU_ASSERT(abort_status == SPDK_BDEV_IO_STATUS_PENDING); 1095 poll_threads(); 1096 CU_ASSERT(abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 1097 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_ABORTED); 1098 1099 /* Send an I/O on thread 1. The QoS thread is not running here. */ 1100 status = SPDK_BDEV_IO_STATUS_PENDING; 1101 set_thread(1); 1102 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status); 1103 CU_ASSERT(rc == 0); 1104 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 1105 poll_threads(); 1106 /* Send an abort to the I/O on the same thread. */ 1107 abort_status = SPDK_BDEV_IO_STATUS_PENDING; 1108 rc = spdk_bdev_abort(g_desc, io_ch[1], &status, io_during_io_done, &abort_status); 1109 CU_ASSERT(rc == 0); 1110 CU_ASSERT(abort_status == SPDK_BDEV_IO_STATUS_PENDING); 1111 poll_threads(); 1112 /* Complete the I/O with failure and the abort with success on thread 1. */ 1113 CU_ASSERT(abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 1114 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_ABORTED); 1115 1116 set_thread(0); 1117 1118 /* 1119 * Close the descriptor only, which should stop the qos channel as 1120 * the last descriptor removed. 1121 */ 1122 spdk_bdev_close(g_desc); 1123 poll_threads(); 1124 CU_ASSERT(bdev->internal.qos->ch == NULL); 1125 1126 /* 1127 * Open the bdev again which shall setup the qos channel as the 1128 * channels are valid. 1129 */ 1130 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 1131 poll_threads(); 1132 CU_ASSERT(bdev->internal.qos->ch != NULL); 1133 1134 /* Tear down the channels */ 1135 set_thread(0); 1136 spdk_put_io_channel(io_ch[0]); 1137 set_thread(1); 1138 spdk_put_io_channel(io_ch[1]); 1139 poll_threads(); 1140 set_thread(0); 1141 1142 /* Close the descriptor, which should stop the qos channel */ 1143 spdk_bdev_close(g_desc); 1144 poll_threads(); 1145 CU_ASSERT(bdev->internal.qos->ch == NULL); 1146 1147 /* Open the bdev again, no qos channel setup without valid channels. */ 1148 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 1149 poll_threads(); 1150 CU_ASSERT(bdev->internal.qos->ch == NULL); 1151 1152 /* Create the channels in reverse order. */ 1153 set_thread(1); 1154 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1155 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1156 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 1157 1158 set_thread(0); 1159 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1160 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1161 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 1162 1163 /* Confirm that the qos thread is now thread 1 */ 1164 CU_ASSERT(bdev->internal.qos->ch == bdev_ch[1]); 1165 1166 /* Tear down the channels */ 1167 set_thread(0); 1168 spdk_put_io_channel(io_ch[0]); 1169 set_thread(1); 1170 spdk_put_io_channel(io_ch[1]); 1171 poll_threads(); 1172 1173 set_thread(0); 1174 1175 teardown_test(); 1176 } 1177 1178 static void 1179 io_during_qos_queue(void) 1180 { 1181 struct spdk_io_channel *io_ch[2]; 1182 struct spdk_bdev_channel *bdev_ch[2]; 1183 struct spdk_bdev *bdev; 1184 enum spdk_bdev_io_status status0, status1, status2; 1185 int rc; 1186 1187 setup_test(); 1188 MOCK_SET(spdk_get_ticks, 0); 1189 1190 /* Enable QoS */ 1191 bdev = &g_bdev.bdev; 1192 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 1193 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 1194 1195 /* 1196 * Enable read/write IOPS, read only byte per sec, write only 1197 * byte per sec and read/write byte per sec rate limits. 1198 * In this case, both read only and write only byte per sec 1199 * rate limit will take effect. 1200 */ 1201 /* 4000 read/write I/O per second, or 4 per millisecond */ 1202 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 4000; 1203 /* 8K byte per millisecond with 4K block size */ 1204 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 1205 /* 4K byte per millisecond with 4K block size */ 1206 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 4096000; 1207 /* 4K byte per millisecond with 4K block size */ 1208 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 4096000; 1209 1210 g_get_io_channel = true; 1211 1212 /* Create channels */ 1213 set_thread(0); 1214 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1215 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1216 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 1217 1218 set_thread(1); 1219 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1220 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1221 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 1222 1223 /* Send two read I/Os */ 1224 status1 = SPDK_BDEV_IO_STATUS_PENDING; 1225 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 1226 CU_ASSERT(rc == 0); 1227 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 1228 set_thread(0); 1229 status0 = SPDK_BDEV_IO_STATUS_PENDING; 1230 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 1231 CU_ASSERT(rc == 0); 1232 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 1233 /* Send one write I/O */ 1234 status2 = SPDK_BDEV_IO_STATUS_PENDING; 1235 rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status2); 1236 CU_ASSERT(rc == 0); 1237 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_PENDING); 1238 1239 /* Complete any I/O that arrived at the disk */ 1240 poll_threads(); 1241 set_thread(1); 1242 stub_complete_io(g_bdev.io_target, 0); 1243 set_thread(0); 1244 stub_complete_io(g_bdev.io_target, 0); 1245 poll_threads(); 1246 1247 /* Only one of the two read I/Os should complete. (logical XOR) */ 1248 if (status0 == SPDK_BDEV_IO_STATUS_SUCCESS) { 1249 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 1250 } else { 1251 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 1252 } 1253 /* The write I/O should complete. */ 1254 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_SUCCESS); 1255 1256 /* Advance in time by a millisecond */ 1257 spdk_delay_us(1000); 1258 1259 /* Complete more I/O */ 1260 poll_threads(); 1261 set_thread(1); 1262 stub_complete_io(g_bdev.io_target, 0); 1263 set_thread(0); 1264 stub_complete_io(g_bdev.io_target, 0); 1265 poll_threads(); 1266 1267 /* Now the second read I/O should be done */ 1268 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 1269 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 1270 1271 /* Tear down the channels */ 1272 set_thread(1); 1273 spdk_put_io_channel(io_ch[1]); 1274 set_thread(0); 1275 spdk_put_io_channel(io_ch[0]); 1276 poll_threads(); 1277 1278 teardown_test(); 1279 } 1280 1281 static void 1282 io_during_qos_reset(void) 1283 { 1284 struct spdk_io_channel *io_ch[2]; 1285 struct spdk_bdev_channel *bdev_ch[2]; 1286 struct spdk_bdev *bdev; 1287 enum spdk_bdev_io_status status0, status1, reset_status; 1288 int rc; 1289 1290 setup_test(); 1291 MOCK_SET(spdk_get_ticks, 0); 1292 1293 /* Enable QoS */ 1294 bdev = &g_bdev.bdev; 1295 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 1296 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 1297 1298 /* 1299 * Enable read/write IOPS, write only byte per sec and 1300 * read/write byte per second rate limits. 1301 * In this case, read/write byte per second rate limit will 1302 * take effect first. 1303 */ 1304 /* 2000 read/write I/O per second, or 2 per millisecond */ 1305 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 1306 /* 4K byte per millisecond with 4K block size */ 1307 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 4096000; 1308 /* 8K byte per millisecond with 4K block size */ 1309 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 8192000; 1310 1311 g_get_io_channel = true; 1312 1313 /* Create channels */ 1314 set_thread(0); 1315 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1316 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1317 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 1318 1319 set_thread(1); 1320 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1321 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1322 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 1323 1324 /* Send two I/O. One of these gets queued by QoS. The other is sitting at the disk. */ 1325 status1 = SPDK_BDEV_IO_STATUS_PENDING; 1326 rc = spdk_bdev_write_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 1327 CU_ASSERT(rc == 0); 1328 set_thread(0); 1329 status0 = SPDK_BDEV_IO_STATUS_PENDING; 1330 rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 1331 CU_ASSERT(rc == 0); 1332 1333 poll_threads(); 1334 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 1335 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 1336 1337 /* Reset the bdev. */ 1338 reset_status = SPDK_BDEV_IO_STATUS_PENDING; 1339 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &reset_status); 1340 CU_ASSERT(rc == 0); 1341 1342 /* Complete any I/O that arrived at the disk */ 1343 poll_threads(); 1344 set_thread(1); 1345 stub_complete_io(g_bdev.io_target, 0); 1346 set_thread(0); 1347 stub_complete_io(g_bdev.io_target, 0); 1348 poll_threads(); 1349 1350 CU_ASSERT(reset_status == SPDK_BDEV_IO_STATUS_SUCCESS); 1351 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_ABORTED); 1352 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_ABORTED); 1353 1354 /* Tear down the channels */ 1355 set_thread(1); 1356 spdk_put_io_channel(io_ch[1]); 1357 set_thread(0); 1358 spdk_put_io_channel(io_ch[0]); 1359 poll_threads(); 1360 1361 teardown_test(); 1362 } 1363 1364 static void 1365 enomem_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1366 { 1367 enum spdk_bdev_io_status *status = cb_arg; 1368 1369 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 1370 spdk_bdev_free_io(bdev_io); 1371 } 1372 1373 static void 1374 enomem(void) 1375 { 1376 struct spdk_io_channel *io_ch; 1377 struct spdk_bdev_channel *bdev_ch; 1378 struct spdk_bdev_shared_resource *shared_resource; 1379 struct ut_bdev_channel *ut_ch; 1380 const uint32_t IO_ARRAY_SIZE = 64; 1381 const uint32_t AVAIL = 20; 1382 enum spdk_bdev_io_status status[IO_ARRAY_SIZE], status_reset; 1383 uint32_t nomem_cnt, i; 1384 struct spdk_bdev_io *first_io; 1385 int rc; 1386 1387 setup_test(); 1388 1389 set_thread(0); 1390 io_ch = spdk_bdev_get_io_channel(g_desc); 1391 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1392 shared_resource = bdev_ch->shared_resource; 1393 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1394 ut_ch->avail_cnt = AVAIL; 1395 1396 /* First submit a number of IOs equal to what the channel can support. */ 1397 for (i = 0; i < AVAIL; i++) { 1398 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1399 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1400 CU_ASSERT(rc == 0); 1401 } 1402 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1403 1404 /* 1405 * Next, submit one additional I/O. This one should fail with ENOMEM and then go onto 1406 * the enomem_io list. 1407 */ 1408 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1409 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1410 CU_ASSERT(rc == 0); 1411 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1412 first_io = TAILQ_FIRST(&shared_resource->nomem_io); 1413 1414 /* 1415 * Now submit a bunch more I/O. These should all fail with ENOMEM and get queued behind 1416 * the first_io above. 1417 */ 1418 for (i = AVAIL + 1; i < IO_ARRAY_SIZE; i++) { 1419 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1420 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1421 CU_ASSERT(rc == 0); 1422 } 1423 1424 /* Assert that first_io is still at the head of the list. */ 1425 CU_ASSERT(TAILQ_FIRST(&shared_resource->nomem_io) == first_io); 1426 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == (IO_ARRAY_SIZE - AVAIL)); 1427 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 1428 CU_ASSERT(shared_resource->nomem_threshold == (AVAIL - NOMEM_THRESHOLD_COUNT)); 1429 1430 /* 1431 * Complete 1 I/O only. The key check here is bdev_io_tailq_cnt - this should not have 1432 * changed since completing just 1 I/O should not trigger retrying the queued nomem_io 1433 * list. 1434 */ 1435 stub_complete_io(g_bdev.io_target, 1); 1436 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 1437 1438 /* 1439 * Complete enough I/O to hit the nomem_threshold. This should trigger retrying nomem_io, 1440 * and we should see I/O get resubmitted to the test bdev module. 1441 */ 1442 stub_complete_io(g_bdev.io_target, NOMEM_THRESHOLD_COUNT - 1); 1443 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) < nomem_cnt); 1444 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 1445 1446 /* Complete 1 I/O only. This should not trigger retrying the queued nomem_io. */ 1447 stub_complete_io(g_bdev.io_target, 1); 1448 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 1449 1450 /* 1451 * Send a reset and confirm that all I/O are completed, including the ones that 1452 * were queued on the nomem_io list. 1453 */ 1454 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 1455 rc = spdk_bdev_reset(g_desc, io_ch, enomem_done, &status_reset); 1456 poll_threads(); 1457 CU_ASSERT(rc == 0); 1458 /* This will complete the reset. */ 1459 stub_complete_io(g_bdev.io_target, 0); 1460 1461 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == 0); 1462 CU_ASSERT(shared_resource->io_outstanding == 0); 1463 1464 spdk_put_io_channel(io_ch); 1465 poll_threads(); 1466 teardown_test(); 1467 } 1468 1469 static void 1470 enomem_multi_bdev(void) 1471 { 1472 struct spdk_io_channel *io_ch; 1473 struct spdk_bdev_channel *bdev_ch; 1474 struct spdk_bdev_shared_resource *shared_resource; 1475 struct ut_bdev_channel *ut_ch; 1476 const uint32_t IO_ARRAY_SIZE = 64; 1477 const uint32_t AVAIL = 20; 1478 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1479 uint32_t i; 1480 struct ut_bdev *second_bdev; 1481 struct spdk_bdev_desc *second_desc = NULL; 1482 struct spdk_bdev_channel *second_bdev_ch; 1483 struct spdk_io_channel *second_ch; 1484 int rc; 1485 1486 setup_test(); 1487 1488 /* Register second bdev with the same io_target */ 1489 second_bdev = calloc(1, sizeof(*second_bdev)); 1490 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1491 register_bdev(second_bdev, "ut_bdev2", g_bdev.io_target); 1492 spdk_bdev_open_ext("ut_bdev2", true, _bdev_event_cb, NULL, &second_desc); 1493 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1494 1495 set_thread(0); 1496 io_ch = spdk_bdev_get_io_channel(g_desc); 1497 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1498 shared_resource = bdev_ch->shared_resource; 1499 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1500 ut_ch->avail_cnt = AVAIL; 1501 1502 second_ch = spdk_bdev_get_io_channel(second_desc); 1503 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1504 SPDK_CU_ASSERT_FATAL(shared_resource == second_bdev_ch->shared_resource); 1505 1506 /* Saturate io_target through bdev A. */ 1507 for (i = 0; i < AVAIL; i++) { 1508 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1509 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1510 CU_ASSERT(rc == 0); 1511 } 1512 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1513 1514 /* 1515 * Now submit I/O through the second bdev. This should fail with ENOMEM 1516 * and then go onto the nomem_io list. 1517 */ 1518 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1519 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1520 CU_ASSERT(rc == 0); 1521 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1522 1523 /* Complete first bdev's I/O. This should retry sending second bdev's nomem_io */ 1524 stub_complete_io(g_bdev.io_target, AVAIL); 1525 1526 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&shared_resource->nomem_io)); 1527 CU_ASSERT(shared_resource->io_outstanding == 1); 1528 1529 /* Now complete our retried I/O */ 1530 stub_complete_io(g_bdev.io_target, 1); 1531 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == 0); 1532 1533 spdk_put_io_channel(io_ch); 1534 spdk_put_io_channel(second_ch); 1535 spdk_bdev_close(second_desc); 1536 unregister_bdev(second_bdev); 1537 poll_threads(); 1538 free(second_bdev); 1539 teardown_test(); 1540 } 1541 1542 static void 1543 enomem_multi_bdev_unregister(void) 1544 { 1545 struct spdk_io_channel *io_ch; 1546 struct spdk_bdev_channel *bdev_ch; 1547 struct spdk_bdev_shared_resource *shared_resource; 1548 struct ut_bdev_channel *ut_ch; 1549 const uint32_t IO_ARRAY_SIZE = 64; 1550 const uint32_t AVAIL = 20; 1551 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1552 uint32_t i; 1553 int rc; 1554 1555 setup_test(); 1556 1557 set_thread(0); 1558 io_ch = spdk_bdev_get_io_channel(g_desc); 1559 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1560 shared_resource = bdev_ch->shared_resource; 1561 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1562 ut_ch->avail_cnt = AVAIL; 1563 1564 /* Saturate io_target through the bdev. */ 1565 for (i = 0; i < AVAIL; i++) { 1566 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1567 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1568 CU_ASSERT(rc == 0); 1569 } 1570 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1571 1572 /* 1573 * Now submit I/O through the bdev. This should fail with ENOMEM 1574 * and then go onto the nomem_io list. 1575 */ 1576 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1577 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1578 CU_ASSERT(rc == 0); 1579 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1580 1581 /* Unregister the bdev to abort the IOs from nomem_io queue. */ 1582 unregister_bdev(&g_bdev); 1583 CU_ASSERT(status[AVAIL] == SPDK_BDEV_IO_STATUS_FAILED); 1584 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&shared_resource->nomem_io)); 1585 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == AVAIL); 1586 1587 /* Complete the bdev's I/O. */ 1588 stub_complete_io(g_bdev.io_target, AVAIL); 1589 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == 0); 1590 1591 spdk_put_io_channel(io_ch); 1592 poll_threads(); 1593 teardown_test(); 1594 } 1595 1596 static void 1597 enomem_multi_io_target(void) 1598 { 1599 struct spdk_io_channel *io_ch; 1600 struct spdk_bdev_channel *bdev_ch; 1601 struct ut_bdev_channel *ut_ch; 1602 const uint32_t IO_ARRAY_SIZE = 64; 1603 const uint32_t AVAIL = 20; 1604 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1605 uint32_t i; 1606 int new_io_device; 1607 struct ut_bdev *second_bdev; 1608 struct spdk_bdev_desc *second_desc = NULL; 1609 struct spdk_bdev_channel *second_bdev_ch; 1610 struct spdk_io_channel *second_ch; 1611 int rc; 1612 1613 setup_test(); 1614 1615 /* Create new io_target and a second bdev using it */ 1616 spdk_io_device_register(&new_io_device, stub_create_ch, stub_destroy_ch, 1617 sizeof(struct ut_bdev_channel), NULL); 1618 second_bdev = calloc(1, sizeof(*second_bdev)); 1619 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1620 register_bdev(second_bdev, "ut_bdev2", &new_io_device); 1621 spdk_bdev_open_ext("ut_bdev2", true, _bdev_event_cb, NULL, &second_desc); 1622 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1623 1624 set_thread(0); 1625 io_ch = spdk_bdev_get_io_channel(g_desc); 1626 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1627 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1628 ut_ch->avail_cnt = AVAIL; 1629 1630 /* Different io_target should imply a different shared_resource */ 1631 second_ch = spdk_bdev_get_io_channel(second_desc); 1632 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1633 SPDK_CU_ASSERT_FATAL(bdev_ch->shared_resource != second_bdev_ch->shared_resource); 1634 1635 /* Saturate io_target through bdev A. */ 1636 for (i = 0; i < AVAIL; i++) { 1637 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1638 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1639 CU_ASSERT(rc == 0); 1640 } 1641 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1642 1643 /* Issue one more I/O to fill ENOMEM list. */ 1644 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1645 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1646 CU_ASSERT(rc == 0); 1647 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1648 1649 /* 1650 * Now submit I/O through the second bdev. This should go through and complete 1651 * successfully because we're using a different io_device underneath. 1652 */ 1653 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1654 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1655 CU_ASSERT(rc == 0); 1656 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&second_bdev_ch->shared_resource->nomem_io)); 1657 stub_complete_io(second_bdev->io_target, 1); 1658 1659 /* Cleanup; Complete outstanding I/O. */ 1660 stub_complete_io(g_bdev.io_target, AVAIL); 1661 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1662 /* Complete the ENOMEM I/O */ 1663 stub_complete_io(g_bdev.io_target, 1); 1664 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1665 1666 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1667 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1668 spdk_put_io_channel(io_ch); 1669 spdk_put_io_channel(second_ch); 1670 spdk_bdev_close(second_desc); 1671 unregister_bdev(second_bdev); 1672 spdk_io_device_unregister(&new_io_device, NULL); 1673 poll_threads(); 1674 free(second_bdev); 1675 teardown_test(); 1676 } 1677 1678 static void 1679 qos_dynamic_enable_done(void *cb_arg, int status) 1680 { 1681 int *rc = cb_arg; 1682 *rc = status; 1683 } 1684 1685 static void 1686 qos_dynamic_enable(void) 1687 { 1688 struct spdk_io_channel *io_ch[2]; 1689 struct spdk_bdev_channel *bdev_ch[2]; 1690 struct spdk_bdev *bdev; 1691 enum spdk_bdev_io_status bdev_io_status[2]; 1692 uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {}; 1693 int status, second_status, rc, i; 1694 1695 setup_test(); 1696 MOCK_SET(spdk_get_ticks, 0); 1697 1698 for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) { 1699 limits[i] = UINT64_MAX; 1700 } 1701 1702 bdev = &g_bdev.bdev; 1703 1704 g_get_io_channel = true; 1705 1706 /* Create channels */ 1707 set_thread(0); 1708 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1709 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1710 CU_ASSERT(bdev_ch[0]->flags == 0); 1711 1712 set_thread(1); 1713 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1714 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1715 CU_ASSERT(bdev_ch[1]->flags == 0); 1716 1717 set_thread(0); 1718 1719 /* 1720 * Enable QoS: Read/Write IOPS, Read/Write byte, 1721 * Read only byte and Write only byte per second 1722 * rate limits. 1723 * More than 10 I/Os allowed per timeslice. 1724 */ 1725 status = -1; 1726 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1727 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 100; 1728 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 100; 1729 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 10; 1730 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1731 poll_threads(); 1732 CU_ASSERT(status == 0); 1733 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1734 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1735 1736 /* 1737 * Submit and complete 10 I/O to fill the QoS allotment for this timeslice. 1738 * Additional I/O will then be queued. 1739 */ 1740 set_thread(0); 1741 for (i = 0; i < 10; i++) { 1742 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1743 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1744 CU_ASSERT(rc == 0); 1745 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1746 poll_thread(0); 1747 stub_complete_io(g_bdev.io_target, 0); 1748 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1749 } 1750 1751 /* 1752 * Send two more I/O. These I/O will be queued since the current timeslice allotment has been 1753 * filled already. We want to test that when QoS is disabled that these two I/O: 1754 * 1) are not aborted 1755 * 2) are sent back to their original thread for resubmission 1756 */ 1757 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1758 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1759 CU_ASSERT(rc == 0); 1760 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1761 set_thread(1); 1762 bdev_io_status[1] = SPDK_BDEV_IO_STATUS_PENDING; 1763 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &bdev_io_status[1]); 1764 CU_ASSERT(rc == 0); 1765 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1766 poll_threads(); 1767 1768 /* 1769 * Disable QoS: Read/Write IOPS, Read/Write byte, 1770 * Read only byte rate limits 1771 */ 1772 status = -1; 1773 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1774 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 0; 1775 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 0; 1776 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1777 poll_threads(); 1778 CU_ASSERT(status == 0); 1779 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1780 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1781 1782 /* Disable QoS: Write only Byte per second rate limit */ 1783 status = -1; 1784 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 0; 1785 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1786 poll_threads(); 1787 CU_ASSERT(status == 0); 1788 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1789 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1790 1791 /* 1792 * All I/O should have been resubmitted back on their original thread. Complete 1793 * all I/O on thread 0, and ensure that only the thread 0 I/O was completed. 1794 */ 1795 set_thread(0); 1796 stub_complete_io(g_bdev.io_target, 0); 1797 poll_threads(); 1798 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1799 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1800 1801 /* Now complete all I/O on thread 1 and ensure the thread 1 I/O was completed. */ 1802 set_thread(1); 1803 stub_complete_io(g_bdev.io_target, 0); 1804 poll_threads(); 1805 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_SUCCESS); 1806 1807 /* Disable QoS again */ 1808 status = -1; 1809 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1810 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1811 poll_threads(); 1812 CU_ASSERT(status == 0); /* This should succeed */ 1813 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1814 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1815 1816 /* Enable QoS on thread 0 */ 1817 status = -1; 1818 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1819 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1820 poll_threads(); 1821 CU_ASSERT(status == 0); 1822 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1823 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1824 1825 /* Disable QoS on thread 1 */ 1826 set_thread(1); 1827 status = -1; 1828 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1829 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1830 /* Don't poll yet. This should leave the channels with QoS enabled */ 1831 CU_ASSERT(status == -1); 1832 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1833 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1834 1835 /* Enable QoS. This should immediately fail because the previous disable QoS hasn't completed. */ 1836 second_status = 0; 1837 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 10; 1838 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &second_status); 1839 poll_threads(); 1840 CU_ASSERT(status == 0); /* The disable should succeed */ 1841 CU_ASSERT(second_status < 0); /* The enable should fail */ 1842 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1843 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1844 1845 /* Enable QoS on thread 1. This should succeed now that the disable has completed. */ 1846 status = -1; 1847 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1848 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1849 poll_threads(); 1850 CU_ASSERT(status == 0); 1851 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1852 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1853 1854 /* Tear down the channels */ 1855 set_thread(0); 1856 spdk_put_io_channel(io_ch[0]); 1857 set_thread(1); 1858 spdk_put_io_channel(io_ch[1]); 1859 poll_threads(); 1860 1861 set_thread(0); 1862 teardown_test(); 1863 } 1864 1865 static void 1866 histogram_status_cb(void *cb_arg, int status) 1867 { 1868 g_status = status; 1869 } 1870 1871 static void 1872 histogram_data_cb(void *cb_arg, int status, struct spdk_histogram_data *histogram) 1873 { 1874 g_status = status; 1875 g_histogram = histogram; 1876 } 1877 1878 static void 1879 histogram_io_count(void *ctx, uint64_t start, uint64_t end, uint64_t count, 1880 uint64_t total, uint64_t so_far) 1881 { 1882 g_count += count; 1883 } 1884 1885 static void 1886 bdev_histograms_mt(void) 1887 { 1888 struct spdk_io_channel *ch[2]; 1889 struct spdk_histogram_data *histogram; 1890 uint8_t buf[4096]; 1891 int status = false; 1892 int rc; 1893 1894 1895 setup_test(); 1896 1897 set_thread(0); 1898 ch[0] = spdk_bdev_get_io_channel(g_desc); 1899 CU_ASSERT(ch[0] != NULL); 1900 1901 set_thread(1); 1902 ch[1] = spdk_bdev_get_io_channel(g_desc); 1903 CU_ASSERT(ch[1] != NULL); 1904 1905 1906 /* Enable histogram */ 1907 spdk_bdev_histogram_enable(&g_bdev.bdev, histogram_status_cb, NULL, true); 1908 poll_threads(); 1909 CU_ASSERT(g_status == 0); 1910 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == true); 1911 1912 /* Allocate histogram */ 1913 histogram = spdk_histogram_data_alloc(); 1914 1915 /* Check if histogram is zeroed */ 1916 spdk_bdev_histogram_get(&g_bdev.bdev, histogram, histogram_data_cb, NULL); 1917 poll_threads(); 1918 CU_ASSERT(g_status == 0); 1919 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 1920 1921 g_count = 0; 1922 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 1923 1924 CU_ASSERT(g_count == 0); 1925 1926 set_thread(0); 1927 rc = spdk_bdev_write_blocks(g_desc, ch[0], &buf, 0, 1, io_during_io_done, &status); 1928 CU_ASSERT(rc == 0); 1929 1930 spdk_delay_us(10); 1931 stub_complete_io(g_bdev.io_target, 1); 1932 poll_threads(); 1933 CU_ASSERT(status == true); 1934 1935 1936 set_thread(1); 1937 rc = spdk_bdev_read_blocks(g_desc, ch[1], &buf, 0, 1, io_during_io_done, &status); 1938 CU_ASSERT(rc == 0); 1939 1940 spdk_delay_us(10); 1941 stub_complete_io(g_bdev.io_target, 1); 1942 poll_threads(); 1943 CU_ASSERT(status == true); 1944 1945 set_thread(0); 1946 1947 /* Check if histogram gathered data from all I/O channels */ 1948 spdk_bdev_histogram_get(&g_bdev.bdev, histogram, histogram_data_cb, NULL); 1949 poll_threads(); 1950 CU_ASSERT(g_status == 0); 1951 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == true); 1952 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 1953 1954 g_count = 0; 1955 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 1956 CU_ASSERT(g_count == 2); 1957 1958 /* Disable histogram */ 1959 spdk_bdev_histogram_enable(&g_bdev.bdev, histogram_status_cb, NULL, false); 1960 poll_threads(); 1961 CU_ASSERT(g_status == 0); 1962 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == false); 1963 1964 spdk_histogram_data_free(histogram); 1965 1966 /* Tear down the channels */ 1967 set_thread(0); 1968 spdk_put_io_channel(ch[0]); 1969 set_thread(1); 1970 spdk_put_io_channel(ch[1]); 1971 poll_threads(); 1972 set_thread(0); 1973 teardown_test(); 1974 1975 } 1976 1977 struct timeout_io_cb_arg { 1978 struct iovec iov; 1979 uint8_t type; 1980 }; 1981 1982 static int 1983 bdev_channel_count_submitted_io(struct spdk_bdev_channel *ch) 1984 { 1985 struct spdk_bdev_io *bdev_io; 1986 int n = 0; 1987 1988 if (!ch) { 1989 return -1; 1990 } 1991 1992 TAILQ_FOREACH(bdev_io, &ch->io_submitted, internal.ch_link) { 1993 n++; 1994 } 1995 1996 return n; 1997 } 1998 1999 static void 2000 bdev_channel_io_timeout_cb(void *cb_arg, struct spdk_bdev_io *bdev_io) 2001 { 2002 struct timeout_io_cb_arg *ctx = cb_arg; 2003 2004 ctx->type = bdev_io->type; 2005 ctx->iov.iov_base = bdev_io->iov.iov_base; 2006 ctx->iov.iov_len = bdev_io->iov.iov_len; 2007 } 2008 2009 static bool g_io_done; 2010 2011 static void 2012 io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 2013 { 2014 g_io_done = true; 2015 spdk_bdev_free_io(bdev_io); 2016 } 2017 2018 static void 2019 bdev_set_io_timeout_mt(void) 2020 { 2021 struct spdk_io_channel *ch[3]; 2022 struct spdk_bdev_channel *bdev_ch[3]; 2023 struct timeout_io_cb_arg cb_arg; 2024 2025 setup_test(); 2026 2027 g_bdev.bdev.optimal_io_boundary = 16; 2028 g_bdev.bdev.split_on_optimal_io_boundary = true; 2029 2030 set_thread(0); 2031 ch[0] = spdk_bdev_get_io_channel(g_desc); 2032 CU_ASSERT(ch[0] != NULL); 2033 2034 set_thread(1); 2035 ch[1] = spdk_bdev_get_io_channel(g_desc); 2036 CU_ASSERT(ch[1] != NULL); 2037 2038 set_thread(2); 2039 ch[2] = spdk_bdev_get_io_channel(g_desc); 2040 CU_ASSERT(ch[2] != NULL); 2041 2042 /* Multi-thread mode 2043 * 1, Check the poller was registered successfully 2044 * 2, Check the timeout IO and ensure the IO was the submitted by user 2045 * 3, Check the link int the bdev_ch works right. 2046 * 4, Close desc and put io channel during the timeout poller is polling 2047 */ 2048 2049 /* In desc thread set the timeout */ 2050 set_thread(0); 2051 CU_ASSERT(spdk_bdev_set_timeout(g_desc, 5, bdev_channel_io_timeout_cb, &cb_arg) == 0); 2052 CU_ASSERT(g_desc->io_timeout_poller != NULL); 2053 CU_ASSERT(g_desc->cb_fn == bdev_channel_io_timeout_cb); 2054 CU_ASSERT(g_desc->cb_arg == &cb_arg); 2055 2056 /* check the IO submitted list and timeout handler */ 2057 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[0], (void *)0x2000, 0, 1, io_done, NULL) == 0); 2058 bdev_ch[0] = spdk_io_channel_get_ctx(ch[0]); 2059 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[0]) == 1); 2060 2061 set_thread(1); 2062 CU_ASSERT(spdk_bdev_write_blocks(g_desc, ch[1], (void *)0x1000, 0, 1, io_done, NULL) == 0); 2063 bdev_ch[1] = spdk_io_channel_get_ctx(ch[1]); 2064 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[1]) == 1); 2065 2066 /* Now test that a single-vector command is split correctly. 2067 * Offset 14, length 8, payload 0xF000 2068 * Child - Offset 14, length 2, payload 0xF000 2069 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 2070 * 2071 * Set up the expected values before calling spdk_bdev_read_blocks 2072 */ 2073 set_thread(2); 2074 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[2], (void *)0xF000, 14, 8, io_done, NULL) == 0); 2075 bdev_ch[2] = spdk_io_channel_get_ctx(ch[2]); 2076 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[2]) == 3); 2077 2078 set_thread(0); 2079 memset(&cb_arg, 0, sizeof(cb_arg)); 2080 spdk_delay_us(3 * spdk_get_ticks_hz()); 2081 poll_threads(); 2082 CU_ASSERT(cb_arg.type == 0); 2083 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 2084 CU_ASSERT(cb_arg.iov.iov_len == 0); 2085 2086 /* Now the time reach the limit */ 2087 spdk_delay_us(3 * spdk_get_ticks_hz()); 2088 poll_thread(0); 2089 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_READ); 2090 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x2000); 2091 CU_ASSERT(cb_arg.iov.iov_len == 1 * g_bdev.bdev.blocklen); 2092 stub_complete_io(g_bdev.io_target, 1); 2093 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[0]) == 0); 2094 2095 memset(&cb_arg, 0, sizeof(cb_arg)); 2096 set_thread(1); 2097 poll_thread(1); 2098 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 2099 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x1000); 2100 CU_ASSERT(cb_arg.iov.iov_len == 1 * g_bdev.bdev.blocklen); 2101 stub_complete_io(g_bdev.io_target, 1); 2102 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[1]) == 0); 2103 2104 memset(&cb_arg, 0, sizeof(cb_arg)); 2105 set_thread(2); 2106 poll_thread(2); 2107 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_READ); 2108 CU_ASSERT(cb_arg.iov.iov_base == (void *)0xF000); 2109 CU_ASSERT(cb_arg.iov.iov_len == 8 * g_bdev.bdev.blocklen); 2110 stub_complete_io(g_bdev.io_target, 1); 2111 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[2]) == 2); 2112 stub_complete_io(g_bdev.io_target, 1); 2113 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[2]) == 0); 2114 2115 /* Run poll_timeout_done() it means complete the timeout poller */ 2116 set_thread(0); 2117 poll_thread(0); 2118 CU_ASSERT(g_desc->refs == 0); 2119 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[0], (void *)0x1000, 0, 1, io_done, NULL) == 0); 2120 set_thread(1); 2121 CU_ASSERT(spdk_bdev_write_blocks(g_desc, ch[1], (void *)0x2000, 0, 2, io_done, NULL) == 0); 2122 set_thread(2); 2123 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[2], (void *)0x3000, 0, 3, io_done, NULL) == 0); 2124 2125 /* Trigger timeout poller to run again, desc->refs is incremented. 2126 * In thread 0 we destroy the io channel before timeout poller runs. 2127 * Timeout callback is not called on thread 0. 2128 */ 2129 spdk_delay_us(6 * spdk_get_ticks_hz()); 2130 memset(&cb_arg, 0, sizeof(cb_arg)); 2131 set_thread(0); 2132 stub_complete_io(g_bdev.io_target, 1); 2133 spdk_put_io_channel(ch[0]); 2134 poll_thread(0); 2135 CU_ASSERT(g_desc->refs == 1) 2136 CU_ASSERT(cb_arg.type == 0); 2137 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 2138 CU_ASSERT(cb_arg.iov.iov_len == 0); 2139 2140 /* In thread 1 timeout poller runs then we destroy the io channel 2141 * Timeout callback is called on thread 1. 2142 */ 2143 memset(&cb_arg, 0, sizeof(cb_arg)); 2144 set_thread(1); 2145 poll_thread(1); 2146 stub_complete_io(g_bdev.io_target, 1); 2147 spdk_put_io_channel(ch[1]); 2148 poll_thread(1); 2149 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 2150 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x2000); 2151 CU_ASSERT(cb_arg.iov.iov_len == 2 * g_bdev.bdev.blocklen); 2152 2153 /* Close the desc. 2154 * Unregister the timeout poller first. 2155 * Then decrement desc->refs but it's not zero yet so desc is not freed. 2156 */ 2157 set_thread(0); 2158 spdk_bdev_close(g_desc); 2159 CU_ASSERT(g_desc->refs == 1); 2160 CU_ASSERT(g_desc->io_timeout_poller == NULL); 2161 2162 /* Timeout poller runs on thread 2 then we destroy the io channel. 2163 * Desc is closed so we would exit the timeout poller directly. 2164 * timeout callback is not called on thread 2. 2165 */ 2166 memset(&cb_arg, 0, sizeof(cb_arg)); 2167 set_thread(2); 2168 poll_thread(2); 2169 stub_complete_io(g_bdev.io_target, 1); 2170 spdk_put_io_channel(ch[2]); 2171 poll_thread(2); 2172 CU_ASSERT(cb_arg.type == 0); 2173 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 2174 CU_ASSERT(cb_arg.iov.iov_len == 0); 2175 2176 set_thread(0); 2177 poll_thread(0); 2178 g_teardown_done = false; 2179 unregister_bdev(&g_bdev); 2180 spdk_io_device_unregister(&g_io_device, NULL); 2181 spdk_bdev_finish(finish_cb, NULL); 2182 spdk_iobuf_finish(finish_cb, NULL); 2183 poll_threads(); 2184 memset(&g_bdev, 0, sizeof(g_bdev)); 2185 CU_ASSERT(g_teardown_done == true); 2186 g_teardown_done = false; 2187 free_threads(); 2188 free_cores(); 2189 } 2190 2191 static bool g_io_done2; 2192 static bool g_lock_lba_range_done; 2193 static bool g_unlock_lba_range_done; 2194 2195 static void 2196 io_done2(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 2197 { 2198 g_io_done2 = true; 2199 spdk_bdev_free_io(bdev_io); 2200 } 2201 2202 static void 2203 lock_lba_range_done(struct lba_range *range, void *ctx, int status) 2204 { 2205 g_lock_lba_range_done = true; 2206 } 2207 2208 static void 2209 unlock_lba_range_done(struct lba_range *range, void *ctx, int status) 2210 { 2211 g_unlock_lba_range_done = true; 2212 } 2213 2214 static uint32_t 2215 stub_channel_outstanding_cnt(void *io_target) 2216 { 2217 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 2218 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 2219 uint32_t outstanding_cnt; 2220 2221 outstanding_cnt = ch->outstanding_cnt; 2222 2223 spdk_put_io_channel(_ch); 2224 return outstanding_cnt; 2225 } 2226 2227 static void 2228 lock_lba_range_then_submit_io(void) 2229 { 2230 struct spdk_bdev_desc *desc = NULL; 2231 void *io_target; 2232 struct spdk_io_channel *io_ch[3]; 2233 struct spdk_bdev_channel *bdev_ch[3]; 2234 struct lba_range *range; 2235 char buf[4096]; 2236 int ctx0, ctx1, ctx2; 2237 int rc; 2238 2239 setup_test(); 2240 2241 io_target = g_bdev.io_target; 2242 desc = g_desc; 2243 2244 set_thread(0); 2245 io_ch[0] = spdk_bdev_get_io_channel(desc); 2246 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 2247 CU_ASSERT(io_ch[0] != NULL); 2248 2249 set_thread(1); 2250 io_ch[1] = spdk_bdev_get_io_channel(desc); 2251 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 2252 CU_ASSERT(io_ch[1] != NULL); 2253 2254 set_thread(0); 2255 g_lock_lba_range_done = false; 2256 rc = bdev_lock_lba_range(desc, io_ch[0], 20, 10, lock_lba_range_done, &ctx0); 2257 CU_ASSERT(rc == 0); 2258 poll_threads(); 2259 2260 /* The lock should immediately become valid, since there are no outstanding 2261 * write I/O. 2262 */ 2263 CU_ASSERT(g_lock_lba_range_done == true); 2264 range = TAILQ_FIRST(&bdev_ch[0]->locked_ranges); 2265 SPDK_CU_ASSERT_FATAL(range != NULL); 2266 CU_ASSERT(range->offset == 20); 2267 CU_ASSERT(range->length == 10); 2268 CU_ASSERT(range->owner_ch == bdev_ch[0]); 2269 2270 g_io_done = false; 2271 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 2272 rc = spdk_bdev_read_blocks(desc, io_ch[0], buf, 20, 1, io_done, &ctx0); 2273 CU_ASSERT(rc == 0); 2274 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 2275 2276 stub_complete_io(io_target, 1); 2277 poll_threads(); 2278 CU_ASSERT(g_io_done == true); 2279 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 2280 2281 /* Try a write I/O. This should actually be allowed to execute, since the channel 2282 * holding the lock is submitting the write I/O. 2283 */ 2284 g_io_done = false; 2285 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 2286 rc = spdk_bdev_write_blocks(desc, io_ch[0], buf, 20, 1, io_done, &ctx0); 2287 CU_ASSERT(rc == 0); 2288 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 2289 2290 stub_complete_io(io_target, 1); 2291 poll_threads(); 2292 CU_ASSERT(g_io_done == true); 2293 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 2294 2295 /* Try a write I/O. This should get queued in the io_locked tailq. */ 2296 set_thread(1); 2297 g_io_done = false; 2298 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[1]->io_locked)); 2299 rc = spdk_bdev_write_blocks(desc, io_ch[1], buf, 20, 1, io_done, &ctx1); 2300 CU_ASSERT(rc == 0); 2301 poll_threads(); 2302 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 0); 2303 CU_ASSERT(!TAILQ_EMPTY(&bdev_ch[1]->io_locked)); 2304 CU_ASSERT(g_io_done == false); 2305 2306 /* Try to unlock the lba range using thread 1's io_ch. This should fail. */ 2307 rc = bdev_unlock_lba_range(desc, io_ch[1], 20, 10, unlock_lba_range_done, &ctx1); 2308 CU_ASSERT(rc == -EINVAL); 2309 2310 /* Now create a new channel and submit a write I/O with it. This should also be queued. 2311 * The new channel should inherit the active locks from the bdev's internal list. 2312 */ 2313 set_thread(2); 2314 io_ch[2] = spdk_bdev_get_io_channel(desc); 2315 bdev_ch[2] = spdk_io_channel_get_ctx(io_ch[2]); 2316 CU_ASSERT(io_ch[2] != NULL); 2317 2318 g_io_done2 = false; 2319 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[2]->io_locked)); 2320 rc = spdk_bdev_write_blocks(desc, io_ch[2], buf, 22, 2, io_done2, &ctx2); 2321 CU_ASSERT(rc == 0); 2322 poll_threads(); 2323 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 0); 2324 CU_ASSERT(!TAILQ_EMPTY(&bdev_ch[2]->io_locked)); 2325 CU_ASSERT(g_io_done2 == false); 2326 2327 set_thread(0); 2328 rc = bdev_unlock_lba_range(desc, io_ch[0], 20, 10, unlock_lba_range_done, &ctx0); 2329 CU_ASSERT(rc == 0); 2330 poll_threads(); 2331 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->locked_ranges)); 2332 2333 /* The LBA range is unlocked, so the write IOs should now have started execution. */ 2334 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[1]->io_locked)); 2335 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[2]->io_locked)); 2336 2337 set_thread(1); 2338 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 2339 stub_complete_io(io_target, 1); 2340 set_thread(2); 2341 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 2342 stub_complete_io(io_target, 1); 2343 2344 poll_threads(); 2345 CU_ASSERT(g_io_done == true); 2346 CU_ASSERT(g_io_done2 == true); 2347 2348 /* Tear down the channels */ 2349 set_thread(0); 2350 spdk_put_io_channel(io_ch[0]); 2351 set_thread(1); 2352 spdk_put_io_channel(io_ch[1]); 2353 set_thread(2); 2354 spdk_put_io_channel(io_ch[2]); 2355 poll_threads(); 2356 set_thread(0); 2357 teardown_test(); 2358 } 2359 2360 /* spdk_bdev_reset() freezes and unfreezes I/O channels by using spdk_for_each_channel(). 2361 * spdk_bdev_unregister() calls spdk_io_device_unregister() in the end. However 2362 * spdk_io_device_unregister() fails if it is called while executing spdk_for_each_channel(). 2363 * Hence, in this case, spdk_io_device_unregister() is deferred until spdk_bdev_reset() 2364 * completes. Test this behavior. 2365 */ 2366 static void 2367 unregister_during_reset(void) 2368 { 2369 struct spdk_io_channel *io_ch[2]; 2370 bool done_reset = false, done_unregister = false; 2371 int rc; 2372 2373 setup_test(); 2374 set_thread(0); 2375 2376 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 2377 SPDK_CU_ASSERT_FATAL(io_ch[0] != NULL); 2378 2379 set_thread(1); 2380 2381 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 2382 SPDK_CU_ASSERT_FATAL(io_ch[1] != NULL); 2383 2384 set_thread(0); 2385 2386 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 2387 2388 rc = spdk_bdev_reset(g_desc, io_ch[0], reset_done, &done_reset); 2389 CU_ASSERT(rc == 0); 2390 2391 set_thread(0); 2392 2393 poll_thread_times(0, 1); 2394 2395 spdk_bdev_close(g_desc); 2396 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done_unregister); 2397 2398 CU_ASSERT(done_reset == false); 2399 CU_ASSERT(done_unregister == false); 2400 2401 poll_threads(); 2402 2403 stub_complete_io(g_bdev.io_target, 0); 2404 2405 poll_threads(); 2406 2407 CU_ASSERT(done_reset == true); 2408 CU_ASSERT(done_unregister == false); 2409 2410 spdk_put_io_channel(io_ch[0]); 2411 2412 set_thread(1); 2413 2414 spdk_put_io_channel(io_ch[1]); 2415 2416 poll_threads(); 2417 2418 CU_ASSERT(done_unregister == true); 2419 2420 /* Restore the original g_bdev so that we can use teardown_test(). */ 2421 set_thread(0); 2422 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 2423 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 2424 teardown_test(); 2425 } 2426 2427 static void 2428 bdev_init_wt_cb(void *done, int rc) 2429 { 2430 } 2431 2432 static int 2433 wrong_thread_setup(void) 2434 { 2435 allocate_cores(1); 2436 allocate_threads(2); 2437 set_thread(0); 2438 2439 spdk_io_device_register(&g_accel_io_device, ut_accel_ch_create_cb, 2440 ut_accel_ch_destroy_cb, 0, NULL); 2441 spdk_bdev_initialize(bdev_init_wt_cb, NULL); 2442 spdk_io_device_register(&g_io_device, stub_create_ch, stub_destroy_ch, 2443 sizeof(struct ut_bdev_channel), NULL); 2444 2445 set_thread(1); 2446 2447 return 0; 2448 } 2449 2450 static int 2451 wrong_thread_teardown(void) 2452 { 2453 int rc = 0; 2454 2455 set_thread(0); 2456 2457 g_teardown_done = false; 2458 spdk_io_device_unregister(&g_io_device, NULL); 2459 spdk_bdev_finish(finish_cb, NULL); 2460 poll_threads(); 2461 memset(&g_bdev, 0, sizeof(g_bdev)); 2462 if (!g_teardown_done) { 2463 fprintf(stderr, "%s:%d %s: teardown not done\n", __FILE__, __LINE__, __func__); 2464 rc = -1; 2465 } 2466 g_teardown_done = false; 2467 2468 spdk_io_device_unregister(&g_accel_io_device, NULL); 2469 free_threads(); 2470 free_cores(); 2471 2472 return rc; 2473 } 2474 2475 static void 2476 _bdev_unregistered_wt(void *ctx, int rc) 2477 { 2478 struct spdk_thread **threadp = ctx; 2479 2480 *threadp = spdk_get_thread(); 2481 } 2482 2483 static void 2484 spdk_bdev_register_wt(void) 2485 { 2486 struct spdk_bdev bdev = { 0 }; 2487 int rc; 2488 struct spdk_thread *unreg_thread; 2489 2490 bdev.name = "wt_bdev"; 2491 bdev.fn_table = &fn_table; 2492 bdev.module = &bdev_ut_if; 2493 bdev.blocklen = 4096; 2494 bdev.blockcnt = 1024; 2495 2496 /* Can register only on app thread */ 2497 rc = spdk_bdev_register(&bdev); 2498 CU_ASSERT(rc == -EINVAL); 2499 2500 /* Can unregister on any thread */ 2501 set_thread(0); 2502 rc = spdk_bdev_register(&bdev); 2503 CU_ASSERT(rc == 0); 2504 set_thread(1); 2505 unreg_thread = NULL; 2506 spdk_bdev_unregister(&bdev, _bdev_unregistered_wt, &unreg_thread); 2507 poll_threads(); 2508 CU_ASSERT(unreg_thread == spdk_get_thread()); 2509 2510 /* Can unregister by name on any thread */ 2511 set_thread(0); 2512 rc = spdk_bdev_register(&bdev); 2513 CU_ASSERT(rc == 0); 2514 set_thread(1); 2515 unreg_thread = NULL; 2516 rc = spdk_bdev_unregister_by_name(bdev.name, bdev.module, _bdev_unregistered_wt, 2517 &unreg_thread); 2518 CU_ASSERT(rc == 0); 2519 poll_threads(); 2520 CU_ASSERT(unreg_thread == spdk_get_thread()); 2521 } 2522 2523 static void 2524 wait_for_examine_cb(void *arg) 2525 { 2526 struct spdk_thread **thread = arg; 2527 2528 *thread = spdk_get_thread(); 2529 } 2530 2531 static void 2532 spdk_bdev_examine_wt(void) 2533 { 2534 int rc; 2535 bool save_auto_examine = g_bdev_opts.bdev_auto_examine; 2536 struct spdk_thread *thread; 2537 2538 g_bdev_opts.bdev_auto_examine = false; 2539 2540 set_thread(0); 2541 register_bdev(&g_bdev, "ut_bdev_wt", &g_io_device); 2542 CU_ASSERT(spdk_bdev_get_by_name("ut_bdev_wt") != NULL); 2543 set_thread(1); 2544 2545 /* Can examine only on the app thread */ 2546 rc = spdk_bdev_examine("ut_bdev_wt"); 2547 CU_ASSERT(rc == -EINVAL); 2548 unregister_bdev(&g_bdev); 2549 CU_ASSERT(spdk_bdev_get_by_name("ut_bdev_wt") == NULL); 2550 2551 /* Can wait for examine on app thread, callback called on app thread. */ 2552 set_thread(0); 2553 register_bdev(&g_bdev, "ut_bdev_wt", &g_io_device); 2554 CU_ASSERT(spdk_bdev_get_by_name("ut_bdev_wt") != NULL); 2555 thread = NULL; 2556 rc = spdk_bdev_wait_for_examine(wait_for_examine_cb, &thread); 2557 CU_ASSERT(rc == 0); 2558 poll_threads(); 2559 CU_ASSERT(thread == spdk_get_thread()); 2560 unregister_bdev(&g_bdev); 2561 CU_ASSERT(spdk_bdev_get_by_name("ut_bdev_wt") == NULL); 2562 2563 /* Can wait for examine on non-app thread, callback called on same thread. */ 2564 set_thread(0); 2565 register_bdev(&g_bdev, "ut_bdev_wt", &g_io_device); 2566 CU_ASSERT(spdk_bdev_get_by_name("ut_bdev_wt") != NULL); 2567 thread = NULL; 2568 rc = spdk_bdev_wait_for_examine(wait_for_examine_cb, &thread); 2569 CU_ASSERT(rc == 0); 2570 poll_threads(); 2571 CU_ASSERT(thread == spdk_get_thread()); 2572 unregister_bdev(&g_bdev); 2573 CU_ASSERT(spdk_bdev_get_by_name("ut_bdev_wt") == NULL); 2574 2575 unregister_bdev(&g_bdev); 2576 g_bdev_opts.bdev_auto_examine = save_auto_examine; 2577 } 2578 2579 static void 2580 event_notify_and_close(void) 2581 { 2582 int resize_notify_count = 0; 2583 struct spdk_bdev_desc *desc = NULL; 2584 struct spdk_bdev *bdev; 2585 int rc; 2586 2587 setup_test(); 2588 set_thread(0); 2589 2590 /* setup_test() automatically opens the bdev, but this test needs to do 2591 * that in a different way. */ 2592 spdk_bdev_close(g_desc); 2593 poll_threads(); 2594 2595 set_thread(1); 2596 2597 rc = spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, &resize_notify_count, &desc); 2598 CU_ASSERT(rc == 0); 2599 SPDK_CU_ASSERT_FATAL(desc != NULL); 2600 2601 bdev = spdk_bdev_desc_get_bdev(desc); 2602 SPDK_CU_ASSERT_FATAL(bdev != NULL); 2603 2604 /* Test a normal case that a resize event is notified. */ 2605 set_thread(0); 2606 2607 rc = spdk_bdev_notify_blockcnt_change(bdev, 1024 * 2); 2608 CU_ASSERT(rc == 0); 2609 CU_ASSERT(bdev->blockcnt == 1024 * 2); 2610 CU_ASSERT(desc->refs == 1); 2611 CU_ASSERT(resize_notify_count == 0); 2612 2613 poll_threads(); 2614 2615 CU_ASSERT(desc->refs == 0); 2616 CU_ASSERT(resize_notify_count == 1); 2617 2618 /* Test a complex case if the bdev is closed after two event_notify messages are sent, 2619 * then both event_notify messages are discarded and the desc is freed. 2620 */ 2621 rc = spdk_bdev_notify_blockcnt_change(bdev, 1024 * 3); 2622 CU_ASSERT(rc == 0); 2623 CU_ASSERT(bdev->blockcnt == 1024 * 3); 2624 CU_ASSERT(desc->refs == 1); 2625 CU_ASSERT(resize_notify_count == 1); 2626 2627 rc = spdk_bdev_notify_blockcnt_change(bdev, 1024 * 4); 2628 CU_ASSERT(rc == 0); 2629 CU_ASSERT(bdev->blockcnt == 1024 * 4); 2630 CU_ASSERT(desc->refs == 2); 2631 CU_ASSERT(resize_notify_count == 1); 2632 2633 set_thread(1); 2634 2635 spdk_bdev_close(desc); 2636 CU_ASSERT(desc->closed == true); 2637 CU_ASSERT(desc->refs == 2); 2638 CU_ASSERT(resize_notify_count == 1); 2639 2640 poll_threads(); 2641 2642 CU_ASSERT(resize_notify_count == 1); 2643 2644 set_thread(0); 2645 2646 /* Restore g_desc. Then, we can execute teardown_test(). */ 2647 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 2648 teardown_test(); 2649 } 2650 2651 /* There was a bug that bdev_channel_poll_qos() called spdk_for_each_channel() 2652 * after spdk_io_device_unregister() is called for a bdev. 2653 * 2654 * This occurred in the following sequence. 2655 * - There was a bdev and a channel for it. 2656 * - QoS was enabled and started. 2657 * - spdk_bdev_unregister() was called. However, there was a open descriptor. 2658 * Hence, remove notification was sent and unregistration was pending. 2659 * - Receiving a event notification, spdk_put_io_channel() and spdk_bdev_close() were 2660 * called. In spdk_bdev_close(), the existing QoS was unbound and a message was sent 2661 * to it, and then the pending spdk_io_device_unregister() was finally executed. 2662 * - If bdev_channel_poll_qos() was executed before the message was processed, 2663 * bdev_channel_poll_qos() called spdk_bdev_for_each_channel() and hit assert(). 2664 * 2665 * The fix was in this case bdev_channel_poll_qos() returned immediately because QoS 2666 * was not enabled. bdev_qos_destroy() created a new disabled QoS and swapped it with 2667 * the existing QoS. 2668 * 2669 * This test case was added to avoid degradation in future. 2670 */ 2671 static void 2672 unregister_and_qos_poller(void) 2673 { 2674 struct spdk_io_channel *io_ch; 2675 struct spdk_bdev_channel *bdev_ch; 2676 struct spdk_bdev_desc *desc = NULL; 2677 struct spdk_bdev_qos *old_qos; 2678 uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {}; 2679 bool remove_notify = false, done_unregister = false; 2680 int status = -1, rc; 2681 2682 setup_test(); 2683 set_thread(0); 2684 2685 MOCK_SET(spdk_get_ticks, 10); 2686 2687 /* setup_test() automatically opens the bdev, but this test needs to do 2688 * that in a different way. 2689 */ 2690 spdk_bdev_close(g_desc); 2691 poll_threads(); 2692 2693 /* We want to get remove event notification to check if unregistration 2694 * is deferred. 2695 */ 2696 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, &remove_notify, &desc); 2697 SPDK_CU_ASSERT_FATAL(desc != NULL); 2698 CU_ASSERT(remove_notify == false); 2699 2700 io_ch = spdk_bdev_get_io_channel(desc); 2701 SPDK_CU_ASSERT_FATAL(io_ch != NULL); 2702 bdev_ch = spdk_io_channel_get_ctx(io_ch); 2703 2704 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 2705 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 0; 2706 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 0; 2707 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 0; 2708 spdk_bdev_set_qos_rate_limits(&g_bdev.bdev, limits, qos_dynamic_enable_done, &status); 2709 poll_threads(); 2710 CU_ASSERT(status == 0); 2711 CU_ASSERT((bdev_ch->flags & BDEV_CH_QOS_ENABLED) != 0); 2712 2713 old_qos = g_bdev.bdev.internal.qos; 2714 CU_ASSERT(old_qos != NULL); 2715 2716 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done_unregister); 2717 CU_ASSERT(done_unregister == false); 2718 CU_ASSERT(remove_notify == false); 2719 2720 poll_threads(); 2721 CU_ASSERT(done_unregister == false); 2722 CU_ASSERT(remove_notify == true); 2723 2724 spdk_put_io_channel(io_ch); 2725 spdk_bdev_close(desc); 2726 2727 CU_ASSERT(g_bdev.bdev.internal.qos != NULL); 2728 CU_ASSERT(g_bdev.bdev.internal.qos->thread == NULL); 2729 CU_ASSERT(old_qos != g_bdev.bdev.internal.qos); 2730 2731 /* bdev_channel_poll_qos() has a chance to be executed in this small window. */ 2732 spdk_delay_us(SPDK_BDEV_QOS_TIMESLICE_IN_USEC); 2733 2734 rc = bdev_channel_poll_qos(&g_bdev.bdev); 2735 CU_ASSERT(rc == SPDK_POLLER_IDLE); 2736 2737 poll_threads(); 2738 2739 CU_ASSERT(done_unregister == true); 2740 2741 /* Restore the original g_bdev so that we can use teardown_test(). */ 2742 set_thread(0); 2743 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 2744 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 2745 teardown_test(); 2746 } 2747 2748 int 2749 main(int argc, char **argv) 2750 { 2751 CU_pSuite suite = NULL; 2752 CU_pSuite suite_wt = NULL; 2753 unsigned int num_failures; 2754 2755 CU_initialize_registry(); 2756 2757 suite = CU_add_suite("bdev", NULL, NULL); 2758 suite_wt = CU_add_suite("bdev_wrong_thread", wrong_thread_setup, wrong_thread_teardown); 2759 2760 CU_ADD_TEST(suite, basic); 2761 CU_ADD_TEST(suite, unregister_and_close); 2762 CU_ADD_TEST(suite, unregister_and_close_different_threads); 2763 CU_ADD_TEST(suite, basic_qos); 2764 CU_ADD_TEST(suite, put_channel_during_reset); 2765 CU_ADD_TEST(suite, aborted_reset); 2766 CU_ADD_TEST(suite, aborted_reset_no_outstanding_io); 2767 CU_ADD_TEST(suite, io_during_reset); 2768 CU_ADD_TEST(suite, reset_completions); 2769 CU_ADD_TEST(suite, io_during_qos_queue); 2770 CU_ADD_TEST(suite, io_during_qos_reset); 2771 CU_ADD_TEST(suite, enomem); 2772 CU_ADD_TEST(suite, enomem_multi_bdev); 2773 CU_ADD_TEST(suite, enomem_multi_bdev_unregister); 2774 CU_ADD_TEST(suite, enomem_multi_io_target); 2775 CU_ADD_TEST(suite, qos_dynamic_enable); 2776 CU_ADD_TEST(suite, bdev_histograms_mt); 2777 CU_ADD_TEST(suite, bdev_set_io_timeout_mt); 2778 CU_ADD_TEST(suite, lock_lba_range_then_submit_io); 2779 CU_ADD_TEST(suite, unregister_during_reset); 2780 CU_ADD_TEST(suite_wt, spdk_bdev_register_wt); 2781 CU_ADD_TEST(suite_wt, spdk_bdev_examine_wt); 2782 CU_ADD_TEST(suite, event_notify_and_close); 2783 CU_ADD_TEST(suite, unregister_and_qos_poller); 2784 2785 num_failures = spdk_ut_run_tests(argc, argv, NULL); 2786 CU_cleanup_registry(); 2787 return num_failures; 2788 } 2789