1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk_cunit.h" 7 8 #include "common/lib/ut_multithread.c" 9 #include "unit/lib/json_mock.c" 10 11 #include "spdk/config.h" 12 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 13 #undef SPDK_CONFIG_VTUNE 14 15 #include "bdev/bdev.c" 16 17 #define BDEV_UT_NUM_THREADS 3 18 19 DEFINE_STUB(spdk_notify_send, uint64_t, (const char *type, const char *ctx), 0); 20 DEFINE_STUB(spdk_notify_type_register, struct spdk_notify_type *, (const char *type), NULL); 21 DEFINE_STUB_V(spdk_scsi_nvme_translate, (const struct spdk_bdev_io *bdev_io, int *sc, int *sk, 22 int *asc, int *ascq)); 23 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, (struct spdk_memory_domain *domain), 24 "test_domain"); 25 DEFINE_STUB(spdk_memory_domain_get_dma_device_type, enum spdk_dma_device_type, 26 (struct spdk_memory_domain *domain), 0); 27 28 DEFINE_RETURN_MOCK(spdk_memory_domain_pull_data, int); 29 int 30 spdk_memory_domain_pull_data(struct spdk_memory_domain *src_domain, void *src_domain_ctx, 31 struct iovec *src_iov, uint32_t src_iov_cnt, struct iovec *dst_iov, uint32_t dst_iov_cnt, 32 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) 33 { 34 HANDLE_RETURN_MOCK(spdk_memory_domain_pull_data); 35 36 cpl_cb(cpl_cb_arg, 0); 37 return 0; 38 } 39 40 DEFINE_RETURN_MOCK(spdk_memory_domain_push_data, int); 41 int 42 spdk_memory_domain_push_data(struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 43 struct iovec *dst_iov, uint32_t dst_iovcnt, struct iovec *src_iov, uint32_t src_iovcnt, 44 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) 45 { 46 HANDLE_RETURN_MOCK(spdk_memory_domain_push_data); 47 48 cpl_cb(cpl_cb_arg, 0); 49 return 0; 50 } 51 52 struct ut_bdev { 53 struct spdk_bdev bdev; 54 void *io_target; 55 }; 56 57 struct ut_bdev_channel { 58 TAILQ_HEAD(, spdk_bdev_io) outstanding_io; 59 uint32_t outstanding_cnt; 60 uint32_t avail_cnt; 61 }; 62 63 int g_io_device; 64 struct ut_bdev g_bdev; 65 struct spdk_bdev_desc *g_desc; 66 bool g_teardown_done = false; 67 bool g_get_io_channel = true; 68 bool g_create_ch = true; 69 bool g_init_complete_called = false; 70 bool g_fini_start_called = true; 71 int g_status = 0; 72 int g_count = 0; 73 struct spdk_histogram_data *g_histogram = NULL; 74 75 static int 76 stub_create_ch(void *io_device, void *ctx_buf) 77 { 78 struct ut_bdev_channel *ch = ctx_buf; 79 80 if (g_create_ch == false) { 81 return -1; 82 } 83 84 TAILQ_INIT(&ch->outstanding_io); 85 ch->outstanding_cnt = 0; 86 /* 87 * When avail gets to 0, the submit_request function will return ENOMEM. 88 * Most tests to not want ENOMEM to occur, so by default set this to a 89 * big value that won't get hit. The ENOMEM tests can then override this 90 * value to something much smaller to induce ENOMEM conditions. 91 */ 92 ch->avail_cnt = 2048; 93 return 0; 94 } 95 96 static void 97 stub_destroy_ch(void *io_device, void *ctx_buf) 98 { 99 } 100 101 static struct spdk_io_channel * 102 stub_get_io_channel(void *ctx) 103 { 104 struct ut_bdev *ut_bdev = ctx; 105 106 if (g_get_io_channel == true) { 107 return spdk_get_io_channel(ut_bdev->io_target); 108 } else { 109 return NULL; 110 } 111 } 112 113 static int 114 stub_destruct(void *ctx) 115 { 116 return 0; 117 } 118 119 static void 120 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 121 { 122 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 123 struct spdk_bdev_io *io; 124 125 if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { 126 while (!TAILQ_EMPTY(&ch->outstanding_io)) { 127 io = TAILQ_FIRST(&ch->outstanding_io); 128 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 129 ch->outstanding_cnt--; 130 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_ABORTED); 131 ch->avail_cnt++; 132 } 133 } else if (bdev_io->type == SPDK_BDEV_IO_TYPE_ABORT) { 134 TAILQ_FOREACH(io, &ch->outstanding_io, module_link) { 135 if (io == bdev_io->u.abort.bio_to_abort) { 136 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 137 ch->outstanding_cnt--; 138 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_ABORTED); 139 ch->avail_cnt++; 140 141 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); 142 return; 143 } 144 } 145 146 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 147 return; 148 } 149 150 if (ch->avail_cnt > 0) { 151 TAILQ_INSERT_TAIL(&ch->outstanding_io, bdev_io, module_link); 152 ch->outstanding_cnt++; 153 ch->avail_cnt--; 154 } else { 155 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); 156 } 157 } 158 159 static uint32_t 160 stub_complete_io(void *io_target, uint32_t num_to_complete) 161 { 162 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 163 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 164 struct spdk_bdev_io *io; 165 bool complete_all = (num_to_complete == 0); 166 uint32_t num_completed = 0; 167 168 while (complete_all || num_completed < num_to_complete) { 169 if (TAILQ_EMPTY(&ch->outstanding_io)) { 170 break; 171 } 172 io = TAILQ_FIRST(&ch->outstanding_io); 173 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 174 ch->outstanding_cnt--; 175 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_SUCCESS); 176 ch->avail_cnt++; 177 num_completed++; 178 } 179 spdk_put_io_channel(_ch); 180 return num_completed; 181 } 182 183 static bool 184 stub_io_type_supported(void *ctx, enum spdk_bdev_io_type type) 185 { 186 return true; 187 } 188 189 static struct spdk_bdev_fn_table fn_table = { 190 .get_io_channel = stub_get_io_channel, 191 .destruct = stub_destruct, 192 .submit_request = stub_submit_request, 193 .io_type_supported = stub_io_type_supported, 194 }; 195 196 struct spdk_bdev_module bdev_ut_if; 197 198 static int 199 module_init(void) 200 { 201 spdk_bdev_module_init_done(&bdev_ut_if); 202 return 0; 203 } 204 205 static void 206 module_fini(void) 207 { 208 } 209 210 static void 211 init_complete(void) 212 { 213 g_init_complete_called = true; 214 } 215 216 static void 217 fini_start(void) 218 { 219 g_fini_start_called = true; 220 } 221 222 struct spdk_bdev_module bdev_ut_if = { 223 .name = "bdev_ut", 224 .module_init = module_init, 225 .module_fini = module_fini, 226 .async_init = true, 227 .init_complete = init_complete, 228 .fini_start = fini_start, 229 }; 230 231 SPDK_BDEV_MODULE_REGISTER(bdev_ut, &bdev_ut_if) 232 233 static void 234 register_bdev(struct ut_bdev *ut_bdev, char *name, void *io_target) 235 { 236 memset(ut_bdev, 0, sizeof(*ut_bdev)); 237 238 ut_bdev->io_target = io_target; 239 ut_bdev->bdev.ctxt = ut_bdev; 240 ut_bdev->bdev.name = name; 241 ut_bdev->bdev.fn_table = &fn_table; 242 ut_bdev->bdev.module = &bdev_ut_if; 243 ut_bdev->bdev.blocklen = 4096; 244 ut_bdev->bdev.blockcnt = 1024; 245 246 spdk_bdev_register(&ut_bdev->bdev); 247 } 248 249 static void 250 unregister_bdev(struct ut_bdev *ut_bdev) 251 { 252 /* Handle any deferred messages. */ 253 poll_threads(); 254 spdk_bdev_unregister(&ut_bdev->bdev, NULL, NULL); 255 /* Handle the async bdev unregister. */ 256 poll_threads(); 257 } 258 259 static void 260 bdev_init_cb(void *done, int rc) 261 { 262 CU_ASSERT(rc == 0); 263 *(bool *)done = true; 264 } 265 266 static void 267 _bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, 268 void *event_ctx) 269 { 270 switch (type) { 271 case SPDK_BDEV_EVENT_REMOVE: 272 if (event_ctx != NULL) { 273 *(bool *)event_ctx = true; 274 } 275 break; 276 default: 277 CU_ASSERT(false); 278 break; 279 } 280 } 281 282 static void 283 setup_test(void) 284 { 285 bool done = false; 286 287 allocate_cores(BDEV_UT_NUM_THREADS); 288 allocate_threads(BDEV_UT_NUM_THREADS); 289 set_thread(0); 290 spdk_bdev_initialize(bdev_init_cb, &done); 291 spdk_io_device_register(&g_io_device, stub_create_ch, stub_destroy_ch, 292 sizeof(struct ut_bdev_channel), NULL); 293 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 294 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 295 } 296 297 static void 298 finish_cb(void *cb_arg) 299 { 300 g_teardown_done = true; 301 } 302 303 static void 304 teardown_test(void) 305 { 306 set_thread(0); 307 g_teardown_done = false; 308 spdk_bdev_close(g_desc); 309 g_desc = NULL; 310 unregister_bdev(&g_bdev); 311 spdk_io_device_unregister(&g_io_device, NULL); 312 spdk_bdev_finish(finish_cb, NULL); 313 poll_threads(); 314 memset(&g_bdev, 0, sizeof(g_bdev)); 315 CU_ASSERT(g_teardown_done == true); 316 g_teardown_done = false; 317 free_threads(); 318 free_cores(); 319 } 320 321 static uint32_t 322 bdev_io_tailq_cnt(bdev_io_tailq_t *tailq) 323 { 324 struct spdk_bdev_io *io; 325 uint32_t cnt = 0; 326 327 TAILQ_FOREACH(io, tailq, internal.link) { 328 cnt++; 329 } 330 331 return cnt; 332 } 333 334 static void 335 basic(void) 336 { 337 g_init_complete_called = false; 338 setup_test(); 339 CU_ASSERT(g_init_complete_called == true); 340 341 set_thread(0); 342 343 g_get_io_channel = false; 344 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 345 CU_ASSERT(g_ut_threads[0].ch == NULL); 346 347 g_get_io_channel = true; 348 g_create_ch = false; 349 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 350 CU_ASSERT(g_ut_threads[0].ch == NULL); 351 352 g_get_io_channel = true; 353 g_create_ch = true; 354 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 355 CU_ASSERT(g_ut_threads[0].ch != NULL); 356 spdk_put_io_channel(g_ut_threads[0].ch); 357 358 g_fini_start_called = false; 359 teardown_test(); 360 CU_ASSERT(g_fini_start_called == true); 361 } 362 363 static void 364 _bdev_unregistered(void *done, int rc) 365 { 366 CU_ASSERT(rc == 0); 367 *(bool *)done = true; 368 } 369 370 static void 371 unregister_and_close(void) 372 { 373 bool done, remove_notify; 374 struct spdk_bdev_desc *desc = NULL; 375 376 setup_test(); 377 set_thread(0); 378 379 /* setup_test() automatically opens the bdev, 380 * but this test needs to do that in a different 381 * way. */ 382 spdk_bdev_close(g_desc); 383 poll_threads(); 384 385 /* Try hotremoving a bdev with descriptors which don't provide 386 * any context to the notification callback */ 387 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &desc); 388 SPDK_CU_ASSERT_FATAL(desc != NULL); 389 390 /* There is an open descriptor on the device. Unregister it, 391 * which can't proceed until the descriptor is closed. */ 392 done = false; 393 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 394 395 /* Poll the threads to allow all events to be processed */ 396 poll_threads(); 397 398 /* Make sure the bdev was not unregistered. We still have a 399 * descriptor open */ 400 CU_ASSERT(done == false); 401 402 spdk_bdev_close(desc); 403 poll_threads(); 404 desc = NULL; 405 406 /* The unregister should have completed */ 407 CU_ASSERT(done == true); 408 409 410 /* Register the bdev again */ 411 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 412 413 remove_notify = false; 414 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, &remove_notify, &desc); 415 SPDK_CU_ASSERT_FATAL(desc != NULL); 416 CU_ASSERT(remove_notify == false); 417 418 /* There is an open descriptor on the device. Unregister it, 419 * which can't proceed until the descriptor is closed. */ 420 done = false; 421 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 422 /* No polling has occurred, so neither of these should execute */ 423 CU_ASSERT(remove_notify == false); 424 CU_ASSERT(done == false); 425 426 /* Prior to the unregister completing, close the descriptor */ 427 spdk_bdev_close(desc); 428 429 /* Poll the threads to allow all events to be processed */ 430 poll_threads(); 431 432 /* Remove notify should not have been called because the 433 * descriptor is already closed. */ 434 CU_ASSERT(remove_notify == false); 435 436 /* The unregister should have completed */ 437 CU_ASSERT(done == true); 438 439 /* Restore the original g_bdev so that we can use teardown_test(). */ 440 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 441 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 442 teardown_test(); 443 } 444 445 static void 446 reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 447 { 448 bool *done = cb_arg; 449 450 CU_ASSERT(success == true); 451 *done = true; 452 spdk_bdev_free_io(bdev_io); 453 } 454 455 static void 456 put_channel_during_reset(void) 457 { 458 struct spdk_io_channel *io_ch; 459 bool done = false; 460 461 setup_test(); 462 463 set_thread(0); 464 io_ch = spdk_bdev_get_io_channel(g_desc); 465 CU_ASSERT(io_ch != NULL); 466 467 /* 468 * Start a reset, but then put the I/O channel before 469 * the deferred messages for the reset get a chance to 470 * execute. 471 */ 472 spdk_bdev_reset(g_desc, io_ch, reset_done, &done); 473 spdk_put_io_channel(io_ch); 474 poll_threads(); 475 stub_complete_io(g_bdev.io_target, 0); 476 477 teardown_test(); 478 } 479 480 static void 481 aborted_reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 482 { 483 enum spdk_bdev_io_status *status = cb_arg; 484 485 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 486 spdk_bdev_free_io(bdev_io); 487 } 488 489 static void io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); 490 491 static void 492 aborted_reset(void) 493 { 494 struct spdk_io_channel *io_ch[2]; 495 enum spdk_bdev_io_status status1 = SPDK_BDEV_IO_STATUS_PENDING, 496 status2 = SPDK_BDEV_IO_STATUS_PENDING; 497 498 setup_test(); 499 500 set_thread(0); 501 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 502 CU_ASSERT(io_ch[0] != NULL); 503 spdk_bdev_reset(g_desc, io_ch[0], aborted_reset_done, &status1); 504 poll_threads(); 505 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 506 507 /* 508 * First reset has been submitted on ch0. Now submit a second 509 * reset on ch1 which will get queued since there is already a 510 * reset in progress. 511 */ 512 set_thread(1); 513 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 514 CU_ASSERT(io_ch[1] != NULL); 515 spdk_bdev_reset(g_desc, io_ch[1], aborted_reset_done, &status2); 516 poll_threads(); 517 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 518 519 /* 520 * Now destroy ch1. This will abort the queued reset. Check that 521 * the second reset was completed with failed status. Also check 522 * that bdev->internal.reset_in_progress != NULL, since the 523 * original reset has not been completed yet. This ensures that 524 * the bdev code is correctly noticing that the failed reset is 525 * *not* the one that had been submitted to the bdev module. 526 */ 527 set_thread(1); 528 spdk_put_io_channel(io_ch[1]); 529 poll_threads(); 530 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_FAILED); 531 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 532 533 /* 534 * Now complete the first reset, verify that it completed with SUCCESS 535 * status and that bdev->internal.reset_in_progress is also set back to NULL. 536 */ 537 set_thread(0); 538 spdk_put_io_channel(io_ch[0]); 539 stub_complete_io(g_bdev.io_target, 0); 540 poll_threads(); 541 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 542 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 543 544 teardown_test(); 545 } 546 547 static void 548 aborted_reset_no_outstanding_io(void) 549 { 550 struct spdk_io_channel *io_ch[2]; 551 struct spdk_bdev_channel *bdev_ch[2]; 552 struct spdk_bdev *bdev[2]; 553 enum spdk_bdev_io_status status1 = SPDK_BDEV_IO_STATUS_PENDING, 554 status2 = SPDK_BDEV_IO_STATUS_PENDING; 555 556 setup_test(); 557 558 /* 559 * This time we test the reset without any outstanding IO 560 * present on the bdev channel, so both resets should finish 561 * immediately. 562 */ 563 564 set_thread(0); 565 /* Set reset_io_drain_timeout to allow bdev 566 * reset to stay pending until we call abort. */ 567 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 568 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 569 bdev[0] = bdev_ch[0]->bdev; 570 bdev[0]->reset_io_drain_timeout = SPDK_BDEV_RESET_IO_DRAIN_RECOMMENDED_VALUE; 571 CU_ASSERT(io_ch[0] != NULL); 572 spdk_bdev_reset(g_desc, io_ch[0], aborted_reset_done, &status1); 573 poll_threads(); 574 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 575 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 576 spdk_put_io_channel(io_ch[0]); 577 578 set_thread(1); 579 /* Set reset_io_drain_timeout to allow bdev 580 * reset to stay pending until we call abort. */ 581 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 582 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 583 bdev[1] = bdev_ch[1]->bdev; 584 bdev[1]->reset_io_drain_timeout = SPDK_BDEV_RESET_IO_DRAIN_RECOMMENDED_VALUE; 585 CU_ASSERT(io_ch[1] != NULL); 586 spdk_bdev_reset(g_desc, io_ch[1], aborted_reset_done, &status2); 587 poll_threads(); 588 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 589 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_SUCCESS); 590 spdk_put_io_channel(io_ch[1]); 591 592 stub_complete_io(g_bdev.io_target, 0); 593 poll_threads(); 594 595 teardown_test(); 596 } 597 598 599 static void 600 io_during_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 601 { 602 enum spdk_bdev_io_status *status = cb_arg; 603 604 *status = bdev_io->internal.status; 605 spdk_bdev_free_io(bdev_io); 606 } 607 608 static void 609 io_during_reset(void) 610 { 611 struct spdk_io_channel *io_ch[2]; 612 struct spdk_bdev_channel *bdev_ch[2]; 613 enum spdk_bdev_io_status status0, status1, status_reset; 614 int rc; 615 616 setup_test(); 617 618 /* 619 * First test normal case - submit an I/O on each of two channels (with no resets) 620 * and verify they complete successfully. 621 */ 622 set_thread(0); 623 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 624 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 625 CU_ASSERT(bdev_ch[0]->flags == 0); 626 status0 = SPDK_BDEV_IO_STATUS_PENDING; 627 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 628 CU_ASSERT(rc == 0); 629 630 set_thread(1); 631 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 632 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 633 CU_ASSERT(bdev_ch[1]->flags == 0); 634 status1 = SPDK_BDEV_IO_STATUS_PENDING; 635 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 636 CU_ASSERT(rc == 0); 637 638 poll_threads(); 639 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 640 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 641 642 set_thread(0); 643 stub_complete_io(g_bdev.io_target, 0); 644 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 645 646 set_thread(1); 647 stub_complete_io(g_bdev.io_target, 0); 648 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 649 650 /* 651 * Now submit a reset, and leave it pending while we submit I/O on two different 652 * channels. These I/O should be failed by the bdev layer since the reset is in 653 * progress. 654 */ 655 set_thread(0); 656 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 657 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &status_reset); 658 CU_ASSERT(rc == 0); 659 660 CU_ASSERT(bdev_ch[0]->flags == 0); 661 CU_ASSERT(bdev_ch[1]->flags == 0); 662 poll_threads(); 663 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_RESET_IN_PROGRESS); 664 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_RESET_IN_PROGRESS); 665 666 set_thread(0); 667 status0 = SPDK_BDEV_IO_STATUS_PENDING; 668 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 669 CU_ASSERT(rc == 0); 670 671 set_thread(1); 672 status1 = SPDK_BDEV_IO_STATUS_PENDING; 673 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 674 CU_ASSERT(rc == 0); 675 676 /* 677 * A reset is in progress so these read I/O should complete with aborted. Note that we 678 * need to poll_threads() since I/O completed inline have their completion deferred. 679 */ 680 poll_threads(); 681 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 682 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_ABORTED); 683 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_ABORTED); 684 685 /* 686 * Complete the reset 687 */ 688 set_thread(0); 689 stub_complete_io(g_bdev.io_target, 0); 690 691 /* 692 * Only poll thread 0. We should not get a completion. 693 */ 694 poll_thread(0); 695 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 696 697 /* 698 * Poll both thread 0 and 1 so the messages can propagate and we 699 * get a completion. 700 */ 701 poll_threads(); 702 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 703 704 spdk_put_io_channel(io_ch[0]); 705 set_thread(1); 706 spdk_put_io_channel(io_ch[1]); 707 poll_threads(); 708 709 teardown_test(); 710 } 711 712 static uint32_t 713 count_queued_resets(void *io_target) 714 { 715 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 716 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 717 struct spdk_bdev_io *io; 718 uint32_t submitted_resets = 0; 719 720 TAILQ_FOREACH(io, &ch->outstanding_io, module_link) { 721 if (io->type == SPDK_BDEV_IO_TYPE_RESET) { 722 submitted_resets++; 723 } 724 } 725 726 spdk_put_io_channel(_ch); 727 728 return submitted_resets; 729 } 730 731 static void 732 reset_completions(void) 733 { 734 struct spdk_io_channel *io_ch; 735 struct spdk_bdev_channel *bdev_ch; 736 struct spdk_bdev *bdev; 737 enum spdk_bdev_io_status status0, status_reset; 738 int rc, iter; 739 740 setup_test(); 741 742 /* This test covers four test cases: 743 * 1) reset_io_drain_timeout of a bdev is greater than 0 744 * 2) No outstandind IO are present on any bdev channel 745 * 3) Outstanding IO finish during bdev reset 746 * 4) Outstanding IO do not finish before reset is done waiting 747 * for them. 748 * 749 * Above conditions mainly affect the timing of bdev reset completion 750 * and whether a reset should be skipped via spdk_bdev_io_complete() 751 * or sent down to the underlying bdev module via bdev_io_submit_reset(). */ 752 753 /* Test preparation */ 754 set_thread(0); 755 io_ch = spdk_bdev_get_io_channel(g_desc); 756 bdev_ch = spdk_io_channel_get_ctx(io_ch); 757 CU_ASSERT(bdev_ch->flags == 0); 758 759 760 /* Test case 1) reset_io_drain_timeout set to 0. Reset should be sent down immediately. */ 761 bdev = &g_bdev.bdev; 762 bdev->reset_io_drain_timeout = 0; 763 764 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 765 rc = spdk_bdev_reset(g_desc, io_ch, io_during_io_done, &status_reset); 766 CU_ASSERT(rc == 0); 767 poll_threads(); 768 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 1); 769 770 /* Call reset completion inside bdev module. */ 771 stub_complete_io(g_bdev.io_target, 0); 772 poll_threads(); 773 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 774 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 775 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 776 777 778 /* Test case 2) no outstanding IO are present. Reset should perform one iteration over 779 * channels and then be skipped. */ 780 bdev->reset_io_drain_timeout = SPDK_BDEV_RESET_IO_DRAIN_RECOMMENDED_VALUE; 781 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 782 783 rc = spdk_bdev_reset(g_desc, io_ch, io_during_io_done, &status_reset); 784 CU_ASSERT(rc == 0); 785 poll_threads(); 786 /* Reset was never submitted to the bdev module. */ 787 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 788 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 789 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 790 791 792 /* Test case 3) outstanding IO finish during bdev reset procedure. Reset should initiate 793 * wait poller to check for IO completions every second, until reset_io_drain_timeout is 794 * reached, but finish earlier than this threshold. */ 795 status0 = SPDK_BDEV_IO_STATUS_PENDING; 796 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 797 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, io_during_io_done, &status0); 798 CU_ASSERT(rc == 0); 799 800 rc = spdk_bdev_reset(g_desc, io_ch, io_during_io_done, &status_reset); 801 CU_ASSERT(rc == 0); 802 poll_threads(); 803 /* The reset just started and should not have been submitted yet. */ 804 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 805 806 poll_threads(); 807 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 808 /* Let the poller wait for about half the time then complete outstanding IO. */ 809 for (iter = 0; iter < 2; iter++) { 810 /* Reset is still processing and not submitted at this point. */ 811 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 812 spdk_delay_us(1000 * 1000); 813 poll_threads(); 814 poll_threads(); 815 } 816 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 817 stub_complete_io(g_bdev.io_target, 0); 818 poll_threads(); 819 spdk_delay_us(BDEV_RESET_CHECK_OUTSTANDING_IO_PERIOD); 820 poll_threads(); 821 poll_threads(); 822 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 823 /* Sending reset to the bdev module has been skipped. */ 824 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 825 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 826 827 828 /* Test case 4) outstanding IO are still present after reset_io_drain_timeout 829 * seconds have passed. */ 830 status0 = SPDK_BDEV_IO_STATUS_PENDING; 831 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 832 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, io_during_io_done, &status0); 833 CU_ASSERT(rc == 0); 834 835 rc = spdk_bdev_reset(g_desc, io_ch, io_during_io_done, &status_reset); 836 CU_ASSERT(rc == 0); 837 poll_threads(); 838 /* The reset just started and should not have been submitted yet. */ 839 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 840 841 poll_threads(); 842 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 843 /* Let the poller wait for reset_io_drain_timeout seconds. */ 844 for (iter = 0; iter < bdev->reset_io_drain_timeout; iter++) { 845 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 0); 846 spdk_delay_us(BDEV_RESET_CHECK_OUTSTANDING_IO_PERIOD); 847 poll_threads(); 848 poll_threads(); 849 } 850 851 /* After timing out, the reset should have been sent to the module. */ 852 CU_ASSERT(count_queued_resets(g_bdev.io_target) == 1); 853 /* Complete reset submitted to the module and the read IO. */ 854 stub_complete_io(g_bdev.io_target, 0); 855 poll_threads(); 856 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 857 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 858 859 860 /* Destroy the channel and end the test. */ 861 spdk_put_io_channel(io_ch); 862 poll_threads(); 863 864 teardown_test(); 865 } 866 867 868 static void 869 basic_qos(void) 870 { 871 struct spdk_io_channel *io_ch[2]; 872 struct spdk_bdev_channel *bdev_ch[2]; 873 struct spdk_bdev *bdev; 874 enum spdk_bdev_io_status status, abort_status; 875 int rc; 876 877 setup_test(); 878 879 /* Enable QoS */ 880 bdev = &g_bdev.bdev; 881 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 882 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 883 TAILQ_INIT(&bdev->internal.qos->queued); 884 /* 885 * Enable read/write IOPS, read only byte per second and 886 * read/write byte per second rate limits. 887 * In this case, all rate limits will take equal effect. 888 */ 889 /* 2000 read/write I/O per second, or 2 per millisecond */ 890 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 891 /* 8K read/write byte per millisecond with 4K block size */ 892 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 893 /* 8K read only byte per millisecond with 4K block size */ 894 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 8192000; 895 896 g_get_io_channel = true; 897 898 set_thread(0); 899 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 900 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 901 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 902 903 set_thread(1); 904 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 905 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 906 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 907 908 /* 909 * Send an I/O on thread 0, which is where the QoS thread is running. 910 */ 911 set_thread(0); 912 status = SPDK_BDEV_IO_STATUS_PENDING; 913 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status); 914 CU_ASSERT(rc == 0); 915 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 916 poll_threads(); 917 stub_complete_io(g_bdev.io_target, 0); 918 poll_threads(); 919 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 920 921 /* Send an I/O on thread 1. The QoS thread is not running here. */ 922 status = SPDK_BDEV_IO_STATUS_PENDING; 923 set_thread(1); 924 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status); 925 CU_ASSERT(rc == 0); 926 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 927 poll_threads(); 928 /* Complete I/O on thread 1. This should not complete the I/O we submitted */ 929 stub_complete_io(g_bdev.io_target, 0); 930 poll_threads(); 931 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 932 /* Now complete I/O on thread 0 */ 933 set_thread(0); 934 poll_threads(); 935 stub_complete_io(g_bdev.io_target, 0); 936 poll_threads(); 937 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 938 939 /* Reset rate limit for the next test cases. */ 940 spdk_delay_us(SPDK_BDEV_QOS_TIMESLICE_IN_USEC); 941 poll_threads(); 942 943 /* 944 * Test abort request when QoS is enabled. 945 */ 946 947 /* Send an I/O on thread 0, which is where the QoS thread is running. */ 948 set_thread(0); 949 status = SPDK_BDEV_IO_STATUS_PENDING; 950 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status); 951 CU_ASSERT(rc == 0); 952 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 953 /* Send an abort to the I/O on the same thread. */ 954 abort_status = SPDK_BDEV_IO_STATUS_PENDING; 955 rc = spdk_bdev_abort(g_desc, io_ch[0], &status, io_during_io_done, &abort_status); 956 CU_ASSERT(rc == 0); 957 CU_ASSERT(abort_status == SPDK_BDEV_IO_STATUS_PENDING); 958 poll_threads(); 959 CU_ASSERT(abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 960 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_ABORTED); 961 962 /* Send an I/O on thread 1. The QoS thread is not running here. */ 963 status = SPDK_BDEV_IO_STATUS_PENDING; 964 set_thread(1); 965 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status); 966 CU_ASSERT(rc == 0); 967 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 968 poll_threads(); 969 /* Send an abort to the I/O on the same thread. */ 970 abort_status = SPDK_BDEV_IO_STATUS_PENDING; 971 rc = spdk_bdev_abort(g_desc, io_ch[1], &status, io_during_io_done, &abort_status); 972 CU_ASSERT(rc == 0); 973 CU_ASSERT(abort_status == SPDK_BDEV_IO_STATUS_PENDING); 974 poll_threads(); 975 /* Complete the I/O with failure and the abort with success on thread 1. */ 976 CU_ASSERT(abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 977 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_ABORTED); 978 979 set_thread(0); 980 981 /* 982 * Close the descriptor only, which should stop the qos channel as 983 * the last descriptor removed. 984 */ 985 spdk_bdev_close(g_desc); 986 poll_threads(); 987 CU_ASSERT(bdev->internal.qos->ch == NULL); 988 989 /* 990 * Open the bdev again which shall setup the qos channel as the 991 * channels are valid. 992 */ 993 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 994 poll_threads(); 995 CU_ASSERT(bdev->internal.qos->ch != NULL); 996 997 /* Tear down the channels */ 998 set_thread(0); 999 spdk_put_io_channel(io_ch[0]); 1000 set_thread(1); 1001 spdk_put_io_channel(io_ch[1]); 1002 poll_threads(); 1003 set_thread(0); 1004 1005 /* Close the descriptor, which should stop the qos channel */ 1006 spdk_bdev_close(g_desc); 1007 poll_threads(); 1008 CU_ASSERT(bdev->internal.qos->ch == NULL); 1009 1010 /* Open the bdev again, no qos channel setup without valid channels. */ 1011 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 1012 poll_threads(); 1013 CU_ASSERT(bdev->internal.qos->ch == NULL); 1014 1015 /* Create the channels in reverse order. */ 1016 set_thread(1); 1017 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1018 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1019 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 1020 1021 set_thread(0); 1022 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1023 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1024 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 1025 1026 /* Confirm that the qos thread is now thread 1 */ 1027 CU_ASSERT(bdev->internal.qos->ch == bdev_ch[1]); 1028 1029 /* Tear down the channels */ 1030 set_thread(0); 1031 spdk_put_io_channel(io_ch[0]); 1032 set_thread(1); 1033 spdk_put_io_channel(io_ch[1]); 1034 poll_threads(); 1035 1036 set_thread(0); 1037 1038 teardown_test(); 1039 } 1040 1041 static void 1042 io_during_qos_queue(void) 1043 { 1044 struct spdk_io_channel *io_ch[2]; 1045 struct spdk_bdev_channel *bdev_ch[2]; 1046 struct spdk_bdev *bdev; 1047 enum spdk_bdev_io_status status0, status1, status2; 1048 int rc; 1049 1050 setup_test(); 1051 MOCK_SET(spdk_get_ticks, 0); 1052 1053 /* Enable QoS */ 1054 bdev = &g_bdev.bdev; 1055 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 1056 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 1057 TAILQ_INIT(&bdev->internal.qos->queued); 1058 /* 1059 * Enable read/write IOPS, read only byte per sec, write only 1060 * byte per sec and read/write byte per sec rate limits. 1061 * In this case, both read only and write only byte per sec 1062 * rate limit will take effect. 1063 */ 1064 /* 4000 read/write I/O per second, or 4 per millisecond */ 1065 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 4000; 1066 /* 8K byte per millisecond with 4K block size */ 1067 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 1068 /* 4K byte per millisecond with 4K block size */ 1069 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 4096000; 1070 /* 4K byte per millisecond with 4K block size */ 1071 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 4096000; 1072 1073 g_get_io_channel = true; 1074 1075 /* Create channels */ 1076 set_thread(0); 1077 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1078 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1079 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 1080 1081 set_thread(1); 1082 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1083 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1084 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 1085 1086 /* Send two read I/Os */ 1087 status1 = SPDK_BDEV_IO_STATUS_PENDING; 1088 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 1089 CU_ASSERT(rc == 0); 1090 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 1091 set_thread(0); 1092 status0 = SPDK_BDEV_IO_STATUS_PENDING; 1093 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 1094 CU_ASSERT(rc == 0); 1095 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 1096 /* Send one write I/O */ 1097 status2 = SPDK_BDEV_IO_STATUS_PENDING; 1098 rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status2); 1099 CU_ASSERT(rc == 0); 1100 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_PENDING); 1101 1102 /* Complete any I/O that arrived at the disk */ 1103 poll_threads(); 1104 set_thread(1); 1105 stub_complete_io(g_bdev.io_target, 0); 1106 set_thread(0); 1107 stub_complete_io(g_bdev.io_target, 0); 1108 poll_threads(); 1109 1110 /* Only one of the two read I/Os should complete. (logical XOR) */ 1111 if (status0 == SPDK_BDEV_IO_STATUS_SUCCESS) { 1112 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 1113 } else { 1114 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 1115 } 1116 /* The write I/O should complete. */ 1117 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_SUCCESS); 1118 1119 /* Advance in time by a millisecond */ 1120 spdk_delay_us(1000); 1121 1122 /* Complete more I/O */ 1123 poll_threads(); 1124 set_thread(1); 1125 stub_complete_io(g_bdev.io_target, 0); 1126 set_thread(0); 1127 stub_complete_io(g_bdev.io_target, 0); 1128 poll_threads(); 1129 1130 /* Now the second read I/O should be done */ 1131 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 1132 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 1133 1134 /* Tear down the channels */ 1135 set_thread(1); 1136 spdk_put_io_channel(io_ch[1]); 1137 set_thread(0); 1138 spdk_put_io_channel(io_ch[0]); 1139 poll_threads(); 1140 1141 teardown_test(); 1142 } 1143 1144 static void 1145 io_during_qos_reset(void) 1146 { 1147 struct spdk_io_channel *io_ch[2]; 1148 struct spdk_bdev_channel *bdev_ch[2]; 1149 struct spdk_bdev *bdev; 1150 enum spdk_bdev_io_status status0, status1, reset_status; 1151 int rc; 1152 1153 setup_test(); 1154 MOCK_SET(spdk_get_ticks, 0); 1155 1156 /* Enable QoS */ 1157 bdev = &g_bdev.bdev; 1158 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 1159 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 1160 TAILQ_INIT(&bdev->internal.qos->queued); 1161 /* 1162 * Enable read/write IOPS, write only byte per sec and 1163 * read/write byte per second rate limits. 1164 * In this case, read/write byte per second rate limit will 1165 * take effect first. 1166 */ 1167 /* 2000 read/write I/O per second, or 2 per millisecond */ 1168 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 1169 /* 4K byte per millisecond with 4K block size */ 1170 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 4096000; 1171 /* 8K byte per millisecond with 4K block size */ 1172 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 8192000; 1173 1174 g_get_io_channel = true; 1175 1176 /* Create channels */ 1177 set_thread(0); 1178 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1179 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1180 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 1181 1182 set_thread(1); 1183 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1184 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1185 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 1186 1187 /* Send two I/O. One of these gets queued by QoS. The other is sitting at the disk. */ 1188 status1 = SPDK_BDEV_IO_STATUS_PENDING; 1189 rc = spdk_bdev_write_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 1190 CU_ASSERT(rc == 0); 1191 set_thread(0); 1192 status0 = SPDK_BDEV_IO_STATUS_PENDING; 1193 rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 1194 CU_ASSERT(rc == 0); 1195 1196 poll_threads(); 1197 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 1198 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 1199 1200 /* Reset the bdev. */ 1201 reset_status = SPDK_BDEV_IO_STATUS_PENDING; 1202 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &reset_status); 1203 CU_ASSERT(rc == 0); 1204 1205 /* Complete any I/O that arrived at the disk */ 1206 poll_threads(); 1207 set_thread(1); 1208 stub_complete_io(g_bdev.io_target, 0); 1209 set_thread(0); 1210 stub_complete_io(g_bdev.io_target, 0); 1211 poll_threads(); 1212 1213 CU_ASSERT(reset_status == SPDK_BDEV_IO_STATUS_SUCCESS); 1214 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_ABORTED); 1215 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_ABORTED); 1216 1217 /* Tear down the channels */ 1218 set_thread(1); 1219 spdk_put_io_channel(io_ch[1]); 1220 set_thread(0); 1221 spdk_put_io_channel(io_ch[0]); 1222 poll_threads(); 1223 1224 teardown_test(); 1225 } 1226 1227 static void 1228 enomem_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1229 { 1230 enum spdk_bdev_io_status *status = cb_arg; 1231 1232 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 1233 spdk_bdev_free_io(bdev_io); 1234 } 1235 1236 static void 1237 enomem(void) 1238 { 1239 struct spdk_io_channel *io_ch; 1240 struct spdk_bdev_channel *bdev_ch; 1241 struct spdk_bdev_shared_resource *shared_resource; 1242 struct ut_bdev_channel *ut_ch; 1243 const uint32_t IO_ARRAY_SIZE = 64; 1244 const uint32_t AVAIL = 20; 1245 enum spdk_bdev_io_status status[IO_ARRAY_SIZE], status_reset; 1246 uint32_t nomem_cnt, i; 1247 struct spdk_bdev_io *first_io; 1248 int rc; 1249 1250 setup_test(); 1251 1252 set_thread(0); 1253 io_ch = spdk_bdev_get_io_channel(g_desc); 1254 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1255 shared_resource = bdev_ch->shared_resource; 1256 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1257 ut_ch->avail_cnt = AVAIL; 1258 1259 /* First submit a number of IOs equal to what the channel can support. */ 1260 for (i = 0; i < AVAIL; i++) { 1261 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1262 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1263 CU_ASSERT(rc == 0); 1264 } 1265 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1266 1267 /* 1268 * Next, submit one additional I/O. This one should fail with ENOMEM and then go onto 1269 * the enomem_io list. 1270 */ 1271 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1272 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1273 CU_ASSERT(rc == 0); 1274 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1275 first_io = TAILQ_FIRST(&shared_resource->nomem_io); 1276 1277 /* 1278 * Now submit a bunch more I/O. These should all fail with ENOMEM and get queued behind 1279 * the first_io above. 1280 */ 1281 for (i = AVAIL + 1; i < IO_ARRAY_SIZE; i++) { 1282 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1283 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1284 CU_ASSERT(rc == 0); 1285 } 1286 1287 /* Assert that first_io is still at the head of the list. */ 1288 CU_ASSERT(TAILQ_FIRST(&shared_resource->nomem_io) == first_io); 1289 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == (IO_ARRAY_SIZE - AVAIL)); 1290 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 1291 CU_ASSERT(shared_resource->nomem_threshold == (AVAIL - NOMEM_THRESHOLD_COUNT)); 1292 1293 /* 1294 * Complete 1 I/O only. The key check here is bdev_io_tailq_cnt - this should not have 1295 * changed since completing just 1 I/O should not trigger retrying the queued nomem_io 1296 * list. 1297 */ 1298 stub_complete_io(g_bdev.io_target, 1); 1299 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 1300 1301 /* 1302 * Complete enough I/O to hit the nomem_threshold. This should trigger retrying nomem_io, 1303 * and we should see I/O get resubmitted to the test bdev module. 1304 */ 1305 stub_complete_io(g_bdev.io_target, NOMEM_THRESHOLD_COUNT - 1); 1306 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) < nomem_cnt); 1307 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 1308 1309 /* Complete 1 I/O only. This should not trigger retrying the queued nomem_io. */ 1310 stub_complete_io(g_bdev.io_target, 1); 1311 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 1312 1313 /* 1314 * Send a reset and confirm that all I/O are completed, including the ones that 1315 * were queued on the nomem_io list. 1316 */ 1317 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 1318 rc = spdk_bdev_reset(g_desc, io_ch, enomem_done, &status_reset); 1319 poll_threads(); 1320 CU_ASSERT(rc == 0); 1321 /* This will complete the reset. */ 1322 stub_complete_io(g_bdev.io_target, 0); 1323 1324 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == 0); 1325 CU_ASSERT(shared_resource->io_outstanding == 0); 1326 1327 spdk_put_io_channel(io_ch); 1328 poll_threads(); 1329 teardown_test(); 1330 } 1331 1332 static void 1333 enomem_multi_bdev(void) 1334 { 1335 struct spdk_io_channel *io_ch; 1336 struct spdk_bdev_channel *bdev_ch; 1337 struct spdk_bdev_shared_resource *shared_resource; 1338 struct ut_bdev_channel *ut_ch; 1339 const uint32_t IO_ARRAY_SIZE = 64; 1340 const uint32_t AVAIL = 20; 1341 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1342 uint32_t i; 1343 struct ut_bdev *second_bdev; 1344 struct spdk_bdev_desc *second_desc = NULL; 1345 struct spdk_bdev_channel *second_bdev_ch; 1346 struct spdk_io_channel *second_ch; 1347 int rc; 1348 1349 setup_test(); 1350 1351 /* Register second bdev with the same io_target */ 1352 second_bdev = calloc(1, sizeof(*second_bdev)); 1353 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1354 register_bdev(second_bdev, "ut_bdev2", g_bdev.io_target); 1355 spdk_bdev_open_ext("ut_bdev2", true, _bdev_event_cb, NULL, &second_desc); 1356 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1357 1358 set_thread(0); 1359 io_ch = spdk_bdev_get_io_channel(g_desc); 1360 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1361 shared_resource = bdev_ch->shared_resource; 1362 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1363 ut_ch->avail_cnt = AVAIL; 1364 1365 second_ch = spdk_bdev_get_io_channel(second_desc); 1366 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1367 SPDK_CU_ASSERT_FATAL(shared_resource == second_bdev_ch->shared_resource); 1368 1369 /* Saturate io_target through bdev A. */ 1370 for (i = 0; i < AVAIL; i++) { 1371 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1372 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1373 CU_ASSERT(rc == 0); 1374 } 1375 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1376 1377 /* 1378 * Now submit I/O through the second bdev. This should fail with ENOMEM 1379 * and then go onto the nomem_io list. 1380 */ 1381 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1382 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1383 CU_ASSERT(rc == 0); 1384 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1385 1386 /* Complete first bdev's I/O. This should retry sending second bdev's nomem_io */ 1387 stub_complete_io(g_bdev.io_target, AVAIL); 1388 1389 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&shared_resource->nomem_io)); 1390 CU_ASSERT(shared_resource->io_outstanding == 1); 1391 1392 /* Now complete our retried I/O */ 1393 stub_complete_io(g_bdev.io_target, 1); 1394 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == 0); 1395 1396 spdk_put_io_channel(io_ch); 1397 spdk_put_io_channel(second_ch); 1398 spdk_bdev_close(second_desc); 1399 unregister_bdev(second_bdev); 1400 poll_threads(); 1401 free(second_bdev); 1402 teardown_test(); 1403 } 1404 1405 static void 1406 enomem_multi_bdev_unregister(void) 1407 { 1408 struct spdk_io_channel *io_ch; 1409 struct spdk_bdev_channel *bdev_ch; 1410 struct spdk_bdev_shared_resource *shared_resource; 1411 struct ut_bdev_channel *ut_ch; 1412 const uint32_t IO_ARRAY_SIZE = 64; 1413 const uint32_t AVAIL = 20; 1414 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1415 uint32_t i; 1416 int rc; 1417 1418 setup_test(); 1419 1420 set_thread(0); 1421 io_ch = spdk_bdev_get_io_channel(g_desc); 1422 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1423 shared_resource = bdev_ch->shared_resource; 1424 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1425 ut_ch->avail_cnt = AVAIL; 1426 1427 /* Saturate io_target through the bdev. */ 1428 for (i = 0; i < AVAIL; i++) { 1429 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1430 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1431 CU_ASSERT(rc == 0); 1432 } 1433 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1434 1435 /* 1436 * Now submit I/O through the bdev. This should fail with ENOMEM 1437 * and then go onto the nomem_io list. 1438 */ 1439 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1440 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1441 CU_ASSERT(rc == 0); 1442 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1443 1444 /* Unregister the bdev to abort the IOs from nomem_io queue. */ 1445 unregister_bdev(&g_bdev); 1446 CU_ASSERT(status[AVAIL] == SPDK_BDEV_IO_STATUS_FAILED); 1447 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&shared_resource->nomem_io)); 1448 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == AVAIL); 1449 1450 /* Complete the bdev's I/O. */ 1451 stub_complete_io(g_bdev.io_target, AVAIL); 1452 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == 0); 1453 1454 spdk_put_io_channel(io_ch); 1455 poll_threads(); 1456 teardown_test(); 1457 } 1458 1459 static void 1460 enomem_multi_io_target(void) 1461 { 1462 struct spdk_io_channel *io_ch; 1463 struct spdk_bdev_channel *bdev_ch; 1464 struct ut_bdev_channel *ut_ch; 1465 const uint32_t IO_ARRAY_SIZE = 64; 1466 const uint32_t AVAIL = 20; 1467 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1468 uint32_t i; 1469 int new_io_device; 1470 struct ut_bdev *second_bdev; 1471 struct spdk_bdev_desc *second_desc = NULL; 1472 struct spdk_bdev_channel *second_bdev_ch; 1473 struct spdk_io_channel *second_ch; 1474 int rc; 1475 1476 setup_test(); 1477 1478 /* Create new io_target and a second bdev using it */ 1479 spdk_io_device_register(&new_io_device, stub_create_ch, stub_destroy_ch, 1480 sizeof(struct ut_bdev_channel), NULL); 1481 second_bdev = calloc(1, sizeof(*second_bdev)); 1482 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1483 register_bdev(second_bdev, "ut_bdev2", &new_io_device); 1484 spdk_bdev_open_ext("ut_bdev2", true, _bdev_event_cb, NULL, &second_desc); 1485 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1486 1487 set_thread(0); 1488 io_ch = spdk_bdev_get_io_channel(g_desc); 1489 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1490 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1491 ut_ch->avail_cnt = AVAIL; 1492 1493 /* Different io_target should imply a different shared_resource */ 1494 second_ch = spdk_bdev_get_io_channel(second_desc); 1495 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1496 SPDK_CU_ASSERT_FATAL(bdev_ch->shared_resource != second_bdev_ch->shared_resource); 1497 1498 /* Saturate io_target through bdev A. */ 1499 for (i = 0; i < AVAIL; i++) { 1500 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1501 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1502 CU_ASSERT(rc == 0); 1503 } 1504 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1505 1506 /* Issue one more I/O to fill ENOMEM list. */ 1507 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1508 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1509 CU_ASSERT(rc == 0); 1510 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1511 1512 /* 1513 * Now submit I/O through the second bdev. This should go through and complete 1514 * successfully because we're using a different io_device underneath. 1515 */ 1516 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1517 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1518 CU_ASSERT(rc == 0); 1519 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&second_bdev_ch->shared_resource->nomem_io)); 1520 stub_complete_io(second_bdev->io_target, 1); 1521 1522 /* Cleanup; Complete outstanding I/O. */ 1523 stub_complete_io(g_bdev.io_target, AVAIL); 1524 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1525 /* Complete the ENOMEM I/O */ 1526 stub_complete_io(g_bdev.io_target, 1); 1527 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1528 1529 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1530 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1531 spdk_put_io_channel(io_ch); 1532 spdk_put_io_channel(second_ch); 1533 spdk_bdev_close(second_desc); 1534 unregister_bdev(second_bdev); 1535 spdk_io_device_unregister(&new_io_device, NULL); 1536 poll_threads(); 1537 free(second_bdev); 1538 teardown_test(); 1539 } 1540 1541 static void 1542 qos_dynamic_enable_done(void *cb_arg, int status) 1543 { 1544 int *rc = cb_arg; 1545 *rc = status; 1546 } 1547 1548 static void 1549 qos_dynamic_enable(void) 1550 { 1551 struct spdk_io_channel *io_ch[2]; 1552 struct spdk_bdev_channel *bdev_ch[2]; 1553 struct spdk_bdev *bdev; 1554 enum spdk_bdev_io_status bdev_io_status[2]; 1555 uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {}; 1556 int status, second_status, rc, i; 1557 1558 setup_test(); 1559 MOCK_SET(spdk_get_ticks, 0); 1560 1561 for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) { 1562 limits[i] = UINT64_MAX; 1563 } 1564 1565 bdev = &g_bdev.bdev; 1566 1567 g_get_io_channel = true; 1568 1569 /* Create channels */ 1570 set_thread(0); 1571 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1572 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1573 CU_ASSERT(bdev_ch[0]->flags == 0); 1574 1575 set_thread(1); 1576 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1577 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1578 CU_ASSERT(bdev_ch[1]->flags == 0); 1579 1580 set_thread(0); 1581 1582 /* 1583 * Enable QoS: Read/Write IOPS, Read/Write byte, 1584 * Read only byte and Write only byte per second 1585 * rate limits. 1586 * More than 10 I/Os allowed per timeslice. 1587 */ 1588 status = -1; 1589 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1590 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 100; 1591 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 100; 1592 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 10; 1593 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1594 poll_threads(); 1595 CU_ASSERT(status == 0); 1596 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1597 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1598 1599 /* 1600 * Submit and complete 10 I/O to fill the QoS allotment for this timeslice. 1601 * Additional I/O will then be queued. 1602 */ 1603 set_thread(0); 1604 for (i = 0; i < 10; i++) { 1605 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1606 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1607 CU_ASSERT(rc == 0); 1608 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1609 poll_thread(0); 1610 stub_complete_io(g_bdev.io_target, 0); 1611 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1612 } 1613 1614 /* 1615 * Send two more I/O. These I/O will be queued since the current timeslice allotment has been 1616 * filled already. We want to test that when QoS is disabled that these two I/O: 1617 * 1) are not aborted 1618 * 2) are sent back to their original thread for resubmission 1619 */ 1620 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1621 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1622 CU_ASSERT(rc == 0); 1623 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1624 set_thread(1); 1625 bdev_io_status[1] = SPDK_BDEV_IO_STATUS_PENDING; 1626 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &bdev_io_status[1]); 1627 CU_ASSERT(rc == 0); 1628 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1629 poll_threads(); 1630 1631 /* 1632 * Disable QoS: Read/Write IOPS, Read/Write byte, 1633 * Read only byte rate limits 1634 */ 1635 status = -1; 1636 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1637 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 0; 1638 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 0; 1639 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1640 poll_threads(); 1641 CU_ASSERT(status == 0); 1642 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1643 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1644 1645 /* Disable QoS: Write only Byte per second rate limit */ 1646 status = -1; 1647 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 0; 1648 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1649 poll_threads(); 1650 CU_ASSERT(status == 0); 1651 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1652 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1653 1654 /* 1655 * All I/O should have been resubmitted back on their original thread. Complete 1656 * all I/O on thread 0, and ensure that only the thread 0 I/O was completed. 1657 */ 1658 set_thread(0); 1659 stub_complete_io(g_bdev.io_target, 0); 1660 poll_threads(); 1661 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1662 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1663 1664 /* Now complete all I/O on thread 1 and ensure the thread 1 I/O was completed. */ 1665 set_thread(1); 1666 stub_complete_io(g_bdev.io_target, 0); 1667 poll_threads(); 1668 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_SUCCESS); 1669 1670 /* Disable QoS again */ 1671 status = -1; 1672 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1673 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1674 poll_threads(); 1675 CU_ASSERT(status == 0); /* This should succeed */ 1676 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1677 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1678 1679 /* Enable QoS on thread 0 */ 1680 status = -1; 1681 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1682 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1683 poll_threads(); 1684 CU_ASSERT(status == 0); 1685 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1686 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1687 1688 /* Disable QoS on thread 1 */ 1689 set_thread(1); 1690 status = -1; 1691 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1692 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1693 /* Don't poll yet. This should leave the channels with QoS enabled */ 1694 CU_ASSERT(status == -1); 1695 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1696 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1697 1698 /* Enable QoS. This should immediately fail because the previous disable QoS hasn't completed. */ 1699 second_status = 0; 1700 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 10; 1701 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &second_status); 1702 poll_threads(); 1703 CU_ASSERT(status == 0); /* The disable should succeed */ 1704 CU_ASSERT(second_status < 0); /* The enable should fail */ 1705 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1706 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1707 1708 /* Enable QoS on thread 1. This should succeed now that the disable has completed. */ 1709 status = -1; 1710 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1711 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1712 poll_threads(); 1713 CU_ASSERT(status == 0); 1714 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1715 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1716 1717 /* Tear down the channels */ 1718 set_thread(0); 1719 spdk_put_io_channel(io_ch[0]); 1720 set_thread(1); 1721 spdk_put_io_channel(io_ch[1]); 1722 poll_threads(); 1723 1724 set_thread(0); 1725 teardown_test(); 1726 } 1727 1728 static void 1729 histogram_status_cb(void *cb_arg, int status) 1730 { 1731 g_status = status; 1732 } 1733 1734 static void 1735 histogram_data_cb(void *cb_arg, int status, struct spdk_histogram_data *histogram) 1736 { 1737 g_status = status; 1738 g_histogram = histogram; 1739 } 1740 1741 static void 1742 histogram_io_count(void *ctx, uint64_t start, uint64_t end, uint64_t count, 1743 uint64_t total, uint64_t so_far) 1744 { 1745 g_count += count; 1746 } 1747 1748 static void 1749 bdev_histograms_mt(void) 1750 { 1751 struct spdk_io_channel *ch[2]; 1752 struct spdk_histogram_data *histogram; 1753 uint8_t buf[4096]; 1754 int status = false; 1755 int rc; 1756 1757 1758 setup_test(); 1759 1760 set_thread(0); 1761 ch[0] = spdk_bdev_get_io_channel(g_desc); 1762 CU_ASSERT(ch[0] != NULL); 1763 1764 set_thread(1); 1765 ch[1] = spdk_bdev_get_io_channel(g_desc); 1766 CU_ASSERT(ch[1] != NULL); 1767 1768 1769 /* Enable histogram */ 1770 spdk_bdev_histogram_enable(&g_bdev.bdev, histogram_status_cb, NULL, true); 1771 poll_threads(); 1772 CU_ASSERT(g_status == 0); 1773 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == true); 1774 1775 /* Allocate histogram */ 1776 histogram = spdk_histogram_data_alloc(); 1777 1778 /* Check if histogram is zeroed */ 1779 spdk_bdev_histogram_get(&g_bdev.bdev, histogram, histogram_data_cb, NULL); 1780 poll_threads(); 1781 CU_ASSERT(g_status == 0); 1782 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 1783 1784 g_count = 0; 1785 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 1786 1787 CU_ASSERT(g_count == 0); 1788 1789 set_thread(0); 1790 rc = spdk_bdev_write_blocks(g_desc, ch[0], &buf, 0, 1, io_during_io_done, &status); 1791 CU_ASSERT(rc == 0); 1792 1793 spdk_delay_us(10); 1794 stub_complete_io(g_bdev.io_target, 1); 1795 poll_threads(); 1796 CU_ASSERT(status == true); 1797 1798 1799 set_thread(1); 1800 rc = spdk_bdev_read_blocks(g_desc, ch[1], &buf, 0, 1, io_during_io_done, &status); 1801 CU_ASSERT(rc == 0); 1802 1803 spdk_delay_us(10); 1804 stub_complete_io(g_bdev.io_target, 1); 1805 poll_threads(); 1806 CU_ASSERT(status == true); 1807 1808 set_thread(0); 1809 1810 /* Check if histogram gathered data from all I/O channels */ 1811 spdk_bdev_histogram_get(&g_bdev.bdev, histogram, histogram_data_cb, NULL); 1812 poll_threads(); 1813 CU_ASSERT(g_status == 0); 1814 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == true); 1815 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 1816 1817 g_count = 0; 1818 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 1819 CU_ASSERT(g_count == 2); 1820 1821 /* Disable histogram */ 1822 spdk_bdev_histogram_enable(&g_bdev.bdev, histogram_status_cb, NULL, false); 1823 poll_threads(); 1824 CU_ASSERT(g_status == 0); 1825 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == false); 1826 1827 spdk_histogram_data_free(histogram); 1828 1829 /* Tear down the channels */ 1830 set_thread(0); 1831 spdk_put_io_channel(ch[0]); 1832 set_thread(1); 1833 spdk_put_io_channel(ch[1]); 1834 poll_threads(); 1835 set_thread(0); 1836 teardown_test(); 1837 1838 } 1839 1840 struct timeout_io_cb_arg { 1841 struct iovec iov; 1842 uint8_t type; 1843 }; 1844 1845 static int 1846 bdev_channel_count_submitted_io(struct spdk_bdev_channel *ch) 1847 { 1848 struct spdk_bdev_io *bdev_io; 1849 int n = 0; 1850 1851 if (!ch) { 1852 return -1; 1853 } 1854 1855 TAILQ_FOREACH(bdev_io, &ch->io_submitted, internal.ch_link) { 1856 n++; 1857 } 1858 1859 return n; 1860 } 1861 1862 static void 1863 bdev_channel_io_timeout_cb(void *cb_arg, struct spdk_bdev_io *bdev_io) 1864 { 1865 struct timeout_io_cb_arg *ctx = cb_arg; 1866 1867 ctx->type = bdev_io->type; 1868 ctx->iov.iov_base = bdev_io->iov.iov_base; 1869 ctx->iov.iov_len = bdev_io->iov.iov_len; 1870 } 1871 1872 static bool g_io_done; 1873 1874 static void 1875 io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1876 { 1877 g_io_done = true; 1878 spdk_bdev_free_io(bdev_io); 1879 } 1880 1881 static void 1882 bdev_set_io_timeout_mt(void) 1883 { 1884 struct spdk_io_channel *ch[3]; 1885 struct spdk_bdev_channel *bdev_ch[3]; 1886 struct timeout_io_cb_arg cb_arg; 1887 1888 setup_test(); 1889 1890 g_bdev.bdev.optimal_io_boundary = 16; 1891 g_bdev.bdev.split_on_optimal_io_boundary = true; 1892 1893 set_thread(0); 1894 ch[0] = spdk_bdev_get_io_channel(g_desc); 1895 CU_ASSERT(ch[0] != NULL); 1896 1897 set_thread(1); 1898 ch[1] = spdk_bdev_get_io_channel(g_desc); 1899 CU_ASSERT(ch[1] != NULL); 1900 1901 set_thread(2); 1902 ch[2] = spdk_bdev_get_io_channel(g_desc); 1903 CU_ASSERT(ch[2] != NULL); 1904 1905 /* Multi-thread mode 1906 * 1, Check the poller was registered successfully 1907 * 2, Check the timeout IO and ensure the IO was the submitted by user 1908 * 3, Check the link int the bdev_ch works right. 1909 * 4, Close desc and put io channel during the timeout poller is polling 1910 */ 1911 1912 /* In desc thread set the timeout */ 1913 set_thread(0); 1914 CU_ASSERT(spdk_bdev_set_timeout(g_desc, 5, bdev_channel_io_timeout_cb, &cb_arg) == 0); 1915 CU_ASSERT(g_desc->io_timeout_poller != NULL); 1916 CU_ASSERT(g_desc->cb_fn == bdev_channel_io_timeout_cb); 1917 CU_ASSERT(g_desc->cb_arg == &cb_arg); 1918 1919 /* check the IO submitted list and timeout handler */ 1920 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[0], (void *)0x2000, 0, 1, io_done, NULL) == 0); 1921 bdev_ch[0] = spdk_io_channel_get_ctx(ch[0]); 1922 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[0]) == 1); 1923 1924 set_thread(1); 1925 CU_ASSERT(spdk_bdev_write_blocks(g_desc, ch[1], (void *)0x1000, 0, 1, io_done, NULL) == 0); 1926 bdev_ch[1] = spdk_io_channel_get_ctx(ch[1]); 1927 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[1]) == 1); 1928 1929 /* Now test that a single-vector command is split correctly. 1930 * Offset 14, length 8, payload 0xF000 1931 * Child - Offset 14, length 2, payload 0xF000 1932 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 1933 * 1934 * Set up the expected values before calling spdk_bdev_read_blocks 1935 */ 1936 set_thread(2); 1937 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[2], (void *)0xF000, 14, 8, io_done, NULL) == 0); 1938 bdev_ch[2] = spdk_io_channel_get_ctx(ch[2]); 1939 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[2]) == 3); 1940 1941 set_thread(0); 1942 memset(&cb_arg, 0, sizeof(cb_arg)); 1943 spdk_delay_us(3 * spdk_get_ticks_hz()); 1944 poll_threads(); 1945 CU_ASSERT(cb_arg.type == 0); 1946 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 1947 CU_ASSERT(cb_arg.iov.iov_len == 0); 1948 1949 /* Now the time reach the limit */ 1950 spdk_delay_us(3 * spdk_get_ticks_hz()); 1951 poll_thread(0); 1952 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_READ); 1953 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x2000); 1954 CU_ASSERT(cb_arg.iov.iov_len == 1 * g_bdev.bdev.blocklen); 1955 stub_complete_io(g_bdev.io_target, 1); 1956 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[0]) == 0); 1957 1958 memset(&cb_arg, 0, sizeof(cb_arg)); 1959 set_thread(1); 1960 poll_thread(1); 1961 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 1962 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x1000); 1963 CU_ASSERT(cb_arg.iov.iov_len == 1 * g_bdev.bdev.blocklen); 1964 stub_complete_io(g_bdev.io_target, 1); 1965 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[1]) == 0); 1966 1967 memset(&cb_arg, 0, sizeof(cb_arg)); 1968 set_thread(2); 1969 poll_thread(2); 1970 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_READ); 1971 CU_ASSERT(cb_arg.iov.iov_base == (void *)0xF000); 1972 CU_ASSERT(cb_arg.iov.iov_len == 8 * g_bdev.bdev.blocklen); 1973 stub_complete_io(g_bdev.io_target, 1); 1974 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[2]) == 2); 1975 stub_complete_io(g_bdev.io_target, 1); 1976 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[2]) == 0); 1977 1978 /* Run poll_timeout_done() it means complete the timeout poller */ 1979 set_thread(0); 1980 poll_thread(0); 1981 CU_ASSERT(g_desc->refs == 0); 1982 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[0], (void *)0x1000, 0, 1, io_done, NULL) == 0); 1983 set_thread(1); 1984 CU_ASSERT(spdk_bdev_write_blocks(g_desc, ch[1], (void *)0x2000, 0, 2, io_done, NULL) == 0); 1985 set_thread(2); 1986 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[2], (void *)0x3000, 0, 3, io_done, NULL) == 0); 1987 1988 /* Trigger timeout poller to run again, desc->refs is incremented. 1989 * In thread 0 we destroy the io channel before timeout poller runs. 1990 * Timeout callback is not called on thread 0. 1991 */ 1992 spdk_delay_us(6 * spdk_get_ticks_hz()); 1993 memset(&cb_arg, 0, sizeof(cb_arg)); 1994 set_thread(0); 1995 stub_complete_io(g_bdev.io_target, 1); 1996 spdk_put_io_channel(ch[0]); 1997 poll_thread(0); 1998 CU_ASSERT(g_desc->refs == 1) 1999 CU_ASSERT(cb_arg.type == 0); 2000 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 2001 CU_ASSERT(cb_arg.iov.iov_len == 0); 2002 2003 /* In thread 1 timeout poller runs then we destroy the io channel 2004 * Timeout callback is called on thread 1. 2005 */ 2006 memset(&cb_arg, 0, sizeof(cb_arg)); 2007 set_thread(1); 2008 poll_thread(1); 2009 stub_complete_io(g_bdev.io_target, 1); 2010 spdk_put_io_channel(ch[1]); 2011 poll_thread(1); 2012 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 2013 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x2000); 2014 CU_ASSERT(cb_arg.iov.iov_len == 2 * g_bdev.bdev.blocklen); 2015 2016 /* Close the desc. 2017 * Unregister the timeout poller first. 2018 * Then decrement desc->refs but it's not zero yet so desc is not freed. 2019 */ 2020 set_thread(0); 2021 spdk_bdev_close(g_desc); 2022 CU_ASSERT(g_desc->refs == 1); 2023 CU_ASSERT(g_desc->io_timeout_poller == NULL); 2024 2025 /* Timeout poller runs on thread 2 then we destroy the io channel. 2026 * Desc is closed so we would exit the timeout poller directly. 2027 * timeout callback is not called on thread 2. 2028 */ 2029 memset(&cb_arg, 0, sizeof(cb_arg)); 2030 set_thread(2); 2031 poll_thread(2); 2032 stub_complete_io(g_bdev.io_target, 1); 2033 spdk_put_io_channel(ch[2]); 2034 poll_thread(2); 2035 CU_ASSERT(cb_arg.type == 0); 2036 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 2037 CU_ASSERT(cb_arg.iov.iov_len == 0); 2038 2039 set_thread(0); 2040 poll_thread(0); 2041 g_teardown_done = false; 2042 unregister_bdev(&g_bdev); 2043 spdk_io_device_unregister(&g_io_device, NULL); 2044 spdk_bdev_finish(finish_cb, NULL); 2045 poll_threads(); 2046 memset(&g_bdev, 0, sizeof(g_bdev)); 2047 CU_ASSERT(g_teardown_done == true); 2048 g_teardown_done = false; 2049 free_threads(); 2050 free_cores(); 2051 } 2052 2053 static bool g_io_done2; 2054 static bool g_lock_lba_range_done; 2055 static bool g_unlock_lba_range_done; 2056 2057 static void 2058 io_done2(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 2059 { 2060 g_io_done2 = true; 2061 spdk_bdev_free_io(bdev_io); 2062 } 2063 2064 static void 2065 lock_lba_range_done(void *ctx, int status) 2066 { 2067 g_lock_lba_range_done = true; 2068 } 2069 2070 static void 2071 unlock_lba_range_done(void *ctx, int status) 2072 { 2073 g_unlock_lba_range_done = true; 2074 } 2075 2076 static uint32_t 2077 stub_channel_outstanding_cnt(void *io_target) 2078 { 2079 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 2080 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 2081 uint32_t outstanding_cnt; 2082 2083 outstanding_cnt = ch->outstanding_cnt; 2084 2085 spdk_put_io_channel(_ch); 2086 return outstanding_cnt; 2087 } 2088 2089 static void 2090 lock_lba_range_then_submit_io(void) 2091 { 2092 struct spdk_bdev_desc *desc = NULL; 2093 void *io_target; 2094 struct spdk_io_channel *io_ch[3]; 2095 struct spdk_bdev_channel *bdev_ch[3]; 2096 struct lba_range *range; 2097 char buf[4096]; 2098 int ctx0, ctx1, ctx2; 2099 int rc; 2100 2101 setup_test(); 2102 2103 io_target = g_bdev.io_target; 2104 desc = g_desc; 2105 2106 set_thread(0); 2107 io_ch[0] = spdk_bdev_get_io_channel(desc); 2108 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 2109 CU_ASSERT(io_ch[0] != NULL); 2110 2111 set_thread(1); 2112 io_ch[1] = spdk_bdev_get_io_channel(desc); 2113 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 2114 CU_ASSERT(io_ch[1] != NULL); 2115 2116 set_thread(0); 2117 g_lock_lba_range_done = false; 2118 rc = bdev_lock_lba_range(desc, io_ch[0], 20, 10, lock_lba_range_done, &ctx0); 2119 CU_ASSERT(rc == 0); 2120 poll_threads(); 2121 2122 /* The lock should immediately become valid, since there are no outstanding 2123 * write I/O. 2124 */ 2125 CU_ASSERT(g_lock_lba_range_done == true); 2126 range = TAILQ_FIRST(&bdev_ch[0]->locked_ranges); 2127 SPDK_CU_ASSERT_FATAL(range != NULL); 2128 CU_ASSERT(range->offset == 20); 2129 CU_ASSERT(range->length == 10); 2130 CU_ASSERT(range->owner_ch == bdev_ch[0]); 2131 2132 g_io_done = false; 2133 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 2134 rc = spdk_bdev_read_blocks(desc, io_ch[0], buf, 20, 1, io_done, &ctx0); 2135 CU_ASSERT(rc == 0); 2136 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 2137 2138 stub_complete_io(io_target, 1); 2139 poll_threads(); 2140 CU_ASSERT(g_io_done == true); 2141 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 2142 2143 /* Try a write I/O. This should actually be allowed to execute, since the channel 2144 * holding the lock is submitting the write I/O. 2145 */ 2146 g_io_done = false; 2147 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 2148 rc = spdk_bdev_write_blocks(desc, io_ch[0], buf, 20, 1, io_done, &ctx0); 2149 CU_ASSERT(rc == 0); 2150 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 2151 2152 stub_complete_io(io_target, 1); 2153 poll_threads(); 2154 CU_ASSERT(g_io_done == true); 2155 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 2156 2157 /* Try a write I/O. This should get queued in the io_locked tailq. */ 2158 set_thread(1); 2159 g_io_done = false; 2160 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[1]->io_locked)); 2161 rc = spdk_bdev_write_blocks(desc, io_ch[1], buf, 20, 1, io_done, &ctx1); 2162 CU_ASSERT(rc == 0); 2163 poll_threads(); 2164 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 0); 2165 CU_ASSERT(!TAILQ_EMPTY(&bdev_ch[1]->io_locked)); 2166 CU_ASSERT(g_io_done == false); 2167 2168 /* Try to unlock the lba range using thread 1's io_ch. This should fail. */ 2169 rc = bdev_unlock_lba_range(desc, io_ch[1], 20, 10, unlock_lba_range_done, &ctx1); 2170 CU_ASSERT(rc == -EINVAL); 2171 2172 /* Now create a new channel and submit a write I/O with it. This should also be queued. 2173 * The new channel should inherit the active locks from the bdev's internal list. 2174 */ 2175 set_thread(2); 2176 io_ch[2] = spdk_bdev_get_io_channel(desc); 2177 bdev_ch[2] = spdk_io_channel_get_ctx(io_ch[2]); 2178 CU_ASSERT(io_ch[2] != NULL); 2179 2180 g_io_done2 = false; 2181 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[2]->io_locked)); 2182 rc = spdk_bdev_write_blocks(desc, io_ch[2], buf, 22, 2, io_done2, &ctx2); 2183 CU_ASSERT(rc == 0); 2184 poll_threads(); 2185 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 0); 2186 CU_ASSERT(!TAILQ_EMPTY(&bdev_ch[2]->io_locked)); 2187 CU_ASSERT(g_io_done2 == false); 2188 2189 set_thread(0); 2190 rc = bdev_unlock_lba_range(desc, io_ch[0], 20, 10, unlock_lba_range_done, &ctx0); 2191 CU_ASSERT(rc == 0); 2192 poll_threads(); 2193 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->locked_ranges)); 2194 2195 /* The LBA range is unlocked, so the write IOs should now have started execution. */ 2196 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[1]->io_locked)); 2197 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[2]->io_locked)); 2198 2199 set_thread(1); 2200 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 2201 stub_complete_io(io_target, 1); 2202 set_thread(2); 2203 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 2204 stub_complete_io(io_target, 1); 2205 2206 poll_threads(); 2207 CU_ASSERT(g_io_done == true); 2208 CU_ASSERT(g_io_done2 == true); 2209 2210 /* Tear down the channels */ 2211 set_thread(0); 2212 spdk_put_io_channel(io_ch[0]); 2213 set_thread(1); 2214 spdk_put_io_channel(io_ch[1]); 2215 set_thread(2); 2216 spdk_put_io_channel(io_ch[2]); 2217 poll_threads(); 2218 set_thread(0); 2219 teardown_test(); 2220 } 2221 2222 /* spdk_bdev_reset() freezes and unfreezes I/O channels by using spdk_for_each_channel(). 2223 * spdk_bdev_unregister() calls spdk_io_device_unregister() in the end. However 2224 * spdk_io_device_unregister() fails if it is called while executing spdk_for_each_channel(). 2225 * Hence, in this case, spdk_io_device_unregister() is deferred until spdk_bdev_reset() 2226 * completes. Test this behavior. 2227 */ 2228 static void 2229 unregister_during_reset(void) 2230 { 2231 struct spdk_io_channel *io_ch[2]; 2232 bool done_reset = false, done_unregister = false; 2233 int rc; 2234 2235 setup_test(); 2236 set_thread(0); 2237 2238 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 2239 SPDK_CU_ASSERT_FATAL(io_ch[0] != NULL); 2240 2241 set_thread(1); 2242 2243 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 2244 SPDK_CU_ASSERT_FATAL(io_ch[1] != NULL); 2245 2246 set_thread(0); 2247 2248 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 2249 2250 rc = spdk_bdev_reset(g_desc, io_ch[0], reset_done, &done_reset); 2251 CU_ASSERT(rc == 0); 2252 2253 set_thread(0); 2254 2255 poll_thread_times(0, 1); 2256 2257 spdk_bdev_close(g_desc); 2258 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done_unregister); 2259 2260 CU_ASSERT(done_reset == false); 2261 CU_ASSERT(done_unregister == false); 2262 2263 poll_threads(); 2264 2265 stub_complete_io(g_bdev.io_target, 0); 2266 2267 poll_threads(); 2268 2269 CU_ASSERT(done_reset == true); 2270 CU_ASSERT(done_unregister == false); 2271 2272 spdk_put_io_channel(io_ch[0]); 2273 2274 set_thread(1); 2275 2276 spdk_put_io_channel(io_ch[1]); 2277 2278 poll_threads(); 2279 2280 CU_ASSERT(done_unregister == true); 2281 2282 /* Restore the original g_bdev so that we can use teardown_test(). */ 2283 set_thread(0); 2284 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 2285 spdk_bdev_open_ext("ut_bdev", true, _bdev_event_cb, NULL, &g_desc); 2286 teardown_test(); 2287 } 2288 2289 int 2290 main(int argc, char **argv) 2291 { 2292 CU_pSuite suite = NULL; 2293 unsigned int num_failures; 2294 2295 CU_set_error_action(CUEA_ABORT); 2296 CU_initialize_registry(); 2297 2298 suite = CU_add_suite("bdev", NULL, NULL); 2299 2300 CU_ADD_TEST(suite, basic); 2301 CU_ADD_TEST(suite, unregister_and_close); 2302 CU_ADD_TEST(suite, basic_qos); 2303 CU_ADD_TEST(suite, put_channel_during_reset); 2304 CU_ADD_TEST(suite, aborted_reset); 2305 CU_ADD_TEST(suite, aborted_reset_no_outstanding_io); 2306 CU_ADD_TEST(suite, io_during_reset); 2307 CU_ADD_TEST(suite, reset_completions); 2308 CU_ADD_TEST(suite, io_during_qos_queue); 2309 CU_ADD_TEST(suite, io_during_qos_reset); 2310 CU_ADD_TEST(suite, enomem); 2311 CU_ADD_TEST(suite, enomem_multi_bdev); 2312 CU_ADD_TEST(suite, enomem_multi_bdev_unregister); 2313 CU_ADD_TEST(suite, enomem_multi_io_target); 2314 CU_ADD_TEST(suite, qos_dynamic_enable); 2315 CU_ADD_TEST(suite, bdev_histograms_mt); 2316 CU_ADD_TEST(suite, bdev_set_io_timeout_mt); 2317 CU_ADD_TEST(suite, lock_lba_range_then_submit_io); 2318 CU_ADD_TEST(suite, unregister_during_reset); 2319 2320 CU_basic_set_mode(CU_BRM_VERBOSE); 2321 CU_basic_run_tests(); 2322 num_failures = CU_get_number_of_failures(); 2323 CU_cleanup_registry(); 2324 return num_failures; 2325 } 2326