1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk_cunit.h" 35 36 #include "common/lib/ut_multithread.c" 37 #include "unit/lib/json_mock.c" 38 39 #include "spdk/config.h" 40 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 41 #undef SPDK_CONFIG_VTUNE 42 43 #include "bdev/bdev.c" 44 45 #define BDEV_UT_NUM_THREADS 3 46 47 DEFINE_STUB(spdk_conf_find_section, struct spdk_conf_section *, (struct spdk_conf *cp, 48 const char *name), NULL); 49 DEFINE_STUB(spdk_conf_section_get_nmval, char *, 50 (struct spdk_conf_section *sp, const char *key, int idx1, int idx2), NULL); 51 DEFINE_STUB(spdk_conf_section_get_intval, int, (struct spdk_conf_section *sp, const char *key), -1); 52 53 struct spdk_trace_histories *g_trace_histories; 54 DEFINE_STUB_V(spdk_trace_add_register_fn, (struct spdk_trace_register_fn *reg_fn)); 55 DEFINE_STUB_V(spdk_trace_register_owner, (uint8_t type, char id_prefix)); 56 DEFINE_STUB_V(spdk_trace_register_object, (uint8_t type, char id_prefix)); 57 DEFINE_STUB_V(spdk_trace_register_description, (const char *name, 58 uint16_t tpoint_id, uint8_t owner_type, 59 uint8_t object_type, uint8_t new_object, 60 uint8_t arg1_type, const char *arg1_name)); 61 DEFINE_STUB_V(_spdk_trace_record, (uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, 62 uint32_t size, uint64_t object_id, uint64_t arg1)); 63 DEFINE_STUB(spdk_notify_send, uint64_t, (const char *type, const char *ctx), 0); 64 DEFINE_STUB(spdk_notify_type_register, struct spdk_notify_type *, (const char *type), NULL); 65 66 struct ut_bdev { 67 struct spdk_bdev bdev; 68 void *io_target; 69 }; 70 71 struct ut_bdev_channel { 72 TAILQ_HEAD(, spdk_bdev_io) outstanding_io; 73 uint32_t outstanding_cnt; 74 uint32_t avail_cnt; 75 }; 76 77 int g_io_device; 78 struct ut_bdev g_bdev; 79 struct spdk_bdev_desc *g_desc; 80 bool g_teardown_done = false; 81 bool g_get_io_channel = true; 82 bool g_create_ch = true; 83 bool g_init_complete_called = false; 84 bool g_fini_start_called = true; 85 int g_status = 0; 86 int g_count = 0; 87 struct spdk_histogram_data *g_histogram = NULL; 88 89 static int 90 stub_create_ch(void *io_device, void *ctx_buf) 91 { 92 struct ut_bdev_channel *ch = ctx_buf; 93 94 if (g_create_ch == false) { 95 return -1; 96 } 97 98 TAILQ_INIT(&ch->outstanding_io); 99 ch->outstanding_cnt = 0; 100 /* 101 * When avail gets to 0, the submit_request function will return ENOMEM. 102 * Most tests to not want ENOMEM to occur, so by default set this to a 103 * big value that won't get hit. The ENOMEM tests can then override this 104 * value to something much smaller to induce ENOMEM conditions. 105 */ 106 ch->avail_cnt = 2048; 107 return 0; 108 } 109 110 static void 111 stub_destroy_ch(void *io_device, void *ctx_buf) 112 { 113 } 114 115 static struct spdk_io_channel * 116 stub_get_io_channel(void *ctx) 117 { 118 struct ut_bdev *ut_bdev = ctx; 119 120 if (g_get_io_channel == true) { 121 return spdk_get_io_channel(ut_bdev->io_target); 122 } else { 123 return NULL; 124 } 125 } 126 127 static int 128 stub_destruct(void *ctx) 129 { 130 return 0; 131 } 132 133 static void 134 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 135 { 136 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 137 138 if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { 139 struct spdk_bdev_io *io; 140 141 while (!TAILQ_EMPTY(&ch->outstanding_io)) { 142 io = TAILQ_FIRST(&ch->outstanding_io); 143 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 144 ch->outstanding_cnt--; 145 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_FAILED); 146 ch->avail_cnt++; 147 } 148 } 149 150 if (ch->avail_cnt > 0) { 151 TAILQ_INSERT_TAIL(&ch->outstanding_io, bdev_io, module_link); 152 ch->outstanding_cnt++; 153 ch->avail_cnt--; 154 } else { 155 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); 156 } 157 } 158 159 static uint32_t 160 stub_complete_io(void *io_target, uint32_t num_to_complete) 161 { 162 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 163 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 164 struct spdk_bdev_io *io; 165 bool complete_all = (num_to_complete == 0); 166 uint32_t num_completed = 0; 167 168 while (complete_all || num_completed < num_to_complete) { 169 if (TAILQ_EMPTY(&ch->outstanding_io)) { 170 break; 171 } 172 io = TAILQ_FIRST(&ch->outstanding_io); 173 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 174 ch->outstanding_cnt--; 175 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_SUCCESS); 176 ch->avail_cnt++; 177 num_completed++; 178 } 179 180 spdk_put_io_channel(_ch); 181 return num_completed; 182 } 183 184 static struct spdk_bdev_fn_table fn_table = { 185 .get_io_channel = stub_get_io_channel, 186 .destruct = stub_destruct, 187 .submit_request = stub_submit_request, 188 }; 189 190 struct spdk_bdev_module bdev_ut_if; 191 192 static int 193 module_init(void) 194 { 195 spdk_bdev_module_init_done(&bdev_ut_if); 196 return 0; 197 } 198 199 static void 200 module_fini(void) 201 { 202 } 203 204 static void 205 init_complete(void) 206 { 207 g_init_complete_called = true; 208 } 209 210 static void 211 fini_start(void) 212 { 213 g_fini_start_called = true; 214 } 215 216 struct spdk_bdev_module bdev_ut_if = { 217 .name = "bdev_ut", 218 .module_init = module_init, 219 .module_fini = module_fini, 220 .async_init = true, 221 .init_complete = init_complete, 222 .fini_start = fini_start, 223 }; 224 225 SPDK_BDEV_MODULE_REGISTER(bdev_ut, &bdev_ut_if) 226 227 static void 228 register_bdev(struct ut_bdev *ut_bdev, char *name, void *io_target) 229 { 230 memset(ut_bdev, 0, sizeof(*ut_bdev)); 231 232 ut_bdev->io_target = io_target; 233 ut_bdev->bdev.ctxt = ut_bdev; 234 ut_bdev->bdev.name = name; 235 ut_bdev->bdev.fn_table = &fn_table; 236 ut_bdev->bdev.module = &bdev_ut_if; 237 ut_bdev->bdev.blocklen = 4096; 238 ut_bdev->bdev.blockcnt = 1024; 239 240 spdk_bdev_register(&ut_bdev->bdev); 241 } 242 243 static void 244 unregister_bdev(struct ut_bdev *ut_bdev) 245 { 246 /* Handle any deferred messages. */ 247 poll_threads(); 248 spdk_bdev_unregister(&ut_bdev->bdev, NULL, NULL); 249 } 250 251 static void 252 bdev_init_cb(void *done, int rc) 253 { 254 CU_ASSERT(rc == 0); 255 *(bool *)done = true; 256 } 257 258 static void 259 setup_test(void) 260 { 261 bool done = false; 262 263 allocate_threads(BDEV_UT_NUM_THREADS); 264 set_thread(0); 265 spdk_bdev_initialize(bdev_init_cb, &done); 266 spdk_io_device_register(&g_io_device, stub_create_ch, stub_destroy_ch, 267 sizeof(struct ut_bdev_channel), NULL); 268 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 269 spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &g_desc); 270 } 271 272 static void 273 finish_cb(void *cb_arg) 274 { 275 g_teardown_done = true; 276 } 277 278 static void 279 teardown_test(void) 280 { 281 set_thread(0); 282 g_teardown_done = false; 283 spdk_bdev_close(g_desc); 284 g_desc = NULL; 285 unregister_bdev(&g_bdev); 286 spdk_io_device_unregister(&g_io_device, NULL); 287 spdk_bdev_finish(finish_cb, NULL); 288 poll_threads(); 289 memset(&g_bdev, 0, sizeof(g_bdev)); 290 CU_ASSERT(g_teardown_done == true); 291 g_teardown_done = false; 292 free_threads(); 293 } 294 295 static uint32_t 296 bdev_io_tailq_cnt(bdev_io_tailq_t *tailq) 297 { 298 struct spdk_bdev_io *io; 299 uint32_t cnt = 0; 300 301 TAILQ_FOREACH(io, tailq, internal.link) { 302 cnt++; 303 } 304 305 return cnt; 306 } 307 308 static void 309 basic(void) 310 { 311 g_init_complete_called = false; 312 setup_test(); 313 CU_ASSERT(g_init_complete_called == true); 314 315 set_thread(0); 316 317 g_get_io_channel = false; 318 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 319 CU_ASSERT(g_ut_threads[0].ch == NULL); 320 321 g_get_io_channel = true; 322 g_create_ch = false; 323 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 324 CU_ASSERT(g_ut_threads[0].ch == NULL); 325 326 g_get_io_channel = true; 327 g_create_ch = true; 328 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 329 CU_ASSERT(g_ut_threads[0].ch != NULL); 330 spdk_put_io_channel(g_ut_threads[0].ch); 331 332 g_fini_start_called = false; 333 teardown_test(); 334 CU_ASSERT(g_fini_start_called == true); 335 } 336 337 static void 338 _bdev_removed(void *done) 339 { 340 *(bool *)done = true; 341 } 342 343 static void 344 _bdev_unregistered(void *done, int rc) 345 { 346 CU_ASSERT(rc == 0); 347 *(bool *)done = true; 348 } 349 350 static void 351 unregister_and_close(void) 352 { 353 bool done, remove_notify; 354 struct spdk_bdev_desc *desc = NULL; 355 356 setup_test(); 357 set_thread(0); 358 359 /* setup_test() automatically opens the bdev, 360 * but this test needs to do that in a different 361 * way. */ 362 spdk_bdev_close(g_desc); 363 poll_threads(); 364 365 /* Try hotremoving a bdev with descriptors which don't provide 366 * the notification callback */ 367 spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &desc); 368 SPDK_CU_ASSERT_FATAL(desc != NULL); 369 370 /* There is an open descriptor on the device. Unregister it, 371 * which can't proceed until the descriptor is closed. */ 372 done = false; 373 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 374 375 /* Poll the threads to allow all events to be processed */ 376 poll_threads(); 377 378 /* Make sure the bdev was not unregistered. We still have a 379 * descriptor open */ 380 CU_ASSERT(done == false); 381 382 spdk_bdev_close(desc); 383 poll_threads(); 384 desc = NULL; 385 386 /* The unregister should have completed */ 387 CU_ASSERT(done == true); 388 389 390 /* Register the bdev again */ 391 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 392 393 remove_notify = false; 394 spdk_bdev_open(&g_bdev.bdev, true, _bdev_removed, &remove_notify, &desc); 395 SPDK_CU_ASSERT_FATAL(desc != NULL); 396 CU_ASSERT(remove_notify == false); 397 398 /* There is an open descriptor on the device. Unregister it, 399 * which can't proceed until the descriptor is closed. */ 400 done = false; 401 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 402 /* No polling has occurred, so neither of these should execute */ 403 CU_ASSERT(remove_notify == false); 404 CU_ASSERT(done == false); 405 406 /* Prior to the unregister completing, close the descriptor */ 407 spdk_bdev_close(desc); 408 409 /* Poll the threads to allow all events to be processed */ 410 poll_threads(); 411 412 /* Remove notify should not have been called because the 413 * descriptor is already closed. */ 414 CU_ASSERT(remove_notify == false); 415 416 /* The unregister should have completed */ 417 CU_ASSERT(done == true); 418 419 /* Restore the original g_bdev so that we can use teardown_test(). */ 420 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 421 spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &g_desc); 422 teardown_test(); 423 } 424 425 static void 426 reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 427 { 428 bool *done = cb_arg; 429 430 CU_ASSERT(success == true); 431 *done = true; 432 spdk_bdev_free_io(bdev_io); 433 } 434 435 static void 436 put_channel_during_reset(void) 437 { 438 struct spdk_io_channel *io_ch; 439 bool done = false; 440 441 setup_test(); 442 443 set_thread(0); 444 io_ch = spdk_bdev_get_io_channel(g_desc); 445 CU_ASSERT(io_ch != NULL); 446 447 /* 448 * Start a reset, but then put the I/O channel before 449 * the deferred messages for the reset get a chance to 450 * execute. 451 */ 452 spdk_bdev_reset(g_desc, io_ch, reset_done, &done); 453 spdk_put_io_channel(io_ch); 454 poll_threads(); 455 stub_complete_io(g_bdev.io_target, 0); 456 457 teardown_test(); 458 } 459 460 static void 461 aborted_reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 462 { 463 enum spdk_bdev_io_status *status = cb_arg; 464 465 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 466 spdk_bdev_free_io(bdev_io); 467 } 468 469 static void 470 aborted_reset(void) 471 { 472 struct spdk_io_channel *io_ch[2]; 473 enum spdk_bdev_io_status status1 = SPDK_BDEV_IO_STATUS_PENDING, 474 status2 = SPDK_BDEV_IO_STATUS_PENDING; 475 476 setup_test(); 477 478 set_thread(0); 479 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 480 CU_ASSERT(io_ch[0] != NULL); 481 spdk_bdev_reset(g_desc, io_ch[0], aborted_reset_done, &status1); 482 poll_threads(); 483 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 484 485 /* 486 * First reset has been submitted on ch0. Now submit a second 487 * reset on ch1 which will get queued since there is already a 488 * reset in progress. 489 */ 490 set_thread(1); 491 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 492 CU_ASSERT(io_ch[1] != NULL); 493 spdk_bdev_reset(g_desc, io_ch[1], aborted_reset_done, &status2); 494 poll_threads(); 495 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 496 497 /* 498 * Now destroy ch1. This will abort the queued reset. Check that 499 * the second reset was completed with failed status. Also check 500 * that bdev->internal.reset_in_progress != NULL, since the 501 * original reset has not been completed yet. This ensures that 502 * the bdev code is correctly noticing that the failed reset is 503 * *not* the one that had been submitted to the bdev module. 504 */ 505 set_thread(1); 506 spdk_put_io_channel(io_ch[1]); 507 poll_threads(); 508 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_FAILED); 509 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 510 511 /* 512 * Now complete the first reset, verify that it completed with SUCCESS 513 * status and that bdev->internal.reset_in_progress is also set back to NULL. 514 */ 515 set_thread(0); 516 spdk_put_io_channel(io_ch[0]); 517 stub_complete_io(g_bdev.io_target, 0); 518 poll_threads(); 519 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 520 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 521 522 teardown_test(); 523 } 524 525 static void 526 io_during_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 527 { 528 enum spdk_bdev_io_status *status = cb_arg; 529 530 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 531 spdk_bdev_free_io(bdev_io); 532 } 533 534 static void 535 io_during_reset(void) 536 { 537 struct spdk_io_channel *io_ch[2]; 538 struct spdk_bdev_channel *bdev_ch[2]; 539 enum spdk_bdev_io_status status0, status1, status_reset; 540 int rc; 541 542 setup_test(); 543 544 /* 545 * First test normal case - submit an I/O on each of two channels (with no resets) 546 * and verify they complete successfully. 547 */ 548 set_thread(0); 549 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 550 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 551 CU_ASSERT(bdev_ch[0]->flags == 0); 552 status0 = SPDK_BDEV_IO_STATUS_PENDING; 553 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 554 CU_ASSERT(rc == 0); 555 556 set_thread(1); 557 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 558 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 559 CU_ASSERT(bdev_ch[1]->flags == 0); 560 status1 = SPDK_BDEV_IO_STATUS_PENDING; 561 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 562 CU_ASSERT(rc == 0); 563 564 poll_threads(); 565 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 566 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 567 568 set_thread(0); 569 stub_complete_io(g_bdev.io_target, 0); 570 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 571 572 set_thread(1); 573 stub_complete_io(g_bdev.io_target, 0); 574 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 575 576 /* 577 * Now submit a reset, and leave it pending while we submit I/O on two different 578 * channels. These I/O should be failed by the bdev layer since the reset is in 579 * progress. 580 */ 581 set_thread(0); 582 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 583 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &status_reset); 584 CU_ASSERT(rc == 0); 585 586 CU_ASSERT(bdev_ch[0]->flags == 0); 587 CU_ASSERT(bdev_ch[1]->flags == 0); 588 poll_threads(); 589 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_RESET_IN_PROGRESS); 590 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_RESET_IN_PROGRESS); 591 592 set_thread(0); 593 status0 = SPDK_BDEV_IO_STATUS_PENDING; 594 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 595 CU_ASSERT(rc == 0); 596 597 set_thread(1); 598 status1 = SPDK_BDEV_IO_STATUS_PENDING; 599 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 600 CU_ASSERT(rc == 0); 601 602 /* 603 * A reset is in progress so these read I/O should complete with failure. Note that we 604 * need to poll_threads() since I/O completed inline have their completion deferred. 605 */ 606 poll_threads(); 607 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 608 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 609 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 610 611 /* 612 * Complete the reset 613 */ 614 set_thread(0); 615 stub_complete_io(g_bdev.io_target, 0); 616 617 /* 618 * Only poll thread 0. We should not get a completion. 619 */ 620 poll_thread(0); 621 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 622 623 /* 624 * Poll both thread 0 and 1 so the messages can propagate and we 625 * get a completion. 626 */ 627 poll_threads(); 628 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 629 630 spdk_put_io_channel(io_ch[0]); 631 set_thread(1); 632 spdk_put_io_channel(io_ch[1]); 633 poll_threads(); 634 635 teardown_test(); 636 } 637 638 static void 639 basic_qos(void) 640 { 641 struct spdk_io_channel *io_ch[2]; 642 struct spdk_bdev_channel *bdev_ch[2]; 643 struct spdk_bdev *bdev; 644 enum spdk_bdev_io_status status; 645 int rc; 646 647 setup_test(); 648 649 /* Enable QoS */ 650 bdev = &g_bdev.bdev; 651 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 652 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 653 TAILQ_INIT(&bdev->internal.qos->queued); 654 /* 655 * Enable read/write IOPS, read only byte per second and 656 * read/write byte per second rate limits. 657 * In this case, all rate limits will take equal effect. 658 */ 659 /* 2000 read/write I/O per second, or 2 per millisecond */ 660 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 661 /* 8K read/write byte per millisecond with 4K block size */ 662 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 663 /* 8K read only byte per millisecond with 4K block size */ 664 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 8192000; 665 666 g_get_io_channel = true; 667 668 set_thread(0); 669 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 670 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 671 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 672 673 set_thread(1); 674 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 675 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 676 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 677 678 /* 679 * Send an I/O on thread 0, which is where the QoS thread is running. 680 */ 681 set_thread(0); 682 status = SPDK_BDEV_IO_STATUS_PENDING; 683 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status); 684 CU_ASSERT(rc == 0); 685 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 686 poll_threads(); 687 stub_complete_io(g_bdev.io_target, 0); 688 poll_threads(); 689 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 690 691 /* Send an I/O on thread 1. The QoS thread is not running here. */ 692 status = SPDK_BDEV_IO_STATUS_PENDING; 693 set_thread(1); 694 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status); 695 CU_ASSERT(rc == 0); 696 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 697 poll_threads(); 698 /* Complete I/O on thread 1. This should not complete the I/O we submitted */ 699 stub_complete_io(g_bdev.io_target, 0); 700 poll_threads(); 701 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 702 /* Now complete I/O on thread 0 */ 703 set_thread(0); 704 poll_threads(); 705 stub_complete_io(g_bdev.io_target, 0); 706 poll_threads(); 707 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 708 709 /* 710 * Close the descriptor only, which should stop the qos channel as 711 * the last descriptor removed. 712 */ 713 spdk_bdev_close(g_desc); 714 poll_threads(); 715 CU_ASSERT(bdev->internal.qos->ch == NULL); 716 717 /* 718 * Open the bdev again which shall setup the qos channel as the 719 * channels are valid. 720 */ 721 spdk_bdev_open(bdev, true, NULL, NULL, &g_desc); 722 poll_threads(); 723 CU_ASSERT(bdev->internal.qos->ch != NULL); 724 725 /* Tear down the channels */ 726 set_thread(0); 727 spdk_put_io_channel(io_ch[0]); 728 set_thread(1); 729 spdk_put_io_channel(io_ch[1]); 730 poll_threads(); 731 set_thread(0); 732 733 /* Close the descriptor, which should stop the qos channel */ 734 spdk_bdev_close(g_desc); 735 poll_threads(); 736 CU_ASSERT(bdev->internal.qos->ch == NULL); 737 738 /* Open the bdev again, no qos channel setup without valid channels. */ 739 spdk_bdev_open(bdev, true, NULL, NULL, &g_desc); 740 poll_threads(); 741 CU_ASSERT(bdev->internal.qos->ch == NULL); 742 743 /* Create the channels in reverse order. */ 744 set_thread(1); 745 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 746 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 747 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 748 749 set_thread(0); 750 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 751 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 752 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 753 754 /* Confirm that the qos thread is now thread 1 */ 755 CU_ASSERT(bdev->internal.qos->ch == bdev_ch[1]); 756 757 /* Tear down the channels */ 758 set_thread(0); 759 spdk_put_io_channel(io_ch[0]); 760 set_thread(1); 761 spdk_put_io_channel(io_ch[1]); 762 poll_threads(); 763 764 set_thread(0); 765 766 teardown_test(); 767 } 768 769 static void 770 io_during_qos_queue(void) 771 { 772 struct spdk_io_channel *io_ch[2]; 773 struct spdk_bdev_channel *bdev_ch[2]; 774 struct spdk_bdev *bdev; 775 enum spdk_bdev_io_status status0, status1, status2; 776 int rc; 777 778 setup_test(); 779 MOCK_SET(spdk_get_ticks, 0); 780 781 /* Enable QoS */ 782 bdev = &g_bdev.bdev; 783 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 784 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 785 TAILQ_INIT(&bdev->internal.qos->queued); 786 /* 787 * Enable read/write IOPS, read only byte per sec, write only 788 * byte per sec and read/write byte per sec rate limits. 789 * In this case, both read only and write only byte per sec 790 * rate limit will take effect. 791 */ 792 /* 4000 read/write I/O per second, or 4 per millisecond */ 793 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 4000; 794 /* 8K byte per millisecond with 4K block size */ 795 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 796 /* 4K byte per millisecond with 4K block size */ 797 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 4096000; 798 /* 4K byte per millisecond with 4K block size */ 799 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 4096000; 800 801 g_get_io_channel = true; 802 803 /* Create channels */ 804 set_thread(0); 805 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 806 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 807 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 808 809 set_thread(1); 810 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 811 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 812 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 813 814 /* Send two read I/Os */ 815 status1 = SPDK_BDEV_IO_STATUS_PENDING; 816 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 817 CU_ASSERT(rc == 0); 818 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 819 set_thread(0); 820 status0 = SPDK_BDEV_IO_STATUS_PENDING; 821 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 822 CU_ASSERT(rc == 0); 823 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 824 /* Send one write I/O */ 825 status2 = SPDK_BDEV_IO_STATUS_PENDING; 826 rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status2); 827 CU_ASSERT(rc == 0); 828 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_PENDING); 829 830 /* Complete any I/O that arrived at the disk */ 831 poll_threads(); 832 set_thread(1); 833 stub_complete_io(g_bdev.io_target, 0); 834 set_thread(0); 835 stub_complete_io(g_bdev.io_target, 0); 836 poll_threads(); 837 838 /* Only one of the two read I/Os should complete. (logical XOR) */ 839 if (status0 == SPDK_BDEV_IO_STATUS_SUCCESS) { 840 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 841 } else { 842 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 843 } 844 /* The write I/O should complete. */ 845 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_SUCCESS); 846 847 /* Advance in time by a millisecond */ 848 spdk_delay_us(1000); 849 850 /* Complete more I/O */ 851 poll_threads(); 852 set_thread(1); 853 stub_complete_io(g_bdev.io_target, 0); 854 set_thread(0); 855 stub_complete_io(g_bdev.io_target, 0); 856 poll_threads(); 857 858 /* Now the second read I/O should be done */ 859 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 860 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 861 862 /* Tear down the channels */ 863 set_thread(1); 864 spdk_put_io_channel(io_ch[1]); 865 set_thread(0); 866 spdk_put_io_channel(io_ch[0]); 867 poll_threads(); 868 869 teardown_test(); 870 } 871 872 static void 873 io_during_qos_reset(void) 874 { 875 struct spdk_io_channel *io_ch[2]; 876 struct spdk_bdev_channel *bdev_ch[2]; 877 struct spdk_bdev *bdev; 878 enum spdk_bdev_io_status status0, status1, reset_status; 879 int rc; 880 881 setup_test(); 882 MOCK_SET(spdk_get_ticks, 0); 883 884 /* Enable QoS */ 885 bdev = &g_bdev.bdev; 886 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 887 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 888 TAILQ_INIT(&bdev->internal.qos->queued); 889 /* 890 * Enable read/write IOPS, write only byte per sec and 891 * read/write byte per second rate limits. 892 * In this case, read/write byte per second rate limit will 893 * take effect first. 894 */ 895 /* 2000 read/write I/O per second, or 2 per millisecond */ 896 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 897 /* 4K byte per millisecond with 4K block size */ 898 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 4096000; 899 /* 8K byte per millisecond with 4K block size */ 900 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 8192000; 901 902 g_get_io_channel = true; 903 904 /* Create channels */ 905 set_thread(0); 906 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 907 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 908 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 909 910 set_thread(1); 911 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 912 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 913 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 914 915 /* Send two I/O. One of these gets queued by QoS. The other is sitting at the disk. */ 916 status1 = SPDK_BDEV_IO_STATUS_PENDING; 917 rc = spdk_bdev_write_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 918 CU_ASSERT(rc == 0); 919 set_thread(0); 920 status0 = SPDK_BDEV_IO_STATUS_PENDING; 921 rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 922 CU_ASSERT(rc == 0); 923 924 poll_threads(); 925 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 926 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 927 928 /* Reset the bdev. */ 929 reset_status = SPDK_BDEV_IO_STATUS_PENDING; 930 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &reset_status); 931 CU_ASSERT(rc == 0); 932 933 /* Complete any I/O that arrived at the disk */ 934 poll_threads(); 935 set_thread(1); 936 stub_complete_io(g_bdev.io_target, 0); 937 set_thread(0); 938 stub_complete_io(g_bdev.io_target, 0); 939 poll_threads(); 940 941 CU_ASSERT(reset_status == SPDK_BDEV_IO_STATUS_SUCCESS); 942 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 943 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 944 945 /* Tear down the channels */ 946 set_thread(1); 947 spdk_put_io_channel(io_ch[1]); 948 set_thread(0); 949 spdk_put_io_channel(io_ch[0]); 950 poll_threads(); 951 952 teardown_test(); 953 } 954 955 static void 956 enomem_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 957 { 958 enum spdk_bdev_io_status *status = cb_arg; 959 960 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 961 spdk_bdev_free_io(bdev_io); 962 } 963 964 static void 965 enomem(void) 966 { 967 struct spdk_io_channel *io_ch; 968 struct spdk_bdev_channel *bdev_ch; 969 struct spdk_bdev_shared_resource *shared_resource; 970 struct ut_bdev_channel *ut_ch; 971 const uint32_t IO_ARRAY_SIZE = 64; 972 const uint32_t AVAIL = 20; 973 enum spdk_bdev_io_status status[IO_ARRAY_SIZE], status_reset; 974 uint32_t nomem_cnt, i; 975 struct spdk_bdev_io *first_io; 976 int rc; 977 978 setup_test(); 979 980 set_thread(0); 981 io_ch = spdk_bdev_get_io_channel(g_desc); 982 bdev_ch = spdk_io_channel_get_ctx(io_ch); 983 shared_resource = bdev_ch->shared_resource; 984 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 985 ut_ch->avail_cnt = AVAIL; 986 987 /* First submit a number of IOs equal to what the channel can support. */ 988 for (i = 0; i < AVAIL; i++) { 989 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 990 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 991 CU_ASSERT(rc == 0); 992 } 993 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 994 995 /* 996 * Next, submit one additional I/O. This one should fail with ENOMEM and then go onto 997 * the enomem_io list. 998 */ 999 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1000 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1001 CU_ASSERT(rc == 0); 1002 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1003 first_io = TAILQ_FIRST(&shared_resource->nomem_io); 1004 1005 /* 1006 * Now submit a bunch more I/O. These should all fail with ENOMEM and get queued behind 1007 * the first_io above. 1008 */ 1009 for (i = AVAIL + 1; i < IO_ARRAY_SIZE; i++) { 1010 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1011 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1012 CU_ASSERT(rc == 0); 1013 } 1014 1015 /* Assert that first_io is still at the head of the list. */ 1016 CU_ASSERT(TAILQ_FIRST(&shared_resource->nomem_io) == first_io); 1017 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == (IO_ARRAY_SIZE - AVAIL)); 1018 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 1019 CU_ASSERT(shared_resource->nomem_threshold == (AVAIL - NOMEM_THRESHOLD_COUNT)); 1020 1021 /* 1022 * Complete 1 I/O only. The key check here is bdev_io_tailq_cnt - this should not have 1023 * changed since completing just 1 I/O should not trigger retrying the queued nomem_io 1024 * list. 1025 */ 1026 stub_complete_io(g_bdev.io_target, 1); 1027 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 1028 1029 /* 1030 * Complete enough I/O to hit the nomem_theshold. This should trigger retrying nomem_io, 1031 * and we should see I/O get resubmitted to the test bdev module. 1032 */ 1033 stub_complete_io(g_bdev.io_target, NOMEM_THRESHOLD_COUNT - 1); 1034 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) < nomem_cnt); 1035 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 1036 1037 /* Complete 1 I/O only. This should not trigger retrying the queued nomem_io. */ 1038 stub_complete_io(g_bdev.io_target, 1); 1039 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 1040 1041 /* 1042 * Send a reset and confirm that all I/O are completed, including the ones that 1043 * were queued on the nomem_io list. 1044 */ 1045 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 1046 rc = spdk_bdev_reset(g_desc, io_ch, enomem_done, &status_reset); 1047 poll_threads(); 1048 CU_ASSERT(rc == 0); 1049 /* This will complete the reset. */ 1050 stub_complete_io(g_bdev.io_target, 0); 1051 1052 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == 0); 1053 CU_ASSERT(shared_resource->io_outstanding == 0); 1054 1055 spdk_put_io_channel(io_ch); 1056 poll_threads(); 1057 teardown_test(); 1058 } 1059 1060 static void 1061 enomem_multi_bdev(void) 1062 { 1063 struct spdk_io_channel *io_ch; 1064 struct spdk_bdev_channel *bdev_ch; 1065 struct spdk_bdev_shared_resource *shared_resource; 1066 struct ut_bdev_channel *ut_ch; 1067 const uint32_t IO_ARRAY_SIZE = 64; 1068 const uint32_t AVAIL = 20; 1069 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1070 uint32_t i; 1071 struct ut_bdev *second_bdev; 1072 struct spdk_bdev_desc *second_desc = NULL; 1073 struct spdk_bdev_channel *second_bdev_ch; 1074 struct spdk_io_channel *second_ch; 1075 int rc; 1076 1077 setup_test(); 1078 1079 /* Register second bdev with the same io_target */ 1080 second_bdev = calloc(1, sizeof(*second_bdev)); 1081 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1082 register_bdev(second_bdev, "ut_bdev2", g_bdev.io_target); 1083 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 1084 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1085 1086 set_thread(0); 1087 io_ch = spdk_bdev_get_io_channel(g_desc); 1088 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1089 shared_resource = bdev_ch->shared_resource; 1090 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1091 ut_ch->avail_cnt = AVAIL; 1092 1093 second_ch = spdk_bdev_get_io_channel(second_desc); 1094 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1095 SPDK_CU_ASSERT_FATAL(shared_resource == second_bdev_ch->shared_resource); 1096 1097 /* Saturate io_target through bdev A. */ 1098 for (i = 0; i < AVAIL; i++) { 1099 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1100 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1101 CU_ASSERT(rc == 0); 1102 } 1103 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1104 1105 /* 1106 * Now submit I/O through the second bdev. This should fail with ENOMEM 1107 * and then go onto the nomem_io list. 1108 */ 1109 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1110 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1111 CU_ASSERT(rc == 0); 1112 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1113 1114 /* Complete first bdev's I/O. This should retry sending second bdev's nomem_io */ 1115 stub_complete_io(g_bdev.io_target, AVAIL); 1116 1117 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&shared_resource->nomem_io)); 1118 CU_ASSERT(shared_resource->io_outstanding == 1); 1119 1120 /* Now complete our retried I/O */ 1121 stub_complete_io(g_bdev.io_target, 1); 1122 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == 0); 1123 1124 spdk_put_io_channel(io_ch); 1125 spdk_put_io_channel(second_ch); 1126 spdk_bdev_close(second_desc); 1127 unregister_bdev(second_bdev); 1128 poll_threads(); 1129 free(second_bdev); 1130 teardown_test(); 1131 } 1132 1133 1134 static void 1135 enomem_multi_io_target(void) 1136 { 1137 struct spdk_io_channel *io_ch; 1138 struct spdk_bdev_channel *bdev_ch; 1139 struct ut_bdev_channel *ut_ch; 1140 const uint32_t IO_ARRAY_SIZE = 64; 1141 const uint32_t AVAIL = 20; 1142 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1143 uint32_t i; 1144 int new_io_device; 1145 struct ut_bdev *second_bdev; 1146 struct spdk_bdev_desc *second_desc = NULL; 1147 struct spdk_bdev_channel *second_bdev_ch; 1148 struct spdk_io_channel *second_ch; 1149 int rc; 1150 1151 setup_test(); 1152 1153 /* Create new io_target and a second bdev using it */ 1154 spdk_io_device_register(&new_io_device, stub_create_ch, stub_destroy_ch, 1155 sizeof(struct ut_bdev_channel), NULL); 1156 second_bdev = calloc(1, sizeof(*second_bdev)); 1157 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1158 register_bdev(second_bdev, "ut_bdev2", &new_io_device); 1159 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 1160 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1161 1162 set_thread(0); 1163 io_ch = spdk_bdev_get_io_channel(g_desc); 1164 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1165 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1166 ut_ch->avail_cnt = AVAIL; 1167 1168 /* Different io_target should imply a different shared_resource */ 1169 second_ch = spdk_bdev_get_io_channel(second_desc); 1170 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1171 SPDK_CU_ASSERT_FATAL(bdev_ch->shared_resource != second_bdev_ch->shared_resource); 1172 1173 /* Saturate io_target through bdev A. */ 1174 for (i = 0; i < AVAIL; i++) { 1175 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1176 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1177 CU_ASSERT(rc == 0); 1178 } 1179 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1180 1181 /* Issue one more I/O to fill ENOMEM list. */ 1182 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1183 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1184 CU_ASSERT(rc == 0); 1185 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1186 1187 /* 1188 * Now submit I/O through the second bdev. This should go through and complete 1189 * successfully because we're using a different io_device underneath. 1190 */ 1191 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1192 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1193 CU_ASSERT(rc == 0); 1194 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&second_bdev_ch->shared_resource->nomem_io)); 1195 stub_complete_io(second_bdev->io_target, 1); 1196 1197 /* Cleanup; Complete outstanding I/O. */ 1198 stub_complete_io(g_bdev.io_target, AVAIL); 1199 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1200 /* Complete the ENOMEM I/O */ 1201 stub_complete_io(g_bdev.io_target, 1); 1202 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1203 1204 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1205 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1206 spdk_put_io_channel(io_ch); 1207 spdk_put_io_channel(second_ch); 1208 spdk_bdev_close(second_desc); 1209 unregister_bdev(second_bdev); 1210 spdk_io_device_unregister(&new_io_device, NULL); 1211 poll_threads(); 1212 free(second_bdev); 1213 teardown_test(); 1214 } 1215 1216 static void 1217 qos_dynamic_enable_done(void *cb_arg, int status) 1218 { 1219 int *rc = cb_arg; 1220 *rc = status; 1221 } 1222 1223 static void 1224 qos_dynamic_enable(void) 1225 { 1226 struct spdk_io_channel *io_ch[2]; 1227 struct spdk_bdev_channel *bdev_ch[2]; 1228 struct spdk_bdev *bdev; 1229 enum spdk_bdev_io_status bdev_io_status[2]; 1230 uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {}; 1231 int status, second_status, rc, i; 1232 1233 setup_test(); 1234 MOCK_SET(spdk_get_ticks, 0); 1235 1236 for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) { 1237 limits[i] = UINT64_MAX; 1238 } 1239 1240 bdev = &g_bdev.bdev; 1241 1242 g_get_io_channel = true; 1243 1244 /* Create channels */ 1245 set_thread(0); 1246 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1247 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1248 CU_ASSERT(bdev_ch[0]->flags == 0); 1249 1250 set_thread(1); 1251 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1252 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1253 CU_ASSERT(bdev_ch[1]->flags == 0); 1254 1255 set_thread(0); 1256 1257 /* 1258 * Enable QoS: Read/Write IOPS, Read/Write byte, 1259 * Read only byte and Write only byte per second 1260 * rate limits. 1261 * More than 10 I/Os allowed per timeslice. 1262 */ 1263 status = -1; 1264 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1265 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 100; 1266 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 100; 1267 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 10; 1268 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1269 poll_threads(); 1270 CU_ASSERT(status == 0); 1271 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1272 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1273 1274 /* 1275 * Submit and complete 10 I/O to fill the QoS allotment for this timeslice. 1276 * Additional I/O will then be queued. 1277 */ 1278 set_thread(0); 1279 for (i = 0; i < 10; i++) { 1280 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1281 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1282 CU_ASSERT(rc == 0); 1283 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1284 poll_thread(0); 1285 stub_complete_io(g_bdev.io_target, 0); 1286 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1287 } 1288 1289 /* 1290 * Send two more I/O. These I/O will be queued since the current timeslice allotment has been 1291 * filled already. We want to test that when QoS is disabled that these two I/O: 1292 * 1) are not aborted 1293 * 2) are sent back to their original thread for resubmission 1294 */ 1295 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1296 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1297 CU_ASSERT(rc == 0); 1298 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1299 set_thread(1); 1300 bdev_io_status[1] = SPDK_BDEV_IO_STATUS_PENDING; 1301 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &bdev_io_status[1]); 1302 CU_ASSERT(rc == 0); 1303 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1304 poll_threads(); 1305 1306 /* 1307 * Disable QoS: Read/Write IOPS, Read/Write byte, 1308 * Read only byte rate limits 1309 */ 1310 status = -1; 1311 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1312 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 0; 1313 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 0; 1314 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1315 poll_threads(); 1316 CU_ASSERT(status == 0); 1317 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1318 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1319 1320 /* Disable QoS: Write only Byte per second rate limit */ 1321 status = -1; 1322 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 0; 1323 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1324 poll_threads(); 1325 CU_ASSERT(status == 0); 1326 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1327 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1328 1329 /* 1330 * All I/O should have been resubmitted back on their original thread. Complete 1331 * all I/O on thread 0, and ensure that only the thread 0 I/O was completed. 1332 */ 1333 set_thread(0); 1334 stub_complete_io(g_bdev.io_target, 0); 1335 poll_threads(); 1336 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1337 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1338 1339 /* Now complete all I/O on thread 1 and ensure the thread 1 I/O was completed. */ 1340 set_thread(1); 1341 stub_complete_io(g_bdev.io_target, 0); 1342 poll_threads(); 1343 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_SUCCESS); 1344 1345 /* Disable QoS again */ 1346 status = -1; 1347 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1348 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1349 poll_threads(); 1350 CU_ASSERT(status == 0); /* This should succeed */ 1351 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1352 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1353 1354 /* Enable QoS on thread 0 */ 1355 status = -1; 1356 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1357 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1358 poll_threads(); 1359 CU_ASSERT(status == 0); 1360 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1361 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1362 1363 /* Disable QoS on thread 1 */ 1364 set_thread(1); 1365 status = -1; 1366 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1367 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1368 /* Don't poll yet. This should leave the channels with QoS enabled */ 1369 CU_ASSERT(status == -1); 1370 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1371 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1372 1373 /* Enable QoS. This should immediately fail because the previous disable QoS hasn't completed. */ 1374 second_status = 0; 1375 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 10; 1376 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &second_status); 1377 poll_threads(); 1378 CU_ASSERT(status == 0); /* The disable should succeed */ 1379 CU_ASSERT(second_status < 0); /* The enable should fail */ 1380 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1381 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1382 1383 /* Enable QoS on thread 1. This should succeed now that the disable has completed. */ 1384 status = -1; 1385 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1386 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1387 poll_threads(); 1388 CU_ASSERT(status == 0); 1389 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1390 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1391 1392 /* Tear down the channels */ 1393 set_thread(0); 1394 spdk_put_io_channel(io_ch[0]); 1395 set_thread(1); 1396 spdk_put_io_channel(io_ch[1]); 1397 poll_threads(); 1398 1399 set_thread(0); 1400 teardown_test(); 1401 } 1402 1403 static void 1404 histogram_status_cb(void *cb_arg, int status) 1405 { 1406 g_status = status; 1407 } 1408 1409 static void 1410 histogram_data_cb(void *cb_arg, int status, struct spdk_histogram_data *histogram) 1411 { 1412 g_status = status; 1413 g_histogram = histogram; 1414 } 1415 1416 static void 1417 histogram_io_count(void *ctx, uint64_t start, uint64_t end, uint64_t count, 1418 uint64_t total, uint64_t so_far) 1419 { 1420 g_count += count; 1421 } 1422 1423 static void 1424 bdev_histograms_mt(void) 1425 { 1426 struct spdk_io_channel *ch[2]; 1427 struct spdk_histogram_data *histogram; 1428 uint8_t buf[4096]; 1429 int status = false; 1430 int rc; 1431 1432 1433 setup_test(); 1434 1435 set_thread(0); 1436 ch[0] = spdk_bdev_get_io_channel(g_desc); 1437 CU_ASSERT(ch[0] != NULL); 1438 1439 set_thread(1); 1440 ch[1] = spdk_bdev_get_io_channel(g_desc); 1441 CU_ASSERT(ch[1] != NULL); 1442 1443 1444 /* Enable histogram */ 1445 spdk_bdev_histogram_enable(&g_bdev.bdev, histogram_status_cb, NULL, true); 1446 poll_threads(); 1447 CU_ASSERT(g_status == 0); 1448 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == true); 1449 1450 /* Allocate histogram */ 1451 histogram = spdk_histogram_data_alloc(); 1452 1453 /* Check if histogram is zeroed */ 1454 spdk_bdev_histogram_get(&g_bdev.bdev, histogram, histogram_data_cb, NULL); 1455 poll_threads(); 1456 CU_ASSERT(g_status == 0); 1457 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 1458 1459 g_count = 0; 1460 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 1461 1462 CU_ASSERT(g_count == 0); 1463 1464 set_thread(0); 1465 rc = spdk_bdev_write_blocks(g_desc, ch[0], &buf, 0, 1, io_during_io_done, &status); 1466 CU_ASSERT(rc == 0); 1467 1468 spdk_delay_us(10); 1469 stub_complete_io(g_bdev.io_target, 1); 1470 poll_threads(); 1471 CU_ASSERT(status == true); 1472 1473 1474 set_thread(1); 1475 rc = spdk_bdev_read_blocks(g_desc, ch[1], &buf, 0, 1, io_during_io_done, &status); 1476 CU_ASSERT(rc == 0); 1477 1478 spdk_delay_us(10); 1479 stub_complete_io(g_bdev.io_target, 1); 1480 poll_threads(); 1481 CU_ASSERT(status == true); 1482 1483 set_thread(0); 1484 1485 /* Check if histogram gathered data from all I/O channels */ 1486 spdk_bdev_histogram_get(&g_bdev.bdev, histogram, histogram_data_cb, NULL); 1487 poll_threads(); 1488 CU_ASSERT(g_status == 0); 1489 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == true); 1490 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 1491 1492 g_count = 0; 1493 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 1494 CU_ASSERT(g_count == 2); 1495 1496 /* Disable histogram */ 1497 spdk_bdev_histogram_enable(&g_bdev.bdev, histogram_status_cb, NULL, false); 1498 poll_threads(); 1499 CU_ASSERT(g_status == 0); 1500 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == false); 1501 1502 spdk_histogram_data_free(histogram); 1503 1504 /* Tear down the channels */ 1505 set_thread(0); 1506 spdk_put_io_channel(ch[0]); 1507 set_thread(1); 1508 spdk_put_io_channel(ch[1]); 1509 poll_threads(); 1510 set_thread(0); 1511 teardown_test(); 1512 1513 } 1514 1515 struct timeout_io_cb_arg { 1516 struct iovec iov; 1517 uint8_t type; 1518 }; 1519 1520 static int 1521 bdev_channel_count_submitted_io(struct spdk_bdev_channel *ch) 1522 { 1523 struct spdk_bdev_io *bdev_io; 1524 int n = 0; 1525 1526 if (!ch) { 1527 return -1; 1528 } 1529 1530 TAILQ_FOREACH(bdev_io, &ch->io_submitted, internal.ch_link) { 1531 n++; 1532 } 1533 1534 return n; 1535 } 1536 1537 static void 1538 bdev_channel_io_timeout_cb(void *cb_arg, struct spdk_bdev_io *bdev_io) 1539 { 1540 struct timeout_io_cb_arg *ctx = cb_arg; 1541 1542 ctx->type = bdev_io->type; 1543 ctx->iov.iov_base = bdev_io->iov.iov_base; 1544 ctx->iov.iov_len = bdev_io->iov.iov_len; 1545 } 1546 1547 static bool g_io_done; 1548 1549 static void 1550 io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1551 { 1552 g_io_done = true; 1553 spdk_bdev_free_io(bdev_io); 1554 } 1555 1556 static void 1557 bdev_set_io_timeout_mt(void) 1558 { 1559 struct spdk_io_channel *ch[3]; 1560 struct spdk_bdev_channel *bdev_ch[3]; 1561 struct timeout_io_cb_arg cb_arg; 1562 1563 setup_test(); 1564 1565 g_bdev.bdev.optimal_io_boundary = 16; 1566 g_bdev.bdev.split_on_optimal_io_boundary = true; 1567 1568 set_thread(0); 1569 ch[0] = spdk_bdev_get_io_channel(g_desc); 1570 CU_ASSERT(ch[0] != NULL); 1571 1572 set_thread(1); 1573 ch[1] = spdk_bdev_get_io_channel(g_desc); 1574 CU_ASSERT(ch[1] != NULL); 1575 1576 set_thread(2); 1577 ch[2] = spdk_bdev_get_io_channel(g_desc); 1578 CU_ASSERT(ch[2] != NULL); 1579 1580 /* Multi-thread mode 1581 * 1, Check the poller was registered successfully 1582 * 2, Check the timeout IO and ensure the IO was the submitted by user 1583 * 3, Check the link int the bdev_ch works right. 1584 * 4, Close desc and put io channel during the timeout poller is polling 1585 */ 1586 1587 /* In desc thread set the timeout */ 1588 set_thread(0); 1589 CU_ASSERT(spdk_bdev_set_timeout(g_desc, 5, bdev_channel_io_timeout_cb, &cb_arg) == 0); 1590 CU_ASSERT(g_desc->io_timeout_poller != NULL); 1591 CU_ASSERT(g_desc->cb_fn == bdev_channel_io_timeout_cb); 1592 CU_ASSERT(g_desc->cb_arg == &cb_arg); 1593 1594 /* check the IO submitted list and timeout handler */ 1595 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[0], (void *)0x2000, 0, 1, io_done, NULL) == 0); 1596 bdev_ch[0] = spdk_io_channel_get_ctx(ch[0]); 1597 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[0]) == 1); 1598 1599 set_thread(1); 1600 CU_ASSERT(spdk_bdev_write_blocks(g_desc, ch[1], (void *)0x1000, 0, 1, io_done, NULL) == 0); 1601 bdev_ch[1] = spdk_io_channel_get_ctx(ch[1]); 1602 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[1]) == 1); 1603 1604 /* Now test that a single-vector command is split correctly. 1605 * Offset 14, length 8, payload 0xF000 1606 * Child - Offset 14, length 2, payload 0xF000 1607 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 1608 * 1609 * Set up the expected values before calling spdk_bdev_read_blocks 1610 */ 1611 set_thread(2); 1612 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[2], (void *)0xF000, 14, 8, io_done, NULL) == 0); 1613 bdev_ch[2] = spdk_io_channel_get_ctx(ch[2]); 1614 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[2]) == 1); 1615 1616 set_thread(0); 1617 memset(&cb_arg, 0, sizeof(cb_arg)); 1618 spdk_delay_us(3 * spdk_get_ticks_hz()); 1619 poll_threads(); 1620 CU_ASSERT(cb_arg.type == 0); 1621 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 1622 CU_ASSERT(cb_arg.iov.iov_len == 0); 1623 1624 /* Now the time reach the limit */ 1625 spdk_delay_us(3 * spdk_get_ticks_hz()); 1626 poll_thread(0); 1627 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_READ); 1628 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x2000); 1629 CU_ASSERT(cb_arg.iov.iov_len == 1 * g_bdev.bdev.blocklen); 1630 stub_complete_io(g_bdev.io_target, 1); 1631 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[0]) == 0); 1632 1633 memset(&cb_arg, 0, sizeof(cb_arg)); 1634 set_thread(1); 1635 poll_thread(1); 1636 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 1637 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x1000); 1638 CU_ASSERT(cb_arg.iov.iov_len == 1 * g_bdev.bdev.blocklen); 1639 stub_complete_io(g_bdev.io_target, 1); 1640 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[1]) == 0); 1641 1642 memset(&cb_arg, 0, sizeof(cb_arg)); 1643 set_thread(2); 1644 poll_thread(2); 1645 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_READ); 1646 CU_ASSERT(cb_arg.iov.iov_base == (void *)0xF000); 1647 CU_ASSERT(cb_arg.iov.iov_len == 8 * g_bdev.bdev.blocklen); 1648 stub_complete_io(g_bdev.io_target, 1); 1649 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[2]) == 1); 1650 stub_complete_io(g_bdev.io_target, 1); 1651 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch[2]) == 0); 1652 1653 /* Run poll_timeout_done() it means complete the timeout poller */ 1654 set_thread(0); 1655 poll_thread(0); 1656 CU_ASSERT(g_desc->refs == 0); 1657 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[0], (void *)0x1000, 0, 1, io_done, NULL) == 0); 1658 set_thread(1); 1659 CU_ASSERT(spdk_bdev_write_blocks(g_desc, ch[1], (void *)0x2000, 0, 2, io_done, NULL) == 0); 1660 set_thread(2); 1661 CU_ASSERT(spdk_bdev_read_blocks(g_desc, ch[2], (void *)0x3000, 0, 3, io_done, NULL) == 0); 1662 1663 /* Trigger timeout poller to run again, desc->refs is incremented. 1664 * In thread 0 we destroy the io channel before timeout poller runs. 1665 * Timeout callback is not called on thread 0. 1666 */ 1667 spdk_delay_us(6 * spdk_get_ticks_hz()); 1668 memset(&cb_arg, 0, sizeof(cb_arg)); 1669 set_thread(0); 1670 stub_complete_io(g_bdev.io_target, 1); 1671 spdk_put_io_channel(ch[0]); 1672 poll_thread(0); 1673 CU_ASSERT(g_desc->refs == 1) 1674 CU_ASSERT(cb_arg.type == 0); 1675 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 1676 CU_ASSERT(cb_arg.iov.iov_len == 0); 1677 1678 /* In thread 1 timeout poller runs then we destroy the io channel 1679 * Timeout callback is called on thread 1. 1680 */ 1681 memset(&cb_arg, 0, sizeof(cb_arg)); 1682 set_thread(1); 1683 poll_thread(1); 1684 stub_complete_io(g_bdev.io_target, 1); 1685 spdk_put_io_channel(ch[1]); 1686 poll_thread(1); 1687 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 1688 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x2000); 1689 CU_ASSERT(cb_arg.iov.iov_len == 2 * g_bdev.bdev.blocklen); 1690 1691 /* Close the desc. 1692 * Unregister the timeout poller first. 1693 * Then decrement desc->refs but it's not zero yet so desc is not freed. 1694 */ 1695 set_thread(0); 1696 spdk_bdev_close(g_desc); 1697 CU_ASSERT(g_desc->refs == 1); 1698 CU_ASSERT(g_desc->io_timeout_poller == NULL); 1699 1700 /* Timeout poller runs on thread 2 then we destroy the io channel. 1701 * Desc is closed so we would exit the timeout poller directly. 1702 * timeout callback is not called on thread 2. 1703 */ 1704 memset(&cb_arg, 0, sizeof(cb_arg)); 1705 set_thread(2); 1706 poll_thread(2); 1707 stub_complete_io(g_bdev.io_target, 1); 1708 spdk_put_io_channel(ch[2]); 1709 poll_thread(2); 1710 CU_ASSERT(cb_arg.type == 0); 1711 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 1712 CU_ASSERT(cb_arg.iov.iov_len == 0); 1713 1714 set_thread(0); 1715 poll_thread(0); 1716 g_teardown_done = false; 1717 unregister_bdev(&g_bdev); 1718 spdk_io_device_unregister(&g_io_device, NULL); 1719 spdk_bdev_finish(finish_cb, NULL); 1720 poll_threads(); 1721 memset(&g_bdev, 0, sizeof(g_bdev)); 1722 CU_ASSERT(g_teardown_done == true); 1723 g_teardown_done = false; 1724 free_threads(); 1725 } 1726 1727 static bool g_io_done2; 1728 static bool g_lock_lba_range_done; 1729 static bool g_unlock_lba_range_done; 1730 1731 static void 1732 io_done2(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1733 { 1734 g_io_done2 = true; 1735 spdk_bdev_free_io(bdev_io); 1736 } 1737 1738 static void 1739 lock_lba_range_done(void *ctx, int status) 1740 { 1741 g_lock_lba_range_done = true; 1742 } 1743 1744 static void 1745 unlock_lba_range_done(void *ctx, int status) 1746 { 1747 g_unlock_lba_range_done = true; 1748 } 1749 1750 static uint32_t 1751 stub_channel_outstanding_cnt(void *io_target) 1752 { 1753 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 1754 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 1755 uint32_t outstanding_cnt; 1756 1757 outstanding_cnt = ch->outstanding_cnt; 1758 1759 spdk_put_io_channel(_ch); 1760 return outstanding_cnt; 1761 } 1762 1763 static void 1764 lock_lba_range_then_submit_io(void) 1765 { 1766 struct spdk_bdev_desc *desc = NULL; 1767 void *io_target; 1768 struct spdk_io_channel *io_ch[3]; 1769 struct spdk_bdev_channel *bdev_ch[3]; 1770 struct lba_range *range; 1771 char buf[4096]; 1772 int ctx0, ctx1, ctx2; 1773 int rc; 1774 1775 setup_test(); 1776 1777 io_target = g_bdev.io_target; 1778 desc = g_desc; 1779 1780 set_thread(0); 1781 io_ch[0] = spdk_bdev_get_io_channel(desc); 1782 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1783 CU_ASSERT(io_ch[0] != NULL); 1784 1785 set_thread(1); 1786 io_ch[1] = spdk_bdev_get_io_channel(desc); 1787 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1788 CU_ASSERT(io_ch[1] != NULL); 1789 1790 set_thread(0); 1791 g_lock_lba_range_done = false; 1792 rc = bdev_lock_lba_range(desc, io_ch[0], 20, 10, lock_lba_range_done, &ctx0); 1793 CU_ASSERT(rc == 0); 1794 poll_threads(); 1795 1796 /* The lock should immediately become valid, since there are no outstanding 1797 * write I/O. 1798 */ 1799 CU_ASSERT(g_lock_lba_range_done == true); 1800 range = TAILQ_FIRST(&bdev_ch[0]->locked_ranges); 1801 SPDK_CU_ASSERT_FATAL(range != NULL); 1802 CU_ASSERT(range->offset == 20); 1803 CU_ASSERT(range->length == 10); 1804 CU_ASSERT(range->owner_ch == bdev_ch[0]); 1805 1806 g_io_done = false; 1807 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 1808 rc = spdk_bdev_read_blocks(desc, io_ch[0], buf, 20, 1, io_done, &ctx0); 1809 CU_ASSERT(rc == 0); 1810 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 1811 1812 stub_complete_io(io_target, 1); 1813 poll_threads(); 1814 CU_ASSERT(g_io_done == true); 1815 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 1816 1817 /* Try a write I/O. This should actually be allowed to execute, since the channel 1818 * holding the lock is submitting the write I/O. 1819 */ 1820 g_io_done = false; 1821 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 1822 rc = spdk_bdev_write_blocks(desc, io_ch[0], buf, 20, 1, io_done, &ctx0); 1823 CU_ASSERT(rc == 0); 1824 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 1825 1826 stub_complete_io(io_target, 1); 1827 poll_threads(); 1828 CU_ASSERT(g_io_done == true); 1829 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->io_locked)); 1830 1831 /* Try a write I/O. This should get queued in the io_locked tailq. */ 1832 set_thread(1); 1833 g_io_done = false; 1834 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[1]->io_locked)); 1835 rc = spdk_bdev_write_blocks(desc, io_ch[1], buf, 20, 1, io_done, &ctx1); 1836 CU_ASSERT(rc == 0); 1837 poll_threads(); 1838 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 0); 1839 CU_ASSERT(!TAILQ_EMPTY(&bdev_ch[1]->io_locked)); 1840 CU_ASSERT(g_io_done == false); 1841 1842 /* Try to unlock the lba range using thread 1's io_ch. This should fail. */ 1843 rc = bdev_unlock_lba_range(desc, io_ch[1], 20, 10, unlock_lba_range_done, &ctx1); 1844 CU_ASSERT(rc == -EINVAL); 1845 1846 /* Now create a new channel and submit a write I/O with it. This should also be queued. 1847 * The new channel should inherit the active locks from the bdev's internal list. 1848 */ 1849 set_thread(2); 1850 io_ch[2] = spdk_bdev_get_io_channel(desc); 1851 bdev_ch[2] = spdk_io_channel_get_ctx(io_ch[2]); 1852 CU_ASSERT(io_ch[2] != NULL); 1853 1854 g_io_done2 = false; 1855 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[2]->io_locked)); 1856 rc = spdk_bdev_write_blocks(desc, io_ch[2], buf, 22, 2, io_done2, &ctx2); 1857 CU_ASSERT(rc == 0); 1858 poll_threads(); 1859 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 0); 1860 CU_ASSERT(!TAILQ_EMPTY(&bdev_ch[2]->io_locked)); 1861 CU_ASSERT(g_io_done2 == false); 1862 1863 set_thread(0); 1864 rc = bdev_unlock_lba_range(desc, io_ch[0], 20, 10, unlock_lba_range_done, &ctx0); 1865 CU_ASSERT(rc == 0); 1866 poll_threads(); 1867 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[0]->locked_ranges)); 1868 1869 /* The LBA range is unlocked, so the write IOs should now have started execution. */ 1870 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[1]->io_locked)); 1871 CU_ASSERT(TAILQ_EMPTY(&bdev_ch[2]->io_locked)); 1872 1873 set_thread(1); 1874 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 1875 stub_complete_io(io_target, 1); 1876 set_thread(2); 1877 CU_ASSERT(stub_channel_outstanding_cnt(io_target) == 1); 1878 stub_complete_io(io_target, 1); 1879 1880 poll_threads(); 1881 CU_ASSERT(g_io_done == true); 1882 CU_ASSERT(g_io_done2 == true); 1883 1884 /* Tear down the channels */ 1885 set_thread(0); 1886 spdk_put_io_channel(io_ch[0]); 1887 set_thread(1); 1888 spdk_put_io_channel(io_ch[1]); 1889 set_thread(2); 1890 spdk_put_io_channel(io_ch[2]); 1891 poll_threads(); 1892 set_thread(0); 1893 teardown_test(); 1894 } 1895 1896 int 1897 main(int argc, char **argv) 1898 { 1899 CU_pSuite suite = NULL; 1900 unsigned int num_failures; 1901 1902 CU_set_error_action(CUEA_ABORT); 1903 CU_initialize_registry(); 1904 1905 suite = CU_add_suite("bdev", NULL, NULL); 1906 1907 CU_ADD_TEST(suite, basic); 1908 CU_ADD_TEST(suite, unregister_and_close); 1909 CU_ADD_TEST(suite, basic_qos); 1910 CU_ADD_TEST(suite, put_channel_during_reset); 1911 CU_ADD_TEST(suite, aborted_reset); 1912 CU_ADD_TEST(suite, io_during_reset); 1913 CU_ADD_TEST(suite, io_during_qos_queue); 1914 CU_ADD_TEST(suite, io_during_qos_reset); 1915 CU_ADD_TEST(suite, enomem); 1916 CU_ADD_TEST(suite, enomem_multi_bdev); 1917 CU_ADD_TEST(suite, enomem_multi_io_target); 1918 CU_ADD_TEST(suite, qos_dynamic_enable); 1919 CU_ADD_TEST(suite, bdev_histograms_mt); 1920 CU_ADD_TEST(suite, bdev_set_io_timeout_mt); 1921 CU_ADD_TEST(suite, lock_lba_range_then_submit_io); 1922 1923 CU_basic_set_mode(CU_BRM_VERBOSE); 1924 CU_basic_run_tests(); 1925 num_failures = CU_get_number_of_failures(); 1926 CU_cleanup_registry(); 1927 return num_failures; 1928 } 1929