1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk_cunit.h" 35 36 #include "common/lib/ut_multithread.c" 37 #include "unit/lib/json_mock.c" 38 39 #include "spdk/config.h" 40 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 41 #undef SPDK_CONFIG_VTUNE 42 43 #include "bdev/bdev.c" 44 45 #define BDEV_UT_NUM_THREADS 3 46 47 DEFINE_STUB(spdk_conf_find_section, struct spdk_conf_section *, (struct spdk_conf *cp, 48 const char *name), NULL); 49 DEFINE_STUB(spdk_conf_section_get_nmval, char *, 50 (struct spdk_conf_section *sp, const char *key, int idx1, int idx2), NULL); 51 DEFINE_STUB(spdk_conf_section_get_intval, int, (struct spdk_conf_section *sp, const char *key), -1); 52 53 struct spdk_trace_histories *g_trace_histories; 54 DEFINE_STUB_V(spdk_trace_add_register_fn, (struct spdk_trace_register_fn *reg_fn)); 55 DEFINE_STUB_V(spdk_trace_register_owner, (uint8_t type, char id_prefix)); 56 DEFINE_STUB_V(spdk_trace_register_object, (uint8_t type, char id_prefix)); 57 DEFINE_STUB_V(spdk_trace_register_description, (const char *name, const char *short_name, 58 uint16_t tpoint_id, uint8_t owner_type, 59 uint8_t object_type, uint8_t new_object, 60 uint8_t arg1_is_ptr, const char *arg1_name)); 61 DEFINE_STUB_V(_spdk_trace_record, (uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, 62 uint32_t size, uint64_t object_id, uint64_t arg1)); 63 64 struct ut_bdev { 65 struct spdk_bdev bdev; 66 void *io_target; 67 }; 68 69 struct ut_bdev_channel { 70 TAILQ_HEAD(, spdk_bdev_io) outstanding_io; 71 uint32_t outstanding_cnt; 72 uint32_t avail_cnt; 73 }; 74 75 int g_io_device; 76 struct ut_bdev g_bdev; 77 struct spdk_bdev_desc *g_desc; 78 bool g_teardown_done = false; 79 bool g_get_io_channel = true; 80 bool g_create_ch = true; 81 bool g_init_complete_called = false; 82 bool g_fini_start_called = true; 83 int g_status = 0; 84 int g_count = 0; 85 struct spdk_histogram_data *g_histogram = NULL; 86 87 static int 88 stub_create_ch(void *io_device, void *ctx_buf) 89 { 90 struct ut_bdev_channel *ch = ctx_buf; 91 92 if (g_create_ch == false) { 93 return -1; 94 } 95 96 TAILQ_INIT(&ch->outstanding_io); 97 ch->outstanding_cnt = 0; 98 /* 99 * When avail gets to 0, the submit_request function will return ENOMEM. 100 * Most tests to not want ENOMEM to occur, so by default set this to a 101 * big value that won't get hit. The ENOMEM tests can then override this 102 * value to something much smaller to induce ENOMEM conditions. 103 */ 104 ch->avail_cnt = 2048; 105 return 0; 106 } 107 108 static void 109 stub_destroy_ch(void *io_device, void *ctx_buf) 110 { 111 } 112 113 static struct spdk_io_channel * 114 stub_get_io_channel(void *ctx) 115 { 116 struct ut_bdev *ut_bdev = ctx; 117 118 if (g_get_io_channel == true) { 119 return spdk_get_io_channel(ut_bdev->io_target); 120 } else { 121 return NULL; 122 } 123 } 124 125 static int 126 stub_destruct(void *ctx) 127 { 128 return 0; 129 } 130 131 static void 132 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 133 { 134 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 135 136 if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { 137 struct spdk_bdev_io *io; 138 139 while (!TAILQ_EMPTY(&ch->outstanding_io)) { 140 io = TAILQ_FIRST(&ch->outstanding_io); 141 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 142 ch->outstanding_cnt--; 143 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_FAILED); 144 ch->avail_cnt++; 145 } 146 } 147 148 if (ch->avail_cnt > 0) { 149 TAILQ_INSERT_TAIL(&ch->outstanding_io, bdev_io, module_link); 150 ch->outstanding_cnt++; 151 ch->avail_cnt--; 152 } else { 153 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); 154 } 155 } 156 157 static uint32_t 158 stub_complete_io(void *io_target, uint32_t num_to_complete) 159 { 160 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 161 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 162 struct spdk_bdev_io *io; 163 bool complete_all = (num_to_complete == 0); 164 uint32_t num_completed = 0; 165 166 while (complete_all || num_completed < num_to_complete) { 167 if (TAILQ_EMPTY(&ch->outstanding_io)) { 168 break; 169 } 170 io = TAILQ_FIRST(&ch->outstanding_io); 171 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 172 ch->outstanding_cnt--; 173 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_SUCCESS); 174 ch->avail_cnt++; 175 num_completed++; 176 } 177 178 spdk_put_io_channel(_ch); 179 return num_completed; 180 } 181 182 static struct spdk_bdev_fn_table fn_table = { 183 .get_io_channel = stub_get_io_channel, 184 .destruct = stub_destruct, 185 .submit_request = stub_submit_request, 186 }; 187 188 static int 189 module_init(void) 190 { 191 return 0; 192 } 193 194 static void 195 module_fini(void) 196 { 197 } 198 199 static void 200 init_complete(void) 201 { 202 g_init_complete_called = true; 203 } 204 205 static void 206 fini_start(void) 207 { 208 g_fini_start_called = true; 209 } 210 211 struct spdk_bdev_module bdev_ut_if = { 212 .name = "bdev_ut", 213 .module_init = module_init, 214 .module_fini = module_fini, 215 .init_complete = init_complete, 216 .fini_start = fini_start, 217 }; 218 219 SPDK_BDEV_MODULE_REGISTER(bdev_ut, &bdev_ut_if) 220 221 static void 222 register_bdev(struct ut_bdev *ut_bdev, char *name, void *io_target) 223 { 224 memset(ut_bdev, 0, sizeof(*ut_bdev)); 225 226 ut_bdev->io_target = io_target; 227 ut_bdev->bdev.ctxt = ut_bdev; 228 ut_bdev->bdev.name = name; 229 ut_bdev->bdev.fn_table = &fn_table; 230 ut_bdev->bdev.module = &bdev_ut_if; 231 ut_bdev->bdev.blocklen = 4096; 232 ut_bdev->bdev.blockcnt = 1024; 233 234 spdk_bdev_register(&ut_bdev->bdev); 235 } 236 237 static void 238 unregister_bdev(struct ut_bdev *ut_bdev) 239 { 240 /* Handle any deferred messages. */ 241 poll_threads(); 242 spdk_bdev_unregister(&ut_bdev->bdev, NULL, NULL); 243 } 244 245 static void 246 bdev_init_cb(void *done, int rc) 247 { 248 CU_ASSERT(rc == 0); 249 *(bool *)done = true; 250 } 251 252 static void 253 setup_test(void) 254 { 255 bool done = false; 256 257 allocate_threads(BDEV_UT_NUM_THREADS); 258 set_thread(0); 259 spdk_bdev_initialize(bdev_init_cb, &done); 260 spdk_io_device_register(&g_io_device, stub_create_ch, stub_destroy_ch, 261 sizeof(struct ut_bdev_channel), NULL); 262 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 263 spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &g_desc); 264 } 265 266 static void 267 finish_cb(void *cb_arg) 268 { 269 g_teardown_done = true; 270 } 271 272 static void 273 teardown_test(void) 274 { 275 set_thread(0); 276 g_teardown_done = false; 277 spdk_bdev_close(g_desc); 278 g_desc = NULL; 279 unregister_bdev(&g_bdev); 280 spdk_io_device_unregister(&g_io_device, NULL); 281 spdk_bdev_finish(finish_cb, NULL); 282 poll_threads(); 283 memset(&g_bdev, 0, sizeof(g_bdev)); 284 CU_ASSERT(g_teardown_done == true); 285 g_teardown_done = false; 286 free_threads(); 287 } 288 289 static uint32_t 290 bdev_io_tailq_cnt(bdev_io_tailq_t *tailq) 291 { 292 struct spdk_bdev_io *io; 293 uint32_t cnt = 0; 294 295 TAILQ_FOREACH(io, tailq, internal.link) { 296 cnt++; 297 } 298 299 return cnt; 300 } 301 302 static void 303 basic(void) 304 { 305 g_init_complete_called = false; 306 setup_test(); 307 CU_ASSERT(g_init_complete_called == true); 308 309 set_thread(0); 310 311 g_get_io_channel = false; 312 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 313 CU_ASSERT(g_ut_threads[0].ch == NULL); 314 315 g_get_io_channel = true; 316 g_create_ch = false; 317 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 318 CU_ASSERT(g_ut_threads[0].ch == NULL); 319 320 g_get_io_channel = true; 321 g_create_ch = true; 322 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 323 CU_ASSERT(g_ut_threads[0].ch != NULL); 324 spdk_put_io_channel(g_ut_threads[0].ch); 325 326 g_fini_start_called = false; 327 teardown_test(); 328 CU_ASSERT(g_fini_start_called == true); 329 } 330 331 static void 332 _bdev_removed(void *done) 333 { 334 *(bool *)done = true; 335 } 336 337 static void 338 _bdev_unregistered(void *done, int rc) 339 { 340 CU_ASSERT(rc == 0); 341 *(bool *)done = true; 342 } 343 344 static void 345 unregister_and_close(void) 346 { 347 bool done, remove_notify; 348 struct spdk_bdev_desc *desc; 349 350 setup_test(); 351 set_thread(0); 352 353 /* setup_test() automatically opens the bdev, 354 * but this test needs to do that in a different 355 * way. */ 356 spdk_bdev_close(g_desc); 357 poll_threads(); 358 359 remove_notify = false; 360 spdk_bdev_open(&g_bdev.bdev, true, _bdev_removed, &remove_notify, &desc); 361 CU_ASSERT(remove_notify == false); 362 CU_ASSERT(desc != NULL); 363 364 /* There is an open descriptor on the device. Unregister it, 365 * which can't proceed until the descriptor is closed. */ 366 done = false; 367 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 368 /* No polling has occurred, so neither of these should execute */ 369 CU_ASSERT(remove_notify == false); 370 CU_ASSERT(done == false); 371 372 /* Prior to the unregister completing, close the descriptor */ 373 spdk_bdev_close(desc); 374 375 /* Poll the threads to allow all events to be processed */ 376 poll_threads(); 377 378 /* Remove notify should not have been called because the 379 * descriptor is already closed. */ 380 CU_ASSERT(remove_notify == false); 381 382 /* The unregister should have completed */ 383 CU_ASSERT(done == true); 384 385 spdk_bdev_finish(finish_cb, NULL); 386 poll_threads(); 387 free_threads(); 388 } 389 390 static void 391 reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 392 { 393 bool *done = cb_arg; 394 395 CU_ASSERT(success == true); 396 *done = true; 397 spdk_bdev_free_io(bdev_io); 398 } 399 400 static void 401 put_channel_during_reset(void) 402 { 403 struct spdk_io_channel *io_ch; 404 bool done = false; 405 406 setup_test(); 407 408 set_thread(0); 409 io_ch = spdk_bdev_get_io_channel(g_desc); 410 CU_ASSERT(io_ch != NULL); 411 412 /* 413 * Start a reset, but then put the I/O channel before 414 * the deferred messages for the reset get a chance to 415 * execute. 416 */ 417 spdk_bdev_reset(g_desc, io_ch, reset_done, &done); 418 spdk_put_io_channel(io_ch); 419 poll_threads(); 420 stub_complete_io(g_bdev.io_target, 0); 421 422 teardown_test(); 423 } 424 425 static void 426 aborted_reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 427 { 428 enum spdk_bdev_io_status *status = cb_arg; 429 430 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 431 spdk_bdev_free_io(bdev_io); 432 } 433 434 static void 435 aborted_reset(void) 436 { 437 struct spdk_io_channel *io_ch[2]; 438 enum spdk_bdev_io_status status1 = SPDK_BDEV_IO_STATUS_PENDING, 439 status2 = SPDK_BDEV_IO_STATUS_PENDING; 440 441 setup_test(); 442 443 set_thread(0); 444 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 445 CU_ASSERT(io_ch[0] != NULL); 446 spdk_bdev_reset(g_desc, io_ch[0], aborted_reset_done, &status1); 447 poll_threads(); 448 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 449 450 /* 451 * First reset has been submitted on ch0. Now submit a second 452 * reset on ch1 which will get queued since there is already a 453 * reset in progress. 454 */ 455 set_thread(1); 456 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 457 CU_ASSERT(io_ch[1] != NULL); 458 spdk_bdev_reset(g_desc, io_ch[1], aborted_reset_done, &status2); 459 poll_threads(); 460 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 461 462 /* 463 * Now destroy ch1. This will abort the queued reset. Check that 464 * the second reset was completed with failed status. Also check 465 * that bdev->internal.reset_in_progress != NULL, since the 466 * original reset has not been completed yet. This ensures that 467 * the bdev code is correctly noticing that the failed reset is 468 * *not* the one that had been submitted to the bdev module. 469 */ 470 set_thread(1); 471 spdk_put_io_channel(io_ch[1]); 472 poll_threads(); 473 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_FAILED); 474 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 475 476 /* 477 * Now complete the first reset, verify that it completed with SUCCESS 478 * status and that bdev->internal.reset_in_progress is also set back to NULL. 479 */ 480 set_thread(0); 481 spdk_put_io_channel(io_ch[0]); 482 stub_complete_io(g_bdev.io_target, 0); 483 poll_threads(); 484 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 485 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 486 487 teardown_test(); 488 } 489 490 static void 491 io_during_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 492 { 493 enum spdk_bdev_io_status *status = cb_arg; 494 495 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 496 spdk_bdev_free_io(bdev_io); 497 } 498 499 static void 500 io_during_reset(void) 501 { 502 struct spdk_io_channel *io_ch[2]; 503 struct spdk_bdev_channel *bdev_ch[2]; 504 enum spdk_bdev_io_status status0, status1, status_reset; 505 int rc; 506 507 setup_test(); 508 509 /* 510 * First test normal case - submit an I/O on each of two channels (with no resets) 511 * and verify they complete successfully. 512 */ 513 set_thread(0); 514 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 515 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 516 CU_ASSERT(bdev_ch[0]->flags == 0); 517 status0 = SPDK_BDEV_IO_STATUS_PENDING; 518 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 519 CU_ASSERT(rc == 0); 520 521 set_thread(1); 522 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 523 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 524 CU_ASSERT(bdev_ch[1]->flags == 0); 525 status1 = SPDK_BDEV_IO_STATUS_PENDING; 526 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 527 CU_ASSERT(rc == 0); 528 529 poll_threads(); 530 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 531 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 532 533 set_thread(0); 534 stub_complete_io(g_bdev.io_target, 0); 535 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 536 537 set_thread(1); 538 stub_complete_io(g_bdev.io_target, 0); 539 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 540 541 /* 542 * Now submit a reset, and leave it pending while we submit I/O on two different 543 * channels. These I/O should be failed by the bdev layer since the reset is in 544 * progress. 545 */ 546 set_thread(0); 547 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 548 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &status_reset); 549 CU_ASSERT(rc == 0); 550 551 CU_ASSERT(bdev_ch[0]->flags == 0); 552 CU_ASSERT(bdev_ch[1]->flags == 0); 553 poll_threads(); 554 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_RESET_IN_PROGRESS); 555 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_RESET_IN_PROGRESS); 556 557 set_thread(0); 558 status0 = SPDK_BDEV_IO_STATUS_PENDING; 559 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 560 CU_ASSERT(rc == 0); 561 562 set_thread(1); 563 status1 = SPDK_BDEV_IO_STATUS_PENDING; 564 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 565 CU_ASSERT(rc == 0); 566 567 /* 568 * A reset is in progress so these read I/O should complete with failure. Note that we 569 * need to poll_threads() since I/O completed inline have their completion deferred. 570 */ 571 poll_threads(); 572 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 573 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 574 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 575 576 /* 577 * Complete the reset 578 */ 579 set_thread(0); 580 stub_complete_io(g_bdev.io_target, 0); 581 582 /* 583 * Only poll thread 0. We should not get a completion. 584 */ 585 poll_thread(0); 586 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 587 588 /* 589 * Poll both thread 0 and 1 so the messages can propagate and we 590 * get a completion. 591 */ 592 poll_threads(); 593 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 594 595 spdk_put_io_channel(io_ch[0]); 596 set_thread(1); 597 spdk_put_io_channel(io_ch[1]); 598 poll_threads(); 599 600 teardown_test(); 601 } 602 603 static void 604 basic_qos(void) 605 { 606 struct spdk_io_channel *io_ch[2]; 607 struct spdk_bdev_channel *bdev_ch[2]; 608 struct spdk_bdev *bdev; 609 enum spdk_bdev_io_status status; 610 int rc; 611 612 setup_test(); 613 614 /* Enable QoS */ 615 bdev = &g_bdev.bdev; 616 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 617 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 618 TAILQ_INIT(&bdev->internal.qos->queued); 619 /* 620 * Enable read/write IOPS, read only byte per second and 621 * read/write byte per second rate limits. 622 * In this case, all rate limits will take equal effect. 623 */ 624 /* 2000 read/write I/O per second, or 2 per millisecond */ 625 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 626 /* 8K read/write byte per millisecond with 4K block size */ 627 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 628 /* 8K read only byte per millisecond with 4K block size */ 629 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 8192000; 630 631 g_get_io_channel = true; 632 633 set_thread(0); 634 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 635 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 636 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 637 638 set_thread(1); 639 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 640 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 641 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 642 643 /* 644 * Send an I/O on thread 0, which is where the QoS thread is running. 645 */ 646 set_thread(0); 647 status = SPDK_BDEV_IO_STATUS_PENDING; 648 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status); 649 CU_ASSERT(rc == 0); 650 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 651 poll_threads(); 652 stub_complete_io(g_bdev.io_target, 0); 653 poll_threads(); 654 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 655 656 /* Send an I/O on thread 1. The QoS thread is not running here. */ 657 status = SPDK_BDEV_IO_STATUS_PENDING; 658 set_thread(1); 659 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status); 660 CU_ASSERT(rc == 0); 661 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 662 poll_threads(); 663 /* Complete I/O on thread 1. This should not complete the I/O we submitted */ 664 stub_complete_io(g_bdev.io_target, 0); 665 poll_threads(); 666 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 667 /* Now complete I/O on thread 0 */ 668 set_thread(0); 669 poll_threads(); 670 stub_complete_io(g_bdev.io_target, 0); 671 poll_threads(); 672 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 673 674 /* 675 * Close the descriptor only, which should stop the qos channel as 676 * the last descriptor removed. 677 */ 678 spdk_bdev_close(g_desc); 679 poll_threads(); 680 CU_ASSERT(bdev->internal.qos->ch == NULL); 681 682 /* 683 * Open the bdev again which shall setup the qos channel as the 684 * channels are valid. 685 */ 686 spdk_bdev_open(bdev, true, NULL, NULL, &g_desc); 687 poll_threads(); 688 CU_ASSERT(bdev->internal.qos->ch != NULL); 689 690 /* Tear down the channels */ 691 set_thread(0); 692 spdk_put_io_channel(io_ch[0]); 693 set_thread(1); 694 spdk_put_io_channel(io_ch[1]); 695 poll_threads(); 696 set_thread(0); 697 698 /* Close the descriptor, which should stop the qos channel */ 699 spdk_bdev_close(g_desc); 700 poll_threads(); 701 CU_ASSERT(bdev->internal.qos->ch == NULL); 702 703 /* Open the bdev again, no qos channel setup without valid channels. */ 704 spdk_bdev_open(bdev, true, NULL, NULL, &g_desc); 705 poll_threads(); 706 CU_ASSERT(bdev->internal.qos->ch == NULL); 707 708 /* Create the channels in reverse order. */ 709 set_thread(1); 710 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 711 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 712 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 713 714 set_thread(0); 715 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 716 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 717 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 718 719 /* Confirm that the qos thread is now thread 1 */ 720 CU_ASSERT(bdev->internal.qos->ch == bdev_ch[1]); 721 722 /* Tear down the channels */ 723 set_thread(0); 724 spdk_put_io_channel(io_ch[0]); 725 set_thread(1); 726 spdk_put_io_channel(io_ch[1]); 727 poll_threads(); 728 729 set_thread(0); 730 731 teardown_test(); 732 } 733 734 static void 735 io_during_qos_queue(void) 736 { 737 struct spdk_io_channel *io_ch[2]; 738 struct spdk_bdev_channel *bdev_ch[2]; 739 struct spdk_bdev *bdev; 740 enum spdk_bdev_io_status status0, status1, status2; 741 int rc; 742 743 setup_test(); 744 MOCK_SET(spdk_get_ticks, 0); 745 746 /* Enable QoS */ 747 bdev = &g_bdev.bdev; 748 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 749 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 750 TAILQ_INIT(&bdev->internal.qos->queued); 751 /* 752 * Enable read/write IOPS, read only byte per sec, write only 753 * byte per sec and read/write byte per sec rate limits. 754 * In this case, both read only and write only byte per sec 755 * rate limit will take effect. 756 */ 757 /* 4000 read/write I/O per second, or 4 per millisecond */ 758 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 4000; 759 /* 8K byte per millisecond with 4K block size */ 760 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 761 /* 4K byte per millisecond with 4K block size */ 762 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT].limit = 4096000; 763 /* 4K byte per millisecond with 4K block size */ 764 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 4096000; 765 766 g_get_io_channel = true; 767 768 /* Create channels */ 769 set_thread(0); 770 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 771 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 772 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 773 774 set_thread(1); 775 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 776 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 777 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 778 779 /* Send two read I/Os */ 780 status1 = SPDK_BDEV_IO_STATUS_PENDING; 781 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 782 CU_ASSERT(rc == 0); 783 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 784 set_thread(0); 785 status0 = SPDK_BDEV_IO_STATUS_PENDING; 786 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 787 CU_ASSERT(rc == 0); 788 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 789 /* Send one write I/O */ 790 status2 = SPDK_BDEV_IO_STATUS_PENDING; 791 rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status2); 792 CU_ASSERT(rc == 0); 793 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_PENDING); 794 795 /* Complete any I/O that arrived at the disk */ 796 poll_threads(); 797 set_thread(1); 798 stub_complete_io(g_bdev.io_target, 0); 799 set_thread(0); 800 stub_complete_io(g_bdev.io_target, 0); 801 poll_threads(); 802 803 /* Only one of the two read I/Os should complete. (logical XOR) */ 804 if (status0 == SPDK_BDEV_IO_STATUS_SUCCESS) { 805 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 806 } else { 807 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 808 } 809 /* The write I/O should complete. */ 810 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_SUCCESS); 811 812 /* Advance in time by a millisecond */ 813 spdk_delay_us(1000); 814 815 /* Complete more I/O */ 816 poll_threads(); 817 set_thread(1); 818 stub_complete_io(g_bdev.io_target, 0); 819 set_thread(0); 820 stub_complete_io(g_bdev.io_target, 0); 821 poll_threads(); 822 823 /* Now the second read I/O should be done */ 824 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 825 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 826 827 /* Tear down the channels */ 828 set_thread(1); 829 spdk_put_io_channel(io_ch[1]); 830 set_thread(0); 831 spdk_put_io_channel(io_ch[0]); 832 poll_threads(); 833 834 teardown_test(); 835 } 836 837 static void 838 io_during_qos_reset(void) 839 { 840 struct spdk_io_channel *io_ch[2]; 841 struct spdk_bdev_channel *bdev_ch[2]; 842 struct spdk_bdev *bdev; 843 enum spdk_bdev_io_status status0, status1, reset_status; 844 int rc; 845 846 setup_test(); 847 MOCK_SET(spdk_get_ticks, 0); 848 849 /* Enable QoS */ 850 bdev = &g_bdev.bdev; 851 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 852 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 853 TAILQ_INIT(&bdev->internal.qos->queued); 854 /* 855 * Enable read/write IOPS, write only byte per sec and 856 * read/write byte per second rate limits. 857 * In this case, read/write byte per second rate limit will 858 * take effect first. 859 */ 860 /* 2000 read/write I/O per second, or 2 per millisecond */ 861 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 862 /* 4K byte per millisecond with 4K block size */ 863 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 4096000; 864 /* 8K byte per millisecond with 4K block size */ 865 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT].limit = 8192000; 866 867 g_get_io_channel = true; 868 869 /* Create channels */ 870 set_thread(0); 871 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 872 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 873 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 874 875 set_thread(1); 876 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 877 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 878 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 879 880 /* Send two I/O. One of these gets queued by QoS. The other is sitting at the disk. */ 881 status1 = SPDK_BDEV_IO_STATUS_PENDING; 882 rc = spdk_bdev_write_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 883 CU_ASSERT(rc == 0); 884 set_thread(0); 885 status0 = SPDK_BDEV_IO_STATUS_PENDING; 886 rc = spdk_bdev_write_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 887 CU_ASSERT(rc == 0); 888 889 poll_threads(); 890 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 891 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 892 893 /* Reset the bdev. */ 894 reset_status = SPDK_BDEV_IO_STATUS_PENDING; 895 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &reset_status); 896 CU_ASSERT(rc == 0); 897 898 /* Complete any I/O that arrived at the disk */ 899 poll_threads(); 900 set_thread(1); 901 stub_complete_io(g_bdev.io_target, 0); 902 set_thread(0); 903 stub_complete_io(g_bdev.io_target, 0); 904 poll_threads(); 905 906 CU_ASSERT(reset_status == SPDK_BDEV_IO_STATUS_SUCCESS); 907 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 908 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 909 910 /* Tear down the channels */ 911 set_thread(1); 912 spdk_put_io_channel(io_ch[1]); 913 set_thread(0); 914 spdk_put_io_channel(io_ch[0]); 915 poll_threads(); 916 917 teardown_test(); 918 } 919 920 static void 921 enomem_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 922 { 923 enum spdk_bdev_io_status *status = cb_arg; 924 925 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 926 spdk_bdev_free_io(bdev_io); 927 } 928 929 static void 930 enomem(void) 931 { 932 struct spdk_io_channel *io_ch; 933 struct spdk_bdev_channel *bdev_ch; 934 struct spdk_bdev_shared_resource *shared_resource; 935 struct ut_bdev_channel *ut_ch; 936 const uint32_t IO_ARRAY_SIZE = 64; 937 const uint32_t AVAIL = 20; 938 enum spdk_bdev_io_status status[IO_ARRAY_SIZE], status_reset; 939 uint32_t nomem_cnt, i; 940 struct spdk_bdev_io *first_io; 941 int rc; 942 943 setup_test(); 944 945 set_thread(0); 946 io_ch = spdk_bdev_get_io_channel(g_desc); 947 bdev_ch = spdk_io_channel_get_ctx(io_ch); 948 shared_resource = bdev_ch->shared_resource; 949 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 950 ut_ch->avail_cnt = AVAIL; 951 952 /* First submit a number of IOs equal to what the channel can support. */ 953 for (i = 0; i < AVAIL; i++) { 954 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 955 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 956 CU_ASSERT(rc == 0); 957 } 958 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 959 960 /* 961 * Next, submit one additional I/O. This one should fail with ENOMEM and then go onto 962 * the enomem_io list. 963 */ 964 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 965 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 966 CU_ASSERT(rc == 0); 967 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 968 first_io = TAILQ_FIRST(&shared_resource->nomem_io); 969 970 /* 971 * Now submit a bunch more I/O. These should all fail with ENOMEM and get queued behind 972 * the first_io above. 973 */ 974 for (i = AVAIL + 1; i < IO_ARRAY_SIZE; i++) { 975 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 976 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 977 CU_ASSERT(rc == 0); 978 } 979 980 /* Assert that first_io is still at the head of the list. */ 981 CU_ASSERT(TAILQ_FIRST(&shared_resource->nomem_io) == first_io); 982 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == (IO_ARRAY_SIZE - AVAIL)); 983 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 984 CU_ASSERT(shared_resource->nomem_threshold == (AVAIL - NOMEM_THRESHOLD_COUNT)); 985 986 /* 987 * Complete 1 I/O only. The key check here is bdev_io_tailq_cnt - this should not have 988 * changed since completing just 1 I/O should not trigger retrying the queued nomem_io 989 * list. 990 */ 991 stub_complete_io(g_bdev.io_target, 1); 992 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 993 994 /* 995 * Complete enough I/O to hit the nomem_theshold. This should trigger retrying nomem_io, 996 * and we should see I/O get resubmitted to the test bdev module. 997 */ 998 stub_complete_io(g_bdev.io_target, NOMEM_THRESHOLD_COUNT - 1); 999 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) < nomem_cnt); 1000 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 1001 1002 /* Complete 1 I/O only. This should not trigger retrying the queued nomem_io. */ 1003 stub_complete_io(g_bdev.io_target, 1); 1004 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 1005 1006 /* 1007 * Send a reset and confirm that all I/O are completed, including the ones that 1008 * were queued on the nomem_io list. 1009 */ 1010 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 1011 rc = spdk_bdev_reset(g_desc, io_ch, enomem_done, &status_reset); 1012 poll_threads(); 1013 CU_ASSERT(rc == 0); 1014 /* This will complete the reset. */ 1015 stub_complete_io(g_bdev.io_target, 0); 1016 1017 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == 0); 1018 CU_ASSERT(shared_resource->io_outstanding == 0); 1019 1020 spdk_put_io_channel(io_ch); 1021 poll_threads(); 1022 teardown_test(); 1023 } 1024 1025 static void 1026 enomem_multi_bdev(void) 1027 { 1028 struct spdk_io_channel *io_ch; 1029 struct spdk_bdev_channel *bdev_ch; 1030 struct spdk_bdev_shared_resource *shared_resource; 1031 struct ut_bdev_channel *ut_ch; 1032 const uint32_t IO_ARRAY_SIZE = 64; 1033 const uint32_t AVAIL = 20; 1034 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1035 uint32_t i; 1036 struct ut_bdev *second_bdev; 1037 struct spdk_bdev_desc *second_desc = NULL; 1038 struct spdk_bdev_channel *second_bdev_ch; 1039 struct spdk_io_channel *second_ch; 1040 int rc; 1041 1042 setup_test(); 1043 1044 /* Register second bdev with the same io_target */ 1045 second_bdev = calloc(1, sizeof(*second_bdev)); 1046 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1047 register_bdev(second_bdev, "ut_bdev2", g_bdev.io_target); 1048 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 1049 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1050 1051 set_thread(0); 1052 io_ch = spdk_bdev_get_io_channel(g_desc); 1053 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1054 shared_resource = bdev_ch->shared_resource; 1055 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1056 ut_ch->avail_cnt = AVAIL; 1057 1058 second_ch = spdk_bdev_get_io_channel(second_desc); 1059 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1060 SPDK_CU_ASSERT_FATAL(shared_resource == second_bdev_ch->shared_resource); 1061 1062 /* Saturate io_target through bdev A. */ 1063 for (i = 0; i < AVAIL; i++) { 1064 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1065 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1066 CU_ASSERT(rc == 0); 1067 } 1068 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1069 1070 /* 1071 * Now submit I/O through the second bdev. This should fail with ENOMEM 1072 * and then go onto the nomem_io list. 1073 */ 1074 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1075 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1076 CU_ASSERT(rc == 0); 1077 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1078 1079 /* Complete first bdev's I/O. This should retry sending second bdev's nomem_io */ 1080 stub_complete_io(g_bdev.io_target, AVAIL); 1081 1082 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&shared_resource->nomem_io)); 1083 CU_ASSERT(shared_resource->io_outstanding == 1); 1084 1085 /* Now complete our retried I/O */ 1086 stub_complete_io(g_bdev.io_target, 1); 1087 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == 0); 1088 1089 spdk_put_io_channel(io_ch); 1090 spdk_put_io_channel(second_ch); 1091 spdk_bdev_close(second_desc); 1092 unregister_bdev(second_bdev); 1093 poll_threads(); 1094 free(second_bdev); 1095 teardown_test(); 1096 } 1097 1098 1099 static void 1100 enomem_multi_io_target(void) 1101 { 1102 struct spdk_io_channel *io_ch; 1103 struct spdk_bdev_channel *bdev_ch; 1104 struct ut_bdev_channel *ut_ch; 1105 const uint32_t IO_ARRAY_SIZE = 64; 1106 const uint32_t AVAIL = 20; 1107 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1108 uint32_t i; 1109 int new_io_device; 1110 struct ut_bdev *second_bdev; 1111 struct spdk_bdev_desc *second_desc = NULL; 1112 struct spdk_bdev_channel *second_bdev_ch; 1113 struct spdk_io_channel *second_ch; 1114 int rc; 1115 1116 setup_test(); 1117 1118 /* Create new io_target and a second bdev using it */ 1119 spdk_io_device_register(&new_io_device, stub_create_ch, stub_destroy_ch, 1120 sizeof(struct ut_bdev_channel), NULL); 1121 second_bdev = calloc(1, sizeof(*second_bdev)); 1122 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1123 register_bdev(second_bdev, "ut_bdev2", &new_io_device); 1124 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 1125 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1126 1127 set_thread(0); 1128 io_ch = spdk_bdev_get_io_channel(g_desc); 1129 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1130 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1131 ut_ch->avail_cnt = AVAIL; 1132 1133 /* Different io_target should imply a different shared_resource */ 1134 second_ch = spdk_bdev_get_io_channel(second_desc); 1135 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1136 SPDK_CU_ASSERT_FATAL(bdev_ch->shared_resource != second_bdev_ch->shared_resource); 1137 1138 /* Saturate io_target through bdev A. */ 1139 for (i = 0; i < AVAIL; i++) { 1140 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1141 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1142 CU_ASSERT(rc == 0); 1143 } 1144 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1145 1146 /* Issue one more I/O to fill ENOMEM list. */ 1147 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1148 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1149 CU_ASSERT(rc == 0); 1150 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1151 1152 /* 1153 * Now submit I/O through the second bdev. This should go through and complete 1154 * successfully because we're using a different io_device underneath. 1155 */ 1156 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1157 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1158 CU_ASSERT(rc == 0); 1159 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&second_bdev_ch->shared_resource->nomem_io)); 1160 stub_complete_io(second_bdev->io_target, 1); 1161 1162 /* Cleanup; Complete outstanding I/O. */ 1163 stub_complete_io(g_bdev.io_target, AVAIL); 1164 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1165 /* Complete the ENOMEM I/O */ 1166 stub_complete_io(g_bdev.io_target, 1); 1167 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1168 1169 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1170 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1171 spdk_put_io_channel(io_ch); 1172 spdk_put_io_channel(second_ch); 1173 spdk_bdev_close(second_desc); 1174 unregister_bdev(second_bdev); 1175 spdk_io_device_unregister(&new_io_device, NULL); 1176 poll_threads(); 1177 free(second_bdev); 1178 teardown_test(); 1179 } 1180 1181 static void 1182 qos_dynamic_enable_done(void *cb_arg, int status) 1183 { 1184 int *rc = cb_arg; 1185 *rc = status; 1186 } 1187 1188 static void 1189 qos_dynamic_enable(void) 1190 { 1191 struct spdk_io_channel *io_ch[2]; 1192 struct spdk_bdev_channel *bdev_ch[2]; 1193 struct spdk_bdev *bdev; 1194 enum spdk_bdev_io_status bdev_io_status[2]; 1195 uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {}; 1196 int status, second_status, rc, i; 1197 1198 setup_test(); 1199 MOCK_SET(spdk_get_ticks, 0); 1200 1201 for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) { 1202 limits[i] = UINT64_MAX; 1203 } 1204 1205 bdev = &g_bdev.bdev; 1206 1207 g_get_io_channel = true; 1208 1209 /* Create channels */ 1210 set_thread(0); 1211 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1212 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1213 CU_ASSERT(bdev_ch[0]->flags == 0); 1214 1215 set_thread(1); 1216 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1217 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1218 CU_ASSERT(bdev_ch[1]->flags == 0); 1219 1220 set_thread(0); 1221 1222 /* 1223 * Enable QoS: Read/Write IOPS, Read/Write byte, 1224 * Read only byte and Write only byte per second 1225 * rate limits. 1226 * More than 10 I/Os allowed per timeslice. 1227 */ 1228 status = -1; 1229 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1230 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 100; 1231 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 100; 1232 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 10; 1233 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1234 poll_threads(); 1235 CU_ASSERT(status == 0); 1236 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1237 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1238 1239 /* 1240 * Submit and complete 10 I/O to fill the QoS allotment for this timeslice. 1241 * Additional I/O will then be queued. 1242 */ 1243 set_thread(0); 1244 for (i = 0; i < 10; i++) { 1245 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1246 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1247 CU_ASSERT(rc == 0); 1248 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1249 poll_thread(0); 1250 stub_complete_io(g_bdev.io_target, 0); 1251 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1252 } 1253 1254 /* 1255 * Send two more I/O. These I/O will be queued since the current timeslice allotment has been 1256 * filled already. We want to test that when QoS is disabled that these two I/O: 1257 * 1) are not aborted 1258 * 2) are sent back to their original thread for resubmission 1259 */ 1260 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1261 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1262 CU_ASSERT(rc == 0); 1263 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1264 set_thread(1); 1265 bdev_io_status[1] = SPDK_BDEV_IO_STATUS_PENDING; 1266 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &bdev_io_status[1]); 1267 CU_ASSERT(rc == 0); 1268 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1269 poll_threads(); 1270 1271 /* 1272 * Disable QoS: Read/Write IOPS, Read/Write byte, 1273 * Read only byte rate limits 1274 */ 1275 status = -1; 1276 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1277 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 0; 1278 limits[SPDK_BDEV_QOS_R_BPS_RATE_LIMIT] = 0; 1279 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1280 poll_threads(); 1281 CU_ASSERT(status == 0); 1282 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1283 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1284 1285 /* Disable QoS: Write only Byte per second rate limit */ 1286 status = -1; 1287 limits[SPDK_BDEV_QOS_W_BPS_RATE_LIMIT] = 0; 1288 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1289 poll_threads(); 1290 CU_ASSERT(status == 0); 1291 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1292 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1293 1294 /* 1295 * All I/O should have been resubmitted back on their original thread. Complete 1296 * all I/O on thread 0, and ensure that only the thread 0 I/O was completed. 1297 */ 1298 set_thread(0); 1299 stub_complete_io(g_bdev.io_target, 0); 1300 poll_threads(); 1301 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1302 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1303 1304 /* Now complete all I/O on thread 1 and ensure the thread 1 I/O was completed. */ 1305 set_thread(1); 1306 stub_complete_io(g_bdev.io_target, 0); 1307 poll_threads(); 1308 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_SUCCESS); 1309 1310 /* Disable QoS again */ 1311 status = -1; 1312 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1313 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1314 poll_threads(); 1315 CU_ASSERT(status == 0); /* This should succeed */ 1316 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1317 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1318 1319 /* Enable QoS on thread 0 */ 1320 status = -1; 1321 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1322 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1323 poll_threads(); 1324 CU_ASSERT(status == 0); 1325 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1326 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1327 1328 /* Disable QoS on thread 1 */ 1329 set_thread(1); 1330 status = -1; 1331 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1332 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1333 /* Don't poll yet. This should leave the channels with QoS enabled */ 1334 CU_ASSERT(status == -1); 1335 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1336 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1337 1338 /* Enable QoS. This should immediately fail because the previous disable QoS hasn't completed. */ 1339 second_status = 0; 1340 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 10; 1341 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &second_status); 1342 poll_threads(); 1343 CU_ASSERT(status == 0); /* The disable should succeed */ 1344 CU_ASSERT(second_status < 0); /* The enable should fail */ 1345 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1346 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1347 1348 /* Enable QoS on thread 1. This should succeed now that the disable has completed. */ 1349 status = -1; 1350 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1351 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1352 poll_threads(); 1353 CU_ASSERT(status == 0); 1354 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1355 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1356 1357 /* Tear down the channels */ 1358 set_thread(0); 1359 spdk_put_io_channel(io_ch[0]); 1360 set_thread(1); 1361 spdk_put_io_channel(io_ch[1]); 1362 poll_threads(); 1363 1364 set_thread(0); 1365 teardown_test(); 1366 } 1367 1368 static void 1369 histogram_status_cb(void *cb_arg, int status) 1370 { 1371 g_status = status; 1372 } 1373 1374 static void 1375 histogram_data_cb(void *cb_arg, int status, struct spdk_histogram_data *histogram) 1376 { 1377 g_status = status; 1378 g_histogram = histogram; 1379 } 1380 1381 static void 1382 histogram_io_count(void *ctx, uint64_t start, uint64_t end, uint64_t count, 1383 uint64_t total, uint64_t so_far) 1384 { 1385 g_count += count; 1386 } 1387 1388 static void 1389 bdev_histograms_mt(void) 1390 { 1391 struct spdk_io_channel *ch[2]; 1392 struct spdk_histogram_data *histogram; 1393 uint8_t buf[4096]; 1394 int status = false; 1395 int rc; 1396 1397 1398 setup_test(); 1399 1400 set_thread(0); 1401 ch[0] = spdk_bdev_get_io_channel(g_desc); 1402 CU_ASSERT(ch[0] != NULL); 1403 1404 set_thread(1); 1405 ch[1] = spdk_bdev_get_io_channel(g_desc); 1406 CU_ASSERT(ch[1] != NULL); 1407 1408 1409 /* Enable histogram */ 1410 spdk_bdev_histogram_enable(&g_bdev.bdev, histogram_status_cb, NULL, true); 1411 poll_threads(); 1412 CU_ASSERT(g_status == 0); 1413 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == true); 1414 1415 /* Allocate histogram */ 1416 histogram = spdk_histogram_data_alloc(); 1417 1418 /* Check if histogram is zeroed */ 1419 spdk_bdev_histogram_get(&g_bdev.bdev, histogram, histogram_data_cb, NULL); 1420 poll_threads(); 1421 CU_ASSERT(g_status == 0); 1422 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 1423 1424 g_count = 0; 1425 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 1426 1427 CU_ASSERT(g_count == 0); 1428 1429 set_thread(0); 1430 rc = spdk_bdev_write_blocks(g_desc, ch[0], &buf, 0, 1, io_during_io_done, &status); 1431 CU_ASSERT(rc == 0); 1432 1433 spdk_delay_us(10); 1434 stub_complete_io(g_bdev.io_target, 1); 1435 poll_threads(); 1436 CU_ASSERT(status == true); 1437 1438 1439 set_thread(1); 1440 rc = spdk_bdev_read_blocks(g_desc, ch[1], &buf, 0, 1, io_during_io_done, &status); 1441 CU_ASSERT(rc == 0); 1442 1443 spdk_delay_us(10); 1444 stub_complete_io(g_bdev.io_target, 1); 1445 poll_threads(); 1446 CU_ASSERT(status == true); 1447 1448 set_thread(0); 1449 1450 /* Check if histogram gathered data from all I/O channels */ 1451 spdk_bdev_histogram_get(&g_bdev.bdev, histogram, histogram_data_cb, NULL); 1452 poll_threads(); 1453 CU_ASSERT(g_status == 0); 1454 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == true); 1455 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 1456 1457 g_count = 0; 1458 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 1459 CU_ASSERT(g_count == 2); 1460 1461 /* Disable histogram */ 1462 spdk_bdev_histogram_enable(&g_bdev.bdev, histogram_status_cb, NULL, false); 1463 poll_threads(); 1464 CU_ASSERT(g_status == 0); 1465 CU_ASSERT(g_bdev.bdev.internal.histogram_enabled == false); 1466 1467 spdk_histogram_data_free(g_histogram); 1468 } 1469 1470 int 1471 main(int argc, char **argv) 1472 { 1473 CU_pSuite suite = NULL; 1474 unsigned int num_failures; 1475 1476 if (CU_initialize_registry() != CUE_SUCCESS) { 1477 return CU_get_error(); 1478 } 1479 1480 suite = CU_add_suite("bdev", NULL, NULL); 1481 if (suite == NULL) { 1482 CU_cleanup_registry(); 1483 return CU_get_error(); 1484 } 1485 1486 if ( 1487 CU_add_test(suite, "basic", basic) == NULL || 1488 CU_add_test(suite, "unregister_and_close", unregister_and_close) == NULL || 1489 CU_add_test(suite, "basic_qos", basic_qos) == NULL || 1490 CU_add_test(suite, "put_channel_during_reset", put_channel_during_reset) == NULL || 1491 CU_add_test(suite, "aborted_reset", aborted_reset) == NULL || 1492 CU_add_test(suite, "io_during_reset", io_during_reset) == NULL || 1493 CU_add_test(suite, "io_during_qos_queue", io_during_qos_queue) == NULL || 1494 CU_add_test(suite, "io_during_qos_reset", io_during_qos_reset) == NULL || 1495 CU_add_test(suite, "enomem", enomem) == NULL || 1496 CU_add_test(suite, "enomem_multi_bdev", enomem_multi_bdev) == NULL || 1497 CU_add_test(suite, "enomem_multi_io_target", enomem_multi_io_target) == NULL || 1498 CU_add_test(suite, "qos_dynamic_enable", qos_dynamic_enable) == NULL || 1499 CU_add_test(suite, "bdev_histograms_mt", bdev_histograms_mt) == NULL 1500 ) { 1501 CU_cleanup_registry(); 1502 return CU_get_error(); 1503 } 1504 1505 CU_basic_set_mode(CU_BRM_VERBOSE); 1506 CU_basic_run_tests(); 1507 num_failures = CU_get_number_of_failures(); 1508 CU_cleanup_registry(); 1509 return num_failures; 1510 } 1511