1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk_cunit.h" 35 36 #include "common/lib/test_env.c" 37 #include "common/lib/ut_multithread.c" 38 #include "unit/lib/json_mock.c" 39 40 #include "spdk/config.h" 41 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 42 #undef SPDK_CONFIG_VTUNE 43 44 #include "bdev/bdev.c" 45 46 #define BDEV_UT_NUM_THREADS 3 47 48 DEFINE_STUB_V(spdk_scsi_nvme_translate, (const struct spdk_bdev_io *bdev_io, 49 int *sc, int *sk, int *asc, int *ascq)); 50 51 DEFINE_STUB(spdk_conf_find_section, struct spdk_conf_section *, (struct spdk_conf *cp, 52 const char *name), NULL); 53 DEFINE_STUB(spdk_conf_section_get_nmval, char *, 54 (struct spdk_conf_section *sp, const char *key, int idx1, int idx2), NULL); 55 DEFINE_STUB(spdk_conf_section_get_intval, int, (struct spdk_conf_section *sp, const char *key), -1); 56 57 struct spdk_trace_histories *g_trace_histories; 58 DEFINE_STUB_V(spdk_trace_add_register_fn, (struct spdk_trace_register_fn *reg_fn)); 59 DEFINE_STUB_V(spdk_trace_register_owner, (uint8_t type, char id_prefix)); 60 DEFINE_STUB_V(spdk_trace_register_object, (uint8_t type, char id_prefix)); 61 DEFINE_STUB_V(spdk_trace_register_description, (const char *name, const char *short_name, 62 uint16_t tpoint_id, uint8_t owner_type, 63 uint8_t object_type, uint8_t new_object, 64 uint8_t arg1_is_ptr, const char *arg1_name)); 65 DEFINE_STUB_V(_spdk_trace_record, (uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, 66 uint32_t size, uint64_t object_id, uint64_t arg1)); 67 68 struct ut_bdev { 69 struct spdk_bdev bdev; 70 void *io_target; 71 }; 72 73 struct ut_bdev_channel { 74 TAILQ_HEAD(, spdk_bdev_io) outstanding_io; 75 uint32_t outstanding_cnt; 76 uint32_t avail_cnt; 77 }; 78 79 int g_io_device; 80 struct ut_bdev g_bdev; 81 struct spdk_bdev_desc *g_desc; 82 bool g_teardown_done = false; 83 bool g_get_io_channel = true; 84 bool g_create_ch = true; 85 bool g_init_complete_called = false; 86 bool g_fini_start_called = true; 87 88 static int 89 stub_create_ch(void *io_device, void *ctx_buf) 90 { 91 struct ut_bdev_channel *ch = ctx_buf; 92 93 if (g_create_ch == false) { 94 return -1; 95 } 96 97 TAILQ_INIT(&ch->outstanding_io); 98 ch->outstanding_cnt = 0; 99 /* 100 * When avail gets to 0, the submit_request function will return ENOMEM. 101 * Most tests to not want ENOMEM to occur, so by default set this to a 102 * big value that won't get hit. The ENOMEM tests can then override this 103 * value to something much smaller to induce ENOMEM conditions. 104 */ 105 ch->avail_cnt = 2048; 106 return 0; 107 } 108 109 static void 110 stub_destroy_ch(void *io_device, void *ctx_buf) 111 { 112 } 113 114 static struct spdk_io_channel * 115 stub_get_io_channel(void *ctx) 116 { 117 struct ut_bdev *ut_bdev = ctx; 118 119 if (g_get_io_channel == true) { 120 return spdk_get_io_channel(ut_bdev->io_target); 121 } else { 122 return NULL; 123 } 124 } 125 126 static int 127 stub_destruct(void *ctx) 128 { 129 return 0; 130 } 131 132 static void 133 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 134 { 135 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 136 137 if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { 138 struct spdk_bdev_io *io; 139 140 while (!TAILQ_EMPTY(&ch->outstanding_io)) { 141 io = TAILQ_FIRST(&ch->outstanding_io); 142 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 143 ch->outstanding_cnt--; 144 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_FAILED); 145 ch->avail_cnt++; 146 } 147 } 148 149 if (ch->avail_cnt > 0) { 150 TAILQ_INSERT_TAIL(&ch->outstanding_io, bdev_io, module_link); 151 ch->outstanding_cnt++; 152 ch->avail_cnt--; 153 } else { 154 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); 155 } 156 } 157 158 static uint32_t 159 stub_complete_io(void *io_target, uint32_t num_to_complete) 160 { 161 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 162 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 163 struct spdk_bdev_io *io; 164 bool complete_all = (num_to_complete == 0); 165 uint32_t num_completed = 0; 166 167 while (complete_all || num_completed < num_to_complete) { 168 if (TAILQ_EMPTY(&ch->outstanding_io)) { 169 break; 170 } 171 io = TAILQ_FIRST(&ch->outstanding_io); 172 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 173 ch->outstanding_cnt--; 174 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_SUCCESS); 175 ch->avail_cnt++; 176 num_completed++; 177 } 178 179 spdk_put_io_channel(_ch); 180 return num_completed; 181 } 182 183 static struct spdk_bdev_fn_table fn_table = { 184 .get_io_channel = stub_get_io_channel, 185 .destruct = stub_destruct, 186 .submit_request = stub_submit_request, 187 }; 188 189 static int 190 module_init(void) 191 { 192 return 0; 193 } 194 195 static void 196 module_fini(void) 197 { 198 } 199 200 static void 201 init_complete(void) 202 { 203 g_init_complete_called = true; 204 } 205 206 static void 207 fini_start(void) 208 { 209 g_fini_start_called = true; 210 } 211 212 struct spdk_bdev_module bdev_ut_if = { 213 .name = "bdev_ut", 214 .module_init = module_init, 215 .module_fini = module_fini, 216 .init_complete = init_complete, 217 .fini_start = fini_start, 218 }; 219 220 SPDK_BDEV_MODULE_REGISTER(&bdev_ut_if) 221 222 static void 223 register_bdev(struct ut_bdev *ut_bdev, char *name, void *io_target) 224 { 225 memset(ut_bdev, 0, sizeof(*ut_bdev)); 226 227 ut_bdev->io_target = io_target; 228 ut_bdev->bdev.ctxt = ut_bdev; 229 ut_bdev->bdev.name = name; 230 ut_bdev->bdev.fn_table = &fn_table; 231 ut_bdev->bdev.module = &bdev_ut_if; 232 ut_bdev->bdev.blocklen = 4096; 233 ut_bdev->bdev.blockcnt = 1024; 234 235 spdk_bdev_register(&ut_bdev->bdev); 236 } 237 238 static void 239 unregister_bdev(struct ut_bdev *ut_bdev) 240 { 241 /* Handle any deferred messages. */ 242 poll_threads(); 243 spdk_bdev_unregister(&ut_bdev->bdev, NULL, NULL); 244 } 245 246 static void 247 bdev_init_cb(void *done, int rc) 248 { 249 CU_ASSERT(rc == 0); 250 *(bool *)done = true; 251 } 252 253 static void 254 setup_test(void) 255 { 256 bool done = false; 257 258 allocate_threads(BDEV_UT_NUM_THREADS); 259 set_thread(0); 260 spdk_bdev_initialize(bdev_init_cb, &done); 261 spdk_io_device_register(&g_io_device, stub_create_ch, stub_destroy_ch, 262 sizeof(struct ut_bdev_channel), NULL); 263 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 264 spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &g_desc); 265 } 266 267 static void 268 finish_cb(void *cb_arg) 269 { 270 g_teardown_done = true; 271 } 272 273 static void 274 teardown_test(void) 275 { 276 set_thread(0); 277 g_teardown_done = false; 278 spdk_bdev_close(g_desc); 279 g_desc = NULL; 280 unregister_bdev(&g_bdev); 281 spdk_io_device_unregister(&g_io_device, NULL); 282 spdk_bdev_finish(finish_cb, NULL); 283 poll_threads(); 284 memset(&g_bdev, 0, sizeof(g_bdev)); 285 CU_ASSERT(g_teardown_done == true); 286 g_teardown_done = false; 287 free_threads(); 288 } 289 290 static uint32_t 291 bdev_io_tailq_cnt(bdev_io_tailq_t *tailq) 292 { 293 struct spdk_bdev_io *io; 294 uint32_t cnt = 0; 295 296 TAILQ_FOREACH(io, tailq, internal.link) { 297 cnt++; 298 } 299 300 return cnt; 301 } 302 303 static void 304 basic(void) 305 { 306 g_init_complete_called = false; 307 setup_test(); 308 CU_ASSERT(g_init_complete_called == true); 309 310 set_thread(0); 311 312 g_get_io_channel = false; 313 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 314 CU_ASSERT(g_ut_threads[0].ch == NULL); 315 316 g_get_io_channel = true; 317 g_create_ch = false; 318 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 319 CU_ASSERT(g_ut_threads[0].ch == NULL); 320 321 g_get_io_channel = true; 322 g_create_ch = true; 323 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 324 CU_ASSERT(g_ut_threads[0].ch != NULL); 325 spdk_put_io_channel(g_ut_threads[0].ch); 326 327 g_fini_start_called = false; 328 teardown_test(); 329 CU_ASSERT(g_fini_start_called == true); 330 } 331 332 static void 333 _bdev_removed(void *done) 334 { 335 *(bool *)done = true; 336 } 337 338 static void 339 _bdev_unregistered(void *done, int rc) 340 { 341 CU_ASSERT(rc == 0); 342 *(bool *)done = true; 343 } 344 345 static void 346 unregister_and_close(void) 347 { 348 bool done, remove_notify; 349 struct spdk_bdev_desc *desc; 350 351 setup_test(); 352 set_thread(0); 353 354 /* setup_test() automatically opens the bdev, 355 * but this test needs to do that in a different 356 * way. */ 357 spdk_bdev_close(g_desc); 358 poll_threads(); 359 360 remove_notify = false; 361 spdk_bdev_open(&g_bdev.bdev, true, _bdev_removed, &remove_notify, &desc); 362 CU_ASSERT(remove_notify == false); 363 CU_ASSERT(desc != NULL); 364 365 /* There is an open descriptor on the device. Unregister it, 366 * which can't proceed until the descriptor is closed. */ 367 done = false; 368 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 369 /* No polling has occurred, so neither of these should execute */ 370 CU_ASSERT(remove_notify == false); 371 CU_ASSERT(done == false); 372 373 /* Prior to the unregister completing, close the descriptor */ 374 spdk_bdev_close(desc); 375 376 /* Poll the threads to allow all events to be processed */ 377 poll_threads(); 378 379 /* Remove notify should not have been called because the 380 * descriptor is already closed. */ 381 CU_ASSERT(remove_notify == false); 382 383 /* The unregister should have completed */ 384 CU_ASSERT(done == true); 385 386 spdk_bdev_finish(finish_cb, NULL); 387 poll_threads(); 388 free_threads(); 389 } 390 391 static void 392 reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 393 { 394 bool *done = cb_arg; 395 396 CU_ASSERT(success == true); 397 *done = true; 398 spdk_bdev_free_io(bdev_io); 399 } 400 401 static void 402 put_channel_during_reset(void) 403 { 404 struct spdk_io_channel *io_ch; 405 bool done = false; 406 407 setup_test(); 408 409 set_thread(0); 410 io_ch = spdk_bdev_get_io_channel(g_desc); 411 CU_ASSERT(io_ch != NULL); 412 413 /* 414 * Start a reset, but then put the I/O channel before 415 * the deferred messages for the reset get a chance to 416 * execute. 417 */ 418 spdk_bdev_reset(g_desc, io_ch, reset_done, &done); 419 spdk_put_io_channel(io_ch); 420 poll_threads(); 421 stub_complete_io(g_bdev.io_target, 0); 422 423 teardown_test(); 424 } 425 426 static void 427 aborted_reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 428 { 429 enum spdk_bdev_io_status *status = cb_arg; 430 431 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 432 spdk_bdev_free_io(bdev_io); 433 } 434 435 static void 436 aborted_reset(void) 437 { 438 struct spdk_io_channel *io_ch[2]; 439 enum spdk_bdev_io_status status1 = SPDK_BDEV_IO_STATUS_PENDING, 440 status2 = SPDK_BDEV_IO_STATUS_PENDING; 441 442 setup_test(); 443 444 set_thread(0); 445 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 446 CU_ASSERT(io_ch[0] != NULL); 447 spdk_bdev_reset(g_desc, io_ch[0], aborted_reset_done, &status1); 448 poll_threads(); 449 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 450 451 /* 452 * First reset has been submitted on ch0. Now submit a second 453 * reset on ch1 which will get queued since there is already a 454 * reset in progress. 455 */ 456 set_thread(1); 457 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 458 CU_ASSERT(io_ch[1] != NULL); 459 spdk_bdev_reset(g_desc, io_ch[1], aborted_reset_done, &status2); 460 poll_threads(); 461 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 462 463 /* 464 * Now destroy ch1. This will abort the queued reset. Check that 465 * the second reset was completed with failed status. Also check 466 * that bdev->internal.reset_in_progress != NULL, since the 467 * original reset has not been completed yet. This ensures that 468 * the bdev code is correctly noticing that the failed reset is 469 * *not* the one that had been submitted to the bdev module. 470 */ 471 set_thread(1); 472 spdk_put_io_channel(io_ch[1]); 473 poll_threads(); 474 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_FAILED); 475 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 476 477 /* 478 * Now complete the first reset, verify that it completed with SUCCESS 479 * status and that bdev->internal.reset_in_progress is also set back to NULL. 480 */ 481 set_thread(0); 482 spdk_put_io_channel(io_ch[0]); 483 stub_complete_io(g_bdev.io_target, 0); 484 poll_threads(); 485 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 486 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 487 488 teardown_test(); 489 } 490 491 static void 492 io_during_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 493 { 494 enum spdk_bdev_io_status *status = cb_arg; 495 496 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 497 spdk_bdev_free_io(bdev_io); 498 } 499 500 static void 501 io_during_reset(void) 502 { 503 struct spdk_io_channel *io_ch[2]; 504 struct spdk_bdev_channel *bdev_ch[2]; 505 enum spdk_bdev_io_status status0, status1, status_reset; 506 int rc; 507 508 setup_test(); 509 510 /* 511 * First test normal case - submit an I/O on each of two channels (with no resets) 512 * and verify they complete successfully. 513 */ 514 set_thread(0); 515 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 516 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 517 CU_ASSERT(bdev_ch[0]->flags == 0); 518 status0 = SPDK_BDEV_IO_STATUS_PENDING; 519 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 520 CU_ASSERT(rc == 0); 521 522 set_thread(1); 523 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 524 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 525 CU_ASSERT(bdev_ch[1]->flags == 0); 526 status1 = SPDK_BDEV_IO_STATUS_PENDING; 527 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 528 CU_ASSERT(rc == 0); 529 530 poll_threads(); 531 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 532 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 533 534 set_thread(0); 535 stub_complete_io(g_bdev.io_target, 0); 536 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 537 538 set_thread(1); 539 stub_complete_io(g_bdev.io_target, 0); 540 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 541 542 /* 543 * Now submit a reset, and leave it pending while we submit I/O on two different 544 * channels. These I/O should be failed by the bdev layer since the reset is in 545 * progress. 546 */ 547 set_thread(0); 548 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 549 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &status_reset); 550 CU_ASSERT(rc == 0); 551 552 CU_ASSERT(bdev_ch[0]->flags == 0); 553 CU_ASSERT(bdev_ch[1]->flags == 0); 554 poll_threads(); 555 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_RESET_IN_PROGRESS); 556 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_RESET_IN_PROGRESS); 557 558 set_thread(0); 559 status0 = SPDK_BDEV_IO_STATUS_PENDING; 560 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 561 CU_ASSERT(rc == 0); 562 563 set_thread(1); 564 status1 = SPDK_BDEV_IO_STATUS_PENDING; 565 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 566 CU_ASSERT(rc == 0); 567 568 /* 569 * A reset is in progress so these read I/O should complete with failure. Note that we 570 * need to poll_threads() since I/O completed inline have their completion deferred. 571 */ 572 poll_threads(); 573 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 574 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 575 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 576 577 /* 578 * Complete the reset 579 */ 580 set_thread(0); 581 stub_complete_io(g_bdev.io_target, 0); 582 583 /* 584 * Only poll thread 0. We should not get a completion. 585 */ 586 poll_thread(0); 587 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 588 589 /* 590 * Poll both thread 0 and 1 so the messages can propagate and we 591 * get a completion. 592 */ 593 poll_threads(); 594 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 595 596 spdk_put_io_channel(io_ch[0]); 597 set_thread(1); 598 spdk_put_io_channel(io_ch[1]); 599 poll_threads(); 600 601 teardown_test(); 602 } 603 604 static void 605 basic_qos(void) 606 { 607 struct spdk_io_channel *io_ch[2]; 608 struct spdk_bdev_channel *bdev_ch[2]; 609 struct spdk_bdev *bdev; 610 enum spdk_bdev_io_status status; 611 int rc; 612 613 setup_test(); 614 615 /* Enable QoS */ 616 bdev = &g_bdev.bdev; 617 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 618 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 619 TAILQ_INIT(&bdev->internal.qos->queued); 620 /* 621 * Enable both IOPS and bandwidth rate limits. 622 * In this case, both rate limits will take equal effect. 623 */ 624 /* 2000 I/O per second, or 2 per millisecond */ 625 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 626 /* 8K byte per millisecond with 4K block size */ 627 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 628 629 g_get_io_channel = true; 630 631 set_thread(0); 632 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 633 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 634 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 635 636 set_thread(1); 637 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 638 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 639 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 640 641 /* 642 * Send an I/O on thread 0, which is where the QoS thread is running. 643 */ 644 set_thread(0); 645 status = SPDK_BDEV_IO_STATUS_PENDING; 646 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status); 647 CU_ASSERT(rc == 0); 648 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 649 poll_threads(); 650 stub_complete_io(g_bdev.io_target, 0); 651 poll_threads(); 652 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 653 654 /* Send an I/O on thread 1. The QoS thread is not running here. */ 655 status = SPDK_BDEV_IO_STATUS_PENDING; 656 set_thread(1); 657 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status); 658 CU_ASSERT(rc == 0); 659 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 660 poll_threads(); 661 /* Complete I/O on thread 1. This should not complete the I/O we submitted */ 662 stub_complete_io(g_bdev.io_target, 0); 663 poll_threads(); 664 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 665 /* Now complete I/O on thread 0 */ 666 set_thread(0); 667 poll_threads(); 668 stub_complete_io(g_bdev.io_target, 0); 669 poll_threads(); 670 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 671 672 /* Tear down the channels */ 673 set_thread(0); 674 spdk_put_io_channel(io_ch[0]); 675 set_thread(1); 676 spdk_put_io_channel(io_ch[1]); 677 poll_threads(); 678 set_thread(0); 679 680 /* Close the descriptor, which should stop the qos channel */ 681 spdk_bdev_close(g_desc); 682 poll_threads(); 683 CU_ASSERT(bdev->internal.qos->ch == NULL); 684 685 spdk_bdev_open(bdev, true, NULL, NULL, &g_desc); 686 687 /* Create the channels in reverse order. */ 688 set_thread(1); 689 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 690 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 691 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 692 693 set_thread(0); 694 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 695 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 696 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 697 698 /* Confirm that the qos thread is now thread 1 */ 699 CU_ASSERT(bdev->internal.qos->ch == bdev_ch[1]); 700 701 /* Tear down the channels */ 702 set_thread(0); 703 spdk_put_io_channel(io_ch[0]); 704 set_thread(1); 705 spdk_put_io_channel(io_ch[1]); 706 poll_threads(); 707 708 set_thread(0); 709 710 teardown_test(); 711 } 712 713 static void 714 io_during_qos_queue(void) 715 { 716 struct spdk_io_channel *io_ch[2]; 717 struct spdk_bdev_channel *bdev_ch[2]; 718 struct spdk_bdev *bdev; 719 enum spdk_bdev_io_status status0, status1; 720 int rc; 721 722 setup_test(); 723 reset_time(); 724 725 /* Enable QoS */ 726 bdev = &g_bdev.bdev; 727 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 728 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 729 TAILQ_INIT(&bdev->internal.qos->queued); 730 /* 731 * Enable both IOPS and bandwidth rate limits. 732 * In this case, IOPS rate limit will take effect first. 733 */ 734 /* 1000 I/O per second, or 1 per millisecond */ 735 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 1000; 736 /* 8K byte per millisecond with 4K block size */ 737 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 738 739 g_get_io_channel = true; 740 741 /* Create channels */ 742 set_thread(0); 743 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 744 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 745 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 746 747 set_thread(1); 748 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 749 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 750 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 751 752 /* Send two I/O */ 753 status1 = SPDK_BDEV_IO_STATUS_PENDING; 754 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 755 CU_ASSERT(rc == 0); 756 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 757 set_thread(0); 758 status0 = SPDK_BDEV_IO_STATUS_PENDING; 759 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 760 CU_ASSERT(rc == 0); 761 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 762 763 /* Complete any I/O that arrived at the disk */ 764 poll_threads(); 765 set_thread(1); 766 stub_complete_io(g_bdev.io_target, 0); 767 set_thread(0); 768 stub_complete_io(g_bdev.io_target, 0); 769 poll_threads(); 770 771 /* Only one of the I/O should complete. (logical XOR) */ 772 if (status0 == SPDK_BDEV_IO_STATUS_SUCCESS) { 773 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 774 } else { 775 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 776 } 777 778 /* Advance in time by a millisecond */ 779 increment_time(1000); 780 781 /* Complete more I/O */ 782 poll_threads(); 783 set_thread(1); 784 stub_complete_io(g_bdev.io_target, 0); 785 set_thread(0); 786 stub_complete_io(g_bdev.io_target, 0); 787 poll_threads(); 788 789 /* Now the second I/O should be done */ 790 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 791 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 792 793 /* Tear down the channels */ 794 set_thread(1); 795 spdk_put_io_channel(io_ch[1]); 796 set_thread(0); 797 spdk_put_io_channel(io_ch[0]); 798 poll_threads(); 799 800 teardown_test(); 801 } 802 803 static void 804 io_during_qos_reset(void) 805 { 806 struct spdk_io_channel *io_ch[2]; 807 struct spdk_bdev_channel *bdev_ch[2]; 808 struct spdk_bdev *bdev; 809 enum spdk_bdev_io_status status0, status1, reset_status; 810 int rc; 811 812 setup_test(); 813 reset_time(); 814 815 /* Enable QoS */ 816 bdev = &g_bdev.bdev; 817 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 818 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 819 TAILQ_INIT(&bdev->internal.qos->queued); 820 /* 821 * Enable both IOPS and bandwidth rate limits. 822 * In this case, bandwidth rate limit will take effect first. 823 */ 824 /* 2000 I/O per second, or 2 per millisecond */ 825 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 826 /* 4K byte per millisecond with 4K block size */ 827 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 4096000; 828 829 g_get_io_channel = true; 830 831 /* Create channels */ 832 set_thread(0); 833 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 834 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 835 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 836 837 set_thread(1); 838 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 839 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 840 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 841 842 /* Send two I/O. One of these gets queued by QoS. The other is sitting at the disk. */ 843 status1 = SPDK_BDEV_IO_STATUS_PENDING; 844 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 845 CU_ASSERT(rc == 0); 846 set_thread(0); 847 status0 = SPDK_BDEV_IO_STATUS_PENDING; 848 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 849 CU_ASSERT(rc == 0); 850 851 poll_threads(); 852 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 853 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 854 855 /* Reset the bdev. */ 856 reset_status = SPDK_BDEV_IO_STATUS_PENDING; 857 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &reset_status); 858 CU_ASSERT(rc == 0); 859 860 /* Complete any I/O that arrived at the disk */ 861 poll_threads(); 862 set_thread(1); 863 stub_complete_io(g_bdev.io_target, 0); 864 set_thread(0); 865 stub_complete_io(g_bdev.io_target, 0); 866 poll_threads(); 867 868 CU_ASSERT(reset_status == SPDK_BDEV_IO_STATUS_SUCCESS); 869 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 870 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 871 872 /* Tear down the channels */ 873 set_thread(1); 874 spdk_put_io_channel(io_ch[1]); 875 set_thread(0); 876 spdk_put_io_channel(io_ch[0]); 877 poll_threads(); 878 879 teardown_test(); 880 } 881 882 static void 883 enomem_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 884 { 885 enum spdk_bdev_io_status *status = cb_arg; 886 887 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 888 spdk_bdev_free_io(bdev_io); 889 } 890 891 static void 892 enomem(void) 893 { 894 struct spdk_io_channel *io_ch; 895 struct spdk_bdev_channel *bdev_ch; 896 struct spdk_bdev_shared_resource *shared_resource; 897 struct ut_bdev_channel *ut_ch; 898 const uint32_t IO_ARRAY_SIZE = 64; 899 const uint32_t AVAIL = 20; 900 enum spdk_bdev_io_status status[IO_ARRAY_SIZE], status_reset; 901 uint32_t nomem_cnt, i; 902 struct spdk_bdev_io *first_io; 903 int rc; 904 905 setup_test(); 906 907 set_thread(0); 908 io_ch = spdk_bdev_get_io_channel(g_desc); 909 bdev_ch = spdk_io_channel_get_ctx(io_ch); 910 shared_resource = bdev_ch->shared_resource; 911 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 912 ut_ch->avail_cnt = AVAIL; 913 914 /* First submit a number of IOs equal to what the channel can support. */ 915 for (i = 0; i < AVAIL; i++) { 916 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 917 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 918 CU_ASSERT(rc == 0); 919 } 920 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 921 922 /* 923 * Next, submit one additional I/O. This one should fail with ENOMEM and then go onto 924 * the enomem_io list. 925 */ 926 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 927 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 928 CU_ASSERT(rc == 0); 929 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 930 first_io = TAILQ_FIRST(&shared_resource->nomem_io); 931 932 /* 933 * Now submit a bunch more I/O. These should all fail with ENOMEM and get queued behind 934 * the first_io above. 935 */ 936 for (i = AVAIL + 1; i < IO_ARRAY_SIZE; i++) { 937 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 938 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 939 CU_ASSERT(rc == 0); 940 } 941 942 /* Assert that first_io is still at the head of the list. */ 943 CU_ASSERT(TAILQ_FIRST(&shared_resource->nomem_io) == first_io); 944 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == (IO_ARRAY_SIZE - AVAIL)); 945 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 946 CU_ASSERT(shared_resource->nomem_threshold == (AVAIL - NOMEM_THRESHOLD_COUNT)); 947 948 /* 949 * Complete 1 I/O only. The key check here is bdev_io_tailq_cnt - this should not have 950 * changed since completing just 1 I/O should not trigger retrying the queued nomem_io 951 * list. 952 */ 953 stub_complete_io(g_bdev.io_target, 1); 954 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 955 956 /* 957 * Complete enough I/O to hit the nomem_theshold. This should trigger retrying nomem_io, 958 * and we should see I/O get resubmitted to the test bdev module. 959 */ 960 stub_complete_io(g_bdev.io_target, NOMEM_THRESHOLD_COUNT - 1); 961 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) < nomem_cnt); 962 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 963 964 /* Complete 1 I/O only. This should not trigger retrying the queued nomem_io. */ 965 stub_complete_io(g_bdev.io_target, 1); 966 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 967 968 /* 969 * Send a reset and confirm that all I/O are completed, including the ones that 970 * were queued on the nomem_io list. 971 */ 972 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 973 rc = spdk_bdev_reset(g_desc, io_ch, enomem_done, &status_reset); 974 poll_threads(); 975 CU_ASSERT(rc == 0); 976 /* This will complete the reset. */ 977 stub_complete_io(g_bdev.io_target, 0); 978 979 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == 0); 980 CU_ASSERT(shared_resource->io_outstanding == 0); 981 982 spdk_put_io_channel(io_ch); 983 poll_threads(); 984 teardown_test(); 985 } 986 987 static void 988 enomem_multi_bdev(void) 989 { 990 struct spdk_io_channel *io_ch; 991 struct spdk_bdev_channel *bdev_ch; 992 struct spdk_bdev_shared_resource *shared_resource; 993 struct ut_bdev_channel *ut_ch; 994 const uint32_t IO_ARRAY_SIZE = 64; 995 const uint32_t AVAIL = 20; 996 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 997 uint32_t i; 998 struct ut_bdev *second_bdev; 999 struct spdk_bdev_desc *second_desc = NULL; 1000 struct spdk_bdev_channel *second_bdev_ch; 1001 struct spdk_io_channel *second_ch; 1002 int rc; 1003 1004 setup_test(); 1005 1006 /* Register second bdev with the same io_target */ 1007 second_bdev = calloc(1, sizeof(*second_bdev)); 1008 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1009 register_bdev(second_bdev, "ut_bdev2", g_bdev.io_target); 1010 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 1011 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1012 1013 set_thread(0); 1014 io_ch = spdk_bdev_get_io_channel(g_desc); 1015 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1016 shared_resource = bdev_ch->shared_resource; 1017 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1018 ut_ch->avail_cnt = AVAIL; 1019 1020 second_ch = spdk_bdev_get_io_channel(second_desc); 1021 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1022 SPDK_CU_ASSERT_FATAL(shared_resource == second_bdev_ch->shared_resource); 1023 1024 /* Saturate io_target through bdev A. */ 1025 for (i = 0; i < AVAIL; i++) { 1026 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1027 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1028 CU_ASSERT(rc == 0); 1029 } 1030 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1031 1032 /* 1033 * Now submit I/O through the second bdev. This should fail with ENOMEM 1034 * and then go onto the nomem_io list. 1035 */ 1036 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1037 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1038 CU_ASSERT(rc == 0); 1039 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1040 1041 /* Complete first bdev's I/O. This should retry sending second bdev's nomem_io */ 1042 stub_complete_io(g_bdev.io_target, AVAIL); 1043 1044 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&shared_resource->nomem_io)); 1045 CU_ASSERT(shared_resource->io_outstanding == 1); 1046 1047 /* Now complete our retried I/O */ 1048 stub_complete_io(g_bdev.io_target, 1); 1049 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == 0); 1050 1051 spdk_put_io_channel(io_ch); 1052 spdk_put_io_channel(second_ch); 1053 spdk_bdev_close(second_desc); 1054 unregister_bdev(second_bdev); 1055 poll_threads(); 1056 free(second_bdev); 1057 teardown_test(); 1058 } 1059 1060 1061 static void 1062 enomem_multi_io_target(void) 1063 { 1064 struct spdk_io_channel *io_ch; 1065 struct spdk_bdev_channel *bdev_ch; 1066 struct ut_bdev_channel *ut_ch; 1067 const uint32_t IO_ARRAY_SIZE = 64; 1068 const uint32_t AVAIL = 20; 1069 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1070 uint32_t i; 1071 int new_io_device; 1072 struct ut_bdev *second_bdev; 1073 struct spdk_bdev_desc *second_desc = NULL; 1074 struct spdk_bdev_channel *second_bdev_ch; 1075 struct spdk_io_channel *second_ch; 1076 int rc; 1077 1078 setup_test(); 1079 1080 /* Create new io_target and a second bdev using it */ 1081 spdk_io_device_register(&new_io_device, stub_create_ch, stub_destroy_ch, 1082 sizeof(struct ut_bdev_channel), NULL); 1083 second_bdev = calloc(1, sizeof(*second_bdev)); 1084 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1085 register_bdev(second_bdev, "ut_bdev2", &new_io_device); 1086 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 1087 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1088 1089 set_thread(0); 1090 io_ch = spdk_bdev_get_io_channel(g_desc); 1091 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1092 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1093 ut_ch->avail_cnt = AVAIL; 1094 1095 /* Different io_target should imply a different shared_resource */ 1096 second_ch = spdk_bdev_get_io_channel(second_desc); 1097 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1098 SPDK_CU_ASSERT_FATAL(bdev_ch->shared_resource != second_bdev_ch->shared_resource); 1099 1100 /* Saturate io_target through bdev A. */ 1101 for (i = 0; i < AVAIL; i++) { 1102 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1103 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1104 CU_ASSERT(rc == 0); 1105 } 1106 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1107 1108 /* Issue one more I/O to fill ENOMEM list. */ 1109 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1110 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1111 CU_ASSERT(rc == 0); 1112 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1113 1114 /* 1115 * Now submit I/O through the second bdev. This should go through and complete 1116 * successfully because we're using a different io_device underneath. 1117 */ 1118 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1119 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1120 CU_ASSERT(rc == 0); 1121 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&second_bdev_ch->shared_resource->nomem_io)); 1122 stub_complete_io(second_bdev->io_target, 1); 1123 1124 /* Cleanup; Complete outstanding I/O. */ 1125 stub_complete_io(g_bdev.io_target, AVAIL); 1126 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1127 /* Complete the ENOMEM I/O */ 1128 stub_complete_io(g_bdev.io_target, 1); 1129 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1130 1131 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1132 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1133 spdk_put_io_channel(io_ch); 1134 spdk_put_io_channel(second_ch); 1135 spdk_bdev_close(second_desc); 1136 unregister_bdev(second_bdev); 1137 spdk_io_device_unregister(&new_io_device, NULL); 1138 poll_threads(); 1139 free(second_bdev); 1140 teardown_test(); 1141 } 1142 1143 static void 1144 qos_dynamic_enable_done(void *cb_arg, int status) 1145 { 1146 int *rc = cb_arg; 1147 *rc = status; 1148 } 1149 1150 static void 1151 qos_dynamic_enable(void) 1152 { 1153 struct spdk_io_channel *io_ch[2]; 1154 struct spdk_bdev_channel *bdev_ch[2]; 1155 struct spdk_bdev *bdev; 1156 enum spdk_bdev_io_status bdev_io_status[2]; 1157 uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {}; 1158 int status, second_status, rc, i; 1159 1160 setup_test(); 1161 reset_time(); 1162 1163 for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) { 1164 limits[i] = UINT64_MAX; 1165 } 1166 1167 bdev = &g_bdev.bdev; 1168 1169 g_get_io_channel = true; 1170 1171 /* Create channels */ 1172 set_thread(0); 1173 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1174 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1175 CU_ASSERT(bdev_ch[0]->flags == 0); 1176 1177 set_thread(1); 1178 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1179 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1180 CU_ASSERT(bdev_ch[1]->flags == 0); 1181 1182 set_thread(0); 1183 1184 /* 1185 * Enable QoS: IOPS and byte per second rate limits. 1186 * More than 10 I/Os allowed per timeslice. 1187 */ 1188 status = -1; 1189 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1190 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 100; 1191 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1192 poll_threads(); 1193 CU_ASSERT(status == 0); 1194 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1195 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1196 1197 /* 1198 * Submit and complete 10 I/O to fill the QoS allotment for this timeslice. 1199 * Additional I/O will then be queued. 1200 */ 1201 set_thread(0); 1202 for (i = 0; i < 10; i++) { 1203 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1204 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1205 CU_ASSERT(rc == 0); 1206 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1207 poll_thread(0); 1208 stub_complete_io(g_bdev.io_target, 0); 1209 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1210 } 1211 1212 /* 1213 * Send two more I/O. These I/O will be queued since the current timeslice allotment has been 1214 * filled already. We want to test that when QoS is disabled that these two I/O: 1215 * 1) are not aborted 1216 * 2) are sent back to their original thread for resubmission 1217 */ 1218 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1219 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1220 CU_ASSERT(rc == 0); 1221 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1222 set_thread(1); 1223 bdev_io_status[1] = SPDK_BDEV_IO_STATUS_PENDING; 1224 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &bdev_io_status[1]); 1225 CU_ASSERT(rc == 0); 1226 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1227 poll_threads(); 1228 1229 /* Disable QoS: IOPS rate limit */ 1230 status = -1; 1231 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1232 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1233 poll_threads(); 1234 CU_ASSERT(status == 0); 1235 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1236 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1237 1238 /* Disable QoS: Byte per second rate limit */ 1239 status = -1; 1240 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 0; 1241 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1242 poll_threads(); 1243 CU_ASSERT(status == 0); 1244 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1245 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1246 1247 /* 1248 * All I/O should have been resubmitted back on their original thread. Complete 1249 * all I/O on thread 0, and ensure that only the thread 0 I/O was completed. 1250 */ 1251 set_thread(0); 1252 stub_complete_io(g_bdev.io_target, 0); 1253 poll_threads(); 1254 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1255 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1256 1257 /* Now complete all I/O on thread 1 and ensure the thread 1 I/O was completed. */ 1258 set_thread(1); 1259 stub_complete_io(g_bdev.io_target, 0); 1260 poll_threads(); 1261 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_SUCCESS); 1262 1263 /* Disable QoS again */ 1264 status = -1; 1265 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1266 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1267 poll_threads(); 1268 CU_ASSERT(status == 0); /* This should succeed */ 1269 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1270 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1271 1272 /* Enable QoS on thread 0 */ 1273 status = -1; 1274 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1275 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1276 poll_threads(); 1277 CU_ASSERT(status == 0); 1278 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1279 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1280 1281 /* Disable QoS on thread 1 */ 1282 set_thread(1); 1283 status = -1; 1284 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1285 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1286 /* Don't poll yet. This should leave the channels with QoS enabled */ 1287 CU_ASSERT(status == -1); 1288 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1289 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1290 1291 /* Enable QoS. This should immediately fail because the previous disable QoS hasn't completed. */ 1292 second_status = 0; 1293 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 10; 1294 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &second_status); 1295 poll_threads(); 1296 CU_ASSERT(status == 0); /* The disable should succeed */ 1297 CU_ASSERT(second_status < 0); /* The enable should fail */ 1298 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1299 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1300 1301 /* Enable QoS on thread 1. This should succeed now that the disable has completed. */ 1302 status = -1; 1303 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1304 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1305 poll_threads(); 1306 CU_ASSERT(status == 0); 1307 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1308 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1309 1310 /* Tear down the channels */ 1311 set_thread(0); 1312 spdk_put_io_channel(io_ch[0]); 1313 set_thread(1); 1314 spdk_put_io_channel(io_ch[1]); 1315 poll_threads(); 1316 1317 set_thread(0); 1318 teardown_test(); 1319 } 1320 1321 int 1322 main(int argc, char **argv) 1323 { 1324 CU_pSuite suite = NULL; 1325 unsigned int num_failures; 1326 1327 if (CU_initialize_registry() != CUE_SUCCESS) { 1328 return CU_get_error(); 1329 } 1330 1331 suite = CU_add_suite("bdev", NULL, NULL); 1332 if (suite == NULL) { 1333 CU_cleanup_registry(); 1334 return CU_get_error(); 1335 } 1336 1337 if ( 1338 CU_add_test(suite, "basic", basic) == NULL || 1339 CU_add_test(suite, "unregister_and_close", unregister_and_close) == NULL || 1340 CU_add_test(suite, "basic_qos", basic_qos) == NULL || 1341 CU_add_test(suite, "put_channel_during_reset", put_channel_during_reset) == NULL || 1342 CU_add_test(suite, "aborted_reset", aborted_reset) == NULL || 1343 CU_add_test(suite, "io_during_reset", io_during_reset) == NULL || 1344 CU_add_test(suite, "io_during_qos_queue", io_during_qos_queue) == NULL || 1345 CU_add_test(suite, "io_during_qos_reset", io_during_qos_reset) == NULL || 1346 CU_add_test(suite, "enomem", enomem) == NULL || 1347 CU_add_test(suite, "enomem_multi_bdev", enomem_multi_bdev) == NULL || 1348 CU_add_test(suite, "enomem_multi_io_target", enomem_multi_io_target) == NULL || 1349 CU_add_test(suite, "qos_dynamic_enable", qos_dynamic_enable) == NULL 1350 ) { 1351 CU_cleanup_registry(); 1352 return CU_get_error(); 1353 } 1354 1355 CU_basic_set_mode(CU_BRM_VERBOSE); 1356 CU_basic_run_tests(); 1357 num_failures = CU_get_number_of_failures(); 1358 CU_cleanup_registry(); 1359 return num_failures; 1360 } 1361