1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk_cunit.h" 35 36 #include "common/lib/ut_multithread.c" 37 #include "unit/lib/json_mock.c" 38 39 #include "spdk/config.h" 40 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 41 #undef SPDK_CONFIG_VTUNE 42 43 #include "bdev/bdev.c" 44 45 #define BDEV_UT_NUM_THREADS 3 46 47 DEFINE_STUB(spdk_conf_find_section, struct spdk_conf_section *, (struct spdk_conf *cp, 48 const char *name), NULL); 49 DEFINE_STUB(spdk_conf_section_get_nmval, char *, 50 (struct spdk_conf_section *sp, const char *key, int idx1, int idx2), NULL); 51 DEFINE_STUB(spdk_conf_section_get_intval, int, (struct spdk_conf_section *sp, const char *key), -1); 52 53 struct spdk_trace_histories *g_trace_histories; 54 DEFINE_STUB_V(spdk_trace_add_register_fn, (struct spdk_trace_register_fn *reg_fn)); 55 DEFINE_STUB_V(spdk_trace_register_owner, (uint8_t type, char id_prefix)); 56 DEFINE_STUB_V(spdk_trace_register_object, (uint8_t type, char id_prefix)); 57 DEFINE_STUB_V(spdk_trace_register_description, (const char *name, const char *short_name, 58 uint16_t tpoint_id, uint8_t owner_type, 59 uint8_t object_type, uint8_t new_object, 60 uint8_t arg1_is_ptr, const char *arg1_name)); 61 DEFINE_STUB_V(_spdk_trace_record, (uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, 62 uint32_t size, uint64_t object_id, uint64_t arg1)); 63 64 struct ut_bdev { 65 struct spdk_bdev bdev; 66 void *io_target; 67 }; 68 69 struct ut_bdev_channel { 70 TAILQ_HEAD(, spdk_bdev_io) outstanding_io; 71 uint32_t outstanding_cnt; 72 uint32_t avail_cnt; 73 }; 74 75 int g_io_device; 76 struct ut_bdev g_bdev; 77 struct spdk_bdev_desc *g_desc; 78 bool g_teardown_done = false; 79 bool g_get_io_channel = true; 80 bool g_create_ch = true; 81 bool g_init_complete_called = false; 82 bool g_fini_start_called = true; 83 84 static int 85 stub_create_ch(void *io_device, void *ctx_buf) 86 { 87 struct ut_bdev_channel *ch = ctx_buf; 88 89 if (g_create_ch == false) { 90 return -1; 91 } 92 93 TAILQ_INIT(&ch->outstanding_io); 94 ch->outstanding_cnt = 0; 95 /* 96 * When avail gets to 0, the submit_request function will return ENOMEM. 97 * Most tests to not want ENOMEM to occur, so by default set this to a 98 * big value that won't get hit. The ENOMEM tests can then override this 99 * value to something much smaller to induce ENOMEM conditions. 100 */ 101 ch->avail_cnt = 2048; 102 return 0; 103 } 104 105 static void 106 stub_destroy_ch(void *io_device, void *ctx_buf) 107 { 108 } 109 110 static struct spdk_io_channel * 111 stub_get_io_channel(void *ctx) 112 { 113 struct ut_bdev *ut_bdev = ctx; 114 115 if (g_get_io_channel == true) { 116 return spdk_get_io_channel(ut_bdev->io_target); 117 } else { 118 return NULL; 119 } 120 } 121 122 static int 123 stub_destruct(void *ctx) 124 { 125 return 0; 126 } 127 128 static void 129 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 130 { 131 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 132 133 if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { 134 struct spdk_bdev_io *io; 135 136 while (!TAILQ_EMPTY(&ch->outstanding_io)) { 137 io = TAILQ_FIRST(&ch->outstanding_io); 138 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 139 ch->outstanding_cnt--; 140 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_FAILED); 141 ch->avail_cnt++; 142 } 143 } 144 145 if (ch->avail_cnt > 0) { 146 TAILQ_INSERT_TAIL(&ch->outstanding_io, bdev_io, module_link); 147 ch->outstanding_cnt++; 148 ch->avail_cnt--; 149 } else { 150 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); 151 } 152 } 153 154 static uint32_t 155 stub_complete_io(void *io_target, uint32_t num_to_complete) 156 { 157 struct spdk_io_channel *_ch = spdk_get_io_channel(io_target); 158 struct ut_bdev_channel *ch = spdk_io_channel_get_ctx(_ch); 159 struct spdk_bdev_io *io; 160 bool complete_all = (num_to_complete == 0); 161 uint32_t num_completed = 0; 162 163 while (complete_all || num_completed < num_to_complete) { 164 if (TAILQ_EMPTY(&ch->outstanding_io)) { 165 break; 166 } 167 io = TAILQ_FIRST(&ch->outstanding_io); 168 TAILQ_REMOVE(&ch->outstanding_io, io, module_link); 169 ch->outstanding_cnt--; 170 spdk_bdev_io_complete(io, SPDK_BDEV_IO_STATUS_SUCCESS); 171 ch->avail_cnt++; 172 num_completed++; 173 } 174 175 spdk_put_io_channel(_ch); 176 return num_completed; 177 } 178 179 static struct spdk_bdev_fn_table fn_table = { 180 .get_io_channel = stub_get_io_channel, 181 .destruct = stub_destruct, 182 .submit_request = stub_submit_request, 183 }; 184 185 static int 186 module_init(void) 187 { 188 return 0; 189 } 190 191 static void 192 module_fini(void) 193 { 194 } 195 196 static void 197 init_complete(void) 198 { 199 g_init_complete_called = true; 200 } 201 202 static void 203 fini_start(void) 204 { 205 g_fini_start_called = true; 206 } 207 208 struct spdk_bdev_module bdev_ut_if = { 209 .name = "bdev_ut", 210 .module_init = module_init, 211 .module_fini = module_fini, 212 .init_complete = init_complete, 213 .fini_start = fini_start, 214 }; 215 216 SPDK_BDEV_MODULE_REGISTER(&bdev_ut_if) 217 218 static void 219 register_bdev(struct ut_bdev *ut_bdev, char *name, void *io_target) 220 { 221 memset(ut_bdev, 0, sizeof(*ut_bdev)); 222 223 ut_bdev->io_target = io_target; 224 ut_bdev->bdev.ctxt = ut_bdev; 225 ut_bdev->bdev.name = name; 226 ut_bdev->bdev.fn_table = &fn_table; 227 ut_bdev->bdev.module = &bdev_ut_if; 228 ut_bdev->bdev.blocklen = 4096; 229 ut_bdev->bdev.blockcnt = 1024; 230 231 spdk_bdev_register(&ut_bdev->bdev); 232 } 233 234 static void 235 unregister_bdev(struct ut_bdev *ut_bdev) 236 { 237 /* Handle any deferred messages. */ 238 poll_threads(); 239 spdk_bdev_unregister(&ut_bdev->bdev, NULL, NULL); 240 } 241 242 static void 243 bdev_init_cb(void *done, int rc) 244 { 245 CU_ASSERT(rc == 0); 246 *(bool *)done = true; 247 } 248 249 static void 250 setup_test(void) 251 { 252 bool done = false; 253 254 allocate_threads(BDEV_UT_NUM_THREADS); 255 set_thread(0); 256 spdk_bdev_initialize(bdev_init_cb, &done); 257 spdk_io_device_register(&g_io_device, stub_create_ch, stub_destroy_ch, 258 sizeof(struct ut_bdev_channel), NULL); 259 register_bdev(&g_bdev, "ut_bdev", &g_io_device); 260 spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &g_desc); 261 } 262 263 static void 264 finish_cb(void *cb_arg) 265 { 266 g_teardown_done = true; 267 } 268 269 static void 270 teardown_test(void) 271 { 272 set_thread(0); 273 g_teardown_done = false; 274 spdk_bdev_close(g_desc); 275 g_desc = NULL; 276 unregister_bdev(&g_bdev); 277 spdk_io_device_unregister(&g_io_device, NULL); 278 spdk_bdev_finish(finish_cb, NULL); 279 poll_threads(); 280 memset(&g_bdev, 0, sizeof(g_bdev)); 281 CU_ASSERT(g_teardown_done == true); 282 g_teardown_done = false; 283 free_threads(); 284 } 285 286 static uint32_t 287 bdev_io_tailq_cnt(bdev_io_tailq_t *tailq) 288 { 289 struct spdk_bdev_io *io; 290 uint32_t cnt = 0; 291 292 TAILQ_FOREACH(io, tailq, internal.link) { 293 cnt++; 294 } 295 296 return cnt; 297 } 298 299 static void 300 basic(void) 301 { 302 g_init_complete_called = false; 303 setup_test(); 304 CU_ASSERT(g_init_complete_called == true); 305 306 set_thread(0); 307 308 g_get_io_channel = false; 309 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 310 CU_ASSERT(g_ut_threads[0].ch == NULL); 311 312 g_get_io_channel = true; 313 g_create_ch = false; 314 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 315 CU_ASSERT(g_ut_threads[0].ch == NULL); 316 317 g_get_io_channel = true; 318 g_create_ch = true; 319 g_ut_threads[0].ch = spdk_bdev_get_io_channel(g_desc); 320 CU_ASSERT(g_ut_threads[0].ch != NULL); 321 spdk_put_io_channel(g_ut_threads[0].ch); 322 323 g_fini_start_called = false; 324 teardown_test(); 325 CU_ASSERT(g_fini_start_called == true); 326 } 327 328 static void 329 _bdev_removed(void *done) 330 { 331 *(bool *)done = true; 332 } 333 334 static void 335 _bdev_unregistered(void *done, int rc) 336 { 337 CU_ASSERT(rc == 0); 338 *(bool *)done = true; 339 } 340 341 static void 342 unregister_and_close(void) 343 { 344 bool done, remove_notify; 345 struct spdk_bdev_desc *desc; 346 347 setup_test(); 348 set_thread(0); 349 350 /* setup_test() automatically opens the bdev, 351 * but this test needs to do that in a different 352 * way. */ 353 spdk_bdev_close(g_desc); 354 poll_threads(); 355 356 remove_notify = false; 357 spdk_bdev_open(&g_bdev.bdev, true, _bdev_removed, &remove_notify, &desc); 358 CU_ASSERT(remove_notify == false); 359 CU_ASSERT(desc != NULL); 360 361 /* There is an open descriptor on the device. Unregister it, 362 * which can't proceed until the descriptor is closed. */ 363 done = false; 364 spdk_bdev_unregister(&g_bdev.bdev, _bdev_unregistered, &done); 365 /* No polling has occurred, so neither of these should execute */ 366 CU_ASSERT(remove_notify == false); 367 CU_ASSERT(done == false); 368 369 /* Prior to the unregister completing, close the descriptor */ 370 spdk_bdev_close(desc); 371 372 /* Poll the threads to allow all events to be processed */ 373 poll_threads(); 374 375 /* Remove notify should not have been called because the 376 * descriptor is already closed. */ 377 CU_ASSERT(remove_notify == false); 378 379 /* The unregister should have completed */ 380 CU_ASSERT(done == true); 381 382 spdk_bdev_finish(finish_cb, NULL); 383 poll_threads(); 384 free_threads(); 385 } 386 387 static void 388 reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 389 { 390 bool *done = cb_arg; 391 392 CU_ASSERT(success == true); 393 *done = true; 394 spdk_bdev_free_io(bdev_io); 395 } 396 397 static void 398 put_channel_during_reset(void) 399 { 400 struct spdk_io_channel *io_ch; 401 bool done = false; 402 403 setup_test(); 404 405 set_thread(0); 406 io_ch = spdk_bdev_get_io_channel(g_desc); 407 CU_ASSERT(io_ch != NULL); 408 409 /* 410 * Start a reset, but then put the I/O channel before 411 * the deferred messages for the reset get a chance to 412 * execute. 413 */ 414 spdk_bdev_reset(g_desc, io_ch, reset_done, &done); 415 spdk_put_io_channel(io_ch); 416 poll_threads(); 417 stub_complete_io(g_bdev.io_target, 0); 418 419 teardown_test(); 420 } 421 422 static void 423 aborted_reset_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 424 { 425 enum spdk_bdev_io_status *status = cb_arg; 426 427 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 428 spdk_bdev_free_io(bdev_io); 429 } 430 431 static void 432 aborted_reset(void) 433 { 434 struct spdk_io_channel *io_ch[2]; 435 enum spdk_bdev_io_status status1 = SPDK_BDEV_IO_STATUS_PENDING, 436 status2 = SPDK_BDEV_IO_STATUS_PENDING; 437 438 setup_test(); 439 440 set_thread(0); 441 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 442 CU_ASSERT(io_ch[0] != NULL); 443 spdk_bdev_reset(g_desc, io_ch[0], aborted_reset_done, &status1); 444 poll_threads(); 445 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 446 447 /* 448 * First reset has been submitted on ch0. Now submit a second 449 * reset on ch1 which will get queued since there is already a 450 * reset in progress. 451 */ 452 set_thread(1); 453 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 454 CU_ASSERT(io_ch[1] != NULL); 455 spdk_bdev_reset(g_desc, io_ch[1], aborted_reset_done, &status2); 456 poll_threads(); 457 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 458 459 /* 460 * Now destroy ch1. This will abort the queued reset. Check that 461 * the second reset was completed with failed status. Also check 462 * that bdev->internal.reset_in_progress != NULL, since the 463 * original reset has not been completed yet. This ensures that 464 * the bdev code is correctly noticing that the failed reset is 465 * *not* the one that had been submitted to the bdev module. 466 */ 467 set_thread(1); 468 spdk_put_io_channel(io_ch[1]); 469 poll_threads(); 470 CU_ASSERT(status2 == SPDK_BDEV_IO_STATUS_FAILED); 471 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress != NULL); 472 473 /* 474 * Now complete the first reset, verify that it completed with SUCCESS 475 * status and that bdev->internal.reset_in_progress is also set back to NULL. 476 */ 477 set_thread(0); 478 spdk_put_io_channel(io_ch[0]); 479 stub_complete_io(g_bdev.io_target, 0); 480 poll_threads(); 481 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 482 CU_ASSERT(g_bdev.bdev.internal.reset_in_progress == NULL); 483 484 teardown_test(); 485 } 486 487 static void 488 io_during_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 489 { 490 enum spdk_bdev_io_status *status = cb_arg; 491 492 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 493 spdk_bdev_free_io(bdev_io); 494 } 495 496 static void 497 io_during_reset(void) 498 { 499 struct spdk_io_channel *io_ch[2]; 500 struct spdk_bdev_channel *bdev_ch[2]; 501 enum spdk_bdev_io_status status0, status1, status_reset; 502 int rc; 503 504 setup_test(); 505 506 /* 507 * First test normal case - submit an I/O on each of two channels (with no resets) 508 * and verify they complete successfully. 509 */ 510 set_thread(0); 511 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 512 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 513 CU_ASSERT(bdev_ch[0]->flags == 0); 514 status0 = SPDK_BDEV_IO_STATUS_PENDING; 515 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 516 CU_ASSERT(rc == 0); 517 518 set_thread(1); 519 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 520 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 521 CU_ASSERT(bdev_ch[1]->flags == 0); 522 status1 = SPDK_BDEV_IO_STATUS_PENDING; 523 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 524 CU_ASSERT(rc == 0); 525 526 poll_threads(); 527 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 528 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 529 530 set_thread(0); 531 stub_complete_io(g_bdev.io_target, 0); 532 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 533 534 set_thread(1); 535 stub_complete_io(g_bdev.io_target, 0); 536 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 537 538 /* 539 * Now submit a reset, and leave it pending while we submit I/O on two different 540 * channels. These I/O should be failed by the bdev layer since the reset is in 541 * progress. 542 */ 543 set_thread(0); 544 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 545 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &status_reset); 546 CU_ASSERT(rc == 0); 547 548 CU_ASSERT(bdev_ch[0]->flags == 0); 549 CU_ASSERT(bdev_ch[1]->flags == 0); 550 poll_threads(); 551 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_RESET_IN_PROGRESS); 552 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_RESET_IN_PROGRESS); 553 554 set_thread(0); 555 status0 = SPDK_BDEV_IO_STATUS_PENDING; 556 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 557 CU_ASSERT(rc == 0); 558 559 set_thread(1); 560 status1 = SPDK_BDEV_IO_STATUS_PENDING; 561 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 562 CU_ASSERT(rc == 0); 563 564 /* 565 * A reset is in progress so these read I/O should complete with failure. Note that we 566 * need to poll_threads() since I/O completed inline have their completion deferred. 567 */ 568 poll_threads(); 569 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 570 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 571 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 572 573 /* 574 * Complete the reset 575 */ 576 set_thread(0); 577 stub_complete_io(g_bdev.io_target, 0); 578 579 /* 580 * Only poll thread 0. We should not get a completion. 581 */ 582 poll_thread(0); 583 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_PENDING); 584 585 /* 586 * Poll both thread 0 and 1 so the messages can propagate and we 587 * get a completion. 588 */ 589 poll_threads(); 590 CU_ASSERT(status_reset == SPDK_BDEV_IO_STATUS_SUCCESS); 591 592 spdk_put_io_channel(io_ch[0]); 593 set_thread(1); 594 spdk_put_io_channel(io_ch[1]); 595 poll_threads(); 596 597 teardown_test(); 598 } 599 600 static void 601 basic_qos(void) 602 { 603 struct spdk_io_channel *io_ch[2]; 604 struct spdk_bdev_channel *bdev_ch[2]; 605 struct spdk_bdev *bdev; 606 enum spdk_bdev_io_status status; 607 int rc; 608 609 setup_test(); 610 611 /* Enable QoS */ 612 bdev = &g_bdev.bdev; 613 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 614 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 615 TAILQ_INIT(&bdev->internal.qos->queued); 616 /* 617 * Enable both IOPS and bandwidth rate limits. 618 * In this case, both rate limits will take equal effect. 619 */ 620 /* 2000 I/O per second, or 2 per millisecond */ 621 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 622 /* 8K byte per millisecond with 4K block size */ 623 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 624 625 g_get_io_channel = true; 626 627 set_thread(0); 628 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 629 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 630 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 631 632 set_thread(1); 633 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 634 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 635 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 636 637 /* 638 * Send an I/O on thread 0, which is where the QoS thread is running. 639 */ 640 set_thread(0); 641 status = SPDK_BDEV_IO_STATUS_PENDING; 642 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status); 643 CU_ASSERT(rc == 0); 644 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 645 poll_threads(); 646 stub_complete_io(g_bdev.io_target, 0); 647 poll_threads(); 648 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 649 650 /* Send an I/O on thread 1. The QoS thread is not running here. */ 651 status = SPDK_BDEV_IO_STATUS_PENDING; 652 set_thread(1); 653 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status); 654 CU_ASSERT(rc == 0); 655 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 656 poll_threads(); 657 /* Complete I/O on thread 1. This should not complete the I/O we submitted */ 658 stub_complete_io(g_bdev.io_target, 0); 659 poll_threads(); 660 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); 661 /* Now complete I/O on thread 0 */ 662 set_thread(0); 663 poll_threads(); 664 stub_complete_io(g_bdev.io_target, 0); 665 poll_threads(); 666 CU_ASSERT(status == SPDK_BDEV_IO_STATUS_SUCCESS); 667 668 /* Tear down the channels */ 669 set_thread(0); 670 spdk_put_io_channel(io_ch[0]); 671 set_thread(1); 672 spdk_put_io_channel(io_ch[1]); 673 poll_threads(); 674 set_thread(0); 675 676 /* Close the descriptor, which should stop the qos channel */ 677 spdk_bdev_close(g_desc); 678 poll_threads(); 679 CU_ASSERT(bdev->internal.qos->ch == NULL); 680 681 spdk_bdev_open(bdev, true, NULL, NULL, &g_desc); 682 683 /* Create the channels in reverse order. */ 684 set_thread(1); 685 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 686 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 687 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 688 689 set_thread(0); 690 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 691 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 692 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 693 694 /* Confirm that the qos thread is now thread 1 */ 695 CU_ASSERT(bdev->internal.qos->ch == bdev_ch[1]); 696 697 /* Tear down the channels */ 698 set_thread(0); 699 spdk_put_io_channel(io_ch[0]); 700 set_thread(1); 701 spdk_put_io_channel(io_ch[1]); 702 poll_threads(); 703 704 set_thread(0); 705 706 teardown_test(); 707 } 708 709 static void 710 io_during_qos_queue(void) 711 { 712 struct spdk_io_channel *io_ch[2]; 713 struct spdk_bdev_channel *bdev_ch[2]; 714 struct spdk_bdev *bdev; 715 enum spdk_bdev_io_status status0, status1; 716 int rc; 717 718 setup_test(); 719 MOCK_SET(spdk_get_ticks, 0); 720 721 /* Enable QoS */ 722 bdev = &g_bdev.bdev; 723 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 724 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 725 TAILQ_INIT(&bdev->internal.qos->queued); 726 /* 727 * Enable both IOPS and bandwidth rate limits. 728 * In this case, IOPS rate limit will take effect first. 729 */ 730 /* 1000 I/O per second, or 1 per millisecond */ 731 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 1000; 732 /* 8K byte per millisecond with 4K block size */ 733 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 8192000; 734 735 g_get_io_channel = true; 736 737 /* Create channels */ 738 set_thread(0); 739 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 740 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 741 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 742 743 set_thread(1); 744 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 745 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 746 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 747 748 /* Send two I/O */ 749 status1 = SPDK_BDEV_IO_STATUS_PENDING; 750 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 751 CU_ASSERT(rc == 0); 752 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 753 set_thread(0); 754 status0 = SPDK_BDEV_IO_STATUS_PENDING; 755 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 756 CU_ASSERT(rc == 0); 757 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 758 759 /* Complete any I/O that arrived at the disk */ 760 poll_threads(); 761 set_thread(1); 762 stub_complete_io(g_bdev.io_target, 0); 763 set_thread(0); 764 stub_complete_io(g_bdev.io_target, 0); 765 poll_threads(); 766 767 /* Only one of the I/O should complete. (logical XOR) */ 768 if (status0 == SPDK_BDEV_IO_STATUS_SUCCESS) { 769 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 770 } else { 771 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 772 } 773 774 /* Advance in time by a millisecond */ 775 spdk_delay_us(1000); 776 777 /* Complete more I/O */ 778 poll_threads(); 779 set_thread(1); 780 stub_complete_io(g_bdev.io_target, 0); 781 set_thread(0); 782 stub_complete_io(g_bdev.io_target, 0); 783 poll_threads(); 784 785 /* Now the second I/O should be done */ 786 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_SUCCESS); 787 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_SUCCESS); 788 789 /* Tear down the channels */ 790 set_thread(1); 791 spdk_put_io_channel(io_ch[1]); 792 set_thread(0); 793 spdk_put_io_channel(io_ch[0]); 794 poll_threads(); 795 796 teardown_test(); 797 } 798 799 static void 800 io_during_qos_reset(void) 801 { 802 struct spdk_io_channel *io_ch[2]; 803 struct spdk_bdev_channel *bdev_ch[2]; 804 struct spdk_bdev *bdev; 805 enum spdk_bdev_io_status status0, status1, reset_status; 806 int rc; 807 808 setup_test(); 809 MOCK_SET(spdk_get_ticks, 0); 810 811 /* Enable QoS */ 812 bdev = &g_bdev.bdev; 813 bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos)); 814 SPDK_CU_ASSERT_FATAL(bdev->internal.qos != NULL); 815 TAILQ_INIT(&bdev->internal.qos->queued); 816 /* 817 * Enable both IOPS and bandwidth rate limits. 818 * In this case, bandwidth rate limit will take effect first. 819 */ 820 /* 2000 I/O per second, or 2 per millisecond */ 821 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT].limit = 2000; 822 /* 4K byte per millisecond with 4K block size */ 823 bdev->internal.qos->rate_limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT].limit = 4096000; 824 825 g_get_io_channel = true; 826 827 /* Create channels */ 828 set_thread(0); 829 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 830 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 831 CU_ASSERT(bdev_ch[0]->flags == BDEV_CH_QOS_ENABLED); 832 833 set_thread(1); 834 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 835 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 836 CU_ASSERT(bdev_ch[1]->flags == BDEV_CH_QOS_ENABLED); 837 838 /* Send two I/O. One of these gets queued by QoS. The other is sitting at the disk. */ 839 status1 = SPDK_BDEV_IO_STATUS_PENDING; 840 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &status1); 841 CU_ASSERT(rc == 0); 842 set_thread(0); 843 status0 = SPDK_BDEV_IO_STATUS_PENDING; 844 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &status0); 845 CU_ASSERT(rc == 0); 846 847 poll_threads(); 848 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_PENDING); 849 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_PENDING); 850 851 /* Reset the bdev. */ 852 reset_status = SPDK_BDEV_IO_STATUS_PENDING; 853 rc = spdk_bdev_reset(g_desc, io_ch[0], io_during_io_done, &reset_status); 854 CU_ASSERT(rc == 0); 855 856 /* Complete any I/O that arrived at the disk */ 857 poll_threads(); 858 set_thread(1); 859 stub_complete_io(g_bdev.io_target, 0); 860 set_thread(0); 861 stub_complete_io(g_bdev.io_target, 0); 862 poll_threads(); 863 864 CU_ASSERT(reset_status == SPDK_BDEV_IO_STATUS_SUCCESS); 865 CU_ASSERT(status0 == SPDK_BDEV_IO_STATUS_FAILED); 866 CU_ASSERT(status1 == SPDK_BDEV_IO_STATUS_FAILED); 867 868 /* Tear down the channels */ 869 set_thread(1); 870 spdk_put_io_channel(io_ch[1]); 871 set_thread(0); 872 spdk_put_io_channel(io_ch[0]); 873 poll_threads(); 874 875 teardown_test(); 876 } 877 878 static void 879 enomem_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 880 { 881 enum spdk_bdev_io_status *status = cb_arg; 882 883 *status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; 884 spdk_bdev_free_io(bdev_io); 885 } 886 887 static void 888 enomem(void) 889 { 890 struct spdk_io_channel *io_ch; 891 struct spdk_bdev_channel *bdev_ch; 892 struct spdk_bdev_shared_resource *shared_resource; 893 struct ut_bdev_channel *ut_ch; 894 const uint32_t IO_ARRAY_SIZE = 64; 895 const uint32_t AVAIL = 20; 896 enum spdk_bdev_io_status status[IO_ARRAY_SIZE], status_reset; 897 uint32_t nomem_cnt, i; 898 struct spdk_bdev_io *first_io; 899 int rc; 900 901 setup_test(); 902 903 set_thread(0); 904 io_ch = spdk_bdev_get_io_channel(g_desc); 905 bdev_ch = spdk_io_channel_get_ctx(io_ch); 906 shared_resource = bdev_ch->shared_resource; 907 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 908 ut_ch->avail_cnt = AVAIL; 909 910 /* First submit a number of IOs equal to what the channel can support. */ 911 for (i = 0; i < AVAIL; i++) { 912 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 913 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 914 CU_ASSERT(rc == 0); 915 } 916 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 917 918 /* 919 * Next, submit one additional I/O. This one should fail with ENOMEM and then go onto 920 * the enomem_io list. 921 */ 922 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 923 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 924 CU_ASSERT(rc == 0); 925 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 926 first_io = TAILQ_FIRST(&shared_resource->nomem_io); 927 928 /* 929 * Now submit a bunch more I/O. These should all fail with ENOMEM and get queued behind 930 * the first_io above. 931 */ 932 for (i = AVAIL + 1; i < IO_ARRAY_SIZE; i++) { 933 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 934 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 935 CU_ASSERT(rc == 0); 936 } 937 938 /* Assert that first_io is still at the head of the list. */ 939 CU_ASSERT(TAILQ_FIRST(&shared_resource->nomem_io) == first_io); 940 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == (IO_ARRAY_SIZE - AVAIL)); 941 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 942 CU_ASSERT(shared_resource->nomem_threshold == (AVAIL - NOMEM_THRESHOLD_COUNT)); 943 944 /* 945 * Complete 1 I/O only. The key check here is bdev_io_tailq_cnt - this should not have 946 * changed since completing just 1 I/O should not trigger retrying the queued nomem_io 947 * list. 948 */ 949 stub_complete_io(g_bdev.io_target, 1); 950 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 951 952 /* 953 * Complete enough I/O to hit the nomem_theshold. This should trigger retrying nomem_io, 954 * and we should see I/O get resubmitted to the test bdev module. 955 */ 956 stub_complete_io(g_bdev.io_target, NOMEM_THRESHOLD_COUNT - 1); 957 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) < nomem_cnt); 958 nomem_cnt = bdev_io_tailq_cnt(&shared_resource->nomem_io); 959 960 /* Complete 1 I/O only. This should not trigger retrying the queued nomem_io. */ 961 stub_complete_io(g_bdev.io_target, 1); 962 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == nomem_cnt); 963 964 /* 965 * Send a reset and confirm that all I/O are completed, including the ones that 966 * were queued on the nomem_io list. 967 */ 968 status_reset = SPDK_BDEV_IO_STATUS_PENDING; 969 rc = spdk_bdev_reset(g_desc, io_ch, enomem_done, &status_reset); 970 poll_threads(); 971 CU_ASSERT(rc == 0); 972 /* This will complete the reset. */ 973 stub_complete_io(g_bdev.io_target, 0); 974 975 CU_ASSERT(bdev_io_tailq_cnt(&shared_resource->nomem_io) == 0); 976 CU_ASSERT(shared_resource->io_outstanding == 0); 977 978 spdk_put_io_channel(io_ch); 979 poll_threads(); 980 teardown_test(); 981 } 982 983 static void 984 enomem_multi_bdev(void) 985 { 986 struct spdk_io_channel *io_ch; 987 struct spdk_bdev_channel *bdev_ch; 988 struct spdk_bdev_shared_resource *shared_resource; 989 struct ut_bdev_channel *ut_ch; 990 const uint32_t IO_ARRAY_SIZE = 64; 991 const uint32_t AVAIL = 20; 992 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 993 uint32_t i; 994 struct ut_bdev *second_bdev; 995 struct spdk_bdev_desc *second_desc = NULL; 996 struct spdk_bdev_channel *second_bdev_ch; 997 struct spdk_io_channel *second_ch; 998 int rc; 999 1000 setup_test(); 1001 1002 /* Register second bdev with the same io_target */ 1003 second_bdev = calloc(1, sizeof(*second_bdev)); 1004 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1005 register_bdev(second_bdev, "ut_bdev2", g_bdev.io_target); 1006 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 1007 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1008 1009 set_thread(0); 1010 io_ch = spdk_bdev_get_io_channel(g_desc); 1011 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1012 shared_resource = bdev_ch->shared_resource; 1013 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1014 ut_ch->avail_cnt = AVAIL; 1015 1016 second_ch = spdk_bdev_get_io_channel(second_desc); 1017 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1018 SPDK_CU_ASSERT_FATAL(shared_resource == second_bdev_ch->shared_resource); 1019 1020 /* Saturate io_target through bdev A. */ 1021 for (i = 0; i < AVAIL; i++) { 1022 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1023 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1024 CU_ASSERT(rc == 0); 1025 } 1026 CU_ASSERT(TAILQ_EMPTY(&shared_resource->nomem_io)); 1027 1028 /* 1029 * Now submit I/O through the second bdev. This should fail with ENOMEM 1030 * and then go onto the nomem_io list. 1031 */ 1032 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1033 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1034 CU_ASSERT(rc == 0); 1035 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&shared_resource->nomem_io)); 1036 1037 /* Complete first bdev's I/O. This should retry sending second bdev's nomem_io */ 1038 stub_complete_io(g_bdev.io_target, AVAIL); 1039 1040 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&shared_resource->nomem_io)); 1041 CU_ASSERT(shared_resource->io_outstanding == 1); 1042 1043 /* Now complete our retried I/O */ 1044 stub_complete_io(g_bdev.io_target, 1); 1045 SPDK_CU_ASSERT_FATAL(shared_resource->io_outstanding == 0); 1046 1047 spdk_put_io_channel(io_ch); 1048 spdk_put_io_channel(second_ch); 1049 spdk_bdev_close(second_desc); 1050 unregister_bdev(second_bdev); 1051 poll_threads(); 1052 free(second_bdev); 1053 teardown_test(); 1054 } 1055 1056 1057 static void 1058 enomem_multi_io_target(void) 1059 { 1060 struct spdk_io_channel *io_ch; 1061 struct spdk_bdev_channel *bdev_ch; 1062 struct ut_bdev_channel *ut_ch; 1063 const uint32_t IO_ARRAY_SIZE = 64; 1064 const uint32_t AVAIL = 20; 1065 enum spdk_bdev_io_status status[IO_ARRAY_SIZE]; 1066 uint32_t i; 1067 int new_io_device; 1068 struct ut_bdev *second_bdev; 1069 struct spdk_bdev_desc *second_desc = NULL; 1070 struct spdk_bdev_channel *second_bdev_ch; 1071 struct spdk_io_channel *second_ch; 1072 int rc; 1073 1074 setup_test(); 1075 1076 /* Create new io_target and a second bdev using it */ 1077 spdk_io_device_register(&new_io_device, stub_create_ch, stub_destroy_ch, 1078 sizeof(struct ut_bdev_channel), NULL); 1079 second_bdev = calloc(1, sizeof(*second_bdev)); 1080 SPDK_CU_ASSERT_FATAL(second_bdev != NULL); 1081 register_bdev(second_bdev, "ut_bdev2", &new_io_device); 1082 spdk_bdev_open(&second_bdev->bdev, true, NULL, NULL, &second_desc); 1083 SPDK_CU_ASSERT_FATAL(second_desc != NULL); 1084 1085 set_thread(0); 1086 io_ch = spdk_bdev_get_io_channel(g_desc); 1087 bdev_ch = spdk_io_channel_get_ctx(io_ch); 1088 ut_ch = spdk_io_channel_get_ctx(bdev_ch->channel); 1089 ut_ch->avail_cnt = AVAIL; 1090 1091 /* Different io_target should imply a different shared_resource */ 1092 second_ch = spdk_bdev_get_io_channel(second_desc); 1093 second_bdev_ch = spdk_io_channel_get_ctx(second_ch); 1094 SPDK_CU_ASSERT_FATAL(bdev_ch->shared_resource != second_bdev_ch->shared_resource); 1095 1096 /* Saturate io_target through bdev A. */ 1097 for (i = 0; i < AVAIL; i++) { 1098 status[i] = SPDK_BDEV_IO_STATUS_PENDING; 1099 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[i]); 1100 CU_ASSERT(rc == 0); 1101 } 1102 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1103 1104 /* Issue one more I/O to fill ENOMEM list. */ 1105 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1106 rc = spdk_bdev_read_blocks(g_desc, io_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1107 CU_ASSERT(rc == 0); 1108 SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1109 1110 /* 1111 * Now submit I/O through the second bdev. This should go through and complete 1112 * successfully because we're using a different io_device underneath. 1113 */ 1114 status[AVAIL] = SPDK_BDEV_IO_STATUS_PENDING; 1115 rc = spdk_bdev_read_blocks(second_desc, second_ch, NULL, 0, 1, enomem_done, &status[AVAIL]); 1116 CU_ASSERT(rc == 0); 1117 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&second_bdev_ch->shared_resource->nomem_io)); 1118 stub_complete_io(second_bdev->io_target, 1); 1119 1120 /* Cleanup; Complete outstanding I/O. */ 1121 stub_complete_io(g_bdev.io_target, AVAIL); 1122 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1123 /* Complete the ENOMEM I/O */ 1124 stub_complete_io(g_bdev.io_target, 1); 1125 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1126 1127 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev_ch->shared_resource->nomem_io)); 1128 CU_ASSERT(bdev_ch->shared_resource->io_outstanding == 0); 1129 spdk_put_io_channel(io_ch); 1130 spdk_put_io_channel(second_ch); 1131 spdk_bdev_close(second_desc); 1132 unregister_bdev(second_bdev); 1133 spdk_io_device_unregister(&new_io_device, NULL); 1134 poll_threads(); 1135 free(second_bdev); 1136 teardown_test(); 1137 } 1138 1139 static void 1140 qos_dynamic_enable_done(void *cb_arg, int status) 1141 { 1142 int *rc = cb_arg; 1143 *rc = status; 1144 } 1145 1146 static void 1147 qos_dynamic_enable(void) 1148 { 1149 struct spdk_io_channel *io_ch[2]; 1150 struct spdk_bdev_channel *bdev_ch[2]; 1151 struct spdk_bdev *bdev; 1152 enum spdk_bdev_io_status bdev_io_status[2]; 1153 uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {}; 1154 int status, second_status, rc, i; 1155 1156 setup_test(); 1157 MOCK_SET(spdk_get_ticks, 0); 1158 1159 for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) { 1160 limits[i] = UINT64_MAX; 1161 } 1162 1163 bdev = &g_bdev.bdev; 1164 1165 g_get_io_channel = true; 1166 1167 /* Create channels */ 1168 set_thread(0); 1169 io_ch[0] = spdk_bdev_get_io_channel(g_desc); 1170 bdev_ch[0] = spdk_io_channel_get_ctx(io_ch[0]); 1171 CU_ASSERT(bdev_ch[0]->flags == 0); 1172 1173 set_thread(1); 1174 io_ch[1] = spdk_bdev_get_io_channel(g_desc); 1175 bdev_ch[1] = spdk_io_channel_get_ctx(io_ch[1]); 1176 CU_ASSERT(bdev_ch[1]->flags == 0); 1177 1178 set_thread(0); 1179 1180 /* 1181 * Enable QoS: IOPS and byte per second rate limits. 1182 * More than 10 I/Os allowed per timeslice. 1183 */ 1184 status = -1; 1185 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1186 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 100; 1187 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1188 poll_threads(); 1189 CU_ASSERT(status == 0); 1190 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1191 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1192 1193 /* 1194 * Submit and complete 10 I/O to fill the QoS allotment for this timeslice. 1195 * Additional I/O will then be queued. 1196 */ 1197 set_thread(0); 1198 for (i = 0; i < 10; i++) { 1199 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1200 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1201 CU_ASSERT(rc == 0); 1202 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1203 poll_thread(0); 1204 stub_complete_io(g_bdev.io_target, 0); 1205 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1206 } 1207 1208 /* 1209 * Send two more I/O. These I/O will be queued since the current timeslice allotment has been 1210 * filled already. We want to test that when QoS is disabled that these two I/O: 1211 * 1) are not aborted 1212 * 2) are sent back to their original thread for resubmission 1213 */ 1214 bdev_io_status[0] = SPDK_BDEV_IO_STATUS_PENDING; 1215 rc = spdk_bdev_read_blocks(g_desc, io_ch[0], NULL, 0, 1, io_during_io_done, &bdev_io_status[0]); 1216 CU_ASSERT(rc == 0); 1217 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_PENDING); 1218 set_thread(1); 1219 bdev_io_status[1] = SPDK_BDEV_IO_STATUS_PENDING; 1220 rc = spdk_bdev_read_blocks(g_desc, io_ch[1], NULL, 0, 1, io_during_io_done, &bdev_io_status[1]); 1221 CU_ASSERT(rc == 0); 1222 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1223 poll_threads(); 1224 1225 /* Disable QoS: IOPS rate limit */ 1226 status = -1; 1227 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1228 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1229 poll_threads(); 1230 CU_ASSERT(status == 0); 1231 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1232 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1233 1234 /* Disable QoS: Byte per second rate limit */ 1235 status = -1; 1236 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 0; 1237 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1238 poll_threads(); 1239 CU_ASSERT(status == 0); 1240 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1241 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1242 1243 /* 1244 * All I/O should have been resubmitted back on their original thread. Complete 1245 * all I/O on thread 0, and ensure that only the thread 0 I/O was completed. 1246 */ 1247 set_thread(0); 1248 stub_complete_io(g_bdev.io_target, 0); 1249 poll_threads(); 1250 CU_ASSERT(bdev_io_status[0] == SPDK_BDEV_IO_STATUS_SUCCESS); 1251 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_PENDING); 1252 1253 /* Now complete all I/O on thread 1 and ensure the thread 1 I/O was completed. */ 1254 set_thread(1); 1255 stub_complete_io(g_bdev.io_target, 0); 1256 poll_threads(); 1257 CU_ASSERT(bdev_io_status[1] == SPDK_BDEV_IO_STATUS_SUCCESS); 1258 1259 /* Disable QoS again */ 1260 status = -1; 1261 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1262 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1263 poll_threads(); 1264 CU_ASSERT(status == 0); /* This should succeed */ 1265 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1266 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1267 1268 /* Enable QoS on thread 0 */ 1269 status = -1; 1270 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1271 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1272 poll_threads(); 1273 CU_ASSERT(status == 0); 1274 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1275 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1276 1277 /* Disable QoS on thread 1 */ 1278 set_thread(1); 1279 status = -1; 1280 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 0; 1281 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1282 /* Don't poll yet. This should leave the channels with QoS enabled */ 1283 CU_ASSERT(status == -1); 1284 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1285 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1286 1287 /* Enable QoS. This should immediately fail because the previous disable QoS hasn't completed. */ 1288 second_status = 0; 1289 limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT] = 10; 1290 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &second_status); 1291 poll_threads(); 1292 CU_ASSERT(status == 0); /* The disable should succeed */ 1293 CU_ASSERT(second_status < 0); /* The enable should fail */ 1294 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) == 0); 1295 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) == 0); 1296 1297 /* Enable QoS on thread 1. This should succeed now that the disable has completed. */ 1298 status = -1; 1299 limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = 10000; 1300 spdk_bdev_set_qos_rate_limits(bdev, limits, qos_dynamic_enable_done, &status); 1301 poll_threads(); 1302 CU_ASSERT(status == 0); 1303 CU_ASSERT((bdev_ch[0]->flags & BDEV_CH_QOS_ENABLED) != 0); 1304 CU_ASSERT((bdev_ch[1]->flags & BDEV_CH_QOS_ENABLED) != 0); 1305 1306 /* Tear down the channels */ 1307 set_thread(0); 1308 spdk_put_io_channel(io_ch[0]); 1309 set_thread(1); 1310 spdk_put_io_channel(io_ch[1]); 1311 poll_threads(); 1312 1313 set_thread(0); 1314 teardown_test(); 1315 } 1316 1317 int 1318 main(int argc, char **argv) 1319 { 1320 CU_pSuite suite = NULL; 1321 unsigned int num_failures; 1322 1323 if (CU_initialize_registry() != CUE_SUCCESS) { 1324 return CU_get_error(); 1325 } 1326 1327 suite = CU_add_suite("bdev", NULL, NULL); 1328 if (suite == NULL) { 1329 CU_cleanup_registry(); 1330 return CU_get_error(); 1331 } 1332 1333 if ( 1334 CU_add_test(suite, "basic", basic) == NULL || 1335 CU_add_test(suite, "unregister_and_close", unregister_and_close) == NULL || 1336 CU_add_test(suite, "basic_qos", basic_qos) == NULL || 1337 CU_add_test(suite, "put_channel_during_reset", put_channel_during_reset) == NULL || 1338 CU_add_test(suite, "aborted_reset", aborted_reset) == NULL || 1339 CU_add_test(suite, "io_during_reset", io_during_reset) == NULL || 1340 CU_add_test(suite, "io_during_qos_queue", io_during_qos_queue) == NULL || 1341 CU_add_test(suite, "io_during_qos_reset", io_during_qos_reset) == NULL || 1342 CU_add_test(suite, "enomem", enomem) == NULL || 1343 CU_add_test(suite, "enomem_multi_bdev", enomem_multi_bdev) == NULL || 1344 CU_add_test(suite, "enomem_multi_io_target", enomem_multi_io_target) == NULL || 1345 CU_add_test(suite, "qos_dynamic_enable", qos_dynamic_enable) == NULL 1346 ) { 1347 CU_cleanup_registry(); 1348 return CU_get_error(); 1349 } 1350 1351 CU_basic_set_mode(CU_BRM_VERBOSE); 1352 CU_basic_run_tests(); 1353 num_failures = CU_get_number_of_failures(); 1354 CU_cleanup_registry(); 1355 return num_failures; 1356 } 1357