1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk_cunit.h" 35 36 #include "common/lib/ut_multithread.c" 37 #include "unit/lib/json_mock.c" 38 39 #include "spdk/config.h" 40 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 41 #undef SPDK_CONFIG_VTUNE 42 43 #include "bdev/bdev.c" 44 45 struct spdk_trace_histories *g_trace_histories; 46 DEFINE_STUB_V(spdk_trace_add_register_fn, (struct spdk_trace_register_fn *reg_fn)); 47 DEFINE_STUB_V(spdk_trace_register_owner, (uint8_t type, char id_prefix)); 48 DEFINE_STUB_V(spdk_trace_register_object, (uint8_t type, char id_prefix)); 49 DEFINE_STUB_V(spdk_trace_register_description, (const char *name, 50 uint16_t tpoint_id, uint8_t owner_type, 51 uint8_t object_type, uint8_t new_object, 52 uint8_t arg1_type, const char *arg1_name)); 53 DEFINE_STUB_V(_spdk_trace_record, (uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, 54 uint32_t size, uint64_t object_id, uint64_t arg1)); 55 DEFINE_STUB(spdk_notify_send, uint64_t, (const char *type, const char *ctx), 0); 56 DEFINE_STUB(spdk_notify_type_register, struct spdk_notify_type *, (const char *type), NULL); 57 58 59 int g_status; 60 int g_count; 61 enum spdk_bdev_event_type g_event_type1; 62 enum spdk_bdev_event_type g_event_type2; 63 struct spdk_histogram_data *g_histogram; 64 void *g_unregister_arg; 65 int g_unregister_rc; 66 67 void 68 spdk_scsi_nvme_translate(const struct spdk_bdev_io *bdev_io, 69 int *sc, int *sk, int *asc, int *ascq) 70 { 71 } 72 73 static int 74 null_init(void) 75 { 76 return 0; 77 } 78 79 static int 80 null_clean(void) 81 { 82 return 0; 83 } 84 85 static int 86 stub_destruct(void *ctx) 87 { 88 return 0; 89 } 90 91 struct ut_expected_io { 92 uint8_t type; 93 uint64_t offset; 94 uint64_t length; 95 int iovcnt; 96 struct iovec iov[BDEV_IO_NUM_CHILD_IOV]; 97 void *md_buf; 98 TAILQ_ENTRY(ut_expected_io) link; 99 }; 100 101 struct bdev_ut_channel { 102 TAILQ_HEAD(, spdk_bdev_io) outstanding_io; 103 uint32_t outstanding_io_count; 104 TAILQ_HEAD(, ut_expected_io) expected_io; 105 }; 106 107 static bool g_io_done; 108 static struct spdk_bdev_io *g_bdev_io; 109 static enum spdk_bdev_io_status g_io_status; 110 static enum spdk_bdev_io_status g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 111 static uint32_t g_bdev_ut_io_device; 112 static struct bdev_ut_channel *g_bdev_ut_channel; 113 static void *g_compare_read_buf; 114 static uint32_t g_compare_read_buf_len; 115 static void *g_compare_write_buf; 116 static uint32_t g_compare_write_buf_len; 117 static bool g_abort_done; 118 static enum spdk_bdev_io_status g_abort_status; 119 120 static struct ut_expected_io * 121 ut_alloc_expected_io(uint8_t type, uint64_t offset, uint64_t length, int iovcnt) 122 { 123 struct ut_expected_io *expected_io; 124 125 expected_io = calloc(1, sizeof(*expected_io)); 126 SPDK_CU_ASSERT_FATAL(expected_io != NULL); 127 128 expected_io->type = type; 129 expected_io->offset = offset; 130 expected_io->length = length; 131 expected_io->iovcnt = iovcnt; 132 133 return expected_io; 134 } 135 136 static void 137 ut_expected_io_set_iov(struct ut_expected_io *expected_io, int pos, void *base, size_t len) 138 { 139 expected_io->iov[pos].iov_base = base; 140 expected_io->iov[pos].iov_len = len; 141 } 142 143 static void 144 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 145 { 146 struct bdev_ut_channel *ch = spdk_io_channel_get_ctx(_ch); 147 struct ut_expected_io *expected_io; 148 struct iovec *iov, *expected_iov; 149 struct spdk_bdev_io *bio_to_abort; 150 int i; 151 152 g_bdev_io = bdev_io; 153 154 if (g_compare_read_buf && bdev_io->type == SPDK_BDEV_IO_TYPE_READ) { 155 uint32_t len = bdev_io->u.bdev.iovs[0].iov_len; 156 157 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 158 CU_ASSERT(g_compare_read_buf_len == len); 159 memcpy(bdev_io->u.bdev.iovs[0].iov_base, g_compare_read_buf, len); 160 } 161 162 if (g_compare_write_buf && bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) { 163 uint32_t len = bdev_io->u.bdev.iovs[0].iov_len; 164 165 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 166 CU_ASSERT(g_compare_write_buf_len == len); 167 memcpy(g_compare_write_buf, bdev_io->u.bdev.iovs[0].iov_base, len); 168 } 169 170 if (g_compare_read_buf && bdev_io->type == SPDK_BDEV_IO_TYPE_COMPARE) { 171 uint32_t len = bdev_io->u.bdev.iovs[0].iov_len; 172 173 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 174 CU_ASSERT(g_compare_read_buf_len == len); 175 if (memcmp(bdev_io->u.bdev.iovs[0].iov_base, g_compare_read_buf, len)) { 176 g_io_exp_status = SPDK_BDEV_IO_STATUS_MISCOMPARE; 177 } 178 } 179 180 if (bdev_io->type == SPDK_BDEV_IO_TYPE_ABORT) { 181 if (g_io_exp_status == SPDK_BDEV_IO_STATUS_SUCCESS) { 182 TAILQ_FOREACH(bio_to_abort, &ch->outstanding_io, module_link) { 183 if (bio_to_abort == bdev_io->u.abort.bio_to_abort) { 184 TAILQ_REMOVE(&ch->outstanding_io, bio_to_abort, module_link); 185 ch->outstanding_io_count--; 186 spdk_bdev_io_complete(bio_to_abort, SPDK_BDEV_IO_STATUS_FAILED); 187 break; 188 } 189 } 190 } 191 } 192 193 TAILQ_INSERT_TAIL(&ch->outstanding_io, bdev_io, module_link); 194 ch->outstanding_io_count++; 195 196 expected_io = TAILQ_FIRST(&ch->expected_io); 197 if (expected_io == NULL) { 198 return; 199 } 200 TAILQ_REMOVE(&ch->expected_io, expected_io, link); 201 202 if (expected_io->type != SPDK_BDEV_IO_TYPE_INVALID) { 203 CU_ASSERT(bdev_io->type == expected_io->type); 204 } 205 206 if (expected_io->md_buf != NULL) { 207 CU_ASSERT(expected_io->md_buf == bdev_io->u.bdev.md_buf); 208 } 209 210 if (expected_io->length == 0) { 211 free(expected_io); 212 return; 213 } 214 215 CU_ASSERT(expected_io->offset == bdev_io->u.bdev.offset_blocks); 216 CU_ASSERT(expected_io->length = bdev_io->u.bdev.num_blocks); 217 218 if (expected_io->iovcnt == 0) { 219 free(expected_io); 220 /* UNMAP, WRITE_ZEROES and FLUSH don't have iovs, so we can just return now. */ 221 return; 222 } 223 224 CU_ASSERT(expected_io->iovcnt == bdev_io->u.bdev.iovcnt); 225 for (i = 0; i < expected_io->iovcnt; i++) { 226 iov = &bdev_io->u.bdev.iovs[i]; 227 expected_iov = &expected_io->iov[i]; 228 CU_ASSERT(iov->iov_len == expected_iov->iov_len); 229 CU_ASSERT(iov->iov_base == expected_iov->iov_base); 230 } 231 232 free(expected_io); 233 } 234 235 static void 236 stub_submit_request_get_buf_cb(struct spdk_io_channel *_ch, 237 struct spdk_bdev_io *bdev_io, bool success) 238 { 239 CU_ASSERT(success == true); 240 241 stub_submit_request(_ch, bdev_io); 242 } 243 244 static void 245 stub_submit_request_get_buf(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 246 { 247 spdk_bdev_io_get_buf(bdev_io, stub_submit_request_get_buf_cb, 248 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); 249 } 250 251 static uint32_t 252 stub_complete_io(uint32_t num_to_complete) 253 { 254 struct bdev_ut_channel *ch = g_bdev_ut_channel; 255 struct spdk_bdev_io *bdev_io; 256 static enum spdk_bdev_io_status io_status; 257 uint32_t num_completed = 0; 258 259 while (num_completed < num_to_complete) { 260 if (TAILQ_EMPTY(&ch->outstanding_io)) { 261 break; 262 } 263 bdev_io = TAILQ_FIRST(&ch->outstanding_io); 264 TAILQ_REMOVE(&ch->outstanding_io, bdev_io, module_link); 265 ch->outstanding_io_count--; 266 io_status = g_io_exp_status == SPDK_BDEV_IO_STATUS_SUCCESS ? SPDK_BDEV_IO_STATUS_SUCCESS : 267 g_io_exp_status; 268 spdk_bdev_io_complete(bdev_io, io_status); 269 num_completed++; 270 } 271 272 return num_completed; 273 } 274 275 static struct spdk_io_channel * 276 bdev_ut_get_io_channel(void *ctx) 277 { 278 return spdk_get_io_channel(&g_bdev_ut_io_device); 279 } 280 281 static bool g_io_types_supported[SPDK_BDEV_NUM_IO_TYPES] = { 282 [SPDK_BDEV_IO_TYPE_READ] = true, 283 [SPDK_BDEV_IO_TYPE_WRITE] = true, 284 [SPDK_BDEV_IO_TYPE_COMPARE] = true, 285 [SPDK_BDEV_IO_TYPE_UNMAP] = true, 286 [SPDK_BDEV_IO_TYPE_FLUSH] = true, 287 [SPDK_BDEV_IO_TYPE_RESET] = true, 288 [SPDK_BDEV_IO_TYPE_NVME_ADMIN] = true, 289 [SPDK_BDEV_IO_TYPE_NVME_IO] = true, 290 [SPDK_BDEV_IO_TYPE_NVME_IO_MD] = true, 291 [SPDK_BDEV_IO_TYPE_WRITE_ZEROES] = true, 292 [SPDK_BDEV_IO_TYPE_ZCOPY] = true, 293 [SPDK_BDEV_IO_TYPE_ABORT] = true, 294 }; 295 296 static void 297 ut_enable_io_type(enum spdk_bdev_io_type io_type, bool enable) 298 { 299 g_io_types_supported[io_type] = enable; 300 } 301 302 static bool 303 stub_io_type_supported(void *_bdev, enum spdk_bdev_io_type io_type) 304 { 305 return g_io_types_supported[io_type]; 306 } 307 308 static struct spdk_bdev_fn_table fn_table = { 309 .destruct = stub_destruct, 310 .submit_request = stub_submit_request, 311 .get_io_channel = bdev_ut_get_io_channel, 312 .io_type_supported = stub_io_type_supported, 313 }; 314 315 static int 316 bdev_ut_create_ch(void *io_device, void *ctx_buf) 317 { 318 struct bdev_ut_channel *ch = ctx_buf; 319 320 CU_ASSERT(g_bdev_ut_channel == NULL); 321 g_bdev_ut_channel = ch; 322 323 TAILQ_INIT(&ch->outstanding_io); 324 ch->outstanding_io_count = 0; 325 TAILQ_INIT(&ch->expected_io); 326 return 0; 327 } 328 329 static void 330 bdev_ut_destroy_ch(void *io_device, void *ctx_buf) 331 { 332 CU_ASSERT(g_bdev_ut_channel != NULL); 333 g_bdev_ut_channel = NULL; 334 } 335 336 struct spdk_bdev_module bdev_ut_if; 337 338 static int 339 bdev_ut_module_init(void) 340 { 341 spdk_io_device_register(&g_bdev_ut_io_device, bdev_ut_create_ch, bdev_ut_destroy_ch, 342 sizeof(struct bdev_ut_channel), NULL); 343 spdk_bdev_module_init_done(&bdev_ut_if); 344 return 0; 345 } 346 347 static void 348 bdev_ut_module_fini(void) 349 { 350 spdk_io_device_unregister(&g_bdev_ut_io_device, NULL); 351 } 352 353 struct spdk_bdev_module bdev_ut_if = { 354 .name = "bdev_ut", 355 .module_init = bdev_ut_module_init, 356 .module_fini = bdev_ut_module_fini, 357 .async_init = true, 358 }; 359 360 static void vbdev_ut_examine(struct spdk_bdev *bdev); 361 362 static int 363 vbdev_ut_module_init(void) 364 { 365 return 0; 366 } 367 368 static void 369 vbdev_ut_module_fini(void) 370 { 371 } 372 373 struct spdk_bdev_module vbdev_ut_if = { 374 .name = "vbdev_ut", 375 .module_init = vbdev_ut_module_init, 376 .module_fini = vbdev_ut_module_fini, 377 .examine_config = vbdev_ut_examine, 378 }; 379 380 SPDK_BDEV_MODULE_REGISTER(bdev_ut, &bdev_ut_if) 381 SPDK_BDEV_MODULE_REGISTER(vbdev_ut, &vbdev_ut_if) 382 383 static void 384 vbdev_ut_examine(struct spdk_bdev *bdev) 385 { 386 spdk_bdev_module_examine_done(&vbdev_ut_if); 387 } 388 389 static struct spdk_bdev * 390 allocate_bdev(char *name) 391 { 392 struct spdk_bdev *bdev; 393 int rc; 394 395 bdev = calloc(1, sizeof(*bdev)); 396 SPDK_CU_ASSERT_FATAL(bdev != NULL); 397 398 bdev->name = name; 399 bdev->fn_table = &fn_table; 400 bdev->module = &bdev_ut_if; 401 bdev->blockcnt = 1024; 402 bdev->blocklen = 512; 403 404 rc = spdk_bdev_register(bdev); 405 CU_ASSERT(rc == 0); 406 407 return bdev; 408 } 409 410 static struct spdk_bdev * 411 allocate_vbdev(char *name) 412 { 413 struct spdk_bdev *bdev; 414 int rc; 415 416 bdev = calloc(1, sizeof(*bdev)); 417 SPDK_CU_ASSERT_FATAL(bdev != NULL); 418 419 bdev->name = name; 420 bdev->fn_table = &fn_table; 421 bdev->module = &vbdev_ut_if; 422 423 rc = spdk_bdev_register(bdev); 424 CU_ASSERT(rc == 0); 425 426 return bdev; 427 } 428 429 static void 430 free_bdev(struct spdk_bdev *bdev) 431 { 432 spdk_bdev_unregister(bdev, NULL, NULL); 433 poll_threads(); 434 memset(bdev, 0xFF, sizeof(*bdev)); 435 free(bdev); 436 } 437 438 static void 439 free_vbdev(struct spdk_bdev *bdev) 440 { 441 spdk_bdev_unregister(bdev, NULL, NULL); 442 poll_threads(); 443 memset(bdev, 0xFF, sizeof(*bdev)); 444 free(bdev); 445 } 446 447 static void 448 get_device_stat_cb(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat, void *cb_arg, int rc) 449 { 450 const char *bdev_name; 451 452 CU_ASSERT(bdev != NULL); 453 CU_ASSERT(rc == 0); 454 bdev_name = spdk_bdev_get_name(bdev); 455 CU_ASSERT_STRING_EQUAL(bdev_name, "bdev0"); 456 457 free(stat); 458 459 *(bool *)cb_arg = true; 460 } 461 462 static void 463 bdev_unregister_cb(void *cb_arg, int rc) 464 { 465 g_unregister_arg = cb_arg; 466 g_unregister_rc = rc; 467 } 468 469 static void 470 bdev_ut_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 471 { 472 } 473 474 static void 475 bdev_open_cb1(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 476 { 477 struct spdk_bdev_desc *desc = *(struct spdk_bdev_desc **)event_ctx; 478 479 g_event_type1 = type; 480 if (SPDK_BDEV_EVENT_REMOVE == type) { 481 spdk_bdev_close(desc); 482 } 483 } 484 485 static void 486 bdev_open_cb2(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 487 { 488 struct spdk_bdev_desc *desc = *(struct spdk_bdev_desc **)event_ctx; 489 490 g_event_type2 = type; 491 if (SPDK_BDEV_EVENT_REMOVE == type) { 492 spdk_bdev_close(desc); 493 } 494 } 495 496 static void 497 get_device_stat_test(void) 498 { 499 struct spdk_bdev *bdev; 500 struct spdk_bdev_io_stat *stat; 501 bool done; 502 503 bdev = allocate_bdev("bdev0"); 504 stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 505 if (stat == NULL) { 506 free_bdev(bdev); 507 return; 508 } 509 510 done = false; 511 spdk_bdev_get_device_stat(bdev, stat, get_device_stat_cb, &done); 512 while (!done) { poll_threads(); } 513 514 free_bdev(bdev); 515 } 516 517 static void 518 open_write_test(void) 519 { 520 struct spdk_bdev *bdev[9]; 521 struct spdk_bdev_desc *desc[9] = {}; 522 int rc; 523 524 /* 525 * Create a tree of bdevs to test various open w/ write cases. 526 * 527 * bdev0 through bdev3 are physical block devices, such as NVMe 528 * namespaces or Ceph block devices. 529 * 530 * bdev4 is a virtual bdev with multiple base bdevs. This models 531 * caching or RAID use cases. 532 * 533 * bdev5 through bdev7 are all virtual bdevs with the same base 534 * bdev (except bdev7). This models partitioning or logical volume 535 * use cases. 536 * 537 * bdev7 is a virtual bdev with multiple base bdevs. One of base bdevs 538 * (bdev2) is shared with other virtual bdevs: bdev5 and bdev6. This 539 * models caching, RAID, partitioning or logical volumes use cases. 540 * 541 * bdev8 is a virtual bdev with multiple base bdevs, but these 542 * base bdevs are themselves virtual bdevs. 543 * 544 * bdev8 545 * | 546 * +----------+ 547 * | | 548 * bdev4 bdev5 bdev6 bdev7 549 * | | | | 550 * +---+---+ +---+ + +---+---+ 551 * | | \ | / \ 552 * bdev0 bdev1 bdev2 bdev3 553 */ 554 555 bdev[0] = allocate_bdev("bdev0"); 556 rc = spdk_bdev_module_claim_bdev(bdev[0], NULL, &bdev_ut_if); 557 CU_ASSERT(rc == 0); 558 559 bdev[1] = allocate_bdev("bdev1"); 560 rc = spdk_bdev_module_claim_bdev(bdev[1], NULL, &bdev_ut_if); 561 CU_ASSERT(rc == 0); 562 563 bdev[2] = allocate_bdev("bdev2"); 564 rc = spdk_bdev_module_claim_bdev(bdev[2], NULL, &bdev_ut_if); 565 CU_ASSERT(rc == 0); 566 567 bdev[3] = allocate_bdev("bdev3"); 568 rc = spdk_bdev_module_claim_bdev(bdev[3], NULL, &bdev_ut_if); 569 CU_ASSERT(rc == 0); 570 571 bdev[4] = allocate_vbdev("bdev4"); 572 rc = spdk_bdev_module_claim_bdev(bdev[4], NULL, &bdev_ut_if); 573 CU_ASSERT(rc == 0); 574 575 bdev[5] = allocate_vbdev("bdev5"); 576 rc = spdk_bdev_module_claim_bdev(bdev[5], NULL, &bdev_ut_if); 577 CU_ASSERT(rc == 0); 578 579 bdev[6] = allocate_vbdev("bdev6"); 580 581 bdev[7] = allocate_vbdev("bdev7"); 582 583 bdev[8] = allocate_vbdev("bdev8"); 584 585 /* Open bdev0 read-only. This should succeed. */ 586 rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc[0]); 587 CU_ASSERT(rc == 0); 588 SPDK_CU_ASSERT_FATAL(desc[0] != NULL); 589 CU_ASSERT(bdev[0] == spdk_bdev_desc_get_bdev(desc[0])); 590 spdk_bdev_close(desc[0]); 591 592 /* 593 * Open bdev1 read/write. This should fail since bdev1 has been claimed 594 * by a vbdev module. 595 */ 596 rc = spdk_bdev_open_ext("bdev1", true, bdev_ut_event_cb, NULL, &desc[1]); 597 CU_ASSERT(rc == -EPERM); 598 599 /* 600 * Open bdev4 read/write. This should fail since bdev3 has been claimed 601 * by a vbdev module. 602 */ 603 rc = spdk_bdev_open_ext("bdev4", true, bdev_ut_event_cb, NULL, &desc[4]); 604 CU_ASSERT(rc == -EPERM); 605 606 /* Open bdev4 read-only. This should succeed. */ 607 rc = spdk_bdev_open_ext("bdev4", false, bdev_ut_event_cb, NULL, &desc[4]); 608 CU_ASSERT(rc == 0); 609 SPDK_CU_ASSERT_FATAL(desc[4] != NULL); 610 CU_ASSERT(bdev[4] == spdk_bdev_desc_get_bdev(desc[4])); 611 spdk_bdev_close(desc[4]); 612 613 /* 614 * Open bdev8 read/write. This should succeed since it is a leaf 615 * bdev. 616 */ 617 rc = spdk_bdev_open_ext("bdev8", true, bdev_ut_event_cb, NULL, &desc[8]); 618 CU_ASSERT(rc == 0); 619 SPDK_CU_ASSERT_FATAL(desc[8] != NULL); 620 CU_ASSERT(bdev[8] == spdk_bdev_desc_get_bdev(desc[8])); 621 spdk_bdev_close(desc[8]); 622 623 /* 624 * Open bdev5 read/write. This should fail since bdev4 has been claimed 625 * by a vbdev module. 626 */ 627 rc = spdk_bdev_open_ext("bdev5", true, bdev_ut_event_cb, NULL, &desc[5]); 628 CU_ASSERT(rc == -EPERM); 629 630 /* Open bdev4 read-only. This should succeed. */ 631 rc = spdk_bdev_open_ext("bdev5", false, bdev_ut_event_cb, NULL, &desc[5]); 632 CU_ASSERT(rc == 0); 633 SPDK_CU_ASSERT_FATAL(desc[5] != NULL); 634 CU_ASSERT(bdev[5] == spdk_bdev_desc_get_bdev(desc[5])); 635 spdk_bdev_close(desc[5]); 636 637 free_vbdev(bdev[8]); 638 639 free_vbdev(bdev[5]); 640 free_vbdev(bdev[6]); 641 free_vbdev(bdev[7]); 642 643 free_vbdev(bdev[4]); 644 645 free_bdev(bdev[0]); 646 free_bdev(bdev[1]); 647 free_bdev(bdev[2]); 648 free_bdev(bdev[3]); 649 } 650 651 static void 652 bytes_to_blocks_test(void) 653 { 654 struct spdk_bdev bdev; 655 uint64_t offset_blocks, num_blocks; 656 657 memset(&bdev, 0, sizeof(bdev)); 658 659 bdev.blocklen = 512; 660 661 /* All parameters valid */ 662 offset_blocks = 0; 663 num_blocks = 0; 664 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 512, &offset_blocks, 1024, &num_blocks) == 0); 665 CU_ASSERT(offset_blocks == 1); 666 CU_ASSERT(num_blocks == 2); 667 668 /* Offset not a block multiple */ 669 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 3, &offset_blocks, 512, &num_blocks) != 0); 670 671 /* Length not a block multiple */ 672 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 512, &offset_blocks, 3, &num_blocks) != 0); 673 674 /* In case blocklen not the power of two */ 675 bdev.blocklen = 100; 676 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 100, &offset_blocks, 200, &num_blocks) == 0); 677 CU_ASSERT(offset_blocks == 1); 678 CU_ASSERT(num_blocks == 2); 679 680 /* Offset not a block multiple */ 681 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 3, &offset_blocks, 100, &num_blocks) != 0); 682 683 /* Length not a block multiple */ 684 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 100, &offset_blocks, 3, &num_blocks) != 0); 685 } 686 687 static void 688 num_blocks_test(void) 689 { 690 struct spdk_bdev bdev; 691 struct spdk_bdev_desc *desc = NULL; 692 struct spdk_bdev_desc *desc_ext = NULL; 693 int rc; 694 695 memset(&bdev, 0, sizeof(bdev)); 696 bdev.name = "num_blocks"; 697 bdev.fn_table = &fn_table; 698 bdev.module = &bdev_ut_if; 699 spdk_bdev_register(&bdev); 700 spdk_bdev_notify_blockcnt_change(&bdev, 50); 701 702 /* Growing block number */ 703 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 70) == 0); 704 /* Shrinking block number */ 705 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 30) == 0); 706 707 /* In case bdev opened */ 708 rc = spdk_bdev_open(&bdev, false, NULL, NULL, &desc); 709 CU_ASSERT(rc == 0); 710 SPDK_CU_ASSERT_FATAL(desc != NULL); 711 712 /* Growing block number */ 713 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 80) == 0); 714 /* Shrinking block number */ 715 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 20) != 0); 716 717 /* In case bdev opened with ext API */ 718 rc = spdk_bdev_open_ext("num_blocks", false, bdev_open_cb1, &desc_ext, &desc_ext); 719 CU_ASSERT(rc == 0); 720 SPDK_CU_ASSERT_FATAL(desc_ext != NULL); 721 CU_ASSERT(&bdev == spdk_bdev_desc_get_bdev(desc_ext)); 722 723 g_event_type1 = 0xFF; 724 /* Growing block number */ 725 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 90) == 0); 726 727 poll_threads(); 728 CU_ASSERT_EQUAL(g_event_type1, SPDK_BDEV_EVENT_RESIZE); 729 730 g_event_type1 = 0xFF; 731 /* Growing block number and closing */ 732 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 100) == 0); 733 734 spdk_bdev_close(desc); 735 spdk_bdev_close(desc_ext); 736 spdk_bdev_unregister(&bdev, NULL, NULL); 737 738 poll_threads(); 739 740 /* Callback is not called for closed device */ 741 CU_ASSERT_EQUAL(g_event_type1, 0xFF); 742 } 743 744 static void 745 io_valid_test(void) 746 { 747 struct spdk_bdev bdev; 748 749 memset(&bdev, 0, sizeof(bdev)); 750 751 bdev.blocklen = 512; 752 spdk_bdev_notify_blockcnt_change(&bdev, 100); 753 754 /* All parameters valid */ 755 CU_ASSERT(bdev_io_valid_blocks(&bdev, 1, 2) == true); 756 757 /* Last valid block */ 758 CU_ASSERT(bdev_io_valid_blocks(&bdev, 99, 1) == true); 759 760 /* Offset past end of bdev */ 761 CU_ASSERT(bdev_io_valid_blocks(&bdev, 100, 1) == false); 762 763 /* Offset + length past end of bdev */ 764 CU_ASSERT(bdev_io_valid_blocks(&bdev, 99, 2) == false); 765 766 /* Offset near end of uint64_t range (2^64 - 1) */ 767 CU_ASSERT(bdev_io_valid_blocks(&bdev, 18446744073709551615ULL, 1) == false); 768 } 769 770 static void 771 alias_add_del_test(void) 772 { 773 struct spdk_bdev *bdev[3]; 774 int rc; 775 776 /* Creating and registering bdevs */ 777 bdev[0] = allocate_bdev("bdev0"); 778 SPDK_CU_ASSERT_FATAL(bdev[0] != 0); 779 780 bdev[1] = allocate_bdev("bdev1"); 781 SPDK_CU_ASSERT_FATAL(bdev[1] != 0); 782 783 bdev[2] = allocate_bdev("bdev2"); 784 SPDK_CU_ASSERT_FATAL(bdev[2] != 0); 785 786 poll_threads(); 787 788 /* 789 * Trying adding an alias identical to name. 790 * Alias is identical to name, so it can not be added to aliases list 791 */ 792 rc = spdk_bdev_alias_add(bdev[0], bdev[0]->name); 793 CU_ASSERT(rc == -EEXIST); 794 795 /* 796 * Trying to add empty alias, 797 * this one should fail 798 */ 799 rc = spdk_bdev_alias_add(bdev[0], NULL); 800 CU_ASSERT(rc == -EINVAL); 801 802 /* Trying adding same alias to two different registered bdevs */ 803 804 /* Alias is used first time, so this one should pass */ 805 rc = spdk_bdev_alias_add(bdev[0], "proper alias 0"); 806 CU_ASSERT(rc == 0); 807 808 /* Alias was added to another bdev, so this one should fail */ 809 rc = spdk_bdev_alias_add(bdev[1], "proper alias 0"); 810 CU_ASSERT(rc == -EEXIST); 811 812 /* Alias is used first time, so this one should pass */ 813 rc = spdk_bdev_alias_add(bdev[1], "proper alias 1"); 814 CU_ASSERT(rc == 0); 815 816 /* Trying removing an alias from registered bdevs */ 817 818 /* Alias is not on a bdev aliases list, so this one should fail */ 819 rc = spdk_bdev_alias_del(bdev[0], "not existing"); 820 CU_ASSERT(rc == -ENOENT); 821 822 /* Alias is present on a bdev aliases list, so this one should pass */ 823 rc = spdk_bdev_alias_del(bdev[0], "proper alias 0"); 824 CU_ASSERT(rc == 0); 825 826 /* Alias is present on a bdev aliases list, so this one should pass */ 827 rc = spdk_bdev_alias_del(bdev[1], "proper alias 1"); 828 CU_ASSERT(rc == 0); 829 830 /* Trying to remove name instead of alias, so this one should fail, name cannot be changed or removed */ 831 rc = spdk_bdev_alias_del(bdev[0], bdev[0]->name); 832 CU_ASSERT(rc != 0); 833 834 /* Trying to del all alias from empty alias list */ 835 spdk_bdev_alias_del_all(bdev[2]); 836 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev[2]->aliases)); 837 838 /* Trying to del all alias from non-empty alias list */ 839 rc = spdk_bdev_alias_add(bdev[2], "alias0"); 840 CU_ASSERT(rc == 0); 841 rc = spdk_bdev_alias_add(bdev[2], "alias1"); 842 CU_ASSERT(rc == 0); 843 spdk_bdev_alias_del_all(bdev[2]); 844 CU_ASSERT(TAILQ_EMPTY(&bdev[2]->aliases)); 845 846 /* Unregister and free bdevs */ 847 spdk_bdev_unregister(bdev[0], NULL, NULL); 848 spdk_bdev_unregister(bdev[1], NULL, NULL); 849 spdk_bdev_unregister(bdev[2], NULL, NULL); 850 851 poll_threads(); 852 853 free(bdev[0]); 854 free(bdev[1]); 855 free(bdev[2]); 856 } 857 858 static void 859 io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 860 { 861 g_io_done = true; 862 g_io_status = bdev_io->internal.status; 863 spdk_bdev_free_io(bdev_io); 864 } 865 866 static void 867 bdev_init_cb(void *arg, int rc) 868 { 869 CU_ASSERT(rc == 0); 870 } 871 872 static void 873 bdev_fini_cb(void *arg) 874 { 875 } 876 877 struct bdev_ut_io_wait_entry { 878 struct spdk_bdev_io_wait_entry entry; 879 struct spdk_io_channel *io_ch; 880 struct spdk_bdev_desc *desc; 881 bool submitted; 882 }; 883 884 static void 885 io_wait_cb(void *arg) 886 { 887 struct bdev_ut_io_wait_entry *entry = arg; 888 int rc; 889 890 rc = spdk_bdev_read_blocks(entry->desc, entry->io_ch, NULL, 0, 1, io_done, NULL); 891 CU_ASSERT(rc == 0); 892 entry->submitted = true; 893 } 894 895 static void 896 bdev_io_types_test(void) 897 { 898 struct spdk_bdev *bdev; 899 struct spdk_bdev_desc *desc = NULL; 900 struct spdk_io_channel *io_ch; 901 struct spdk_bdev_opts bdev_opts = { 902 .bdev_io_pool_size = 4, 903 .bdev_io_cache_size = 2, 904 }; 905 int rc; 906 907 rc = spdk_bdev_set_opts(&bdev_opts); 908 CU_ASSERT(rc == 0); 909 spdk_bdev_initialize(bdev_init_cb, NULL); 910 poll_threads(); 911 912 bdev = allocate_bdev("bdev0"); 913 914 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 915 CU_ASSERT(rc == 0); 916 poll_threads(); 917 SPDK_CU_ASSERT_FATAL(desc != NULL); 918 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 919 io_ch = spdk_bdev_get_io_channel(desc); 920 CU_ASSERT(io_ch != NULL); 921 922 /* WRITE and WRITE ZEROES are not supported */ 923 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, false); 924 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE, false); 925 rc = spdk_bdev_write_zeroes_blocks(desc, io_ch, 0, 128, io_done, NULL); 926 CU_ASSERT(rc == -ENOTSUP); 927 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, true); 928 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE, true); 929 930 spdk_put_io_channel(io_ch); 931 spdk_bdev_close(desc); 932 free_bdev(bdev); 933 spdk_bdev_finish(bdev_fini_cb, NULL); 934 poll_threads(); 935 } 936 937 static void 938 bdev_io_wait_test(void) 939 { 940 struct spdk_bdev *bdev; 941 struct spdk_bdev_desc *desc = NULL; 942 struct spdk_io_channel *io_ch; 943 struct spdk_bdev_opts bdev_opts = { 944 .bdev_io_pool_size = 4, 945 .bdev_io_cache_size = 2, 946 }; 947 struct bdev_ut_io_wait_entry io_wait_entry; 948 struct bdev_ut_io_wait_entry io_wait_entry2; 949 int rc; 950 951 rc = spdk_bdev_set_opts(&bdev_opts); 952 CU_ASSERT(rc == 0); 953 spdk_bdev_initialize(bdev_init_cb, NULL); 954 poll_threads(); 955 956 bdev = allocate_bdev("bdev0"); 957 958 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 959 CU_ASSERT(rc == 0); 960 poll_threads(); 961 SPDK_CU_ASSERT_FATAL(desc != NULL); 962 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 963 io_ch = spdk_bdev_get_io_channel(desc); 964 CU_ASSERT(io_ch != NULL); 965 966 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 967 CU_ASSERT(rc == 0); 968 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 969 CU_ASSERT(rc == 0); 970 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 971 CU_ASSERT(rc == 0); 972 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 973 CU_ASSERT(rc == 0); 974 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 975 976 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 977 CU_ASSERT(rc == -ENOMEM); 978 979 io_wait_entry.entry.bdev = bdev; 980 io_wait_entry.entry.cb_fn = io_wait_cb; 981 io_wait_entry.entry.cb_arg = &io_wait_entry; 982 io_wait_entry.io_ch = io_ch; 983 io_wait_entry.desc = desc; 984 io_wait_entry.submitted = false; 985 /* Cannot use the same io_wait_entry for two different calls. */ 986 memcpy(&io_wait_entry2, &io_wait_entry, sizeof(io_wait_entry)); 987 io_wait_entry2.entry.cb_arg = &io_wait_entry2; 988 989 /* Queue two I/O waits. */ 990 rc = spdk_bdev_queue_io_wait(bdev, io_ch, &io_wait_entry.entry); 991 CU_ASSERT(rc == 0); 992 CU_ASSERT(io_wait_entry.submitted == false); 993 rc = spdk_bdev_queue_io_wait(bdev, io_ch, &io_wait_entry2.entry); 994 CU_ASSERT(rc == 0); 995 CU_ASSERT(io_wait_entry2.submitted == false); 996 997 stub_complete_io(1); 998 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 999 CU_ASSERT(io_wait_entry.submitted == true); 1000 CU_ASSERT(io_wait_entry2.submitted == false); 1001 1002 stub_complete_io(1); 1003 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 1004 CU_ASSERT(io_wait_entry2.submitted == true); 1005 1006 stub_complete_io(4); 1007 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1008 1009 spdk_put_io_channel(io_ch); 1010 spdk_bdev_close(desc); 1011 free_bdev(bdev); 1012 spdk_bdev_finish(bdev_fini_cb, NULL); 1013 poll_threads(); 1014 } 1015 1016 static void 1017 bdev_io_spans_split_test(void) 1018 { 1019 struct spdk_bdev bdev; 1020 struct spdk_bdev_io bdev_io; 1021 struct iovec iov[BDEV_IO_NUM_CHILD_IOV]; 1022 1023 memset(&bdev, 0, sizeof(bdev)); 1024 bdev_io.u.bdev.iovs = iov; 1025 1026 bdev.optimal_io_boundary = 0; 1027 bdev.max_segment_size = 0; 1028 bdev.max_num_segments = 0; 1029 bdev_io.bdev = &bdev; 1030 1031 /* bdev has no optimal_io_boundary and max_size set - so this should return false. */ 1032 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1033 1034 bdev.split_on_optimal_io_boundary = true; 1035 bdev.optimal_io_boundary = 32; 1036 bdev_io.type = SPDK_BDEV_IO_TYPE_RESET; 1037 1038 /* RESETs are not based on LBAs - so this should return false. */ 1039 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1040 1041 bdev_io.type = SPDK_BDEV_IO_TYPE_READ; 1042 bdev_io.u.bdev.offset_blocks = 0; 1043 bdev_io.u.bdev.num_blocks = 32; 1044 1045 /* This I/O run right up to, but does not cross, the boundary - so this should return false. */ 1046 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1047 1048 bdev_io.u.bdev.num_blocks = 33; 1049 1050 /* This I/O spans a boundary. */ 1051 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1052 1053 bdev_io.u.bdev.num_blocks = 32; 1054 bdev.max_segment_size = 512 * 32; 1055 bdev.max_num_segments = 1; 1056 bdev_io.u.bdev.iovcnt = 1; 1057 iov[0].iov_len = 512; 1058 1059 /* Does not cross and exceed max_size or max_segs */ 1060 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1061 1062 bdev.split_on_optimal_io_boundary = false; 1063 bdev.max_segment_size = 512; 1064 bdev.max_num_segments = 1; 1065 bdev_io.u.bdev.iovcnt = 2; 1066 1067 /* Exceed max_segs */ 1068 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1069 1070 bdev.max_num_segments = 2; 1071 iov[0].iov_len = 513; 1072 iov[1].iov_len = 512; 1073 1074 /* Exceed max_sizes */ 1075 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1076 } 1077 1078 static void 1079 bdev_io_boundary_split_test(void) 1080 { 1081 struct spdk_bdev *bdev; 1082 struct spdk_bdev_desc *desc = NULL; 1083 struct spdk_io_channel *io_ch; 1084 struct spdk_bdev_opts bdev_opts = { 1085 .bdev_io_pool_size = 512, 1086 .bdev_io_cache_size = 64, 1087 }; 1088 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 1089 struct ut_expected_io *expected_io; 1090 void *md_buf = (void *)0xFF000000; 1091 uint64_t i; 1092 int rc; 1093 1094 rc = spdk_bdev_set_opts(&bdev_opts); 1095 CU_ASSERT(rc == 0); 1096 spdk_bdev_initialize(bdev_init_cb, NULL); 1097 1098 bdev = allocate_bdev("bdev0"); 1099 1100 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 1101 CU_ASSERT(rc == 0); 1102 SPDK_CU_ASSERT_FATAL(desc != NULL); 1103 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 1104 io_ch = spdk_bdev_get_io_channel(desc); 1105 CU_ASSERT(io_ch != NULL); 1106 1107 bdev->optimal_io_boundary = 16; 1108 bdev->split_on_optimal_io_boundary = false; 1109 1110 g_io_done = false; 1111 1112 /* First test that the I/O does not get split if split_on_optimal_io_boundary == false. */ 1113 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 8, 1); 1114 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 8 * 512); 1115 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1116 1117 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL); 1118 CU_ASSERT(rc == 0); 1119 CU_ASSERT(g_io_done == false); 1120 1121 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1122 stub_complete_io(1); 1123 CU_ASSERT(g_io_done == true); 1124 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1125 1126 bdev->split_on_optimal_io_boundary = true; 1127 bdev->md_interleave = false; 1128 bdev->md_len = 8; 1129 1130 /* Now test that a single-vector command is split correctly. 1131 * Offset 14, length 8, payload 0xF000 1132 * Child - Offset 14, length 2, payload 0xF000 1133 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 1134 * 1135 * Set up the expected values before calling spdk_bdev_read_blocks 1136 */ 1137 g_io_done = false; 1138 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 1); 1139 expected_io->md_buf = md_buf; 1140 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 2 * 512); 1141 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1142 1143 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 6, 1); 1144 expected_io->md_buf = md_buf + 2 * 8; 1145 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 6 * 512); 1146 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1147 1148 /* spdk_bdev_read_blocks will submit the first child immediately. */ 1149 rc = spdk_bdev_read_blocks_with_md(desc, io_ch, (void *)0xF000, md_buf, 1150 14, 8, io_done, NULL); 1151 CU_ASSERT(rc == 0); 1152 CU_ASSERT(g_io_done == false); 1153 1154 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1155 stub_complete_io(2); 1156 CU_ASSERT(g_io_done == true); 1157 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1158 1159 /* Now set up a more complex, multi-vector command that needs to be split, 1160 * including splitting iovecs. 1161 */ 1162 iov[0].iov_base = (void *)0x10000; 1163 iov[0].iov_len = 512; 1164 iov[1].iov_base = (void *)0x20000; 1165 iov[1].iov_len = 20 * 512; 1166 iov[2].iov_base = (void *)0x30000; 1167 iov[2].iov_len = 11 * 512; 1168 1169 g_io_done = false; 1170 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 2); 1171 expected_io->md_buf = md_buf; 1172 ut_expected_io_set_iov(expected_io, 0, (void *)0x10000, 512); 1173 ut_expected_io_set_iov(expected_io, 1, (void *)0x20000, 512); 1174 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1175 1176 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 16, 1); 1177 expected_io->md_buf = md_buf + 2 * 8; 1178 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 512), 16 * 512); 1179 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1180 1181 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 14, 2); 1182 expected_io->md_buf = md_buf + 18 * 8; 1183 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 17 * 512), 3 * 512); 1184 ut_expected_io_set_iov(expected_io, 1, (void *)0x30000, 11 * 512); 1185 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1186 1187 rc = spdk_bdev_writev_blocks_with_md(desc, io_ch, iov, 3, md_buf, 1188 14, 32, io_done, NULL); 1189 CU_ASSERT(rc == 0); 1190 CU_ASSERT(g_io_done == false); 1191 1192 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 1193 stub_complete_io(3); 1194 CU_ASSERT(g_io_done == true); 1195 1196 /* Test multi vector command that needs to be split by strip and then needs to be 1197 * split further due to the capacity of child iovs. 1198 */ 1199 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV * 2; i++) { 1200 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1201 iov[i].iov_len = 512; 1202 } 1203 1204 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1205 g_io_done = false; 1206 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, BDEV_IO_NUM_CHILD_IOV, 1207 BDEV_IO_NUM_CHILD_IOV); 1208 expected_io->md_buf = md_buf; 1209 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1210 ut_expected_io_set_iov(expected_io, i, (void *)((i + 1) * 0x10000), 512); 1211 } 1212 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1213 1214 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1215 BDEV_IO_NUM_CHILD_IOV, BDEV_IO_NUM_CHILD_IOV); 1216 expected_io->md_buf = md_buf + BDEV_IO_NUM_CHILD_IOV * 8; 1217 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1218 ut_expected_io_set_iov(expected_io, i, 1219 (void *)((i + 1 + BDEV_IO_NUM_CHILD_IOV) * 0x10000), 512); 1220 } 1221 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1222 1223 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, md_buf, 1224 0, BDEV_IO_NUM_CHILD_IOV * 2, io_done, NULL); 1225 CU_ASSERT(rc == 0); 1226 CU_ASSERT(g_io_done == false); 1227 1228 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1229 stub_complete_io(1); 1230 CU_ASSERT(g_io_done == false); 1231 1232 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1233 stub_complete_io(1); 1234 CU_ASSERT(g_io_done == true); 1235 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1236 1237 /* Test multi vector command that needs to be split by strip and then needs to be 1238 * split further due to the capacity of child iovs. In this case, the length of 1239 * the rest of iovec array with an I/O boundary is the multiple of block size. 1240 */ 1241 1242 /* Fill iovec array for exactly one boundary. The iovec cnt for this boundary 1243 * is BDEV_IO_NUM_CHILD_IOV + 1, which exceeds the capacity of child iovs. 1244 */ 1245 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1246 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1247 iov[i].iov_len = 512; 1248 } 1249 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1250 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1251 iov[i].iov_len = 256; 1252 } 1253 iov[BDEV_IO_NUM_CHILD_IOV].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000); 1254 iov[BDEV_IO_NUM_CHILD_IOV].iov_len = 512; 1255 1256 /* Add an extra iovec to trigger split */ 1257 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000); 1258 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_len = 512; 1259 1260 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1261 g_io_done = false; 1262 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 1263 BDEV_IO_NUM_CHILD_IOV - 1, BDEV_IO_NUM_CHILD_IOV); 1264 expected_io->md_buf = md_buf; 1265 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1266 ut_expected_io_set_iov(expected_io, i, 1267 (void *)((i + 1) * 0x10000), 512); 1268 } 1269 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1270 ut_expected_io_set_iov(expected_io, i, 1271 (void *)((i + 1) * 0x10000), 256); 1272 } 1273 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1274 1275 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV - 1, 1276 1, 1); 1277 expected_io->md_buf = md_buf + (BDEV_IO_NUM_CHILD_IOV - 1) * 8; 1278 ut_expected_io_set_iov(expected_io, 0, 1279 (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000), 512); 1280 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1281 1282 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1283 1, 1); 1284 expected_io->md_buf = md_buf + BDEV_IO_NUM_CHILD_IOV * 8; 1285 ut_expected_io_set_iov(expected_io, 0, 1286 (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000), 512); 1287 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1288 1289 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 2, md_buf, 1290 0, BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1291 CU_ASSERT(rc == 0); 1292 CU_ASSERT(g_io_done == false); 1293 1294 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1295 stub_complete_io(1); 1296 CU_ASSERT(g_io_done == false); 1297 1298 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1299 stub_complete_io(2); 1300 CU_ASSERT(g_io_done == true); 1301 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1302 1303 /* Test multi vector command that needs to be split by strip and then needs to be 1304 * split further due to the capacity of child iovs, the child request offset should 1305 * be rewind to last aligned offset and go success without error. 1306 */ 1307 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 1; i++) { 1308 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1309 iov[i].iov_len = 512; 1310 } 1311 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_base = (void *)(BDEV_IO_NUM_CHILD_IOV * 0x10000); 1312 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_len = 256; 1313 1314 iov[BDEV_IO_NUM_CHILD_IOV].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000); 1315 iov[BDEV_IO_NUM_CHILD_IOV].iov_len = 256; 1316 1317 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000); 1318 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_len = 512; 1319 1320 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1321 g_io_done = false; 1322 g_io_status = 0; 1323 /* The first expected io should be start from offset 0 to BDEV_IO_NUM_CHILD_IOV - 1 */ 1324 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 1325 BDEV_IO_NUM_CHILD_IOV - 1, BDEV_IO_NUM_CHILD_IOV - 1); 1326 expected_io->md_buf = md_buf; 1327 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 1; i++) { 1328 ut_expected_io_set_iov(expected_io, i, 1329 (void *)((i + 1) * 0x10000), 512); 1330 } 1331 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1332 /* The second expected io should be start from offset BDEV_IO_NUM_CHILD_IOV - 1 to BDEV_IO_NUM_CHILD_IOV */ 1333 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV - 1, 1334 1, 2); 1335 expected_io->md_buf = md_buf + (BDEV_IO_NUM_CHILD_IOV - 1) * 8; 1336 ut_expected_io_set_iov(expected_io, 0, 1337 (void *)(BDEV_IO_NUM_CHILD_IOV * 0x10000), 256); 1338 ut_expected_io_set_iov(expected_io, 1, 1339 (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000), 256); 1340 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1341 /* The third expected io should be start from offset BDEV_IO_NUM_CHILD_IOV to BDEV_IO_NUM_CHILD_IOV + 1 */ 1342 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1343 1, 1); 1344 expected_io->md_buf = md_buf + BDEV_IO_NUM_CHILD_IOV * 8; 1345 ut_expected_io_set_iov(expected_io, 0, 1346 (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000), 512); 1347 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1348 1349 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, md_buf, 1350 0, BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1351 CU_ASSERT(rc == 0); 1352 CU_ASSERT(g_io_done == false); 1353 1354 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1355 stub_complete_io(1); 1356 CU_ASSERT(g_io_done == false); 1357 1358 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1359 stub_complete_io(2); 1360 CU_ASSERT(g_io_done == true); 1361 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1362 1363 /* Test multi vector command that needs to be split due to the IO boundary and 1364 * the capacity of child iovs. Especially test the case when the command is 1365 * split due to the capacity of child iovs, the tail address is not aligned with 1366 * block size and is rewinded to the aligned address. 1367 * 1368 * The iovecs used in read request is complex but is based on the data 1369 * collected in the real issue. We change the base addresses but keep the lengths 1370 * not to loose the credibility of the test. 1371 */ 1372 bdev->optimal_io_boundary = 128; 1373 g_io_done = false; 1374 g_io_status = 0; 1375 1376 for (i = 0; i < 31; i++) { 1377 iov[i].iov_base = (void *)(0xFEED0000000 + (i << 20)); 1378 iov[i].iov_len = 1024; 1379 } 1380 iov[31].iov_base = (void *)0xFEED1F00000; 1381 iov[31].iov_len = 32768; 1382 iov[32].iov_base = (void *)0xFEED2000000; 1383 iov[32].iov_len = 160; 1384 iov[33].iov_base = (void *)0xFEED2100000; 1385 iov[33].iov_len = 4096; 1386 iov[34].iov_base = (void *)0xFEED2200000; 1387 iov[34].iov_len = 4096; 1388 iov[35].iov_base = (void *)0xFEED2300000; 1389 iov[35].iov_len = 4096; 1390 iov[36].iov_base = (void *)0xFEED2400000; 1391 iov[36].iov_len = 4096; 1392 iov[37].iov_base = (void *)0xFEED2500000; 1393 iov[37].iov_len = 4096; 1394 iov[38].iov_base = (void *)0xFEED2600000; 1395 iov[38].iov_len = 4096; 1396 iov[39].iov_base = (void *)0xFEED2700000; 1397 iov[39].iov_len = 4096; 1398 iov[40].iov_base = (void *)0xFEED2800000; 1399 iov[40].iov_len = 4096; 1400 iov[41].iov_base = (void *)0xFEED2900000; 1401 iov[41].iov_len = 4096; 1402 iov[42].iov_base = (void *)0xFEED2A00000; 1403 iov[42].iov_len = 4096; 1404 iov[43].iov_base = (void *)0xFEED2B00000; 1405 iov[43].iov_len = 12288; 1406 iov[44].iov_base = (void *)0xFEED2C00000; 1407 iov[44].iov_len = 8192; 1408 iov[45].iov_base = (void *)0xFEED2F00000; 1409 iov[45].iov_len = 4096; 1410 iov[46].iov_base = (void *)0xFEED3000000; 1411 iov[46].iov_len = 4096; 1412 iov[47].iov_base = (void *)0xFEED3100000; 1413 iov[47].iov_len = 4096; 1414 iov[48].iov_base = (void *)0xFEED3200000; 1415 iov[48].iov_len = 24576; 1416 iov[49].iov_base = (void *)0xFEED3300000; 1417 iov[49].iov_len = 16384; 1418 iov[50].iov_base = (void *)0xFEED3400000; 1419 iov[50].iov_len = 12288; 1420 iov[51].iov_base = (void *)0xFEED3500000; 1421 iov[51].iov_len = 4096; 1422 iov[52].iov_base = (void *)0xFEED3600000; 1423 iov[52].iov_len = 4096; 1424 iov[53].iov_base = (void *)0xFEED3700000; 1425 iov[53].iov_len = 4096; 1426 iov[54].iov_base = (void *)0xFEED3800000; 1427 iov[54].iov_len = 28672; 1428 iov[55].iov_base = (void *)0xFEED3900000; 1429 iov[55].iov_len = 20480; 1430 iov[56].iov_base = (void *)0xFEED3A00000; 1431 iov[56].iov_len = 4096; 1432 iov[57].iov_base = (void *)0xFEED3B00000; 1433 iov[57].iov_len = 12288; 1434 iov[58].iov_base = (void *)0xFEED3C00000; 1435 iov[58].iov_len = 4096; 1436 iov[59].iov_base = (void *)0xFEED3D00000; 1437 iov[59].iov_len = 4096; 1438 iov[60].iov_base = (void *)0xFEED3E00000; 1439 iov[60].iov_len = 352; 1440 1441 /* The 1st child IO must be from iov[0] to iov[31] split by the capacity 1442 * of child iovs, 1443 */ 1444 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 126, 32); 1445 expected_io->md_buf = md_buf; 1446 for (i = 0; i < 32; i++) { 1447 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 1448 } 1449 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1450 1451 /* The 2nd child IO must be from iov[32] to the first 864 bytes of iov[33] 1452 * split by the IO boundary requirement. 1453 */ 1454 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 126, 2, 2); 1455 expected_io->md_buf = md_buf + 126 * 8; 1456 ut_expected_io_set_iov(expected_io, 0, iov[32].iov_base, iov[32].iov_len); 1457 ut_expected_io_set_iov(expected_io, 1, iov[33].iov_base, 864); 1458 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1459 1460 /* The 3rd child IO must be from the remaining 3232 bytes of iov[33] to 1461 * the first 864 bytes of iov[46] split by the IO boundary requirement. 1462 */ 1463 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 128, 128, 14); 1464 expected_io->md_buf = md_buf + 128 * 8; 1465 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[33].iov_base + 864), 1466 iov[33].iov_len - 864); 1467 ut_expected_io_set_iov(expected_io, 1, iov[34].iov_base, iov[34].iov_len); 1468 ut_expected_io_set_iov(expected_io, 2, iov[35].iov_base, iov[35].iov_len); 1469 ut_expected_io_set_iov(expected_io, 3, iov[36].iov_base, iov[36].iov_len); 1470 ut_expected_io_set_iov(expected_io, 4, iov[37].iov_base, iov[37].iov_len); 1471 ut_expected_io_set_iov(expected_io, 5, iov[38].iov_base, iov[38].iov_len); 1472 ut_expected_io_set_iov(expected_io, 6, iov[39].iov_base, iov[39].iov_len); 1473 ut_expected_io_set_iov(expected_io, 7, iov[40].iov_base, iov[40].iov_len); 1474 ut_expected_io_set_iov(expected_io, 8, iov[41].iov_base, iov[41].iov_len); 1475 ut_expected_io_set_iov(expected_io, 9, iov[42].iov_base, iov[42].iov_len); 1476 ut_expected_io_set_iov(expected_io, 10, iov[43].iov_base, iov[43].iov_len); 1477 ut_expected_io_set_iov(expected_io, 11, iov[44].iov_base, iov[44].iov_len); 1478 ut_expected_io_set_iov(expected_io, 12, iov[45].iov_base, iov[45].iov_len); 1479 ut_expected_io_set_iov(expected_io, 13, iov[46].iov_base, 864); 1480 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1481 1482 /* The 4th child IO must be from the remaining 3232 bytes of iov[46] to the 1483 * first 864 bytes of iov[52] split by the IO boundary requirement. 1484 */ 1485 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 256, 128, 7); 1486 expected_io->md_buf = md_buf + 256 * 8; 1487 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[46].iov_base + 864), 1488 iov[46].iov_len - 864); 1489 ut_expected_io_set_iov(expected_io, 1, iov[47].iov_base, iov[47].iov_len); 1490 ut_expected_io_set_iov(expected_io, 2, iov[48].iov_base, iov[48].iov_len); 1491 ut_expected_io_set_iov(expected_io, 3, iov[49].iov_base, iov[49].iov_len); 1492 ut_expected_io_set_iov(expected_io, 4, iov[50].iov_base, iov[50].iov_len); 1493 ut_expected_io_set_iov(expected_io, 5, iov[51].iov_base, iov[51].iov_len); 1494 ut_expected_io_set_iov(expected_io, 6, iov[52].iov_base, 864); 1495 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1496 1497 /* The 5th child IO must be from the remaining 3232 bytes of iov[52] to 1498 * the first 4096 bytes of iov[57] split by the IO boundary requirement. 1499 */ 1500 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 384, 128, 6); 1501 expected_io->md_buf = md_buf + 384 * 8; 1502 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[52].iov_base + 864), 1503 iov[52].iov_len - 864); 1504 ut_expected_io_set_iov(expected_io, 1, iov[53].iov_base, iov[53].iov_len); 1505 ut_expected_io_set_iov(expected_io, 2, iov[54].iov_base, iov[54].iov_len); 1506 ut_expected_io_set_iov(expected_io, 3, iov[55].iov_base, iov[55].iov_len); 1507 ut_expected_io_set_iov(expected_io, 4, iov[56].iov_base, iov[56].iov_len); 1508 ut_expected_io_set_iov(expected_io, 5, iov[57].iov_base, 4960); 1509 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1510 1511 /* The 6th child IO must be from the remaining 7328 bytes of iov[57] 1512 * to the first 3936 bytes of iov[58] split by the capacity of child iovs. 1513 */ 1514 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 512, 30, 3); 1515 expected_io->md_buf = md_buf + 512 * 8; 1516 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[57].iov_base + 4960), 1517 iov[57].iov_len - 4960); 1518 ut_expected_io_set_iov(expected_io, 1, iov[58].iov_base, iov[58].iov_len); 1519 ut_expected_io_set_iov(expected_io, 2, iov[59].iov_base, 3936); 1520 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1521 1522 /* The 7th child IO is from the remaining 160 bytes of iov[59] and iov[60]. */ 1523 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 542, 1, 2); 1524 expected_io->md_buf = md_buf + 542 * 8; 1525 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[59].iov_base + 3936), 1526 iov[59].iov_len - 3936); 1527 ut_expected_io_set_iov(expected_io, 1, iov[60].iov_base, iov[60].iov_len); 1528 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1529 1530 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, 61, md_buf, 1531 0, 543, io_done, NULL); 1532 CU_ASSERT(rc == 0); 1533 CU_ASSERT(g_io_done == false); 1534 1535 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1536 stub_complete_io(1); 1537 CU_ASSERT(g_io_done == false); 1538 1539 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 5); 1540 stub_complete_io(5); 1541 CU_ASSERT(g_io_done == false); 1542 1543 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1544 stub_complete_io(1); 1545 CU_ASSERT(g_io_done == true); 1546 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1547 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 1548 1549 /* Test a WRITE_ZEROES that would span an I/O boundary. WRITE_ZEROES should not be 1550 * split, so test that. 1551 */ 1552 bdev->optimal_io_boundary = 15; 1553 g_io_done = false; 1554 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, 9, 36, 0); 1555 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1556 1557 rc = spdk_bdev_write_zeroes_blocks(desc, io_ch, 9, 36, io_done, NULL); 1558 CU_ASSERT(rc == 0); 1559 CU_ASSERT(g_io_done == false); 1560 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1561 stub_complete_io(1); 1562 CU_ASSERT(g_io_done == true); 1563 1564 /* Test an UNMAP. This should also not be split. */ 1565 bdev->optimal_io_boundary = 16; 1566 g_io_done = false; 1567 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, 15, 2, 0); 1568 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1569 1570 rc = spdk_bdev_unmap_blocks(desc, io_ch, 15, 2, io_done, NULL); 1571 CU_ASSERT(rc == 0); 1572 CU_ASSERT(g_io_done == false); 1573 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1574 stub_complete_io(1); 1575 CU_ASSERT(g_io_done == true); 1576 1577 /* Test a FLUSH. This should also not be split. */ 1578 bdev->optimal_io_boundary = 16; 1579 g_io_done = false; 1580 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_FLUSH, 15, 2, 0); 1581 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1582 1583 rc = spdk_bdev_flush_blocks(desc, io_ch, 15, 2, io_done, NULL); 1584 CU_ASSERT(rc == 0); 1585 CU_ASSERT(g_io_done == false); 1586 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1587 stub_complete_io(1); 1588 CU_ASSERT(g_io_done == true); 1589 1590 CU_ASSERT(TAILQ_EMPTY(&g_bdev_ut_channel->expected_io)); 1591 1592 /* Children requests return an error status */ 1593 bdev->optimal_io_boundary = 16; 1594 iov[0].iov_base = (void *)0x10000; 1595 iov[0].iov_len = 512 * 64; 1596 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 1597 g_io_done = false; 1598 g_io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1599 1600 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, 1, 1, 64, io_done, NULL); 1601 CU_ASSERT(rc == 0); 1602 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 5); 1603 stub_complete_io(4); 1604 CU_ASSERT(g_io_done == false); 1605 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 1606 stub_complete_io(1); 1607 CU_ASSERT(g_io_done == true); 1608 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 1609 1610 /* Test if a multi vector command terminated with failure before continueing 1611 * splitting process when one of child I/O failed. 1612 * The multi vector command is as same as the above that needs to be split by strip 1613 * and then needs to be split further due to the capacity of child iovs. 1614 */ 1615 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 1; i++) { 1616 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1617 iov[i].iov_len = 512; 1618 } 1619 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_base = (void *)(BDEV_IO_NUM_CHILD_IOV * 0x10000); 1620 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_len = 256; 1621 1622 iov[BDEV_IO_NUM_CHILD_IOV].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000); 1623 iov[BDEV_IO_NUM_CHILD_IOV].iov_len = 256; 1624 1625 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000); 1626 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_len = 512; 1627 1628 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1629 1630 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 1631 g_io_done = false; 1632 g_io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1633 1634 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, 0, 1635 BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1636 CU_ASSERT(rc == 0); 1637 CU_ASSERT(g_io_done == false); 1638 1639 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1640 stub_complete_io(1); 1641 CU_ASSERT(g_io_done == true); 1642 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 1643 1644 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1645 1646 /* for this test we will create the following conditions to hit the code path where 1647 * we are trying to send and IO following a split that has no iovs because we had to 1648 * trim them for alignment reasons. 1649 * 1650 * - 16K boundary, our IO will start at offset 0 with a length of 0x4200 1651 * - Our IOVs are 0x212 in size so that we run into the 16K boundary at child IOV 1652 * position 30 and overshoot by 0x2e. 1653 * - That means we'll send the IO and loop back to pick up the remaining bytes at 1654 * child IOV index 31. When we do, we find that we have to shorten index 31 by 0x2e 1655 * which eliniates that vector so we just send the first split IO with 30 vectors 1656 * and let the completion pick up the last 2 vectors. 1657 */ 1658 bdev->optimal_io_boundary = 32; 1659 bdev->split_on_optimal_io_boundary = true; 1660 g_io_done = false; 1661 1662 /* Init all parent IOVs to 0x212 */ 1663 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV + 2; i++) { 1664 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1665 iov[i].iov_len = 0x212; 1666 } 1667 1668 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, BDEV_IO_NUM_CHILD_IOV, 1669 BDEV_IO_NUM_CHILD_IOV - 1); 1670 /* expect 0-29 to be 1:1 with the parent iov */ 1671 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1672 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 1673 } 1674 1675 /* expect index 30 to be shortened to 0x1e4 (0x212 - 0x1e) because of the alignment 1676 * where 0x1e is the amount we overshot the 16K boundary 1677 */ 1678 ut_expected_io_set_iov(expected_io, BDEV_IO_NUM_CHILD_IOV - 2, 1679 (void *)(iov[BDEV_IO_NUM_CHILD_IOV - 2].iov_base), 0x1e4); 1680 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1681 1682 /* 2nd child IO will have 2 remaining vectors, one to pick up from the one that was 1683 * shortened that take it to the next boundary and then a final one to get us to 1684 * 0x4200 bytes for the IO. 1685 */ 1686 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1687 BDEV_IO_NUM_CHILD_IOV, 2); 1688 /* position 30 picked up the remaining bytes to the next boundary */ 1689 ut_expected_io_set_iov(expected_io, 0, 1690 (void *)(iov[BDEV_IO_NUM_CHILD_IOV - 2].iov_base + 0x1e4), 0x2e); 1691 1692 /* position 31 picked the the rest of the trasnfer to get us to 0x4200 */ 1693 ut_expected_io_set_iov(expected_io, 1, 1694 (void *)(iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_base), 0x1d2); 1695 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1696 1697 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 1, 0, 1698 BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1699 CU_ASSERT(rc == 0); 1700 CU_ASSERT(g_io_done == false); 1701 1702 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1703 stub_complete_io(1); 1704 CU_ASSERT(g_io_done == false); 1705 1706 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1707 stub_complete_io(1); 1708 CU_ASSERT(g_io_done == true); 1709 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1710 1711 spdk_put_io_channel(io_ch); 1712 spdk_bdev_close(desc); 1713 free_bdev(bdev); 1714 spdk_bdev_finish(bdev_fini_cb, NULL); 1715 poll_threads(); 1716 } 1717 1718 static void 1719 bdev_io_max_size_and_segment_split_test(void) 1720 { 1721 struct spdk_bdev *bdev; 1722 struct spdk_bdev_desc *desc = NULL; 1723 struct spdk_io_channel *io_ch; 1724 struct spdk_bdev_opts bdev_opts = { 1725 .bdev_io_pool_size = 512, 1726 .bdev_io_cache_size = 64, 1727 }; 1728 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 1729 struct ut_expected_io *expected_io; 1730 uint64_t i; 1731 int rc; 1732 1733 rc = spdk_bdev_set_opts(&bdev_opts); 1734 CU_ASSERT(rc == 0); 1735 spdk_bdev_initialize(bdev_init_cb, NULL); 1736 1737 bdev = allocate_bdev("bdev0"); 1738 1739 rc = spdk_bdev_open_ext(bdev->name, true, bdev_ut_event_cb, NULL, &desc); 1740 CU_ASSERT(rc == 0); 1741 SPDK_CU_ASSERT_FATAL(desc != NULL); 1742 io_ch = spdk_bdev_get_io_channel(desc); 1743 CU_ASSERT(io_ch != NULL); 1744 1745 bdev->split_on_optimal_io_boundary = false; 1746 bdev->optimal_io_boundary = 0; 1747 1748 /* Case 0 max_num_segments == 0. 1749 * but segment size 2 * 512 > 512 1750 */ 1751 bdev->max_segment_size = 512; 1752 bdev->max_num_segments = 0; 1753 g_io_done = false; 1754 1755 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 2); 1756 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512); 1757 ut_expected_io_set_iov(expected_io, 1, (void *)(0xF000 + 512), 512); 1758 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1759 1760 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 2, io_done, NULL); 1761 CU_ASSERT(rc == 0); 1762 CU_ASSERT(g_io_done == false); 1763 1764 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1765 stub_complete_io(1); 1766 CU_ASSERT(g_io_done == true); 1767 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1768 1769 /* Case 1 max_segment_size == 0 1770 * but iov num 2 > 1. 1771 */ 1772 bdev->max_segment_size = 0; 1773 bdev->max_num_segments = 1; 1774 g_io_done = false; 1775 1776 iov[0].iov_base = (void *)0x10000; 1777 iov[0].iov_len = 512; 1778 iov[1].iov_base = (void *)0x20000; 1779 iov[1].iov_len = 8 * 512; 1780 1781 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 1, 1); 1782 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, iov[0].iov_len); 1783 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1784 1785 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 15, 8, 1); 1786 ut_expected_io_set_iov(expected_io, 0, iov[1].iov_base, iov[1].iov_len); 1787 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1788 1789 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, 2, 14, 9, io_done, NULL); 1790 CU_ASSERT(rc == 0); 1791 CU_ASSERT(g_io_done == false); 1792 1793 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1794 stub_complete_io(2); 1795 CU_ASSERT(g_io_done == true); 1796 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1797 1798 /* Test that a non-vector command is split correctly. 1799 * Set up the expected values before calling spdk_bdev_read_blocks 1800 */ 1801 bdev->max_segment_size = 512; 1802 bdev->max_num_segments = 1; 1803 g_io_done = false; 1804 1805 /* Child IO 0 */ 1806 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 1, 1); 1807 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512); 1808 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1809 1810 /* Child IO 1 */ 1811 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 15, 1, 1); 1812 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 1 * 512), 512); 1813 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1814 1815 /* spdk_bdev_read_blocks will submit the first child immediately. */ 1816 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 2, io_done, NULL); 1817 CU_ASSERT(rc == 0); 1818 CU_ASSERT(g_io_done == false); 1819 1820 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1821 stub_complete_io(2); 1822 CU_ASSERT(g_io_done == true); 1823 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1824 1825 /* Now set up a more complex, multi-vector command that needs to be split, 1826 * including splitting iovecs. 1827 */ 1828 bdev->max_segment_size = 2 * 512; 1829 bdev->max_num_segments = 1; 1830 g_io_done = false; 1831 1832 iov[0].iov_base = (void *)0x10000; 1833 iov[0].iov_len = 2 * 512; 1834 iov[1].iov_base = (void *)0x20000; 1835 iov[1].iov_len = 4 * 512; 1836 iov[2].iov_base = (void *)0x30000; 1837 iov[2].iov_len = 6 * 512; 1838 1839 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 1); 1840 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, 512 * 2); 1841 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1842 1843 /* Split iov[1].size to 2 iov entries then split the segments */ 1844 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 2, 1); 1845 ut_expected_io_set_iov(expected_io, 0, iov[1].iov_base, 512 * 2); 1846 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1847 1848 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 18, 2, 1); 1849 ut_expected_io_set_iov(expected_io, 0, iov[1].iov_base + 512 * 2, 512 * 2); 1850 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1851 1852 /* Split iov[2].size to 3 iov entries then split the segments */ 1853 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 20, 2, 1); 1854 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base, 512 * 2); 1855 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1856 1857 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 22, 2, 1); 1858 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 2, 512 * 2); 1859 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1860 1861 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 24, 2, 1); 1862 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 4, 512 * 2); 1863 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1864 1865 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 3, 14, 12, io_done, NULL); 1866 CU_ASSERT(rc == 0); 1867 CU_ASSERT(g_io_done == false); 1868 1869 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 6); 1870 stub_complete_io(6); 1871 CU_ASSERT(g_io_done == true); 1872 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1873 1874 /* Test multi vector command that needs to be split by strip and then needs to be 1875 * split further due to the capacity of parent IO child iovs. 1876 */ 1877 bdev->max_segment_size = 512; 1878 bdev->max_num_segments = 1; 1879 g_io_done = false; 1880 1881 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1882 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1883 iov[i].iov_len = 512 * 2; 1884 } 1885 1886 /* Each input iov.size is split into 2 iovs, 1887 * half of the input iov can fill all child iov entries of a single IO. 1888 */ 1889 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV / 2; i++) { 1890 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 2 * i, 1, 1); 1891 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base, 512); 1892 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1893 1894 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 2 * i + 1, 1, 1); 1895 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base + 512, 512); 1896 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1897 } 1898 1899 /* The remaining iov is split in the second round */ 1900 for (i = BDEV_IO_NUM_CHILD_IOV / 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1901 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 2, 1, 1); 1902 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base, 512); 1903 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1904 1905 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 2 + 1, 1, 1); 1906 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base + 512, 512); 1907 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1908 } 1909 1910 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV, 0, 1911 BDEV_IO_NUM_CHILD_IOV * 2, io_done, NULL); 1912 CU_ASSERT(rc == 0); 1913 CU_ASSERT(g_io_done == false); 1914 1915 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == BDEV_IO_NUM_CHILD_IOV); 1916 stub_complete_io(BDEV_IO_NUM_CHILD_IOV); 1917 CU_ASSERT(g_io_done == false); 1918 1919 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == BDEV_IO_NUM_CHILD_IOV); 1920 stub_complete_io(BDEV_IO_NUM_CHILD_IOV); 1921 CU_ASSERT(g_io_done == true); 1922 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1923 1924 /* A wrong case, a child IO that is divided does 1925 * not meet the principle of multiples of block size, 1926 * and exits with error 1927 */ 1928 bdev->max_segment_size = 512; 1929 bdev->max_num_segments = 1; 1930 g_io_done = false; 1931 1932 iov[0].iov_base = (void *)0x10000; 1933 iov[0].iov_len = 512 + 256; 1934 iov[1].iov_base = (void *)0x20000; 1935 iov[1].iov_len = 256; 1936 1937 /* iov[0] is split to 512 and 256. 1938 * 256 is less than a block size, and it is found 1939 * in the next round of split that it is the first child IO smaller than 1940 * the block size, so the error exit 1941 */ 1942 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 1, 1); 1943 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, 512); 1944 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1945 1946 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, 2, 0, 2, io_done, NULL); 1947 CU_ASSERT(rc == 0); 1948 CU_ASSERT(g_io_done == false); 1949 1950 /* First child IO is OK */ 1951 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1952 stub_complete_io(1); 1953 CU_ASSERT(g_io_done == true); 1954 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1955 1956 /* error exit */ 1957 stub_complete_io(1); 1958 CU_ASSERT(g_io_done == true); 1959 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 1960 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1961 1962 /* Test multi vector command that needs to be split by strip and then needs to be 1963 * split further due to the capacity of child iovs. 1964 * 1965 * In this case, the last two iovs need to be split, but it will exceed the capacity 1966 * of child iovs, so it needs to wait until the first batch completed. 1967 */ 1968 bdev->max_segment_size = 512; 1969 bdev->max_num_segments = BDEV_IO_NUM_CHILD_IOV; 1970 g_io_done = false; 1971 1972 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1973 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1974 iov[i].iov_len = 512; 1975 } 1976 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1977 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1978 iov[i].iov_len = 512 * 2; 1979 } 1980 1981 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 1982 BDEV_IO_NUM_CHILD_IOV, BDEV_IO_NUM_CHILD_IOV); 1983 /* 0 ~ (BDEV_IO_NUM_CHILD_IOV - 2) Will not be split */ 1984 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1985 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 1986 } 1987 /* (BDEV_IO_NUM_CHILD_IOV - 2) is split */ 1988 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, 512); 1989 ut_expected_io_set_iov(expected_io, i + 1, iov[i].iov_base + 512, 512); 1990 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1991 1992 /* Child iov entries exceed the max num of parent IO so split it in next round */ 1993 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 2, 2); 1994 ut_expected_io_set_iov(expected_io, 0, iov[i + 1].iov_base, 512); 1995 ut_expected_io_set_iov(expected_io, 1, iov[i + 1].iov_base + 512, 512); 1996 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1997 1998 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV, 0, 1999 BDEV_IO_NUM_CHILD_IOV + 2, io_done, NULL); 2000 CU_ASSERT(rc == 0); 2001 CU_ASSERT(g_io_done == false); 2002 2003 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2004 stub_complete_io(1); 2005 CU_ASSERT(g_io_done == false); 2006 2007 /* Next round */ 2008 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2009 stub_complete_io(1); 2010 CU_ASSERT(g_io_done == true); 2011 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2012 2013 /* This case is similar to the previous one, but the io composed of 2014 * the last few entries of child iov is not enough for a blocklen, so they 2015 * cannot be put into this IO, but wait until the next time. 2016 */ 2017 bdev->max_segment_size = 512; 2018 bdev->max_num_segments = BDEV_IO_NUM_CHILD_IOV; 2019 g_io_done = false; 2020 2021 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2022 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2023 iov[i].iov_len = 512; 2024 } 2025 2026 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV + 2; i++) { 2027 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2028 iov[i].iov_len = 128; 2029 } 2030 2031 /* First child iovcnt is't BDEV_IO_NUM_CHILD_IOV but BDEV_IO_NUM_CHILD_IOV - 2. 2032 * Because the left 2 iov is not enough for a blocklen. 2033 */ 2034 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 2035 BDEV_IO_NUM_CHILD_IOV - 2, BDEV_IO_NUM_CHILD_IOV - 2); 2036 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2037 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 2038 } 2039 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2040 2041 /* The second child io waits until the end of the first child io before executing. 2042 * Because the iovcnt of the two IOs exceeds the child iovcnt of the parent IO. 2043 * BDEV_IO_NUM_CHILD_IOV - 2 to BDEV_IO_NUM_CHILD_IOV + 2 2044 */ 2045 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV - 2, 2046 1, 4); 2047 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base, iov[i].iov_len); 2048 ut_expected_io_set_iov(expected_io, 1, iov[i + 1].iov_base, iov[i + 1].iov_len); 2049 ut_expected_io_set_iov(expected_io, 2, iov[i + 2].iov_base, iov[i + 2].iov_len); 2050 ut_expected_io_set_iov(expected_io, 3, iov[i + 3].iov_base, iov[i + 3].iov_len); 2051 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2052 2053 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 2, 0, 2054 BDEV_IO_NUM_CHILD_IOV - 1, io_done, NULL); 2055 CU_ASSERT(rc == 0); 2056 CU_ASSERT(g_io_done == false); 2057 2058 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2059 stub_complete_io(1); 2060 CU_ASSERT(g_io_done == false); 2061 2062 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2063 stub_complete_io(1); 2064 CU_ASSERT(g_io_done == true); 2065 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2066 2067 /* A very complicated case. Each sg entry exceeds max_segment_size and 2068 * needs to be split. At the same time, child io must be a multiple of blocklen. 2069 * At the same time, child iovcnt exceeds parent iovcnt. 2070 */ 2071 bdev->max_segment_size = 512 + 128; 2072 bdev->max_num_segments = 3; 2073 g_io_done = false; 2074 2075 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2076 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2077 iov[i].iov_len = 512 + 256; 2078 } 2079 2080 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV + 2; i++) { 2081 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2082 iov[i].iov_len = 512 + 128; 2083 } 2084 2085 /* Child IOs use 9 entries per for() round and 3 * 9 = 27 child iov entries. 2086 * Consume 4 parent IO iov entries per for() round and 6 block size. 2087 * Generate 9 child IOs. 2088 */ 2089 for (i = 0; i < 3; i++) { 2090 uint32_t j = i * 4; 2091 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 6, 2, 3); 2092 ut_expected_io_set_iov(expected_io, 0, iov[j].iov_base, 640); 2093 ut_expected_io_set_iov(expected_io, 1, iov[j].iov_base + 640, 128); 2094 ut_expected_io_set_iov(expected_io, 2, iov[j + 1].iov_base, 256); 2095 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2096 2097 /* Child io must be a multiple of blocklen 2098 * iov[j + 2] must be split. If the third entry is also added, 2099 * the multiple of blocklen cannot be guaranteed. But it still 2100 * occupies one iov entry of the parent child iov. 2101 */ 2102 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 6 + 2, 2, 2); 2103 ut_expected_io_set_iov(expected_io, 0, iov[j + 1].iov_base + 256, 512); 2104 ut_expected_io_set_iov(expected_io, 1, iov[j + 2].iov_base, 512); 2105 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2106 2107 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 6 + 4, 2, 3); 2108 ut_expected_io_set_iov(expected_io, 0, iov[j + 2].iov_base + 512, 256); 2109 ut_expected_io_set_iov(expected_io, 1, iov[j + 3].iov_base, 640); 2110 ut_expected_io_set_iov(expected_io, 2, iov[j + 3].iov_base + 640, 128); 2111 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2112 } 2113 2114 /* Child iov position at 27, the 10th child IO 2115 * iov entry index is 3 * 4 and offset is 3 * 6 2116 */ 2117 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 18, 2, 3); 2118 ut_expected_io_set_iov(expected_io, 0, iov[12].iov_base, 640); 2119 ut_expected_io_set_iov(expected_io, 1, iov[12].iov_base + 640, 128); 2120 ut_expected_io_set_iov(expected_io, 2, iov[13].iov_base, 256); 2121 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2122 2123 /* Child iov position at 30, the 11th child IO */ 2124 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 20, 2, 2); 2125 ut_expected_io_set_iov(expected_io, 0, iov[13].iov_base + 256, 512); 2126 ut_expected_io_set_iov(expected_io, 1, iov[14].iov_base, 512); 2127 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2128 2129 /* The 2nd split round and iovpos is 0, the 12th child IO */ 2130 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 22, 2, 3); 2131 ut_expected_io_set_iov(expected_io, 0, iov[14].iov_base + 512, 256); 2132 ut_expected_io_set_iov(expected_io, 1, iov[15].iov_base, 640); 2133 ut_expected_io_set_iov(expected_io, 2, iov[15].iov_base + 640, 128); 2134 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2135 2136 /* Consume 9 child IOs and 27 child iov entries. 2137 * Consume 4 parent IO iov entries per for() round and 6 block size. 2138 * Parent IO iov index start from 16 and block offset start from 24 2139 */ 2140 for (i = 0; i < 3; i++) { 2141 uint32_t j = i * 4 + 16; 2142 uint32_t offset = i * 6 + 24; 2143 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset, 2, 3); 2144 ut_expected_io_set_iov(expected_io, 0, iov[j].iov_base, 640); 2145 ut_expected_io_set_iov(expected_io, 1, iov[j].iov_base + 640, 128); 2146 ut_expected_io_set_iov(expected_io, 2, iov[j + 1].iov_base, 256); 2147 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2148 2149 /* Child io must be a multiple of blocklen 2150 * iov[j + 2] must be split. If the third entry is also added, 2151 * the multiple of blocklen cannot be guaranteed. But it still 2152 * occupies one iov entry of the parent child iov. 2153 */ 2154 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset + 2, 2, 2); 2155 ut_expected_io_set_iov(expected_io, 0, iov[j + 1].iov_base + 256, 512); 2156 ut_expected_io_set_iov(expected_io, 1, iov[j + 2].iov_base, 512); 2157 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2158 2159 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset + 4, 2, 3); 2160 ut_expected_io_set_iov(expected_io, 0, iov[j + 2].iov_base + 512, 256); 2161 ut_expected_io_set_iov(expected_io, 1, iov[j + 3].iov_base, 640); 2162 ut_expected_io_set_iov(expected_io, 2, iov[j + 3].iov_base + 640, 128); 2163 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2164 } 2165 2166 /* The 22th child IO, child iov position at 30 */ 2167 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 42, 1, 1); 2168 ut_expected_io_set_iov(expected_io, 0, iov[28].iov_base, 512); 2169 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2170 2171 /* The third round */ 2172 /* Here is the 23nd child IO and child iovpos is 0 */ 2173 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 43, 2, 3); 2174 ut_expected_io_set_iov(expected_io, 0, iov[28].iov_base + 512, 256); 2175 ut_expected_io_set_iov(expected_io, 1, iov[29].iov_base, 640); 2176 ut_expected_io_set_iov(expected_io, 2, iov[29].iov_base + 640, 128); 2177 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2178 2179 /* The 24th child IO */ 2180 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 45, 3, 3); 2181 ut_expected_io_set_iov(expected_io, 0, iov[30].iov_base, 640); 2182 ut_expected_io_set_iov(expected_io, 1, iov[31].iov_base, 640); 2183 ut_expected_io_set_iov(expected_io, 2, iov[32].iov_base, 256); 2184 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2185 2186 /* The 25th child IO */ 2187 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 48, 2, 2); 2188 ut_expected_io_set_iov(expected_io, 0, iov[32].iov_base + 256, 384); 2189 ut_expected_io_set_iov(expected_io, 1, iov[33].iov_base, 640); 2190 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2191 2192 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 2, 0, 2193 50, io_done, NULL); 2194 CU_ASSERT(rc == 0); 2195 CU_ASSERT(g_io_done == false); 2196 2197 /* Parent IO supports up to 32 child iovs, so it is calculated that 2198 * a maximum of 11 IOs can be split at a time, and the 2199 * splitting will continue after the first batch is over. 2200 */ 2201 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 11); 2202 stub_complete_io(11); 2203 CU_ASSERT(g_io_done == false); 2204 2205 /* The 2nd round */ 2206 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 11); 2207 stub_complete_io(11); 2208 CU_ASSERT(g_io_done == false); 2209 2210 /* The last round */ 2211 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 2212 stub_complete_io(3); 2213 CU_ASSERT(g_io_done == true); 2214 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2215 2216 /* Test an WRITE_ZEROES. This should also not be split. */ 2217 bdev->max_segment_size = 512; 2218 bdev->max_num_segments = 1; 2219 g_io_done = false; 2220 2221 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, 9, 36, 0); 2222 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2223 2224 rc = spdk_bdev_write_zeroes_blocks(desc, io_ch, 9, 36, io_done, NULL); 2225 CU_ASSERT(rc == 0); 2226 CU_ASSERT(g_io_done == false); 2227 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2228 stub_complete_io(1); 2229 CU_ASSERT(g_io_done == true); 2230 2231 /* Test an UNMAP. This should also not be split. */ 2232 g_io_done = false; 2233 2234 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, 15, 4, 0); 2235 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2236 2237 rc = spdk_bdev_unmap_blocks(desc, io_ch, 15, 4, io_done, NULL); 2238 CU_ASSERT(rc == 0); 2239 CU_ASSERT(g_io_done == false); 2240 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2241 stub_complete_io(1); 2242 CU_ASSERT(g_io_done == true); 2243 2244 /* Test a FLUSH. This should also not be split. */ 2245 g_io_done = false; 2246 2247 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_FLUSH, 15, 4, 0); 2248 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2249 2250 rc = spdk_bdev_flush_blocks(desc, io_ch, 15, 2, io_done, NULL); 2251 CU_ASSERT(rc == 0); 2252 CU_ASSERT(g_io_done == false); 2253 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2254 stub_complete_io(1); 2255 CU_ASSERT(g_io_done == true); 2256 2257 spdk_put_io_channel(io_ch); 2258 spdk_bdev_close(desc); 2259 free_bdev(bdev); 2260 spdk_bdev_finish(bdev_fini_cb, NULL); 2261 poll_threads(); 2262 } 2263 2264 static void 2265 bdev_io_mix_split_test(void) 2266 { 2267 struct spdk_bdev *bdev; 2268 struct spdk_bdev_desc *desc = NULL; 2269 struct spdk_io_channel *io_ch; 2270 struct spdk_bdev_opts bdev_opts = { 2271 .bdev_io_pool_size = 512, 2272 .bdev_io_cache_size = 64, 2273 }; 2274 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 2275 struct ut_expected_io *expected_io; 2276 uint64_t i; 2277 int rc; 2278 2279 rc = spdk_bdev_set_opts(&bdev_opts); 2280 CU_ASSERT(rc == 0); 2281 spdk_bdev_initialize(bdev_init_cb, NULL); 2282 2283 bdev = allocate_bdev("bdev0"); 2284 2285 rc = spdk_bdev_open_ext(bdev->name, true, bdev_ut_event_cb, NULL, &desc); 2286 CU_ASSERT(rc == 0); 2287 SPDK_CU_ASSERT_FATAL(desc != NULL); 2288 io_ch = spdk_bdev_get_io_channel(desc); 2289 CU_ASSERT(io_ch != NULL); 2290 2291 /* First case optimal_io_boundary == max_segment_size * max_num_segments */ 2292 bdev->split_on_optimal_io_boundary = true; 2293 bdev->optimal_io_boundary = 16; 2294 2295 bdev->max_segment_size = 512; 2296 bdev->max_num_segments = 16; 2297 g_io_done = false; 2298 2299 /* IO crossing the IO boundary requires split 2300 * Total 2 child IOs. 2301 */ 2302 2303 /* The 1st child IO split the segment_size to multiple segment entry */ 2304 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 2); 2305 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512); 2306 ut_expected_io_set_iov(expected_io, 1, (void *)(0xF000 + 512), 512); 2307 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2308 2309 /* The 2nd child IO split the segment_size to multiple segment entry */ 2310 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 2, 2); 2311 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 512); 2312 ut_expected_io_set_iov(expected_io, 1, (void *)(0xF000 + 3 * 512), 512); 2313 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2314 2315 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 4, io_done, NULL); 2316 CU_ASSERT(rc == 0); 2317 CU_ASSERT(g_io_done == false); 2318 2319 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2320 stub_complete_io(2); 2321 CU_ASSERT(g_io_done == true); 2322 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2323 2324 /* Second case optimal_io_boundary > max_segment_size * max_num_segments */ 2325 bdev->max_segment_size = 15 * 512; 2326 bdev->max_num_segments = 1; 2327 g_io_done = false; 2328 2329 /* IO crossing the IO boundary requires split. 2330 * The 1st child IO segment size exceeds the max_segment_size, 2331 * So 1st child IO will be splitted to multiple segment entry. 2332 * Then it split to 2 child IOs because of the max_num_segments. 2333 * Total 3 child IOs. 2334 */ 2335 2336 /* The first 2 IOs are in an IO boundary. 2337 * Because the optimal_io_boundary > max_segment_size * max_num_segments 2338 * So it split to the first 2 IOs. 2339 */ 2340 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 15, 1); 2341 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512 * 15); 2342 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2343 2344 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 15, 1, 1); 2345 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 512 * 15), 512); 2346 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2347 2348 /* The 3rd Child IO is because of the io boundary */ 2349 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 2, 1); 2350 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 512 * 16), 512 * 2); 2351 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2352 2353 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 0, 18, io_done, NULL); 2354 CU_ASSERT(rc == 0); 2355 CU_ASSERT(g_io_done == false); 2356 2357 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 2358 stub_complete_io(3); 2359 CU_ASSERT(g_io_done == true); 2360 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2361 2362 /* Third case optimal_io_boundary < max_segment_size * max_num_segments */ 2363 bdev->max_segment_size = 17 * 512; 2364 bdev->max_num_segments = 1; 2365 g_io_done = false; 2366 2367 /* IO crossing the IO boundary requires split. 2368 * Child IO does not split. 2369 * Total 2 child IOs. 2370 */ 2371 2372 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 16, 1); 2373 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512 * 16); 2374 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2375 2376 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 2, 1); 2377 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 512 * 16), 512 * 2); 2378 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2379 2380 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 0, 18, io_done, NULL); 2381 CU_ASSERT(rc == 0); 2382 CU_ASSERT(g_io_done == false); 2383 2384 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2385 stub_complete_io(2); 2386 CU_ASSERT(g_io_done == true); 2387 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2388 2389 /* Now set up a more complex, multi-vector command that needs to be split, 2390 * including splitting iovecs. 2391 * optimal_io_boundary < max_segment_size * max_num_segments 2392 */ 2393 bdev->max_segment_size = 3 * 512; 2394 bdev->max_num_segments = 6; 2395 g_io_done = false; 2396 2397 iov[0].iov_base = (void *)0x10000; 2398 iov[0].iov_len = 4 * 512; 2399 iov[1].iov_base = (void *)0x20000; 2400 iov[1].iov_len = 4 * 512; 2401 iov[2].iov_base = (void *)0x30000; 2402 iov[2].iov_len = 10 * 512; 2403 2404 /* IO crossing the IO boundary requires split. 2405 * The 1st child IO segment size exceeds the max_segment_size and after 2406 * splitting segment_size, the num_segments exceeds max_num_segments. 2407 * So 1st child IO will be splitted to 2 child IOs. 2408 * Total 3 child IOs. 2409 */ 2410 2411 /* The first 2 IOs are in an IO boundary. 2412 * After splitting segmemt size the segment num exceeds. 2413 * So it splits to 2 child IOs. 2414 */ 2415 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 0, 14, 6); 2416 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, 512 * 3); 2417 ut_expected_io_set_iov(expected_io, 1, iov[0].iov_base + 512 * 3, 512); 2418 ut_expected_io_set_iov(expected_io, 2, iov[1].iov_base, 512 * 3); 2419 ut_expected_io_set_iov(expected_io, 3, iov[1].iov_base + 512 * 3, 512); 2420 ut_expected_io_set_iov(expected_io, 4, iov[2].iov_base, 512 * 3); 2421 ut_expected_io_set_iov(expected_io, 5, iov[2].iov_base + 512 * 3, 512 * 3); 2422 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2423 2424 /* The 2nd child IO has the left segment entry */ 2425 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 1); 2426 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 6, 512 * 2); 2427 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2428 2429 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 2, 1); 2430 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 8, 512 * 2); 2431 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2432 2433 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 3, 0, 18, io_done, NULL); 2434 CU_ASSERT(rc == 0); 2435 CU_ASSERT(g_io_done == false); 2436 2437 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 2438 stub_complete_io(3); 2439 CU_ASSERT(g_io_done == true); 2440 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2441 2442 /* A very complicated case. Each sg entry exceeds max_segment_size 2443 * and split on io boundary. 2444 * optimal_io_boundary < max_segment_size * max_num_segments 2445 */ 2446 bdev->max_segment_size = 3 * 512; 2447 bdev->max_num_segments = BDEV_IO_NUM_CHILD_IOV; 2448 g_io_done = false; 2449 2450 for (i = 0; i < 20; i++) { 2451 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2452 iov[i].iov_len = 512 * 4; 2453 } 2454 2455 /* IO crossing the IO boundary requires split. 2456 * 80 block length can split 5 child IOs base on offset and IO boundary. 2457 * Each iov entry needs to be splitted to 2 entries because of max_segment_size 2458 * Total 5 child IOs. 2459 */ 2460 2461 /* 4 iov entries are in an IO boundary and each iov entry splits to 2. 2462 * So each child IO occupies 8 child iov entries. 2463 */ 2464 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 0, 16, 8); 2465 for (i = 0; i < 4; i++) { 2466 int iovcnt = i * 2; 2467 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2468 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2469 } 2470 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2471 2472 /* 2nd child IO and total 16 child iov entries of parent IO */ 2473 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 16, 8); 2474 for (i = 4; i < 8; i++) { 2475 int iovcnt = (i - 4) * 2; 2476 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2477 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2478 } 2479 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2480 2481 /* 3rd child IO and total 24 child iov entries of parent IO */ 2482 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 16, 8); 2483 for (i = 8; i < 12; i++) { 2484 int iovcnt = (i - 8) * 2; 2485 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2486 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2487 } 2488 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2489 2490 /* 4th child IO and total 32 child iov entries of parent IO */ 2491 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 48, 16, 8); 2492 for (i = 12; i < 16; i++) { 2493 int iovcnt = (i - 12) * 2; 2494 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2495 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2496 } 2497 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2498 2499 /* 5th child IO and because of the child iov entry it should be splitted 2500 * in next round. 2501 */ 2502 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 64, 16, 8); 2503 for (i = 16; i < 20; i++) { 2504 int iovcnt = (i - 16) * 2; 2505 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2506 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2507 } 2508 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2509 2510 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 20, 0, 80, io_done, NULL); 2511 CU_ASSERT(rc == 0); 2512 CU_ASSERT(g_io_done == false); 2513 2514 /* First split round */ 2515 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 2516 stub_complete_io(4); 2517 CU_ASSERT(g_io_done == false); 2518 2519 /* Second split round */ 2520 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2521 stub_complete_io(1); 2522 CU_ASSERT(g_io_done == true); 2523 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2524 2525 spdk_put_io_channel(io_ch); 2526 spdk_bdev_close(desc); 2527 free_bdev(bdev); 2528 spdk_bdev_finish(bdev_fini_cb, NULL); 2529 poll_threads(); 2530 } 2531 2532 static void 2533 bdev_io_split_with_io_wait(void) 2534 { 2535 struct spdk_bdev *bdev; 2536 struct spdk_bdev_desc *desc = NULL; 2537 struct spdk_io_channel *io_ch; 2538 struct spdk_bdev_channel *channel; 2539 struct spdk_bdev_mgmt_channel *mgmt_ch; 2540 struct spdk_bdev_opts bdev_opts = { 2541 .bdev_io_pool_size = 2, 2542 .bdev_io_cache_size = 1, 2543 }; 2544 struct iovec iov[3]; 2545 struct ut_expected_io *expected_io; 2546 int rc; 2547 2548 rc = spdk_bdev_set_opts(&bdev_opts); 2549 CU_ASSERT(rc == 0); 2550 spdk_bdev_initialize(bdev_init_cb, NULL); 2551 2552 bdev = allocate_bdev("bdev0"); 2553 2554 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 2555 CU_ASSERT(rc == 0); 2556 CU_ASSERT(desc != NULL); 2557 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 2558 io_ch = spdk_bdev_get_io_channel(desc); 2559 CU_ASSERT(io_ch != NULL); 2560 channel = spdk_io_channel_get_ctx(io_ch); 2561 mgmt_ch = channel->shared_resource->mgmt_ch; 2562 2563 bdev->optimal_io_boundary = 16; 2564 bdev->split_on_optimal_io_boundary = true; 2565 2566 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 2567 CU_ASSERT(rc == 0); 2568 2569 /* Now test that a single-vector command is split correctly. 2570 * Offset 14, length 8, payload 0xF000 2571 * Child - Offset 14, length 2, payload 0xF000 2572 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 2573 * 2574 * Set up the expected values before calling spdk_bdev_read_blocks 2575 */ 2576 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 1); 2577 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 2 * 512); 2578 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2579 2580 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 6, 1); 2581 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 6 * 512); 2582 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2583 2584 /* The following children will be submitted sequentially due to the capacity of 2585 * spdk_bdev_io. 2586 */ 2587 2588 /* The first child I/O will be queued to wait until an spdk_bdev_io becomes available */ 2589 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL); 2590 CU_ASSERT(rc == 0); 2591 CU_ASSERT(!TAILQ_EMPTY(&mgmt_ch->io_wait_queue)); 2592 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2593 2594 /* Completing the first read I/O will submit the first child */ 2595 stub_complete_io(1); 2596 CU_ASSERT(TAILQ_EMPTY(&mgmt_ch->io_wait_queue)); 2597 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2598 2599 /* Completing the first child will submit the second child */ 2600 stub_complete_io(1); 2601 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2602 2603 /* Complete the second child I/O. This should result in our callback getting 2604 * invoked since the parent I/O is now complete. 2605 */ 2606 stub_complete_io(1); 2607 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2608 2609 /* Now set up a more complex, multi-vector command that needs to be split, 2610 * including splitting iovecs. 2611 */ 2612 iov[0].iov_base = (void *)0x10000; 2613 iov[0].iov_len = 512; 2614 iov[1].iov_base = (void *)0x20000; 2615 iov[1].iov_len = 20 * 512; 2616 iov[2].iov_base = (void *)0x30000; 2617 iov[2].iov_len = 11 * 512; 2618 2619 g_io_done = false; 2620 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 2); 2621 ut_expected_io_set_iov(expected_io, 0, (void *)0x10000, 512); 2622 ut_expected_io_set_iov(expected_io, 1, (void *)0x20000, 512); 2623 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2624 2625 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 16, 1); 2626 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 512), 16 * 512); 2627 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2628 2629 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 14, 2); 2630 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 17 * 512), 3 * 512); 2631 ut_expected_io_set_iov(expected_io, 1, (void *)0x30000, 11 * 512); 2632 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2633 2634 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 3, 14, 32, io_done, NULL); 2635 CU_ASSERT(rc == 0); 2636 CU_ASSERT(g_io_done == false); 2637 2638 /* The following children will be submitted sequentially due to the capacity of 2639 * spdk_bdev_io. 2640 */ 2641 2642 /* Completing the first child will submit the second child */ 2643 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2644 stub_complete_io(1); 2645 CU_ASSERT(g_io_done == false); 2646 2647 /* Completing the second child will submit the third child */ 2648 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2649 stub_complete_io(1); 2650 CU_ASSERT(g_io_done == false); 2651 2652 /* Completing the third child will result in our callback getting invoked 2653 * since the parent I/O is now complete. 2654 */ 2655 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2656 stub_complete_io(1); 2657 CU_ASSERT(g_io_done == true); 2658 2659 CU_ASSERT(TAILQ_EMPTY(&g_bdev_ut_channel->expected_io)); 2660 2661 spdk_put_io_channel(io_ch); 2662 spdk_bdev_close(desc); 2663 free_bdev(bdev); 2664 spdk_bdev_finish(bdev_fini_cb, NULL); 2665 poll_threads(); 2666 } 2667 2668 static void 2669 bdev_io_alignment(void) 2670 { 2671 struct spdk_bdev *bdev; 2672 struct spdk_bdev_desc *desc = NULL; 2673 struct spdk_io_channel *io_ch; 2674 struct spdk_bdev_opts bdev_opts = { 2675 .bdev_io_pool_size = 20, 2676 .bdev_io_cache_size = 2, 2677 }; 2678 int rc; 2679 void *buf = NULL; 2680 struct iovec iovs[2]; 2681 int iovcnt; 2682 uint64_t alignment; 2683 2684 rc = spdk_bdev_set_opts(&bdev_opts); 2685 CU_ASSERT(rc == 0); 2686 spdk_bdev_initialize(bdev_init_cb, NULL); 2687 2688 fn_table.submit_request = stub_submit_request_get_buf; 2689 bdev = allocate_bdev("bdev0"); 2690 2691 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 2692 CU_ASSERT(rc == 0); 2693 CU_ASSERT(desc != NULL); 2694 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 2695 io_ch = spdk_bdev_get_io_channel(desc); 2696 CU_ASSERT(io_ch != NULL); 2697 2698 /* Create aligned buffer */ 2699 rc = posix_memalign(&buf, 4096, 8192); 2700 SPDK_CU_ASSERT_FATAL(rc == 0); 2701 2702 /* Pass aligned single buffer with no alignment required */ 2703 alignment = 1; 2704 bdev->required_alignment = spdk_u32log2(alignment); 2705 2706 rc = spdk_bdev_write_blocks(desc, io_ch, buf, 0, 1, io_done, NULL); 2707 CU_ASSERT(rc == 0); 2708 stub_complete_io(1); 2709 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2710 alignment)); 2711 2712 rc = spdk_bdev_read_blocks(desc, io_ch, buf, 0, 1, io_done, NULL); 2713 CU_ASSERT(rc == 0); 2714 stub_complete_io(1); 2715 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2716 alignment)); 2717 2718 /* Pass unaligned single buffer with no alignment required */ 2719 alignment = 1; 2720 bdev->required_alignment = spdk_u32log2(alignment); 2721 2722 rc = spdk_bdev_write_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 2723 CU_ASSERT(rc == 0); 2724 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2725 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == buf + 4); 2726 stub_complete_io(1); 2727 2728 rc = spdk_bdev_read_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 2729 CU_ASSERT(rc == 0); 2730 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2731 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == buf + 4); 2732 stub_complete_io(1); 2733 2734 /* Pass unaligned single buffer with 512 alignment required */ 2735 alignment = 512; 2736 bdev->required_alignment = spdk_u32log2(alignment); 2737 2738 rc = spdk_bdev_write_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 2739 CU_ASSERT(rc == 0); 2740 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 2741 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2742 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2743 alignment)); 2744 stub_complete_io(1); 2745 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2746 2747 rc = spdk_bdev_read_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 2748 CU_ASSERT(rc == 0); 2749 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 2750 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2751 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2752 alignment)); 2753 stub_complete_io(1); 2754 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2755 2756 /* Pass unaligned single buffer with 4096 alignment required */ 2757 alignment = 4096; 2758 bdev->required_alignment = spdk_u32log2(alignment); 2759 2760 rc = spdk_bdev_write_blocks(desc, io_ch, buf + 8, 0, 1, io_done, NULL); 2761 CU_ASSERT(rc == 0); 2762 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 2763 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2764 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2765 alignment)); 2766 stub_complete_io(1); 2767 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2768 2769 rc = spdk_bdev_read_blocks(desc, io_ch, buf + 8, 0, 1, io_done, NULL); 2770 CU_ASSERT(rc == 0); 2771 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 2772 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2773 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2774 alignment)); 2775 stub_complete_io(1); 2776 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2777 2778 /* Pass aligned iovs with no alignment required */ 2779 alignment = 1; 2780 bdev->required_alignment = spdk_u32log2(alignment); 2781 2782 iovcnt = 1; 2783 iovs[0].iov_base = buf; 2784 iovs[0].iov_len = 512; 2785 2786 rc = spdk_bdev_writev(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2787 CU_ASSERT(rc == 0); 2788 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2789 stub_complete_io(1); 2790 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 2791 2792 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2793 CU_ASSERT(rc == 0); 2794 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2795 stub_complete_io(1); 2796 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 2797 2798 /* Pass unaligned iovs with no alignment required */ 2799 alignment = 1; 2800 bdev->required_alignment = spdk_u32log2(alignment); 2801 2802 iovcnt = 2; 2803 iovs[0].iov_base = buf + 16; 2804 iovs[0].iov_len = 256; 2805 iovs[1].iov_base = buf + 16 + 256 + 32; 2806 iovs[1].iov_len = 256; 2807 2808 rc = spdk_bdev_writev(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2809 CU_ASSERT(rc == 0); 2810 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2811 stub_complete_io(1); 2812 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 2813 2814 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2815 CU_ASSERT(rc == 0); 2816 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2817 stub_complete_io(1); 2818 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 2819 2820 /* Pass unaligned iov with 2048 alignment required */ 2821 alignment = 2048; 2822 bdev->required_alignment = spdk_u32log2(alignment); 2823 2824 iovcnt = 2; 2825 iovs[0].iov_base = buf + 16; 2826 iovs[0].iov_len = 256; 2827 iovs[1].iov_base = buf + 16 + 256 + 32; 2828 iovs[1].iov_len = 256; 2829 2830 rc = spdk_bdev_writev(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2831 CU_ASSERT(rc == 0); 2832 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == iovcnt); 2833 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2834 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2835 alignment)); 2836 stub_complete_io(1); 2837 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2838 2839 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2840 CU_ASSERT(rc == 0); 2841 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == iovcnt); 2842 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2843 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2844 alignment)); 2845 stub_complete_io(1); 2846 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2847 2848 /* Pass iov without allocated buffer without alignment required */ 2849 alignment = 1; 2850 bdev->required_alignment = spdk_u32log2(alignment); 2851 2852 iovcnt = 1; 2853 iovs[0].iov_base = NULL; 2854 iovs[0].iov_len = 0; 2855 2856 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2857 CU_ASSERT(rc == 0); 2858 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2859 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2860 alignment)); 2861 stub_complete_io(1); 2862 2863 /* Pass iov without allocated buffer with 1024 alignment required */ 2864 alignment = 1024; 2865 bdev->required_alignment = spdk_u32log2(alignment); 2866 2867 iovcnt = 1; 2868 iovs[0].iov_base = NULL; 2869 iovs[0].iov_len = 0; 2870 2871 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2872 CU_ASSERT(rc == 0); 2873 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2874 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2875 alignment)); 2876 stub_complete_io(1); 2877 2878 spdk_put_io_channel(io_ch); 2879 spdk_bdev_close(desc); 2880 free_bdev(bdev); 2881 fn_table.submit_request = stub_submit_request; 2882 spdk_bdev_finish(bdev_fini_cb, NULL); 2883 poll_threads(); 2884 2885 free(buf); 2886 } 2887 2888 static void 2889 bdev_io_alignment_with_boundary(void) 2890 { 2891 struct spdk_bdev *bdev; 2892 struct spdk_bdev_desc *desc = NULL; 2893 struct spdk_io_channel *io_ch; 2894 struct spdk_bdev_opts bdev_opts = { 2895 .bdev_io_pool_size = 20, 2896 .bdev_io_cache_size = 2, 2897 }; 2898 int rc; 2899 void *buf = NULL; 2900 struct iovec iovs[2]; 2901 int iovcnt; 2902 uint64_t alignment; 2903 2904 rc = spdk_bdev_set_opts(&bdev_opts); 2905 CU_ASSERT(rc == 0); 2906 spdk_bdev_initialize(bdev_init_cb, NULL); 2907 2908 fn_table.submit_request = stub_submit_request_get_buf; 2909 bdev = allocate_bdev("bdev0"); 2910 2911 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 2912 CU_ASSERT(rc == 0); 2913 CU_ASSERT(desc != NULL); 2914 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 2915 io_ch = spdk_bdev_get_io_channel(desc); 2916 CU_ASSERT(io_ch != NULL); 2917 2918 /* Create aligned buffer */ 2919 rc = posix_memalign(&buf, 4096, 131072); 2920 SPDK_CU_ASSERT_FATAL(rc == 0); 2921 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 2922 2923 /* 512 * 3 with 2 IO boundary, allocate small data buffer from bdev layer */ 2924 alignment = 512; 2925 bdev->required_alignment = spdk_u32log2(alignment); 2926 bdev->optimal_io_boundary = 2; 2927 bdev->split_on_optimal_io_boundary = true; 2928 2929 iovcnt = 1; 2930 iovs[0].iov_base = NULL; 2931 iovs[0].iov_len = 512 * 3; 2932 2933 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 3, io_done, NULL); 2934 CU_ASSERT(rc == 0); 2935 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2936 stub_complete_io(2); 2937 2938 /* 8KiB with 16 IO boundary, allocate large data buffer from bdev layer */ 2939 alignment = 512; 2940 bdev->required_alignment = spdk_u32log2(alignment); 2941 bdev->optimal_io_boundary = 16; 2942 bdev->split_on_optimal_io_boundary = true; 2943 2944 iovcnt = 1; 2945 iovs[0].iov_base = NULL; 2946 iovs[0].iov_len = 512 * 16; 2947 2948 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 16, io_done, NULL); 2949 CU_ASSERT(rc == 0); 2950 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2951 stub_complete_io(2); 2952 2953 /* 512 * 160 with 128 IO boundary, 63.5KiB + 16.5KiB for the two children requests */ 2954 alignment = 512; 2955 bdev->required_alignment = spdk_u32log2(alignment); 2956 bdev->optimal_io_boundary = 128; 2957 bdev->split_on_optimal_io_boundary = true; 2958 2959 iovcnt = 1; 2960 iovs[0].iov_base = buf + 16; 2961 iovs[0].iov_len = 512 * 160; 2962 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 160, io_done, NULL); 2963 CU_ASSERT(rc == 0); 2964 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2965 stub_complete_io(2); 2966 2967 /* 512 * 3 with 2 IO boundary */ 2968 alignment = 512; 2969 bdev->required_alignment = spdk_u32log2(alignment); 2970 bdev->optimal_io_boundary = 2; 2971 bdev->split_on_optimal_io_boundary = true; 2972 2973 iovcnt = 2; 2974 iovs[0].iov_base = buf + 16; 2975 iovs[0].iov_len = 512; 2976 iovs[1].iov_base = buf + 16 + 512 + 32; 2977 iovs[1].iov_len = 1024; 2978 2979 rc = spdk_bdev_writev_blocks(desc, io_ch, iovs, iovcnt, 1, 3, io_done, NULL); 2980 CU_ASSERT(rc == 0); 2981 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2982 stub_complete_io(2); 2983 2984 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 3, io_done, NULL); 2985 CU_ASSERT(rc == 0); 2986 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2987 stub_complete_io(2); 2988 2989 /* 512 * 64 with 32 IO boundary */ 2990 bdev->optimal_io_boundary = 32; 2991 iovcnt = 2; 2992 iovs[0].iov_base = buf + 16; 2993 iovs[0].iov_len = 16384; 2994 iovs[1].iov_base = buf + 16 + 16384 + 32; 2995 iovs[1].iov_len = 16384; 2996 2997 rc = spdk_bdev_writev_blocks(desc, io_ch, iovs, iovcnt, 1, 64, io_done, NULL); 2998 CU_ASSERT(rc == 0); 2999 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 3000 stub_complete_io(3); 3001 3002 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 64, io_done, NULL); 3003 CU_ASSERT(rc == 0); 3004 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 3005 stub_complete_io(3); 3006 3007 /* 512 * 160 with 32 IO boundary */ 3008 iovcnt = 1; 3009 iovs[0].iov_base = buf + 16; 3010 iovs[0].iov_len = 16384 + 65536; 3011 3012 rc = spdk_bdev_writev_blocks(desc, io_ch, iovs, iovcnt, 1, 160, io_done, NULL); 3013 CU_ASSERT(rc == 0); 3014 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 6); 3015 stub_complete_io(6); 3016 3017 spdk_put_io_channel(io_ch); 3018 spdk_bdev_close(desc); 3019 free_bdev(bdev); 3020 fn_table.submit_request = stub_submit_request; 3021 spdk_bdev_finish(bdev_fini_cb, NULL); 3022 poll_threads(); 3023 3024 free(buf); 3025 } 3026 3027 static void 3028 histogram_status_cb(void *cb_arg, int status) 3029 { 3030 g_status = status; 3031 } 3032 3033 static void 3034 histogram_data_cb(void *cb_arg, int status, struct spdk_histogram_data *histogram) 3035 { 3036 g_status = status; 3037 g_histogram = histogram; 3038 } 3039 3040 static void 3041 histogram_io_count(void *ctx, uint64_t start, uint64_t end, uint64_t count, 3042 uint64_t total, uint64_t so_far) 3043 { 3044 g_count += count; 3045 } 3046 3047 static void 3048 bdev_histograms(void) 3049 { 3050 struct spdk_bdev *bdev; 3051 struct spdk_bdev_desc *desc = NULL; 3052 struct spdk_io_channel *ch; 3053 struct spdk_histogram_data *histogram; 3054 uint8_t buf[4096]; 3055 int rc; 3056 3057 spdk_bdev_initialize(bdev_init_cb, NULL); 3058 3059 bdev = allocate_bdev("bdev"); 3060 3061 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3062 CU_ASSERT(rc == 0); 3063 CU_ASSERT(desc != NULL); 3064 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3065 3066 ch = spdk_bdev_get_io_channel(desc); 3067 CU_ASSERT(ch != NULL); 3068 3069 /* Enable histogram */ 3070 g_status = -1; 3071 spdk_bdev_histogram_enable(bdev, histogram_status_cb, NULL, true); 3072 poll_threads(); 3073 CU_ASSERT(g_status == 0); 3074 CU_ASSERT(bdev->internal.histogram_enabled == true); 3075 3076 /* Allocate histogram */ 3077 histogram = spdk_histogram_data_alloc(); 3078 SPDK_CU_ASSERT_FATAL(histogram != NULL); 3079 3080 /* Check if histogram is zeroed */ 3081 spdk_bdev_histogram_get(bdev, histogram, histogram_data_cb, NULL); 3082 poll_threads(); 3083 CU_ASSERT(g_status == 0); 3084 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 3085 3086 g_count = 0; 3087 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 3088 3089 CU_ASSERT(g_count == 0); 3090 3091 rc = spdk_bdev_write_blocks(desc, ch, buf, 0, 1, io_done, NULL); 3092 CU_ASSERT(rc == 0); 3093 3094 spdk_delay_us(10); 3095 stub_complete_io(1); 3096 poll_threads(); 3097 3098 rc = spdk_bdev_read_blocks(desc, ch, buf, 0, 1, io_done, NULL); 3099 CU_ASSERT(rc == 0); 3100 3101 spdk_delay_us(10); 3102 stub_complete_io(1); 3103 poll_threads(); 3104 3105 /* Check if histogram gathered data from all I/O channels */ 3106 g_histogram = NULL; 3107 spdk_bdev_histogram_get(bdev, histogram, histogram_data_cb, NULL); 3108 poll_threads(); 3109 CU_ASSERT(g_status == 0); 3110 CU_ASSERT(bdev->internal.histogram_enabled == true); 3111 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 3112 3113 g_count = 0; 3114 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 3115 CU_ASSERT(g_count == 2); 3116 3117 /* Disable histogram */ 3118 spdk_bdev_histogram_enable(bdev, histogram_status_cb, NULL, false); 3119 poll_threads(); 3120 CU_ASSERT(g_status == 0); 3121 CU_ASSERT(bdev->internal.histogram_enabled == false); 3122 3123 /* Try to run histogram commands on disabled bdev */ 3124 spdk_bdev_histogram_get(bdev, histogram, histogram_data_cb, NULL); 3125 poll_threads(); 3126 CU_ASSERT(g_status == -EFAULT); 3127 3128 spdk_histogram_data_free(histogram); 3129 spdk_put_io_channel(ch); 3130 spdk_bdev_close(desc); 3131 free_bdev(bdev); 3132 spdk_bdev_finish(bdev_fini_cb, NULL); 3133 poll_threads(); 3134 } 3135 3136 static void 3137 _bdev_compare(bool emulated) 3138 { 3139 struct spdk_bdev *bdev; 3140 struct spdk_bdev_desc *desc = NULL; 3141 struct spdk_io_channel *ioch; 3142 struct ut_expected_io *expected_io; 3143 uint64_t offset, num_blocks; 3144 uint32_t num_completed; 3145 char aa_buf[512]; 3146 char bb_buf[512]; 3147 struct iovec compare_iov; 3148 uint8_t io_type; 3149 int rc; 3150 3151 if (emulated) { 3152 io_type = SPDK_BDEV_IO_TYPE_READ; 3153 } else { 3154 io_type = SPDK_BDEV_IO_TYPE_COMPARE; 3155 } 3156 3157 memset(aa_buf, 0xaa, sizeof(aa_buf)); 3158 memset(bb_buf, 0xbb, sizeof(bb_buf)); 3159 3160 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = !emulated; 3161 3162 spdk_bdev_initialize(bdev_init_cb, NULL); 3163 fn_table.submit_request = stub_submit_request_get_buf; 3164 bdev = allocate_bdev("bdev"); 3165 3166 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3167 CU_ASSERT_EQUAL(rc, 0); 3168 SPDK_CU_ASSERT_FATAL(desc != NULL); 3169 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3170 ioch = spdk_bdev_get_io_channel(desc); 3171 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3172 3173 fn_table.submit_request = stub_submit_request_get_buf; 3174 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3175 3176 offset = 50; 3177 num_blocks = 1; 3178 compare_iov.iov_base = aa_buf; 3179 compare_iov.iov_len = sizeof(aa_buf); 3180 3181 expected_io = ut_alloc_expected_io(io_type, offset, num_blocks, 0); 3182 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3183 3184 g_io_done = false; 3185 g_compare_read_buf = aa_buf; 3186 g_compare_read_buf_len = sizeof(aa_buf); 3187 rc = spdk_bdev_comparev_blocks(desc, ioch, &compare_iov, 1, offset, num_blocks, io_done, NULL); 3188 CU_ASSERT_EQUAL(rc, 0); 3189 num_completed = stub_complete_io(1); 3190 CU_ASSERT_EQUAL(num_completed, 1); 3191 CU_ASSERT(g_io_done == true); 3192 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3193 3194 expected_io = ut_alloc_expected_io(io_type, offset, num_blocks, 0); 3195 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3196 3197 g_io_done = false; 3198 g_compare_read_buf = bb_buf; 3199 g_compare_read_buf_len = sizeof(bb_buf); 3200 rc = spdk_bdev_comparev_blocks(desc, ioch, &compare_iov, 1, offset, num_blocks, io_done, NULL); 3201 CU_ASSERT_EQUAL(rc, 0); 3202 num_completed = stub_complete_io(1); 3203 CU_ASSERT_EQUAL(num_completed, 1); 3204 CU_ASSERT(g_io_done == true); 3205 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_MISCOMPARE); 3206 3207 spdk_put_io_channel(ioch); 3208 spdk_bdev_close(desc); 3209 free_bdev(bdev); 3210 fn_table.submit_request = stub_submit_request; 3211 spdk_bdev_finish(bdev_fini_cb, NULL); 3212 poll_threads(); 3213 3214 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = true; 3215 3216 g_compare_read_buf = NULL; 3217 } 3218 3219 static void 3220 bdev_compare(void) 3221 { 3222 _bdev_compare(true); 3223 _bdev_compare(false); 3224 } 3225 3226 static void 3227 bdev_compare_and_write(void) 3228 { 3229 struct spdk_bdev *bdev; 3230 struct spdk_bdev_desc *desc = NULL; 3231 struct spdk_io_channel *ioch; 3232 struct ut_expected_io *expected_io; 3233 uint64_t offset, num_blocks; 3234 uint32_t num_completed; 3235 char aa_buf[512]; 3236 char bb_buf[512]; 3237 char cc_buf[512]; 3238 char write_buf[512]; 3239 struct iovec compare_iov; 3240 struct iovec write_iov; 3241 int rc; 3242 3243 memset(aa_buf, 0xaa, sizeof(aa_buf)); 3244 memset(bb_buf, 0xbb, sizeof(bb_buf)); 3245 memset(cc_buf, 0xcc, sizeof(cc_buf)); 3246 3247 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = false; 3248 3249 spdk_bdev_initialize(bdev_init_cb, NULL); 3250 fn_table.submit_request = stub_submit_request_get_buf; 3251 bdev = allocate_bdev("bdev"); 3252 3253 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3254 CU_ASSERT_EQUAL(rc, 0); 3255 SPDK_CU_ASSERT_FATAL(desc != NULL); 3256 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3257 ioch = spdk_bdev_get_io_channel(desc); 3258 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3259 3260 fn_table.submit_request = stub_submit_request_get_buf; 3261 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3262 3263 offset = 50; 3264 num_blocks = 1; 3265 compare_iov.iov_base = aa_buf; 3266 compare_iov.iov_len = sizeof(aa_buf); 3267 write_iov.iov_base = bb_buf; 3268 write_iov.iov_len = sizeof(bb_buf); 3269 3270 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset, num_blocks, 0); 3271 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3272 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, offset, num_blocks, 0); 3273 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3274 3275 g_io_done = false; 3276 g_compare_read_buf = aa_buf; 3277 g_compare_read_buf_len = sizeof(aa_buf); 3278 memset(write_buf, 0, sizeof(write_buf)); 3279 g_compare_write_buf = write_buf; 3280 g_compare_write_buf_len = sizeof(write_buf); 3281 rc = spdk_bdev_comparev_and_writev_blocks(desc, ioch, &compare_iov, 1, &write_iov, 1, 3282 offset, num_blocks, io_done, NULL); 3283 /* Trigger range locking */ 3284 poll_threads(); 3285 CU_ASSERT_EQUAL(rc, 0); 3286 num_completed = stub_complete_io(1); 3287 CU_ASSERT_EQUAL(num_completed, 1); 3288 CU_ASSERT(g_io_done == false); 3289 num_completed = stub_complete_io(1); 3290 /* Trigger range unlocking */ 3291 poll_threads(); 3292 CU_ASSERT_EQUAL(num_completed, 1); 3293 CU_ASSERT(g_io_done == true); 3294 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3295 CU_ASSERT(memcmp(write_buf, bb_buf, sizeof(write_buf)) == 0); 3296 3297 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset, num_blocks, 0); 3298 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3299 3300 g_io_done = false; 3301 g_compare_read_buf = cc_buf; 3302 g_compare_read_buf_len = sizeof(cc_buf); 3303 memset(write_buf, 0, sizeof(write_buf)); 3304 g_compare_write_buf = write_buf; 3305 g_compare_write_buf_len = sizeof(write_buf); 3306 rc = spdk_bdev_comparev_and_writev_blocks(desc, ioch, &compare_iov, 1, &write_iov, 1, 3307 offset, num_blocks, io_done, NULL); 3308 /* Trigger range locking */ 3309 poll_threads(); 3310 CU_ASSERT_EQUAL(rc, 0); 3311 num_completed = stub_complete_io(1); 3312 /* Trigger range unlocking earlier because we expect error here */ 3313 poll_threads(); 3314 CU_ASSERT_EQUAL(num_completed, 1); 3315 CU_ASSERT(g_io_done == true); 3316 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_MISCOMPARE); 3317 num_completed = stub_complete_io(1); 3318 CU_ASSERT_EQUAL(num_completed, 0); 3319 3320 spdk_put_io_channel(ioch); 3321 spdk_bdev_close(desc); 3322 free_bdev(bdev); 3323 fn_table.submit_request = stub_submit_request; 3324 spdk_bdev_finish(bdev_fini_cb, NULL); 3325 poll_threads(); 3326 3327 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = true; 3328 3329 g_compare_read_buf = NULL; 3330 g_compare_write_buf = NULL; 3331 } 3332 3333 static void 3334 bdev_write_zeroes(void) 3335 { 3336 struct spdk_bdev *bdev; 3337 struct spdk_bdev_desc *desc = NULL; 3338 struct spdk_io_channel *ioch; 3339 struct ut_expected_io *expected_io; 3340 uint64_t offset, num_io_blocks, num_blocks; 3341 uint32_t num_completed, num_requests; 3342 int rc; 3343 3344 spdk_bdev_initialize(bdev_init_cb, NULL); 3345 bdev = allocate_bdev("bdev"); 3346 3347 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3348 CU_ASSERT_EQUAL(rc, 0); 3349 SPDK_CU_ASSERT_FATAL(desc != NULL); 3350 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3351 ioch = spdk_bdev_get_io_channel(desc); 3352 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3353 3354 fn_table.submit_request = stub_submit_request; 3355 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3356 3357 /* First test that if the bdev supports write_zeroes, the request won't be split */ 3358 bdev->md_len = 0; 3359 bdev->blocklen = 4096; 3360 num_blocks = (ZERO_BUFFER_SIZE / bdev->blocklen) * 2; 3361 3362 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, 0, num_blocks, 0); 3363 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3364 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3365 CU_ASSERT_EQUAL(rc, 0); 3366 num_completed = stub_complete_io(1); 3367 CU_ASSERT_EQUAL(num_completed, 1); 3368 3369 /* Check that if write zeroes is not supported it'll be replaced by regular writes */ 3370 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, false); 3371 num_io_blocks = ZERO_BUFFER_SIZE / bdev->blocklen; 3372 num_requests = 2; 3373 num_blocks = (ZERO_BUFFER_SIZE / bdev->blocklen) * num_requests; 3374 3375 for (offset = 0; offset < num_requests; ++offset) { 3376 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 3377 offset * num_io_blocks, num_io_blocks, 0); 3378 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3379 } 3380 3381 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3382 CU_ASSERT_EQUAL(rc, 0); 3383 num_completed = stub_complete_io(num_requests); 3384 CU_ASSERT_EQUAL(num_completed, num_requests); 3385 3386 /* Check that the splitting is correct if bdev has interleaved metadata */ 3387 bdev->md_interleave = true; 3388 bdev->md_len = 64; 3389 bdev->blocklen = 4096 + 64; 3390 num_blocks = (ZERO_BUFFER_SIZE / bdev->blocklen) * 2; 3391 3392 num_requests = offset = 0; 3393 while (offset < num_blocks) { 3394 num_io_blocks = spdk_min(ZERO_BUFFER_SIZE / bdev->blocklen, num_blocks - offset); 3395 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 3396 offset, num_io_blocks, 0); 3397 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3398 offset += num_io_blocks; 3399 num_requests++; 3400 } 3401 3402 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3403 CU_ASSERT_EQUAL(rc, 0); 3404 num_completed = stub_complete_io(num_requests); 3405 CU_ASSERT_EQUAL(num_completed, num_requests); 3406 num_completed = stub_complete_io(num_requests); 3407 assert(num_completed == 0); 3408 3409 /* Check the the same for separate metadata buffer */ 3410 bdev->md_interleave = false; 3411 bdev->md_len = 64; 3412 bdev->blocklen = 4096; 3413 3414 num_requests = offset = 0; 3415 while (offset < num_blocks) { 3416 num_io_blocks = spdk_min(ZERO_BUFFER_SIZE / (bdev->blocklen + bdev->md_len), num_blocks); 3417 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 3418 offset, num_io_blocks, 0); 3419 expected_io->md_buf = (char *)g_bdev_mgr.zero_buffer + num_io_blocks * bdev->blocklen; 3420 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3421 offset += num_io_blocks; 3422 num_requests++; 3423 } 3424 3425 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3426 CU_ASSERT_EQUAL(rc, 0); 3427 num_completed = stub_complete_io(num_requests); 3428 CU_ASSERT_EQUAL(num_completed, num_requests); 3429 3430 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, true); 3431 spdk_put_io_channel(ioch); 3432 spdk_bdev_close(desc); 3433 free_bdev(bdev); 3434 spdk_bdev_finish(bdev_fini_cb, NULL); 3435 poll_threads(); 3436 } 3437 3438 static void 3439 bdev_open_while_hotremove(void) 3440 { 3441 struct spdk_bdev *bdev; 3442 struct spdk_bdev_desc *desc[2] = {}; 3443 int rc; 3444 3445 bdev = allocate_bdev("bdev"); 3446 3447 rc = spdk_bdev_open_ext("bdev", false, bdev_ut_event_cb, NULL, &desc[0]); 3448 CU_ASSERT(rc == 0); 3449 SPDK_CU_ASSERT_FATAL(desc[0] != NULL); 3450 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc[0])); 3451 3452 spdk_bdev_unregister(bdev, NULL, NULL); 3453 3454 rc = spdk_bdev_open_ext("bdev", false, bdev_ut_event_cb, NULL, &desc[1]); 3455 CU_ASSERT(rc == -ENODEV); 3456 SPDK_CU_ASSERT_FATAL(desc[1] == NULL); 3457 3458 spdk_bdev_close(desc[0]); 3459 free_bdev(bdev); 3460 } 3461 3462 static void 3463 bdev_close_while_hotremove(void) 3464 { 3465 struct spdk_bdev *bdev; 3466 struct spdk_bdev_desc *desc = NULL; 3467 int rc = 0; 3468 3469 bdev = allocate_bdev("bdev"); 3470 3471 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb1, &desc, &desc); 3472 CU_ASSERT_EQUAL(rc, 0); 3473 SPDK_CU_ASSERT_FATAL(desc != NULL); 3474 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3475 3476 /* Simulate hot-unplug by unregistering bdev */ 3477 g_event_type1 = 0xFF; 3478 g_unregister_arg = NULL; 3479 g_unregister_rc = -1; 3480 spdk_bdev_unregister(bdev, bdev_unregister_cb, (void *)0x12345678); 3481 /* Close device while remove event is in flight */ 3482 spdk_bdev_close(desc); 3483 3484 /* Ensure that unregister callback is delayed */ 3485 CU_ASSERT_EQUAL(g_unregister_arg, NULL); 3486 CU_ASSERT_EQUAL(g_unregister_rc, -1); 3487 3488 poll_threads(); 3489 3490 /* Event callback shall not be issued because device was closed */ 3491 CU_ASSERT_EQUAL(g_event_type1, 0xFF); 3492 /* Unregister callback is issued */ 3493 CU_ASSERT_EQUAL(g_unregister_arg, (void *)0x12345678); 3494 CU_ASSERT_EQUAL(g_unregister_rc, 0); 3495 3496 free_bdev(bdev); 3497 } 3498 3499 static void 3500 bdev_open_ext(void) 3501 { 3502 struct spdk_bdev *bdev; 3503 struct spdk_bdev_desc *desc1 = NULL; 3504 struct spdk_bdev_desc *desc2 = NULL; 3505 int rc = 0; 3506 3507 bdev = allocate_bdev("bdev"); 3508 3509 rc = spdk_bdev_open_ext("bdev", true, NULL, NULL, &desc1); 3510 CU_ASSERT_EQUAL(rc, -EINVAL); 3511 3512 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb1, &desc1, &desc1); 3513 CU_ASSERT_EQUAL(rc, 0); 3514 3515 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb2, &desc2, &desc2); 3516 CU_ASSERT_EQUAL(rc, 0); 3517 3518 g_event_type1 = 0xFF; 3519 g_event_type2 = 0xFF; 3520 3521 /* Simulate hot-unplug by unregistering bdev */ 3522 spdk_bdev_unregister(bdev, NULL, NULL); 3523 poll_threads(); 3524 3525 /* Check if correct events have been triggered in event callback fn */ 3526 CU_ASSERT_EQUAL(g_event_type1, SPDK_BDEV_EVENT_REMOVE); 3527 CU_ASSERT_EQUAL(g_event_type2, SPDK_BDEV_EVENT_REMOVE); 3528 3529 free_bdev(bdev); 3530 poll_threads(); 3531 } 3532 3533 struct timeout_io_cb_arg { 3534 struct iovec iov; 3535 uint8_t type; 3536 }; 3537 3538 static int 3539 bdev_channel_count_submitted_io(struct spdk_bdev_channel *ch) 3540 { 3541 struct spdk_bdev_io *bdev_io; 3542 int n = 0; 3543 3544 if (!ch) { 3545 return -1; 3546 } 3547 3548 TAILQ_FOREACH(bdev_io, &ch->io_submitted, internal.ch_link) { 3549 n++; 3550 } 3551 3552 return n; 3553 } 3554 3555 static void 3556 bdev_channel_io_timeout_cb(void *cb_arg, struct spdk_bdev_io *bdev_io) 3557 { 3558 struct timeout_io_cb_arg *ctx = cb_arg; 3559 3560 ctx->type = bdev_io->type; 3561 ctx->iov.iov_base = bdev_io->iov.iov_base; 3562 ctx->iov.iov_len = bdev_io->iov.iov_len; 3563 } 3564 3565 static void 3566 bdev_set_io_timeout(void) 3567 { 3568 struct spdk_bdev *bdev; 3569 struct spdk_bdev_desc *desc = NULL; 3570 struct spdk_io_channel *io_ch = NULL; 3571 struct spdk_bdev_channel *bdev_ch = NULL; 3572 struct timeout_io_cb_arg cb_arg; 3573 3574 spdk_bdev_initialize(bdev_init_cb, NULL); 3575 3576 bdev = allocate_bdev("bdev"); 3577 3578 CU_ASSERT(spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc) == 0); 3579 SPDK_CU_ASSERT_FATAL(desc != NULL); 3580 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3581 3582 io_ch = spdk_bdev_get_io_channel(desc); 3583 CU_ASSERT(io_ch != NULL); 3584 3585 bdev_ch = spdk_io_channel_get_ctx(io_ch); 3586 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->io_submitted)); 3587 3588 /* This is the part1. 3589 * We will check the bdev_ch->io_submitted list 3590 * TO make sure that it can link IOs and only the user submitted IOs 3591 */ 3592 CU_ASSERT(spdk_bdev_read(desc, io_ch, (void *)0x1000, 0, 4096, io_done, NULL) == 0); 3593 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 3594 CU_ASSERT(spdk_bdev_write(desc, io_ch, (void *)0x2000, 0, 4096, io_done, NULL) == 0); 3595 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 2); 3596 stub_complete_io(1); 3597 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 3598 stub_complete_io(1); 3599 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 0); 3600 3601 /* Split IO */ 3602 bdev->optimal_io_boundary = 16; 3603 bdev->split_on_optimal_io_boundary = true; 3604 3605 /* Now test that a single-vector command is split correctly. 3606 * Offset 14, length 8, payload 0xF000 3607 * Child - Offset 14, length 2, payload 0xF000 3608 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 3609 * 3610 * Set up the expected values before calling spdk_bdev_read_blocks 3611 */ 3612 CU_ASSERT(spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL) == 0); 3613 /* We count all submitted IOs including IO that are generated by splitting. */ 3614 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 3); 3615 stub_complete_io(1); 3616 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 2); 3617 stub_complete_io(1); 3618 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 0); 3619 3620 /* Also include the reset IO */ 3621 CU_ASSERT(spdk_bdev_reset(desc, io_ch, io_done, NULL) == 0); 3622 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 3623 poll_threads(); 3624 stub_complete_io(1); 3625 poll_threads(); 3626 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 0); 3627 3628 /* This is part2 3629 * Test the desc timeout poller register 3630 */ 3631 3632 /* Successfully set the timeout */ 3633 CU_ASSERT(spdk_bdev_set_timeout(desc, 30, bdev_channel_io_timeout_cb, &cb_arg) == 0); 3634 CU_ASSERT(desc->io_timeout_poller != NULL); 3635 CU_ASSERT(desc->timeout_in_sec == 30); 3636 CU_ASSERT(desc->cb_fn == bdev_channel_io_timeout_cb); 3637 CU_ASSERT(desc->cb_arg == &cb_arg); 3638 3639 /* Change the timeout limit */ 3640 CU_ASSERT(spdk_bdev_set_timeout(desc, 20, bdev_channel_io_timeout_cb, &cb_arg) == 0); 3641 CU_ASSERT(desc->io_timeout_poller != NULL); 3642 CU_ASSERT(desc->timeout_in_sec == 20); 3643 CU_ASSERT(desc->cb_fn == bdev_channel_io_timeout_cb); 3644 CU_ASSERT(desc->cb_arg == &cb_arg); 3645 3646 /* Disable the timeout */ 3647 CU_ASSERT(spdk_bdev_set_timeout(desc, 0, NULL, NULL) == 0); 3648 CU_ASSERT(desc->io_timeout_poller == NULL); 3649 3650 /* This the part3 3651 * We will test to catch timeout IO and check whether the IO is 3652 * the submitted one. 3653 */ 3654 memset(&cb_arg, 0, sizeof(cb_arg)); 3655 CU_ASSERT(spdk_bdev_set_timeout(desc, 30, bdev_channel_io_timeout_cb, &cb_arg) == 0); 3656 CU_ASSERT(spdk_bdev_write_blocks(desc, io_ch, (void *)0x1000, 0, 1, io_done, NULL) == 0); 3657 3658 /* Don't reach the limit */ 3659 spdk_delay_us(15 * spdk_get_ticks_hz()); 3660 poll_threads(); 3661 CU_ASSERT(cb_arg.type == 0); 3662 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 3663 CU_ASSERT(cb_arg.iov.iov_len == 0); 3664 3665 /* 15 + 15 = 30 reach the limit */ 3666 spdk_delay_us(15 * spdk_get_ticks_hz()); 3667 poll_threads(); 3668 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 3669 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x1000); 3670 CU_ASSERT(cb_arg.iov.iov_len == 1 * bdev->blocklen); 3671 stub_complete_io(1); 3672 3673 /* Use the same split IO above and check the IO */ 3674 memset(&cb_arg, 0, sizeof(cb_arg)); 3675 CU_ASSERT(spdk_bdev_write_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL) == 0); 3676 3677 /* The first child complete in time */ 3678 spdk_delay_us(15 * spdk_get_ticks_hz()); 3679 poll_threads(); 3680 stub_complete_io(1); 3681 CU_ASSERT(cb_arg.type == 0); 3682 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 3683 CU_ASSERT(cb_arg.iov.iov_len == 0); 3684 3685 /* The second child reach the limit */ 3686 spdk_delay_us(15 * spdk_get_ticks_hz()); 3687 poll_threads(); 3688 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 3689 CU_ASSERT(cb_arg.iov.iov_base == (void *)0xF000); 3690 CU_ASSERT(cb_arg.iov.iov_len == 8 * bdev->blocklen); 3691 stub_complete_io(1); 3692 3693 /* Also include the reset IO */ 3694 memset(&cb_arg, 0, sizeof(cb_arg)); 3695 CU_ASSERT(spdk_bdev_reset(desc, io_ch, io_done, NULL) == 0); 3696 spdk_delay_us(30 * spdk_get_ticks_hz()); 3697 poll_threads(); 3698 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_RESET); 3699 stub_complete_io(1); 3700 poll_threads(); 3701 3702 spdk_put_io_channel(io_ch); 3703 spdk_bdev_close(desc); 3704 free_bdev(bdev); 3705 spdk_bdev_finish(bdev_fini_cb, NULL); 3706 poll_threads(); 3707 } 3708 3709 static void 3710 lba_range_overlap(void) 3711 { 3712 struct lba_range r1, r2; 3713 3714 r1.offset = 100; 3715 r1.length = 50; 3716 3717 r2.offset = 0; 3718 r2.length = 1; 3719 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 3720 3721 r2.offset = 0; 3722 r2.length = 100; 3723 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 3724 3725 r2.offset = 0; 3726 r2.length = 110; 3727 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 3728 3729 r2.offset = 100; 3730 r2.length = 10; 3731 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 3732 3733 r2.offset = 110; 3734 r2.length = 20; 3735 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 3736 3737 r2.offset = 140; 3738 r2.length = 150; 3739 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 3740 3741 r2.offset = 130; 3742 r2.length = 200; 3743 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 3744 3745 r2.offset = 150; 3746 r2.length = 100; 3747 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 3748 3749 r2.offset = 110; 3750 r2.length = 0; 3751 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 3752 } 3753 3754 static bool g_lock_lba_range_done; 3755 static bool g_unlock_lba_range_done; 3756 3757 static void 3758 lock_lba_range_done(void *ctx, int status) 3759 { 3760 g_lock_lba_range_done = true; 3761 } 3762 3763 static void 3764 unlock_lba_range_done(void *ctx, int status) 3765 { 3766 g_unlock_lba_range_done = true; 3767 } 3768 3769 static void 3770 lock_lba_range_check_ranges(void) 3771 { 3772 struct spdk_bdev *bdev; 3773 struct spdk_bdev_desc *desc = NULL; 3774 struct spdk_io_channel *io_ch; 3775 struct spdk_bdev_channel *channel; 3776 struct lba_range *range; 3777 int ctx1; 3778 int rc; 3779 3780 spdk_bdev_initialize(bdev_init_cb, NULL); 3781 3782 bdev = allocate_bdev("bdev0"); 3783 3784 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 3785 CU_ASSERT(rc == 0); 3786 CU_ASSERT(desc != NULL); 3787 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3788 io_ch = spdk_bdev_get_io_channel(desc); 3789 CU_ASSERT(io_ch != NULL); 3790 channel = spdk_io_channel_get_ctx(io_ch); 3791 3792 g_lock_lba_range_done = false; 3793 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 3794 CU_ASSERT(rc == 0); 3795 poll_threads(); 3796 3797 CU_ASSERT(g_lock_lba_range_done == true); 3798 range = TAILQ_FIRST(&channel->locked_ranges); 3799 SPDK_CU_ASSERT_FATAL(range != NULL); 3800 CU_ASSERT(range->offset == 20); 3801 CU_ASSERT(range->length == 10); 3802 CU_ASSERT(range->owner_ch == channel); 3803 3804 /* Unlocks must exactly match a lock. */ 3805 g_unlock_lba_range_done = false; 3806 rc = bdev_unlock_lba_range(desc, io_ch, 20, 1, unlock_lba_range_done, &ctx1); 3807 CU_ASSERT(rc == -EINVAL); 3808 CU_ASSERT(g_unlock_lba_range_done == false); 3809 3810 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, unlock_lba_range_done, &ctx1); 3811 CU_ASSERT(rc == 0); 3812 spdk_delay_us(100); 3813 poll_threads(); 3814 3815 CU_ASSERT(g_unlock_lba_range_done == true); 3816 CU_ASSERT(TAILQ_EMPTY(&channel->locked_ranges)); 3817 3818 spdk_put_io_channel(io_ch); 3819 spdk_bdev_close(desc); 3820 free_bdev(bdev); 3821 spdk_bdev_finish(bdev_fini_cb, NULL); 3822 poll_threads(); 3823 } 3824 3825 static void 3826 lock_lba_range_with_io_outstanding(void) 3827 { 3828 struct spdk_bdev *bdev; 3829 struct spdk_bdev_desc *desc = NULL; 3830 struct spdk_io_channel *io_ch; 3831 struct spdk_bdev_channel *channel; 3832 struct lba_range *range; 3833 char buf[4096]; 3834 int ctx1; 3835 int rc; 3836 3837 spdk_bdev_initialize(bdev_init_cb, NULL); 3838 3839 bdev = allocate_bdev("bdev0"); 3840 3841 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 3842 CU_ASSERT(rc == 0); 3843 CU_ASSERT(desc != NULL); 3844 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3845 io_ch = spdk_bdev_get_io_channel(desc); 3846 CU_ASSERT(io_ch != NULL); 3847 channel = spdk_io_channel_get_ctx(io_ch); 3848 3849 g_io_done = false; 3850 rc = spdk_bdev_read_blocks(desc, io_ch, buf, 20, 1, io_done, &ctx1); 3851 CU_ASSERT(rc == 0); 3852 3853 g_lock_lba_range_done = false; 3854 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 3855 CU_ASSERT(rc == 0); 3856 poll_threads(); 3857 3858 /* The lock should immediately become valid, since there are no outstanding 3859 * write I/O. 3860 */ 3861 CU_ASSERT(g_io_done == false); 3862 CU_ASSERT(g_lock_lba_range_done == true); 3863 range = TAILQ_FIRST(&channel->locked_ranges); 3864 SPDK_CU_ASSERT_FATAL(range != NULL); 3865 CU_ASSERT(range->offset == 20); 3866 CU_ASSERT(range->length == 10); 3867 CU_ASSERT(range->owner_ch == channel); 3868 CU_ASSERT(range->locked_ctx == &ctx1); 3869 3870 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 3871 CU_ASSERT(rc == 0); 3872 stub_complete_io(1); 3873 spdk_delay_us(100); 3874 poll_threads(); 3875 3876 CU_ASSERT(TAILQ_EMPTY(&channel->locked_ranges)); 3877 3878 /* Now try again, but with a write I/O. */ 3879 g_io_done = false; 3880 rc = spdk_bdev_write_blocks(desc, io_ch, buf, 20, 1, io_done, &ctx1); 3881 CU_ASSERT(rc == 0); 3882 3883 g_lock_lba_range_done = false; 3884 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 3885 CU_ASSERT(rc == 0); 3886 poll_threads(); 3887 3888 /* The lock should not be fully valid yet, since a write I/O is outstanding. 3889 * But note that the range should be on the channel's locked_list, to make sure no 3890 * new write I/O are started. 3891 */ 3892 CU_ASSERT(g_io_done == false); 3893 CU_ASSERT(g_lock_lba_range_done == false); 3894 range = TAILQ_FIRST(&channel->locked_ranges); 3895 SPDK_CU_ASSERT_FATAL(range != NULL); 3896 CU_ASSERT(range->offset == 20); 3897 CU_ASSERT(range->length == 10); 3898 3899 /* Complete the write I/O. This should make the lock valid (checked by confirming 3900 * our callback was invoked). 3901 */ 3902 stub_complete_io(1); 3903 spdk_delay_us(100); 3904 poll_threads(); 3905 CU_ASSERT(g_io_done == true); 3906 CU_ASSERT(g_lock_lba_range_done == true); 3907 3908 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, unlock_lba_range_done, &ctx1); 3909 CU_ASSERT(rc == 0); 3910 poll_threads(); 3911 3912 CU_ASSERT(TAILQ_EMPTY(&channel->locked_ranges)); 3913 3914 spdk_put_io_channel(io_ch); 3915 spdk_bdev_close(desc); 3916 free_bdev(bdev); 3917 spdk_bdev_finish(bdev_fini_cb, NULL); 3918 poll_threads(); 3919 } 3920 3921 static void 3922 lock_lba_range_overlapped(void) 3923 { 3924 struct spdk_bdev *bdev; 3925 struct spdk_bdev_desc *desc = NULL; 3926 struct spdk_io_channel *io_ch; 3927 struct spdk_bdev_channel *channel; 3928 struct lba_range *range; 3929 int ctx1; 3930 int rc; 3931 3932 spdk_bdev_initialize(bdev_init_cb, NULL); 3933 3934 bdev = allocate_bdev("bdev0"); 3935 3936 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 3937 CU_ASSERT(rc == 0); 3938 CU_ASSERT(desc != NULL); 3939 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3940 io_ch = spdk_bdev_get_io_channel(desc); 3941 CU_ASSERT(io_ch != NULL); 3942 channel = spdk_io_channel_get_ctx(io_ch); 3943 3944 /* Lock range 20-29. */ 3945 g_lock_lba_range_done = false; 3946 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 3947 CU_ASSERT(rc == 0); 3948 poll_threads(); 3949 3950 CU_ASSERT(g_lock_lba_range_done == true); 3951 range = TAILQ_FIRST(&channel->locked_ranges); 3952 SPDK_CU_ASSERT_FATAL(range != NULL); 3953 CU_ASSERT(range->offset == 20); 3954 CU_ASSERT(range->length == 10); 3955 3956 /* Try to lock range 25-39. It should not lock immediately, since it overlaps with 3957 * 20-29. 3958 */ 3959 g_lock_lba_range_done = false; 3960 rc = bdev_lock_lba_range(desc, io_ch, 25, 15, lock_lba_range_done, &ctx1); 3961 CU_ASSERT(rc == 0); 3962 poll_threads(); 3963 3964 CU_ASSERT(g_lock_lba_range_done == false); 3965 range = TAILQ_FIRST(&bdev->internal.pending_locked_ranges); 3966 SPDK_CU_ASSERT_FATAL(range != NULL); 3967 CU_ASSERT(range->offset == 25); 3968 CU_ASSERT(range->length == 15); 3969 3970 /* Unlock 20-29. This should result in range 25-39 now getting locked since it 3971 * no longer overlaps with an active lock. 3972 */ 3973 g_unlock_lba_range_done = false; 3974 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, unlock_lba_range_done, &ctx1); 3975 CU_ASSERT(rc == 0); 3976 poll_threads(); 3977 3978 CU_ASSERT(g_unlock_lba_range_done == true); 3979 CU_ASSERT(TAILQ_EMPTY(&bdev->internal.pending_locked_ranges)); 3980 range = TAILQ_FIRST(&channel->locked_ranges); 3981 SPDK_CU_ASSERT_FATAL(range != NULL); 3982 CU_ASSERT(range->offset == 25); 3983 CU_ASSERT(range->length == 15); 3984 3985 /* Lock 40-59. This should immediately lock since it does not overlap with the 3986 * currently active 25-39 lock. 3987 */ 3988 g_lock_lba_range_done = false; 3989 rc = bdev_lock_lba_range(desc, io_ch, 40, 20, lock_lba_range_done, &ctx1); 3990 CU_ASSERT(rc == 0); 3991 poll_threads(); 3992 3993 CU_ASSERT(g_lock_lba_range_done == true); 3994 range = TAILQ_FIRST(&bdev->internal.locked_ranges); 3995 SPDK_CU_ASSERT_FATAL(range != NULL); 3996 range = TAILQ_NEXT(range, tailq); 3997 SPDK_CU_ASSERT_FATAL(range != NULL); 3998 CU_ASSERT(range->offset == 40); 3999 CU_ASSERT(range->length == 20); 4000 4001 /* Try to lock 35-44. Note that this overlaps with both 25-39 and 40-59. */ 4002 g_lock_lba_range_done = false; 4003 rc = bdev_lock_lba_range(desc, io_ch, 35, 10, lock_lba_range_done, &ctx1); 4004 CU_ASSERT(rc == 0); 4005 poll_threads(); 4006 4007 CU_ASSERT(g_lock_lba_range_done == false); 4008 range = TAILQ_FIRST(&bdev->internal.pending_locked_ranges); 4009 SPDK_CU_ASSERT_FATAL(range != NULL); 4010 CU_ASSERT(range->offset == 35); 4011 CU_ASSERT(range->length == 10); 4012 4013 /* Unlock 25-39. Make sure that 35-44 is still in the pending list, since 4014 * the 40-59 lock is still active. 4015 */ 4016 g_unlock_lba_range_done = false; 4017 rc = bdev_unlock_lba_range(desc, io_ch, 25, 15, unlock_lba_range_done, &ctx1); 4018 CU_ASSERT(rc == 0); 4019 poll_threads(); 4020 4021 CU_ASSERT(g_unlock_lba_range_done == true); 4022 CU_ASSERT(g_lock_lba_range_done == false); 4023 range = TAILQ_FIRST(&bdev->internal.pending_locked_ranges); 4024 SPDK_CU_ASSERT_FATAL(range != NULL); 4025 CU_ASSERT(range->offset == 35); 4026 CU_ASSERT(range->length == 10); 4027 4028 /* Unlock 40-59. This should result in 35-44 now getting locked, since there are 4029 * no longer any active overlapping locks. 4030 */ 4031 g_unlock_lba_range_done = false; 4032 rc = bdev_unlock_lba_range(desc, io_ch, 40, 20, unlock_lba_range_done, &ctx1); 4033 CU_ASSERT(rc == 0); 4034 poll_threads(); 4035 4036 CU_ASSERT(g_unlock_lba_range_done == true); 4037 CU_ASSERT(g_lock_lba_range_done == true); 4038 CU_ASSERT(TAILQ_EMPTY(&bdev->internal.pending_locked_ranges)); 4039 range = TAILQ_FIRST(&bdev->internal.locked_ranges); 4040 SPDK_CU_ASSERT_FATAL(range != NULL); 4041 CU_ASSERT(range->offset == 35); 4042 CU_ASSERT(range->length == 10); 4043 4044 /* Finally, unlock 35-44. */ 4045 g_unlock_lba_range_done = false; 4046 rc = bdev_unlock_lba_range(desc, io_ch, 35, 10, unlock_lba_range_done, &ctx1); 4047 CU_ASSERT(rc == 0); 4048 poll_threads(); 4049 4050 CU_ASSERT(g_unlock_lba_range_done == true); 4051 CU_ASSERT(TAILQ_EMPTY(&bdev->internal.locked_ranges)); 4052 4053 spdk_put_io_channel(io_ch); 4054 spdk_bdev_close(desc); 4055 free_bdev(bdev); 4056 spdk_bdev_finish(bdev_fini_cb, NULL); 4057 poll_threads(); 4058 } 4059 4060 static void 4061 abort_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 4062 { 4063 g_abort_done = true; 4064 g_abort_status = bdev_io->internal.status; 4065 spdk_bdev_free_io(bdev_io); 4066 } 4067 4068 static void 4069 bdev_io_abort(void) 4070 { 4071 struct spdk_bdev *bdev; 4072 struct spdk_bdev_desc *desc = NULL; 4073 struct spdk_io_channel *io_ch; 4074 struct spdk_bdev_channel *channel; 4075 struct spdk_bdev_mgmt_channel *mgmt_ch; 4076 struct spdk_bdev_opts bdev_opts = { 4077 .bdev_io_pool_size = 7, 4078 .bdev_io_cache_size = 2, 4079 }; 4080 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 4081 uint64_t io_ctx1 = 0, io_ctx2 = 0, i; 4082 int rc; 4083 4084 rc = spdk_bdev_set_opts(&bdev_opts); 4085 CU_ASSERT(rc == 0); 4086 spdk_bdev_initialize(bdev_init_cb, NULL); 4087 4088 bdev = allocate_bdev("bdev0"); 4089 4090 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 4091 CU_ASSERT(rc == 0); 4092 CU_ASSERT(desc != NULL); 4093 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4094 io_ch = spdk_bdev_get_io_channel(desc); 4095 CU_ASSERT(io_ch != NULL); 4096 channel = spdk_io_channel_get_ctx(io_ch); 4097 mgmt_ch = channel->shared_resource->mgmt_ch; 4098 4099 g_abort_done = false; 4100 4101 ut_enable_io_type(SPDK_BDEV_IO_TYPE_ABORT, false); 4102 4103 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4104 CU_ASSERT(rc == -ENOTSUP); 4105 4106 ut_enable_io_type(SPDK_BDEV_IO_TYPE_ABORT, true); 4107 4108 rc = spdk_bdev_abort(desc, io_ch, &io_ctx2, abort_done, NULL); 4109 CU_ASSERT(rc == 0); 4110 CU_ASSERT(g_abort_done == true); 4111 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_FAILED); 4112 4113 /* Test the case that the target I/O was successfully aborted. */ 4114 g_io_done = false; 4115 4116 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, &io_ctx1); 4117 CU_ASSERT(rc == 0); 4118 CU_ASSERT(g_io_done == false); 4119 4120 g_abort_done = false; 4121 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4122 4123 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4124 CU_ASSERT(rc == 0); 4125 CU_ASSERT(g_io_done == true); 4126 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 4127 stub_complete_io(1); 4128 CU_ASSERT(g_abort_done == true); 4129 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4130 4131 /* Test the case that the target I/O was not aborted because it completed 4132 * in the middle of execution of the abort. 4133 */ 4134 g_io_done = false; 4135 4136 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, &io_ctx1); 4137 CU_ASSERT(rc == 0); 4138 CU_ASSERT(g_io_done == false); 4139 4140 g_abort_done = false; 4141 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 4142 4143 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4144 CU_ASSERT(rc == 0); 4145 CU_ASSERT(g_io_done == false); 4146 4147 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4148 stub_complete_io(1); 4149 CU_ASSERT(g_io_done == true); 4150 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4151 4152 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 4153 stub_complete_io(1); 4154 CU_ASSERT(g_abort_done == true); 4155 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4156 4157 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4158 4159 bdev->optimal_io_boundary = 16; 4160 bdev->split_on_optimal_io_boundary = true; 4161 4162 /* Test that a single-vector command which is split is aborted correctly. 4163 * Offset 14, length 8, payload 0xF000 4164 * Child - Offset 14, length 2, payload 0xF000 4165 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 4166 */ 4167 g_io_done = false; 4168 4169 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, &io_ctx1); 4170 CU_ASSERT(rc == 0); 4171 CU_ASSERT(g_io_done == false); 4172 4173 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 4174 4175 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4176 4177 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4178 CU_ASSERT(rc == 0); 4179 CU_ASSERT(g_io_done == true); 4180 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 4181 stub_complete_io(2); 4182 CU_ASSERT(g_abort_done == true); 4183 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4184 4185 /* Test that a multi-vector command that needs to be split by strip and then 4186 * needs to be split is aborted correctly. Abort is requested before the second 4187 * child I/O was submitted. The parent I/O should complete with failure without 4188 * submitting the second child I/O. 4189 */ 4190 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV * 2; i++) { 4191 iov[i].iov_base = (void *)((i + 1) * 0x10000); 4192 iov[i].iov_len = 512; 4193 } 4194 4195 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 4196 g_io_done = false; 4197 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, 0, 4198 BDEV_IO_NUM_CHILD_IOV * 2, io_done, &io_ctx1); 4199 CU_ASSERT(rc == 0); 4200 CU_ASSERT(g_io_done == false); 4201 4202 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 4203 4204 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4205 4206 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4207 CU_ASSERT(rc == 0); 4208 CU_ASSERT(g_io_done == true); 4209 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 4210 stub_complete_io(1); 4211 CU_ASSERT(g_abort_done == true); 4212 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4213 4214 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4215 4216 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 4217 4218 bdev->optimal_io_boundary = 16; 4219 g_io_done = false; 4220 4221 /* Test that a ingle-vector command which is split is aborted correctly. 4222 * Differently from the above, the child abort request will be submitted 4223 * sequentially due to the capacity of spdk_bdev_io. 4224 */ 4225 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 50, io_done, &io_ctx1); 4226 CU_ASSERT(rc == 0); 4227 CU_ASSERT(g_io_done == false); 4228 4229 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 4230 4231 g_abort_done = false; 4232 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4233 4234 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4235 CU_ASSERT(rc == 0); 4236 CU_ASSERT(!TAILQ_EMPTY(&mgmt_ch->io_wait_queue)); 4237 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 4238 4239 stub_complete_io(1); 4240 CU_ASSERT(g_io_done == true); 4241 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 4242 stub_complete_io(3); 4243 CU_ASSERT(g_abort_done == true); 4244 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4245 4246 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4247 4248 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 4249 4250 spdk_put_io_channel(io_ch); 4251 spdk_bdev_close(desc); 4252 free_bdev(bdev); 4253 spdk_bdev_finish(bdev_fini_cb, NULL); 4254 poll_threads(); 4255 } 4256 4257 int 4258 main(int argc, char **argv) 4259 { 4260 CU_pSuite suite = NULL; 4261 unsigned int num_failures; 4262 4263 CU_set_error_action(CUEA_ABORT); 4264 CU_initialize_registry(); 4265 4266 suite = CU_add_suite("bdev", null_init, null_clean); 4267 4268 CU_ADD_TEST(suite, bytes_to_blocks_test); 4269 CU_ADD_TEST(suite, num_blocks_test); 4270 CU_ADD_TEST(suite, io_valid_test); 4271 CU_ADD_TEST(suite, open_write_test); 4272 CU_ADD_TEST(suite, alias_add_del_test); 4273 CU_ADD_TEST(suite, get_device_stat_test); 4274 CU_ADD_TEST(suite, bdev_io_types_test); 4275 CU_ADD_TEST(suite, bdev_io_wait_test); 4276 CU_ADD_TEST(suite, bdev_io_spans_split_test); 4277 CU_ADD_TEST(suite, bdev_io_boundary_split_test); 4278 CU_ADD_TEST(suite, bdev_io_max_size_and_segment_split_test); 4279 CU_ADD_TEST(suite, bdev_io_mix_split_test); 4280 CU_ADD_TEST(suite, bdev_io_split_with_io_wait); 4281 CU_ADD_TEST(suite, bdev_io_alignment_with_boundary); 4282 CU_ADD_TEST(suite, bdev_io_alignment); 4283 CU_ADD_TEST(suite, bdev_histograms); 4284 CU_ADD_TEST(suite, bdev_write_zeroes); 4285 CU_ADD_TEST(suite, bdev_compare_and_write); 4286 CU_ADD_TEST(suite, bdev_compare); 4287 CU_ADD_TEST(suite, bdev_open_while_hotremove); 4288 CU_ADD_TEST(suite, bdev_close_while_hotremove); 4289 CU_ADD_TEST(suite, bdev_open_ext); 4290 CU_ADD_TEST(suite, bdev_set_io_timeout); 4291 CU_ADD_TEST(suite, lba_range_overlap); 4292 CU_ADD_TEST(suite, lock_lba_range_check_ranges); 4293 CU_ADD_TEST(suite, lock_lba_range_with_io_outstanding); 4294 CU_ADD_TEST(suite, lock_lba_range_overlapped); 4295 CU_ADD_TEST(suite, bdev_io_abort); 4296 4297 allocate_cores(1); 4298 allocate_threads(1); 4299 set_thread(0); 4300 4301 CU_basic_set_mode(CU_BRM_VERBOSE); 4302 CU_basic_run_tests(); 4303 num_failures = CU_get_number_of_failures(); 4304 CU_cleanup_registry(); 4305 4306 free_threads(); 4307 free_cores(); 4308 4309 return num_failures; 4310 } 4311