1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk_cunit.h" 35 36 #include "common/lib/ut_multithread.c" 37 #include "unit/lib/json_mock.c" 38 39 #include "spdk/config.h" 40 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 41 #undef SPDK_CONFIG_VTUNE 42 43 #include "bdev/bdev.c" 44 45 struct spdk_trace_histories *g_trace_histories; 46 DEFINE_STUB_V(spdk_trace_add_register_fn, (struct spdk_trace_register_fn *reg_fn)); 47 DEFINE_STUB_V(spdk_trace_register_owner, (uint8_t type, char id_prefix)); 48 DEFINE_STUB_V(spdk_trace_register_object, (uint8_t type, char id_prefix)); 49 DEFINE_STUB_V(spdk_trace_register_description, (const char *name, 50 uint16_t tpoint_id, uint8_t owner_type, 51 uint8_t object_type, uint8_t new_object, 52 uint8_t arg1_type, const char *arg1_name)); 53 DEFINE_STUB_V(_spdk_trace_record, (uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, 54 uint32_t size, uint64_t object_id, uint64_t arg1)); 55 DEFINE_STUB(spdk_notify_send, uint64_t, (const char *type, const char *ctx), 0); 56 DEFINE_STUB(spdk_notify_type_register, struct spdk_notify_type *, (const char *type), NULL); 57 58 59 int g_status; 60 int g_count; 61 enum spdk_bdev_event_type g_event_type1; 62 enum spdk_bdev_event_type g_event_type2; 63 struct spdk_histogram_data *g_histogram; 64 void *g_unregister_arg; 65 int g_unregister_rc; 66 67 void 68 spdk_scsi_nvme_translate(const struct spdk_bdev_io *bdev_io, 69 int *sc, int *sk, int *asc, int *ascq) 70 { 71 } 72 73 static int 74 null_init(void) 75 { 76 return 0; 77 } 78 79 static int 80 null_clean(void) 81 { 82 return 0; 83 } 84 85 static int 86 stub_destruct(void *ctx) 87 { 88 return 0; 89 } 90 91 struct ut_expected_io { 92 uint8_t type; 93 uint64_t offset; 94 uint64_t length; 95 int iovcnt; 96 struct iovec iov[BDEV_IO_NUM_CHILD_IOV]; 97 void *md_buf; 98 TAILQ_ENTRY(ut_expected_io) link; 99 }; 100 101 struct bdev_ut_channel { 102 TAILQ_HEAD(, spdk_bdev_io) outstanding_io; 103 uint32_t outstanding_io_count; 104 TAILQ_HEAD(, ut_expected_io) expected_io; 105 }; 106 107 static bool g_io_done; 108 static struct spdk_bdev_io *g_bdev_io; 109 static enum spdk_bdev_io_status g_io_status; 110 static enum spdk_bdev_io_status g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 111 static uint32_t g_bdev_ut_io_device; 112 static struct bdev_ut_channel *g_bdev_ut_channel; 113 static void *g_compare_read_buf; 114 static uint32_t g_compare_read_buf_len; 115 static void *g_compare_write_buf; 116 static uint32_t g_compare_write_buf_len; 117 static bool g_abort_done; 118 static enum spdk_bdev_io_status g_abort_status; 119 static void *g_zcopy_read_buf; 120 static uint32_t g_zcopy_read_buf_len; 121 static void *g_zcopy_write_buf; 122 static uint32_t g_zcopy_write_buf_len; 123 static struct spdk_bdev_io *g_zcopy_bdev_io; 124 125 static struct ut_expected_io * 126 ut_alloc_expected_io(uint8_t type, uint64_t offset, uint64_t length, int iovcnt) 127 { 128 struct ut_expected_io *expected_io; 129 130 expected_io = calloc(1, sizeof(*expected_io)); 131 SPDK_CU_ASSERT_FATAL(expected_io != NULL); 132 133 expected_io->type = type; 134 expected_io->offset = offset; 135 expected_io->length = length; 136 expected_io->iovcnt = iovcnt; 137 138 return expected_io; 139 } 140 141 static void 142 ut_expected_io_set_iov(struct ut_expected_io *expected_io, int pos, void *base, size_t len) 143 { 144 expected_io->iov[pos].iov_base = base; 145 expected_io->iov[pos].iov_len = len; 146 } 147 148 static void 149 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 150 { 151 struct bdev_ut_channel *ch = spdk_io_channel_get_ctx(_ch); 152 struct ut_expected_io *expected_io; 153 struct iovec *iov, *expected_iov; 154 struct spdk_bdev_io *bio_to_abort; 155 int i; 156 157 g_bdev_io = bdev_io; 158 159 if (g_compare_read_buf && bdev_io->type == SPDK_BDEV_IO_TYPE_READ) { 160 uint32_t len = bdev_io->u.bdev.iovs[0].iov_len; 161 162 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 163 CU_ASSERT(g_compare_read_buf_len == len); 164 memcpy(bdev_io->u.bdev.iovs[0].iov_base, g_compare_read_buf, len); 165 } 166 167 if (g_compare_write_buf && bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) { 168 uint32_t len = bdev_io->u.bdev.iovs[0].iov_len; 169 170 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 171 CU_ASSERT(g_compare_write_buf_len == len); 172 memcpy(g_compare_write_buf, bdev_io->u.bdev.iovs[0].iov_base, len); 173 } 174 175 if (g_compare_read_buf && bdev_io->type == SPDK_BDEV_IO_TYPE_COMPARE) { 176 uint32_t len = bdev_io->u.bdev.iovs[0].iov_len; 177 178 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 179 CU_ASSERT(g_compare_read_buf_len == len); 180 if (memcmp(bdev_io->u.bdev.iovs[0].iov_base, g_compare_read_buf, len)) { 181 g_io_exp_status = SPDK_BDEV_IO_STATUS_MISCOMPARE; 182 } 183 } 184 185 if (bdev_io->type == SPDK_BDEV_IO_TYPE_ABORT) { 186 if (g_io_exp_status == SPDK_BDEV_IO_STATUS_SUCCESS) { 187 TAILQ_FOREACH(bio_to_abort, &ch->outstanding_io, module_link) { 188 if (bio_to_abort == bdev_io->u.abort.bio_to_abort) { 189 TAILQ_REMOVE(&ch->outstanding_io, bio_to_abort, module_link); 190 ch->outstanding_io_count--; 191 spdk_bdev_io_complete(bio_to_abort, SPDK_BDEV_IO_STATUS_FAILED); 192 break; 193 } 194 } 195 } 196 } 197 198 if (bdev_io->type == SPDK_BDEV_IO_TYPE_ZCOPY) { 199 if (bdev_io->u.bdev.zcopy.start) { 200 g_zcopy_bdev_io = bdev_io; 201 if (bdev_io->u.bdev.zcopy.populate) { 202 /* Start of a read */ 203 CU_ASSERT(g_zcopy_read_buf != NULL); 204 CU_ASSERT(g_zcopy_read_buf_len > 0); 205 bdev_io->u.bdev.iovs[0].iov_base = g_zcopy_read_buf; 206 bdev_io->u.bdev.iovs[0].iov_len = g_zcopy_read_buf_len; 207 bdev_io->u.bdev.iovcnt = 1; 208 } else { 209 /* Start of a write */ 210 CU_ASSERT(g_zcopy_write_buf != NULL); 211 CU_ASSERT(g_zcopy_write_buf_len > 0); 212 bdev_io->u.bdev.iovs[0].iov_base = g_zcopy_write_buf; 213 bdev_io->u.bdev.iovs[0].iov_len = g_zcopy_write_buf_len; 214 bdev_io->u.bdev.iovcnt = 1; 215 } 216 } else { 217 if (bdev_io->u.bdev.zcopy.commit) { 218 /* End of write */ 219 CU_ASSERT(bdev_io->u.bdev.iovs[0].iov_base == g_zcopy_write_buf); 220 CU_ASSERT(bdev_io->u.bdev.iovs[0].iov_len == g_zcopy_write_buf_len); 221 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 222 g_zcopy_write_buf = NULL; 223 g_zcopy_write_buf_len = 0; 224 } else { 225 /* End of read */ 226 CU_ASSERT(bdev_io->u.bdev.iovs[0].iov_base == g_zcopy_read_buf); 227 CU_ASSERT(bdev_io->u.bdev.iovs[0].iov_len == g_zcopy_read_buf_len); 228 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 229 g_zcopy_read_buf = NULL; 230 g_zcopy_read_buf_len = 0; 231 } 232 } 233 } 234 235 TAILQ_INSERT_TAIL(&ch->outstanding_io, bdev_io, module_link); 236 ch->outstanding_io_count++; 237 238 expected_io = TAILQ_FIRST(&ch->expected_io); 239 if (expected_io == NULL) { 240 return; 241 } 242 TAILQ_REMOVE(&ch->expected_io, expected_io, link); 243 244 if (expected_io->type != SPDK_BDEV_IO_TYPE_INVALID) { 245 CU_ASSERT(bdev_io->type == expected_io->type); 246 } 247 248 if (expected_io->md_buf != NULL) { 249 CU_ASSERT(expected_io->md_buf == bdev_io->u.bdev.md_buf); 250 } 251 252 if (expected_io->length == 0) { 253 free(expected_io); 254 return; 255 } 256 257 CU_ASSERT(expected_io->offset == bdev_io->u.bdev.offset_blocks); 258 CU_ASSERT(expected_io->length = bdev_io->u.bdev.num_blocks); 259 260 if (expected_io->iovcnt == 0) { 261 free(expected_io); 262 /* UNMAP, WRITE_ZEROES and FLUSH don't have iovs, so we can just return now. */ 263 return; 264 } 265 266 CU_ASSERT(expected_io->iovcnt == bdev_io->u.bdev.iovcnt); 267 for (i = 0; i < expected_io->iovcnt; i++) { 268 iov = &bdev_io->u.bdev.iovs[i]; 269 expected_iov = &expected_io->iov[i]; 270 CU_ASSERT(iov->iov_len == expected_iov->iov_len); 271 CU_ASSERT(iov->iov_base == expected_iov->iov_base); 272 } 273 274 free(expected_io); 275 } 276 277 static void 278 stub_submit_request_get_buf_cb(struct spdk_io_channel *_ch, 279 struct spdk_bdev_io *bdev_io, bool success) 280 { 281 CU_ASSERT(success == true); 282 283 stub_submit_request(_ch, bdev_io); 284 } 285 286 static void 287 stub_submit_request_get_buf(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 288 { 289 spdk_bdev_io_get_buf(bdev_io, stub_submit_request_get_buf_cb, 290 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); 291 } 292 293 static uint32_t 294 stub_complete_io(uint32_t num_to_complete) 295 { 296 struct bdev_ut_channel *ch = g_bdev_ut_channel; 297 struct spdk_bdev_io *bdev_io; 298 static enum spdk_bdev_io_status io_status; 299 uint32_t num_completed = 0; 300 301 while (num_completed < num_to_complete) { 302 if (TAILQ_EMPTY(&ch->outstanding_io)) { 303 break; 304 } 305 bdev_io = TAILQ_FIRST(&ch->outstanding_io); 306 TAILQ_REMOVE(&ch->outstanding_io, bdev_io, module_link); 307 ch->outstanding_io_count--; 308 io_status = g_io_exp_status == SPDK_BDEV_IO_STATUS_SUCCESS ? SPDK_BDEV_IO_STATUS_SUCCESS : 309 g_io_exp_status; 310 spdk_bdev_io_complete(bdev_io, io_status); 311 num_completed++; 312 } 313 314 return num_completed; 315 } 316 317 static struct spdk_io_channel * 318 bdev_ut_get_io_channel(void *ctx) 319 { 320 return spdk_get_io_channel(&g_bdev_ut_io_device); 321 } 322 323 static bool g_io_types_supported[SPDK_BDEV_NUM_IO_TYPES] = { 324 [SPDK_BDEV_IO_TYPE_READ] = true, 325 [SPDK_BDEV_IO_TYPE_WRITE] = true, 326 [SPDK_BDEV_IO_TYPE_COMPARE] = true, 327 [SPDK_BDEV_IO_TYPE_UNMAP] = true, 328 [SPDK_BDEV_IO_TYPE_FLUSH] = true, 329 [SPDK_BDEV_IO_TYPE_RESET] = true, 330 [SPDK_BDEV_IO_TYPE_NVME_ADMIN] = true, 331 [SPDK_BDEV_IO_TYPE_NVME_IO] = true, 332 [SPDK_BDEV_IO_TYPE_NVME_IO_MD] = true, 333 [SPDK_BDEV_IO_TYPE_WRITE_ZEROES] = true, 334 [SPDK_BDEV_IO_TYPE_ZCOPY] = true, 335 [SPDK_BDEV_IO_TYPE_ABORT] = true, 336 }; 337 338 static void 339 ut_enable_io_type(enum spdk_bdev_io_type io_type, bool enable) 340 { 341 g_io_types_supported[io_type] = enable; 342 } 343 344 static bool 345 stub_io_type_supported(void *_bdev, enum spdk_bdev_io_type io_type) 346 { 347 return g_io_types_supported[io_type]; 348 } 349 350 static struct spdk_bdev_fn_table fn_table = { 351 .destruct = stub_destruct, 352 .submit_request = stub_submit_request, 353 .get_io_channel = bdev_ut_get_io_channel, 354 .io_type_supported = stub_io_type_supported, 355 }; 356 357 static int 358 bdev_ut_create_ch(void *io_device, void *ctx_buf) 359 { 360 struct bdev_ut_channel *ch = ctx_buf; 361 362 CU_ASSERT(g_bdev_ut_channel == NULL); 363 g_bdev_ut_channel = ch; 364 365 TAILQ_INIT(&ch->outstanding_io); 366 ch->outstanding_io_count = 0; 367 TAILQ_INIT(&ch->expected_io); 368 return 0; 369 } 370 371 static void 372 bdev_ut_destroy_ch(void *io_device, void *ctx_buf) 373 { 374 CU_ASSERT(g_bdev_ut_channel != NULL); 375 g_bdev_ut_channel = NULL; 376 } 377 378 struct spdk_bdev_module bdev_ut_if; 379 380 static int 381 bdev_ut_module_init(void) 382 { 383 spdk_io_device_register(&g_bdev_ut_io_device, bdev_ut_create_ch, bdev_ut_destroy_ch, 384 sizeof(struct bdev_ut_channel), NULL); 385 spdk_bdev_module_init_done(&bdev_ut_if); 386 return 0; 387 } 388 389 static void 390 bdev_ut_module_fini(void) 391 { 392 spdk_io_device_unregister(&g_bdev_ut_io_device, NULL); 393 } 394 395 struct spdk_bdev_module bdev_ut_if = { 396 .name = "bdev_ut", 397 .module_init = bdev_ut_module_init, 398 .module_fini = bdev_ut_module_fini, 399 .async_init = true, 400 }; 401 402 static void vbdev_ut_examine(struct spdk_bdev *bdev); 403 404 static int 405 vbdev_ut_module_init(void) 406 { 407 return 0; 408 } 409 410 static void 411 vbdev_ut_module_fini(void) 412 { 413 } 414 415 struct spdk_bdev_module vbdev_ut_if = { 416 .name = "vbdev_ut", 417 .module_init = vbdev_ut_module_init, 418 .module_fini = vbdev_ut_module_fini, 419 .examine_config = vbdev_ut_examine, 420 }; 421 422 SPDK_BDEV_MODULE_REGISTER(bdev_ut, &bdev_ut_if) 423 SPDK_BDEV_MODULE_REGISTER(vbdev_ut, &vbdev_ut_if) 424 425 static void 426 vbdev_ut_examine(struct spdk_bdev *bdev) 427 { 428 spdk_bdev_module_examine_done(&vbdev_ut_if); 429 } 430 431 static struct spdk_bdev * 432 allocate_bdev(char *name) 433 { 434 struct spdk_bdev *bdev; 435 int rc; 436 437 bdev = calloc(1, sizeof(*bdev)); 438 SPDK_CU_ASSERT_FATAL(bdev != NULL); 439 440 bdev->name = name; 441 bdev->fn_table = &fn_table; 442 bdev->module = &bdev_ut_if; 443 bdev->blockcnt = 1024; 444 bdev->blocklen = 512; 445 446 rc = spdk_bdev_register(bdev); 447 CU_ASSERT(rc == 0); 448 449 return bdev; 450 } 451 452 static struct spdk_bdev * 453 allocate_vbdev(char *name) 454 { 455 struct spdk_bdev *bdev; 456 int rc; 457 458 bdev = calloc(1, sizeof(*bdev)); 459 SPDK_CU_ASSERT_FATAL(bdev != NULL); 460 461 bdev->name = name; 462 bdev->fn_table = &fn_table; 463 bdev->module = &vbdev_ut_if; 464 465 rc = spdk_bdev_register(bdev); 466 CU_ASSERT(rc == 0); 467 468 return bdev; 469 } 470 471 static void 472 free_bdev(struct spdk_bdev *bdev) 473 { 474 spdk_bdev_unregister(bdev, NULL, NULL); 475 poll_threads(); 476 memset(bdev, 0xFF, sizeof(*bdev)); 477 free(bdev); 478 } 479 480 static void 481 free_vbdev(struct spdk_bdev *bdev) 482 { 483 spdk_bdev_unregister(bdev, NULL, NULL); 484 poll_threads(); 485 memset(bdev, 0xFF, sizeof(*bdev)); 486 free(bdev); 487 } 488 489 static void 490 get_device_stat_cb(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat, void *cb_arg, int rc) 491 { 492 const char *bdev_name; 493 494 CU_ASSERT(bdev != NULL); 495 CU_ASSERT(rc == 0); 496 bdev_name = spdk_bdev_get_name(bdev); 497 CU_ASSERT_STRING_EQUAL(bdev_name, "bdev0"); 498 499 free(stat); 500 501 *(bool *)cb_arg = true; 502 } 503 504 static void 505 bdev_unregister_cb(void *cb_arg, int rc) 506 { 507 g_unregister_arg = cb_arg; 508 g_unregister_rc = rc; 509 } 510 511 static void 512 bdev_ut_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 513 { 514 } 515 516 static void 517 bdev_open_cb1(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 518 { 519 struct spdk_bdev_desc *desc = *(struct spdk_bdev_desc **)event_ctx; 520 521 g_event_type1 = type; 522 if (SPDK_BDEV_EVENT_REMOVE == type) { 523 spdk_bdev_close(desc); 524 } 525 } 526 527 static void 528 bdev_open_cb2(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 529 { 530 struct spdk_bdev_desc *desc = *(struct spdk_bdev_desc **)event_ctx; 531 532 g_event_type2 = type; 533 if (SPDK_BDEV_EVENT_REMOVE == type) { 534 spdk_bdev_close(desc); 535 } 536 } 537 538 static void 539 get_device_stat_test(void) 540 { 541 struct spdk_bdev *bdev; 542 struct spdk_bdev_io_stat *stat; 543 bool done; 544 545 bdev = allocate_bdev("bdev0"); 546 stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 547 if (stat == NULL) { 548 free_bdev(bdev); 549 return; 550 } 551 552 done = false; 553 spdk_bdev_get_device_stat(bdev, stat, get_device_stat_cb, &done); 554 while (!done) { poll_threads(); } 555 556 free_bdev(bdev); 557 } 558 559 static void 560 open_write_test(void) 561 { 562 struct spdk_bdev *bdev[9]; 563 struct spdk_bdev_desc *desc[9] = {}; 564 int rc; 565 566 /* 567 * Create a tree of bdevs to test various open w/ write cases. 568 * 569 * bdev0 through bdev3 are physical block devices, such as NVMe 570 * namespaces or Ceph block devices. 571 * 572 * bdev4 is a virtual bdev with multiple base bdevs. This models 573 * caching or RAID use cases. 574 * 575 * bdev5 through bdev7 are all virtual bdevs with the same base 576 * bdev (except bdev7). This models partitioning or logical volume 577 * use cases. 578 * 579 * bdev7 is a virtual bdev with multiple base bdevs. One of base bdevs 580 * (bdev2) is shared with other virtual bdevs: bdev5 and bdev6. This 581 * models caching, RAID, partitioning or logical volumes use cases. 582 * 583 * bdev8 is a virtual bdev with multiple base bdevs, but these 584 * base bdevs are themselves virtual bdevs. 585 * 586 * bdev8 587 * | 588 * +----------+ 589 * | | 590 * bdev4 bdev5 bdev6 bdev7 591 * | | | | 592 * +---+---+ +---+ + +---+---+ 593 * | | \ | / \ 594 * bdev0 bdev1 bdev2 bdev3 595 */ 596 597 bdev[0] = allocate_bdev("bdev0"); 598 rc = spdk_bdev_module_claim_bdev(bdev[0], NULL, &bdev_ut_if); 599 CU_ASSERT(rc == 0); 600 601 bdev[1] = allocate_bdev("bdev1"); 602 rc = spdk_bdev_module_claim_bdev(bdev[1], NULL, &bdev_ut_if); 603 CU_ASSERT(rc == 0); 604 605 bdev[2] = allocate_bdev("bdev2"); 606 rc = spdk_bdev_module_claim_bdev(bdev[2], NULL, &bdev_ut_if); 607 CU_ASSERT(rc == 0); 608 609 bdev[3] = allocate_bdev("bdev3"); 610 rc = spdk_bdev_module_claim_bdev(bdev[3], NULL, &bdev_ut_if); 611 CU_ASSERT(rc == 0); 612 613 bdev[4] = allocate_vbdev("bdev4"); 614 rc = spdk_bdev_module_claim_bdev(bdev[4], NULL, &bdev_ut_if); 615 CU_ASSERT(rc == 0); 616 617 bdev[5] = allocate_vbdev("bdev5"); 618 rc = spdk_bdev_module_claim_bdev(bdev[5], NULL, &bdev_ut_if); 619 CU_ASSERT(rc == 0); 620 621 bdev[6] = allocate_vbdev("bdev6"); 622 623 bdev[7] = allocate_vbdev("bdev7"); 624 625 bdev[8] = allocate_vbdev("bdev8"); 626 627 /* Open bdev0 read-only. This should succeed. */ 628 rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc[0]); 629 CU_ASSERT(rc == 0); 630 SPDK_CU_ASSERT_FATAL(desc[0] != NULL); 631 CU_ASSERT(bdev[0] == spdk_bdev_desc_get_bdev(desc[0])); 632 spdk_bdev_close(desc[0]); 633 634 /* 635 * Open bdev1 read/write. This should fail since bdev1 has been claimed 636 * by a vbdev module. 637 */ 638 rc = spdk_bdev_open_ext("bdev1", true, bdev_ut_event_cb, NULL, &desc[1]); 639 CU_ASSERT(rc == -EPERM); 640 641 /* 642 * Open bdev4 read/write. This should fail since bdev3 has been claimed 643 * by a vbdev module. 644 */ 645 rc = spdk_bdev_open_ext("bdev4", true, bdev_ut_event_cb, NULL, &desc[4]); 646 CU_ASSERT(rc == -EPERM); 647 648 /* Open bdev4 read-only. This should succeed. */ 649 rc = spdk_bdev_open_ext("bdev4", false, bdev_ut_event_cb, NULL, &desc[4]); 650 CU_ASSERT(rc == 0); 651 SPDK_CU_ASSERT_FATAL(desc[4] != NULL); 652 CU_ASSERT(bdev[4] == spdk_bdev_desc_get_bdev(desc[4])); 653 spdk_bdev_close(desc[4]); 654 655 /* 656 * Open bdev8 read/write. This should succeed since it is a leaf 657 * bdev. 658 */ 659 rc = spdk_bdev_open_ext("bdev8", true, bdev_ut_event_cb, NULL, &desc[8]); 660 CU_ASSERT(rc == 0); 661 SPDK_CU_ASSERT_FATAL(desc[8] != NULL); 662 CU_ASSERT(bdev[8] == spdk_bdev_desc_get_bdev(desc[8])); 663 spdk_bdev_close(desc[8]); 664 665 /* 666 * Open bdev5 read/write. This should fail since bdev4 has been claimed 667 * by a vbdev module. 668 */ 669 rc = spdk_bdev_open_ext("bdev5", true, bdev_ut_event_cb, NULL, &desc[5]); 670 CU_ASSERT(rc == -EPERM); 671 672 /* Open bdev4 read-only. This should succeed. */ 673 rc = spdk_bdev_open_ext("bdev5", false, bdev_ut_event_cb, NULL, &desc[5]); 674 CU_ASSERT(rc == 0); 675 SPDK_CU_ASSERT_FATAL(desc[5] != NULL); 676 CU_ASSERT(bdev[5] == spdk_bdev_desc_get_bdev(desc[5])); 677 spdk_bdev_close(desc[5]); 678 679 free_vbdev(bdev[8]); 680 681 free_vbdev(bdev[5]); 682 free_vbdev(bdev[6]); 683 free_vbdev(bdev[7]); 684 685 free_vbdev(bdev[4]); 686 687 free_bdev(bdev[0]); 688 free_bdev(bdev[1]); 689 free_bdev(bdev[2]); 690 free_bdev(bdev[3]); 691 } 692 693 static void 694 bytes_to_blocks_test(void) 695 { 696 struct spdk_bdev bdev; 697 uint64_t offset_blocks, num_blocks; 698 699 memset(&bdev, 0, sizeof(bdev)); 700 701 bdev.blocklen = 512; 702 703 /* All parameters valid */ 704 offset_blocks = 0; 705 num_blocks = 0; 706 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 512, &offset_blocks, 1024, &num_blocks) == 0); 707 CU_ASSERT(offset_blocks == 1); 708 CU_ASSERT(num_blocks == 2); 709 710 /* Offset not a block multiple */ 711 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 3, &offset_blocks, 512, &num_blocks) != 0); 712 713 /* Length not a block multiple */ 714 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 512, &offset_blocks, 3, &num_blocks) != 0); 715 716 /* In case blocklen not the power of two */ 717 bdev.blocklen = 100; 718 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 100, &offset_blocks, 200, &num_blocks) == 0); 719 CU_ASSERT(offset_blocks == 1); 720 CU_ASSERT(num_blocks == 2); 721 722 /* Offset not a block multiple */ 723 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 3, &offset_blocks, 100, &num_blocks) != 0); 724 725 /* Length not a block multiple */ 726 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 100, &offset_blocks, 3, &num_blocks) != 0); 727 } 728 729 static void 730 num_blocks_test(void) 731 { 732 struct spdk_bdev bdev; 733 struct spdk_bdev_desc *desc = NULL; 734 int rc; 735 736 memset(&bdev, 0, sizeof(bdev)); 737 bdev.name = "num_blocks"; 738 bdev.fn_table = &fn_table; 739 bdev.module = &bdev_ut_if; 740 spdk_bdev_register(&bdev); 741 spdk_bdev_notify_blockcnt_change(&bdev, 50); 742 743 /* Growing block number */ 744 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 70) == 0); 745 /* Shrinking block number */ 746 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 30) == 0); 747 748 rc = spdk_bdev_open_ext("num_blocks", false, bdev_open_cb1, &desc, &desc); 749 CU_ASSERT(rc == 0); 750 SPDK_CU_ASSERT_FATAL(desc != NULL); 751 CU_ASSERT(&bdev == spdk_bdev_desc_get_bdev(desc)); 752 753 /* Growing block number */ 754 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 80) == 0); 755 /* Shrinking block number */ 756 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 20) != 0); 757 758 g_event_type1 = 0xFF; 759 /* Growing block number */ 760 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 90) == 0); 761 762 poll_threads(); 763 CU_ASSERT_EQUAL(g_event_type1, SPDK_BDEV_EVENT_RESIZE); 764 765 g_event_type1 = 0xFF; 766 /* Growing block number and closing */ 767 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 100) == 0); 768 769 spdk_bdev_close(desc); 770 spdk_bdev_unregister(&bdev, NULL, NULL); 771 772 poll_threads(); 773 774 /* Callback is not called for closed device */ 775 CU_ASSERT_EQUAL(g_event_type1, 0xFF); 776 } 777 778 static void 779 io_valid_test(void) 780 { 781 struct spdk_bdev bdev; 782 783 memset(&bdev, 0, sizeof(bdev)); 784 785 bdev.blocklen = 512; 786 CU_ASSERT(pthread_mutex_init(&bdev.internal.mutex, NULL) == 0); 787 788 spdk_bdev_notify_blockcnt_change(&bdev, 100); 789 790 /* All parameters valid */ 791 CU_ASSERT(bdev_io_valid_blocks(&bdev, 1, 2) == true); 792 793 /* Last valid block */ 794 CU_ASSERT(bdev_io_valid_blocks(&bdev, 99, 1) == true); 795 796 /* Offset past end of bdev */ 797 CU_ASSERT(bdev_io_valid_blocks(&bdev, 100, 1) == false); 798 799 /* Offset + length past end of bdev */ 800 CU_ASSERT(bdev_io_valid_blocks(&bdev, 99, 2) == false); 801 802 /* Offset near end of uint64_t range (2^64 - 1) */ 803 CU_ASSERT(bdev_io_valid_blocks(&bdev, 18446744073709551615ULL, 1) == false); 804 805 CU_ASSERT(pthread_mutex_destroy(&bdev.internal.mutex) == 0); 806 } 807 808 static void 809 alias_add_del_test(void) 810 { 811 struct spdk_bdev *bdev[3]; 812 int rc; 813 814 /* Creating and registering bdevs */ 815 bdev[0] = allocate_bdev("bdev0"); 816 SPDK_CU_ASSERT_FATAL(bdev[0] != 0); 817 818 bdev[1] = allocate_bdev("bdev1"); 819 SPDK_CU_ASSERT_FATAL(bdev[1] != 0); 820 821 bdev[2] = allocate_bdev("bdev2"); 822 SPDK_CU_ASSERT_FATAL(bdev[2] != 0); 823 824 poll_threads(); 825 826 /* 827 * Trying adding an alias identical to name. 828 * Alias is identical to name, so it can not be added to aliases list 829 */ 830 rc = spdk_bdev_alias_add(bdev[0], bdev[0]->name); 831 CU_ASSERT(rc == -EEXIST); 832 833 /* 834 * Trying to add empty alias, 835 * this one should fail 836 */ 837 rc = spdk_bdev_alias_add(bdev[0], NULL); 838 CU_ASSERT(rc == -EINVAL); 839 840 /* Trying adding same alias to two different registered bdevs */ 841 842 /* Alias is used first time, so this one should pass */ 843 rc = spdk_bdev_alias_add(bdev[0], "proper alias 0"); 844 CU_ASSERT(rc == 0); 845 846 /* Alias was added to another bdev, so this one should fail */ 847 rc = spdk_bdev_alias_add(bdev[1], "proper alias 0"); 848 CU_ASSERT(rc == -EEXIST); 849 850 /* Alias is used first time, so this one should pass */ 851 rc = spdk_bdev_alias_add(bdev[1], "proper alias 1"); 852 CU_ASSERT(rc == 0); 853 854 /* Trying removing an alias from registered bdevs */ 855 856 /* Alias is not on a bdev aliases list, so this one should fail */ 857 rc = spdk_bdev_alias_del(bdev[0], "not existing"); 858 CU_ASSERT(rc == -ENOENT); 859 860 /* Alias is present on a bdev aliases list, so this one should pass */ 861 rc = spdk_bdev_alias_del(bdev[0], "proper alias 0"); 862 CU_ASSERT(rc == 0); 863 864 /* Alias is present on a bdev aliases list, so this one should pass */ 865 rc = spdk_bdev_alias_del(bdev[1], "proper alias 1"); 866 CU_ASSERT(rc == 0); 867 868 /* Trying to remove name instead of alias, so this one should fail, name cannot be changed or removed */ 869 rc = spdk_bdev_alias_del(bdev[0], bdev[0]->name); 870 CU_ASSERT(rc != 0); 871 872 /* Trying to del all alias from empty alias list */ 873 spdk_bdev_alias_del_all(bdev[2]); 874 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev[2]->aliases)); 875 876 /* Trying to del all alias from non-empty alias list */ 877 rc = spdk_bdev_alias_add(bdev[2], "alias0"); 878 CU_ASSERT(rc == 0); 879 rc = spdk_bdev_alias_add(bdev[2], "alias1"); 880 CU_ASSERT(rc == 0); 881 spdk_bdev_alias_del_all(bdev[2]); 882 CU_ASSERT(TAILQ_EMPTY(&bdev[2]->aliases)); 883 884 /* Unregister and free bdevs */ 885 spdk_bdev_unregister(bdev[0], NULL, NULL); 886 spdk_bdev_unregister(bdev[1], NULL, NULL); 887 spdk_bdev_unregister(bdev[2], NULL, NULL); 888 889 poll_threads(); 890 891 free(bdev[0]); 892 free(bdev[1]); 893 free(bdev[2]); 894 } 895 896 static void 897 io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 898 { 899 g_io_done = true; 900 g_io_status = bdev_io->internal.status; 901 if ((bdev_io->type == SPDK_BDEV_IO_TYPE_ZCOPY) && 902 (bdev_io->u.bdev.zcopy.start)) { 903 g_zcopy_bdev_io = bdev_io; 904 } else { 905 spdk_bdev_free_io(bdev_io); 906 g_zcopy_bdev_io = NULL; 907 } 908 } 909 910 static void 911 bdev_init_cb(void *arg, int rc) 912 { 913 CU_ASSERT(rc == 0); 914 } 915 916 static void 917 bdev_fini_cb(void *arg) 918 { 919 } 920 921 struct bdev_ut_io_wait_entry { 922 struct spdk_bdev_io_wait_entry entry; 923 struct spdk_io_channel *io_ch; 924 struct spdk_bdev_desc *desc; 925 bool submitted; 926 }; 927 928 static void 929 io_wait_cb(void *arg) 930 { 931 struct bdev_ut_io_wait_entry *entry = arg; 932 int rc; 933 934 rc = spdk_bdev_read_blocks(entry->desc, entry->io_ch, NULL, 0, 1, io_done, NULL); 935 CU_ASSERT(rc == 0); 936 entry->submitted = true; 937 } 938 939 static void 940 bdev_io_types_test(void) 941 { 942 struct spdk_bdev *bdev; 943 struct spdk_bdev_desc *desc = NULL; 944 struct spdk_io_channel *io_ch; 945 struct spdk_bdev_opts bdev_opts = {}; 946 int rc; 947 948 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 949 bdev_opts.bdev_io_pool_size = 4; 950 bdev_opts.bdev_io_cache_size = 2; 951 952 rc = spdk_bdev_set_opts(&bdev_opts); 953 CU_ASSERT(rc == 0); 954 spdk_bdev_initialize(bdev_init_cb, NULL); 955 poll_threads(); 956 957 bdev = allocate_bdev("bdev0"); 958 959 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 960 CU_ASSERT(rc == 0); 961 poll_threads(); 962 SPDK_CU_ASSERT_FATAL(desc != NULL); 963 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 964 io_ch = spdk_bdev_get_io_channel(desc); 965 CU_ASSERT(io_ch != NULL); 966 967 /* WRITE and WRITE ZEROES are not supported */ 968 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, false); 969 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE, false); 970 rc = spdk_bdev_write_zeroes_blocks(desc, io_ch, 0, 128, io_done, NULL); 971 CU_ASSERT(rc == -ENOTSUP); 972 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, true); 973 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE, true); 974 975 spdk_put_io_channel(io_ch); 976 spdk_bdev_close(desc); 977 free_bdev(bdev); 978 spdk_bdev_finish(bdev_fini_cb, NULL); 979 poll_threads(); 980 } 981 982 static void 983 bdev_io_wait_test(void) 984 { 985 struct spdk_bdev *bdev; 986 struct spdk_bdev_desc *desc = NULL; 987 struct spdk_io_channel *io_ch; 988 struct spdk_bdev_opts bdev_opts = {}; 989 struct bdev_ut_io_wait_entry io_wait_entry; 990 struct bdev_ut_io_wait_entry io_wait_entry2; 991 int rc; 992 993 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 994 bdev_opts.bdev_io_pool_size = 4; 995 bdev_opts.bdev_io_cache_size = 2; 996 997 rc = spdk_bdev_set_opts(&bdev_opts); 998 CU_ASSERT(rc == 0); 999 spdk_bdev_initialize(bdev_init_cb, NULL); 1000 poll_threads(); 1001 1002 bdev = allocate_bdev("bdev0"); 1003 1004 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 1005 CU_ASSERT(rc == 0); 1006 poll_threads(); 1007 SPDK_CU_ASSERT_FATAL(desc != NULL); 1008 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 1009 io_ch = spdk_bdev_get_io_channel(desc); 1010 CU_ASSERT(io_ch != NULL); 1011 1012 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 1013 CU_ASSERT(rc == 0); 1014 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 1015 CU_ASSERT(rc == 0); 1016 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 1017 CU_ASSERT(rc == 0); 1018 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 1019 CU_ASSERT(rc == 0); 1020 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 1021 1022 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 1023 CU_ASSERT(rc == -ENOMEM); 1024 1025 io_wait_entry.entry.bdev = bdev; 1026 io_wait_entry.entry.cb_fn = io_wait_cb; 1027 io_wait_entry.entry.cb_arg = &io_wait_entry; 1028 io_wait_entry.io_ch = io_ch; 1029 io_wait_entry.desc = desc; 1030 io_wait_entry.submitted = false; 1031 /* Cannot use the same io_wait_entry for two different calls. */ 1032 memcpy(&io_wait_entry2, &io_wait_entry, sizeof(io_wait_entry)); 1033 io_wait_entry2.entry.cb_arg = &io_wait_entry2; 1034 1035 /* Queue two I/O waits. */ 1036 rc = spdk_bdev_queue_io_wait(bdev, io_ch, &io_wait_entry.entry); 1037 CU_ASSERT(rc == 0); 1038 CU_ASSERT(io_wait_entry.submitted == false); 1039 rc = spdk_bdev_queue_io_wait(bdev, io_ch, &io_wait_entry2.entry); 1040 CU_ASSERT(rc == 0); 1041 CU_ASSERT(io_wait_entry2.submitted == false); 1042 1043 stub_complete_io(1); 1044 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 1045 CU_ASSERT(io_wait_entry.submitted == true); 1046 CU_ASSERT(io_wait_entry2.submitted == false); 1047 1048 stub_complete_io(1); 1049 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 1050 CU_ASSERT(io_wait_entry2.submitted == true); 1051 1052 stub_complete_io(4); 1053 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1054 1055 spdk_put_io_channel(io_ch); 1056 spdk_bdev_close(desc); 1057 free_bdev(bdev); 1058 spdk_bdev_finish(bdev_fini_cb, NULL); 1059 poll_threads(); 1060 } 1061 1062 static void 1063 bdev_io_spans_split_test(void) 1064 { 1065 struct spdk_bdev bdev; 1066 struct spdk_bdev_io bdev_io; 1067 struct iovec iov[BDEV_IO_NUM_CHILD_IOV]; 1068 1069 memset(&bdev, 0, sizeof(bdev)); 1070 bdev_io.u.bdev.iovs = iov; 1071 1072 bdev_io.type = SPDK_BDEV_IO_TYPE_READ; 1073 bdev.optimal_io_boundary = 0; 1074 bdev.max_segment_size = 0; 1075 bdev.max_num_segments = 0; 1076 bdev_io.bdev = &bdev; 1077 1078 /* bdev has no optimal_io_boundary and max_size set - so this should return false. */ 1079 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1080 1081 bdev.split_on_optimal_io_boundary = true; 1082 bdev.optimal_io_boundary = 32; 1083 bdev_io.type = SPDK_BDEV_IO_TYPE_RESET; 1084 1085 /* RESETs are not based on LBAs - so this should return false. */ 1086 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1087 1088 bdev_io.type = SPDK_BDEV_IO_TYPE_READ; 1089 bdev_io.u.bdev.offset_blocks = 0; 1090 bdev_io.u.bdev.num_blocks = 32; 1091 1092 /* This I/O run right up to, but does not cross, the boundary - so this should return false. */ 1093 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1094 1095 bdev_io.u.bdev.num_blocks = 33; 1096 1097 /* This I/O spans a boundary. */ 1098 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1099 1100 bdev_io.u.bdev.num_blocks = 32; 1101 bdev.max_segment_size = 512 * 32; 1102 bdev.max_num_segments = 1; 1103 bdev_io.u.bdev.iovcnt = 1; 1104 iov[0].iov_len = 512; 1105 1106 /* Does not cross and exceed max_size or max_segs */ 1107 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1108 1109 bdev.split_on_optimal_io_boundary = false; 1110 bdev.max_segment_size = 512; 1111 bdev.max_num_segments = 1; 1112 bdev_io.u.bdev.iovcnt = 2; 1113 1114 /* Exceed max_segs */ 1115 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1116 1117 bdev.max_num_segments = 2; 1118 iov[0].iov_len = 513; 1119 iov[1].iov_len = 512; 1120 1121 /* Exceed max_sizes */ 1122 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1123 } 1124 1125 static void 1126 bdev_io_boundary_split_test(void) 1127 { 1128 struct spdk_bdev *bdev; 1129 struct spdk_bdev_desc *desc = NULL; 1130 struct spdk_io_channel *io_ch; 1131 struct spdk_bdev_opts bdev_opts = {}; 1132 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 1133 struct ut_expected_io *expected_io; 1134 void *md_buf = (void *)0xFF000000; 1135 uint64_t i; 1136 int rc; 1137 1138 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 1139 bdev_opts.bdev_io_pool_size = 512; 1140 bdev_opts.bdev_io_cache_size = 64; 1141 1142 rc = spdk_bdev_set_opts(&bdev_opts); 1143 CU_ASSERT(rc == 0); 1144 spdk_bdev_initialize(bdev_init_cb, NULL); 1145 1146 bdev = allocate_bdev("bdev0"); 1147 1148 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 1149 CU_ASSERT(rc == 0); 1150 SPDK_CU_ASSERT_FATAL(desc != NULL); 1151 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 1152 io_ch = spdk_bdev_get_io_channel(desc); 1153 CU_ASSERT(io_ch != NULL); 1154 1155 bdev->optimal_io_boundary = 16; 1156 bdev->split_on_optimal_io_boundary = false; 1157 1158 g_io_done = false; 1159 1160 /* First test that the I/O does not get split if split_on_optimal_io_boundary == false. */ 1161 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 8, 1); 1162 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 8 * 512); 1163 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1164 1165 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL); 1166 CU_ASSERT(rc == 0); 1167 CU_ASSERT(g_io_done == false); 1168 1169 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1170 stub_complete_io(1); 1171 CU_ASSERT(g_io_done == true); 1172 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1173 1174 bdev->split_on_optimal_io_boundary = true; 1175 bdev->md_interleave = false; 1176 bdev->md_len = 8; 1177 1178 /* Now test that a single-vector command is split correctly. 1179 * Offset 14, length 8, payload 0xF000 1180 * Child - Offset 14, length 2, payload 0xF000 1181 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 1182 * 1183 * Set up the expected values before calling spdk_bdev_read_blocks 1184 */ 1185 g_io_done = false; 1186 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 1); 1187 expected_io->md_buf = md_buf; 1188 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 2 * 512); 1189 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1190 1191 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 6, 1); 1192 expected_io->md_buf = md_buf + 2 * 8; 1193 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 6 * 512); 1194 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1195 1196 /* spdk_bdev_read_blocks will submit the first child immediately. */ 1197 rc = spdk_bdev_read_blocks_with_md(desc, io_ch, (void *)0xF000, md_buf, 1198 14, 8, io_done, NULL); 1199 CU_ASSERT(rc == 0); 1200 CU_ASSERT(g_io_done == false); 1201 1202 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1203 stub_complete_io(2); 1204 CU_ASSERT(g_io_done == true); 1205 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1206 1207 /* Now set up a more complex, multi-vector command that needs to be split, 1208 * including splitting iovecs. 1209 */ 1210 iov[0].iov_base = (void *)0x10000; 1211 iov[0].iov_len = 512; 1212 iov[1].iov_base = (void *)0x20000; 1213 iov[1].iov_len = 20 * 512; 1214 iov[2].iov_base = (void *)0x30000; 1215 iov[2].iov_len = 11 * 512; 1216 1217 g_io_done = false; 1218 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 2); 1219 expected_io->md_buf = md_buf; 1220 ut_expected_io_set_iov(expected_io, 0, (void *)0x10000, 512); 1221 ut_expected_io_set_iov(expected_io, 1, (void *)0x20000, 512); 1222 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1223 1224 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 16, 1); 1225 expected_io->md_buf = md_buf + 2 * 8; 1226 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 512), 16 * 512); 1227 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1228 1229 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 14, 2); 1230 expected_io->md_buf = md_buf + 18 * 8; 1231 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 17 * 512), 3 * 512); 1232 ut_expected_io_set_iov(expected_io, 1, (void *)0x30000, 11 * 512); 1233 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1234 1235 rc = spdk_bdev_writev_blocks_with_md(desc, io_ch, iov, 3, md_buf, 1236 14, 32, io_done, NULL); 1237 CU_ASSERT(rc == 0); 1238 CU_ASSERT(g_io_done == false); 1239 1240 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 1241 stub_complete_io(3); 1242 CU_ASSERT(g_io_done == true); 1243 1244 /* Test multi vector command that needs to be split by strip and then needs to be 1245 * split further due to the capacity of child iovs. 1246 */ 1247 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV * 2; i++) { 1248 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1249 iov[i].iov_len = 512; 1250 } 1251 1252 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1253 g_io_done = false; 1254 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, BDEV_IO_NUM_CHILD_IOV, 1255 BDEV_IO_NUM_CHILD_IOV); 1256 expected_io->md_buf = md_buf; 1257 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1258 ut_expected_io_set_iov(expected_io, i, (void *)((i + 1) * 0x10000), 512); 1259 } 1260 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1261 1262 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1263 BDEV_IO_NUM_CHILD_IOV, BDEV_IO_NUM_CHILD_IOV); 1264 expected_io->md_buf = md_buf + BDEV_IO_NUM_CHILD_IOV * 8; 1265 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1266 ut_expected_io_set_iov(expected_io, i, 1267 (void *)((i + 1 + BDEV_IO_NUM_CHILD_IOV) * 0x10000), 512); 1268 } 1269 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1270 1271 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, md_buf, 1272 0, BDEV_IO_NUM_CHILD_IOV * 2, io_done, NULL); 1273 CU_ASSERT(rc == 0); 1274 CU_ASSERT(g_io_done == false); 1275 1276 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1277 stub_complete_io(1); 1278 CU_ASSERT(g_io_done == false); 1279 1280 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1281 stub_complete_io(1); 1282 CU_ASSERT(g_io_done == true); 1283 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1284 1285 /* Test multi vector command that needs to be split by strip and then needs to be 1286 * split further due to the capacity of child iovs. In this case, the length of 1287 * the rest of iovec array with an I/O boundary is the multiple of block size. 1288 */ 1289 1290 /* Fill iovec array for exactly one boundary. The iovec cnt for this boundary 1291 * is BDEV_IO_NUM_CHILD_IOV + 1, which exceeds the capacity of child iovs. 1292 */ 1293 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1294 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1295 iov[i].iov_len = 512; 1296 } 1297 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1298 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1299 iov[i].iov_len = 256; 1300 } 1301 iov[BDEV_IO_NUM_CHILD_IOV].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000); 1302 iov[BDEV_IO_NUM_CHILD_IOV].iov_len = 512; 1303 1304 /* Add an extra iovec to trigger split */ 1305 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000); 1306 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_len = 512; 1307 1308 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1309 g_io_done = false; 1310 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 1311 BDEV_IO_NUM_CHILD_IOV - 1, BDEV_IO_NUM_CHILD_IOV); 1312 expected_io->md_buf = md_buf; 1313 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1314 ut_expected_io_set_iov(expected_io, i, 1315 (void *)((i + 1) * 0x10000), 512); 1316 } 1317 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1318 ut_expected_io_set_iov(expected_io, i, 1319 (void *)((i + 1) * 0x10000), 256); 1320 } 1321 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1322 1323 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV - 1, 1324 1, 1); 1325 expected_io->md_buf = md_buf + (BDEV_IO_NUM_CHILD_IOV - 1) * 8; 1326 ut_expected_io_set_iov(expected_io, 0, 1327 (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000), 512); 1328 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1329 1330 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1331 1, 1); 1332 expected_io->md_buf = md_buf + BDEV_IO_NUM_CHILD_IOV * 8; 1333 ut_expected_io_set_iov(expected_io, 0, 1334 (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000), 512); 1335 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1336 1337 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 2, md_buf, 1338 0, BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1339 CU_ASSERT(rc == 0); 1340 CU_ASSERT(g_io_done == false); 1341 1342 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1343 stub_complete_io(1); 1344 CU_ASSERT(g_io_done == false); 1345 1346 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1347 stub_complete_io(2); 1348 CU_ASSERT(g_io_done == true); 1349 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1350 1351 /* Test multi vector command that needs to be split by strip and then needs to be 1352 * split further due to the capacity of child iovs, the child request offset should 1353 * be rewind to last aligned offset and go success without error. 1354 */ 1355 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 1; i++) { 1356 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1357 iov[i].iov_len = 512; 1358 } 1359 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_base = (void *)(BDEV_IO_NUM_CHILD_IOV * 0x10000); 1360 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_len = 256; 1361 1362 iov[BDEV_IO_NUM_CHILD_IOV].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000); 1363 iov[BDEV_IO_NUM_CHILD_IOV].iov_len = 256; 1364 1365 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000); 1366 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_len = 512; 1367 1368 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1369 g_io_done = false; 1370 g_io_status = 0; 1371 /* The first expected io should be start from offset 0 to BDEV_IO_NUM_CHILD_IOV - 1 */ 1372 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 1373 BDEV_IO_NUM_CHILD_IOV - 1, BDEV_IO_NUM_CHILD_IOV - 1); 1374 expected_io->md_buf = md_buf; 1375 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 1; i++) { 1376 ut_expected_io_set_iov(expected_io, i, 1377 (void *)((i + 1) * 0x10000), 512); 1378 } 1379 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1380 /* The second expected io should be start from offset BDEV_IO_NUM_CHILD_IOV - 1 to BDEV_IO_NUM_CHILD_IOV */ 1381 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV - 1, 1382 1, 2); 1383 expected_io->md_buf = md_buf + (BDEV_IO_NUM_CHILD_IOV - 1) * 8; 1384 ut_expected_io_set_iov(expected_io, 0, 1385 (void *)(BDEV_IO_NUM_CHILD_IOV * 0x10000), 256); 1386 ut_expected_io_set_iov(expected_io, 1, 1387 (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000), 256); 1388 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1389 /* The third expected io should be start from offset BDEV_IO_NUM_CHILD_IOV to BDEV_IO_NUM_CHILD_IOV + 1 */ 1390 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1391 1, 1); 1392 expected_io->md_buf = md_buf + BDEV_IO_NUM_CHILD_IOV * 8; 1393 ut_expected_io_set_iov(expected_io, 0, 1394 (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000), 512); 1395 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1396 1397 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, md_buf, 1398 0, BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1399 CU_ASSERT(rc == 0); 1400 CU_ASSERT(g_io_done == false); 1401 1402 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1403 stub_complete_io(1); 1404 CU_ASSERT(g_io_done == false); 1405 1406 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1407 stub_complete_io(2); 1408 CU_ASSERT(g_io_done == true); 1409 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1410 1411 /* Test multi vector command that needs to be split due to the IO boundary and 1412 * the capacity of child iovs. Especially test the case when the command is 1413 * split due to the capacity of child iovs, the tail address is not aligned with 1414 * block size and is rewinded to the aligned address. 1415 * 1416 * The iovecs used in read request is complex but is based on the data 1417 * collected in the real issue. We change the base addresses but keep the lengths 1418 * not to loose the credibility of the test. 1419 */ 1420 bdev->optimal_io_boundary = 128; 1421 g_io_done = false; 1422 g_io_status = 0; 1423 1424 for (i = 0; i < 31; i++) { 1425 iov[i].iov_base = (void *)(0xFEED0000000 + (i << 20)); 1426 iov[i].iov_len = 1024; 1427 } 1428 iov[31].iov_base = (void *)0xFEED1F00000; 1429 iov[31].iov_len = 32768; 1430 iov[32].iov_base = (void *)0xFEED2000000; 1431 iov[32].iov_len = 160; 1432 iov[33].iov_base = (void *)0xFEED2100000; 1433 iov[33].iov_len = 4096; 1434 iov[34].iov_base = (void *)0xFEED2200000; 1435 iov[34].iov_len = 4096; 1436 iov[35].iov_base = (void *)0xFEED2300000; 1437 iov[35].iov_len = 4096; 1438 iov[36].iov_base = (void *)0xFEED2400000; 1439 iov[36].iov_len = 4096; 1440 iov[37].iov_base = (void *)0xFEED2500000; 1441 iov[37].iov_len = 4096; 1442 iov[38].iov_base = (void *)0xFEED2600000; 1443 iov[38].iov_len = 4096; 1444 iov[39].iov_base = (void *)0xFEED2700000; 1445 iov[39].iov_len = 4096; 1446 iov[40].iov_base = (void *)0xFEED2800000; 1447 iov[40].iov_len = 4096; 1448 iov[41].iov_base = (void *)0xFEED2900000; 1449 iov[41].iov_len = 4096; 1450 iov[42].iov_base = (void *)0xFEED2A00000; 1451 iov[42].iov_len = 4096; 1452 iov[43].iov_base = (void *)0xFEED2B00000; 1453 iov[43].iov_len = 12288; 1454 iov[44].iov_base = (void *)0xFEED2C00000; 1455 iov[44].iov_len = 8192; 1456 iov[45].iov_base = (void *)0xFEED2F00000; 1457 iov[45].iov_len = 4096; 1458 iov[46].iov_base = (void *)0xFEED3000000; 1459 iov[46].iov_len = 4096; 1460 iov[47].iov_base = (void *)0xFEED3100000; 1461 iov[47].iov_len = 4096; 1462 iov[48].iov_base = (void *)0xFEED3200000; 1463 iov[48].iov_len = 24576; 1464 iov[49].iov_base = (void *)0xFEED3300000; 1465 iov[49].iov_len = 16384; 1466 iov[50].iov_base = (void *)0xFEED3400000; 1467 iov[50].iov_len = 12288; 1468 iov[51].iov_base = (void *)0xFEED3500000; 1469 iov[51].iov_len = 4096; 1470 iov[52].iov_base = (void *)0xFEED3600000; 1471 iov[52].iov_len = 4096; 1472 iov[53].iov_base = (void *)0xFEED3700000; 1473 iov[53].iov_len = 4096; 1474 iov[54].iov_base = (void *)0xFEED3800000; 1475 iov[54].iov_len = 28672; 1476 iov[55].iov_base = (void *)0xFEED3900000; 1477 iov[55].iov_len = 20480; 1478 iov[56].iov_base = (void *)0xFEED3A00000; 1479 iov[56].iov_len = 4096; 1480 iov[57].iov_base = (void *)0xFEED3B00000; 1481 iov[57].iov_len = 12288; 1482 iov[58].iov_base = (void *)0xFEED3C00000; 1483 iov[58].iov_len = 4096; 1484 iov[59].iov_base = (void *)0xFEED3D00000; 1485 iov[59].iov_len = 4096; 1486 iov[60].iov_base = (void *)0xFEED3E00000; 1487 iov[60].iov_len = 352; 1488 1489 /* The 1st child IO must be from iov[0] to iov[31] split by the capacity 1490 * of child iovs, 1491 */ 1492 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 126, 32); 1493 expected_io->md_buf = md_buf; 1494 for (i = 0; i < 32; i++) { 1495 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 1496 } 1497 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1498 1499 /* The 2nd child IO must be from iov[32] to the first 864 bytes of iov[33] 1500 * split by the IO boundary requirement. 1501 */ 1502 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 126, 2, 2); 1503 expected_io->md_buf = md_buf + 126 * 8; 1504 ut_expected_io_set_iov(expected_io, 0, iov[32].iov_base, iov[32].iov_len); 1505 ut_expected_io_set_iov(expected_io, 1, iov[33].iov_base, 864); 1506 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1507 1508 /* The 3rd child IO must be from the remaining 3232 bytes of iov[33] to 1509 * the first 864 bytes of iov[46] split by the IO boundary requirement. 1510 */ 1511 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 128, 128, 14); 1512 expected_io->md_buf = md_buf + 128 * 8; 1513 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[33].iov_base + 864), 1514 iov[33].iov_len - 864); 1515 ut_expected_io_set_iov(expected_io, 1, iov[34].iov_base, iov[34].iov_len); 1516 ut_expected_io_set_iov(expected_io, 2, iov[35].iov_base, iov[35].iov_len); 1517 ut_expected_io_set_iov(expected_io, 3, iov[36].iov_base, iov[36].iov_len); 1518 ut_expected_io_set_iov(expected_io, 4, iov[37].iov_base, iov[37].iov_len); 1519 ut_expected_io_set_iov(expected_io, 5, iov[38].iov_base, iov[38].iov_len); 1520 ut_expected_io_set_iov(expected_io, 6, iov[39].iov_base, iov[39].iov_len); 1521 ut_expected_io_set_iov(expected_io, 7, iov[40].iov_base, iov[40].iov_len); 1522 ut_expected_io_set_iov(expected_io, 8, iov[41].iov_base, iov[41].iov_len); 1523 ut_expected_io_set_iov(expected_io, 9, iov[42].iov_base, iov[42].iov_len); 1524 ut_expected_io_set_iov(expected_io, 10, iov[43].iov_base, iov[43].iov_len); 1525 ut_expected_io_set_iov(expected_io, 11, iov[44].iov_base, iov[44].iov_len); 1526 ut_expected_io_set_iov(expected_io, 12, iov[45].iov_base, iov[45].iov_len); 1527 ut_expected_io_set_iov(expected_io, 13, iov[46].iov_base, 864); 1528 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1529 1530 /* The 4th child IO must be from the remaining 3232 bytes of iov[46] to the 1531 * first 864 bytes of iov[52] split by the IO boundary requirement. 1532 */ 1533 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 256, 128, 7); 1534 expected_io->md_buf = md_buf + 256 * 8; 1535 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[46].iov_base + 864), 1536 iov[46].iov_len - 864); 1537 ut_expected_io_set_iov(expected_io, 1, iov[47].iov_base, iov[47].iov_len); 1538 ut_expected_io_set_iov(expected_io, 2, iov[48].iov_base, iov[48].iov_len); 1539 ut_expected_io_set_iov(expected_io, 3, iov[49].iov_base, iov[49].iov_len); 1540 ut_expected_io_set_iov(expected_io, 4, iov[50].iov_base, iov[50].iov_len); 1541 ut_expected_io_set_iov(expected_io, 5, iov[51].iov_base, iov[51].iov_len); 1542 ut_expected_io_set_iov(expected_io, 6, iov[52].iov_base, 864); 1543 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1544 1545 /* The 5th child IO must be from the remaining 3232 bytes of iov[52] to 1546 * the first 4096 bytes of iov[57] split by the IO boundary requirement. 1547 */ 1548 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 384, 128, 6); 1549 expected_io->md_buf = md_buf + 384 * 8; 1550 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[52].iov_base + 864), 1551 iov[52].iov_len - 864); 1552 ut_expected_io_set_iov(expected_io, 1, iov[53].iov_base, iov[53].iov_len); 1553 ut_expected_io_set_iov(expected_io, 2, iov[54].iov_base, iov[54].iov_len); 1554 ut_expected_io_set_iov(expected_io, 3, iov[55].iov_base, iov[55].iov_len); 1555 ut_expected_io_set_iov(expected_io, 4, iov[56].iov_base, iov[56].iov_len); 1556 ut_expected_io_set_iov(expected_io, 5, iov[57].iov_base, 4960); 1557 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1558 1559 /* The 6th child IO must be from the remaining 7328 bytes of iov[57] 1560 * to the first 3936 bytes of iov[58] split by the capacity of child iovs. 1561 */ 1562 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 512, 30, 3); 1563 expected_io->md_buf = md_buf + 512 * 8; 1564 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[57].iov_base + 4960), 1565 iov[57].iov_len - 4960); 1566 ut_expected_io_set_iov(expected_io, 1, iov[58].iov_base, iov[58].iov_len); 1567 ut_expected_io_set_iov(expected_io, 2, iov[59].iov_base, 3936); 1568 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1569 1570 /* The 7th child IO is from the remaining 160 bytes of iov[59] and iov[60]. */ 1571 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 542, 1, 2); 1572 expected_io->md_buf = md_buf + 542 * 8; 1573 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[59].iov_base + 3936), 1574 iov[59].iov_len - 3936); 1575 ut_expected_io_set_iov(expected_io, 1, iov[60].iov_base, iov[60].iov_len); 1576 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1577 1578 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, 61, md_buf, 1579 0, 543, io_done, NULL); 1580 CU_ASSERT(rc == 0); 1581 CU_ASSERT(g_io_done == false); 1582 1583 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1584 stub_complete_io(1); 1585 CU_ASSERT(g_io_done == false); 1586 1587 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 5); 1588 stub_complete_io(5); 1589 CU_ASSERT(g_io_done == false); 1590 1591 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1592 stub_complete_io(1); 1593 CU_ASSERT(g_io_done == true); 1594 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1595 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 1596 1597 /* Test a WRITE_ZEROES that would span an I/O boundary. WRITE_ZEROES should not be 1598 * split, so test that. 1599 */ 1600 bdev->optimal_io_boundary = 15; 1601 g_io_done = false; 1602 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, 9, 36, 0); 1603 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1604 1605 rc = spdk_bdev_write_zeroes_blocks(desc, io_ch, 9, 36, io_done, NULL); 1606 CU_ASSERT(rc == 0); 1607 CU_ASSERT(g_io_done == false); 1608 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1609 stub_complete_io(1); 1610 CU_ASSERT(g_io_done == true); 1611 1612 /* Test an UNMAP. This should also not be split. */ 1613 bdev->optimal_io_boundary = 16; 1614 g_io_done = false; 1615 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, 15, 2, 0); 1616 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1617 1618 rc = spdk_bdev_unmap_blocks(desc, io_ch, 15, 2, io_done, NULL); 1619 CU_ASSERT(rc == 0); 1620 CU_ASSERT(g_io_done == false); 1621 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1622 stub_complete_io(1); 1623 CU_ASSERT(g_io_done == true); 1624 1625 /* Test a FLUSH. This should also not be split. */ 1626 bdev->optimal_io_boundary = 16; 1627 g_io_done = false; 1628 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_FLUSH, 15, 2, 0); 1629 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1630 1631 rc = spdk_bdev_flush_blocks(desc, io_ch, 15, 2, io_done, NULL); 1632 CU_ASSERT(rc == 0); 1633 CU_ASSERT(g_io_done == false); 1634 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1635 stub_complete_io(1); 1636 CU_ASSERT(g_io_done == true); 1637 1638 CU_ASSERT(TAILQ_EMPTY(&g_bdev_ut_channel->expected_io)); 1639 1640 /* Children requests return an error status */ 1641 bdev->optimal_io_boundary = 16; 1642 iov[0].iov_base = (void *)0x10000; 1643 iov[0].iov_len = 512 * 64; 1644 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 1645 g_io_done = false; 1646 g_io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1647 1648 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, 1, 1, 64, io_done, NULL); 1649 CU_ASSERT(rc == 0); 1650 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 5); 1651 stub_complete_io(4); 1652 CU_ASSERT(g_io_done == false); 1653 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 1654 stub_complete_io(1); 1655 CU_ASSERT(g_io_done == true); 1656 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 1657 1658 /* Test if a multi vector command terminated with failure before continueing 1659 * splitting process when one of child I/O failed. 1660 * The multi vector command is as same as the above that needs to be split by strip 1661 * and then needs to be split further due to the capacity of child iovs. 1662 */ 1663 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 1; i++) { 1664 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1665 iov[i].iov_len = 512; 1666 } 1667 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_base = (void *)(BDEV_IO_NUM_CHILD_IOV * 0x10000); 1668 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_len = 256; 1669 1670 iov[BDEV_IO_NUM_CHILD_IOV].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000); 1671 iov[BDEV_IO_NUM_CHILD_IOV].iov_len = 256; 1672 1673 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000); 1674 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_len = 512; 1675 1676 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1677 1678 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 1679 g_io_done = false; 1680 g_io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1681 1682 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, 0, 1683 BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1684 CU_ASSERT(rc == 0); 1685 CU_ASSERT(g_io_done == false); 1686 1687 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1688 stub_complete_io(1); 1689 CU_ASSERT(g_io_done == true); 1690 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 1691 1692 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1693 1694 /* for this test we will create the following conditions to hit the code path where 1695 * we are trying to send and IO following a split that has no iovs because we had to 1696 * trim them for alignment reasons. 1697 * 1698 * - 16K boundary, our IO will start at offset 0 with a length of 0x4200 1699 * - Our IOVs are 0x212 in size so that we run into the 16K boundary at child IOV 1700 * position 30 and overshoot by 0x2e. 1701 * - That means we'll send the IO and loop back to pick up the remaining bytes at 1702 * child IOV index 31. When we do, we find that we have to shorten index 31 by 0x2e 1703 * which eliniates that vector so we just send the first split IO with 30 vectors 1704 * and let the completion pick up the last 2 vectors. 1705 */ 1706 bdev->optimal_io_boundary = 32; 1707 bdev->split_on_optimal_io_boundary = true; 1708 g_io_done = false; 1709 1710 /* Init all parent IOVs to 0x212 */ 1711 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV + 2; i++) { 1712 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1713 iov[i].iov_len = 0x212; 1714 } 1715 1716 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, BDEV_IO_NUM_CHILD_IOV, 1717 BDEV_IO_NUM_CHILD_IOV - 1); 1718 /* expect 0-29 to be 1:1 with the parent iov */ 1719 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1720 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 1721 } 1722 1723 /* expect index 30 to be shortened to 0x1e4 (0x212 - 0x1e) because of the alignment 1724 * where 0x1e is the amount we overshot the 16K boundary 1725 */ 1726 ut_expected_io_set_iov(expected_io, BDEV_IO_NUM_CHILD_IOV - 2, 1727 (void *)(iov[BDEV_IO_NUM_CHILD_IOV - 2].iov_base), 0x1e4); 1728 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1729 1730 /* 2nd child IO will have 2 remaining vectors, one to pick up from the one that was 1731 * shortened that take it to the next boundary and then a final one to get us to 1732 * 0x4200 bytes for the IO. 1733 */ 1734 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1735 BDEV_IO_NUM_CHILD_IOV, 2); 1736 /* position 30 picked up the remaining bytes to the next boundary */ 1737 ut_expected_io_set_iov(expected_io, 0, 1738 (void *)(iov[BDEV_IO_NUM_CHILD_IOV - 2].iov_base + 0x1e4), 0x2e); 1739 1740 /* position 31 picked the the rest of the trasnfer to get us to 0x4200 */ 1741 ut_expected_io_set_iov(expected_io, 1, 1742 (void *)(iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_base), 0x1d2); 1743 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1744 1745 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 1, 0, 1746 BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1747 CU_ASSERT(rc == 0); 1748 CU_ASSERT(g_io_done == false); 1749 1750 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1751 stub_complete_io(1); 1752 CU_ASSERT(g_io_done == false); 1753 1754 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1755 stub_complete_io(1); 1756 CU_ASSERT(g_io_done == true); 1757 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1758 1759 spdk_put_io_channel(io_ch); 1760 spdk_bdev_close(desc); 1761 free_bdev(bdev); 1762 spdk_bdev_finish(bdev_fini_cb, NULL); 1763 poll_threads(); 1764 } 1765 1766 static void 1767 bdev_io_max_size_and_segment_split_test(void) 1768 { 1769 struct spdk_bdev *bdev; 1770 struct spdk_bdev_desc *desc = NULL; 1771 struct spdk_io_channel *io_ch; 1772 struct spdk_bdev_opts bdev_opts = {}; 1773 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 1774 struct ut_expected_io *expected_io; 1775 uint64_t i; 1776 int rc; 1777 1778 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 1779 bdev_opts.bdev_io_pool_size = 512; 1780 bdev_opts.bdev_io_cache_size = 64; 1781 1782 bdev_opts.opts_size = sizeof(bdev_opts); 1783 rc = spdk_bdev_set_opts(&bdev_opts); 1784 CU_ASSERT(rc == 0); 1785 spdk_bdev_initialize(bdev_init_cb, NULL); 1786 1787 bdev = allocate_bdev("bdev0"); 1788 1789 rc = spdk_bdev_open_ext(bdev->name, true, bdev_ut_event_cb, NULL, &desc); 1790 CU_ASSERT(rc == 0); 1791 SPDK_CU_ASSERT_FATAL(desc != NULL); 1792 io_ch = spdk_bdev_get_io_channel(desc); 1793 CU_ASSERT(io_ch != NULL); 1794 1795 bdev->split_on_optimal_io_boundary = false; 1796 bdev->optimal_io_boundary = 0; 1797 1798 /* Case 0 max_num_segments == 0. 1799 * but segment size 2 * 512 > 512 1800 */ 1801 bdev->max_segment_size = 512; 1802 bdev->max_num_segments = 0; 1803 g_io_done = false; 1804 1805 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 2); 1806 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512); 1807 ut_expected_io_set_iov(expected_io, 1, (void *)(0xF000 + 512), 512); 1808 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1809 1810 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 2, io_done, NULL); 1811 CU_ASSERT(rc == 0); 1812 CU_ASSERT(g_io_done == false); 1813 1814 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1815 stub_complete_io(1); 1816 CU_ASSERT(g_io_done == true); 1817 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1818 1819 /* Case 1 max_segment_size == 0 1820 * but iov num 2 > 1. 1821 */ 1822 bdev->max_segment_size = 0; 1823 bdev->max_num_segments = 1; 1824 g_io_done = false; 1825 1826 iov[0].iov_base = (void *)0x10000; 1827 iov[0].iov_len = 512; 1828 iov[1].iov_base = (void *)0x20000; 1829 iov[1].iov_len = 8 * 512; 1830 1831 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 1, 1); 1832 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, iov[0].iov_len); 1833 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1834 1835 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 15, 8, 1); 1836 ut_expected_io_set_iov(expected_io, 0, iov[1].iov_base, iov[1].iov_len); 1837 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1838 1839 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, 2, 14, 9, io_done, NULL); 1840 CU_ASSERT(rc == 0); 1841 CU_ASSERT(g_io_done == false); 1842 1843 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1844 stub_complete_io(2); 1845 CU_ASSERT(g_io_done == true); 1846 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1847 1848 /* Test that a non-vector command is split correctly. 1849 * Set up the expected values before calling spdk_bdev_read_blocks 1850 */ 1851 bdev->max_segment_size = 512; 1852 bdev->max_num_segments = 1; 1853 g_io_done = false; 1854 1855 /* Child IO 0 */ 1856 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 1, 1); 1857 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512); 1858 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1859 1860 /* Child IO 1 */ 1861 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 15, 1, 1); 1862 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 1 * 512), 512); 1863 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1864 1865 /* spdk_bdev_read_blocks will submit the first child immediately. */ 1866 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 2, io_done, NULL); 1867 CU_ASSERT(rc == 0); 1868 CU_ASSERT(g_io_done == false); 1869 1870 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1871 stub_complete_io(2); 1872 CU_ASSERT(g_io_done == true); 1873 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1874 1875 /* Now set up a more complex, multi-vector command that needs to be split, 1876 * including splitting iovecs. 1877 */ 1878 bdev->max_segment_size = 2 * 512; 1879 bdev->max_num_segments = 1; 1880 g_io_done = false; 1881 1882 iov[0].iov_base = (void *)0x10000; 1883 iov[0].iov_len = 2 * 512; 1884 iov[1].iov_base = (void *)0x20000; 1885 iov[1].iov_len = 4 * 512; 1886 iov[2].iov_base = (void *)0x30000; 1887 iov[2].iov_len = 6 * 512; 1888 1889 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 1); 1890 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, 512 * 2); 1891 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1892 1893 /* Split iov[1].size to 2 iov entries then split the segments */ 1894 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 2, 1); 1895 ut_expected_io_set_iov(expected_io, 0, iov[1].iov_base, 512 * 2); 1896 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1897 1898 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 18, 2, 1); 1899 ut_expected_io_set_iov(expected_io, 0, iov[1].iov_base + 512 * 2, 512 * 2); 1900 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1901 1902 /* Split iov[2].size to 3 iov entries then split the segments */ 1903 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 20, 2, 1); 1904 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base, 512 * 2); 1905 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1906 1907 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 22, 2, 1); 1908 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 2, 512 * 2); 1909 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1910 1911 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 24, 2, 1); 1912 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 4, 512 * 2); 1913 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1914 1915 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 3, 14, 12, io_done, NULL); 1916 CU_ASSERT(rc == 0); 1917 CU_ASSERT(g_io_done == false); 1918 1919 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 6); 1920 stub_complete_io(6); 1921 CU_ASSERT(g_io_done == true); 1922 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1923 1924 /* Test multi vector command that needs to be split by strip and then needs to be 1925 * split further due to the capacity of parent IO child iovs. 1926 */ 1927 bdev->max_segment_size = 512; 1928 bdev->max_num_segments = 1; 1929 g_io_done = false; 1930 1931 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1932 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1933 iov[i].iov_len = 512 * 2; 1934 } 1935 1936 /* Each input iov.size is split into 2 iovs, 1937 * half of the input iov can fill all child iov entries of a single IO. 1938 */ 1939 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV / 2; i++) { 1940 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 2 * i, 1, 1); 1941 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base, 512); 1942 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1943 1944 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 2 * i + 1, 1, 1); 1945 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base + 512, 512); 1946 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1947 } 1948 1949 /* The remaining iov is split in the second round */ 1950 for (i = BDEV_IO_NUM_CHILD_IOV / 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1951 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 2, 1, 1); 1952 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base, 512); 1953 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1954 1955 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 2 + 1, 1, 1); 1956 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base + 512, 512); 1957 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1958 } 1959 1960 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV, 0, 1961 BDEV_IO_NUM_CHILD_IOV * 2, io_done, NULL); 1962 CU_ASSERT(rc == 0); 1963 CU_ASSERT(g_io_done == false); 1964 1965 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == BDEV_IO_NUM_CHILD_IOV); 1966 stub_complete_io(BDEV_IO_NUM_CHILD_IOV); 1967 CU_ASSERT(g_io_done == false); 1968 1969 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == BDEV_IO_NUM_CHILD_IOV); 1970 stub_complete_io(BDEV_IO_NUM_CHILD_IOV); 1971 CU_ASSERT(g_io_done == true); 1972 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1973 1974 /* A wrong case, a child IO that is divided does 1975 * not meet the principle of multiples of block size, 1976 * and exits with error 1977 */ 1978 bdev->max_segment_size = 512; 1979 bdev->max_num_segments = 1; 1980 g_io_done = false; 1981 1982 iov[0].iov_base = (void *)0x10000; 1983 iov[0].iov_len = 512 + 256; 1984 iov[1].iov_base = (void *)0x20000; 1985 iov[1].iov_len = 256; 1986 1987 /* iov[0] is split to 512 and 256. 1988 * 256 is less than a block size, and it is found 1989 * in the next round of split that it is the first child IO smaller than 1990 * the block size, so the error exit 1991 */ 1992 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 1, 1); 1993 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, 512); 1994 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1995 1996 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, 2, 0, 2, io_done, NULL); 1997 CU_ASSERT(rc == 0); 1998 CU_ASSERT(g_io_done == false); 1999 2000 /* First child IO is OK */ 2001 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2002 stub_complete_io(1); 2003 CU_ASSERT(g_io_done == true); 2004 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2005 2006 /* error exit */ 2007 stub_complete_io(1); 2008 CU_ASSERT(g_io_done == true); 2009 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 2010 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2011 2012 /* Test multi vector command that needs to be split by strip and then needs to be 2013 * split further due to the capacity of child iovs. 2014 * 2015 * In this case, the last two iovs need to be split, but it will exceed the capacity 2016 * of child iovs, so it needs to wait until the first batch completed. 2017 */ 2018 bdev->max_segment_size = 512; 2019 bdev->max_num_segments = BDEV_IO_NUM_CHILD_IOV; 2020 g_io_done = false; 2021 2022 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2023 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2024 iov[i].iov_len = 512; 2025 } 2026 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 2027 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2028 iov[i].iov_len = 512 * 2; 2029 } 2030 2031 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 2032 BDEV_IO_NUM_CHILD_IOV, BDEV_IO_NUM_CHILD_IOV); 2033 /* 0 ~ (BDEV_IO_NUM_CHILD_IOV - 2) Will not be split */ 2034 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2035 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 2036 } 2037 /* (BDEV_IO_NUM_CHILD_IOV - 2) is split */ 2038 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, 512); 2039 ut_expected_io_set_iov(expected_io, i + 1, iov[i].iov_base + 512, 512); 2040 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2041 2042 /* Child iov entries exceed the max num of parent IO so split it in next round */ 2043 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 2, 2); 2044 ut_expected_io_set_iov(expected_io, 0, iov[i + 1].iov_base, 512); 2045 ut_expected_io_set_iov(expected_io, 1, iov[i + 1].iov_base + 512, 512); 2046 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2047 2048 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV, 0, 2049 BDEV_IO_NUM_CHILD_IOV + 2, io_done, NULL); 2050 CU_ASSERT(rc == 0); 2051 CU_ASSERT(g_io_done == false); 2052 2053 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2054 stub_complete_io(1); 2055 CU_ASSERT(g_io_done == false); 2056 2057 /* Next round */ 2058 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2059 stub_complete_io(1); 2060 CU_ASSERT(g_io_done == true); 2061 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2062 2063 /* This case is similar to the previous one, but the io composed of 2064 * the last few entries of child iov is not enough for a blocklen, so they 2065 * cannot be put into this IO, but wait until the next time. 2066 */ 2067 bdev->max_segment_size = 512; 2068 bdev->max_num_segments = BDEV_IO_NUM_CHILD_IOV; 2069 g_io_done = false; 2070 2071 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2072 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2073 iov[i].iov_len = 512; 2074 } 2075 2076 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV + 2; i++) { 2077 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2078 iov[i].iov_len = 128; 2079 } 2080 2081 /* First child iovcnt is't BDEV_IO_NUM_CHILD_IOV but BDEV_IO_NUM_CHILD_IOV - 2. 2082 * Because the left 2 iov is not enough for a blocklen. 2083 */ 2084 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 2085 BDEV_IO_NUM_CHILD_IOV - 2, BDEV_IO_NUM_CHILD_IOV - 2); 2086 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2087 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 2088 } 2089 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2090 2091 /* The second child io waits until the end of the first child io before executing. 2092 * Because the iovcnt of the two IOs exceeds the child iovcnt of the parent IO. 2093 * BDEV_IO_NUM_CHILD_IOV - 2 to BDEV_IO_NUM_CHILD_IOV + 2 2094 */ 2095 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV - 2, 2096 1, 4); 2097 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base, iov[i].iov_len); 2098 ut_expected_io_set_iov(expected_io, 1, iov[i + 1].iov_base, iov[i + 1].iov_len); 2099 ut_expected_io_set_iov(expected_io, 2, iov[i + 2].iov_base, iov[i + 2].iov_len); 2100 ut_expected_io_set_iov(expected_io, 3, iov[i + 3].iov_base, iov[i + 3].iov_len); 2101 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2102 2103 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 2, 0, 2104 BDEV_IO_NUM_CHILD_IOV - 1, io_done, NULL); 2105 CU_ASSERT(rc == 0); 2106 CU_ASSERT(g_io_done == false); 2107 2108 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2109 stub_complete_io(1); 2110 CU_ASSERT(g_io_done == false); 2111 2112 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2113 stub_complete_io(1); 2114 CU_ASSERT(g_io_done == true); 2115 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2116 2117 /* A very complicated case. Each sg entry exceeds max_segment_size and 2118 * needs to be split. At the same time, child io must be a multiple of blocklen. 2119 * At the same time, child iovcnt exceeds parent iovcnt. 2120 */ 2121 bdev->max_segment_size = 512 + 128; 2122 bdev->max_num_segments = 3; 2123 g_io_done = false; 2124 2125 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2126 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2127 iov[i].iov_len = 512 + 256; 2128 } 2129 2130 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV + 2; i++) { 2131 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2132 iov[i].iov_len = 512 + 128; 2133 } 2134 2135 /* Child IOs use 9 entries per for() round and 3 * 9 = 27 child iov entries. 2136 * Consume 4 parent IO iov entries per for() round and 6 block size. 2137 * Generate 9 child IOs. 2138 */ 2139 for (i = 0; i < 3; i++) { 2140 uint32_t j = i * 4; 2141 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 6, 2, 3); 2142 ut_expected_io_set_iov(expected_io, 0, iov[j].iov_base, 640); 2143 ut_expected_io_set_iov(expected_io, 1, iov[j].iov_base + 640, 128); 2144 ut_expected_io_set_iov(expected_io, 2, iov[j + 1].iov_base, 256); 2145 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2146 2147 /* Child io must be a multiple of blocklen 2148 * iov[j + 2] must be split. If the third entry is also added, 2149 * the multiple of blocklen cannot be guaranteed. But it still 2150 * occupies one iov entry of the parent child iov. 2151 */ 2152 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 6 + 2, 2, 2); 2153 ut_expected_io_set_iov(expected_io, 0, iov[j + 1].iov_base + 256, 512); 2154 ut_expected_io_set_iov(expected_io, 1, iov[j + 2].iov_base, 512); 2155 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2156 2157 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 6 + 4, 2, 3); 2158 ut_expected_io_set_iov(expected_io, 0, iov[j + 2].iov_base + 512, 256); 2159 ut_expected_io_set_iov(expected_io, 1, iov[j + 3].iov_base, 640); 2160 ut_expected_io_set_iov(expected_io, 2, iov[j + 3].iov_base + 640, 128); 2161 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2162 } 2163 2164 /* Child iov position at 27, the 10th child IO 2165 * iov entry index is 3 * 4 and offset is 3 * 6 2166 */ 2167 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 18, 2, 3); 2168 ut_expected_io_set_iov(expected_io, 0, iov[12].iov_base, 640); 2169 ut_expected_io_set_iov(expected_io, 1, iov[12].iov_base + 640, 128); 2170 ut_expected_io_set_iov(expected_io, 2, iov[13].iov_base, 256); 2171 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2172 2173 /* Child iov position at 30, the 11th child IO */ 2174 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 20, 2, 2); 2175 ut_expected_io_set_iov(expected_io, 0, iov[13].iov_base + 256, 512); 2176 ut_expected_io_set_iov(expected_io, 1, iov[14].iov_base, 512); 2177 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2178 2179 /* The 2nd split round and iovpos is 0, the 12th child IO */ 2180 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 22, 2, 3); 2181 ut_expected_io_set_iov(expected_io, 0, iov[14].iov_base + 512, 256); 2182 ut_expected_io_set_iov(expected_io, 1, iov[15].iov_base, 640); 2183 ut_expected_io_set_iov(expected_io, 2, iov[15].iov_base + 640, 128); 2184 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2185 2186 /* Consume 9 child IOs and 27 child iov entries. 2187 * Consume 4 parent IO iov entries per for() round and 6 block size. 2188 * Parent IO iov index start from 16 and block offset start from 24 2189 */ 2190 for (i = 0; i < 3; i++) { 2191 uint32_t j = i * 4 + 16; 2192 uint32_t offset = i * 6 + 24; 2193 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset, 2, 3); 2194 ut_expected_io_set_iov(expected_io, 0, iov[j].iov_base, 640); 2195 ut_expected_io_set_iov(expected_io, 1, iov[j].iov_base + 640, 128); 2196 ut_expected_io_set_iov(expected_io, 2, iov[j + 1].iov_base, 256); 2197 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2198 2199 /* Child io must be a multiple of blocklen 2200 * iov[j + 2] must be split. If the third entry is also added, 2201 * the multiple of blocklen cannot be guaranteed. But it still 2202 * occupies one iov entry of the parent child iov. 2203 */ 2204 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset + 2, 2, 2); 2205 ut_expected_io_set_iov(expected_io, 0, iov[j + 1].iov_base + 256, 512); 2206 ut_expected_io_set_iov(expected_io, 1, iov[j + 2].iov_base, 512); 2207 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2208 2209 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset + 4, 2, 3); 2210 ut_expected_io_set_iov(expected_io, 0, iov[j + 2].iov_base + 512, 256); 2211 ut_expected_io_set_iov(expected_io, 1, iov[j + 3].iov_base, 640); 2212 ut_expected_io_set_iov(expected_io, 2, iov[j + 3].iov_base + 640, 128); 2213 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2214 } 2215 2216 /* The 22th child IO, child iov position at 30 */ 2217 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 42, 1, 1); 2218 ut_expected_io_set_iov(expected_io, 0, iov[28].iov_base, 512); 2219 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2220 2221 /* The third round */ 2222 /* Here is the 23nd child IO and child iovpos is 0 */ 2223 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 43, 2, 3); 2224 ut_expected_io_set_iov(expected_io, 0, iov[28].iov_base + 512, 256); 2225 ut_expected_io_set_iov(expected_io, 1, iov[29].iov_base, 640); 2226 ut_expected_io_set_iov(expected_io, 2, iov[29].iov_base + 640, 128); 2227 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2228 2229 /* The 24th child IO */ 2230 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 45, 3, 3); 2231 ut_expected_io_set_iov(expected_io, 0, iov[30].iov_base, 640); 2232 ut_expected_io_set_iov(expected_io, 1, iov[31].iov_base, 640); 2233 ut_expected_io_set_iov(expected_io, 2, iov[32].iov_base, 256); 2234 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2235 2236 /* The 25th child IO */ 2237 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 48, 2, 2); 2238 ut_expected_io_set_iov(expected_io, 0, iov[32].iov_base + 256, 384); 2239 ut_expected_io_set_iov(expected_io, 1, iov[33].iov_base, 640); 2240 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2241 2242 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 2, 0, 2243 50, io_done, NULL); 2244 CU_ASSERT(rc == 0); 2245 CU_ASSERT(g_io_done == false); 2246 2247 /* Parent IO supports up to 32 child iovs, so it is calculated that 2248 * a maximum of 11 IOs can be split at a time, and the 2249 * splitting will continue after the first batch is over. 2250 */ 2251 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 11); 2252 stub_complete_io(11); 2253 CU_ASSERT(g_io_done == false); 2254 2255 /* The 2nd round */ 2256 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 11); 2257 stub_complete_io(11); 2258 CU_ASSERT(g_io_done == false); 2259 2260 /* The last round */ 2261 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 2262 stub_complete_io(3); 2263 CU_ASSERT(g_io_done == true); 2264 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2265 2266 /* Test an WRITE_ZEROES. This should also not be split. */ 2267 bdev->max_segment_size = 512; 2268 bdev->max_num_segments = 1; 2269 g_io_done = false; 2270 2271 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, 9, 36, 0); 2272 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2273 2274 rc = spdk_bdev_write_zeroes_blocks(desc, io_ch, 9, 36, io_done, NULL); 2275 CU_ASSERT(rc == 0); 2276 CU_ASSERT(g_io_done == false); 2277 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2278 stub_complete_io(1); 2279 CU_ASSERT(g_io_done == true); 2280 2281 /* Test an UNMAP. This should also not be split. */ 2282 g_io_done = false; 2283 2284 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, 15, 4, 0); 2285 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2286 2287 rc = spdk_bdev_unmap_blocks(desc, io_ch, 15, 4, io_done, NULL); 2288 CU_ASSERT(rc == 0); 2289 CU_ASSERT(g_io_done == false); 2290 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2291 stub_complete_io(1); 2292 CU_ASSERT(g_io_done == true); 2293 2294 /* Test a FLUSH. This should also not be split. */ 2295 g_io_done = false; 2296 2297 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_FLUSH, 15, 4, 0); 2298 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2299 2300 rc = spdk_bdev_flush_blocks(desc, io_ch, 15, 2, io_done, NULL); 2301 CU_ASSERT(rc == 0); 2302 CU_ASSERT(g_io_done == false); 2303 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2304 stub_complete_io(1); 2305 CU_ASSERT(g_io_done == true); 2306 2307 spdk_put_io_channel(io_ch); 2308 spdk_bdev_close(desc); 2309 free_bdev(bdev); 2310 spdk_bdev_finish(bdev_fini_cb, NULL); 2311 poll_threads(); 2312 } 2313 2314 static void 2315 bdev_io_mix_split_test(void) 2316 { 2317 struct spdk_bdev *bdev; 2318 struct spdk_bdev_desc *desc = NULL; 2319 struct spdk_io_channel *io_ch; 2320 struct spdk_bdev_opts bdev_opts = {}; 2321 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 2322 struct ut_expected_io *expected_io; 2323 uint64_t i; 2324 int rc; 2325 2326 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 2327 bdev_opts.bdev_io_pool_size = 512; 2328 bdev_opts.bdev_io_cache_size = 64; 2329 2330 rc = spdk_bdev_set_opts(&bdev_opts); 2331 CU_ASSERT(rc == 0); 2332 spdk_bdev_initialize(bdev_init_cb, NULL); 2333 2334 bdev = allocate_bdev("bdev0"); 2335 2336 rc = spdk_bdev_open_ext(bdev->name, true, bdev_ut_event_cb, NULL, &desc); 2337 CU_ASSERT(rc == 0); 2338 SPDK_CU_ASSERT_FATAL(desc != NULL); 2339 io_ch = spdk_bdev_get_io_channel(desc); 2340 CU_ASSERT(io_ch != NULL); 2341 2342 /* First case optimal_io_boundary == max_segment_size * max_num_segments */ 2343 bdev->split_on_optimal_io_boundary = true; 2344 bdev->optimal_io_boundary = 16; 2345 2346 bdev->max_segment_size = 512; 2347 bdev->max_num_segments = 16; 2348 g_io_done = false; 2349 2350 /* IO crossing the IO boundary requires split 2351 * Total 2 child IOs. 2352 */ 2353 2354 /* The 1st child IO split the segment_size to multiple segment entry */ 2355 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 2); 2356 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512); 2357 ut_expected_io_set_iov(expected_io, 1, (void *)(0xF000 + 512), 512); 2358 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2359 2360 /* The 2nd child IO split the segment_size to multiple segment entry */ 2361 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 2, 2); 2362 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 512); 2363 ut_expected_io_set_iov(expected_io, 1, (void *)(0xF000 + 3 * 512), 512); 2364 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2365 2366 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 4, io_done, NULL); 2367 CU_ASSERT(rc == 0); 2368 CU_ASSERT(g_io_done == false); 2369 2370 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2371 stub_complete_io(2); 2372 CU_ASSERT(g_io_done == true); 2373 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2374 2375 /* Second case optimal_io_boundary > max_segment_size * max_num_segments */ 2376 bdev->max_segment_size = 15 * 512; 2377 bdev->max_num_segments = 1; 2378 g_io_done = false; 2379 2380 /* IO crossing the IO boundary requires split. 2381 * The 1st child IO segment size exceeds the max_segment_size, 2382 * So 1st child IO will be splitted to multiple segment entry. 2383 * Then it split to 2 child IOs because of the max_num_segments. 2384 * Total 3 child IOs. 2385 */ 2386 2387 /* The first 2 IOs are in an IO boundary. 2388 * Because the optimal_io_boundary > max_segment_size * max_num_segments 2389 * So it split to the first 2 IOs. 2390 */ 2391 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 15, 1); 2392 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512 * 15); 2393 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2394 2395 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 15, 1, 1); 2396 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 512 * 15), 512); 2397 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2398 2399 /* The 3rd Child IO is because of the io boundary */ 2400 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 2, 1); 2401 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 512 * 16), 512 * 2); 2402 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2403 2404 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 0, 18, io_done, NULL); 2405 CU_ASSERT(rc == 0); 2406 CU_ASSERT(g_io_done == false); 2407 2408 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 2409 stub_complete_io(3); 2410 CU_ASSERT(g_io_done == true); 2411 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2412 2413 /* Third case optimal_io_boundary < max_segment_size * max_num_segments */ 2414 bdev->max_segment_size = 17 * 512; 2415 bdev->max_num_segments = 1; 2416 g_io_done = false; 2417 2418 /* IO crossing the IO boundary requires split. 2419 * Child IO does not split. 2420 * Total 2 child IOs. 2421 */ 2422 2423 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 16, 1); 2424 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512 * 16); 2425 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2426 2427 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 2, 1); 2428 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 512 * 16), 512 * 2); 2429 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2430 2431 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 0, 18, io_done, NULL); 2432 CU_ASSERT(rc == 0); 2433 CU_ASSERT(g_io_done == false); 2434 2435 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2436 stub_complete_io(2); 2437 CU_ASSERT(g_io_done == true); 2438 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2439 2440 /* Now set up a more complex, multi-vector command that needs to be split, 2441 * including splitting iovecs. 2442 * optimal_io_boundary < max_segment_size * max_num_segments 2443 */ 2444 bdev->max_segment_size = 3 * 512; 2445 bdev->max_num_segments = 6; 2446 g_io_done = false; 2447 2448 iov[0].iov_base = (void *)0x10000; 2449 iov[0].iov_len = 4 * 512; 2450 iov[1].iov_base = (void *)0x20000; 2451 iov[1].iov_len = 4 * 512; 2452 iov[2].iov_base = (void *)0x30000; 2453 iov[2].iov_len = 10 * 512; 2454 2455 /* IO crossing the IO boundary requires split. 2456 * The 1st child IO segment size exceeds the max_segment_size and after 2457 * splitting segment_size, the num_segments exceeds max_num_segments. 2458 * So 1st child IO will be splitted to 2 child IOs. 2459 * Total 3 child IOs. 2460 */ 2461 2462 /* The first 2 IOs are in an IO boundary. 2463 * After splitting segmemt size the segment num exceeds. 2464 * So it splits to 2 child IOs. 2465 */ 2466 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 0, 14, 6); 2467 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, 512 * 3); 2468 ut_expected_io_set_iov(expected_io, 1, iov[0].iov_base + 512 * 3, 512); 2469 ut_expected_io_set_iov(expected_io, 2, iov[1].iov_base, 512 * 3); 2470 ut_expected_io_set_iov(expected_io, 3, iov[1].iov_base + 512 * 3, 512); 2471 ut_expected_io_set_iov(expected_io, 4, iov[2].iov_base, 512 * 3); 2472 ut_expected_io_set_iov(expected_io, 5, iov[2].iov_base + 512 * 3, 512 * 3); 2473 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2474 2475 /* The 2nd child IO has the left segment entry */ 2476 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 1); 2477 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 6, 512 * 2); 2478 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2479 2480 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 2, 1); 2481 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 8, 512 * 2); 2482 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2483 2484 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 3, 0, 18, io_done, NULL); 2485 CU_ASSERT(rc == 0); 2486 CU_ASSERT(g_io_done == false); 2487 2488 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 2489 stub_complete_io(3); 2490 CU_ASSERT(g_io_done == true); 2491 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2492 2493 /* A very complicated case. Each sg entry exceeds max_segment_size 2494 * and split on io boundary. 2495 * optimal_io_boundary < max_segment_size * max_num_segments 2496 */ 2497 bdev->max_segment_size = 3 * 512; 2498 bdev->max_num_segments = BDEV_IO_NUM_CHILD_IOV; 2499 g_io_done = false; 2500 2501 for (i = 0; i < 20; i++) { 2502 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2503 iov[i].iov_len = 512 * 4; 2504 } 2505 2506 /* IO crossing the IO boundary requires split. 2507 * 80 block length can split 5 child IOs base on offset and IO boundary. 2508 * Each iov entry needs to be splitted to 2 entries because of max_segment_size 2509 * Total 5 child IOs. 2510 */ 2511 2512 /* 4 iov entries are in an IO boundary and each iov entry splits to 2. 2513 * So each child IO occupies 8 child iov entries. 2514 */ 2515 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 0, 16, 8); 2516 for (i = 0; i < 4; i++) { 2517 int iovcnt = i * 2; 2518 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2519 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2520 } 2521 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2522 2523 /* 2nd child IO and total 16 child iov entries of parent IO */ 2524 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 16, 8); 2525 for (i = 4; i < 8; i++) { 2526 int iovcnt = (i - 4) * 2; 2527 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2528 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2529 } 2530 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2531 2532 /* 3rd child IO and total 24 child iov entries of parent IO */ 2533 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 16, 8); 2534 for (i = 8; i < 12; i++) { 2535 int iovcnt = (i - 8) * 2; 2536 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2537 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2538 } 2539 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2540 2541 /* 4th child IO and total 32 child iov entries of parent IO */ 2542 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 48, 16, 8); 2543 for (i = 12; i < 16; i++) { 2544 int iovcnt = (i - 12) * 2; 2545 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2546 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2547 } 2548 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2549 2550 /* 5th child IO and because of the child iov entry it should be splitted 2551 * in next round. 2552 */ 2553 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 64, 16, 8); 2554 for (i = 16; i < 20; i++) { 2555 int iovcnt = (i - 16) * 2; 2556 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2557 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2558 } 2559 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2560 2561 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 20, 0, 80, io_done, NULL); 2562 CU_ASSERT(rc == 0); 2563 CU_ASSERT(g_io_done == false); 2564 2565 /* First split round */ 2566 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 2567 stub_complete_io(4); 2568 CU_ASSERT(g_io_done == false); 2569 2570 /* Second split round */ 2571 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2572 stub_complete_io(1); 2573 CU_ASSERT(g_io_done == true); 2574 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2575 2576 spdk_put_io_channel(io_ch); 2577 spdk_bdev_close(desc); 2578 free_bdev(bdev); 2579 spdk_bdev_finish(bdev_fini_cb, NULL); 2580 poll_threads(); 2581 } 2582 2583 static void 2584 bdev_io_split_with_io_wait(void) 2585 { 2586 struct spdk_bdev *bdev; 2587 struct spdk_bdev_desc *desc = NULL; 2588 struct spdk_io_channel *io_ch; 2589 struct spdk_bdev_channel *channel; 2590 struct spdk_bdev_mgmt_channel *mgmt_ch; 2591 struct spdk_bdev_opts bdev_opts = {}; 2592 struct iovec iov[3]; 2593 struct ut_expected_io *expected_io; 2594 int rc; 2595 2596 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 2597 bdev_opts.bdev_io_pool_size = 2; 2598 bdev_opts.bdev_io_cache_size = 1; 2599 2600 rc = spdk_bdev_set_opts(&bdev_opts); 2601 CU_ASSERT(rc == 0); 2602 spdk_bdev_initialize(bdev_init_cb, NULL); 2603 2604 bdev = allocate_bdev("bdev0"); 2605 2606 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 2607 CU_ASSERT(rc == 0); 2608 CU_ASSERT(desc != NULL); 2609 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 2610 io_ch = spdk_bdev_get_io_channel(desc); 2611 CU_ASSERT(io_ch != NULL); 2612 channel = spdk_io_channel_get_ctx(io_ch); 2613 mgmt_ch = channel->shared_resource->mgmt_ch; 2614 2615 bdev->optimal_io_boundary = 16; 2616 bdev->split_on_optimal_io_boundary = true; 2617 2618 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 2619 CU_ASSERT(rc == 0); 2620 2621 /* Now test that a single-vector command is split correctly. 2622 * Offset 14, length 8, payload 0xF000 2623 * Child - Offset 14, length 2, payload 0xF000 2624 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 2625 * 2626 * Set up the expected values before calling spdk_bdev_read_blocks 2627 */ 2628 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 1); 2629 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 2 * 512); 2630 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2631 2632 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 6, 1); 2633 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 6 * 512); 2634 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2635 2636 /* The following children will be submitted sequentially due to the capacity of 2637 * spdk_bdev_io. 2638 */ 2639 2640 /* The first child I/O will be queued to wait until an spdk_bdev_io becomes available */ 2641 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL); 2642 CU_ASSERT(rc == 0); 2643 CU_ASSERT(!TAILQ_EMPTY(&mgmt_ch->io_wait_queue)); 2644 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2645 2646 /* Completing the first read I/O will submit the first child */ 2647 stub_complete_io(1); 2648 CU_ASSERT(TAILQ_EMPTY(&mgmt_ch->io_wait_queue)); 2649 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2650 2651 /* Completing the first child will submit the second child */ 2652 stub_complete_io(1); 2653 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2654 2655 /* Complete the second child I/O. This should result in our callback getting 2656 * invoked since the parent I/O is now complete. 2657 */ 2658 stub_complete_io(1); 2659 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2660 2661 /* Now set up a more complex, multi-vector command that needs to be split, 2662 * including splitting iovecs. 2663 */ 2664 iov[0].iov_base = (void *)0x10000; 2665 iov[0].iov_len = 512; 2666 iov[1].iov_base = (void *)0x20000; 2667 iov[1].iov_len = 20 * 512; 2668 iov[2].iov_base = (void *)0x30000; 2669 iov[2].iov_len = 11 * 512; 2670 2671 g_io_done = false; 2672 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 2); 2673 ut_expected_io_set_iov(expected_io, 0, (void *)0x10000, 512); 2674 ut_expected_io_set_iov(expected_io, 1, (void *)0x20000, 512); 2675 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2676 2677 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 16, 1); 2678 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 512), 16 * 512); 2679 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2680 2681 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 14, 2); 2682 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 17 * 512), 3 * 512); 2683 ut_expected_io_set_iov(expected_io, 1, (void *)0x30000, 11 * 512); 2684 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2685 2686 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 3, 14, 32, io_done, NULL); 2687 CU_ASSERT(rc == 0); 2688 CU_ASSERT(g_io_done == false); 2689 2690 /* The following children will be submitted sequentially due to the capacity of 2691 * spdk_bdev_io. 2692 */ 2693 2694 /* Completing the first child will submit the second child */ 2695 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2696 stub_complete_io(1); 2697 CU_ASSERT(g_io_done == false); 2698 2699 /* Completing the second child will submit the third child */ 2700 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2701 stub_complete_io(1); 2702 CU_ASSERT(g_io_done == false); 2703 2704 /* Completing the third child will result in our callback getting invoked 2705 * since the parent I/O is now complete. 2706 */ 2707 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2708 stub_complete_io(1); 2709 CU_ASSERT(g_io_done == true); 2710 2711 CU_ASSERT(TAILQ_EMPTY(&g_bdev_ut_channel->expected_io)); 2712 2713 spdk_put_io_channel(io_ch); 2714 spdk_bdev_close(desc); 2715 free_bdev(bdev); 2716 spdk_bdev_finish(bdev_fini_cb, NULL); 2717 poll_threads(); 2718 } 2719 2720 static void 2721 bdev_io_alignment(void) 2722 { 2723 struct spdk_bdev *bdev; 2724 struct spdk_bdev_desc *desc = NULL; 2725 struct spdk_io_channel *io_ch; 2726 struct spdk_bdev_opts bdev_opts = {}; 2727 int rc; 2728 void *buf = NULL; 2729 struct iovec iovs[2]; 2730 int iovcnt; 2731 uint64_t alignment; 2732 2733 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 2734 bdev_opts.bdev_io_pool_size = 20; 2735 bdev_opts.bdev_io_cache_size = 2; 2736 2737 rc = spdk_bdev_set_opts(&bdev_opts); 2738 CU_ASSERT(rc == 0); 2739 spdk_bdev_initialize(bdev_init_cb, NULL); 2740 2741 fn_table.submit_request = stub_submit_request_get_buf; 2742 bdev = allocate_bdev("bdev0"); 2743 2744 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 2745 CU_ASSERT(rc == 0); 2746 CU_ASSERT(desc != NULL); 2747 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 2748 io_ch = spdk_bdev_get_io_channel(desc); 2749 CU_ASSERT(io_ch != NULL); 2750 2751 /* Create aligned buffer */ 2752 rc = posix_memalign(&buf, 4096, 8192); 2753 SPDK_CU_ASSERT_FATAL(rc == 0); 2754 2755 /* Pass aligned single buffer with no alignment required */ 2756 alignment = 1; 2757 bdev->required_alignment = spdk_u32log2(alignment); 2758 2759 rc = spdk_bdev_write_blocks(desc, io_ch, buf, 0, 1, io_done, NULL); 2760 CU_ASSERT(rc == 0); 2761 stub_complete_io(1); 2762 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2763 alignment)); 2764 2765 rc = spdk_bdev_read_blocks(desc, io_ch, buf, 0, 1, io_done, NULL); 2766 CU_ASSERT(rc == 0); 2767 stub_complete_io(1); 2768 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2769 alignment)); 2770 2771 /* Pass unaligned single buffer with no alignment required */ 2772 alignment = 1; 2773 bdev->required_alignment = spdk_u32log2(alignment); 2774 2775 rc = spdk_bdev_write_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 2776 CU_ASSERT(rc == 0); 2777 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2778 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == buf + 4); 2779 stub_complete_io(1); 2780 2781 rc = spdk_bdev_read_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 2782 CU_ASSERT(rc == 0); 2783 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2784 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == buf + 4); 2785 stub_complete_io(1); 2786 2787 /* Pass unaligned single buffer with 512 alignment required */ 2788 alignment = 512; 2789 bdev->required_alignment = spdk_u32log2(alignment); 2790 2791 rc = spdk_bdev_write_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 2792 CU_ASSERT(rc == 0); 2793 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 2794 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2795 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2796 alignment)); 2797 stub_complete_io(1); 2798 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2799 2800 rc = spdk_bdev_read_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 2801 CU_ASSERT(rc == 0); 2802 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 2803 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2804 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2805 alignment)); 2806 stub_complete_io(1); 2807 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2808 2809 /* Pass unaligned single buffer with 4096 alignment required */ 2810 alignment = 4096; 2811 bdev->required_alignment = spdk_u32log2(alignment); 2812 2813 rc = spdk_bdev_write_blocks(desc, io_ch, buf + 8, 0, 1, io_done, NULL); 2814 CU_ASSERT(rc == 0); 2815 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 2816 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2817 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2818 alignment)); 2819 stub_complete_io(1); 2820 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2821 2822 rc = spdk_bdev_read_blocks(desc, io_ch, buf + 8, 0, 1, io_done, NULL); 2823 CU_ASSERT(rc == 0); 2824 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 2825 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2826 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2827 alignment)); 2828 stub_complete_io(1); 2829 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2830 2831 /* Pass aligned iovs with no alignment required */ 2832 alignment = 1; 2833 bdev->required_alignment = spdk_u32log2(alignment); 2834 2835 iovcnt = 1; 2836 iovs[0].iov_base = buf; 2837 iovs[0].iov_len = 512; 2838 2839 rc = spdk_bdev_writev(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2840 CU_ASSERT(rc == 0); 2841 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2842 stub_complete_io(1); 2843 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 2844 2845 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2846 CU_ASSERT(rc == 0); 2847 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2848 stub_complete_io(1); 2849 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 2850 2851 /* Pass unaligned iovs with no alignment required */ 2852 alignment = 1; 2853 bdev->required_alignment = spdk_u32log2(alignment); 2854 2855 iovcnt = 2; 2856 iovs[0].iov_base = buf + 16; 2857 iovs[0].iov_len = 256; 2858 iovs[1].iov_base = buf + 16 + 256 + 32; 2859 iovs[1].iov_len = 256; 2860 2861 rc = spdk_bdev_writev(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2862 CU_ASSERT(rc == 0); 2863 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2864 stub_complete_io(1); 2865 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 2866 2867 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2868 CU_ASSERT(rc == 0); 2869 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2870 stub_complete_io(1); 2871 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 2872 2873 /* Pass unaligned iov with 2048 alignment required */ 2874 alignment = 2048; 2875 bdev->required_alignment = spdk_u32log2(alignment); 2876 2877 iovcnt = 2; 2878 iovs[0].iov_base = buf + 16; 2879 iovs[0].iov_len = 256; 2880 iovs[1].iov_base = buf + 16 + 256 + 32; 2881 iovs[1].iov_len = 256; 2882 2883 rc = spdk_bdev_writev(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2884 CU_ASSERT(rc == 0); 2885 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == iovcnt); 2886 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2887 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2888 alignment)); 2889 stub_complete_io(1); 2890 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2891 2892 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2893 CU_ASSERT(rc == 0); 2894 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == iovcnt); 2895 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 2896 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2897 alignment)); 2898 stub_complete_io(1); 2899 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2900 2901 /* Pass iov without allocated buffer without alignment required */ 2902 alignment = 1; 2903 bdev->required_alignment = spdk_u32log2(alignment); 2904 2905 iovcnt = 1; 2906 iovs[0].iov_base = NULL; 2907 iovs[0].iov_len = 0; 2908 2909 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2910 CU_ASSERT(rc == 0); 2911 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2912 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2913 alignment)); 2914 stub_complete_io(1); 2915 2916 /* Pass iov without allocated buffer with 1024 alignment required */ 2917 alignment = 1024; 2918 bdev->required_alignment = spdk_u32log2(alignment); 2919 2920 iovcnt = 1; 2921 iovs[0].iov_base = NULL; 2922 iovs[0].iov_len = 0; 2923 2924 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 2925 CU_ASSERT(rc == 0); 2926 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 2927 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 2928 alignment)); 2929 stub_complete_io(1); 2930 2931 spdk_put_io_channel(io_ch); 2932 spdk_bdev_close(desc); 2933 free_bdev(bdev); 2934 fn_table.submit_request = stub_submit_request; 2935 spdk_bdev_finish(bdev_fini_cb, NULL); 2936 poll_threads(); 2937 2938 free(buf); 2939 } 2940 2941 static void 2942 bdev_io_alignment_with_boundary(void) 2943 { 2944 struct spdk_bdev *bdev; 2945 struct spdk_bdev_desc *desc = NULL; 2946 struct spdk_io_channel *io_ch; 2947 struct spdk_bdev_opts bdev_opts = {}; 2948 int rc; 2949 void *buf = NULL; 2950 struct iovec iovs[2]; 2951 int iovcnt; 2952 uint64_t alignment; 2953 2954 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 2955 bdev_opts.bdev_io_pool_size = 20; 2956 bdev_opts.bdev_io_cache_size = 2; 2957 2958 bdev_opts.opts_size = sizeof(bdev_opts); 2959 rc = spdk_bdev_set_opts(&bdev_opts); 2960 CU_ASSERT(rc == 0); 2961 spdk_bdev_initialize(bdev_init_cb, NULL); 2962 2963 fn_table.submit_request = stub_submit_request_get_buf; 2964 bdev = allocate_bdev("bdev0"); 2965 2966 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 2967 CU_ASSERT(rc == 0); 2968 CU_ASSERT(desc != NULL); 2969 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 2970 io_ch = spdk_bdev_get_io_channel(desc); 2971 CU_ASSERT(io_ch != NULL); 2972 2973 /* Create aligned buffer */ 2974 rc = posix_memalign(&buf, 4096, 131072); 2975 SPDK_CU_ASSERT_FATAL(rc == 0); 2976 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 2977 2978 /* 512 * 3 with 2 IO boundary, allocate small data buffer from bdev layer */ 2979 alignment = 512; 2980 bdev->required_alignment = spdk_u32log2(alignment); 2981 bdev->optimal_io_boundary = 2; 2982 bdev->split_on_optimal_io_boundary = true; 2983 2984 iovcnt = 1; 2985 iovs[0].iov_base = NULL; 2986 iovs[0].iov_len = 512 * 3; 2987 2988 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 3, io_done, NULL); 2989 CU_ASSERT(rc == 0); 2990 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2991 stub_complete_io(2); 2992 2993 /* 8KiB with 16 IO boundary, allocate large data buffer from bdev layer */ 2994 alignment = 512; 2995 bdev->required_alignment = spdk_u32log2(alignment); 2996 bdev->optimal_io_boundary = 16; 2997 bdev->split_on_optimal_io_boundary = true; 2998 2999 iovcnt = 1; 3000 iovs[0].iov_base = NULL; 3001 iovs[0].iov_len = 512 * 16; 3002 3003 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 16, io_done, NULL); 3004 CU_ASSERT(rc == 0); 3005 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 3006 stub_complete_io(2); 3007 3008 /* 512 * 160 with 128 IO boundary, 63.5KiB + 16.5KiB for the two children requests */ 3009 alignment = 512; 3010 bdev->required_alignment = spdk_u32log2(alignment); 3011 bdev->optimal_io_boundary = 128; 3012 bdev->split_on_optimal_io_boundary = true; 3013 3014 iovcnt = 1; 3015 iovs[0].iov_base = buf + 16; 3016 iovs[0].iov_len = 512 * 160; 3017 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 160, io_done, NULL); 3018 CU_ASSERT(rc == 0); 3019 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 3020 stub_complete_io(2); 3021 3022 /* 512 * 3 with 2 IO boundary */ 3023 alignment = 512; 3024 bdev->required_alignment = spdk_u32log2(alignment); 3025 bdev->optimal_io_boundary = 2; 3026 bdev->split_on_optimal_io_boundary = true; 3027 3028 iovcnt = 2; 3029 iovs[0].iov_base = buf + 16; 3030 iovs[0].iov_len = 512; 3031 iovs[1].iov_base = buf + 16 + 512 + 32; 3032 iovs[1].iov_len = 1024; 3033 3034 rc = spdk_bdev_writev_blocks(desc, io_ch, iovs, iovcnt, 1, 3, io_done, NULL); 3035 CU_ASSERT(rc == 0); 3036 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 3037 stub_complete_io(2); 3038 3039 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 3, io_done, NULL); 3040 CU_ASSERT(rc == 0); 3041 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 3042 stub_complete_io(2); 3043 3044 /* 512 * 64 with 32 IO boundary */ 3045 bdev->optimal_io_boundary = 32; 3046 iovcnt = 2; 3047 iovs[0].iov_base = buf + 16; 3048 iovs[0].iov_len = 16384; 3049 iovs[1].iov_base = buf + 16 + 16384 + 32; 3050 iovs[1].iov_len = 16384; 3051 3052 rc = spdk_bdev_writev_blocks(desc, io_ch, iovs, iovcnt, 1, 64, io_done, NULL); 3053 CU_ASSERT(rc == 0); 3054 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 3055 stub_complete_io(3); 3056 3057 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 64, io_done, NULL); 3058 CU_ASSERT(rc == 0); 3059 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 3060 stub_complete_io(3); 3061 3062 /* 512 * 160 with 32 IO boundary */ 3063 iovcnt = 1; 3064 iovs[0].iov_base = buf + 16; 3065 iovs[0].iov_len = 16384 + 65536; 3066 3067 rc = spdk_bdev_writev_blocks(desc, io_ch, iovs, iovcnt, 1, 160, io_done, NULL); 3068 CU_ASSERT(rc == 0); 3069 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 6); 3070 stub_complete_io(6); 3071 3072 spdk_put_io_channel(io_ch); 3073 spdk_bdev_close(desc); 3074 free_bdev(bdev); 3075 fn_table.submit_request = stub_submit_request; 3076 spdk_bdev_finish(bdev_fini_cb, NULL); 3077 poll_threads(); 3078 3079 free(buf); 3080 } 3081 3082 static void 3083 histogram_status_cb(void *cb_arg, int status) 3084 { 3085 g_status = status; 3086 } 3087 3088 static void 3089 histogram_data_cb(void *cb_arg, int status, struct spdk_histogram_data *histogram) 3090 { 3091 g_status = status; 3092 g_histogram = histogram; 3093 } 3094 3095 static void 3096 histogram_io_count(void *ctx, uint64_t start, uint64_t end, uint64_t count, 3097 uint64_t total, uint64_t so_far) 3098 { 3099 g_count += count; 3100 } 3101 3102 static void 3103 bdev_histograms(void) 3104 { 3105 struct spdk_bdev *bdev; 3106 struct spdk_bdev_desc *desc = NULL; 3107 struct spdk_io_channel *ch; 3108 struct spdk_histogram_data *histogram; 3109 uint8_t buf[4096]; 3110 int rc; 3111 3112 spdk_bdev_initialize(bdev_init_cb, NULL); 3113 3114 bdev = allocate_bdev("bdev"); 3115 3116 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3117 CU_ASSERT(rc == 0); 3118 CU_ASSERT(desc != NULL); 3119 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3120 3121 ch = spdk_bdev_get_io_channel(desc); 3122 CU_ASSERT(ch != NULL); 3123 3124 /* Enable histogram */ 3125 g_status = -1; 3126 spdk_bdev_histogram_enable(bdev, histogram_status_cb, NULL, true); 3127 poll_threads(); 3128 CU_ASSERT(g_status == 0); 3129 CU_ASSERT(bdev->internal.histogram_enabled == true); 3130 3131 /* Allocate histogram */ 3132 histogram = spdk_histogram_data_alloc(); 3133 SPDK_CU_ASSERT_FATAL(histogram != NULL); 3134 3135 /* Check if histogram is zeroed */ 3136 spdk_bdev_histogram_get(bdev, histogram, histogram_data_cb, NULL); 3137 poll_threads(); 3138 CU_ASSERT(g_status == 0); 3139 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 3140 3141 g_count = 0; 3142 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 3143 3144 CU_ASSERT(g_count == 0); 3145 3146 rc = spdk_bdev_write_blocks(desc, ch, buf, 0, 1, io_done, NULL); 3147 CU_ASSERT(rc == 0); 3148 3149 spdk_delay_us(10); 3150 stub_complete_io(1); 3151 poll_threads(); 3152 3153 rc = spdk_bdev_read_blocks(desc, ch, buf, 0, 1, io_done, NULL); 3154 CU_ASSERT(rc == 0); 3155 3156 spdk_delay_us(10); 3157 stub_complete_io(1); 3158 poll_threads(); 3159 3160 /* Check if histogram gathered data from all I/O channels */ 3161 g_histogram = NULL; 3162 spdk_bdev_histogram_get(bdev, histogram, histogram_data_cb, NULL); 3163 poll_threads(); 3164 CU_ASSERT(g_status == 0); 3165 CU_ASSERT(bdev->internal.histogram_enabled == true); 3166 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 3167 3168 g_count = 0; 3169 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 3170 CU_ASSERT(g_count == 2); 3171 3172 /* Disable histogram */ 3173 spdk_bdev_histogram_enable(bdev, histogram_status_cb, NULL, false); 3174 poll_threads(); 3175 CU_ASSERT(g_status == 0); 3176 CU_ASSERT(bdev->internal.histogram_enabled == false); 3177 3178 /* Try to run histogram commands on disabled bdev */ 3179 spdk_bdev_histogram_get(bdev, histogram, histogram_data_cb, NULL); 3180 poll_threads(); 3181 CU_ASSERT(g_status == -EFAULT); 3182 3183 spdk_histogram_data_free(histogram); 3184 spdk_put_io_channel(ch); 3185 spdk_bdev_close(desc); 3186 free_bdev(bdev); 3187 spdk_bdev_finish(bdev_fini_cb, NULL); 3188 poll_threads(); 3189 } 3190 3191 static void 3192 _bdev_compare(bool emulated) 3193 { 3194 struct spdk_bdev *bdev; 3195 struct spdk_bdev_desc *desc = NULL; 3196 struct spdk_io_channel *ioch; 3197 struct ut_expected_io *expected_io; 3198 uint64_t offset, num_blocks; 3199 uint32_t num_completed; 3200 char aa_buf[512]; 3201 char bb_buf[512]; 3202 struct iovec compare_iov; 3203 uint8_t io_type; 3204 int rc; 3205 3206 if (emulated) { 3207 io_type = SPDK_BDEV_IO_TYPE_READ; 3208 } else { 3209 io_type = SPDK_BDEV_IO_TYPE_COMPARE; 3210 } 3211 3212 memset(aa_buf, 0xaa, sizeof(aa_buf)); 3213 memset(bb_buf, 0xbb, sizeof(bb_buf)); 3214 3215 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = !emulated; 3216 3217 spdk_bdev_initialize(bdev_init_cb, NULL); 3218 fn_table.submit_request = stub_submit_request_get_buf; 3219 bdev = allocate_bdev("bdev"); 3220 3221 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3222 CU_ASSERT_EQUAL(rc, 0); 3223 SPDK_CU_ASSERT_FATAL(desc != NULL); 3224 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3225 ioch = spdk_bdev_get_io_channel(desc); 3226 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3227 3228 fn_table.submit_request = stub_submit_request_get_buf; 3229 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3230 3231 offset = 50; 3232 num_blocks = 1; 3233 compare_iov.iov_base = aa_buf; 3234 compare_iov.iov_len = sizeof(aa_buf); 3235 3236 expected_io = ut_alloc_expected_io(io_type, offset, num_blocks, 0); 3237 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3238 3239 g_io_done = false; 3240 g_compare_read_buf = aa_buf; 3241 g_compare_read_buf_len = sizeof(aa_buf); 3242 rc = spdk_bdev_comparev_blocks(desc, ioch, &compare_iov, 1, offset, num_blocks, io_done, NULL); 3243 CU_ASSERT_EQUAL(rc, 0); 3244 num_completed = stub_complete_io(1); 3245 CU_ASSERT_EQUAL(num_completed, 1); 3246 CU_ASSERT(g_io_done == true); 3247 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3248 3249 expected_io = ut_alloc_expected_io(io_type, offset, num_blocks, 0); 3250 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3251 3252 g_io_done = false; 3253 g_compare_read_buf = bb_buf; 3254 g_compare_read_buf_len = sizeof(bb_buf); 3255 rc = spdk_bdev_comparev_blocks(desc, ioch, &compare_iov, 1, offset, num_blocks, io_done, NULL); 3256 CU_ASSERT_EQUAL(rc, 0); 3257 num_completed = stub_complete_io(1); 3258 CU_ASSERT_EQUAL(num_completed, 1); 3259 CU_ASSERT(g_io_done == true); 3260 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_MISCOMPARE); 3261 3262 spdk_put_io_channel(ioch); 3263 spdk_bdev_close(desc); 3264 free_bdev(bdev); 3265 fn_table.submit_request = stub_submit_request; 3266 spdk_bdev_finish(bdev_fini_cb, NULL); 3267 poll_threads(); 3268 3269 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = true; 3270 3271 g_compare_read_buf = NULL; 3272 } 3273 3274 static void 3275 bdev_compare(void) 3276 { 3277 _bdev_compare(true); 3278 _bdev_compare(false); 3279 } 3280 3281 static void 3282 bdev_compare_and_write(void) 3283 { 3284 struct spdk_bdev *bdev; 3285 struct spdk_bdev_desc *desc = NULL; 3286 struct spdk_io_channel *ioch; 3287 struct ut_expected_io *expected_io; 3288 uint64_t offset, num_blocks; 3289 uint32_t num_completed; 3290 char aa_buf[512]; 3291 char bb_buf[512]; 3292 char cc_buf[512]; 3293 char write_buf[512]; 3294 struct iovec compare_iov; 3295 struct iovec write_iov; 3296 int rc; 3297 3298 memset(aa_buf, 0xaa, sizeof(aa_buf)); 3299 memset(bb_buf, 0xbb, sizeof(bb_buf)); 3300 memset(cc_buf, 0xcc, sizeof(cc_buf)); 3301 3302 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = false; 3303 3304 spdk_bdev_initialize(bdev_init_cb, NULL); 3305 fn_table.submit_request = stub_submit_request_get_buf; 3306 bdev = allocate_bdev("bdev"); 3307 3308 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3309 CU_ASSERT_EQUAL(rc, 0); 3310 SPDK_CU_ASSERT_FATAL(desc != NULL); 3311 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3312 ioch = spdk_bdev_get_io_channel(desc); 3313 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3314 3315 fn_table.submit_request = stub_submit_request_get_buf; 3316 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3317 3318 offset = 50; 3319 num_blocks = 1; 3320 compare_iov.iov_base = aa_buf; 3321 compare_iov.iov_len = sizeof(aa_buf); 3322 write_iov.iov_base = bb_buf; 3323 write_iov.iov_len = sizeof(bb_buf); 3324 3325 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset, num_blocks, 0); 3326 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3327 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, offset, num_blocks, 0); 3328 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3329 3330 g_io_done = false; 3331 g_compare_read_buf = aa_buf; 3332 g_compare_read_buf_len = sizeof(aa_buf); 3333 memset(write_buf, 0, sizeof(write_buf)); 3334 g_compare_write_buf = write_buf; 3335 g_compare_write_buf_len = sizeof(write_buf); 3336 rc = spdk_bdev_comparev_and_writev_blocks(desc, ioch, &compare_iov, 1, &write_iov, 1, 3337 offset, num_blocks, io_done, NULL); 3338 /* Trigger range locking */ 3339 poll_threads(); 3340 CU_ASSERT_EQUAL(rc, 0); 3341 num_completed = stub_complete_io(1); 3342 CU_ASSERT_EQUAL(num_completed, 1); 3343 CU_ASSERT(g_io_done == false); 3344 num_completed = stub_complete_io(1); 3345 /* Trigger range unlocking */ 3346 poll_threads(); 3347 CU_ASSERT_EQUAL(num_completed, 1); 3348 CU_ASSERT(g_io_done == true); 3349 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3350 CU_ASSERT(memcmp(write_buf, bb_buf, sizeof(write_buf)) == 0); 3351 3352 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset, num_blocks, 0); 3353 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3354 3355 g_io_done = false; 3356 g_compare_read_buf = cc_buf; 3357 g_compare_read_buf_len = sizeof(cc_buf); 3358 memset(write_buf, 0, sizeof(write_buf)); 3359 g_compare_write_buf = write_buf; 3360 g_compare_write_buf_len = sizeof(write_buf); 3361 rc = spdk_bdev_comparev_and_writev_blocks(desc, ioch, &compare_iov, 1, &write_iov, 1, 3362 offset, num_blocks, io_done, NULL); 3363 /* Trigger range locking */ 3364 poll_threads(); 3365 CU_ASSERT_EQUAL(rc, 0); 3366 num_completed = stub_complete_io(1); 3367 /* Trigger range unlocking earlier because we expect error here */ 3368 poll_threads(); 3369 CU_ASSERT_EQUAL(num_completed, 1); 3370 CU_ASSERT(g_io_done == true); 3371 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_MISCOMPARE); 3372 num_completed = stub_complete_io(1); 3373 CU_ASSERT_EQUAL(num_completed, 0); 3374 3375 spdk_put_io_channel(ioch); 3376 spdk_bdev_close(desc); 3377 free_bdev(bdev); 3378 fn_table.submit_request = stub_submit_request; 3379 spdk_bdev_finish(bdev_fini_cb, NULL); 3380 poll_threads(); 3381 3382 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = true; 3383 3384 g_compare_read_buf = NULL; 3385 g_compare_write_buf = NULL; 3386 } 3387 3388 static void 3389 bdev_write_zeroes(void) 3390 { 3391 struct spdk_bdev *bdev; 3392 struct spdk_bdev_desc *desc = NULL; 3393 struct spdk_io_channel *ioch; 3394 struct ut_expected_io *expected_io; 3395 uint64_t offset, num_io_blocks, num_blocks; 3396 uint32_t num_completed, num_requests; 3397 int rc; 3398 3399 spdk_bdev_initialize(bdev_init_cb, NULL); 3400 bdev = allocate_bdev("bdev"); 3401 3402 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3403 CU_ASSERT_EQUAL(rc, 0); 3404 SPDK_CU_ASSERT_FATAL(desc != NULL); 3405 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3406 ioch = spdk_bdev_get_io_channel(desc); 3407 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3408 3409 fn_table.submit_request = stub_submit_request; 3410 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3411 3412 /* First test that if the bdev supports write_zeroes, the request won't be split */ 3413 bdev->md_len = 0; 3414 bdev->blocklen = 4096; 3415 num_blocks = (ZERO_BUFFER_SIZE / bdev->blocklen) * 2; 3416 3417 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, 0, num_blocks, 0); 3418 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3419 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3420 CU_ASSERT_EQUAL(rc, 0); 3421 num_completed = stub_complete_io(1); 3422 CU_ASSERT_EQUAL(num_completed, 1); 3423 3424 /* Check that if write zeroes is not supported it'll be replaced by regular writes */ 3425 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, false); 3426 num_io_blocks = ZERO_BUFFER_SIZE / bdev->blocklen; 3427 num_requests = 2; 3428 num_blocks = (ZERO_BUFFER_SIZE / bdev->blocklen) * num_requests; 3429 3430 for (offset = 0; offset < num_requests; ++offset) { 3431 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 3432 offset * num_io_blocks, num_io_blocks, 0); 3433 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3434 } 3435 3436 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3437 CU_ASSERT_EQUAL(rc, 0); 3438 num_completed = stub_complete_io(num_requests); 3439 CU_ASSERT_EQUAL(num_completed, num_requests); 3440 3441 /* Check that the splitting is correct if bdev has interleaved metadata */ 3442 bdev->md_interleave = true; 3443 bdev->md_len = 64; 3444 bdev->blocklen = 4096 + 64; 3445 num_blocks = (ZERO_BUFFER_SIZE / bdev->blocklen) * 2; 3446 3447 num_requests = offset = 0; 3448 while (offset < num_blocks) { 3449 num_io_blocks = spdk_min(ZERO_BUFFER_SIZE / bdev->blocklen, num_blocks - offset); 3450 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 3451 offset, num_io_blocks, 0); 3452 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3453 offset += num_io_blocks; 3454 num_requests++; 3455 } 3456 3457 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3458 CU_ASSERT_EQUAL(rc, 0); 3459 num_completed = stub_complete_io(num_requests); 3460 CU_ASSERT_EQUAL(num_completed, num_requests); 3461 num_completed = stub_complete_io(num_requests); 3462 assert(num_completed == 0); 3463 3464 /* Check the the same for separate metadata buffer */ 3465 bdev->md_interleave = false; 3466 bdev->md_len = 64; 3467 bdev->blocklen = 4096; 3468 3469 num_requests = offset = 0; 3470 while (offset < num_blocks) { 3471 num_io_blocks = spdk_min(ZERO_BUFFER_SIZE / (bdev->blocklen + bdev->md_len), num_blocks); 3472 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 3473 offset, num_io_blocks, 0); 3474 expected_io->md_buf = (char *)g_bdev_mgr.zero_buffer + num_io_blocks * bdev->blocklen; 3475 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3476 offset += num_io_blocks; 3477 num_requests++; 3478 } 3479 3480 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3481 CU_ASSERT_EQUAL(rc, 0); 3482 num_completed = stub_complete_io(num_requests); 3483 CU_ASSERT_EQUAL(num_completed, num_requests); 3484 3485 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, true); 3486 spdk_put_io_channel(ioch); 3487 spdk_bdev_close(desc); 3488 free_bdev(bdev); 3489 spdk_bdev_finish(bdev_fini_cb, NULL); 3490 poll_threads(); 3491 } 3492 3493 static void 3494 bdev_zcopy_write(void) 3495 { 3496 struct spdk_bdev *bdev; 3497 struct spdk_bdev_desc *desc = NULL; 3498 struct spdk_io_channel *ioch; 3499 struct ut_expected_io *expected_io; 3500 uint64_t offset, num_blocks; 3501 uint32_t num_completed; 3502 char aa_buf[512]; 3503 struct iovec iov; 3504 int rc; 3505 const bool populate = false; 3506 const bool commit = true; 3507 3508 memset(aa_buf, 0xaa, sizeof(aa_buf)); 3509 3510 spdk_bdev_initialize(bdev_init_cb, NULL); 3511 bdev = allocate_bdev("bdev"); 3512 3513 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3514 CU_ASSERT_EQUAL(rc, 0); 3515 SPDK_CU_ASSERT_FATAL(desc != NULL); 3516 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3517 ioch = spdk_bdev_get_io_channel(desc); 3518 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3519 3520 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3521 3522 offset = 50; 3523 num_blocks = 1; 3524 iov.iov_base = NULL; 3525 iov.iov_len = 0; 3526 3527 g_zcopy_read_buf = (void *) 0x1122334455667788UL; 3528 g_zcopy_read_buf_len = (uint32_t) -1; 3529 /* Do a zcopy start for a write (populate=false) */ 3530 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_ZCOPY, offset, num_blocks, 0); 3531 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3532 g_io_done = false; 3533 g_zcopy_write_buf = aa_buf; 3534 g_zcopy_write_buf_len = sizeof(aa_buf); 3535 g_zcopy_bdev_io = NULL; 3536 rc = spdk_bdev_zcopy_start(desc, ioch, &iov, 1, offset, num_blocks, populate, io_done, NULL); 3537 CU_ASSERT_EQUAL(rc, 0); 3538 num_completed = stub_complete_io(1); 3539 CU_ASSERT_EQUAL(num_completed, 1); 3540 CU_ASSERT(g_io_done == true); 3541 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3542 /* Check that the iov has been set up */ 3543 CU_ASSERT(iov.iov_base == g_zcopy_write_buf); 3544 CU_ASSERT(iov.iov_len == g_zcopy_write_buf_len); 3545 /* Check that the bdev_io has been saved */ 3546 CU_ASSERT(g_zcopy_bdev_io != NULL); 3547 /* Now do the zcopy end for a write (commit=true) */ 3548 g_io_done = false; 3549 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_ZCOPY, offset, num_blocks, 0); 3550 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3551 rc = spdk_bdev_zcopy_end(g_zcopy_bdev_io, commit, io_done, NULL); 3552 CU_ASSERT_EQUAL(rc, 0); 3553 num_completed = stub_complete_io(1); 3554 CU_ASSERT_EQUAL(num_completed, 1); 3555 CU_ASSERT(g_io_done == true); 3556 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3557 /* Check the g_zcopy are reset by io_done */ 3558 CU_ASSERT(g_zcopy_write_buf == NULL); 3559 CU_ASSERT(g_zcopy_write_buf_len == 0); 3560 /* Check that io_done has freed the g_zcopy_bdev_io */ 3561 CU_ASSERT(g_zcopy_bdev_io == NULL); 3562 3563 /* Check the zcopy read buffer has not been touched which 3564 * ensures that the correct buffers were used. 3565 */ 3566 CU_ASSERT(g_zcopy_read_buf == (void *) 0x1122334455667788UL); 3567 CU_ASSERT(g_zcopy_read_buf_len == (uint32_t) -1); 3568 3569 spdk_put_io_channel(ioch); 3570 spdk_bdev_close(desc); 3571 free_bdev(bdev); 3572 spdk_bdev_finish(bdev_fini_cb, NULL); 3573 poll_threads(); 3574 } 3575 3576 static void 3577 bdev_zcopy_read(void) 3578 { 3579 struct spdk_bdev *bdev; 3580 struct spdk_bdev_desc *desc = NULL; 3581 struct spdk_io_channel *ioch; 3582 struct ut_expected_io *expected_io; 3583 uint64_t offset, num_blocks; 3584 uint32_t num_completed; 3585 char aa_buf[512]; 3586 struct iovec iov; 3587 int rc; 3588 const bool populate = true; 3589 const bool commit = false; 3590 3591 memset(aa_buf, 0xaa, sizeof(aa_buf)); 3592 3593 spdk_bdev_initialize(bdev_init_cb, NULL); 3594 bdev = allocate_bdev("bdev"); 3595 3596 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3597 CU_ASSERT_EQUAL(rc, 0); 3598 SPDK_CU_ASSERT_FATAL(desc != NULL); 3599 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3600 ioch = spdk_bdev_get_io_channel(desc); 3601 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3602 3603 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3604 3605 offset = 50; 3606 num_blocks = 1; 3607 iov.iov_base = NULL; 3608 iov.iov_len = 0; 3609 3610 g_zcopy_write_buf = (void *) 0x1122334455667788UL; 3611 g_zcopy_write_buf_len = (uint32_t) -1; 3612 3613 /* Do a zcopy start for a read (populate=true) */ 3614 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_ZCOPY, offset, num_blocks, 0); 3615 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3616 g_io_done = false; 3617 g_zcopy_read_buf = aa_buf; 3618 g_zcopy_read_buf_len = sizeof(aa_buf); 3619 g_zcopy_bdev_io = NULL; 3620 rc = spdk_bdev_zcopy_start(desc, ioch, &iov, 1, offset, num_blocks, populate, io_done, NULL); 3621 CU_ASSERT_EQUAL(rc, 0); 3622 num_completed = stub_complete_io(1); 3623 CU_ASSERT_EQUAL(num_completed, 1); 3624 CU_ASSERT(g_io_done == true); 3625 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3626 /* Check that the iov has been set up */ 3627 CU_ASSERT(iov.iov_base == g_zcopy_read_buf); 3628 CU_ASSERT(iov.iov_len == g_zcopy_read_buf_len); 3629 /* Check that the bdev_io has been saved */ 3630 CU_ASSERT(g_zcopy_bdev_io != NULL); 3631 3632 /* Now do the zcopy end for a read (commit=false) */ 3633 g_io_done = false; 3634 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_ZCOPY, offset, num_blocks, 0); 3635 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3636 rc = spdk_bdev_zcopy_end(g_zcopy_bdev_io, commit, io_done, NULL); 3637 CU_ASSERT_EQUAL(rc, 0); 3638 num_completed = stub_complete_io(1); 3639 CU_ASSERT_EQUAL(num_completed, 1); 3640 CU_ASSERT(g_io_done == true); 3641 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3642 /* Check the g_zcopy are reset by io_done */ 3643 CU_ASSERT(g_zcopy_read_buf == NULL); 3644 CU_ASSERT(g_zcopy_read_buf_len == 0); 3645 /* Check that io_done has freed the g_zcopy_bdev_io */ 3646 CU_ASSERT(g_zcopy_bdev_io == NULL); 3647 3648 /* Check the zcopy write buffer has not been touched which 3649 * ensures that the correct buffers were used. 3650 */ 3651 CU_ASSERT(g_zcopy_write_buf == (void *) 0x1122334455667788UL); 3652 CU_ASSERT(g_zcopy_write_buf_len == (uint32_t) -1); 3653 3654 spdk_put_io_channel(ioch); 3655 spdk_bdev_close(desc); 3656 free_bdev(bdev); 3657 spdk_bdev_finish(bdev_fini_cb, NULL); 3658 poll_threads(); 3659 } 3660 3661 static void 3662 bdev_open_while_hotremove(void) 3663 { 3664 struct spdk_bdev *bdev; 3665 struct spdk_bdev_desc *desc[2] = {}; 3666 int rc; 3667 3668 bdev = allocate_bdev("bdev"); 3669 3670 rc = spdk_bdev_open_ext("bdev", false, bdev_ut_event_cb, NULL, &desc[0]); 3671 CU_ASSERT(rc == 0); 3672 SPDK_CU_ASSERT_FATAL(desc[0] != NULL); 3673 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc[0])); 3674 3675 spdk_bdev_unregister(bdev, NULL, NULL); 3676 3677 rc = spdk_bdev_open_ext("bdev", false, bdev_ut_event_cb, NULL, &desc[1]); 3678 CU_ASSERT(rc == -ENODEV); 3679 SPDK_CU_ASSERT_FATAL(desc[1] == NULL); 3680 3681 spdk_bdev_close(desc[0]); 3682 free_bdev(bdev); 3683 } 3684 3685 static void 3686 bdev_close_while_hotremove(void) 3687 { 3688 struct spdk_bdev *bdev; 3689 struct spdk_bdev_desc *desc = NULL; 3690 int rc = 0; 3691 3692 bdev = allocate_bdev("bdev"); 3693 3694 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb1, &desc, &desc); 3695 CU_ASSERT_EQUAL(rc, 0); 3696 SPDK_CU_ASSERT_FATAL(desc != NULL); 3697 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3698 3699 /* Simulate hot-unplug by unregistering bdev */ 3700 g_event_type1 = 0xFF; 3701 g_unregister_arg = NULL; 3702 g_unregister_rc = -1; 3703 spdk_bdev_unregister(bdev, bdev_unregister_cb, (void *)0x12345678); 3704 /* Close device while remove event is in flight */ 3705 spdk_bdev_close(desc); 3706 3707 /* Ensure that unregister callback is delayed */ 3708 CU_ASSERT_EQUAL(g_unregister_arg, NULL); 3709 CU_ASSERT_EQUAL(g_unregister_rc, -1); 3710 3711 poll_threads(); 3712 3713 /* Event callback shall not be issued because device was closed */ 3714 CU_ASSERT_EQUAL(g_event_type1, 0xFF); 3715 /* Unregister callback is issued */ 3716 CU_ASSERT_EQUAL(g_unregister_arg, (void *)0x12345678); 3717 CU_ASSERT_EQUAL(g_unregister_rc, 0); 3718 3719 free_bdev(bdev); 3720 } 3721 3722 static void 3723 bdev_open_ext(void) 3724 { 3725 struct spdk_bdev *bdev; 3726 struct spdk_bdev_desc *desc1 = NULL; 3727 struct spdk_bdev_desc *desc2 = NULL; 3728 int rc = 0; 3729 3730 bdev = allocate_bdev("bdev"); 3731 3732 rc = spdk_bdev_open_ext("bdev", true, NULL, NULL, &desc1); 3733 CU_ASSERT_EQUAL(rc, -EINVAL); 3734 3735 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb1, &desc1, &desc1); 3736 CU_ASSERT_EQUAL(rc, 0); 3737 3738 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb2, &desc2, &desc2); 3739 CU_ASSERT_EQUAL(rc, 0); 3740 3741 g_event_type1 = 0xFF; 3742 g_event_type2 = 0xFF; 3743 3744 /* Simulate hot-unplug by unregistering bdev */ 3745 spdk_bdev_unregister(bdev, NULL, NULL); 3746 poll_threads(); 3747 3748 /* Check if correct events have been triggered in event callback fn */ 3749 CU_ASSERT_EQUAL(g_event_type1, SPDK_BDEV_EVENT_REMOVE); 3750 CU_ASSERT_EQUAL(g_event_type2, SPDK_BDEV_EVENT_REMOVE); 3751 3752 free_bdev(bdev); 3753 poll_threads(); 3754 } 3755 3756 struct timeout_io_cb_arg { 3757 struct iovec iov; 3758 uint8_t type; 3759 }; 3760 3761 static int 3762 bdev_channel_count_submitted_io(struct spdk_bdev_channel *ch) 3763 { 3764 struct spdk_bdev_io *bdev_io; 3765 int n = 0; 3766 3767 if (!ch) { 3768 return -1; 3769 } 3770 3771 TAILQ_FOREACH(bdev_io, &ch->io_submitted, internal.ch_link) { 3772 n++; 3773 } 3774 3775 return n; 3776 } 3777 3778 static void 3779 bdev_channel_io_timeout_cb(void *cb_arg, struct spdk_bdev_io *bdev_io) 3780 { 3781 struct timeout_io_cb_arg *ctx = cb_arg; 3782 3783 ctx->type = bdev_io->type; 3784 ctx->iov.iov_base = bdev_io->iov.iov_base; 3785 ctx->iov.iov_len = bdev_io->iov.iov_len; 3786 } 3787 3788 static void 3789 bdev_set_io_timeout(void) 3790 { 3791 struct spdk_bdev *bdev; 3792 struct spdk_bdev_desc *desc = NULL; 3793 struct spdk_io_channel *io_ch = NULL; 3794 struct spdk_bdev_channel *bdev_ch = NULL; 3795 struct timeout_io_cb_arg cb_arg; 3796 3797 spdk_bdev_initialize(bdev_init_cb, NULL); 3798 3799 bdev = allocate_bdev("bdev"); 3800 3801 CU_ASSERT(spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc) == 0); 3802 SPDK_CU_ASSERT_FATAL(desc != NULL); 3803 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3804 3805 io_ch = spdk_bdev_get_io_channel(desc); 3806 CU_ASSERT(io_ch != NULL); 3807 3808 bdev_ch = spdk_io_channel_get_ctx(io_ch); 3809 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->io_submitted)); 3810 3811 /* This is the part1. 3812 * We will check the bdev_ch->io_submitted list 3813 * TO make sure that it can link IOs and only the user submitted IOs 3814 */ 3815 CU_ASSERT(spdk_bdev_read(desc, io_ch, (void *)0x1000, 0, 4096, io_done, NULL) == 0); 3816 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 3817 CU_ASSERT(spdk_bdev_write(desc, io_ch, (void *)0x2000, 0, 4096, io_done, NULL) == 0); 3818 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 2); 3819 stub_complete_io(1); 3820 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 3821 stub_complete_io(1); 3822 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 0); 3823 3824 /* Split IO */ 3825 bdev->optimal_io_boundary = 16; 3826 bdev->split_on_optimal_io_boundary = true; 3827 3828 /* Now test that a single-vector command is split correctly. 3829 * Offset 14, length 8, payload 0xF000 3830 * Child - Offset 14, length 2, payload 0xF000 3831 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 3832 * 3833 * Set up the expected values before calling spdk_bdev_read_blocks 3834 */ 3835 CU_ASSERT(spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL) == 0); 3836 /* We count all submitted IOs including IO that are generated by splitting. */ 3837 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 3); 3838 stub_complete_io(1); 3839 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 2); 3840 stub_complete_io(1); 3841 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 0); 3842 3843 /* Also include the reset IO */ 3844 CU_ASSERT(spdk_bdev_reset(desc, io_ch, io_done, NULL) == 0); 3845 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 3846 poll_threads(); 3847 stub_complete_io(1); 3848 poll_threads(); 3849 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 0); 3850 3851 /* This is part2 3852 * Test the desc timeout poller register 3853 */ 3854 3855 /* Successfully set the timeout */ 3856 CU_ASSERT(spdk_bdev_set_timeout(desc, 30, bdev_channel_io_timeout_cb, &cb_arg) == 0); 3857 CU_ASSERT(desc->io_timeout_poller != NULL); 3858 CU_ASSERT(desc->timeout_in_sec == 30); 3859 CU_ASSERT(desc->cb_fn == bdev_channel_io_timeout_cb); 3860 CU_ASSERT(desc->cb_arg == &cb_arg); 3861 3862 /* Change the timeout limit */ 3863 CU_ASSERT(spdk_bdev_set_timeout(desc, 20, bdev_channel_io_timeout_cb, &cb_arg) == 0); 3864 CU_ASSERT(desc->io_timeout_poller != NULL); 3865 CU_ASSERT(desc->timeout_in_sec == 20); 3866 CU_ASSERT(desc->cb_fn == bdev_channel_io_timeout_cb); 3867 CU_ASSERT(desc->cb_arg == &cb_arg); 3868 3869 /* Disable the timeout */ 3870 CU_ASSERT(spdk_bdev_set_timeout(desc, 0, NULL, NULL) == 0); 3871 CU_ASSERT(desc->io_timeout_poller == NULL); 3872 3873 /* This the part3 3874 * We will test to catch timeout IO and check whether the IO is 3875 * the submitted one. 3876 */ 3877 memset(&cb_arg, 0, sizeof(cb_arg)); 3878 CU_ASSERT(spdk_bdev_set_timeout(desc, 30, bdev_channel_io_timeout_cb, &cb_arg) == 0); 3879 CU_ASSERT(spdk_bdev_write_blocks(desc, io_ch, (void *)0x1000, 0, 1, io_done, NULL) == 0); 3880 3881 /* Don't reach the limit */ 3882 spdk_delay_us(15 * spdk_get_ticks_hz()); 3883 poll_threads(); 3884 CU_ASSERT(cb_arg.type == 0); 3885 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 3886 CU_ASSERT(cb_arg.iov.iov_len == 0); 3887 3888 /* 15 + 15 = 30 reach the limit */ 3889 spdk_delay_us(15 * spdk_get_ticks_hz()); 3890 poll_threads(); 3891 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 3892 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x1000); 3893 CU_ASSERT(cb_arg.iov.iov_len == 1 * bdev->blocklen); 3894 stub_complete_io(1); 3895 3896 /* Use the same split IO above and check the IO */ 3897 memset(&cb_arg, 0, sizeof(cb_arg)); 3898 CU_ASSERT(spdk_bdev_write_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL) == 0); 3899 3900 /* The first child complete in time */ 3901 spdk_delay_us(15 * spdk_get_ticks_hz()); 3902 poll_threads(); 3903 stub_complete_io(1); 3904 CU_ASSERT(cb_arg.type == 0); 3905 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 3906 CU_ASSERT(cb_arg.iov.iov_len == 0); 3907 3908 /* The second child reach the limit */ 3909 spdk_delay_us(15 * spdk_get_ticks_hz()); 3910 poll_threads(); 3911 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 3912 CU_ASSERT(cb_arg.iov.iov_base == (void *)0xF000); 3913 CU_ASSERT(cb_arg.iov.iov_len == 8 * bdev->blocklen); 3914 stub_complete_io(1); 3915 3916 /* Also include the reset IO */ 3917 memset(&cb_arg, 0, sizeof(cb_arg)); 3918 CU_ASSERT(spdk_bdev_reset(desc, io_ch, io_done, NULL) == 0); 3919 spdk_delay_us(30 * spdk_get_ticks_hz()); 3920 poll_threads(); 3921 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_RESET); 3922 stub_complete_io(1); 3923 poll_threads(); 3924 3925 spdk_put_io_channel(io_ch); 3926 spdk_bdev_close(desc); 3927 free_bdev(bdev); 3928 spdk_bdev_finish(bdev_fini_cb, NULL); 3929 poll_threads(); 3930 } 3931 3932 static void 3933 lba_range_overlap(void) 3934 { 3935 struct lba_range r1, r2; 3936 3937 r1.offset = 100; 3938 r1.length = 50; 3939 3940 r2.offset = 0; 3941 r2.length = 1; 3942 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 3943 3944 r2.offset = 0; 3945 r2.length = 100; 3946 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 3947 3948 r2.offset = 0; 3949 r2.length = 110; 3950 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 3951 3952 r2.offset = 100; 3953 r2.length = 10; 3954 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 3955 3956 r2.offset = 110; 3957 r2.length = 20; 3958 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 3959 3960 r2.offset = 140; 3961 r2.length = 150; 3962 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 3963 3964 r2.offset = 130; 3965 r2.length = 200; 3966 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 3967 3968 r2.offset = 150; 3969 r2.length = 100; 3970 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 3971 3972 r2.offset = 110; 3973 r2.length = 0; 3974 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 3975 } 3976 3977 static bool g_lock_lba_range_done; 3978 static bool g_unlock_lba_range_done; 3979 3980 static void 3981 lock_lba_range_done(void *ctx, int status) 3982 { 3983 g_lock_lba_range_done = true; 3984 } 3985 3986 static void 3987 unlock_lba_range_done(void *ctx, int status) 3988 { 3989 g_unlock_lba_range_done = true; 3990 } 3991 3992 static void 3993 lock_lba_range_check_ranges(void) 3994 { 3995 struct spdk_bdev *bdev; 3996 struct spdk_bdev_desc *desc = NULL; 3997 struct spdk_io_channel *io_ch; 3998 struct spdk_bdev_channel *channel; 3999 struct lba_range *range; 4000 int ctx1; 4001 int rc; 4002 4003 spdk_bdev_initialize(bdev_init_cb, NULL); 4004 4005 bdev = allocate_bdev("bdev0"); 4006 4007 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 4008 CU_ASSERT(rc == 0); 4009 CU_ASSERT(desc != NULL); 4010 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4011 io_ch = spdk_bdev_get_io_channel(desc); 4012 CU_ASSERT(io_ch != NULL); 4013 channel = spdk_io_channel_get_ctx(io_ch); 4014 4015 g_lock_lba_range_done = false; 4016 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 4017 CU_ASSERT(rc == 0); 4018 poll_threads(); 4019 4020 CU_ASSERT(g_lock_lba_range_done == true); 4021 range = TAILQ_FIRST(&channel->locked_ranges); 4022 SPDK_CU_ASSERT_FATAL(range != NULL); 4023 CU_ASSERT(range->offset == 20); 4024 CU_ASSERT(range->length == 10); 4025 CU_ASSERT(range->owner_ch == channel); 4026 4027 /* Unlocks must exactly match a lock. */ 4028 g_unlock_lba_range_done = false; 4029 rc = bdev_unlock_lba_range(desc, io_ch, 20, 1, unlock_lba_range_done, &ctx1); 4030 CU_ASSERT(rc == -EINVAL); 4031 CU_ASSERT(g_unlock_lba_range_done == false); 4032 4033 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, unlock_lba_range_done, &ctx1); 4034 CU_ASSERT(rc == 0); 4035 spdk_delay_us(100); 4036 poll_threads(); 4037 4038 CU_ASSERT(g_unlock_lba_range_done == true); 4039 CU_ASSERT(TAILQ_EMPTY(&channel->locked_ranges)); 4040 4041 spdk_put_io_channel(io_ch); 4042 spdk_bdev_close(desc); 4043 free_bdev(bdev); 4044 spdk_bdev_finish(bdev_fini_cb, NULL); 4045 poll_threads(); 4046 } 4047 4048 static void 4049 lock_lba_range_with_io_outstanding(void) 4050 { 4051 struct spdk_bdev *bdev; 4052 struct spdk_bdev_desc *desc = NULL; 4053 struct spdk_io_channel *io_ch; 4054 struct spdk_bdev_channel *channel; 4055 struct lba_range *range; 4056 char buf[4096]; 4057 int ctx1; 4058 int rc; 4059 4060 spdk_bdev_initialize(bdev_init_cb, NULL); 4061 4062 bdev = allocate_bdev("bdev0"); 4063 4064 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 4065 CU_ASSERT(rc == 0); 4066 CU_ASSERT(desc != NULL); 4067 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4068 io_ch = spdk_bdev_get_io_channel(desc); 4069 CU_ASSERT(io_ch != NULL); 4070 channel = spdk_io_channel_get_ctx(io_ch); 4071 4072 g_io_done = false; 4073 rc = spdk_bdev_read_blocks(desc, io_ch, buf, 20, 1, io_done, &ctx1); 4074 CU_ASSERT(rc == 0); 4075 4076 g_lock_lba_range_done = false; 4077 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 4078 CU_ASSERT(rc == 0); 4079 poll_threads(); 4080 4081 /* The lock should immediately become valid, since there are no outstanding 4082 * write I/O. 4083 */ 4084 CU_ASSERT(g_io_done == false); 4085 CU_ASSERT(g_lock_lba_range_done == true); 4086 range = TAILQ_FIRST(&channel->locked_ranges); 4087 SPDK_CU_ASSERT_FATAL(range != NULL); 4088 CU_ASSERT(range->offset == 20); 4089 CU_ASSERT(range->length == 10); 4090 CU_ASSERT(range->owner_ch == channel); 4091 CU_ASSERT(range->locked_ctx == &ctx1); 4092 4093 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 4094 CU_ASSERT(rc == 0); 4095 stub_complete_io(1); 4096 spdk_delay_us(100); 4097 poll_threads(); 4098 4099 CU_ASSERT(TAILQ_EMPTY(&channel->locked_ranges)); 4100 4101 /* Now try again, but with a write I/O. */ 4102 g_io_done = false; 4103 rc = spdk_bdev_write_blocks(desc, io_ch, buf, 20, 1, io_done, &ctx1); 4104 CU_ASSERT(rc == 0); 4105 4106 g_lock_lba_range_done = false; 4107 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 4108 CU_ASSERT(rc == 0); 4109 poll_threads(); 4110 4111 /* The lock should not be fully valid yet, since a write I/O is outstanding. 4112 * But note that the range should be on the channel's locked_list, to make sure no 4113 * new write I/O are started. 4114 */ 4115 CU_ASSERT(g_io_done == false); 4116 CU_ASSERT(g_lock_lba_range_done == false); 4117 range = TAILQ_FIRST(&channel->locked_ranges); 4118 SPDK_CU_ASSERT_FATAL(range != NULL); 4119 CU_ASSERT(range->offset == 20); 4120 CU_ASSERT(range->length == 10); 4121 4122 /* Complete the write I/O. This should make the lock valid (checked by confirming 4123 * our callback was invoked). 4124 */ 4125 stub_complete_io(1); 4126 spdk_delay_us(100); 4127 poll_threads(); 4128 CU_ASSERT(g_io_done == true); 4129 CU_ASSERT(g_lock_lba_range_done == true); 4130 4131 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, unlock_lba_range_done, &ctx1); 4132 CU_ASSERT(rc == 0); 4133 poll_threads(); 4134 4135 CU_ASSERT(TAILQ_EMPTY(&channel->locked_ranges)); 4136 4137 spdk_put_io_channel(io_ch); 4138 spdk_bdev_close(desc); 4139 free_bdev(bdev); 4140 spdk_bdev_finish(bdev_fini_cb, NULL); 4141 poll_threads(); 4142 } 4143 4144 static void 4145 lock_lba_range_overlapped(void) 4146 { 4147 struct spdk_bdev *bdev; 4148 struct spdk_bdev_desc *desc = NULL; 4149 struct spdk_io_channel *io_ch; 4150 struct spdk_bdev_channel *channel; 4151 struct lba_range *range; 4152 int ctx1; 4153 int rc; 4154 4155 spdk_bdev_initialize(bdev_init_cb, NULL); 4156 4157 bdev = allocate_bdev("bdev0"); 4158 4159 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 4160 CU_ASSERT(rc == 0); 4161 CU_ASSERT(desc != NULL); 4162 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4163 io_ch = spdk_bdev_get_io_channel(desc); 4164 CU_ASSERT(io_ch != NULL); 4165 channel = spdk_io_channel_get_ctx(io_ch); 4166 4167 /* Lock range 20-29. */ 4168 g_lock_lba_range_done = false; 4169 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 4170 CU_ASSERT(rc == 0); 4171 poll_threads(); 4172 4173 CU_ASSERT(g_lock_lba_range_done == true); 4174 range = TAILQ_FIRST(&channel->locked_ranges); 4175 SPDK_CU_ASSERT_FATAL(range != NULL); 4176 CU_ASSERT(range->offset == 20); 4177 CU_ASSERT(range->length == 10); 4178 4179 /* Try to lock range 25-39. It should not lock immediately, since it overlaps with 4180 * 20-29. 4181 */ 4182 g_lock_lba_range_done = false; 4183 rc = bdev_lock_lba_range(desc, io_ch, 25, 15, lock_lba_range_done, &ctx1); 4184 CU_ASSERT(rc == 0); 4185 poll_threads(); 4186 4187 CU_ASSERT(g_lock_lba_range_done == false); 4188 range = TAILQ_FIRST(&bdev->internal.pending_locked_ranges); 4189 SPDK_CU_ASSERT_FATAL(range != NULL); 4190 CU_ASSERT(range->offset == 25); 4191 CU_ASSERT(range->length == 15); 4192 4193 /* Unlock 20-29. This should result in range 25-39 now getting locked since it 4194 * no longer overlaps with an active lock. 4195 */ 4196 g_unlock_lba_range_done = false; 4197 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, unlock_lba_range_done, &ctx1); 4198 CU_ASSERT(rc == 0); 4199 poll_threads(); 4200 4201 CU_ASSERT(g_unlock_lba_range_done == true); 4202 CU_ASSERT(TAILQ_EMPTY(&bdev->internal.pending_locked_ranges)); 4203 range = TAILQ_FIRST(&channel->locked_ranges); 4204 SPDK_CU_ASSERT_FATAL(range != NULL); 4205 CU_ASSERT(range->offset == 25); 4206 CU_ASSERT(range->length == 15); 4207 4208 /* Lock 40-59. This should immediately lock since it does not overlap with the 4209 * currently active 25-39 lock. 4210 */ 4211 g_lock_lba_range_done = false; 4212 rc = bdev_lock_lba_range(desc, io_ch, 40, 20, lock_lba_range_done, &ctx1); 4213 CU_ASSERT(rc == 0); 4214 poll_threads(); 4215 4216 CU_ASSERT(g_lock_lba_range_done == true); 4217 range = TAILQ_FIRST(&bdev->internal.locked_ranges); 4218 SPDK_CU_ASSERT_FATAL(range != NULL); 4219 range = TAILQ_NEXT(range, tailq); 4220 SPDK_CU_ASSERT_FATAL(range != NULL); 4221 CU_ASSERT(range->offset == 40); 4222 CU_ASSERT(range->length == 20); 4223 4224 /* Try to lock 35-44. Note that this overlaps with both 25-39 and 40-59. */ 4225 g_lock_lba_range_done = false; 4226 rc = bdev_lock_lba_range(desc, io_ch, 35, 10, lock_lba_range_done, &ctx1); 4227 CU_ASSERT(rc == 0); 4228 poll_threads(); 4229 4230 CU_ASSERT(g_lock_lba_range_done == false); 4231 range = TAILQ_FIRST(&bdev->internal.pending_locked_ranges); 4232 SPDK_CU_ASSERT_FATAL(range != NULL); 4233 CU_ASSERT(range->offset == 35); 4234 CU_ASSERT(range->length == 10); 4235 4236 /* Unlock 25-39. Make sure that 35-44 is still in the pending list, since 4237 * the 40-59 lock is still active. 4238 */ 4239 g_unlock_lba_range_done = false; 4240 rc = bdev_unlock_lba_range(desc, io_ch, 25, 15, unlock_lba_range_done, &ctx1); 4241 CU_ASSERT(rc == 0); 4242 poll_threads(); 4243 4244 CU_ASSERT(g_unlock_lba_range_done == true); 4245 CU_ASSERT(g_lock_lba_range_done == false); 4246 range = TAILQ_FIRST(&bdev->internal.pending_locked_ranges); 4247 SPDK_CU_ASSERT_FATAL(range != NULL); 4248 CU_ASSERT(range->offset == 35); 4249 CU_ASSERT(range->length == 10); 4250 4251 /* Unlock 40-59. This should result in 35-44 now getting locked, since there are 4252 * no longer any active overlapping locks. 4253 */ 4254 g_unlock_lba_range_done = false; 4255 rc = bdev_unlock_lba_range(desc, io_ch, 40, 20, unlock_lba_range_done, &ctx1); 4256 CU_ASSERT(rc == 0); 4257 poll_threads(); 4258 4259 CU_ASSERT(g_unlock_lba_range_done == true); 4260 CU_ASSERT(g_lock_lba_range_done == true); 4261 CU_ASSERT(TAILQ_EMPTY(&bdev->internal.pending_locked_ranges)); 4262 range = TAILQ_FIRST(&bdev->internal.locked_ranges); 4263 SPDK_CU_ASSERT_FATAL(range != NULL); 4264 CU_ASSERT(range->offset == 35); 4265 CU_ASSERT(range->length == 10); 4266 4267 /* Finally, unlock 35-44. */ 4268 g_unlock_lba_range_done = false; 4269 rc = bdev_unlock_lba_range(desc, io_ch, 35, 10, unlock_lba_range_done, &ctx1); 4270 CU_ASSERT(rc == 0); 4271 poll_threads(); 4272 4273 CU_ASSERT(g_unlock_lba_range_done == true); 4274 CU_ASSERT(TAILQ_EMPTY(&bdev->internal.locked_ranges)); 4275 4276 spdk_put_io_channel(io_ch); 4277 spdk_bdev_close(desc); 4278 free_bdev(bdev); 4279 spdk_bdev_finish(bdev_fini_cb, NULL); 4280 poll_threads(); 4281 } 4282 4283 static void 4284 abort_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 4285 { 4286 g_abort_done = true; 4287 g_abort_status = bdev_io->internal.status; 4288 spdk_bdev_free_io(bdev_io); 4289 } 4290 4291 static void 4292 bdev_io_abort(void) 4293 { 4294 struct spdk_bdev *bdev; 4295 struct spdk_bdev_desc *desc = NULL; 4296 struct spdk_io_channel *io_ch; 4297 struct spdk_bdev_channel *channel; 4298 struct spdk_bdev_mgmt_channel *mgmt_ch; 4299 struct spdk_bdev_opts bdev_opts = {}; 4300 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 4301 uint64_t io_ctx1 = 0, io_ctx2 = 0, i; 4302 int rc; 4303 4304 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 4305 bdev_opts.bdev_io_pool_size = 7; 4306 bdev_opts.bdev_io_cache_size = 2; 4307 4308 rc = spdk_bdev_set_opts(&bdev_opts); 4309 CU_ASSERT(rc == 0); 4310 spdk_bdev_initialize(bdev_init_cb, NULL); 4311 4312 bdev = allocate_bdev("bdev0"); 4313 4314 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 4315 CU_ASSERT(rc == 0); 4316 CU_ASSERT(desc != NULL); 4317 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4318 io_ch = spdk_bdev_get_io_channel(desc); 4319 CU_ASSERT(io_ch != NULL); 4320 channel = spdk_io_channel_get_ctx(io_ch); 4321 mgmt_ch = channel->shared_resource->mgmt_ch; 4322 4323 g_abort_done = false; 4324 4325 ut_enable_io_type(SPDK_BDEV_IO_TYPE_ABORT, false); 4326 4327 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4328 CU_ASSERT(rc == -ENOTSUP); 4329 4330 ut_enable_io_type(SPDK_BDEV_IO_TYPE_ABORT, true); 4331 4332 rc = spdk_bdev_abort(desc, io_ch, &io_ctx2, abort_done, NULL); 4333 CU_ASSERT(rc == 0); 4334 CU_ASSERT(g_abort_done == true); 4335 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_FAILED); 4336 4337 /* Test the case that the target I/O was successfully aborted. */ 4338 g_io_done = false; 4339 4340 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, &io_ctx1); 4341 CU_ASSERT(rc == 0); 4342 CU_ASSERT(g_io_done == false); 4343 4344 g_abort_done = false; 4345 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4346 4347 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4348 CU_ASSERT(rc == 0); 4349 CU_ASSERT(g_io_done == true); 4350 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 4351 stub_complete_io(1); 4352 CU_ASSERT(g_abort_done == true); 4353 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4354 4355 /* Test the case that the target I/O was not aborted because it completed 4356 * in the middle of execution of the abort. 4357 */ 4358 g_io_done = false; 4359 4360 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, &io_ctx1); 4361 CU_ASSERT(rc == 0); 4362 CU_ASSERT(g_io_done == false); 4363 4364 g_abort_done = false; 4365 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 4366 4367 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4368 CU_ASSERT(rc == 0); 4369 CU_ASSERT(g_io_done == false); 4370 4371 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4372 stub_complete_io(1); 4373 CU_ASSERT(g_io_done == true); 4374 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4375 4376 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 4377 stub_complete_io(1); 4378 CU_ASSERT(g_abort_done == true); 4379 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4380 4381 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4382 4383 bdev->optimal_io_boundary = 16; 4384 bdev->split_on_optimal_io_boundary = true; 4385 4386 /* Test that a single-vector command which is split is aborted correctly. 4387 * Offset 14, length 8, payload 0xF000 4388 * Child - Offset 14, length 2, payload 0xF000 4389 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 4390 */ 4391 g_io_done = false; 4392 4393 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, &io_ctx1); 4394 CU_ASSERT(rc == 0); 4395 CU_ASSERT(g_io_done == false); 4396 4397 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 4398 4399 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4400 4401 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4402 CU_ASSERT(rc == 0); 4403 CU_ASSERT(g_io_done == true); 4404 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 4405 stub_complete_io(2); 4406 CU_ASSERT(g_abort_done == true); 4407 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4408 4409 /* Test that a multi-vector command that needs to be split by strip and then 4410 * needs to be split is aborted correctly. Abort is requested before the second 4411 * child I/O was submitted. The parent I/O should complete with failure without 4412 * submitting the second child I/O. 4413 */ 4414 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV * 2; i++) { 4415 iov[i].iov_base = (void *)((i + 1) * 0x10000); 4416 iov[i].iov_len = 512; 4417 } 4418 4419 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 4420 g_io_done = false; 4421 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, 0, 4422 BDEV_IO_NUM_CHILD_IOV * 2, io_done, &io_ctx1); 4423 CU_ASSERT(rc == 0); 4424 CU_ASSERT(g_io_done == false); 4425 4426 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 4427 4428 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4429 4430 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4431 CU_ASSERT(rc == 0); 4432 CU_ASSERT(g_io_done == true); 4433 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 4434 stub_complete_io(1); 4435 CU_ASSERT(g_abort_done == true); 4436 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4437 4438 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4439 4440 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 4441 4442 bdev->optimal_io_boundary = 16; 4443 g_io_done = false; 4444 4445 /* Test that a ingle-vector command which is split is aborted correctly. 4446 * Differently from the above, the child abort request will be submitted 4447 * sequentially due to the capacity of spdk_bdev_io. 4448 */ 4449 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 50, io_done, &io_ctx1); 4450 CU_ASSERT(rc == 0); 4451 CU_ASSERT(g_io_done == false); 4452 4453 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 4454 4455 g_abort_done = false; 4456 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4457 4458 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 4459 CU_ASSERT(rc == 0); 4460 CU_ASSERT(!TAILQ_EMPTY(&mgmt_ch->io_wait_queue)); 4461 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 4462 4463 stub_complete_io(1); 4464 CU_ASSERT(g_io_done == true); 4465 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 4466 stub_complete_io(3); 4467 CU_ASSERT(g_abort_done == true); 4468 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4469 4470 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4471 4472 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 4473 4474 spdk_put_io_channel(io_ch); 4475 spdk_bdev_close(desc); 4476 free_bdev(bdev); 4477 spdk_bdev_finish(bdev_fini_cb, NULL); 4478 poll_threads(); 4479 } 4480 4481 static void 4482 bdev_unmap(void) 4483 { 4484 struct spdk_bdev *bdev; 4485 struct spdk_bdev_desc *desc = NULL; 4486 struct spdk_io_channel *ioch; 4487 struct spdk_bdev_channel *bdev_ch; 4488 struct ut_expected_io *expected_io; 4489 struct spdk_bdev_opts bdev_opts = {}; 4490 uint32_t i, num_outstanding; 4491 uint64_t offset, num_blocks, max_unmap_blocks, num_children; 4492 int rc; 4493 4494 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 4495 bdev_opts.bdev_io_pool_size = 512; 4496 bdev_opts.bdev_io_cache_size = 64; 4497 rc = spdk_bdev_set_opts(&bdev_opts); 4498 CU_ASSERT(rc == 0); 4499 4500 spdk_bdev_initialize(bdev_init_cb, NULL); 4501 bdev = allocate_bdev("bdev"); 4502 4503 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 4504 CU_ASSERT_EQUAL(rc, 0); 4505 SPDK_CU_ASSERT_FATAL(desc != NULL); 4506 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4507 ioch = spdk_bdev_get_io_channel(desc); 4508 SPDK_CU_ASSERT_FATAL(ioch != NULL); 4509 bdev_ch = spdk_io_channel_get_ctx(ioch); 4510 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->io_submitted)); 4511 4512 fn_table.submit_request = stub_submit_request; 4513 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4514 4515 /* Case 1: First test the request won't be split */ 4516 num_blocks = 32; 4517 4518 g_io_done = false; 4519 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, 0, num_blocks, 0); 4520 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 4521 rc = spdk_bdev_unmap_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 4522 CU_ASSERT_EQUAL(rc, 0); 4523 CU_ASSERT(g_io_done == false); 4524 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 4525 stub_complete_io(1); 4526 CU_ASSERT(g_io_done == true); 4527 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 4528 4529 /* Case 2: Test the split with 2 children requests */ 4530 bdev->max_unmap = 8; 4531 bdev->max_unmap_segments = 2; 4532 max_unmap_blocks = bdev->max_unmap * bdev->max_unmap_segments; 4533 num_blocks = max_unmap_blocks * 2; 4534 offset = 0; 4535 4536 g_io_done = false; 4537 for (i = 0; i < 2; i++) { 4538 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, offset, max_unmap_blocks, 0); 4539 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 4540 offset += max_unmap_blocks; 4541 } 4542 4543 rc = spdk_bdev_unmap_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 4544 CU_ASSERT_EQUAL(rc, 0); 4545 CU_ASSERT(g_io_done == false); 4546 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 4547 stub_complete_io(2); 4548 CU_ASSERT(g_io_done == true); 4549 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 4550 4551 /* Case 3: Test the split with 15 children requests, will finish 8 requests first */ 4552 num_children = 15; 4553 num_blocks = max_unmap_blocks * num_children; 4554 g_io_done = false; 4555 offset = 0; 4556 for (i = 0; i < num_children; i++) { 4557 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, offset, max_unmap_blocks, 0); 4558 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 4559 offset += max_unmap_blocks; 4560 } 4561 4562 rc = spdk_bdev_unmap_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 4563 CU_ASSERT_EQUAL(rc, 0); 4564 CU_ASSERT(g_io_done == false); 4565 4566 while (num_children > 0) { 4567 num_outstanding = spdk_min(num_children, SPDK_BDEV_MAX_CHILDREN_UNMAP_REQS); 4568 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == num_outstanding); 4569 stub_complete_io(num_outstanding); 4570 num_children -= num_outstanding; 4571 } 4572 CU_ASSERT(g_io_done == true); 4573 4574 spdk_put_io_channel(ioch); 4575 spdk_bdev_close(desc); 4576 free_bdev(bdev); 4577 spdk_bdev_finish(bdev_fini_cb, NULL); 4578 poll_threads(); 4579 } 4580 4581 static void 4582 bdev_set_options_test(void) 4583 { 4584 struct spdk_bdev_opts bdev_opts = {}; 4585 int rc; 4586 4587 /* Case1: Do not set opts_size */ 4588 rc = spdk_bdev_set_opts(&bdev_opts); 4589 CU_ASSERT(rc == -1); 4590 4591 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 4592 bdev_opts.bdev_io_pool_size = 4; 4593 bdev_opts.bdev_io_cache_size = 2; 4594 bdev_opts.small_buf_pool_size = 4; 4595 4596 /* Case 2: Do not set valid small_buf_pool_size and large_buf_pool_size */ 4597 rc = spdk_bdev_set_opts(&bdev_opts); 4598 CU_ASSERT(rc == -1); 4599 4600 /* Case 3: Do not set valid large_buf_pool_size */ 4601 bdev_opts.small_buf_pool_size = BUF_SMALL_POOL_SIZE; 4602 bdev_opts.large_buf_pool_size = BUF_LARGE_POOL_SIZE - 1; 4603 rc = spdk_bdev_set_opts(&bdev_opts); 4604 CU_ASSERT(rc == -1); 4605 4606 /* Case4: set valid large buf_pool_size */ 4607 bdev_opts.large_buf_pool_size = BUF_LARGE_POOL_SIZE; 4608 rc = spdk_bdev_set_opts(&bdev_opts); 4609 CU_ASSERT(rc == 0); 4610 4611 /* Case5: Set different valid value for small and large buf pool */ 4612 bdev_opts.large_buf_pool_size = BUF_SMALL_POOL_SIZE + 3; 4613 bdev_opts.large_buf_pool_size = BUF_LARGE_POOL_SIZE + 3; 4614 rc = spdk_bdev_set_opts(&bdev_opts); 4615 CU_ASSERT(rc == 0); 4616 } 4617 4618 static uint64_t 4619 get_ns_time(void) 4620 { 4621 int rc; 4622 struct timespec ts; 4623 4624 rc = clock_gettime(CLOCK_MONOTONIC, &ts); 4625 CU_ASSERT(rc == 0); 4626 return ts.tv_sec * 1000 * 1000 * 1000 + ts.tv_nsec; 4627 } 4628 4629 static int 4630 rb_tree_get_height(struct spdk_bdev_name *bdev_name) 4631 { 4632 int h1, h2; 4633 4634 if (bdev_name == NULL) { 4635 return -1; 4636 } else { 4637 h1 = rb_tree_get_height(RB_LEFT(bdev_name, node)); 4638 h2 = rb_tree_get_height(RB_RIGHT(bdev_name, node)); 4639 4640 return spdk_max(h1, h2) + 1; 4641 } 4642 } 4643 4644 static void 4645 bdev_multi_allocation(void) 4646 { 4647 const int max_bdev_num = 1024 * 16; 4648 char name[max_bdev_num][10]; 4649 char noexist_name[] = "invalid_bdev"; 4650 struct spdk_bdev *bdev[max_bdev_num]; 4651 int i, j; 4652 uint64_t last_time; 4653 int bdev_num; 4654 int height; 4655 4656 for (j = 0; j < max_bdev_num; j++) { 4657 snprintf(name[j], sizeof(name[j]), "bdev%d", j); 4658 } 4659 4660 for (i = 0; i < 16; i++) { 4661 last_time = get_ns_time(); 4662 bdev_num = 1024 * (i + 1); 4663 for (j = 0; j < bdev_num; j++) { 4664 bdev[j] = allocate_bdev(name[j]); 4665 height = rb_tree_get_height(&bdev[j]->internal.bdev_name); 4666 CU_ASSERT(height <= (int)(spdk_u32log2(j + 1))); 4667 } 4668 SPDK_NOTICELOG("alloc bdev num %d takes %lu ms\n", bdev_num, 4669 (get_ns_time() - last_time) / 1000 / 1000); 4670 for (j = 0; j < bdev_num; j++) { 4671 CU_ASSERT(spdk_bdev_get_by_name(name[j]) != NULL); 4672 } 4673 CU_ASSERT(spdk_bdev_get_by_name(noexist_name) == NULL); 4674 4675 for (j = 0; j < bdev_num; j++) { 4676 free_bdev(bdev[j]); 4677 } 4678 for (j = 0; j < bdev_num; j++) { 4679 CU_ASSERT(spdk_bdev_get_by_name(name[j]) == NULL); 4680 } 4681 } 4682 } 4683 4684 int 4685 main(int argc, char **argv) 4686 { 4687 CU_pSuite suite = NULL; 4688 unsigned int num_failures; 4689 4690 CU_set_error_action(CUEA_ABORT); 4691 CU_initialize_registry(); 4692 4693 suite = CU_add_suite("bdev", null_init, null_clean); 4694 4695 CU_ADD_TEST(suite, bytes_to_blocks_test); 4696 CU_ADD_TEST(suite, num_blocks_test); 4697 CU_ADD_TEST(suite, io_valid_test); 4698 CU_ADD_TEST(suite, open_write_test); 4699 CU_ADD_TEST(suite, alias_add_del_test); 4700 CU_ADD_TEST(suite, get_device_stat_test); 4701 CU_ADD_TEST(suite, bdev_io_types_test); 4702 CU_ADD_TEST(suite, bdev_io_wait_test); 4703 CU_ADD_TEST(suite, bdev_io_spans_split_test); 4704 CU_ADD_TEST(suite, bdev_io_boundary_split_test); 4705 CU_ADD_TEST(suite, bdev_io_max_size_and_segment_split_test); 4706 CU_ADD_TEST(suite, bdev_io_mix_split_test); 4707 CU_ADD_TEST(suite, bdev_io_split_with_io_wait); 4708 CU_ADD_TEST(suite, bdev_io_alignment_with_boundary); 4709 CU_ADD_TEST(suite, bdev_io_alignment); 4710 CU_ADD_TEST(suite, bdev_histograms); 4711 CU_ADD_TEST(suite, bdev_write_zeroes); 4712 CU_ADD_TEST(suite, bdev_compare_and_write); 4713 CU_ADD_TEST(suite, bdev_compare); 4714 CU_ADD_TEST(suite, bdev_zcopy_write); 4715 CU_ADD_TEST(suite, bdev_zcopy_read); 4716 CU_ADD_TEST(suite, bdev_open_while_hotremove); 4717 CU_ADD_TEST(suite, bdev_close_while_hotremove); 4718 CU_ADD_TEST(suite, bdev_open_ext); 4719 CU_ADD_TEST(suite, bdev_set_io_timeout); 4720 CU_ADD_TEST(suite, lba_range_overlap); 4721 CU_ADD_TEST(suite, lock_lba_range_check_ranges); 4722 CU_ADD_TEST(suite, lock_lba_range_with_io_outstanding); 4723 CU_ADD_TEST(suite, lock_lba_range_overlapped); 4724 CU_ADD_TEST(suite, bdev_io_abort); 4725 CU_ADD_TEST(suite, bdev_unmap); 4726 CU_ADD_TEST(suite, bdev_set_options_test); 4727 CU_ADD_TEST(suite, bdev_multi_allocation); 4728 4729 allocate_cores(1); 4730 allocate_threads(1); 4731 set_thread(0); 4732 4733 CU_basic_set_mode(CU_BRM_VERBOSE); 4734 CU_basic_run_tests(); 4735 num_failures = CU_get_number_of_failures(); 4736 CU_cleanup_registry(); 4737 4738 free_threads(); 4739 free_cores(); 4740 4741 return num_failures; 4742 } 4743