1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_cunit.h" 8 9 #include "common/lib/ut_multithread.c" 10 #include "unit/lib/json_mock.c" 11 12 #include "spdk/config.h" 13 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */ 14 #undef SPDK_CONFIG_VTUNE 15 16 #include "bdev/bdev.c" 17 18 DEFINE_STUB(spdk_notify_send, uint64_t, (const char *type, const char *ctx), 0); 19 DEFINE_STUB(spdk_notify_type_register, struct spdk_notify_type *, (const char *type), NULL); 20 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, (struct spdk_memory_domain *domain), 21 "test_domain"); 22 DEFINE_STUB(spdk_memory_domain_get_dma_device_type, enum spdk_dma_device_type, 23 (struct spdk_memory_domain *domain), 0); 24 25 static bool g_memory_domain_pull_data_called; 26 static bool g_memory_domain_push_data_called; 27 28 DEFINE_RETURN_MOCK(spdk_memory_domain_pull_data, int); 29 int 30 spdk_memory_domain_pull_data(struct spdk_memory_domain *src_domain, void *src_domain_ctx, 31 struct iovec *src_iov, uint32_t src_iov_cnt, struct iovec *dst_iov, uint32_t dst_iov_cnt, 32 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) 33 { 34 g_memory_domain_pull_data_called = true; 35 HANDLE_RETURN_MOCK(spdk_memory_domain_pull_data); 36 cpl_cb(cpl_cb_arg, 0); 37 return 0; 38 } 39 40 DEFINE_RETURN_MOCK(spdk_memory_domain_push_data, int); 41 int 42 spdk_memory_domain_push_data(struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 43 struct iovec *dst_iov, uint32_t dst_iovcnt, struct iovec *src_iov, uint32_t src_iovcnt, 44 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_cb_arg) 45 { 46 g_memory_domain_push_data_called = true; 47 HANDLE_RETURN_MOCK(spdk_memory_domain_push_data); 48 cpl_cb(cpl_cb_arg, 0); 49 return 0; 50 } 51 52 int g_status; 53 int g_count; 54 enum spdk_bdev_event_type g_event_type1; 55 enum spdk_bdev_event_type g_event_type2; 56 enum spdk_bdev_event_type g_event_type3; 57 enum spdk_bdev_event_type g_event_type4; 58 struct spdk_histogram_data *g_histogram; 59 void *g_unregister_arg; 60 int g_unregister_rc; 61 62 void 63 spdk_scsi_nvme_translate(const struct spdk_bdev_io *bdev_io, 64 int *sc, int *sk, int *asc, int *ascq) 65 { 66 } 67 68 static int 69 null_init(void) 70 { 71 return 0; 72 } 73 74 static int 75 null_clean(void) 76 { 77 return 0; 78 } 79 80 static int 81 stub_destruct(void *ctx) 82 { 83 return 0; 84 } 85 86 struct ut_expected_io { 87 uint8_t type; 88 uint64_t offset; 89 uint64_t src_offset; 90 uint64_t length; 91 int iovcnt; 92 struct iovec iov[BDEV_IO_NUM_CHILD_IOV]; 93 void *md_buf; 94 struct spdk_bdev_ext_io_opts *ext_io_opts; 95 bool copy_opts; 96 TAILQ_ENTRY(ut_expected_io) link; 97 }; 98 99 struct bdev_ut_channel { 100 TAILQ_HEAD(, spdk_bdev_io) outstanding_io; 101 uint32_t outstanding_io_count; 102 TAILQ_HEAD(, ut_expected_io) expected_io; 103 }; 104 105 static bool g_io_done; 106 static struct spdk_bdev_io *g_bdev_io; 107 static enum spdk_bdev_io_status g_io_status; 108 static enum spdk_bdev_io_status g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 109 static uint32_t g_bdev_ut_io_device; 110 static struct bdev_ut_channel *g_bdev_ut_channel; 111 static void *g_compare_read_buf; 112 static uint32_t g_compare_read_buf_len; 113 static void *g_compare_write_buf; 114 static uint32_t g_compare_write_buf_len; 115 static void *g_compare_md_buf; 116 static bool g_abort_done; 117 static enum spdk_bdev_io_status g_abort_status; 118 static void *g_zcopy_read_buf; 119 static uint32_t g_zcopy_read_buf_len; 120 static void *g_zcopy_write_buf; 121 static uint32_t g_zcopy_write_buf_len; 122 static struct spdk_bdev_io *g_zcopy_bdev_io; 123 static uint64_t g_seek_data_offset; 124 static uint64_t g_seek_hole_offset; 125 static uint64_t g_seek_offset; 126 127 static struct ut_expected_io * 128 ut_alloc_expected_io(uint8_t type, uint64_t offset, uint64_t length, int iovcnt) 129 { 130 struct ut_expected_io *expected_io; 131 132 expected_io = calloc(1, sizeof(*expected_io)); 133 SPDK_CU_ASSERT_FATAL(expected_io != NULL); 134 135 expected_io->type = type; 136 expected_io->offset = offset; 137 expected_io->length = length; 138 expected_io->iovcnt = iovcnt; 139 140 return expected_io; 141 } 142 143 static struct ut_expected_io * 144 ut_alloc_expected_copy_io(uint8_t type, uint64_t offset, uint64_t src_offset, uint64_t length) 145 { 146 struct ut_expected_io *expected_io; 147 148 expected_io = calloc(1, sizeof(*expected_io)); 149 SPDK_CU_ASSERT_FATAL(expected_io != NULL); 150 151 expected_io->type = type; 152 expected_io->offset = offset; 153 expected_io->src_offset = src_offset; 154 expected_io->length = length; 155 156 return expected_io; 157 } 158 159 static void 160 ut_expected_io_set_iov(struct ut_expected_io *expected_io, int pos, void *base, size_t len) 161 { 162 expected_io->iov[pos].iov_base = base; 163 expected_io->iov[pos].iov_len = len; 164 } 165 166 static void 167 stub_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 168 { 169 struct bdev_ut_channel *ch = spdk_io_channel_get_ctx(_ch); 170 struct ut_expected_io *expected_io; 171 struct iovec *iov, *expected_iov; 172 struct spdk_bdev_io *bio_to_abort; 173 int i; 174 175 g_bdev_io = bdev_io; 176 177 if (g_compare_read_buf && bdev_io->type == SPDK_BDEV_IO_TYPE_READ) { 178 uint32_t len = bdev_io->u.bdev.iovs[0].iov_len; 179 180 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 181 CU_ASSERT(g_compare_read_buf_len == len); 182 memcpy(bdev_io->u.bdev.iovs[0].iov_base, g_compare_read_buf, len); 183 if (bdev_io->bdev->md_len && bdev_io->u.bdev.md_buf && g_compare_md_buf) { 184 memcpy(bdev_io->u.bdev.md_buf, g_compare_md_buf, 185 bdev_io->bdev->md_len * bdev_io->u.bdev.num_blocks); 186 } 187 } 188 189 if (g_compare_write_buf && bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) { 190 uint32_t len = bdev_io->u.bdev.iovs[0].iov_len; 191 192 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 193 CU_ASSERT(g_compare_write_buf_len == len); 194 memcpy(g_compare_write_buf, bdev_io->u.bdev.iovs[0].iov_base, len); 195 } 196 197 if (g_compare_read_buf && bdev_io->type == SPDK_BDEV_IO_TYPE_COMPARE) { 198 uint32_t len = bdev_io->u.bdev.iovs[0].iov_len; 199 200 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 201 CU_ASSERT(g_compare_read_buf_len == len); 202 if (memcmp(bdev_io->u.bdev.iovs[0].iov_base, g_compare_read_buf, len)) { 203 g_io_exp_status = SPDK_BDEV_IO_STATUS_MISCOMPARE; 204 } 205 if (bdev_io->u.bdev.md_buf && 206 memcmp(bdev_io->u.bdev.md_buf, g_compare_md_buf, 207 bdev_io->bdev->md_len * bdev_io->u.bdev.num_blocks)) { 208 g_io_exp_status = SPDK_BDEV_IO_STATUS_MISCOMPARE; 209 } 210 } 211 212 if (bdev_io->type == SPDK_BDEV_IO_TYPE_ABORT) { 213 if (g_io_exp_status == SPDK_BDEV_IO_STATUS_SUCCESS) { 214 TAILQ_FOREACH(bio_to_abort, &ch->outstanding_io, module_link) { 215 if (bio_to_abort == bdev_io->u.abort.bio_to_abort) { 216 TAILQ_REMOVE(&ch->outstanding_io, bio_to_abort, module_link); 217 ch->outstanding_io_count--; 218 spdk_bdev_io_complete(bio_to_abort, SPDK_BDEV_IO_STATUS_FAILED); 219 break; 220 } 221 } 222 } 223 } 224 225 if (bdev_io->type == SPDK_BDEV_IO_TYPE_ZCOPY) { 226 if (bdev_io->u.bdev.zcopy.start) { 227 g_zcopy_bdev_io = bdev_io; 228 if (bdev_io->u.bdev.zcopy.populate) { 229 /* Start of a read */ 230 CU_ASSERT(g_zcopy_read_buf != NULL); 231 CU_ASSERT(g_zcopy_read_buf_len > 0); 232 bdev_io->u.bdev.iovs[0].iov_base = g_zcopy_read_buf; 233 bdev_io->u.bdev.iovs[0].iov_len = g_zcopy_read_buf_len; 234 bdev_io->u.bdev.iovcnt = 1; 235 } else { 236 /* Start of a write */ 237 CU_ASSERT(g_zcopy_write_buf != NULL); 238 CU_ASSERT(g_zcopy_write_buf_len > 0); 239 bdev_io->u.bdev.iovs[0].iov_base = g_zcopy_write_buf; 240 bdev_io->u.bdev.iovs[0].iov_len = g_zcopy_write_buf_len; 241 bdev_io->u.bdev.iovcnt = 1; 242 } 243 } else { 244 if (bdev_io->u.bdev.zcopy.commit) { 245 /* End of write */ 246 CU_ASSERT(bdev_io->u.bdev.iovs[0].iov_base == g_zcopy_write_buf); 247 CU_ASSERT(bdev_io->u.bdev.iovs[0].iov_len == g_zcopy_write_buf_len); 248 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 249 g_zcopy_write_buf = NULL; 250 g_zcopy_write_buf_len = 0; 251 } else { 252 /* End of read */ 253 CU_ASSERT(bdev_io->u.bdev.iovs[0].iov_base == g_zcopy_read_buf); 254 CU_ASSERT(bdev_io->u.bdev.iovs[0].iov_len == g_zcopy_read_buf_len); 255 CU_ASSERT(bdev_io->u.bdev.iovcnt == 1); 256 g_zcopy_read_buf = NULL; 257 g_zcopy_read_buf_len = 0; 258 } 259 } 260 } 261 262 if (bdev_io->type == SPDK_BDEV_IO_TYPE_SEEK_DATA) { 263 bdev_io->u.bdev.seek.offset = g_seek_data_offset; 264 } 265 266 if (bdev_io->type == SPDK_BDEV_IO_TYPE_SEEK_HOLE) { 267 bdev_io->u.bdev.seek.offset = g_seek_hole_offset; 268 } 269 270 TAILQ_INSERT_TAIL(&ch->outstanding_io, bdev_io, module_link); 271 ch->outstanding_io_count++; 272 273 expected_io = TAILQ_FIRST(&ch->expected_io); 274 if (expected_io == NULL) { 275 return; 276 } 277 TAILQ_REMOVE(&ch->expected_io, expected_io, link); 278 279 if (expected_io->type != SPDK_BDEV_IO_TYPE_INVALID) { 280 CU_ASSERT(bdev_io->type == expected_io->type); 281 } 282 283 if (expected_io->md_buf != NULL) { 284 CU_ASSERT(expected_io->md_buf == bdev_io->u.bdev.md_buf); 285 if (bdev_io->u.bdev.ext_opts) { 286 CU_ASSERT(expected_io->md_buf == bdev_io->u.bdev.ext_opts->metadata); 287 } 288 } 289 290 if (expected_io->copy_opts) { 291 if (expected_io->ext_io_opts) { 292 /* opts are not NULL so it should have been copied */ 293 CU_ASSERT(expected_io->ext_io_opts != bdev_io->u.bdev.ext_opts); 294 CU_ASSERT(bdev_io->u.bdev.ext_opts == &bdev_io->internal.ext_opts_copy); 295 /* internal opts always points to opts passed */ 296 CU_ASSERT(expected_io->ext_io_opts == bdev_io->internal.ext_opts); 297 } else { 298 /* passed opts was NULL so we expect bdev_io opts to be NULL */ 299 CU_ASSERT(bdev_io->u.bdev.ext_opts == NULL); 300 } 301 } else { 302 /* opts were not copied so they should be equal */ 303 CU_ASSERT(expected_io->ext_io_opts == bdev_io->u.bdev.ext_opts); 304 } 305 306 if (expected_io->length == 0) { 307 free(expected_io); 308 return; 309 } 310 311 CU_ASSERT(expected_io->offset == bdev_io->u.bdev.offset_blocks); 312 CU_ASSERT(expected_io->length = bdev_io->u.bdev.num_blocks); 313 if (expected_io->type == SPDK_BDEV_IO_TYPE_COPY) { 314 CU_ASSERT(expected_io->src_offset == bdev_io->u.bdev.copy.src_offset_blocks); 315 } 316 317 if (expected_io->iovcnt == 0) { 318 free(expected_io); 319 /* UNMAP, WRITE_ZEROES, FLUSH and COPY don't have iovs, so we can just return now. */ 320 return; 321 } 322 323 CU_ASSERT(expected_io->iovcnt == bdev_io->u.bdev.iovcnt); 324 for (i = 0; i < expected_io->iovcnt; i++) { 325 expected_iov = &expected_io->iov[i]; 326 if (bdev_io->internal.orig_iovcnt == 0) { 327 iov = &bdev_io->u.bdev.iovs[i]; 328 } else { 329 iov = bdev_io->internal.orig_iovs; 330 } 331 CU_ASSERT(iov->iov_len == expected_iov->iov_len); 332 CU_ASSERT(iov->iov_base == expected_iov->iov_base); 333 } 334 335 free(expected_io); 336 } 337 338 static void 339 stub_submit_request_get_buf_cb(struct spdk_io_channel *_ch, 340 struct spdk_bdev_io *bdev_io, bool success) 341 { 342 CU_ASSERT(success == true); 343 344 stub_submit_request(_ch, bdev_io); 345 } 346 347 static void 348 stub_submit_request_get_buf(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 349 { 350 spdk_bdev_io_get_buf(bdev_io, stub_submit_request_get_buf_cb, 351 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); 352 } 353 354 static uint32_t 355 stub_complete_io(uint32_t num_to_complete) 356 { 357 struct bdev_ut_channel *ch = g_bdev_ut_channel; 358 struct spdk_bdev_io *bdev_io; 359 static enum spdk_bdev_io_status io_status; 360 uint32_t num_completed = 0; 361 362 while (num_completed < num_to_complete) { 363 if (TAILQ_EMPTY(&ch->outstanding_io)) { 364 break; 365 } 366 bdev_io = TAILQ_FIRST(&ch->outstanding_io); 367 TAILQ_REMOVE(&ch->outstanding_io, bdev_io, module_link); 368 ch->outstanding_io_count--; 369 io_status = g_io_exp_status == SPDK_BDEV_IO_STATUS_SUCCESS ? SPDK_BDEV_IO_STATUS_SUCCESS : 370 g_io_exp_status; 371 spdk_bdev_io_complete(bdev_io, io_status); 372 num_completed++; 373 } 374 375 return num_completed; 376 } 377 378 static struct spdk_io_channel * 379 bdev_ut_get_io_channel(void *ctx) 380 { 381 return spdk_get_io_channel(&g_bdev_ut_io_device); 382 } 383 384 static bool g_io_types_supported[SPDK_BDEV_NUM_IO_TYPES] = { 385 [SPDK_BDEV_IO_TYPE_READ] = true, 386 [SPDK_BDEV_IO_TYPE_WRITE] = true, 387 [SPDK_BDEV_IO_TYPE_COMPARE] = true, 388 [SPDK_BDEV_IO_TYPE_UNMAP] = true, 389 [SPDK_BDEV_IO_TYPE_FLUSH] = true, 390 [SPDK_BDEV_IO_TYPE_RESET] = true, 391 [SPDK_BDEV_IO_TYPE_NVME_ADMIN] = true, 392 [SPDK_BDEV_IO_TYPE_NVME_IO] = true, 393 [SPDK_BDEV_IO_TYPE_NVME_IO_MD] = true, 394 [SPDK_BDEV_IO_TYPE_WRITE_ZEROES] = true, 395 [SPDK_BDEV_IO_TYPE_ZCOPY] = true, 396 [SPDK_BDEV_IO_TYPE_ABORT] = true, 397 [SPDK_BDEV_IO_TYPE_SEEK_HOLE] = true, 398 [SPDK_BDEV_IO_TYPE_SEEK_DATA] = true, 399 [SPDK_BDEV_IO_TYPE_COPY] = true, 400 }; 401 402 static void 403 ut_enable_io_type(enum spdk_bdev_io_type io_type, bool enable) 404 { 405 g_io_types_supported[io_type] = enable; 406 } 407 408 static bool 409 stub_io_type_supported(void *_bdev, enum spdk_bdev_io_type io_type) 410 { 411 return g_io_types_supported[io_type]; 412 } 413 414 static struct spdk_bdev_fn_table fn_table = { 415 .destruct = stub_destruct, 416 .submit_request = stub_submit_request, 417 .get_io_channel = bdev_ut_get_io_channel, 418 .io_type_supported = stub_io_type_supported, 419 }; 420 421 static int 422 bdev_ut_create_ch(void *io_device, void *ctx_buf) 423 { 424 struct bdev_ut_channel *ch = ctx_buf; 425 426 CU_ASSERT(g_bdev_ut_channel == NULL); 427 g_bdev_ut_channel = ch; 428 429 TAILQ_INIT(&ch->outstanding_io); 430 ch->outstanding_io_count = 0; 431 TAILQ_INIT(&ch->expected_io); 432 return 0; 433 } 434 435 static void 436 bdev_ut_destroy_ch(void *io_device, void *ctx_buf) 437 { 438 CU_ASSERT(g_bdev_ut_channel != NULL); 439 g_bdev_ut_channel = NULL; 440 } 441 442 struct spdk_bdev_module bdev_ut_if; 443 444 static int 445 bdev_ut_module_init(void) 446 { 447 spdk_io_device_register(&g_bdev_ut_io_device, bdev_ut_create_ch, bdev_ut_destroy_ch, 448 sizeof(struct bdev_ut_channel), NULL); 449 spdk_bdev_module_init_done(&bdev_ut_if); 450 return 0; 451 } 452 453 static void 454 bdev_ut_module_fini(void) 455 { 456 spdk_io_device_unregister(&g_bdev_ut_io_device, NULL); 457 } 458 459 struct spdk_bdev_module bdev_ut_if = { 460 .name = "bdev_ut", 461 .module_init = bdev_ut_module_init, 462 .module_fini = bdev_ut_module_fini, 463 .async_init = true, 464 }; 465 466 static void vbdev_ut_examine(struct spdk_bdev *bdev); 467 468 static int 469 vbdev_ut_module_init(void) 470 { 471 return 0; 472 } 473 474 static void 475 vbdev_ut_module_fini(void) 476 { 477 } 478 479 struct spdk_bdev_module vbdev_ut_if = { 480 .name = "vbdev_ut", 481 .module_init = vbdev_ut_module_init, 482 .module_fini = vbdev_ut_module_fini, 483 .examine_config = vbdev_ut_examine, 484 }; 485 486 SPDK_BDEV_MODULE_REGISTER(bdev_ut, &bdev_ut_if) 487 SPDK_BDEV_MODULE_REGISTER(vbdev_ut, &vbdev_ut_if) 488 489 static void 490 vbdev_ut_examine(struct spdk_bdev *bdev) 491 { 492 spdk_bdev_module_examine_done(&vbdev_ut_if); 493 } 494 495 static struct spdk_bdev * 496 allocate_bdev(char *name) 497 { 498 struct spdk_bdev *bdev; 499 int rc; 500 501 bdev = calloc(1, sizeof(*bdev)); 502 SPDK_CU_ASSERT_FATAL(bdev != NULL); 503 504 bdev->name = name; 505 bdev->fn_table = &fn_table; 506 bdev->module = &bdev_ut_if; 507 bdev->blockcnt = 1024; 508 bdev->blocklen = 512; 509 510 rc = spdk_bdev_register(bdev); 511 poll_threads(); 512 CU_ASSERT(rc == 0); 513 514 return bdev; 515 } 516 517 static struct spdk_bdev * 518 allocate_vbdev(char *name) 519 { 520 struct spdk_bdev *bdev; 521 int rc; 522 523 bdev = calloc(1, sizeof(*bdev)); 524 SPDK_CU_ASSERT_FATAL(bdev != NULL); 525 526 bdev->name = name; 527 bdev->fn_table = &fn_table; 528 bdev->module = &vbdev_ut_if; 529 530 rc = spdk_bdev_register(bdev); 531 poll_threads(); 532 CU_ASSERT(rc == 0); 533 534 return bdev; 535 } 536 537 static void 538 free_bdev(struct spdk_bdev *bdev) 539 { 540 spdk_bdev_unregister(bdev, NULL, NULL); 541 poll_threads(); 542 memset(bdev, 0xFF, sizeof(*bdev)); 543 free(bdev); 544 } 545 546 static void 547 free_vbdev(struct spdk_bdev *bdev) 548 { 549 spdk_bdev_unregister(bdev, NULL, NULL); 550 poll_threads(); 551 memset(bdev, 0xFF, sizeof(*bdev)); 552 free(bdev); 553 } 554 555 static void 556 get_device_stat_cb(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat, void *cb_arg, int rc) 557 { 558 const char *bdev_name; 559 560 CU_ASSERT(bdev != NULL); 561 CU_ASSERT(rc == 0); 562 bdev_name = spdk_bdev_get_name(bdev); 563 CU_ASSERT_STRING_EQUAL(bdev_name, "bdev0"); 564 565 free(stat); 566 567 *(bool *)cb_arg = true; 568 } 569 570 static void 571 bdev_unregister_cb(void *cb_arg, int rc) 572 { 573 g_unregister_arg = cb_arg; 574 g_unregister_rc = rc; 575 } 576 577 static void 578 bdev_ut_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 579 { 580 } 581 582 static void 583 bdev_open_cb1(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 584 { 585 struct spdk_bdev_desc *desc = *(struct spdk_bdev_desc **)event_ctx; 586 587 g_event_type1 = type; 588 if (SPDK_BDEV_EVENT_REMOVE == type) { 589 spdk_bdev_close(desc); 590 } 591 } 592 593 static void 594 bdev_open_cb2(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 595 { 596 struct spdk_bdev_desc *desc = *(struct spdk_bdev_desc **)event_ctx; 597 598 g_event_type2 = type; 599 if (SPDK_BDEV_EVENT_REMOVE == type) { 600 spdk_bdev_close(desc); 601 } 602 } 603 604 static void 605 bdev_open_cb3(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 606 { 607 g_event_type3 = type; 608 } 609 610 static void 611 bdev_open_cb4(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) 612 { 613 g_event_type4 = type; 614 } 615 616 static void 617 bdev_seek_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 618 { 619 g_seek_offset = spdk_bdev_io_get_seek_offset(bdev_io); 620 spdk_bdev_free_io(bdev_io); 621 } 622 623 static void 624 get_device_stat_test(void) 625 { 626 struct spdk_bdev *bdev; 627 struct spdk_bdev_io_stat *stat; 628 bool done; 629 630 bdev = allocate_bdev("bdev0"); 631 stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); 632 if (stat == NULL) { 633 free_bdev(bdev); 634 return; 635 } 636 637 done = false; 638 spdk_bdev_get_device_stat(bdev, stat, get_device_stat_cb, &done); 639 while (!done) { poll_threads(); } 640 641 free_bdev(bdev); 642 } 643 644 static void 645 open_write_test(void) 646 { 647 struct spdk_bdev *bdev[9]; 648 struct spdk_bdev_desc *desc[9] = {}; 649 int rc; 650 651 /* 652 * Create a tree of bdevs to test various open w/ write cases. 653 * 654 * bdev0 through bdev3 are physical block devices, such as NVMe 655 * namespaces or Ceph block devices. 656 * 657 * bdev4 is a virtual bdev with multiple base bdevs. This models 658 * caching or RAID use cases. 659 * 660 * bdev5 through bdev7 are all virtual bdevs with the same base 661 * bdev (except bdev7). This models partitioning or logical volume 662 * use cases. 663 * 664 * bdev7 is a virtual bdev with multiple base bdevs. One of base bdevs 665 * (bdev2) is shared with other virtual bdevs: bdev5 and bdev6. This 666 * models caching, RAID, partitioning or logical volumes use cases. 667 * 668 * bdev8 is a virtual bdev with multiple base bdevs, but these 669 * base bdevs are themselves virtual bdevs. 670 * 671 * bdev8 672 * | 673 * +----------+ 674 * | | 675 * bdev4 bdev5 bdev6 bdev7 676 * | | | | 677 * +---+---+ +---+ + +---+---+ 678 * | | \ | / \ 679 * bdev0 bdev1 bdev2 bdev3 680 */ 681 682 bdev[0] = allocate_bdev("bdev0"); 683 rc = spdk_bdev_module_claim_bdev(bdev[0], NULL, &bdev_ut_if); 684 CU_ASSERT(rc == 0); 685 686 bdev[1] = allocate_bdev("bdev1"); 687 rc = spdk_bdev_module_claim_bdev(bdev[1], NULL, &bdev_ut_if); 688 CU_ASSERT(rc == 0); 689 690 bdev[2] = allocate_bdev("bdev2"); 691 rc = spdk_bdev_module_claim_bdev(bdev[2], NULL, &bdev_ut_if); 692 CU_ASSERT(rc == 0); 693 694 bdev[3] = allocate_bdev("bdev3"); 695 rc = spdk_bdev_module_claim_bdev(bdev[3], NULL, &bdev_ut_if); 696 CU_ASSERT(rc == 0); 697 698 bdev[4] = allocate_vbdev("bdev4"); 699 rc = spdk_bdev_module_claim_bdev(bdev[4], NULL, &bdev_ut_if); 700 CU_ASSERT(rc == 0); 701 702 bdev[5] = allocate_vbdev("bdev5"); 703 rc = spdk_bdev_module_claim_bdev(bdev[5], NULL, &bdev_ut_if); 704 CU_ASSERT(rc == 0); 705 706 bdev[6] = allocate_vbdev("bdev6"); 707 708 bdev[7] = allocate_vbdev("bdev7"); 709 710 bdev[8] = allocate_vbdev("bdev8"); 711 712 /* Open bdev0 read-only. This should succeed. */ 713 rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc[0]); 714 CU_ASSERT(rc == 0); 715 SPDK_CU_ASSERT_FATAL(desc[0] != NULL); 716 CU_ASSERT(bdev[0] == spdk_bdev_desc_get_bdev(desc[0])); 717 spdk_bdev_close(desc[0]); 718 719 /* 720 * Open bdev1 read/write. This should fail since bdev1 has been claimed 721 * by a vbdev module. 722 */ 723 rc = spdk_bdev_open_ext("bdev1", true, bdev_ut_event_cb, NULL, &desc[1]); 724 CU_ASSERT(rc == -EPERM); 725 726 /* 727 * Open bdev4 read/write. This should fail since bdev3 has been claimed 728 * by a vbdev module. 729 */ 730 rc = spdk_bdev_open_ext("bdev4", true, bdev_ut_event_cb, NULL, &desc[4]); 731 CU_ASSERT(rc == -EPERM); 732 733 /* Open bdev4 read-only. This should succeed. */ 734 rc = spdk_bdev_open_ext("bdev4", false, bdev_ut_event_cb, NULL, &desc[4]); 735 CU_ASSERT(rc == 0); 736 SPDK_CU_ASSERT_FATAL(desc[4] != NULL); 737 CU_ASSERT(bdev[4] == spdk_bdev_desc_get_bdev(desc[4])); 738 spdk_bdev_close(desc[4]); 739 740 /* 741 * Open bdev8 read/write. This should succeed since it is a leaf 742 * bdev. 743 */ 744 rc = spdk_bdev_open_ext("bdev8", true, bdev_ut_event_cb, NULL, &desc[8]); 745 CU_ASSERT(rc == 0); 746 SPDK_CU_ASSERT_FATAL(desc[8] != NULL); 747 CU_ASSERT(bdev[8] == spdk_bdev_desc_get_bdev(desc[8])); 748 spdk_bdev_close(desc[8]); 749 750 /* 751 * Open bdev5 read/write. This should fail since bdev4 has been claimed 752 * by a vbdev module. 753 */ 754 rc = spdk_bdev_open_ext("bdev5", true, bdev_ut_event_cb, NULL, &desc[5]); 755 CU_ASSERT(rc == -EPERM); 756 757 /* Open bdev4 read-only. This should succeed. */ 758 rc = spdk_bdev_open_ext("bdev5", false, bdev_ut_event_cb, NULL, &desc[5]); 759 CU_ASSERT(rc == 0); 760 SPDK_CU_ASSERT_FATAL(desc[5] != NULL); 761 CU_ASSERT(bdev[5] == spdk_bdev_desc_get_bdev(desc[5])); 762 spdk_bdev_close(desc[5]); 763 764 free_vbdev(bdev[8]); 765 766 free_vbdev(bdev[5]); 767 free_vbdev(bdev[6]); 768 free_vbdev(bdev[7]); 769 770 free_vbdev(bdev[4]); 771 772 free_bdev(bdev[0]); 773 free_bdev(bdev[1]); 774 free_bdev(bdev[2]); 775 free_bdev(bdev[3]); 776 } 777 778 static void 779 claim_test(void) 780 { 781 struct spdk_bdev *bdev; 782 struct spdk_bdev_desc *desc, *open_desc; 783 int rc; 784 uint32_t count; 785 786 /* 787 * A vbdev that uses a read-only bdev may need it to remain read-only. 788 * To do so, it opens the bdev read-only, then claims it without 789 * passing a spdk_bdev_desc. 790 */ 791 bdev = allocate_bdev("bdev0"); 792 rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc); 793 CU_ASSERT(rc == 0); 794 CU_ASSERT(desc->write == false); 795 796 rc = spdk_bdev_module_claim_bdev(bdev, NULL, &bdev_ut_if); 797 CU_ASSERT(rc == 0); 798 CU_ASSERT(bdev->internal.claim_module == &bdev_ut_if); 799 800 /* There should be only one open descriptor and it should still be ro */ 801 count = 0; 802 TAILQ_FOREACH(open_desc, &bdev->internal.open_descs, link) { 803 CU_ASSERT(open_desc == desc); 804 CU_ASSERT(!open_desc->write); 805 count++; 806 } 807 CU_ASSERT(count == 1); 808 809 /* A read-only bdev is upgraded to read-write if desc is passed. */ 810 spdk_bdev_module_release_bdev(bdev); 811 rc = spdk_bdev_module_claim_bdev(bdev, desc, &bdev_ut_if); 812 CU_ASSERT(rc == 0); 813 CU_ASSERT(bdev->internal.claim_module == &bdev_ut_if); 814 815 /* There should be only one open descriptor and it should be rw */ 816 count = 0; 817 TAILQ_FOREACH(open_desc, &bdev->internal.open_descs, link) { 818 CU_ASSERT(open_desc == desc); 819 CU_ASSERT(open_desc->write); 820 count++; 821 } 822 CU_ASSERT(count == 1); 823 824 spdk_bdev_close(desc); 825 free_bdev(bdev); 826 } 827 828 static void 829 bytes_to_blocks_test(void) 830 { 831 struct spdk_bdev bdev; 832 uint64_t offset_blocks, num_blocks; 833 834 memset(&bdev, 0, sizeof(bdev)); 835 836 bdev.blocklen = 512; 837 838 /* All parameters valid */ 839 offset_blocks = 0; 840 num_blocks = 0; 841 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 512, &offset_blocks, 1024, &num_blocks) == 0); 842 CU_ASSERT(offset_blocks == 1); 843 CU_ASSERT(num_blocks == 2); 844 845 /* Offset not a block multiple */ 846 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 3, &offset_blocks, 512, &num_blocks) != 0); 847 848 /* Length not a block multiple */ 849 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 512, &offset_blocks, 3, &num_blocks) != 0); 850 851 /* In case blocklen not the power of two */ 852 bdev.blocklen = 100; 853 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 100, &offset_blocks, 200, &num_blocks) == 0); 854 CU_ASSERT(offset_blocks == 1); 855 CU_ASSERT(num_blocks == 2); 856 857 /* Offset not a block multiple */ 858 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 3, &offset_blocks, 100, &num_blocks) != 0); 859 860 /* Length not a block multiple */ 861 CU_ASSERT(bdev_bytes_to_blocks(&bdev, 100, &offset_blocks, 3, &num_blocks) != 0); 862 } 863 864 static void 865 num_blocks_test(void) 866 { 867 struct spdk_bdev bdev; 868 struct spdk_bdev_desc *desc = NULL; 869 int rc; 870 871 memset(&bdev, 0, sizeof(bdev)); 872 bdev.name = "num_blocks"; 873 bdev.fn_table = &fn_table; 874 bdev.module = &bdev_ut_if; 875 spdk_bdev_register(&bdev); 876 poll_threads(); 877 spdk_bdev_notify_blockcnt_change(&bdev, 50); 878 879 /* Growing block number */ 880 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 70) == 0); 881 /* Shrinking block number */ 882 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 30) == 0); 883 884 rc = spdk_bdev_open_ext("num_blocks", false, bdev_open_cb1, &desc, &desc); 885 CU_ASSERT(rc == 0); 886 SPDK_CU_ASSERT_FATAL(desc != NULL); 887 CU_ASSERT(&bdev == spdk_bdev_desc_get_bdev(desc)); 888 889 /* Growing block number */ 890 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 80) == 0); 891 /* Shrinking block number */ 892 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 20) != 0); 893 894 g_event_type1 = 0xFF; 895 /* Growing block number */ 896 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 90) == 0); 897 898 poll_threads(); 899 CU_ASSERT_EQUAL(g_event_type1, SPDK_BDEV_EVENT_RESIZE); 900 901 g_event_type1 = 0xFF; 902 /* Growing block number and closing */ 903 CU_ASSERT(spdk_bdev_notify_blockcnt_change(&bdev, 100) == 0); 904 905 spdk_bdev_close(desc); 906 spdk_bdev_unregister(&bdev, NULL, NULL); 907 908 poll_threads(); 909 910 /* Callback is not called for closed device */ 911 CU_ASSERT_EQUAL(g_event_type1, 0xFF); 912 } 913 914 static void 915 io_valid_test(void) 916 { 917 struct spdk_bdev bdev; 918 919 memset(&bdev, 0, sizeof(bdev)); 920 921 bdev.blocklen = 512; 922 CU_ASSERT(pthread_mutex_init(&bdev.internal.mutex, NULL) == 0); 923 924 spdk_bdev_notify_blockcnt_change(&bdev, 100); 925 926 /* All parameters valid */ 927 CU_ASSERT(bdev_io_valid_blocks(&bdev, 1, 2) == true); 928 929 /* Last valid block */ 930 CU_ASSERT(bdev_io_valid_blocks(&bdev, 99, 1) == true); 931 932 /* Offset past end of bdev */ 933 CU_ASSERT(bdev_io_valid_blocks(&bdev, 100, 1) == false); 934 935 /* Offset + length past end of bdev */ 936 CU_ASSERT(bdev_io_valid_blocks(&bdev, 99, 2) == false); 937 938 /* Offset near end of uint64_t range (2^64 - 1) */ 939 CU_ASSERT(bdev_io_valid_blocks(&bdev, 18446744073709551615ULL, 1) == false); 940 941 CU_ASSERT(pthread_mutex_destroy(&bdev.internal.mutex) == 0); 942 } 943 944 static void 945 alias_add_del_test(void) 946 { 947 struct spdk_bdev *bdev[3]; 948 int rc; 949 950 /* Creating and registering bdevs */ 951 bdev[0] = allocate_bdev("bdev0"); 952 SPDK_CU_ASSERT_FATAL(bdev[0] != 0); 953 954 bdev[1] = allocate_bdev("bdev1"); 955 SPDK_CU_ASSERT_FATAL(bdev[1] != 0); 956 957 bdev[2] = allocate_bdev("bdev2"); 958 SPDK_CU_ASSERT_FATAL(bdev[2] != 0); 959 960 poll_threads(); 961 962 /* 963 * Trying adding an alias identical to name. 964 * Alias is identical to name, so it can not be added to aliases list 965 */ 966 rc = spdk_bdev_alias_add(bdev[0], bdev[0]->name); 967 CU_ASSERT(rc == -EEXIST); 968 969 /* 970 * Trying to add empty alias, 971 * this one should fail 972 */ 973 rc = spdk_bdev_alias_add(bdev[0], NULL); 974 CU_ASSERT(rc == -EINVAL); 975 976 /* Trying adding same alias to two different registered bdevs */ 977 978 /* Alias is used first time, so this one should pass */ 979 rc = spdk_bdev_alias_add(bdev[0], "proper alias 0"); 980 CU_ASSERT(rc == 0); 981 982 /* Alias was added to another bdev, so this one should fail */ 983 rc = spdk_bdev_alias_add(bdev[1], "proper alias 0"); 984 CU_ASSERT(rc == -EEXIST); 985 986 /* Alias is used first time, so this one should pass */ 987 rc = spdk_bdev_alias_add(bdev[1], "proper alias 1"); 988 CU_ASSERT(rc == 0); 989 990 /* Trying removing an alias from registered bdevs */ 991 992 /* Alias is not on a bdev aliases list, so this one should fail */ 993 rc = spdk_bdev_alias_del(bdev[0], "not existing"); 994 CU_ASSERT(rc == -ENOENT); 995 996 /* Alias is present on a bdev aliases list, so this one should pass */ 997 rc = spdk_bdev_alias_del(bdev[0], "proper alias 0"); 998 CU_ASSERT(rc == 0); 999 1000 /* Alias is present on a bdev aliases list, so this one should pass */ 1001 rc = spdk_bdev_alias_del(bdev[1], "proper alias 1"); 1002 CU_ASSERT(rc == 0); 1003 1004 /* Trying to remove name instead of alias, so this one should fail, name cannot be changed or removed */ 1005 rc = spdk_bdev_alias_del(bdev[0], bdev[0]->name); 1006 CU_ASSERT(rc != 0); 1007 1008 /* Trying to del all alias from empty alias list */ 1009 spdk_bdev_alias_del_all(bdev[2]); 1010 SPDK_CU_ASSERT_FATAL(TAILQ_EMPTY(&bdev[2]->aliases)); 1011 1012 /* Trying to del all alias from non-empty alias list */ 1013 rc = spdk_bdev_alias_add(bdev[2], "alias0"); 1014 CU_ASSERT(rc == 0); 1015 rc = spdk_bdev_alias_add(bdev[2], "alias1"); 1016 CU_ASSERT(rc == 0); 1017 spdk_bdev_alias_del_all(bdev[2]); 1018 CU_ASSERT(TAILQ_EMPTY(&bdev[2]->aliases)); 1019 1020 /* Unregister and free bdevs */ 1021 spdk_bdev_unregister(bdev[0], NULL, NULL); 1022 spdk_bdev_unregister(bdev[1], NULL, NULL); 1023 spdk_bdev_unregister(bdev[2], NULL, NULL); 1024 1025 poll_threads(); 1026 1027 free(bdev[0]); 1028 free(bdev[1]); 1029 free(bdev[2]); 1030 } 1031 1032 static void 1033 io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1034 { 1035 g_io_done = true; 1036 g_io_status = bdev_io->internal.status; 1037 if ((bdev_io->type == SPDK_BDEV_IO_TYPE_ZCOPY) && 1038 (bdev_io->u.bdev.zcopy.start)) { 1039 g_zcopy_bdev_io = bdev_io; 1040 } else { 1041 spdk_bdev_free_io(bdev_io); 1042 g_zcopy_bdev_io = NULL; 1043 } 1044 } 1045 1046 static void 1047 bdev_init_cb(void *arg, int rc) 1048 { 1049 CU_ASSERT(rc == 0); 1050 } 1051 1052 static void 1053 bdev_fini_cb(void *arg) 1054 { 1055 } 1056 1057 struct bdev_ut_io_wait_entry { 1058 struct spdk_bdev_io_wait_entry entry; 1059 struct spdk_io_channel *io_ch; 1060 struct spdk_bdev_desc *desc; 1061 bool submitted; 1062 }; 1063 1064 static void 1065 io_wait_cb(void *arg) 1066 { 1067 struct bdev_ut_io_wait_entry *entry = arg; 1068 int rc; 1069 1070 rc = spdk_bdev_read_blocks(entry->desc, entry->io_ch, NULL, 0, 1, io_done, NULL); 1071 CU_ASSERT(rc == 0); 1072 entry->submitted = true; 1073 } 1074 1075 static void 1076 bdev_io_types_test(void) 1077 { 1078 struct spdk_bdev *bdev; 1079 struct spdk_bdev_desc *desc = NULL; 1080 struct spdk_io_channel *io_ch; 1081 struct spdk_bdev_opts bdev_opts = {}; 1082 int rc; 1083 1084 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 1085 bdev_opts.bdev_io_pool_size = 4; 1086 bdev_opts.bdev_io_cache_size = 2; 1087 1088 rc = spdk_bdev_set_opts(&bdev_opts); 1089 CU_ASSERT(rc == 0); 1090 spdk_bdev_initialize(bdev_init_cb, NULL); 1091 poll_threads(); 1092 1093 bdev = allocate_bdev("bdev0"); 1094 1095 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 1096 CU_ASSERT(rc == 0); 1097 poll_threads(); 1098 SPDK_CU_ASSERT_FATAL(desc != NULL); 1099 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 1100 io_ch = spdk_bdev_get_io_channel(desc); 1101 CU_ASSERT(io_ch != NULL); 1102 1103 /* WRITE and WRITE ZEROES are not supported */ 1104 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, false); 1105 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE, false); 1106 rc = spdk_bdev_write_zeroes_blocks(desc, io_ch, 0, 128, io_done, NULL); 1107 CU_ASSERT(rc == -ENOTSUP); 1108 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, true); 1109 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE, true); 1110 1111 /* COPY is not supported */ 1112 ut_enable_io_type(SPDK_BDEV_IO_TYPE_COPY, false); 1113 rc = spdk_bdev_copy_blocks(desc, io_ch, 128, 0, 128, io_done, NULL); 1114 CU_ASSERT(rc == -ENOTSUP); 1115 ut_enable_io_type(SPDK_BDEV_IO_TYPE_COPY, true); 1116 1117 /* NVME_IO, NVME_IO_MD and NVME_ADMIN are not supported */ 1118 ut_enable_io_type(SPDK_BDEV_IO_TYPE_NVME_IO, false); 1119 ut_enable_io_type(SPDK_BDEV_IO_TYPE_NVME_IO_MD, false); 1120 ut_enable_io_type(SPDK_BDEV_IO_TYPE_NVME_ADMIN, false); 1121 rc = spdk_bdev_nvme_io_passthru(desc, io_ch, NULL, NULL, 0, NULL, NULL); 1122 CU_ASSERT(rc == -ENOTSUP); 1123 rc = spdk_bdev_nvme_io_passthru_md(desc, io_ch, NULL, NULL, 0, NULL, 0, NULL, NULL); 1124 CU_ASSERT(rc == -ENOTSUP); 1125 rc = spdk_bdev_nvme_admin_passthru(desc, io_ch, NULL, NULL, 0, NULL, NULL); 1126 CU_ASSERT(rc == -ENOTSUP); 1127 ut_enable_io_type(SPDK_BDEV_IO_TYPE_NVME_IO, true); 1128 ut_enable_io_type(SPDK_BDEV_IO_TYPE_NVME_IO_MD, true); 1129 ut_enable_io_type(SPDK_BDEV_IO_TYPE_NVME_ADMIN, true); 1130 1131 spdk_put_io_channel(io_ch); 1132 spdk_bdev_close(desc); 1133 free_bdev(bdev); 1134 spdk_bdev_finish(bdev_fini_cb, NULL); 1135 poll_threads(); 1136 } 1137 1138 static void 1139 bdev_io_wait_test(void) 1140 { 1141 struct spdk_bdev *bdev; 1142 struct spdk_bdev_desc *desc = NULL; 1143 struct spdk_io_channel *io_ch; 1144 struct spdk_bdev_opts bdev_opts = {}; 1145 struct bdev_ut_io_wait_entry io_wait_entry; 1146 struct bdev_ut_io_wait_entry io_wait_entry2; 1147 int rc; 1148 1149 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 1150 bdev_opts.bdev_io_pool_size = 4; 1151 bdev_opts.bdev_io_cache_size = 2; 1152 1153 rc = spdk_bdev_set_opts(&bdev_opts); 1154 CU_ASSERT(rc == 0); 1155 spdk_bdev_initialize(bdev_init_cb, NULL); 1156 poll_threads(); 1157 1158 bdev = allocate_bdev("bdev0"); 1159 1160 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 1161 CU_ASSERT(rc == 0); 1162 poll_threads(); 1163 SPDK_CU_ASSERT_FATAL(desc != NULL); 1164 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 1165 io_ch = spdk_bdev_get_io_channel(desc); 1166 CU_ASSERT(io_ch != NULL); 1167 1168 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 1169 CU_ASSERT(rc == 0); 1170 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 1171 CU_ASSERT(rc == 0); 1172 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 1173 CU_ASSERT(rc == 0); 1174 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 1175 CU_ASSERT(rc == 0); 1176 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 1177 1178 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 1179 CU_ASSERT(rc == -ENOMEM); 1180 1181 io_wait_entry.entry.bdev = bdev; 1182 io_wait_entry.entry.cb_fn = io_wait_cb; 1183 io_wait_entry.entry.cb_arg = &io_wait_entry; 1184 io_wait_entry.io_ch = io_ch; 1185 io_wait_entry.desc = desc; 1186 io_wait_entry.submitted = false; 1187 /* Cannot use the same io_wait_entry for two different calls. */ 1188 memcpy(&io_wait_entry2, &io_wait_entry, sizeof(io_wait_entry)); 1189 io_wait_entry2.entry.cb_arg = &io_wait_entry2; 1190 1191 /* Queue two I/O waits. */ 1192 rc = spdk_bdev_queue_io_wait(bdev, io_ch, &io_wait_entry.entry); 1193 CU_ASSERT(rc == 0); 1194 CU_ASSERT(io_wait_entry.submitted == false); 1195 rc = spdk_bdev_queue_io_wait(bdev, io_ch, &io_wait_entry2.entry); 1196 CU_ASSERT(rc == 0); 1197 CU_ASSERT(io_wait_entry2.submitted == false); 1198 1199 stub_complete_io(1); 1200 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 1201 CU_ASSERT(io_wait_entry.submitted == true); 1202 CU_ASSERT(io_wait_entry2.submitted == false); 1203 1204 stub_complete_io(1); 1205 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 1206 CU_ASSERT(io_wait_entry2.submitted == true); 1207 1208 stub_complete_io(4); 1209 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1210 1211 spdk_put_io_channel(io_ch); 1212 spdk_bdev_close(desc); 1213 free_bdev(bdev); 1214 spdk_bdev_finish(bdev_fini_cb, NULL); 1215 poll_threads(); 1216 } 1217 1218 static void 1219 bdev_io_spans_split_test(void) 1220 { 1221 struct spdk_bdev bdev; 1222 struct spdk_bdev_io bdev_io; 1223 struct iovec iov[BDEV_IO_NUM_CHILD_IOV]; 1224 1225 memset(&bdev, 0, sizeof(bdev)); 1226 bdev_io.u.bdev.iovs = iov; 1227 1228 bdev_io.type = SPDK_BDEV_IO_TYPE_READ; 1229 bdev.optimal_io_boundary = 0; 1230 bdev.max_segment_size = 0; 1231 bdev.max_num_segments = 0; 1232 bdev_io.bdev = &bdev; 1233 1234 /* bdev has no optimal_io_boundary and max_size set - so this should return false. */ 1235 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1236 1237 bdev.split_on_optimal_io_boundary = true; 1238 bdev.optimal_io_boundary = 32; 1239 bdev_io.type = SPDK_BDEV_IO_TYPE_RESET; 1240 1241 /* RESETs are not based on LBAs - so this should return false. */ 1242 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1243 1244 bdev_io.type = SPDK_BDEV_IO_TYPE_READ; 1245 bdev_io.u.bdev.offset_blocks = 0; 1246 bdev_io.u.bdev.num_blocks = 32; 1247 1248 /* This I/O run right up to, but does not cross, the boundary - so this should return false. */ 1249 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1250 1251 bdev_io.u.bdev.num_blocks = 33; 1252 1253 /* This I/O spans a boundary. */ 1254 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1255 1256 bdev_io.u.bdev.num_blocks = 32; 1257 bdev.max_segment_size = 512 * 32; 1258 bdev.max_num_segments = 1; 1259 bdev_io.u.bdev.iovcnt = 1; 1260 iov[0].iov_len = 512; 1261 1262 /* Does not cross and exceed max_size or max_segs */ 1263 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1264 1265 bdev.split_on_optimal_io_boundary = false; 1266 bdev.max_segment_size = 512; 1267 bdev.max_num_segments = 1; 1268 bdev_io.u.bdev.iovcnt = 2; 1269 1270 /* Exceed max_segs */ 1271 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1272 1273 bdev.max_num_segments = 2; 1274 iov[0].iov_len = 513; 1275 iov[1].iov_len = 512; 1276 1277 /* Exceed max_sizes */ 1278 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1279 1280 bdev.max_segment_size = 0; 1281 bdev.write_unit_size = 32; 1282 bdev.split_on_write_unit = true; 1283 bdev_io.type = SPDK_BDEV_IO_TYPE_WRITE; 1284 1285 /* This I/O is one write unit */ 1286 CU_ASSERT(bdev_io_should_split(&bdev_io) == false); 1287 1288 bdev_io.u.bdev.num_blocks = 32 * 2; 1289 1290 /* This I/O is more than one write unit */ 1291 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1292 1293 bdev_io.u.bdev.offset_blocks = 1; 1294 bdev_io.u.bdev.num_blocks = 32; 1295 1296 /* This I/O is not aligned to write unit size */ 1297 CU_ASSERT(bdev_io_should_split(&bdev_io) == true); 1298 } 1299 1300 static void 1301 bdev_io_boundary_split_test(void) 1302 { 1303 struct spdk_bdev *bdev; 1304 struct spdk_bdev_desc *desc = NULL; 1305 struct spdk_io_channel *io_ch; 1306 struct spdk_bdev_opts bdev_opts = {}; 1307 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 1308 struct ut_expected_io *expected_io; 1309 void *md_buf = (void *)0xFF000000; 1310 uint64_t i; 1311 int rc; 1312 1313 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 1314 bdev_opts.bdev_io_pool_size = 512; 1315 bdev_opts.bdev_io_cache_size = 64; 1316 1317 rc = spdk_bdev_set_opts(&bdev_opts); 1318 CU_ASSERT(rc == 0); 1319 spdk_bdev_initialize(bdev_init_cb, NULL); 1320 1321 bdev = allocate_bdev("bdev0"); 1322 1323 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 1324 CU_ASSERT(rc == 0); 1325 SPDK_CU_ASSERT_FATAL(desc != NULL); 1326 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 1327 io_ch = spdk_bdev_get_io_channel(desc); 1328 CU_ASSERT(io_ch != NULL); 1329 1330 bdev->optimal_io_boundary = 16; 1331 bdev->split_on_optimal_io_boundary = false; 1332 1333 g_io_done = false; 1334 1335 /* First test that the I/O does not get split if split_on_optimal_io_boundary == false. */ 1336 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 8, 1); 1337 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 8 * 512); 1338 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1339 1340 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL); 1341 CU_ASSERT(rc == 0); 1342 CU_ASSERT(g_io_done == false); 1343 1344 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1345 stub_complete_io(1); 1346 CU_ASSERT(g_io_done == true); 1347 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1348 1349 bdev->split_on_optimal_io_boundary = true; 1350 bdev->md_interleave = false; 1351 bdev->md_len = 8; 1352 1353 /* Now test that a single-vector command is split correctly. 1354 * Offset 14, length 8, payload 0xF000 1355 * Child - Offset 14, length 2, payload 0xF000 1356 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 1357 * 1358 * Set up the expected values before calling spdk_bdev_read_blocks 1359 */ 1360 g_io_done = false; 1361 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 1); 1362 expected_io->md_buf = md_buf; 1363 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 2 * 512); 1364 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1365 1366 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 6, 1); 1367 expected_io->md_buf = md_buf + 2 * 8; 1368 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 6 * 512); 1369 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1370 1371 /* spdk_bdev_read_blocks will submit the first child immediately. */ 1372 rc = spdk_bdev_read_blocks_with_md(desc, io_ch, (void *)0xF000, md_buf, 1373 14, 8, io_done, NULL); 1374 CU_ASSERT(rc == 0); 1375 CU_ASSERT(g_io_done == false); 1376 1377 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1378 stub_complete_io(2); 1379 CU_ASSERT(g_io_done == true); 1380 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1381 1382 /* Now set up a more complex, multi-vector command that needs to be split, 1383 * including splitting iovecs. 1384 */ 1385 iov[0].iov_base = (void *)0x10000; 1386 iov[0].iov_len = 512; 1387 iov[1].iov_base = (void *)0x20000; 1388 iov[1].iov_len = 20 * 512; 1389 iov[2].iov_base = (void *)0x30000; 1390 iov[2].iov_len = 11 * 512; 1391 1392 g_io_done = false; 1393 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 2); 1394 expected_io->md_buf = md_buf; 1395 ut_expected_io_set_iov(expected_io, 0, (void *)0x10000, 512); 1396 ut_expected_io_set_iov(expected_io, 1, (void *)0x20000, 512); 1397 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1398 1399 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 16, 1); 1400 expected_io->md_buf = md_buf + 2 * 8; 1401 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 512), 16 * 512); 1402 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1403 1404 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 14, 2); 1405 expected_io->md_buf = md_buf + 18 * 8; 1406 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 17 * 512), 3 * 512); 1407 ut_expected_io_set_iov(expected_io, 1, (void *)0x30000, 11 * 512); 1408 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1409 1410 rc = spdk_bdev_writev_blocks_with_md(desc, io_ch, iov, 3, md_buf, 1411 14, 32, io_done, NULL); 1412 CU_ASSERT(rc == 0); 1413 CU_ASSERT(g_io_done == false); 1414 1415 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 1416 stub_complete_io(3); 1417 CU_ASSERT(g_io_done == true); 1418 1419 /* Test multi vector command that needs to be split by strip and then needs to be 1420 * split further due to the capacity of child iovs. 1421 */ 1422 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV * 2; i++) { 1423 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1424 iov[i].iov_len = 512; 1425 } 1426 1427 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1428 g_io_done = false; 1429 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, BDEV_IO_NUM_CHILD_IOV, 1430 BDEV_IO_NUM_CHILD_IOV); 1431 expected_io->md_buf = md_buf; 1432 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1433 ut_expected_io_set_iov(expected_io, i, (void *)((i + 1) * 0x10000), 512); 1434 } 1435 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1436 1437 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1438 BDEV_IO_NUM_CHILD_IOV, BDEV_IO_NUM_CHILD_IOV); 1439 expected_io->md_buf = md_buf + BDEV_IO_NUM_CHILD_IOV * 8; 1440 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1441 ut_expected_io_set_iov(expected_io, i, 1442 (void *)((i + 1 + BDEV_IO_NUM_CHILD_IOV) * 0x10000), 512); 1443 } 1444 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1445 1446 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, md_buf, 1447 0, BDEV_IO_NUM_CHILD_IOV * 2, io_done, NULL); 1448 CU_ASSERT(rc == 0); 1449 CU_ASSERT(g_io_done == false); 1450 1451 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1452 stub_complete_io(1); 1453 CU_ASSERT(g_io_done == false); 1454 1455 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1456 stub_complete_io(1); 1457 CU_ASSERT(g_io_done == true); 1458 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1459 1460 /* Test multi vector command that needs to be split by strip and then needs to be 1461 * split further due to the capacity of child iovs. In this case, the length of 1462 * the rest of iovec array with an I/O boundary is the multiple of block size. 1463 */ 1464 1465 /* Fill iovec array for exactly one boundary. The iovec cnt for this boundary 1466 * is BDEV_IO_NUM_CHILD_IOV + 1, which exceeds the capacity of child iovs. 1467 */ 1468 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1469 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1470 iov[i].iov_len = 512; 1471 } 1472 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1473 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1474 iov[i].iov_len = 256; 1475 } 1476 iov[BDEV_IO_NUM_CHILD_IOV].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000); 1477 iov[BDEV_IO_NUM_CHILD_IOV].iov_len = 512; 1478 1479 /* Add an extra iovec to trigger split */ 1480 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000); 1481 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_len = 512; 1482 1483 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1484 g_io_done = false; 1485 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 1486 BDEV_IO_NUM_CHILD_IOV - 1, BDEV_IO_NUM_CHILD_IOV); 1487 expected_io->md_buf = md_buf; 1488 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1489 ut_expected_io_set_iov(expected_io, i, 1490 (void *)((i + 1) * 0x10000), 512); 1491 } 1492 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 1493 ut_expected_io_set_iov(expected_io, i, 1494 (void *)((i + 1) * 0x10000), 256); 1495 } 1496 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1497 1498 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV - 1, 1499 1, 1); 1500 expected_io->md_buf = md_buf + (BDEV_IO_NUM_CHILD_IOV - 1) * 8; 1501 ut_expected_io_set_iov(expected_io, 0, 1502 (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000), 512); 1503 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1504 1505 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1506 1, 1); 1507 expected_io->md_buf = md_buf + BDEV_IO_NUM_CHILD_IOV * 8; 1508 ut_expected_io_set_iov(expected_io, 0, 1509 (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000), 512); 1510 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1511 1512 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 2, md_buf, 1513 0, BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1514 CU_ASSERT(rc == 0); 1515 CU_ASSERT(g_io_done == false); 1516 1517 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1518 stub_complete_io(1); 1519 CU_ASSERT(g_io_done == false); 1520 1521 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1522 stub_complete_io(2); 1523 CU_ASSERT(g_io_done == true); 1524 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1525 1526 /* Test multi vector command that needs to be split by strip and then needs to be 1527 * split further due to the capacity of child iovs, the child request offset should 1528 * be rewind to last aligned offset and go success without error. 1529 */ 1530 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 1; i++) { 1531 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1532 iov[i].iov_len = 512; 1533 } 1534 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_base = (void *)(BDEV_IO_NUM_CHILD_IOV * 0x10000); 1535 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_len = 256; 1536 1537 iov[BDEV_IO_NUM_CHILD_IOV].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000); 1538 iov[BDEV_IO_NUM_CHILD_IOV].iov_len = 256; 1539 1540 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000); 1541 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_len = 512; 1542 1543 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1544 g_io_done = false; 1545 g_io_status = 0; 1546 /* The first expected io should be start from offset 0 to BDEV_IO_NUM_CHILD_IOV - 1 */ 1547 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 1548 BDEV_IO_NUM_CHILD_IOV - 1, BDEV_IO_NUM_CHILD_IOV - 1); 1549 expected_io->md_buf = md_buf; 1550 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 1; i++) { 1551 ut_expected_io_set_iov(expected_io, i, 1552 (void *)((i + 1) * 0x10000), 512); 1553 } 1554 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1555 /* The second expected io should be start from offset BDEV_IO_NUM_CHILD_IOV - 1 to BDEV_IO_NUM_CHILD_IOV */ 1556 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV - 1, 1557 1, 2); 1558 expected_io->md_buf = md_buf + (BDEV_IO_NUM_CHILD_IOV - 1) * 8; 1559 ut_expected_io_set_iov(expected_io, 0, 1560 (void *)(BDEV_IO_NUM_CHILD_IOV * 0x10000), 256); 1561 ut_expected_io_set_iov(expected_io, 1, 1562 (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000), 256); 1563 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1564 /* The third expected io should be start from offset BDEV_IO_NUM_CHILD_IOV to BDEV_IO_NUM_CHILD_IOV + 1 */ 1565 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1566 1, 1); 1567 expected_io->md_buf = md_buf + BDEV_IO_NUM_CHILD_IOV * 8; 1568 ut_expected_io_set_iov(expected_io, 0, 1569 (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000), 512); 1570 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1571 1572 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, md_buf, 1573 0, BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1574 CU_ASSERT(rc == 0); 1575 CU_ASSERT(g_io_done == false); 1576 1577 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1578 stub_complete_io(1); 1579 CU_ASSERT(g_io_done == false); 1580 1581 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 1582 stub_complete_io(2); 1583 CU_ASSERT(g_io_done == true); 1584 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1585 1586 /* Test multi vector command that needs to be split due to the IO boundary and 1587 * the capacity of child iovs. Especially test the case when the command is 1588 * split due to the capacity of child iovs, the tail address is not aligned with 1589 * block size and is rewinded to the aligned address. 1590 * 1591 * The iovecs used in read request is complex but is based on the data 1592 * collected in the real issue. We change the base addresses but keep the lengths 1593 * not to loose the credibility of the test. 1594 */ 1595 bdev->optimal_io_boundary = 128; 1596 g_io_done = false; 1597 g_io_status = 0; 1598 1599 for (i = 0; i < 31; i++) { 1600 iov[i].iov_base = (void *)(0xFEED0000000 + (i << 20)); 1601 iov[i].iov_len = 1024; 1602 } 1603 iov[31].iov_base = (void *)0xFEED1F00000; 1604 iov[31].iov_len = 32768; 1605 iov[32].iov_base = (void *)0xFEED2000000; 1606 iov[32].iov_len = 160; 1607 iov[33].iov_base = (void *)0xFEED2100000; 1608 iov[33].iov_len = 4096; 1609 iov[34].iov_base = (void *)0xFEED2200000; 1610 iov[34].iov_len = 4096; 1611 iov[35].iov_base = (void *)0xFEED2300000; 1612 iov[35].iov_len = 4096; 1613 iov[36].iov_base = (void *)0xFEED2400000; 1614 iov[36].iov_len = 4096; 1615 iov[37].iov_base = (void *)0xFEED2500000; 1616 iov[37].iov_len = 4096; 1617 iov[38].iov_base = (void *)0xFEED2600000; 1618 iov[38].iov_len = 4096; 1619 iov[39].iov_base = (void *)0xFEED2700000; 1620 iov[39].iov_len = 4096; 1621 iov[40].iov_base = (void *)0xFEED2800000; 1622 iov[40].iov_len = 4096; 1623 iov[41].iov_base = (void *)0xFEED2900000; 1624 iov[41].iov_len = 4096; 1625 iov[42].iov_base = (void *)0xFEED2A00000; 1626 iov[42].iov_len = 4096; 1627 iov[43].iov_base = (void *)0xFEED2B00000; 1628 iov[43].iov_len = 12288; 1629 iov[44].iov_base = (void *)0xFEED2C00000; 1630 iov[44].iov_len = 8192; 1631 iov[45].iov_base = (void *)0xFEED2F00000; 1632 iov[45].iov_len = 4096; 1633 iov[46].iov_base = (void *)0xFEED3000000; 1634 iov[46].iov_len = 4096; 1635 iov[47].iov_base = (void *)0xFEED3100000; 1636 iov[47].iov_len = 4096; 1637 iov[48].iov_base = (void *)0xFEED3200000; 1638 iov[48].iov_len = 24576; 1639 iov[49].iov_base = (void *)0xFEED3300000; 1640 iov[49].iov_len = 16384; 1641 iov[50].iov_base = (void *)0xFEED3400000; 1642 iov[50].iov_len = 12288; 1643 iov[51].iov_base = (void *)0xFEED3500000; 1644 iov[51].iov_len = 4096; 1645 iov[52].iov_base = (void *)0xFEED3600000; 1646 iov[52].iov_len = 4096; 1647 iov[53].iov_base = (void *)0xFEED3700000; 1648 iov[53].iov_len = 4096; 1649 iov[54].iov_base = (void *)0xFEED3800000; 1650 iov[54].iov_len = 28672; 1651 iov[55].iov_base = (void *)0xFEED3900000; 1652 iov[55].iov_len = 20480; 1653 iov[56].iov_base = (void *)0xFEED3A00000; 1654 iov[56].iov_len = 4096; 1655 iov[57].iov_base = (void *)0xFEED3B00000; 1656 iov[57].iov_len = 12288; 1657 iov[58].iov_base = (void *)0xFEED3C00000; 1658 iov[58].iov_len = 4096; 1659 iov[59].iov_base = (void *)0xFEED3D00000; 1660 iov[59].iov_len = 4096; 1661 iov[60].iov_base = (void *)0xFEED3E00000; 1662 iov[60].iov_len = 352; 1663 1664 /* The 1st child IO must be from iov[0] to iov[31] split by the capacity 1665 * of child iovs, 1666 */ 1667 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 126, 32); 1668 expected_io->md_buf = md_buf; 1669 for (i = 0; i < 32; i++) { 1670 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 1671 } 1672 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1673 1674 /* The 2nd child IO must be from iov[32] to the first 864 bytes of iov[33] 1675 * split by the IO boundary requirement. 1676 */ 1677 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 126, 2, 2); 1678 expected_io->md_buf = md_buf + 126 * 8; 1679 ut_expected_io_set_iov(expected_io, 0, iov[32].iov_base, iov[32].iov_len); 1680 ut_expected_io_set_iov(expected_io, 1, iov[33].iov_base, 864); 1681 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1682 1683 /* The 3rd child IO must be from the remaining 3232 bytes of iov[33] to 1684 * the first 864 bytes of iov[46] split by the IO boundary requirement. 1685 */ 1686 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 128, 128, 14); 1687 expected_io->md_buf = md_buf + 128 * 8; 1688 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[33].iov_base + 864), 1689 iov[33].iov_len - 864); 1690 ut_expected_io_set_iov(expected_io, 1, iov[34].iov_base, iov[34].iov_len); 1691 ut_expected_io_set_iov(expected_io, 2, iov[35].iov_base, iov[35].iov_len); 1692 ut_expected_io_set_iov(expected_io, 3, iov[36].iov_base, iov[36].iov_len); 1693 ut_expected_io_set_iov(expected_io, 4, iov[37].iov_base, iov[37].iov_len); 1694 ut_expected_io_set_iov(expected_io, 5, iov[38].iov_base, iov[38].iov_len); 1695 ut_expected_io_set_iov(expected_io, 6, iov[39].iov_base, iov[39].iov_len); 1696 ut_expected_io_set_iov(expected_io, 7, iov[40].iov_base, iov[40].iov_len); 1697 ut_expected_io_set_iov(expected_io, 8, iov[41].iov_base, iov[41].iov_len); 1698 ut_expected_io_set_iov(expected_io, 9, iov[42].iov_base, iov[42].iov_len); 1699 ut_expected_io_set_iov(expected_io, 10, iov[43].iov_base, iov[43].iov_len); 1700 ut_expected_io_set_iov(expected_io, 11, iov[44].iov_base, iov[44].iov_len); 1701 ut_expected_io_set_iov(expected_io, 12, iov[45].iov_base, iov[45].iov_len); 1702 ut_expected_io_set_iov(expected_io, 13, iov[46].iov_base, 864); 1703 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1704 1705 /* The 4th child IO must be from the remaining 3232 bytes of iov[46] to the 1706 * first 864 bytes of iov[52] split by the IO boundary requirement. 1707 */ 1708 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 256, 128, 7); 1709 expected_io->md_buf = md_buf + 256 * 8; 1710 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[46].iov_base + 864), 1711 iov[46].iov_len - 864); 1712 ut_expected_io_set_iov(expected_io, 1, iov[47].iov_base, iov[47].iov_len); 1713 ut_expected_io_set_iov(expected_io, 2, iov[48].iov_base, iov[48].iov_len); 1714 ut_expected_io_set_iov(expected_io, 3, iov[49].iov_base, iov[49].iov_len); 1715 ut_expected_io_set_iov(expected_io, 4, iov[50].iov_base, iov[50].iov_len); 1716 ut_expected_io_set_iov(expected_io, 5, iov[51].iov_base, iov[51].iov_len); 1717 ut_expected_io_set_iov(expected_io, 6, iov[52].iov_base, 864); 1718 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1719 1720 /* The 5th child IO must be from the remaining 3232 bytes of iov[52] to 1721 * the first 4096 bytes of iov[57] split by the IO boundary requirement. 1722 */ 1723 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 384, 128, 6); 1724 expected_io->md_buf = md_buf + 384 * 8; 1725 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[52].iov_base + 864), 1726 iov[52].iov_len - 864); 1727 ut_expected_io_set_iov(expected_io, 1, iov[53].iov_base, iov[53].iov_len); 1728 ut_expected_io_set_iov(expected_io, 2, iov[54].iov_base, iov[54].iov_len); 1729 ut_expected_io_set_iov(expected_io, 3, iov[55].iov_base, iov[55].iov_len); 1730 ut_expected_io_set_iov(expected_io, 4, iov[56].iov_base, iov[56].iov_len); 1731 ut_expected_io_set_iov(expected_io, 5, iov[57].iov_base, 4960); 1732 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1733 1734 /* The 6th child IO must be from the remaining 7328 bytes of iov[57] 1735 * to the first 3936 bytes of iov[58] split by the capacity of child iovs. 1736 */ 1737 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 512, 30, 3); 1738 expected_io->md_buf = md_buf + 512 * 8; 1739 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[57].iov_base + 4960), 1740 iov[57].iov_len - 4960); 1741 ut_expected_io_set_iov(expected_io, 1, iov[58].iov_base, iov[58].iov_len); 1742 ut_expected_io_set_iov(expected_io, 2, iov[59].iov_base, 3936); 1743 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1744 1745 /* The 7th child IO is from the remaining 160 bytes of iov[59] and iov[60]. */ 1746 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 542, 1, 2); 1747 expected_io->md_buf = md_buf + 542 * 8; 1748 ut_expected_io_set_iov(expected_io, 0, (void *)((uintptr_t)iov[59].iov_base + 3936), 1749 iov[59].iov_len - 3936); 1750 ut_expected_io_set_iov(expected_io, 1, iov[60].iov_base, iov[60].iov_len); 1751 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1752 1753 rc = spdk_bdev_readv_blocks_with_md(desc, io_ch, iov, 61, md_buf, 1754 0, 543, io_done, NULL); 1755 CU_ASSERT(rc == 0); 1756 CU_ASSERT(g_io_done == false); 1757 1758 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1759 stub_complete_io(1); 1760 CU_ASSERT(g_io_done == false); 1761 1762 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 5); 1763 stub_complete_io(5); 1764 CU_ASSERT(g_io_done == false); 1765 1766 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1767 stub_complete_io(1); 1768 CU_ASSERT(g_io_done == true); 1769 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1770 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 1771 1772 /* Test a WRITE_ZEROES that would span an I/O boundary. WRITE_ZEROES should not be 1773 * split, so test that. 1774 */ 1775 bdev->optimal_io_boundary = 15; 1776 g_io_done = false; 1777 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, 9, 36, 0); 1778 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1779 1780 rc = spdk_bdev_write_zeroes_blocks(desc, io_ch, 9, 36, io_done, NULL); 1781 CU_ASSERT(rc == 0); 1782 CU_ASSERT(g_io_done == false); 1783 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1784 stub_complete_io(1); 1785 CU_ASSERT(g_io_done == true); 1786 1787 /* Test an UNMAP. This should also not be split. */ 1788 bdev->optimal_io_boundary = 16; 1789 g_io_done = false; 1790 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, 15, 2, 0); 1791 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1792 1793 rc = spdk_bdev_unmap_blocks(desc, io_ch, 15, 2, io_done, NULL); 1794 CU_ASSERT(rc == 0); 1795 CU_ASSERT(g_io_done == false); 1796 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1797 stub_complete_io(1); 1798 CU_ASSERT(g_io_done == true); 1799 1800 /* Test a FLUSH. This should also not be split. */ 1801 bdev->optimal_io_boundary = 16; 1802 g_io_done = false; 1803 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_FLUSH, 15, 2, 0); 1804 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1805 1806 rc = spdk_bdev_flush_blocks(desc, io_ch, 15, 2, io_done, NULL); 1807 CU_ASSERT(rc == 0); 1808 CU_ASSERT(g_io_done == false); 1809 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1810 stub_complete_io(1); 1811 CU_ASSERT(g_io_done == true); 1812 1813 /* Test a COPY. This should also not be split. */ 1814 bdev->optimal_io_boundary = 15; 1815 g_io_done = false; 1816 expected_io = ut_alloc_expected_copy_io(SPDK_BDEV_IO_TYPE_COPY, 9, 45, 36); 1817 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1818 1819 rc = spdk_bdev_copy_blocks(desc, io_ch, 9, 45, 36, io_done, NULL); 1820 CU_ASSERT(rc == 0); 1821 CU_ASSERT(g_io_done == false); 1822 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1823 stub_complete_io(1); 1824 CU_ASSERT(g_io_done == true); 1825 1826 CU_ASSERT(TAILQ_EMPTY(&g_bdev_ut_channel->expected_io)); 1827 1828 /* Children requests return an error status */ 1829 bdev->optimal_io_boundary = 16; 1830 iov[0].iov_base = (void *)0x10000; 1831 iov[0].iov_len = 512 * 64; 1832 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 1833 g_io_done = false; 1834 g_io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1835 1836 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, 1, 1, 64, io_done, NULL); 1837 CU_ASSERT(rc == 0); 1838 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 5); 1839 stub_complete_io(4); 1840 CU_ASSERT(g_io_done == false); 1841 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 1842 stub_complete_io(1); 1843 CU_ASSERT(g_io_done == true); 1844 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 1845 1846 /* Test if a multi vector command terminated with failure before continuing 1847 * splitting process when one of child I/O failed. 1848 * The multi vector command is as same as the above that needs to be split by strip 1849 * and then needs to be split further due to the capacity of child iovs. 1850 */ 1851 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 1; i++) { 1852 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1853 iov[i].iov_len = 512; 1854 } 1855 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_base = (void *)(BDEV_IO_NUM_CHILD_IOV * 0x10000); 1856 iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_len = 256; 1857 1858 iov[BDEV_IO_NUM_CHILD_IOV].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 1) * 0x10000); 1859 iov[BDEV_IO_NUM_CHILD_IOV].iov_len = 256; 1860 1861 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_base = (void *)((BDEV_IO_NUM_CHILD_IOV + 2) * 0x10000); 1862 iov[BDEV_IO_NUM_CHILD_IOV + 1].iov_len = 512; 1863 1864 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 1865 1866 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 1867 g_io_done = false; 1868 g_io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1869 1870 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, 0, 1871 BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1872 CU_ASSERT(rc == 0); 1873 CU_ASSERT(g_io_done == false); 1874 1875 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1876 stub_complete_io(1); 1877 CU_ASSERT(g_io_done == true); 1878 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 1879 1880 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 1881 1882 /* for this test we will create the following conditions to hit the code path where 1883 * we are trying to send and IO following a split that has no iovs because we had to 1884 * trim them for alignment reasons. 1885 * 1886 * - 16K boundary, our IO will start at offset 0 with a length of 0x4200 1887 * - Our IOVs are 0x212 in size so that we run into the 16K boundary at child IOV 1888 * position 30 and overshoot by 0x2e. 1889 * - That means we'll send the IO and loop back to pick up the remaining bytes at 1890 * child IOV index 31. When we do, we find that we have to shorten index 31 by 0x2e 1891 * which eliniates that vector so we just send the first split IO with 30 vectors 1892 * and let the completion pick up the last 2 vectors. 1893 */ 1894 bdev->optimal_io_boundary = 32; 1895 bdev->split_on_optimal_io_boundary = true; 1896 g_io_done = false; 1897 1898 /* Init all parent IOVs to 0x212 */ 1899 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV + 2; i++) { 1900 iov[i].iov_base = (void *)((i + 1) * 0x10000); 1901 iov[i].iov_len = 0x212; 1902 } 1903 1904 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, BDEV_IO_NUM_CHILD_IOV, 1905 BDEV_IO_NUM_CHILD_IOV - 1); 1906 /* expect 0-29 to be 1:1 with the parent iov */ 1907 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 1908 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 1909 } 1910 1911 /* expect index 30 to be shortened to 0x1e4 (0x212 - 0x1e) because of the alignment 1912 * where 0x1e is the amount we overshot the 16K boundary 1913 */ 1914 ut_expected_io_set_iov(expected_io, BDEV_IO_NUM_CHILD_IOV - 2, 1915 (void *)(iov[BDEV_IO_NUM_CHILD_IOV - 2].iov_base), 0x1e4); 1916 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1917 1918 /* 2nd child IO will have 2 remaining vectors, one to pick up from the one that was 1919 * shortened that take it to the next boundary and then a final one to get us to 1920 * 0x4200 bytes for the IO. 1921 */ 1922 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 1923 BDEV_IO_NUM_CHILD_IOV, 2); 1924 /* position 30 picked up the remaining bytes to the next boundary */ 1925 ut_expected_io_set_iov(expected_io, 0, 1926 (void *)(iov[BDEV_IO_NUM_CHILD_IOV - 2].iov_base + 0x1e4), 0x2e); 1927 1928 /* position 31 picked the the rest of the transfer to get us to 0x4200 */ 1929 ut_expected_io_set_iov(expected_io, 1, 1930 (void *)(iov[BDEV_IO_NUM_CHILD_IOV - 1].iov_base), 0x1d2); 1931 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1932 1933 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 1, 0, 1934 BDEV_IO_NUM_CHILD_IOV + 1, io_done, NULL); 1935 CU_ASSERT(rc == 0); 1936 CU_ASSERT(g_io_done == false); 1937 1938 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1939 stub_complete_io(1); 1940 CU_ASSERT(g_io_done == false); 1941 1942 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 1943 stub_complete_io(1); 1944 CU_ASSERT(g_io_done == true); 1945 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 1946 1947 spdk_put_io_channel(io_ch); 1948 spdk_bdev_close(desc); 1949 free_bdev(bdev); 1950 spdk_bdev_finish(bdev_fini_cb, NULL); 1951 poll_threads(); 1952 } 1953 1954 static void 1955 bdev_io_max_size_and_segment_split_test(void) 1956 { 1957 struct spdk_bdev *bdev; 1958 struct spdk_bdev_desc *desc = NULL; 1959 struct spdk_io_channel *io_ch; 1960 struct spdk_bdev_opts bdev_opts = {}; 1961 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 1962 struct ut_expected_io *expected_io; 1963 uint64_t i; 1964 int rc; 1965 1966 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 1967 bdev_opts.bdev_io_pool_size = 512; 1968 bdev_opts.bdev_io_cache_size = 64; 1969 1970 bdev_opts.opts_size = sizeof(bdev_opts); 1971 rc = spdk_bdev_set_opts(&bdev_opts); 1972 CU_ASSERT(rc == 0); 1973 spdk_bdev_initialize(bdev_init_cb, NULL); 1974 1975 bdev = allocate_bdev("bdev0"); 1976 1977 rc = spdk_bdev_open_ext(bdev->name, true, bdev_ut_event_cb, NULL, &desc); 1978 CU_ASSERT(rc == 0); 1979 SPDK_CU_ASSERT_FATAL(desc != NULL); 1980 io_ch = spdk_bdev_get_io_channel(desc); 1981 CU_ASSERT(io_ch != NULL); 1982 1983 bdev->split_on_optimal_io_boundary = false; 1984 bdev->optimal_io_boundary = 0; 1985 1986 /* Case 0 max_num_segments == 0. 1987 * but segment size 2 * 512 > 512 1988 */ 1989 bdev->max_segment_size = 512; 1990 bdev->max_num_segments = 0; 1991 g_io_done = false; 1992 1993 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 2); 1994 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512); 1995 ut_expected_io_set_iov(expected_io, 1, (void *)(0xF000 + 512), 512); 1996 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 1997 1998 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 2, io_done, NULL); 1999 CU_ASSERT(rc == 0); 2000 CU_ASSERT(g_io_done == false); 2001 2002 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2003 stub_complete_io(1); 2004 CU_ASSERT(g_io_done == true); 2005 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2006 2007 /* Case 1 max_segment_size == 0 2008 * but iov num 2 > 1. 2009 */ 2010 bdev->max_segment_size = 0; 2011 bdev->max_num_segments = 1; 2012 g_io_done = false; 2013 2014 iov[0].iov_base = (void *)0x10000; 2015 iov[0].iov_len = 512; 2016 iov[1].iov_base = (void *)0x20000; 2017 iov[1].iov_len = 8 * 512; 2018 2019 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 1, 1); 2020 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, iov[0].iov_len); 2021 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2022 2023 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 15, 8, 1); 2024 ut_expected_io_set_iov(expected_io, 0, iov[1].iov_base, iov[1].iov_len); 2025 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2026 2027 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, 2, 14, 9, io_done, NULL); 2028 CU_ASSERT(rc == 0); 2029 CU_ASSERT(g_io_done == false); 2030 2031 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2032 stub_complete_io(2); 2033 CU_ASSERT(g_io_done == true); 2034 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2035 2036 /* Test that a non-vector command is split correctly. 2037 * Set up the expected values before calling spdk_bdev_read_blocks 2038 */ 2039 bdev->max_segment_size = 512; 2040 bdev->max_num_segments = 1; 2041 g_io_done = false; 2042 2043 /* Child IO 0 */ 2044 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 1, 1); 2045 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512); 2046 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2047 2048 /* Child IO 1 */ 2049 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 15, 1, 1); 2050 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 1 * 512), 512); 2051 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2052 2053 /* spdk_bdev_read_blocks will submit the first child immediately. */ 2054 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 2, io_done, NULL); 2055 CU_ASSERT(rc == 0); 2056 CU_ASSERT(g_io_done == false); 2057 2058 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2059 stub_complete_io(2); 2060 CU_ASSERT(g_io_done == true); 2061 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2062 2063 /* Now set up a more complex, multi-vector command that needs to be split, 2064 * including splitting iovecs. 2065 */ 2066 bdev->max_segment_size = 2 * 512; 2067 bdev->max_num_segments = 1; 2068 g_io_done = false; 2069 2070 iov[0].iov_base = (void *)0x10000; 2071 iov[0].iov_len = 2 * 512; 2072 iov[1].iov_base = (void *)0x20000; 2073 iov[1].iov_len = 4 * 512; 2074 iov[2].iov_base = (void *)0x30000; 2075 iov[2].iov_len = 6 * 512; 2076 2077 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 1); 2078 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, 512 * 2); 2079 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2080 2081 /* Split iov[1].size to 2 iov entries then split the segments */ 2082 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 2, 1); 2083 ut_expected_io_set_iov(expected_io, 0, iov[1].iov_base, 512 * 2); 2084 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2085 2086 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 18, 2, 1); 2087 ut_expected_io_set_iov(expected_io, 0, iov[1].iov_base + 512 * 2, 512 * 2); 2088 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2089 2090 /* Split iov[2].size to 3 iov entries then split the segments */ 2091 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 20, 2, 1); 2092 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base, 512 * 2); 2093 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2094 2095 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 22, 2, 1); 2096 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 2, 512 * 2); 2097 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2098 2099 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 24, 2, 1); 2100 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 4, 512 * 2); 2101 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2102 2103 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 3, 14, 12, io_done, NULL); 2104 CU_ASSERT(rc == 0); 2105 CU_ASSERT(g_io_done == false); 2106 2107 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 6); 2108 stub_complete_io(6); 2109 CU_ASSERT(g_io_done == true); 2110 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2111 2112 /* Test multi vector command that needs to be split by strip and then needs to be 2113 * split further due to the capacity of parent IO child iovs. 2114 */ 2115 bdev->max_segment_size = 512; 2116 bdev->max_num_segments = 1; 2117 g_io_done = false; 2118 2119 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV; i++) { 2120 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2121 iov[i].iov_len = 512 * 2; 2122 } 2123 2124 /* Each input iov.size is split into 2 iovs, 2125 * half of the input iov can fill all child iov entries of a single IO. 2126 */ 2127 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV / 2; i++) { 2128 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 2 * i, 1, 1); 2129 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base, 512); 2130 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2131 2132 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 2 * i + 1, 1, 1); 2133 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base + 512, 512); 2134 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2135 } 2136 2137 /* The remaining iov is split in the second round */ 2138 for (i = BDEV_IO_NUM_CHILD_IOV / 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 2139 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 2, 1, 1); 2140 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base, 512); 2141 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2142 2143 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 2 + 1, 1, 1); 2144 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base + 512, 512); 2145 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2146 } 2147 2148 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV, 0, 2149 BDEV_IO_NUM_CHILD_IOV * 2, io_done, NULL); 2150 CU_ASSERT(rc == 0); 2151 CU_ASSERT(g_io_done == false); 2152 2153 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == BDEV_IO_NUM_CHILD_IOV); 2154 stub_complete_io(BDEV_IO_NUM_CHILD_IOV); 2155 CU_ASSERT(g_io_done == false); 2156 2157 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == BDEV_IO_NUM_CHILD_IOV); 2158 stub_complete_io(BDEV_IO_NUM_CHILD_IOV); 2159 CU_ASSERT(g_io_done == true); 2160 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2161 2162 /* A wrong case, a child IO that is divided does 2163 * not meet the principle of multiples of block size, 2164 * and exits with error 2165 */ 2166 bdev->max_segment_size = 512; 2167 bdev->max_num_segments = 1; 2168 g_io_done = false; 2169 2170 iov[0].iov_base = (void *)0x10000; 2171 iov[0].iov_len = 512 + 256; 2172 iov[1].iov_base = (void *)0x20000; 2173 iov[1].iov_len = 256; 2174 2175 /* iov[0] is split to 512 and 256. 2176 * 256 is less than a block size, and it is found 2177 * in the next round of split that it is the first child IO smaller than 2178 * the block size, so the error exit 2179 */ 2180 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 1, 1); 2181 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, 512); 2182 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2183 2184 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, 2, 0, 2, io_done, NULL); 2185 CU_ASSERT(rc == 0); 2186 CU_ASSERT(g_io_done == false); 2187 2188 /* First child IO is OK */ 2189 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2190 stub_complete_io(1); 2191 CU_ASSERT(g_io_done == true); 2192 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2193 2194 /* error exit */ 2195 stub_complete_io(1); 2196 CU_ASSERT(g_io_done == true); 2197 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 2198 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2199 2200 /* Test multi vector command that needs to be split by strip and then needs to be 2201 * split further due to the capacity of child iovs. 2202 * 2203 * In this case, the last two iovs need to be split, but it will exceed the capacity 2204 * of child iovs, so it needs to wait until the first batch completed. 2205 */ 2206 bdev->max_segment_size = 512; 2207 bdev->max_num_segments = BDEV_IO_NUM_CHILD_IOV; 2208 g_io_done = false; 2209 2210 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2211 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2212 iov[i].iov_len = 512; 2213 } 2214 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV; i++) { 2215 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2216 iov[i].iov_len = 512 * 2; 2217 } 2218 2219 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 2220 BDEV_IO_NUM_CHILD_IOV, BDEV_IO_NUM_CHILD_IOV); 2221 /* 0 ~ (BDEV_IO_NUM_CHILD_IOV - 2) Will not be split */ 2222 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2223 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 2224 } 2225 /* (BDEV_IO_NUM_CHILD_IOV - 2) is split */ 2226 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, 512); 2227 ut_expected_io_set_iov(expected_io, i + 1, iov[i].iov_base + 512, 512); 2228 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2229 2230 /* Child iov entries exceed the max num of parent IO so split it in next round */ 2231 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV, 2, 2); 2232 ut_expected_io_set_iov(expected_io, 0, iov[i + 1].iov_base, 512); 2233 ut_expected_io_set_iov(expected_io, 1, iov[i + 1].iov_base + 512, 512); 2234 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2235 2236 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV, 0, 2237 BDEV_IO_NUM_CHILD_IOV + 2, io_done, NULL); 2238 CU_ASSERT(rc == 0); 2239 CU_ASSERT(g_io_done == false); 2240 2241 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2242 stub_complete_io(1); 2243 CU_ASSERT(g_io_done == false); 2244 2245 /* Next round */ 2246 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2247 stub_complete_io(1); 2248 CU_ASSERT(g_io_done == true); 2249 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2250 2251 /* This case is similar to the previous one, but the io composed of 2252 * the last few entries of child iov is not enough for a blocklen, so they 2253 * cannot be put into this IO, but wait until the next time. 2254 */ 2255 bdev->max_segment_size = 512; 2256 bdev->max_num_segments = BDEV_IO_NUM_CHILD_IOV; 2257 g_io_done = false; 2258 2259 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2260 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2261 iov[i].iov_len = 512; 2262 } 2263 2264 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV + 2; i++) { 2265 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2266 iov[i].iov_len = 128; 2267 } 2268 2269 /* First child iovcnt is't BDEV_IO_NUM_CHILD_IOV but BDEV_IO_NUM_CHILD_IOV - 2. 2270 * Because the left 2 iov is not enough for a blocklen. 2271 */ 2272 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 2273 BDEV_IO_NUM_CHILD_IOV - 2, BDEV_IO_NUM_CHILD_IOV - 2); 2274 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2275 ut_expected_io_set_iov(expected_io, i, iov[i].iov_base, iov[i].iov_len); 2276 } 2277 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2278 2279 /* The second child io waits until the end of the first child io before executing. 2280 * Because the iovcnt of the two IOs exceeds the child iovcnt of the parent IO. 2281 * BDEV_IO_NUM_CHILD_IOV - 2 to BDEV_IO_NUM_CHILD_IOV + 2 2282 */ 2283 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, BDEV_IO_NUM_CHILD_IOV - 2, 2284 1, 4); 2285 ut_expected_io_set_iov(expected_io, 0, iov[i].iov_base, iov[i].iov_len); 2286 ut_expected_io_set_iov(expected_io, 1, iov[i + 1].iov_base, iov[i + 1].iov_len); 2287 ut_expected_io_set_iov(expected_io, 2, iov[i + 2].iov_base, iov[i + 2].iov_len); 2288 ut_expected_io_set_iov(expected_io, 3, iov[i + 3].iov_base, iov[i + 3].iov_len); 2289 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2290 2291 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 2, 0, 2292 BDEV_IO_NUM_CHILD_IOV - 1, io_done, NULL); 2293 CU_ASSERT(rc == 0); 2294 CU_ASSERT(g_io_done == false); 2295 2296 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2297 stub_complete_io(1); 2298 CU_ASSERT(g_io_done == false); 2299 2300 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2301 stub_complete_io(1); 2302 CU_ASSERT(g_io_done == true); 2303 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2304 2305 /* A very complicated case. Each sg entry exceeds max_segment_size and 2306 * needs to be split. At the same time, child io must be a multiple of blocklen. 2307 * At the same time, child iovcnt exceeds parent iovcnt. 2308 */ 2309 bdev->max_segment_size = 512 + 128; 2310 bdev->max_num_segments = 3; 2311 g_io_done = false; 2312 2313 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV - 2; i++) { 2314 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2315 iov[i].iov_len = 512 + 256; 2316 } 2317 2318 for (i = BDEV_IO_NUM_CHILD_IOV - 2; i < BDEV_IO_NUM_CHILD_IOV + 2; i++) { 2319 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2320 iov[i].iov_len = 512 + 128; 2321 } 2322 2323 /* Child IOs use 9 entries per for() round and 3 * 9 = 27 child iov entries. 2324 * Consume 4 parent IO iov entries per for() round and 6 block size. 2325 * Generate 9 child IOs. 2326 */ 2327 for (i = 0; i < 3; i++) { 2328 uint32_t j = i * 4; 2329 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 6, 2, 3); 2330 ut_expected_io_set_iov(expected_io, 0, iov[j].iov_base, 640); 2331 ut_expected_io_set_iov(expected_io, 1, iov[j].iov_base + 640, 128); 2332 ut_expected_io_set_iov(expected_io, 2, iov[j + 1].iov_base, 256); 2333 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2334 2335 /* Child io must be a multiple of blocklen 2336 * iov[j + 2] must be split. If the third entry is also added, 2337 * the multiple of blocklen cannot be guaranteed. But it still 2338 * occupies one iov entry of the parent child iov. 2339 */ 2340 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 6 + 2, 2, 2); 2341 ut_expected_io_set_iov(expected_io, 0, iov[j + 1].iov_base + 256, 512); 2342 ut_expected_io_set_iov(expected_io, 1, iov[j + 2].iov_base, 512); 2343 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2344 2345 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, i * 6 + 4, 2, 3); 2346 ut_expected_io_set_iov(expected_io, 0, iov[j + 2].iov_base + 512, 256); 2347 ut_expected_io_set_iov(expected_io, 1, iov[j + 3].iov_base, 640); 2348 ut_expected_io_set_iov(expected_io, 2, iov[j + 3].iov_base + 640, 128); 2349 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2350 } 2351 2352 /* Child iov position at 27, the 10th child IO 2353 * iov entry index is 3 * 4 and offset is 3 * 6 2354 */ 2355 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 18, 2, 3); 2356 ut_expected_io_set_iov(expected_io, 0, iov[12].iov_base, 640); 2357 ut_expected_io_set_iov(expected_io, 1, iov[12].iov_base + 640, 128); 2358 ut_expected_io_set_iov(expected_io, 2, iov[13].iov_base, 256); 2359 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2360 2361 /* Child iov position at 30, the 11th child IO */ 2362 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 20, 2, 2); 2363 ut_expected_io_set_iov(expected_io, 0, iov[13].iov_base + 256, 512); 2364 ut_expected_io_set_iov(expected_io, 1, iov[14].iov_base, 512); 2365 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2366 2367 /* The 2nd split round and iovpos is 0, the 12th child IO */ 2368 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 22, 2, 3); 2369 ut_expected_io_set_iov(expected_io, 0, iov[14].iov_base + 512, 256); 2370 ut_expected_io_set_iov(expected_io, 1, iov[15].iov_base, 640); 2371 ut_expected_io_set_iov(expected_io, 2, iov[15].iov_base + 640, 128); 2372 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2373 2374 /* Consume 9 child IOs and 27 child iov entries. 2375 * Consume 4 parent IO iov entries per for() round and 6 block size. 2376 * Parent IO iov index start from 16 and block offset start from 24 2377 */ 2378 for (i = 0; i < 3; i++) { 2379 uint32_t j = i * 4 + 16; 2380 uint32_t offset = i * 6 + 24; 2381 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset, 2, 3); 2382 ut_expected_io_set_iov(expected_io, 0, iov[j].iov_base, 640); 2383 ut_expected_io_set_iov(expected_io, 1, iov[j].iov_base + 640, 128); 2384 ut_expected_io_set_iov(expected_io, 2, iov[j + 1].iov_base, 256); 2385 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2386 2387 /* Child io must be a multiple of blocklen 2388 * iov[j + 2] must be split. If the third entry is also added, 2389 * the multiple of blocklen cannot be guaranteed. But it still 2390 * occupies one iov entry of the parent child iov. 2391 */ 2392 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset + 2, 2, 2); 2393 ut_expected_io_set_iov(expected_io, 0, iov[j + 1].iov_base + 256, 512); 2394 ut_expected_io_set_iov(expected_io, 1, iov[j + 2].iov_base, 512); 2395 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2396 2397 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset + 4, 2, 3); 2398 ut_expected_io_set_iov(expected_io, 0, iov[j + 2].iov_base + 512, 256); 2399 ut_expected_io_set_iov(expected_io, 1, iov[j + 3].iov_base, 640); 2400 ut_expected_io_set_iov(expected_io, 2, iov[j + 3].iov_base + 640, 128); 2401 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2402 } 2403 2404 /* The 22th child IO, child iov position at 30 */ 2405 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 42, 1, 1); 2406 ut_expected_io_set_iov(expected_io, 0, iov[28].iov_base, 512); 2407 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2408 2409 /* The third round */ 2410 /* Here is the 23nd child IO and child iovpos is 0 */ 2411 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 43, 2, 3); 2412 ut_expected_io_set_iov(expected_io, 0, iov[28].iov_base + 512, 256); 2413 ut_expected_io_set_iov(expected_io, 1, iov[29].iov_base, 640); 2414 ut_expected_io_set_iov(expected_io, 2, iov[29].iov_base + 640, 128); 2415 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2416 2417 /* The 24th child IO */ 2418 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 45, 3, 3); 2419 ut_expected_io_set_iov(expected_io, 0, iov[30].iov_base, 640); 2420 ut_expected_io_set_iov(expected_io, 1, iov[31].iov_base, 640); 2421 ut_expected_io_set_iov(expected_io, 2, iov[32].iov_base, 256); 2422 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2423 2424 /* The 25th child IO */ 2425 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 48, 2, 2); 2426 ut_expected_io_set_iov(expected_io, 0, iov[32].iov_base + 256, 384); 2427 ut_expected_io_set_iov(expected_io, 1, iov[33].iov_base, 640); 2428 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2429 2430 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV + 2, 0, 2431 50, io_done, NULL); 2432 CU_ASSERT(rc == 0); 2433 CU_ASSERT(g_io_done == false); 2434 2435 /* Parent IO supports up to 32 child iovs, so it is calculated that 2436 * a maximum of 11 IOs can be split at a time, and the 2437 * splitting will continue after the first batch is over. 2438 */ 2439 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 11); 2440 stub_complete_io(11); 2441 CU_ASSERT(g_io_done == false); 2442 2443 /* The 2nd round */ 2444 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 11); 2445 stub_complete_io(11); 2446 CU_ASSERT(g_io_done == false); 2447 2448 /* The last round */ 2449 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 2450 stub_complete_io(3); 2451 CU_ASSERT(g_io_done == true); 2452 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2453 2454 /* Test an WRITE_ZEROES. This should also not be split. */ 2455 bdev->max_segment_size = 512; 2456 bdev->max_num_segments = 1; 2457 g_io_done = false; 2458 2459 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, 9, 36, 0); 2460 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2461 2462 rc = spdk_bdev_write_zeroes_blocks(desc, io_ch, 9, 36, io_done, NULL); 2463 CU_ASSERT(rc == 0); 2464 CU_ASSERT(g_io_done == false); 2465 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2466 stub_complete_io(1); 2467 CU_ASSERT(g_io_done == true); 2468 2469 /* Test an UNMAP. This should also not be split. */ 2470 g_io_done = false; 2471 2472 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, 15, 4, 0); 2473 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2474 2475 rc = spdk_bdev_unmap_blocks(desc, io_ch, 15, 4, io_done, NULL); 2476 CU_ASSERT(rc == 0); 2477 CU_ASSERT(g_io_done == false); 2478 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2479 stub_complete_io(1); 2480 CU_ASSERT(g_io_done == true); 2481 2482 /* Test a FLUSH. This should also not be split. */ 2483 g_io_done = false; 2484 2485 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_FLUSH, 15, 4, 0); 2486 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2487 2488 rc = spdk_bdev_flush_blocks(desc, io_ch, 15, 2, io_done, NULL); 2489 CU_ASSERT(rc == 0); 2490 CU_ASSERT(g_io_done == false); 2491 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2492 stub_complete_io(1); 2493 CU_ASSERT(g_io_done == true); 2494 2495 /* Test a COPY. This should also not be split. */ 2496 g_io_done = false; 2497 2498 expected_io = ut_alloc_expected_copy_io(SPDK_BDEV_IO_TYPE_COPY, 9, 45, 36); 2499 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2500 2501 rc = spdk_bdev_copy_blocks(desc, io_ch, 9, 45, 36, io_done, NULL); 2502 CU_ASSERT(rc == 0); 2503 CU_ASSERT(g_io_done == false); 2504 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2505 stub_complete_io(1); 2506 CU_ASSERT(g_io_done == true); 2507 2508 spdk_put_io_channel(io_ch); 2509 spdk_bdev_close(desc); 2510 free_bdev(bdev); 2511 spdk_bdev_finish(bdev_fini_cb, NULL); 2512 poll_threads(); 2513 } 2514 2515 static void 2516 bdev_io_mix_split_test(void) 2517 { 2518 struct spdk_bdev *bdev; 2519 struct spdk_bdev_desc *desc = NULL; 2520 struct spdk_io_channel *io_ch; 2521 struct spdk_bdev_opts bdev_opts = {}; 2522 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 2523 struct ut_expected_io *expected_io; 2524 uint64_t i; 2525 int rc; 2526 2527 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 2528 bdev_opts.bdev_io_pool_size = 512; 2529 bdev_opts.bdev_io_cache_size = 64; 2530 2531 rc = spdk_bdev_set_opts(&bdev_opts); 2532 CU_ASSERT(rc == 0); 2533 spdk_bdev_initialize(bdev_init_cb, NULL); 2534 2535 bdev = allocate_bdev("bdev0"); 2536 2537 rc = spdk_bdev_open_ext(bdev->name, true, bdev_ut_event_cb, NULL, &desc); 2538 CU_ASSERT(rc == 0); 2539 SPDK_CU_ASSERT_FATAL(desc != NULL); 2540 io_ch = spdk_bdev_get_io_channel(desc); 2541 CU_ASSERT(io_ch != NULL); 2542 2543 /* First case optimal_io_boundary == max_segment_size * max_num_segments */ 2544 bdev->split_on_optimal_io_boundary = true; 2545 bdev->optimal_io_boundary = 16; 2546 2547 bdev->max_segment_size = 512; 2548 bdev->max_num_segments = 16; 2549 g_io_done = false; 2550 2551 /* IO crossing the IO boundary requires split 2552 * Total 2 child IOs. 2553 */ 2554 2555 /* The 1st child IO split the segment_size to multiple segment entry */ 2556 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 2); 2557 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512); 2558 ut_expected_io_set_iov(expected_io, 1, (void *)(0xF000 + 512), 512); 2559 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2560 2561 /* The 2nd child IO split the segment_size to multiple segment entry */ 2562 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 2, 2); 2563 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 512); 2564 ut_expected_io_set_iov(expected_io, 1, (void *)(0xF000 + 3 * 512), 512); 2565 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2566 2567 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 4, io_done, NULL); 2568 CU_ASSERT(rc == 0); 2569 CU_ASSERT(g_io_done == false); 2570 2571 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2572 stub_complete_io(2); 2573 CU_ASSERT(g_io_done == true); 2574 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2575 2576 /* Second case optimal_io_boundary > max_segment_size * max_num_segments */ 2577 bdev->max_segment_size = 15 * 512; 2578 bdev->max_num_segments = 1; 2579 g_io_done = false; 2580 2581 /* IO crossing the IO boundary requires split. 2582 * The 1st child IO segment size exceeds the max_segment_size, 2583 * So 1st child IO will be splitted to multiple segment entry. 2584 * Then it split to 2 child IOs because of the max_num_segments. 2585 * Total 3 child IOs. 2586 */ 2587 2588 /* The first 2 IOs are in an IO boundary. 2589 * Because the optimal_io_boundary > max_segment_size * max_num_segments 2590 * So it split to the first 2 IOs. 2591 */ 2592 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 15, 1); 2593 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512 * 15); 2594 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2595 2596 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 15, 1, 1); 2597 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 512 * 15), 512); 2598 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2599 2600 /* The 3rd Child IO is because of the io boundary */ 2601 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 2, 1); 2602 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 512 * 16), 512 * 2); 2603 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2604 2605 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 0, 18, io_done, NULL); 2606 CU_ASSERT(rc == 0); 2607 CU_ASSERT(g_io_done == false); 2608 2609 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 2610 stub_complete_io(3); 2611 CU_ASSERT(g_io_done == true); 2612 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2613 2614 /* Third case optimal_io_boundary < max_segment_size * max_num_segments */ 2615 bdev->max_segment_size = 17 * 512; 2616 bdev->max_num_segments = 1; 2617 g_io_done = false; 2618 2619 /* IO crossing the IO boundary requires split. 2620 * Child IO does not split. 2621 * Total 2 child IOs. 2622 */ 2623 2624 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 0, 16, 1); 2625 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 512 * 16); 2626 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2627 2628 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 2, 1); 2629 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 512 * 16), 512 * 2); 2630 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2631 2632 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 0, 18, io_done, NULL); 2633 CU_ASSERT(rc == 0); 2634 CU_ASSERT(g_io_done == false); 2635 2636 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2637 stub_complete_io(2); 2638 CU_ASSERT(g_io_done == true); 2639 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2640 2641 /* Now set up a more complex, multi-vector command that needs to be split, 2642 * including splitting iovecs. 2643 * optimal_io_boundary < max_segment_size * max_num_segments 2644 */ 2645 bdev->max_segment_size = 3 * 512; 2646 bdev->max_num_segments = 6; 2647 g_io_done = false; 2648 2649 iov[0].iov_base = (void *)0x10000; 2650 iov[0].iov_len = 4 * 512; 2651 iov[1].iov_base = (void *)0x20000; 2652 iov[1].iov_len = 4 * 512; 2653 iov[2].iov_base = (void *)0x30000; 2654 iov[2].iov_len = 10 * 512; 2655 2656 /* IO crossing the IO boundary requires split. 2657 * The 1st child IO segment size exceeds the max_segment_size and after 2658 * splitting segment_size, the num_segments exceeds max_num_segments. 2659 * So 1st child IO will be splitted to 2 child IOs. 2660 * Total 3 child IOs. 2661 */ 2662 2663 /* The first 2 IOs are in an IO boundary. 2664 * After splitting segment size the segment num exceeds. 2665 * So it splits to 2 child IOs. 2666 */ 2667 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 0, 14, 6); 2668 ut_expected_io_set_iov(expected_io, 0, iov[0].iov_base, 512 * 3); 2669 ut_expected_io_set_iov(expected_io, 1, iov[0].iov_base + 512 * 3, 512); 2670 ut_expected_io_set_iov(expected_io, 2, iov[1].iov_base, 512 * 3); 2671 ut_expected_io_set_iov(expected_io, 3, iov[1].iov_base + 512 * 3, 512); 2672 ut_expected_io_set_iov(expected_io, 4, iov[2].iov_base, 512 * 3); 2673 ut_expected_io_set_iov(expected_io, 5, iov[2].iov_base + 512 * 3, 512 * 3); 2674 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2675 2676 /* The 2nd child IO has the left segment entry */ 2677 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 1); 2678 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 6, 512 * 2); 2679 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2680 2681 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 2, 1); 2682 ut_expected_io_set_iov(expected_io, 0, iov[2].iov_base + 512 * 8, 512 * 2); 2683 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2684 2685 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 3, 0, 18, io_done, NULL); 2686 CU_ASSERT(rc == 0); 2687 CU_ASSERT(g_io_done == false); 2688 2689 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 2690 stub_complete_io(3); 2691 CU_ASSERT(g_io_done == true); 2692 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2693 2694 /* A very complicated case. Each sg entry exceeds max_segment_size 2695 * and split on io boundary. 2696 * optimal_io_boundary < max_segment_size * max_num_segments 2697 */ 2698 bdev->max_segment_size = 3 * 512; 2699 bdev->max_num_segments = BDEV_IO_NUM_CHILD_IOV; 2700 g_io_done = false; 2701 2702 for (i = 0; i < 20; i++) { 2703 iov[i].iov_base = (void *)((i + 1) * 0x10000); 2704 iov[i].iov_len = 512 * 4; 2705 } 2706 2707 /* IO crossing the IO boundary requires split. 2708 * 80 block length can split 5 child IOs base on offset and IO boundary. 2709 * Each iov entry needs to be splitted to 2 entries because of max_segment_size 2710 * Total 5 child IOs. 2711 */ 2712 2713 /* 4 iov entries are in an IO boundary and each iov entry splits to 2. 2714 * So each child IO occupies 8 child iov entries. 2715 */ 2716 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 0, 16, 8); 2717 for (i = 0; i < 4; i++) { 2718 int iovcnt = i * 2; 2719 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2720 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2721 } 2722 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2723 2724 /* 2nd child IO and total 16 child iov entries of parent IO */ 2725 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 16, 8); 2726 for (i = 4; i < 8; i++) { 2727 int iovcnt = (i - 4) * 2; 2728 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2729 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2730 } 2731 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2732 2733 /* 3rd child IO and total 24 child iov entries of parent IO */ 2734 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 16, 8); 2735 for (i = 8; i < 12; i++) { 2736 int iovcnt = (i - 8) * 2; 2737 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2738 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2739 } 2740 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2741 2742 /* 4th child IO and total 32 child iov entries of parent IO */ 2743 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 48, 16, 8); 2744 for (i = 12; i < 16; i++) { 2745 int iovcnt = (i - 12) * 2; 2746 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2747 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2748 } 2749 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2750 2751 /* 5th child IO and because of the child iov entry it should be splitted 2752 * in next round. 2753 */ 2754 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 64, 16, 8); 2755 for (i = 16; i < 20; i++) { 2756 int iovcnt = (i - 16) * 2; 2757 ut_expected_io_set_iov(expected_io, iovcnt, iov[i].iov_base, 512 * 3); 2758 ut_expected_io_set_iov(expected_io, iovcnt + 1, iov[i].iov_base + 512 * 3, 512); 2759 } 2760 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2761 2762 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 20, 0, 80, io_done, NULL); 2763 CU_ASSERT(rc == 0); 2764 CU_ASSERT(g_io_done == false); 2765 2766 /* First split round */ 2767 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 2768 stub_complete_io(4); 2769 CU_ASSERT(g_io_done == false); 2770 2771 /* Second split round */ 2772 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2773 stub_complete_io(1); 2774 CU_ASSERT(g_io_done == true); 2775 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2776 2777 spdk_put_io_channel(io_ch); 2778 spdk_bdev_close(desc); 2779 free_bdev(bdev); 2780 spdk_bdev_finish(bdev_fini_cb, NULL); 2781 poll_threads(); 2782 } 2783 2784 static void 2785 bdev_io_split_with_io_wait(void) 2786 { 2787 struct spdk_bdev *bdev; 2788 struct spdk_bdev_desc *desc = NULL; 2789 struct spdk_io_channel *io_ch; 2790 struct spdk_bdev_channel *channel; 2791 struct spdk_bdev_mgmt_channel *mgmt_ch; 2792 struct spdk_bdev_opts bdev_opts = {}; 2793 struct iovec iov[3]; 2794 struct ut_expected_io *expected_io; 2795 int rc; 2796 2797 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 2798 bdev_opts.bdev_io_pool_size = 2; 2799 bdev_opts.bdev_io_cache_size = 1; 2800 2801 rc = spdk_bdev_set_opts(&bdev_opts); 2802 CU_ASSERT(rc == 0); 2803 spdk_bdev_initialize(bdev_init_cb, NULL); 2804 2805 bdev = allocate_bdev("bdev0"); 2806 2807 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 2808 CU_ASSERT(rc == 0); 2809 CU_ASSERT(desc != NULL); 2810 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 2811 io_ch = spdk_bdev_get_io_channel(desc); 2812 CU_ASSERT(io_ch != NULL); 2813 channel = spdk_io_channel_get_ctx(io_ch); 2814 mgmt_ch = channel->shared_resource->mgmt_ch; 2815 2816 bdev->optimal_io_boundary = 16; 2817 bdev->split_on_optimal_io_boundary = true; 2818 2819 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, NULL); 2820 CU_ASSERT(rc == 0); 2821 2822 /* Now test that a single-vector command is split correctly. 2823 * Offset 14, length 8, payload 0xF000 2824 * Child - Offset 14, length 2, payload 0xF000 2825 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 2826 * 2827 * Set up the expected values before calling spdk_bdev_read_blocks 2828 */ 2829 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 1); 2830 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 2 * 512); 2831 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2832 2833 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 6, 1); 2834 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 6 * 512); 2835 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2836 2837 /* The following children will be submitted sequentially due to the capacity of 2838 * spdk_bdev_io. 2839 */ 2840 2841 /* The first child I/O will be queued to wait until an spdk_bdev_io becomes available */ 2842 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL); 2843 CU_ASSERT(rc == 0); 2844 CU_ASSERT(!TAILQ_EMPTY(&mgmt_ch->io_wait_queue)); 2845 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2846 2847 /* Completing the first read I/O will submit the first child */ 2848 stub_complete_io(1); 2849 CU_ASSERT(TAILQ_EMPTY(&mgmt_ch->io_wait_queue)); 2850 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2851 2852 /* Completing the first child will submit the second child */ 2853 stub_complete_io(1); 2854 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2855 2856 /* Complete the second child I/O. This should result in our callback getting 2857 * invoked since the parent I/O is now complete. 2858 */ 2859 stub_complete_io(1); 2860 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2861 2862 /* Now set up a more complex, multi-vector command that needs to be split, 2863 * including splitting iovecs. 2864 */ 2865 iov[0].iov_base = (void *)0x10000; 2866 iov[0].iov_len = 512; 2867 iov[1].iov_base = (void *)0x20000; 2868 iov[1].iov_len = 20 * 512; 2869 iov[2].iov_base = (void *)0x30000; 2870 iov[2].iov_len = 11 * 512; 2871 2872 g_io_done = false; 2873 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 2); 2874 ut_expected_io_set_iov(expected_io, 0, (void *)0x10000, 512); 2875 ut_expected_io_set_iov(expected_io, 1, (void *)0x20000, 512); 2876 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2877 2878 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 16, 1); 2879 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 512), 16 * 512); 2880 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2881 2882 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 14, 2); 2883 ut_expected_io_set_iov(expected_io, 0, (void *)(0x20000 + 17 * 512), 3 * 512); 2884 ut_expected_io_set_iov(expected_io, 1, (void *)0x30000, 11 * 512); 2885 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2886 2887 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, 3, 14, 32, io_done, NULL); 2888 CU_ASSERT(rc == 0); 2889 CU_ASSERT(g_io_done == false); 2890 2891 /* The following children will be submitted sequentially due to the capacity of 2892 * spdk_bdev_io. 2893 */ 2894 2895 /* Completing the first child will submit the second child */ 2896 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2897 stub_complete_io(1); 2898 CU_ASSERT(g_io_done == false); 2899 2900 /* Completing the second child will submit the third child */ 2901 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2902 stub_complete_io(1); 2903 CU_ASSERT(g_io_done == false); 2904 2905 /* Completing the third child will result in our callback getting invoked 2906 * since the parent I/O is now complete. 2907 */ 2908 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 2909 stub_complete_io(1); 2910 CU_ASSERT(g_io_done == true); 2911 2912 CU_ASSERT(TAILQ_EMPTY(&g_bdev_ut_channel->expected_io)); 2913 2914 spdk_put_io_channel(io_ch); 2915 spdk_bdev_close(desc); 2916 free_bdev(bdev); 2917 spdk_bdev_finish(bdev_fini_cb, NULL); 2918 poll_threads(); 2919 } 2920 2921 static void 2922 bdev_io_write_unit_split_test(void) 2923 { 2924 struct spdk_bdev *bdev; 2925 struct spdk_bdev_desc *desc = NULL; 2926 struct spdk_io_channel *io_ch; 2927 struct spdk_bdev_opts bdev_opts = {}; 2928 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 4]; 2929 struct ut_expected_io *expected_io; 2930 uint64_t i; 2931 int rc; 2932 2933 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 2934 bdev_opts.bdev_io_pool_size = 512; 2935 bdev_opts.bdev_io_cache_size = 64; 2936 2937 rc = spdk_bdev_set_opts(&bdev_opts); 2938 CU_ASSERT(rc == 0); 2939 spdk_bdev_initialize(bdev_init_cb, NULL); 2940 2941 bdev = allocate_bdev("bdev0"); 2942 2943 rc = spdk_bdev_open_ext(bdev->name, true, bdev_ut_event_cb, NULL, &desc); 2944 CU_ASSERT(rc == 0); 2945 SPDK_CU_ASSERT_FATAL(desc != NULL); 2946 io_ch = spdk_bdev_get_io_channel(desc); 2947 CU_ASSERT(io_ch != NULL); 2948 2949 /* Write I/O 2x larger than write_unit_size should get split into 2 I/Os */ 2950 bdev->write_unit_size = 32; 2951 bdev->split_on_write_unit = true; 2952 g_io_done = false; 2953 2954 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 0, 32, 1); 2955 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 32 * 512); 2956 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2957 2958 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 32, 1); 2959 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 32 * 512), 32 * 512); 2960 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 2961 2962 rc = spdk_bdev_write_blocks(desc, io_ch, (void *)0xF000, 0, 64, io_done, NULL); 2963 CU_ASSERT(rc == 0); 2964 CU_ASSERT(g_io_done == false); 2965 2966 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2967 stub_complete_io(2); 2968 CU_ASSERT(g_io_done == true); 2969 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2970 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 2971 2972 /* Same as above but with optimal_io_boundary < write_unit_size - the I/O should be split 2973 * based on write_unit_size, not optimal_io_boundary */ 2974 bdev->split_on_optimal_io_boundary = true; 2975 bdev->optimal_io_boundary = 16; 2976 g_io_done = false; 2977 2978 rc = spdk_bdev_write_blocks(desc, io_ch, (void *)0xF000, 0, 64, io_done, NULL); 2979 CU_ASSERT(rc == 0); 2980 CU_ASSERT(g_io_done == false); 2981 2982 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 2983 stub_complete_io(2); 2984 CU_ASSERT(g_io_done == true); 2985 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2986 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 2987 2988 /* Write I/O should fail if it is smaller than write_unit_size */ 2989 g_io_done = false; 2990 2991 rc = spdk_bdev_write_blocks(desc, io_ch, (void *)0xF000, 0, 31, io_done, NULL); 2992 CU_ASSERT(rc == 0); 2993 CU_ASSERT(g_io_done == false); 2994 2995 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2996 poll_threads(); 2997 CU_ASSERT(g_io_done == true); 2998 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 2999 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 3000 3001 /* Same for I/O not aligned to write_unit_size */ 3002 g_io_done = false; 3003 3004 rc = spdk_bdev_write_blocks(desc, io_ch, (void *)0xF000, 1, 32, io_done, NULL); 3005 CU_ASSERT(rc == 0); 3006 CU_ASSERT(g_io_done == false); 3007 3008 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 3009 poll_threads(); 3010 CU_ASSERT(g_io_done == true); 3011 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 3012 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 3013 3014 /* Write should fail if it needs to be split but there are not enough iovs to submit 3015 * an entire write unit */ 3016 bdev->write_unit_size = SPDK_COUNTOF(iov) / 2; 3017 g_io_done = false; 3018 3019 for (i = 0; i < SPDK_COUNTOF(iov); i++) { 3020 iov[i].iov_base = (void *)(0x1000 + 512 * i); 3021 iov[i].iov_len = 512; 3022 } 3023 3024 rc = spdk_bdev_writev_blocks(desc, io_ch, iov, SPDK_COUNTOF(iov), 0, SPDK_COUNTOF(iov), 3025 io_done, NULL); 3026 CU_ASSERT(rc == 0); 3027 CU_ASSERT(g_io_done == false); 3028 3029 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 3030 poll_threads(); 3031 CU_ASSERT(g_io_done == true); 3032 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 3033 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 3034 3035 spdk_put_io_channel(io_ch); 3036 spdk_bdev_close(desc); 3037 free_bdev(bdev); 3038 spdk_bdev_finish(bdev_fini_cb, NULL); 3039 poll_threads(); 3040 } 3041 3042 static void 3043 bdev_io_alignment(void) 3044 { 3045 struct spdk_bdev *bdev; 3046 struct spdk_bdev_desc *desc = NULL; 3047 struct spdk_io_channel *io_ch; 3048 struct spdk_bdev_opts bdev_opts = {}; 3049 int rc; 3050 void *buf = NULL; 3051 struct iovec iovs[2]; 3052 int iovcnt; 3053 uint64_t alignment; 3054 3055 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 3056 bdev_opts.bdev_io_pool_size = 20; 3057 bdev_opts.bdev_io_cache_size = 2; 3058 3059 rc = spdk_bdev_set_opts(&bdev_opts); 3060 CU_ASSERT(rc == 0); 3061 spdk_bdev_initialize(bdev_init_cb, NULL); 3062 3063 fn_table.submit_request = stub_submit_request_get_buf; 3064 bdev = allocate_bdev("bdev0"); 3065 3066 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 3067 CU_ASSERT(rc == 0); 3068 CU_ASSERT(desc != NULL); 3069 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3070 io_ch = spdk_bdev_get_io_channel(desc); 3071 CU_ASSERT(io_ch != NULL); 3072 3073 /* Create aligned buffer */ 3074 rc = posix_memalign(&buf, 4096, 8192); 3075 SPDK_CU_ASSERT_FATAL(rc == 0); 3076 3077 /* Pass aligned single buffer with no alignment required */ 3078 alignment = 1; 3079 bdev->required_alignment = spdk_u32log2(alignment); 3080 3081 rc = spdk_bdev_write_blocks(desc, io_ch, buf, 0, 1, io_done, NULL); 3082 CU_ASSERT(rc == 0); 3083 stub_complete_io(1); 3084 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 3085 alignment)); 3086 3087 rc = spdk_bdev_read_blocks(desc, io_ch, buf, 0, 1, io_done, NULL); 3088 CU_ASSERT(rc == 0); 3089 stub_complete_io(1); 3090 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 3091 alignment)); 3092 3093 /* Pass unaligned single buffer with no alignment required */ 3094 alignment = 1; 3095 bdev->required_alignment = spdk_u32log2(alignment); 3096 3097 rc = spdk_bdev_write_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 3098 CU_ASSERT(rc == 0); 3099 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3100 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == buf + 4); 3101 stub_complete_io(1); 3102 3103 rc = spdk_bdev_read_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 3104 CU_ASSERT(rc == 0); 3105 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3106 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == buf + 4); 3107 stub_complete_io(1); 3108 3109 /* Pass unaligned single buffer with 512 alignment required */ 3110 alignment = 512; 3111 bdev->required_alignment = spdk_u32log2(alignment); 3112 3113 rc = spdk_bdev_write_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 3114 CU_ASSERT(rc == 0); 3115 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 3116 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 3117 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 3118 alignment)); 3119 stub_complete_io(1); 3120 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3121 3122 rc = spdk_bdev_read_blocks(desc, io_ch, buf + 4, 0, 1, io_done, NULL); 3123 CU_ASSERT(rc == 0); 3124 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 3125 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 3126 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 3127 alignment)); 3128 stub_complete_io(1); 3129 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3130 3131 /* Pass unaligned single buffer with 4096 alignment required */ 3132 alignment = 4096; 3133 bdev->required_alignment = spdk_u32log2(alignment); 3134 3135 rc = spdk_bdev_write_blocks(desc, io_ch, buf + 8, 0, 1, io_done, NULL); 3136 CU_ASSERT(rc == 0); 3137 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 3138 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 3139 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 3140 alignment)); 3141 stub_complete_io(1); 3142 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3143 3144 rc = spdk_bdev_read_blocks(desc, io_ch, buf + 8, 0, 1, io_done, NULL); 3145 CU_ASSERT(rc == 0); 3146 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 1); 3147 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 3148 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 3149 alignment)); 3150 stub_complete_io(1); 3151 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3152 3153 /* Pass aligned iovs with no alignment required */ 3154 alignment = 1; 3155 bdev->required_alignment = spdk_u32log2(alignment); 3156 3157 iovcnt = 1; 3158 iovs[0].iov_base = buf; 3159 iovs[0].iov_len = 512; 3160 3161 rc = spdk_bdev_writev(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 3162 CU_ASSERT(rc == 0); 3163 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3164 stub_complete_io(1); 3165 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 3166 3167 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 3168 CU_ASSERT(rc == 0); 3169 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3170 stub_complete_io(1); 3171 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 3172 3173 /* Pass unaligned iovs with no alignment required */ 3174 alignment = 1; 3175 bdev->required_alignment = spdk_u32log2(alignment); 3176 3177 iovcnt = 2; 3178 iovs[0].iov_base = buf + 16; 3179 iovs[0].iov_len = 256; 3180 iovs[1].iov_base = buf + 16 + 256 + 32; 3181 iovs[1].iov_len = 256; 3182 3183 rc = spdk_bdev_writev(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 3184 CU_ASSERT(rc == 0); 3185 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3186 stub_complete_io(1); 3187 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 3188 3189 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 3190 CU_ASSERT(rc == 0); 3191 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3192 stub_complete_io(1); 3193 CU_ASSERT(g_bdev_io->u.bdev.iovs[0].iov_base == iovs[0].iov_base); 3194 3195 /* Pass unaligned iov with 2048 alignment required */ 3196 alignment = 2048; 3197 bdev->required_alignment = spdk_u32log2(alignment); 3198 3199 iovcnt = 2; 3200 iovs[0].iov_base = buf + 16; 3201 iovs[0].iov_len = 256; 3202 iovs[1].iov_base = buf + 16 + 256 + 32; 3203 iovs[1].iov_len = 256; 3204 3205 rc = spdk_bdev_writev(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 3206 CU_ASSERT(rc == 0); 3207 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == iovcnt); 3208 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 3209 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 3210 alignment)); 3211 stub_complete_io(1); 3212 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3213 3214 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 3215 CU_ASSERT(rc == 0); 3216 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == iovcnt); 3217 CU_ASSERT(g_bdev_io->u.bdev.iovs == &g_bdev_io->internal.bounce_iov); 3218 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 3219 alignment)); 3220 stub_complete_io(1); 3221 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3222 3223 /* Pass iov without allocated buffer without alignment required */ 3224 alignment = 1; 3225 bdev->required_alignment = spdk_u32log2(alignment); 3226 3227 iovcnt = 1; 3228 iovs[0].iov_base = NULL; 3229 iovs[0].iov_len = 0; 3230 3231 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 3232 CU_ASSERT(rc == 0); 3233 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3234 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 3235 alignment)); 3236 stub_complete_io(1); 3237 3238 /* Pass iov without allocated buffer with 1024 alignment required */ 3239 alignment = 1024; 3240 bdev->required_alignment = spdk_u32log2(alignment); 3241 3242 iovcnt = 1; 3243 iovs[0].iov_base = NULL; 3244 iovs[0].iov_len = 0; 3245 3246 rc = spdk_bdev_readv(desc, io_ch, iovs, iovcnt, 0, 512, io_done, NULL); 3247 CU_ASSERT(rc == 0); 3248 CU_ASSERT(g_bdev_io->internal.orig_iovcnt == 0); 3249 CU_ASSERT(_are_iovs_aligned(g_bdev_io->u.bdev.iovs, g_bdev_io->u.bdev.iovcnt, 3250 alignment)); 3251 stub_complete_io(1); 3252 3253 spdk_put_io_channel(io_ch); 3254 spdk_bdev_close(desc); 3255 free_bdev(bdev); 3256 fn_table.submit_request = stub_submit_request; 3257 spdk_bdev_finish(bdev_fini_cb, NULL); 3258 poll_threads(); 3259 3260 free(buf); 3261 } 3262 3263 static void 3264 bdev_io_alignment_with_boundary(void) 3265 { 3266 struct spdk_bdev *bdev; 3267 struct spdk_bdev_desc *desc = NULL; 3268 struct spdk_io_channel *io_ch; 3269 struct spdk_bdev_opts bdev_opts = {}; 3270 int rc; 3271 void *buf = NULL; 3272 struct iovec iovs[2]; 3273 int iovcnt; 3274 uint64_t alignment; 3275 3276 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 3277 bdev_opts.bdev_io_pool_size = 20; 3278 bdev_opts.bdev_io_cache_size = 2; 3279 3280 bdev_opts.opts_size = sizeof(bdev_opts); 3281 rc = spdk_bdev_set_opts(&bdev_opts); 3282 CU_ASSERT(rc == 0); 3283 spdk_bdev_initialize(bdev_init_cb, NULL); 3284 3285 fn_table.submit_request = stub_submit_request_get_buf; 3286 bdev = allocate_bdev("bdev0"); 3287 3288 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 3289 CU_ASSERT(rc == 0); 3290 CU_ASSERT(desc != NULL); 3291 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3292 io_ch = spdk_bdev_get_io_channel(desc); 3293 CU_ASSERT(io_ch != NULL); 3294 3295 /* Create aligned buffer */ 3296 rc = posix_memalign(&buf, 4096, 131072); 3297 SPDK_CU_ASSERT_FATAL(rc == 0); 3298 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3299 3300 /* 512 * 3 with 2 IO boundary, allocate small data buffer from bdev layer */ 3301 alignment = 512; 3302 bdev->required_alignment = spdk_u32log2(alignment); 3303 bdev->optimal_io_boundary = 2; 3304 bdev->split_on_optimal_io_boundary = true; 3305 3306 iovcnt = 1; 3307 iovs[0].iov_base = NULL; 3308 iovs[0].iov_len = 512 * 3; 3309 3310 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 3, io_done, NULL); 3311 CU_ASSERT(rc == 0); 3312 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 3313 stub_complete_io(2); 3314 3315 /* 8KiB with 16 IO boundary, allocate large data buffer from bdev layer */ 3316 alignment = 512; 3317 bdev->required_alignment = spdk_u32log2(alignment); 3318 bdev->optimal_io_boundary = 16; 3319 bdev->split_on_optimal_io_boundary = true; 3320 3321 iovcnt = 1; 3322 iovs[0].iov_base = NULL; 3323 iovs[0].iov_len = 512 * 16; 3324 3325 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 16, io_done, NULL); 3326 CU_ASSERT(rc == 0); 3327 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 3328 stub_complete_io(2); 3329 3330 /* 512 * 160 with 128 IO boundary, 63.5KiB + 16.5KiB for the two children requests */ 3331 alignment = 512; 3332 bdev->required_alignment = spdk_u32log2(alignment); 3333 bdev->optimal_io_boundary = 128; 3334 bdev->split_on_optimal_io_boundary = true; 3335 3336 iovcnt = 1; 3337 iovs[0].iov_base = buf + 16; 3338 iovs[0].iov_len = 512 * 160; 3339 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 160, io_done, NULL); 3340 CU_ASSERT(rc == 0); 3341 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 3342 stub_complete_io(2); 3343 3344 /* 512 * 3 with 2 IO boundary */ 3345 alignment = 512; 3346 bdev->required_alignment = spdk_u32log2(alignment); 3347 bdev->optimal_io_boundary = 2; 3348 bdev->split_on_optimal_io_boundary = true; 3349 3350 iovcnt = 2; 3351 iovs[0].iov_base = buf + 16; 3352 iovs[0].iov_len = 512; 3353 iovs[1].iov_base = buf + 16 + 512 + 32; 3354 iovs[1].iov_len = 1024; 3355 3356 rc = spdk_bdev_writev_blocks(desc, io_ch, iovs, iovcnt, 1, 3, io_done, NULL); 3357 CU_ASSERT(rc == 0); 3358 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 3359 stub_complete_io(2); 3360 3361 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 3, io_done, NULL); 3362 CU_ASSERT(rc == 0); 3363 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 3364 stub_complete_io(2); 3365 3366 /* 512 * 64 with 32 IO boundary */ 3367 bdev->optimal_io_boundary = 32; 3368 iovcnt = 2; 3369 iovs[0].iov_base = buf + 16; 3370 iovs[0].iov_len = 16384; 3371 iovs[1].iov_base = buf + 16 + 16384 + 32; 3372 iovs[1].iov_len = 16384; 3373 3374 rc = spdk_bdev_writev_blocks(desc, io_ch, iovs, iovcnt, 1, 64, io_done, NULL); 3375 CU_ASSERT(rc == 0); 3376 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 3377 stub_complete_io(3); 3378 3379 rc = spdk_bdev_readv_blocks(desc, io_ch, iovs, iovcnt, 1, 64, io_done, NULL); 3380 CU_ASSERT(rc == 0); 3381 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 3); 3382 stub_complete_io(3); 3383 3384 /* 512 * 160 with 32 IO boundary */ 3385 iovcnt = 1; 3386 iovs[0].iov_base = buf + 16; 3387 iovs[0].iov_len = 16384 + 65536; 3388 3389 rc = spdk_bdev_writev_blocks(desc, io_ch, iovs, iovcnt, 1, 160, io_done, NULL); 3390 CU_ASSERT(rc == 0); 3391 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 6); 3392 stub_complete_io(6); 3393 3394 spdk_put_io_channel(io_ch); 3395 spdk_bdev_close(desc); 3396 free_bdev(bdev); 3397 fn_table.submit_request = stub_submit_request; 3398 spdk_bdev_finish(bdev_fini_cb, NULL); 3399 poll_threads(); 3400 3401 free(buf); 3402 } 3403 3404 static void 3405 histogram_status_cb(void *cb_arg, int status) 3406 { 3407 g_status = status; 3408 } 3409 3410 static void 3411 histogram_data_cb(void *cb_arg, int status, struct spdk_histogram_data *histogram) 3412 { 3413 g_status = status; 3414 g_histogram = histogram; 3415 } 3416 3417 static void 3418 histogram_io_count(void *ctx, uint64_t start, uint64_t end, uint64_t count, 3419 uint64_t total, uint64_t so_far) 3420 { 3421 g_count += count; 3422 } 3423 3424 static void 3425 bdev_histograms(void) 3426 { 3427 struct spdk_bdev *bdev; 3428 struct spdk_bdev_desc *desc = NULL; 3429 struct spdk_io_channel *ch; 3430 struct spdk_histogram_data *histogram; 3431 uint8_t buf[4096]; 3432 int rc; 3433 3434 spdk_bdev_initialize(bdev_init_cb, NULL); 3435 3436 bdev = allocate_bdev("bdev"); 3437 3438 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3439 CU_ASSERT(rc == 0); 3440 CU_ASSERT(desc != NULL); 3441 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3442 3443 ch = spdk_bdev_get_io_channel(desc); 3444 CU_ASSERT(ch != NULL); 3445 3446 /* Enable histogram */ 3447 g_status = -1; 3448 spdk_bdev_histogram_enable(bdev, histogram_status_cb, NULL, true); 3449 poll_threads(); 3450 CU_ASSERT(g_status == 0); 3451 CU_ASSERT(bdev->internal.histogram_enabled == true); 3452 3453 /* Allocate histogram */ 3454 histogram = spdk_histogram_data_alloc(); 3455 SPDK_CU_ASSERT_FATAL(histogram != NULL); 3456 3457 /* Check if histogram is zeroed */ 3458 spdk_bdev_histogram_get(bdev, histogram, histogram_data_cb, NULL); 3459 poll_threads(); 3460 CU_ASSERT(g_status == 0); 3461 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 3462 3463 g_count = 0; 3464 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 3465 3466 CU_ASSERT(g_count == 0); 3467 3468 rc = spdk_bdev_write_blocks(desc, ch, buf, 0, 1, io_done, NULL); 3469 CU_ASSERT(rc == 0); 3470 3471 spdk_delay_us(10); 3472 stub_complete_io(1); 3473 poll_threads(); 3474 3475 rc = spdk_bdev_read_blocks(desc, ch, buf, 0, 1, io_done, NULL); 3476 CU_ASSERT(rc == 0); 3477 3478 spdk_delay_us(10); 3479 stub_complete_io(1); 3480 poll_threads(); 3481 3482 /* Check if histogram gathered data from all I/O channels */ 3483 g_histogram = NULL; 3484 spdk_bdev_histogram_get(bdev, histogram, histogram_data_cb, NULL); 3485 poll_threads(); 3486 CU_ASSERT(g_status == 0); 3487 CU_ASSERT(bdev->internal.histogram_enabled == true); 3488 SPDK_CU_ASSERT_FATAL(g_histogram != NULL); 3489 3490 g_count = 0; 3491 spdk_histogram_data_iterate(g_histogram, histogram_io_count, NULL); 3492 CU_ASSERT(g_count == 2); 3493 3494 /* Disable histogram */ 3495 spdk_bdev_histogram_enable(bdev, histogram_status_cb, NULL, false); 3496 poll_threads(); 3497 CU_ASSERT(g_status == 0); 3498 CU_ASSERT(bdev->internal.histogram_enabled == false); 3499 3500 /* Try to run histogram commands on disabled bdev */ 3501 spdk_bdev_histogram_get(bdev, histogram, histogram_data_cb, NULL); 3502 poll_threads(); 3503 CU_ASSERT(g_status == -EFAULT); 3504 3505 spdk_histogram_data_free(histogram); 3506 spdk_put_io_channel(ch); 3507 spdk_bdev_close(desc); 3508 free_bdev(bdev); 3509 spdk_bdev_finish(bdev_fini_cb, NULL); 3510 poll_threads(); 3511 } 3512 3513 static void 3514 _bdev_compare(bool emulated) 3515 { 3516 struct spdk_bdev *bdev; 3517 struct spdk_bdev_desc *desc = NULL; 3518 struct spdk_io_channel *ioch; 3519 struct ut_expected_io *expected_io; 3520 uint64_t offset, num_blocks; 3521 uint32_t num_completed; 3522 char aa_buf[512]; 3523 char bb_buf[512]; 3524 struct iovec compare_iov; 3525 uint8_t expected_io_type; 3526 int rc; 3527 3528 if (emulated) { 3529 expected_io_type = SPDK_BDEV_IO_TYPE_READ; 3530 } else { 3531 expected_io_type = SPDK_BDEV_IO_TYPE_COMPARE; 3532 } 3533 3534 memset(aa_buf, 0xaa, sizeof(aa_buf)); 3535 memset(bb_buf, 0xbb, sizeof(bb_buf)); 3536 3537 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = !emulated; 3538 3539 spdk_bdev_initialize(bdev_init_cb, NULL); 3540 fn_table.submit_request = stub_submit_request_get_buf; 3541 bdev = allocate_bdev("bdev"); 3542 3543 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3544 CU_ASSERT_EQUAL(rc, 0); 3545 SPDK_CU_ASSERT_FATAL(desc != NULL); 3546 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3547 ioch = spdk_bdev_get_io_channel(desc); 3548 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3549 3550 fn_table.submit_request = stub_submit_request_get_buf; 3551 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3552 3553 offset = 50; 3554 num_blocks = 1; 3555 compare_iov.iov_base = aa_buf; 3556 compare_iov.iov_len = sizeof(aa_buf); 3557 3558 /* 1. successful compare */ 3559 expected_io = ut_alloc_expected_io(expected_io_type, offset, num_blocks, 0); 3560 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3561 3562 g_io_done = false; 3563 g_compare_read_buf = aa_buf; 3564 g_compare_read_buf_len = sizeof(aa_buf); 3565 rc = spdk_bdev_comparev_blocks(desc, ioch, &compare_iov, 1, offset, num_blocks, io_done, NULL); 3566 CU_ASSERT_EQUAL(rc, 0); 3567 num_completed = stub_complete_io(1); 3568 CU_ASSERT_EQUAL(num_completed, 1); 3569 CU_ASSERT(g_io_done == true); 3570 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3571 3572 /* 2. miscompare */ 3573 expected_io = ut_alloc_expected_io(expected_io_type, offset, num_blocks, 0); 3574 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3575 3576 g_io_done = false; 3577 g_compare_read_buf = bb_buf; 3578 g_compare_read_buf_len = sizeof(bb_buf); 3579 rc = spdk_bdev_comparev_blocks(desc, ioch, &compare_iov, 1, offset, num_blocks, io_done, NULL); 3580 CU_ASSERT_EQUAL(rc, 0); 3581 num_completed = stub_complete_io(1); 3582 CU_ASSERT_EQUAL(num_completed, 1); 3583 CU_ASSERT(g_io_done == true); 3584 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_MISCOMPARE); 3585 3586 spdk_put_io_channel(ioch); 3587 spdk_bdev_close(desc); 3588 free_bdev(bdev); 3589 fn_table.submit_request = stub_submit_request; 3590 spdk_bdev_finish(bdev_fini_cb, NULL); 3591 poll_threads(); 3592 3593 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = true; 3594 3595 g_compare_read_buf = NULL; 3596 } 3597 3598 static void 3599 _bdev_compare_with_md(bool emulated) 3600 { 3601 struct spdk_bdev *bdev; 3602 struct spdk_bdev_desc *desc = NULL; 3603 struct spdk_io_channel *ioch; 3604 struct ut_expected_io *expected_io; 3605 uint64_t offset, num_blocks; 3606 uint32_t num_completed; 3607 char buf[1024 + 16 /* 2 * blocklen + 2 * mdlen */]; 3608 char buf_interleaved_miscompare[1024 + 16 /* 2 * blocklen + 2 * mdlen */]; 3609 char buf_miscompare[1024 /* 2 * blocklen */]; 3610 char md_buf[16]; 3611 char md_buf_miscompare[16]; 3612 struct iovec compare_iov; 3613 uint8_t expected_io_type; 3614 int rc; 3615 3616 if (emulated) { 3617 expected_io_type = SPDK_BDEV_IO_TYPE_READ; 3618 } else { 3619 expected_io_type = SPDK_BDEV_IO_TYPE_COMPARE; 3620 } 3621 3622 memset(buf, 0xaa, sizeof(buf)); 3623 memset(buf_interleaved_miscompare, 0xaa, sizeof(buf_interleaved_miscompare)); 3624 /* make last md different */ 3625 memset(buf_interleaved_miscompare + 1024 + 8, 0xbb, 8); 3626 memset(buf_miscompare, 0xbb, sizeof(buf_miscompare)); 3627 memset(md_buf, 0xaa, 16); 3628 memset(md_buf_miscompare, 0xbb, 16); 3629 3630 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = !emulated; 3631 3632 spdk_bdev_initialize(bdev_init_cb, NULL); 3633 fn_table.submit_request = stub_submit_request_get_buf; 3634 bdev = allocate_bdev("bdev"); 3635 3636 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3637 CU_ASSERT_EQUAL(rc, 0); 3638 SPDK_CU_ASSERT_FATAL(desc != NULL); 3639 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3640 ioch = spdk_bdev_get_io_channel(desc); 3641 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3642 3643 fn_table.submit_request = stub_submit_request_get_buf; 3644 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3645 3646 offset = 50; 3647 num_blocks = 2; 3648 3649 /* interleaved md & data */ 3650 bdev->md_interleave = true; 3651 bdev->md_len = 8; 3652 bdev->blocklen = 512 + 8; 3653 compare_iov.iov_base = buf; 3654 compare_iov.iov_len = sizeof(buf); 3655 3656 /* 1. successful compare with md interleaved */ 3657 expected_io = ut_alloc_expected_io(expected_io_type, offset, num_blocks, 0); 3658 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3659 3660 g_io_done = false; 3661 g_compare_read_buf = buf; 3662 g_compare_read_buf_len = sizeof(buf); 3663 rc = spdk_bdev_comparev_blocks(desc, ioch, &compare_iov, 1, offset, num_blocks, io_done, NULL); 3664 CU_ASSERT_EQUAL(rc, 0); 3665 num_completed = stub_complete_io(1); 3666 CU_ASSERT_EQUAL(num_completed, 1); 3667 CU_ASSERT(g_io_done == true); 3668 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3669 3670 /* 2. miscompare with md interleaved */ 3671 expected_io = ut_alloc_expected_io(expected_io_type, offset, num_blocks, 0); 3672 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3673 3674 g_io_done = false; 3675 g_compare_read_buf = buf_interleaved_miscompare; 3676 g_compare_read_buf_len = sizeof(buf_interleaved_miscompare); 3677 rc = spdk_bdev_comparev_blocks(desc, ioch, &compare_iov, 1, offset, num_blocks, io_done, NULL); 3678 CU_ASSERT_EQUAL(rc, 0); 3679 num_completed = stub_complete_io(1); 3680 CU_ASSERT_EQUAL(num_completed, 1); 3681 CU_ASSERT(g_io_done == true); 3682 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_MISCOMPARE); 3683 3684 /* Separate data & md buffers */ 3685 bdev->md_interleave = false; 3686 bdev->blocklen = 512; 3687 compare_iov.iov_base = buf; 3688 compare_iov.iov_len = 1024; 3689 3690 /* 3. successful compare with md separated */ 3691 expected_io = ut_alloc_expected_io(expected_io_type, offset, num_blocks, 0); 3692 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3693 3694 g_io_done = false; 3695 g_compare_read_buf = buf; 3696 g_compare_read_buf_len = 1024; 3697 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3698 g_compare_md_buf = md_buf; 3699 rc = spdk_bdev_comparev_blocks_with_md(desc, ioch, &compare_iov, 1, md_buf, 3700 offset, num_blocks, io_done, NULL); 3701 CU_ASSERT_EQUAL(rc, 0); 3702 num_completed = stub_complete_io(1); 3703 CU_ASSERT_EQUAL(num_completed, 1); 3704 CU_ASSERT(g_io_done == true); 3705 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3706 3707 /* 4. miscompare with md separated where md buf is different */ 3708 expected_io = ut_alloc_expected_io(expected_io_type, offset, num_blocks, 0); 3709 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3710 3711 g_io_done = false; 3712 g_compare_read_buf = buf; 3713 g_compare_read_buf_len = 1024; 3714 g_compare_md_buf = md_buf_miscompare; 3715 rc = spdk_bdev_comparev_blocks_with_md(desc, ioch, &compare_iov, 1, md_buf, 3716 offset, num_blocks, io_done, NULL); 3717 CU_ASSERT_EQUAL(rc, 0); 3718 num_completed = stub_complete_io(1); 3719 CU_ASSERT_EQUAL(num_completed, 1); 3720 CU_ASSERT(g_io_done == true); 3721 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_MISCOMPARE); 3722 3723 /* 5. miscompare with md separated where buf is different */ 3724 expected_io = ut_alloc_expected_io(expected_io_type, offset, num_blocks, 0); 3725 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3726 3727 g_io_done = false; 3728 g_compare_read_buf = buf_miscompare; 3729 g_compare_read_buf_len = sizeof(buf_miscompare); 3730 g_compare_md_buf = md_buf; 3731 rc = spdk_bdev_comparev_blocks_with_md(desc, ioch, &compare_iov, 1, md_buf, 3732 offset, num_blocks, io_done, NULL); 3733 CU_ASSERT_EQUAL(rc, 0); 3734 num_completed = stub_complete_io(1); 3735 CU_ASSERT_EQUAL(num_completed, 1); 3736 CU_ASSERT(g_io_done == true); 3737 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_MISCOMPARE); 3738 3739 bdev->md_len = 0; 3740 g_compare_md_buf = NULL; 3741 3742 spdk_put_io_channel(ioch); 3743 spdk_bdev_close(desc); 3744 free_bdev(bdev); 3745 fn_table.submit_request = stub_submit_request; 3746 spdk_bdev_finish(bdev_fini_cb, NULL); 3747 poll_threads(); 3748 3749 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = true; 3750 3751 g_compare_read_buf = NULL; 3752 } 3753 3754 static void 3755 bdev_compare(void) 3756 { 3757 _bdev_compare(false); 3758 _bdev_compare_with_md(false); 3759 } 3760 3761 static void 3762 bdev_compare_emulated(void) 3763 { 3764 _bdev_compare(true); 3765 _bdev_compare_with_md(true); 3766 } 3767 3768 static void 3769 bdev_compare_and_write(void) 3770 { 3771 struct spdk_bdev *bdev; 3772 struct spdk_bdev_desc *desc = NULL; 3773 struct spdk_io_channel *ioch; 3774 struct ut_expected_io *expected_io; 3775 uint64_t offset, num_blocks; 3776 uint32_t num_completed; 3777 char aa_buf[512]; 3778 char bb_buf[512]; 3779 char cc_buf[512]; 3780 char write_buf[512]; 3781 struct iovec compare_iov; 3782 struct iovec write_iov; 3783 int rc; 3784 3785 memset(aa_buf, 0xaa, sizeof(aa_buf)); 3786 memset(bb_buf, 0xbb, sizeof(bb_buf)); 3787 memset(cc_buf, 0xcc, sizeof(cc_buf)); 3788 3789 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = false; 3790 3791 spdk_bdev_initialize(bdev_init_cb, NULL); 3792 fn_table.submit_request = stub_submit_request_get_buf; 3793 bdev = allocate_bdev("bdev"); 3794 3795 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3796 CU_ASSERT_EQUAL(rc, 0); 3797 SPDK_CU_ASSERT_FATAL(desc != NULL); 3798 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3799 ioch = spdk_bdev_get_io_channel(desc); 3800 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3801 3802 fn_table.submit_request = stub_submit_request_get_buf; 3803 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3804 3805 offset = 50; 3806 num_blocks = 1; 3807 compare_iov.iov_base = aa_buf; 3808 compare_iov.iov_len = sizeof(aa_buf); 3809 write_iov.iov_base = bb_buf; 3810 write_iov.iov_len = sizeof(bb_buf); 3811 3812 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset, num_blocks, 0); 3813 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3814 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, offset, num_blocks, 0); 3815 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3816 3817 g_io_done = false; 3818 g_compare_read_buf = aa_buf; 3819 g_compare_read_buf_len = sizeof(aa_buf); 3820 memset(write_buf, 0, sizeof(write_buf)); 3821 g_compare_write_buf = write_buf; 3822 g_compare_write_buf_len = sizeof(write_buf); 3823 rc = spdk_bdev_comparev_and_writev_blocks(desc, ioch, &compare_iov, 1, &write_iov, 1, 3824 offset, num_blocks, io_done, NULL); 3825 /* Trigger range locking */ 3826 poll_threads(); 3827 CU_ASSERT_EQUAL(rc, 0); 3828 num_completed = stub_complete_io(1); 3829 CU_ASSERT_EQUAL(num_completed, 1); 3830 CU_ASSERT(g_io_done == false); 3831 num_completed = stub_complete_io(1); 3832 /* Trigger range unlocking */ 3833 poll_threads(); 3834 CU_ASSERT_EQUAL(num_completed, 1); 3835 CU_ASSERT(g_io_done == true); 3836 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 3837 CU_ASSERT(memcmp(write_buf, bb_buf, sizeof(write_buf)) == 0); 3838 3839 /* Test miscompare */ 3840 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, offset, num_blocks, 0); 3841 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3842 3843 g_io_done = false; 3844 g_compare_read_buf = cc_buf; 3845 g_compare_read_buf_len = sizeof(cc_buf); 3846 memset(write_buf, 0, sizeof(write_buf)); 3847 g_compare_write_buf = write_buf; 3848 g_compare_write_buf_len = sizeof(write_buf); 3849 rc = spdk_bdev_comparev_and_writev_blocks(desc, ioch, &compare_iov, 1, &write_iov, 1, 3850 offset, num_blocks, io_done, NULL); 3851 /* Trigger range locking */ 3852 poll_threads(); 3853 CU_ASSERT_EQUAL(rc, 0); 3854 num_completed = stub_complete_io(1); 3855 /* Trigger range unlocking earlier because we expect error here */ 3856 poll_threads(); 3857 CU_ASSERT_EQUAL(num_completed, 1); 3858 CU_ASSERT(g_io_done == true); 3859 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_MISCOMPARE); 3860 num_completed = stub_complete_io(1); 3861 CU_ASSERT_EQUAL(num_completed, 0); 3862 3863 spdk_put_io_channel(ioch); 3864 spdk_bdev_close(desc); 3865 free_bdev(bdev); 3866 fn_table.submit_request = stub_submit_request; 3867 spdk_bdev_finish(bdev_fini_cb, NULL); 3868 poll_threads(); 3869 3870 g_io_types_supported[SPDK_BDEV_IO_TYPE_COMPARE] = true; 3871 3872 g_compare_read_buf = NULL; 3873 g_compare_write_buf = NULL; 3874 } 3875 3876 static void 3877 bdev_write_zeroes(void) 3878 { 3879 struct spdk_bdev *bdev; 3880 struct spdk_bdev_desc *desc = NULL; 3881 struct spdk_io_channel *ioch; 3882 struct ut_expected_io *expected_io; 3883 uint64_t offset, num_io_blocks, num_blocks; 3884 uint32_t num_completed, num_requests; 3885 int rc; 3886 3887 spdk_bdev_initialize(bdev_init_cb, NULL); 3888 bdev = allocate_bdev("bdev"); 3889 3890 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 3891 CU_ASSERT_EQUAL(rc, 0); 3892 SPDK_CU_ASSERT_FATAL(desc != NULL); 3893 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 3894 ioch = spdk_bdev_get_io_channel(desc); 3895 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3896 3897 fn_table.submit_request = stub_submit_request; 3898 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 3899 3900 /* First test that if the bdev supports write_zeroes, the request won't be split */ 3901 bdev->md_len = 0; 3902 bdev->blocklen = 4096; 3903 num_blocks = (ZERO_BUFFER_SIZE / bdev->blocklen) * 2; 3904 3905 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, 0, num_blocks, 0); 3906 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3907 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3908 CU_ASSERT_EQUAL(rc, 0); 3909 num_completed = stub_complete_io(1); 3910 CU_ASSERT_EQUAL(num_completed, 1); 3911 3912 /* Check that if write zeroes is not supported it'll be replaced by regular writes */ 3913 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, false); 3914 num_io_blocks = ZERO_BUFFER_SIZE / bdev->blocklen; 3915 num_requests = 2; 3916 num_blocks = (ZERO_BUFFER_SIZE / bdev->blocklen) * num_requests; 3917 3918 for (offset = 0; offset < num_requests; ++offset) { 3919 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 3920 offset * num_io_blocks, num_io_blocks, 0); 3921 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3922 } 3923 3924 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3925 CU_ASSERT_EQUAL(rc, 0); 3926 num_completed = stub_complete_io(num_requests); 3927 CU_ASSERT_EQUAL(num_completed, num_requests); 3928 3929 /* Check that the splitting is correct if bdev has interleaved metadata */ 3930 bdev->md_interleave = true; 3931 bdev->md_len = 64; 3932 bdev->blocklen = 4096 + 64; 3933 num_blocks = (ZERO_BUFFER_SIZE / bdev->blocklen) * 2; 3934 3935 num_requests = offset = 0; 3936 while (offset < num_blocks) { 3937 num_io_blocks = spdk_min(ZERO_BUFFER_SIZE / bdev->blocklen, num_blocks - offset); 3938 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 3939 offset, num_io_blocks, 0); 3940 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3941 offset += num_io_blocks; 3942 num_requests++; 3943 } 3944 3945 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3946 CU_ASSERT_EQUAL(rc, 0); 3947 num_completed = stub_complete_io(num_requests); 3948 CU_ASSERT_EQUAL(num_completed, num_requests); 3949 num_completed = stub_complete_io(num_requests); 3950 assert(num_completed == 0); 3951 3952 /* Check the the same for separate metadata buffer */ 3953 bdev->md_interleave = false; 3954 bdev->md_len = 64; 3955 bdev->blocklen = 4096; 3956 3957 num_requests = offset = 0; 3958 while (offset < num_blocks) { 3959 num_io_blocks = spdk_min(ZERO_BUFFER_SIZE / (bdev->blocklen + bdev->md_len), num_blocks); 3960 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 3961 offset, num_io_blocks, 0); 3962 expected_io->md_buf = (char *)g_bdev_mgr.zero_buffer + num_io_blocks * bdev->blocklen; 3963 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 3964 offset += num_io_blocks; 3965 num_requests++; 3966 } 3967 3968 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 3969 CU_ASSERT_EQUAL(rc, 0); 3970 num_completed = stub_complete_io(num_requests); 3971 CU_ASSERT_EQUAL(num_completed, num_requests); 3972 3973 ut_enable_io_type(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, true); 3974 spdk_put_io_channel(ioch); 3975 spdk_bdev_close(desc); 3976 free_bdev(bdev); 3977 spdk_bdev_finish(bdev_fini_cb, NULL); 3978 poll_threads(); 3979 } 3980 3981 static void 3982 bdev_zcopy_write(void) 3983 { 3984 struct spdk_bdev *bdev; 3985 struct spdk_bdev_desc *desc = NULL; 3986 struct spdk_io_channel *ioch; 3987 struct ut_expected_io *expected_io; 3988 uint64_t offset, num_blocks; 3989 uint32_t num_completed; 3990 char aa_buf[512]; 3991 struct iovec iov; 3992 int rc; 3993 const bool populate = false; 3994 const bool commit = true; 3995 3996 memset(aa_buf, 0xaa, sizeof(aa_buf)); 3997 3998 spdk_bdev_initialize(bdev_init_cb, NULL); 3999 bdev = allocate_bdev("bdev"); 4000 4001 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 4002 CU_ASSERT_EQUAL(rc, 0); 4003 SPDK_CU_ASSERT_FATAL(desc != NULL); 4004 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4005 ioch = spdk_bdev_get_io_channel(desc); 4006 SPDK_CU_ASSERT_FATAL(ioch != NULL); 4007 4008 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4009 4010 offset = 50; 4011 num_blocks = 1; 4012 iov.iov_base = NULL; 4013 iov.iov_len = 0; 4014 4015 g_zcopy_read_buf = (void *) 0x1122334455667788UL; 4016 g_zcopy_read_buf_len = (uint32_t) -1; 4017 /* Do a zcopy start for a write (populate=false) */ 4018 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_ZCOPY, offset, num_blocks, 0); 4019 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 4020 g_io_done = false; 4021 g_zcopy_write_buf = aa_buf; 4022 g_zcopy_write_buf_len = sizeof(aa_buf); 4023 g_zcopy_bdev_io = NULL; 4024 rc = spdk_bdev_zcopy_start(desc, ioch, &iov, 1, offset, num_blocks, populate, io_done, NULL); 4025 CU_ASSERT_EQUAL(rc, 0); 4026 num_completed = stub_complete_io(1); 4027 CU_ASSERT_EQUAL(num_completed, 1); 4028 CU_ASSERT(g_io_done == true); 4029 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4030 /* Check that the iov has been set up */ 4031 CU_ASSERT(iov.iov_base == g_zcopy_write_buf); 4032 CU_ASSERT(iov.iov_len == g_zcopy_write_buf_len); 4033 /* Check that the bdev_io has been saved */ 4034 CU_ASSERT(g_zcopy_bdev_io != NULL); 4035 /* Now do the zcopy end for a write (commit=true) */ 4036 g_io_done = false; 4037 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_ZCOPY, offset, num_blocks, 0); 4038 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 4039 rc = spdk_bdev_zcopy_end(g_zcopy_bdev_io, commit, io_done, NULL); 4040 CU_ASSERT_EQUAL(rc, 0); 4041 num_completed = stub_complete_io(1); 4042 CU_ASSERT_EQUAL(num_completed, 1); 4043 CU_ASSERT(g_io_done == true); 4044 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4045 /* Check the g_zcopy are reset by io_done */ 4046 CU_ASSERT(g_zcopy_write_buf == NULL); 4047 CU_ASSERT(g_zcopy_write_buf_len == 0); 4048 /* Check that io_done has freed the g_zcopy_bdev_io */ 4049 CU_ASSERT(g_zcopy_bdev_io == NULL); 4050 4051 /* Check the zcopy read buffer has not been touched which 4052 * ensures that the correct buffers were used. 4053 */ 4054 CU_ASSERT(g_zcopy_read_buf == (void *) 0x1122334455667788UL); 4055 CU_ASSERT(g_zcopy_read_buf_len == (uint32_t) -1); 4056 4057 spdk_put_io_channel(ioch); 4058 spdk_bdev_close(desc); 4059 free_bdev(bdev); 4060 spdk_bdev_finish(bdev_fini_cb, NULL); 4061 poll_threads(); 4062 } 4063 4064 static void 4065 bdev_zcopy_read(void) 4066 { 4067 struct spdk_bdev *bdev; 4068 struct spdk_bdev_desc *desc = NULL; 4069 struct spdk_io_channel *ioch; 4070 struct ut_expected_io *expected_io; 4071 uint64_t offset, num_blocks; 4072 uint32_t num_completed; 4073 char aa_buf[512]; 4074 struct iovec iov; 4075 int rc; 4076 const bool populate = true; 4077 const bool commit = false; 4078 4079 memset(aa_buf, 0xaa, sizeof(aa_buf)); 4080 4081 spdk_bdev_initialize(bdev_init_cb, NULL); 4082 bdev = allocate_bdev("bdev"); 4083 4084 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 4085 CU_ASSERT_EQUAL(rc, 0); 4086 SPDK_CU_ASSERT_FATAL(desc != NULL); 4087 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4088 ioch = spdk_bdev_get_io_channel(desc); 4089 SPDK_CU_ASSERT_FATAL(ioch != NULL); 4090 4091 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 4092 4093 offset = 50; 4094 num_blocks = 1; 4095 iov.iov_base = NULL; 4096 iov.iov_len = 0; 4097 4098 g_zcopy_write_buf = (void *) 0x1122334455667788UL; 4099 g_zcopy_write_buf_len = (uint32_t) -1; 4100 4101 /* Do a zcopy start for a read (populate=true) */ 4102 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_ZCOPY, offset, num_blocks, 0); 4103 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 4104 g_io_done = false; 4105 g_zcopy_read_buf = aa_buf; 4106 g_zcopy_read_buf_len = sizeof(aa_buf); 4107 g_zcopy_bdev_io = NULL; 4108 rc = spdk_bdev_zcopy_start(desc, ioch, &iov, 1, offset, num_blocks, populate, io_done, NULL); 4109 CU_ASSERT_EQUAL(rc, 0); 4110 num_completed = stub_complete_io(1); 4111 CU_ASSERT_EQUAL(num_completed, 1); 4112 CU_ASSERT(g_io_done == true); 4113 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4114 /* Check that the iov has been set up */ 4115 CU_ASSERT(iov.iov_base == g_zcopy_read_buf); 4116 CU_ASSERT(iov.iov_len == g_zcopy_read_buf_len); 4117 /* Check that the bdev_io has been saved */ 4118 CU_ASSERT(g_zcopy_bdev_io != NULL); 4119 4120 /* Now do the zcopy end for a read (commit=false) */ 4121 g_io_done = false; 4122 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_ZCOPY, offset, num_blocks, 0); 4123 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 4124 rc = spdk_bdev_zcopy_end(g_zcopy_bdev_io, commit, io_done, NULL); 4125 CU_ASSERT_EQUAL(rc, 0); 4126 num_completed = stub_complete_io(1); 4127 CU_ASSERT_EQUAL(num_completed, 1); 4128 CU_ASSERT(g_io_done == true); 4129 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 4130 /* Check the g_zcopy are reset by io_done */ 4131 CU_ASSERT(g_zcopy_read_buf == NULL); 4132 CU_ASSERT(g_zcopy_read_buf_len == 0); 4133 /* Check that io_done has freed the g_zcopy_bdev_io */ 4134 CU_ASSERT(g_zcopy_bdev_io == NULL); 4135 4136 /* Check the zcopy write buffer has not been touched which 4137 * ensures that the correct buffers were used. 4138 */ 4139 CU_ASSERT(g_zcopy_write_buf == (void *) 0x1122334455667788UL); 4140 CU_ASSERT(g_zcopy_write_buf_len == (uint32_t) -1); 4141 4142 spdk_put_io_channel(ioch); 4143 spdk_bdev_close(desc); 4144 free_bdev(bdev); 4145 spdk_bdev_finish(bdev_fini_cb, NULL); 4146 poll_threads(); 4147 } 4148 4149 static void 4150 bdev_open_while_hotremove(void) 4151 { 4152 struct spdk_bdev *bdev; 4153 struct spdk_bdev_desc *desc[2] = {}; 4154 int rc; 4155 4156 bdev = allocate_bdev("bdev"); 4157 4158 rc = spdk_bdev_open_ext("bdev", false, bdev_ut_event_cb, NULL, &desc[0]); 4159 CU_ASSERT(rc == 0); 4160 SPDK_CU_ASSERT_FATAL(desc[0] != NULL); 4161 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc[0])); 4162 4163 spdk_bdev_unregister(bdev, NULL, NULL); 4164 /* Bdev unregister is handled asynchronously. Poll thread to complete. */ 4165 poll_threads(); 4166 4167 rc = spdk_bdev_open_ext("bdev", false, bdev_ut_event_cb, NULL, &desc[1]); 4168 CU_ASSERT(rc == -ENODEV); 4169 SPDK_CU_ASSERT_FATAL(desc[1] == NULL); 4170 4171 spdk_bdev_close(desc[0]); 4172 free_bdev(bdev); 4173 } 4174 4175 static void 4176 bdev_close_while_hotremove(void) 4177 { 4178 struct spdk_bdev *bdev; 4179 struct spdk_bdev_desc *desc = NULL; 4180 int rc = 0; 4181 4182 bdev = allocate_bdev("bdev"); 4183 4184 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb1, &desc, &desc); 4185 CU_ASSERT_EQUAL(rc, 0); 4186 SPDK_CU_ASSERT_FATAL(desc != NULL); 4187 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4188 4189 /* Simulate hot-unplug by unregistering bdev */ 4190 g_event_type1 = 0xFF; 4191 g_unregister_arg = NULL; 4192 g_unregister_rc = -1; 4193 spdk_bdev_unregister(bdev, bdev_unregister_cb, (void *)0x12345678); 4194 /* Close device while remove event is in flight */ 4195 spdk_bdev_close(desc); 4196 4197 /* Ensure that unregister callback is delayed */ 4198 CU_ASSERT_EQUAL(g_unregister_arg, NULL); 4199 CU_ASSERT_EQUAL(g_unregister_rc, -1); 4200 4201 poll_threads(); 4202 4203 /* Event callback shall not be issued because device was closed */ 4204 CU_ASSERT_EQUAL(g_event_type1, 0xFF); 4205 /* Unregister callback is issued */ 4206 CU_ASSERT_EQUAL(g_unregister_arg, (void *)0x12345678); 4207 CU_ASSERT_EQUAL(g_unregister_rc, 0); 4208 4209 free_bdev(bdev); 4210 } 4211 4212 static void 4213 bdev_open_ext(void) 4214 { 4215 struct spdk_bdev *bdev; 4216 struct spdk_bdev_desc *desc1 = NULL; 4217 struct spdk_bdev_desc *desc2 = NULL; 4218 int rc = 0; 4219 4220 bdev = allocate_bdev("bdev"); 4221 4222 rc = spdk_bdev_open_ext("bdev", true, NULL, NULL, &desc1); 4223 CU_ASSERT_EQUAL(rc, -EINVAL); 4224 4225 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb1, &desc1, &desc1); 4226 CU_ASSERT_EQUAL(rc, 0); 4227 4228 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb2, &desc2, &desc2); 4229 CU_ASSERT_EQUAL(rc, 0); 4230 4231 g_event_type1 = 0xFF; 4232 g_event_type2 = 0xFF; 4233 4234 /* Simulate hot-unplug by unregistering bdev */ 4235 spdk_bdev_unregister(bdev, NULL, NULL); 4236 poll_threads(); 4237 4238 /* Check if correct events have been triggered in event callback fn */ 4239 CU_ASSERT_EQUAL(g_event_type1, SPDK_BDEV_EVENT_REMOVE); 4240 CU_ASSERT_EQUAL(g_event_type2, SPDK_BDEV_EVENT_REMOVE); 4241 4242 free_bdev(bdev); 4243 poll_threads(); 4244 } 4245 4246 static void 4247 bdev_open_ext_unregister(void) 4248 { 4249 struct spdk_bdev *bdev; 4250 struct spdk_bdev_desc *desc1 = NULL; 4251 struct spdk_bdev_desc *desc2 = NULL; 4252 struct spdk_bdev_desc *desc3 = NULL; 4253 struct spdk_bdev_desc *desc4 = NULL; 4254 int rc = 0; 4255 4256 bdev = allocate_bdev("bdev"); 4257 4258 rc = spdk_bdev_open_ext("bdev", true, NULL, NULL, &desc1); 4259 CU_ASSERT_EQUAL(rc, -EINVAL); 4260 4261 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb1, &desc1, &desc1); 4262 CU_ASSERT_EQUAL(rc, 0); 4263 4264 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb2, &desc2, &desc2); 4265 CU_ASSERT_EQUAL(rc, 0); 4266 4267 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb3, &desc3, &desc3); 4268 CU_ASSERT_EQUAL(rc, 0); 4269 4270 rc = spdk_bdev_open_ext("bdev", true, bdev_open_cb4, &desc4, &desc4); 4271 CU_ASSERT_EQUAL(rc, 0); 4272 4273 g_event_type1 = 0xFF; 4274 g_event_type2 = 0xFF; 4275 g_event_type3 = 0xFF; 4276 g_event_type4 = 0xFF; 4277 4278 g_unregister_arg = NULL; 4279 g_unregister_rc = -1; 4280 4281 /* Simulate hot-unplug by unregistering bdev */ 4282 spdk_bdev_unregister(bdev, bdev_unregister_cb, (void *)0x12345678); 4283 4284 /* 4285 * Unregister is handled asynchronously and event callback 4286 * (i.e., above bdev_open_cbN) will be called. 4287 * For bdev_open_cb3 and bdev_open_cb4, it is intended to not 4288 * close the desc3 and desc4 so that the bdev is not closed. 4289 */ 4290 poll_threads(); 4291 4292 /* Check if correct events have been triggered in event callback fn */ 4293 CU_ASSERT_EQUAL(g_event_type1, SPDK_BDEV_EVENT_REMOVE); 4294 CU_ASSERT_EQUAL(g_event_type2, SPDK_BDEV_EVENT_REMOVE); 4295 CU_ASSERT_EQUAL(g_event_type3, SPDK_BDEV_EVENT_REMOVE); 4296 CU_ASSERT_EQUAL(g_event_type4, SPDK_BDEV_EVENT_REMOVE); 4297 4298 /* Check that unregister callback is delayed */ 4299 CU_ASSERT(g_unregister_arg == NULL); 4300 CU_ASSERT(g_unregister_rc == -1); 4301 4302 /* 4303 * Explicitly close desc3. As desc4 is still opened there, the 4304 * unergister callback is still delayed to execute. 4305 */ 4306 spdk_bdev_close(desc3); 4307 CU_ASSERT(g_unregister_arg == NULL); 4308 CU_ASSERT(g_unregister_rc == -1); 4309 4310 /* 4311 * Explicitly close desc4 to trigger the ongoing bdev unregister 4312 * operation after last desc is closed. 4313 */ 4314 spdk_bdev_close(desc4); 4315 4316 /* Poll the thread for the async unregister operation */ 4317 poll_threads(); 4318 4319 /* Check that unregister callback is executed */ 4320 CU_ASSERT(g_unregister_arg == (void *)0x12345678); 4321 CU_ASSERT(g_unregister_rc == 0); 4322 4323 free_bdev(bdev); 4324 poll_threads(); 4325 } 4326 4327 struct timeout_io_cb_arg { 4328 struct iovec iov; 4329 uint8_t type; 4330 }; 4331 4332 static int 4333 bdev_channel_count_submitted_io(struct spdk_bdev_channel *ch) 4334 { 4335 struct spdk_bdev_io *bdev_io; 4336 int n = 0; 4337 4338 if (!ch) { 4339 return -1; 4340 } 4341 4342 TAILQ_FOREACH(bdev_io, &ch->io_submitted, internal.ch_link) { 4343 n++; 4344 } 4345 4346 return n; 4347 } 4348 4349 static void 4350 bdev_channel_io_timeout_cb(void *cb_arg, struct spdk_bdev_io *bdev_io) 4351 { 4352 struct timeout_io_cb_arg *ctx = cb_arg; 4353 4354 ctx->type = bdev_io->type; 4355 ctx->iov.iov_base = bdev_io->iov.iov_base; 4356 ctx->iov.iov_len = bdev_io->iov.iov_len; 4357 } 4358 4359 static void 4360 bdev_set_io_timeout(void) 4361 { 4362 struct spdk_bdev *bdev; 4363 struct spdk_bdev_desc *desc = NULL; 4364 struct spdk_io_channel *io_ch = NULL; 4365 struct spdk_bdev_channel *bdev_ch = NULL; 4366 struct timeout_io_cb_arg cb_arg; 4367 4368 spdk_bdev_initialize(bdev_init_cb, NULL); 4369 4370 bdev = allocate_bdev("bdev"); 4371 4372 CU_ASSERT(spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc) == 0); 4373 SPDK_CU_ASSERT_FATAL(desc != NULL); 4374 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4375 4376 io_ch = spdk_bdev_get_io_channel(desc); 4377 CU_ASSERT(io_ch != NULL); 4378 4379 bdev_ch = spdk_io_channel_get_ctx(io_ch); 4380 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->io_submitted)); 4381 4382 /* This is the part1. 4383 * We will check the bdev_ch->io_submitted list 4384 * TO make sure that it can link IOs and only the user submitted IOs 4385 */ 4386 CU_ASSERT(spdk_bdev_read(desc, io_ch, (void *)0x1000, 0, 4096, io_done, NULL) == 0); 4387 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 4388 CU_ASSERT(spdk_bdev_write(desc, io_ch, (void *)0x2000, 0, 4096, io_done, NULL) == 0); 4389 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 2); 4390 stub_complete_io(1); 4391 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 4392 stub_complete_io(1); 4393 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 0); 4394 4395 /* Split IO */ 4396 bdev->optimal_io_boundary = 16; 4397 bdev->split_on_optimal_io_boundary = true; 4398 4399 /* Now test that a single-vector command is split correctly. 4400 * Offset 14, length 8, payload 0xF000 4401 * Child - Offset 14, length 2, payload 0xF000 4402 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 4403 * 4404 * Set up the expected values before calling spdk_bdev_read_blocks 4405 */ 4406 CU_ASSERT(spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL) == 0); 4407 /* We count all submitted IOs including IO that are generated by splitting. */ 4408 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 3); 4409 stub_complete_io(1); 4410 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 2); 4411 stub_complete_io(1); 4412 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 0); 4413 4414 /* Also include the reset IO */ 4415 CU_ASSERT(spdk_bdev_reset(desc, io_ch, io_done, NULL) == 0); 4416 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 4417 poll_threads(); 4418 stub_complete_io(1); 4419 poll_threads(); 4420 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 0); 4421 4422 /* This is part2 4423 * Test the desc timeout poller register 4424 */ 4425 4426 /* Successfully set the timeout */ 4427 CU_ASSERT(spdk_bdev_set_timeout(desc, 30, bdev_channel_io_timeout_cb, &cb_arg) == 0); 4428 CU_ASSERT(desc->io_timeout_poller != NULL); 4429 CU_ASSERT(desc->timeout_in_sec == 30); 4430 CU_ASSERT(desc->cb_fn == bdev_channel_io_timeout_cb); 4431 CU_ASSERT(desc->cb_arg == &cb_arg); 4432 4433 /* Change the timeout limit */ 4434 CU_ASSERT(spdk_bdev_set_timeout(desc, 20, bdev_channel_io_timeout_cb, &cb_arg) == 0); 4435 CU_ASSERT(desc->io_timeout_poller != NULL); 4436 CU_ASSERT(desc->timeout_in_sec == 20); 4437 CU_ASSERT(desc->cb_fn == bdev_channel_io_timeout_cb); 4438 CU_ASSERT(desc->cb_arg == &cb_arg); 4439 4440 /* Disable the timeout */ 4441 CU_ASSERT(spdk_bdev_set_timeout(desc, 0, NULL, NULL) == 0); 4442 CU_ASSERT(desc->io_timeout_poller == NULL); 4443 4444 /* This the part3 4445 * We will test to catch timeout IO and check whether the IO is 4446 * the submitted one. 4447 */ 4448 memset(&cb_arg, 0, sizeof(cb_arg)); 4449 CU_ASSERT(spdk_bdev_set_timeout(desc, 30, bdev_channel_io_timeout_cb, &cb_arg) == 0); 4450 CU_ASSERT(spdk_bdev_write_blocks(desc, io_ch, (void *)0x1000, 0, 1, io_done, NULL) == 0); 4451 4452 /* Don't reach the limit */ 4453 spdk_delay_us(15 * spdk_get_ticks_hz()); 4454 poll_threads(); 4455 CU_ASSERT(cb_arg.type == 0); 4456 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 4457 CU_ASSERT(cb_arg.iov.iov_len == 0); 4458 4459 /* 15 + 15 = 30 reach the limit */ 4460 spdk_delay_us(15 * spdk_get_ticks_hz()); 4461 poll_threads(); 4462 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 4463 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x1000); 4464 CU_ASSERT(cb_arg.iov.iov_len == 1 * bdev->blocklen); 4465 stub_complete_io(1); 4466 4467 /* Use the same split IO above and check the IO */ 4468 memset(&cb_arg, 0, sizeof(cb_arg)); 4469 CU_ASSERT(spdk_bdev_write_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, NULL) == 0); 4470 4471 /* The first child complete in time */ 4472 spdk_delay_us(15 * spdk_get_ticks_hz()); 4473 poll_threads(); 4474 stub_complete_io(1); 4475 CU_ASSERT(cb_arg.type == 0); 4476 CU_ASSERT(cb_arg.iov.iov_base == (void *)0x0); 4477 CU_ASSERT(cb_arg.iov.iov_len == 0); 4478 4479 /* The second child reach the limit */ 4480 spdk_delay_us(15 * spdk_get_ticks_hz()); 4481 poll_threads(); 4482 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_WRITE); 4483 CU_ASSERT(cb_arg.iov.iov_base == (void *)0xF000); 4484 CU_ASSERT(cb_arg.iov.iov_len == 8 * bdev->blocklen); 4485 stub_complete_io(1); 4486 4487 /* Also include the reset IO */ 4488 memset(&cb_arg, 0, sizeof(cb_arg)); 4489 CU_ASSERT(spdk_bdev_reset(desc, io_ch, io_done, NULL) == 0); 4490 spdk_delay_us(30 * spdk_get_ticks_hz()); 4491 poll_threads(); 4492 CU_ASSERT(cb_arg.type == SPDK_BDEV_IO_TYPE_RESET); 4493 stub_complete_io(1); 4494 poll_threads(); 4495 4496 spdk_put_io_channel(io_ch); 4497 spdk_bdev_close(desc); 4498 free_bdev(bdev); 4499 spdk_bdev_finish(bdev_fini_cb, NULL); 4500 poll_threads(); 4501 } 4502 4503 static void 4504 bdev_set_qd_sampling(void) 4505 { 4506 struct spdk_bdev *bdev; 4507 struct spdk_bdev_desc *desc = NULL; 4508 struct spdk_io_channel *io_ch = NULL; 4509 struct spdk_bdev_channel *bdev_ch = NULL; 4510 struct timeout_io_cb_arg cb_arg; 4511 4512 spdk_bdev_initialize(bdev_init_cb, NULL); 4513 4514 bdev = allocate_bdev("bdev"); 4515 4516 CU_ASSERT(spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc) == 0); 4517 SPDK_CU_ASSERT_FATAL(desc != NULL); 4518 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4519 4520 io_ch = spdk_bdev_get_io_channel(desc); 4521 CU_ASSERT(io_ch != NULL); 4522 4523 bdev_ch = spdk_io_channel_get_ctx(io_ch); 4524 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->io_submitted)); 4525 4526 /* This is the part1. 4527 * We will check the bdev_ch->io_submitted list 4528 * TO make sure that it can link IOs and only the user submitted IOs 4529 */ 4530 CU_ASSERT(spdk_bdev_read(desc, io_ch, (void *)0x1000, 0, 4096, io_done, NULL) == 0); 4531 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 4532 CU_ASSERT(spdk_bdev_write(desc, io_ch, (void *)0x2000, 0, 4096, io_done, NULL) == 0); 4533 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 2); 4534 stub_complete_io(1); 4535 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 4536 stub_complete_io(1); 4537 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 0); 4538 4539 /* This is the part2. 4540 * Test the bdev's qd poller register 4541 */ 4542 /* 1st Successfully set the qd sampling period */ 4543 spdk_bdev_set_qd_sampling_period(bdev, 10); 4544 CU_ASSERT(bdev->internal.new_period == 10); 4545 CU_ASSERT(bdev->internal.period == 10); 4546 CU_ASSERT(bdev->internal.qd_desc != NULL); 4547 poll_threads(); 4548 CU_ASSERT(bdev->internal.qd_poller != NULL); 4549 4550 /* 2nd Change the qd sampling period */ 4551 spdk_bdev_set_qd_sampling_period(bdev, 20); 4552 CU_ASSERT(bdev->internal.new_period == 20); 4553 CU_ASSERT(bdev->internal.period == 10); 4554 CU_ASSERT(bdev->internal.qd_desc != NULL); 4555 poll_threads(); 4556 CU_ASSERT(bdev->internal.qd_poller != NULL); 4557 CU_ASSERT(bdev->internal.period == bdev->internal.new_period); 4558 4559 /* 3rd Change the qd sampling period and verify qd_poll_in_progress */ 4560 spdk_delay_us(20); 4561 poll_thread_times(0, 1); 4562 CU_ASSERT(bdev->internal.qd_poll_in_progress == true); 4563 spdk_bdev_set_qd_sampling_period(bdev, 30); 4564 CU_ASSERT(bdev->internal.new_period == 30); 4565 CU_ASSERT(bdev->internal.period == 20); 4566 poll_threads(); 4567 CU_ASSERT(bdev->internal.qd_poll_in_progress == false); 4568 CU_ASSERT(bdev->internal.period == bdev->internal.new_period); 4569 4570 /* 4th Disable the qd sampling period */ 4571 spdk_bdev_set_qd_sampling_period(bdev, 0); 4572 CU_ASSERT(bdev->internal.new_period == 0); 4573 CU_ASSERT(bdev->internal.period == 30); 4574 poll_threads(); 4575 CU_ASSERT(bdev->internal.qd_poller == NULL); 4576 CU_ASSERT(bdev->internal.period == bdev->internal.new_period); 4577 CU_ASSERT(bdev->internal.qd_desc == NULL); 4578 4579 /* This is the part3. 4580 * We will test the submitted IO and reset works 4581 * properly with the qd sampling. 4582 */ 4583 memset(&cb_arg, 0, sizeof(cb_arg)); 4584 spdk_bdev_set_qd_sampling_period(bdev, 1); 4585 poll_threads(); 4586 4587 CU_ASSERT(spdk_bdev_write(desc, io_ch, (void *)0x2000, 0, 4096, io_done, NULL) == 0); 4588 CU_ASSERT(bdev_channel_count_submitted_io(bdev_ch) == 1); 4589 4590 /* Also include the reset IO */ 4591 memset(&cb_arg, 0, sizeof(cb_arg)); 4592 CU_ASSERT(spdk_bdev_reset(desc, io_ch, io_done, NULL) == 0); 4593 poll_threads(); 4594 4595 /* Close the desc */ 4596 spdk_put_io_channel(io_ch); 4597 spdk_bdev_close(desc); 4598 4599 /* Complete the submitted IO and reset */ 4600 stub_complete_io(2); 4601 poll_threads(); 4602 4603 free_bdev(bdev); 4604 spdk_bdev_finish(bdev_fini_cb, NULL); 4605 poll_threads(); 4606 } 4607 4608 static void 4609 lba_range_overlap(void) 4610 { 4611 struct lba_range r1, r2; 4612 4613 r1.offset = 100; 4614 r1.length = 50; 4615 4616 r2.offset = 0; 4617 r2.length = 1; 4618 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 4619 4620 r2.offset = 0; 4621 r2.length = 100; 4622 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 4623 4624 r2.offset = 0; 4625 r2.length = 110; 4626 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 4627 4628 r2.offset = 100; 4629 r2.length = 10; 4630 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 4631 4632 r2.offset = 110; 4633 r2.length = 20; 4634 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 4635 4636 r2.offset = 140; 4637 r2.length = 150; 4638 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 4639 4640 r2.offset = 130; 4641 r2.length = 200; 4642 CU_ASSERT(bdev_lba_range_overlapped(&r1, &r2)); 4643 4644 r2.offset = 150; 4645 r2.length = 100; 4646 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 4647 4648 r2.offset = 110; 4649 r2.length = 0; 4650 CU_ASSERT(!bdev_lba_range_overlapped(&r1, &r2)); 4651 } 4652 4653 static bool g_lock_lba_range_done; 4654 static bool g_unlock_lba_range_done; 4655 4656 static void 4657 lock_lba_range_done(void *ctx, int status) 4658 { 4659 g_lock_lba_range_done = true; 4660 } 4661 4662 static void 4663 unlock_lba_range_done(void *ctx, int status) 4664 { 4665 g_unlock_lba_range_done = true; 4666 } 4667 4668 static void 4669 lock_lba_range_check_ranges(void) 4670 { 4671 struct spdk_bdev *bdev; 4672 struct spdk_bdev_desc *desc = NULL; 4673 struct spdk_io_channel *io_ch; 4674 struct spdk_bdev_channel *channel; 4675 struct lba_range *range; 4676 int ctx1; 4677 int rc; 4678 4679 spdk_bdev_initialize(bdev_init_cb, NULL); 4680 4681 bdev = allocate_bdev("bdev0"); 4682 4683 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 4684 CU_ASSERT(rc == 0); 4685 CU_ASSERT(desc != NULL); 4686 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4687 io_ch = spdk_bdev_get_io_channel(desc); 4688 CU_ASSERT(io_ch != NULL); 4689 channel = spdk_io_channel_get_ctx(io_ch); 4690 4691 g_lock_lba_range_done = false; 4692 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 4693 CU_ASSERT(rc == 0); 4694 poll_threads(); 4695 4696 CU_ASSERT(g_lock_lba_range_done == true); 4697 range = TAILQ_FIRST(&channel->locked_ranges); 4698 SPDK_CU_ASSERT_FATAL(range != NULL); 4699 CU_ASSERT(range->offset == 20); 4700 CU_ASSERT(range->length == 10); 4701 CU_ASSERT(range->owner_ch == channel); 4702 4703 /* Unlocks must exactly match a lock. */ 4704 g_unlock_lba_range_done = false; 4705 rc = bdev_unlock_lba_range(desc, io_ch, 20, 1, unlock_lba_range_done, &ctx1); 4706 CU_ASSERT(rc == -EINVAL); 4707 CU_ASSERT(g_unlock_lba_range_done == false); 4708 4709 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, unlock_lba_range_done, &ctx1); 4710 CU_ASSERT(rc == 0); 4711 spdk_delay_us(100); 4712 poll_threads(); 4713 4714 CU_ASSERT(g_unlock_lba_range_done == true); 4715 CU_ASSERT(TAILQ_EMPTY(&channel->locked_ranges)); 4716 4717 spdk_put_io_channel(io_ch); 4718 spdk_bdev_close(desc); 4719 free_bdev(bdev); 4720 spdk_bdev_finish(bdev_fini_cb, NULL); 4721 poll_threads(); 4722 } 4723 4724 static void 4725 lock_lba_range_with_io_outstanding(void) 4726 { 4727 struct spdk_bdev *bdev; 4728 struct spdk_bdev_desc *desc = NULL; 4729 struct spdk_io_channel *io_ch; 4730 struct spdk_bdev_channel *channel; 4731 struct lba_range *range; 4732 char buf[4096]; 4733 int ctx1; 4734 int rc; 4735 4736 spdk_bdev_initialize(bdev_init_cb, NULL); 4737 4738 bdev = allocate_bdev("bdev0"); 4739 4740 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 4741 CU_ASSERT(rc == 0); 4742 CU_ASSERT(desc != NULL); 4743 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4744 io_ch = spdk_bdev_get_io_channel(desc); 4745 CU_ASSERT(io_ch != NULL); 4746 channel = spdk_io_channel_get_ctx(io_ch); 4747 4748 g_io_done = false; 4749 rc = spdk_bdev_read_blocks(desc, io_ch, buf, 20, 1, io_done, &ctx1); 4750 CU_ASSERT(rc == 0); 4751 4752 g_lock_lba_range_done = false; 4753 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 4754 CU_ASSERT(rc == 0); 4755 poll_threads(); 4756 4757 /* The lock should immediately become valid, since there are no outstanding 4758 * write I/O. 4759 */ 4760 CU_ASSERT(g_io_done == false); 4761 CU_ASSERT(g_lock_lba_range_done == true); 4762 range = TAILQ_FIRST(&channel->locked_ranges); 4763 SPDK_CU_ASSERT_FATAL(range != NULL); 4764 CU_ASSERT(range->offset == 20); 4765 CU_ASSERT(range->length == 10); 4766 CU_ASSERT(range->owner_ch == channel); 4767 CU_ASSERT(range->locked_ctx == &ctx1); 4768 4769 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 4770 CU_ASSERT(rc == 0); 4771 stub_complete_io(1); 4772 spdk_delay_us(100); 4773 poll_threads(); 4774 4775 CU_ASSERT(TAILQ_EMPTY(&channel->locked_ranges)); 4776 4777 /* Now try again, but with a write I/O. */ 4778 g_io_done = false; 4779 rc = spdk_bdev_write_blocks(desc, io_ch, buf, 20, 1, io_done, &ctx1); 4780 CU_ASSERT(rc == 0); 4781 4782 g_lock_lba_range_done = false; 4783 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 4784 CU_ASSERT(rc == 0); 4785 poll_threads(); 4786 4787 /* The lock should not be fully valid yet, since a write I/O is outstanding. 4788 * But note that the range should be on the channel's locked_list, to make sure no 4789 * new write I/O are started. 4790 */ 4791 CU_ASSERT(g_io_done == false); 4792 CU_ASSERT(g_lock_lba_range_done == false); 4793 range = TAILQ_FIRST(&channel->locked_ranges); 4794 SPDK_CU_ASSERT_FATAL(range != NULL); 4795 CU_ASSERT(range->offset == 20); 4796 CU_ASSERT(range->length == 10); 4797 4798 /* Complete the write I/O. This should make the lock valid (checked by confirming 4799 * our callback was invoked). 4800 */ 4801 stub_complete_io(1); 4802 spdk_delay_us(100); 4803 poll_threads(); 4804 CU_ASSERT(g_io_done == true); 4805 CU_ASSERT(g_lock_lba_range_done == true); 4806 4807 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, unlock_lba_range_done, &ctx1); 4808 CU_ASSERT(rc == 0); 4809 poll_threads(); 4810 4811 CU_ASSERT(TAILQ_EMPTY(&channel->locked_ranges)); 4812 4813 spdk_put_io_channel(io_ch); 4814 spdk_bdev_close(desc); 4815 free_bdev(bdev); 4816 spdk_bdev_finish(bdev_fini_cb, NULL); 4817 poll_threads(); 4818 } 4819 4820 static void 4821 lock_lba_range_overlapped(void) 4822 { 4823 struct spdk_bdev *bdev; 4824 struct spdk_bdev_desc *desc = NULL; 4825 struct spdk_io_channel *io_ch; 4826 struct spdk_bdev_channel *channel; 4827 struct lba_range *range; 4828 int ctx1; 4829 int rc; 4830 4831 spdk_bdev_initialize(bdev_init_cb, NULL); 4832 4833 bdev = allocate_bdev("bdev0"); 4834 4835 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 4836 CU_ASSERT(rc == 0); 4837 CU_ASSERT(desc != NULL); 4838 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4839 io_ch = spdk_bdev_get_io_channel(desc); 4840 CU_ASSERT(io_ch != NULL); 4841 channel = spdk_io_channel_get_ctx(io_ch); 4842 4843 /* Lock range 20-29. */ 4844 g_lock_lba_range_done = false; 4845 rc = bdev_lock_lba_range(desc, io_ch, 20, 10, lock_lba_range_done, &ctx1); 4846 CU_ASSERT(rc == 0); 4847 poll_threads(); 4848 4849 CU_ASSERT(g_lock_lba_range_done == true); 4850 range = TAILQ_FIRST(&channel->locked_ranges); 4851 SPDK_CU_ASSERT_FATAL(range != NULL); 4852 CU_ASSERT(range->offset == 20); 4853 CU_ASSERT(range->length == 10); 4854 4855 /* Try to lock range 25-39. It should not lock immediately, since it overlaps with 4856 * 20-29. 4857 */ 4858 g_lock_lba_range_done = false; 4859 rc = bdev_lock_lba_range(desc, io_ch, 25, 15, lock_lba_range_done, &ctx1); 4860 CU_ASSERT(rc == 0); 4861 poll_threads(); 4862 4863 CU_ASSERT(g_lock_lba_range_done == false); 4864 range = TAILQ_FIRST(&bdev->internal.pending_locked_ranges); 4865 SPDK_CU_ASSERT_FATAL(range != NULL); 4866 CU_ASSERT(range->offset == 25); 4867 CU_ASSERT(range->length == 15); 4868 4869 /* Unlock 20-29. This should result in range 25-39 now getting locked since it 4870 * no longer overlaps with an active lock. 4871 */ 4872 g_unlock_lba_range_done = false; 4873 rc = bdev_unlock_lba_range(desc, io_ch, 20, 10, unlock_lba_range_done, &ctx1); 4874 CU_ASSERT(rc == 0); 4875 poll_threads(); 4876 4877 CU_ASSERT(g_unlock_lba_range_done == true); 4878 CU_ASSERT(TAILQ_EMPTY(&bdev->internal.pending_locked_ranges)); 4879 range = TAILQ_FIRST(&channel->locked_ranges); 4880 SPDK_CU_ASSERT_FATAL(range != NULL); 4881 CU_ASSERT(range->offset == 25); 4882 CU_ASSERT(range->length == 15); 4883 4884 /* Lock 40-59. This should immediately lock since it does not overlap with the 4885 * currently active 25-39 lock. 4886 */ 4887 g_lock_lba_range_done = false; 4888 rc = bdev_lock_lba_range(desc, io_ch, 40, 20, lock_lba_range_done, &ctx1); 4889 CU_ASSERT(rc == 0); 4890 poll_threads(); 4891 4892 CU_ASSERT(g_lock_lba_range_done == true); 4893 range = TAILQ_FIRST(&bdev->internal.locked_ranges); 4894 SPDK_CU_ASSERT_FATAL(range != NULL); 4895 range = TAILQ_NEXT(range, tailq); 4896 SPDK_CU_ASSERT_FATAL(range != NULL); 4897 CU_ASSERT(range->offset == 40); 4898 CU_ASSERT(range->length == 20); 4899 4900 /* Try to lock 35-44. Note that this overlaps with both 25-39 and 40-59. */ 4901 g_lock_lba_range_done = false; 4902 rc = bdev_lock_lba_range(desc, io_ch, 35, 10, lock_lba_range_done, &ctx1); 4903 CU_ASSERT(rc == 0); 4904 poll_threads(); 4905 4906 CU_ASSERT(g_lock_lba_range_done == false); 4907 range = TAILQ_FIRST(&bdev->internal.pending_locked_ranges); 4908 SPDK_CU_ASSERT_FATAL(range != NULL); 4909 CU_ASSERT(range->offset == 35); 4910 CU_ASSERT(range->length == 10); 4911 4912 /* Unlock 25-39. Make sure that 35-44 is still in the pending list, since 4913 * the 40-59 lock is still active. 4914 */ 4915 g_unlock_lba_range_done = false; 4916 rc = bdev_unlock_lba_range(desc, io_ch, 25, 15, unlock_lba_range_done, &ctx1); 4917 CU_ASSERT(rc == 0); 4918 poll_threads(); 4919 4920 CU_ASSERT(g_unlock_lba_range_done == true); 4921 CU_ASSERT(g_lock_lba_range_done == false); 4922 range = TAILQ_FIRST(&bdev->internal.pending_locked_ranges); 4923 SPDK_CU_ASSERT_FATAL(range != NULL); 4924 CU_ASSERT(range->offset == 35); 4925 CU_ASSERT(range->length == 10); 4926 4927 /* Unlock 40-59. This should result in 35-44 now getting locked, since there are 4928 * no longer any active overlapping locks. 4929 */ 4930 g_unlock_lba_range_done = false; 4931 rc = bdev_unlock_lba_range(desc, io_ch, 40, 20, unlock_lba_range_done, &ctx1); 4932 CU_ASSERT(rc == 0); 4933 poll_threads(); 4934 4935 CU_ASSERT(g_unlock_lba_range_done == true); 4936 CU_ASSERT(g_lock_lba_range_done == true); 4937 CU_ASSERT(TAILQ_EMPTY(&bdev->internal.pending_locked_ranges)); 4938 range = TAILQ_FIRST(&bdev->internal.locked_ranges); 4939 SPDK_CU_ASSERT_FATAL(range != NULL); 4940 CU_ASSERT(range->offset == 35); 4941 CU_ASSERT(range->length == 10); 4942 4943 /* Finally, unlock 35-44. */ 4944 g_unlock_lba_range_done = false; 4945 rc = bdev_unlock_lba_range(desc, io_ch, 35, 10, unlock_lba_range_done, &ctx1); 4946 CU_ASSERT(rc == 0); 4947 poll_threads(); 4948 4949 CU_ASSERT(g_unlock_lba_range_done == true); 4950 CU_ASSERT(TAILQ_EMPTY(&bdev->internal.locked_ranges)); 4951 4952 spdk_put_io_channel(io_ch); 4953 spdk_bdev_close(desc); 4954 free_bdev(bdev); 4955 spdk_bdev_finish(bdev_fini_cb, NULL); 4956 poll_threads(); 4957 } 4958 4959 static void 4960 abort_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 4961 { 4962 g_abort_done = true; 4963 g_abort_status = bdev_io->internal.status; 4964 spdk_bdev_free_io(bdev_io); 4965 } 4966 4967 static void 4968 bdev_io_abort(void) 4969 { 4970 struct spdk_bdev *bdev; 4971 struct spdk_bdev_desc *desc = NULL; 4972 struct spdk_io_channel *io_ch; 4973 struct spdk_bdev_channel *channel; 4974 struct spdk_bdev_mgmt_channel *mgmt_ch; 4975 struct spdk_bdev_opts bdev_opts = {}; 4976 struct iovec iov[BDEV_IO_NUM_CHILD_IOV * 2]; 4977 uint64_t io_ctx1 = 0, io_ctx2 = 0, i; 4978 int rc; 4979 4980 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 4981 bdev_opts.bdev_io_pool_size = 7; 4982 bdev_opts.bdev_io_cache_size = 2; 4983 4984 rc = spdk_bdev_set_opts(&bdev_opts); 4985 CU_ASSERT(rc == 0); 4986 spdk_bdev_initialize(bdev_init_cb, NULL); 4987 4988 bdev = allocate_bdev("bdev0"); 4989 4990 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 4991 CU_ASSERT(rc == 0); 4992 CU_ASSERT(desc != NULL); 4993 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 4994 io_ch = spdk_bdev_get_io_channel(desc); 4995 CU_ASSERT(io_ch != NULL); 4996 channel = spdk_io_channel_get_ctx(io_ch); 4997 mgmt_ch = channel->shared_resource->mgmt_ch; 4998 4999 g_abort_done = false; 5000 5001 ut_enable_io_type(SPDK_BDEV_IO_TYPE_ABORT, false); 5002 5003 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 5004 CU_ASSERT(rc == -ENOTSUP); 5005 5006 ut_enable_io_type(SPDK_BDEV_IO_TYPE_ABORT, true); 5007 5008 rc = spdk_bdev_abort(desc, io_ch, &io_ctx2, abort_done, NULL); 5009 CU_ASSERT(rc == 0); 5010 CU_ASSERT(g_abort_done == true); 5011 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_FAILED); 5012 5013 /* Test the case that the target I/O was successfully aborted. */ 5014 g_io_done = false; 5015 5016 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, &io_ctx1); 5017 CU_ASSERT(rc == 0); 5018 CU_ASSERT(g_io_done == false); 5019 5020 g_abort_done = false; 5021 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 5022 5023 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 5024 CU_ASSERT(rc == 0); 5025 CU_ASSERT(g_io_done == true); 5026 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 5027 stub_complete_io(1); 5028 CU_ASSERT(g_abort_done == true); 5029 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 5030 5031 /* Test the case that the target I/O was not aborted because it completed 5032 * in the middle of execution of the abort. 5033 */ 5034 g_io_done = false; 5035 5036 rc = spdk_bdev_read_blocks(desc, io_ch, NULL, 0, 1, io_done, &io_ctx1); 5037 CU_ASSERT(rc == 0); 5038 CU_ASSERT(g_io_done == false); 5039 5040 g_abort_done = false; 5041 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 5042 5043 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 5044 CU_ASSERT(rc == 0); 5045 CU_ASSERT(g_io_done == false); 5046 5047 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 5048 stub_complete_io(1); 5049 CU_ASSERT(g_io_done == true); 5050 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_SUCCESS); 5051 5052 g_io_exp_status = SPDK_BDEV_IO_STATUS_FAILED; 5053 stub_complete_io(1); 5054 CU_ASSERT(g_abort_done == true); 5055 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 5056 5057 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 5058 5059 bdev->optimal_io_boundary = 16; 5060 bdev->split_on_optimal_io_boundary = true; 5061 5062 /* Test that a single-vector command which is split is aborted correctly. 5063 * Offset 14, length 8, payload 0xF000 5064 * Child - Offset 14, length 2, payload 0xF000 5065 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 5066 */ 5067 g_io_done = false; 5068 5069 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 8, io_done, &io_ctx1); 5070 CU_ASSERT(rc == 0); 5071 CU_ASSERT(g_io_done == false); 5072 5073 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 5074 5075 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 5076 5077 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 5078 CU_ASSERT(rc == 0); 5079 CU_ASSERT(g_io_done == true); 5080 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 5081 stub_complete_io(2); 5082 CU_ASSERT(g_abort_done == true); 5083 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 5084 5085 /* Test that a multi-vector command that needs to be split by strip and then 5086 * needs to be split is aborted correctly. Abort is requested before the second 5087 * child I/O was submitted. The parent I/O should complete with failure without 5088 * submitting the second child I/O. 5089 */ 5090 for (i = 0; i < BDEV_IO_NUM_CHILD_IOV * 2; i++) { 5091 iov[i].iov_base = (void *)((i + 1) * 0x10000); 5092 iov[i].iov_len = 512; 5093 } 5094 5095 bdev->optimal_io_boundary = BDEV_IO_NUM_CHILD_IOV; 5096 g_io_done = false; 5097 rc = spdk_bdev_readv_blocks(desc, io_ch, iov, BDEV_IO_NUM_CHILD_IOV * 2, 0, 5098 BDEV_IO_NUM_CHILD_IOV * 2, io_done, &io_ctx1); 5099 CU_ASSERT(rc == 0); 5100 CU_ASSERT(g_io_done == false); 5101 5102 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 5103 5104 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 5105 5106 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 5107 CU_ASSERT(rc == 0); 5108 CU_ASSERT(g_io_done == true); 5109 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 5110 stub_complete_io(1); 5111 CU_ASSERT(g_abort_done == true); 5112 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 5113 5114 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 5115 5116 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 5117 5118 bdev->optimal_io_boundary = 16; 5119 g_io_done = false; 5120 5121 /* Test that a ingle-vector command which is split is aborted correctly. 5122 * Differently from the above, the child abort request will be submitted 5123 * sequentially due to the capacity of spdk_bdev_io. 5124 */ 5125 rc = spdk_bdev_read_blocks(desc, io_ch, (void *)0xF000, 14, 50, io_done, &io_ctx1); 5126 CU_ASSERT(rc == 0); 5127 CU_ASSERT(g_io_done == false); 5128 5129 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 5130 5131 g_abort_done = false; 5132 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 5133 5134 rc = spdk_bdev_abort(desc, io_ch, &io_ctx1, abort_done, NULL); 5135 CU_ASSERT(rc == 0); 5136 CU_ASSERT(!TAILQ_EMPTY(&mgmt_ch->io_wait_queue)); 5137 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 4); 5138 5139 stub_complete_io(1); 5140 CU_ASSERT(g_io_done == true); 5141 CU_ASSERT(g_io_status == SPDK_BDEV_IO_STATUS_FAILED); 5142 stub_complete_io(3); 5143 CU_ASSERT(g_abort_done == true); 5144 CU_ASSERT(g_abort_status == SPDK_BDEV_IO_STATUS_SUCCESS); 5145 5146 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 5147 5148 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 5149 5150 spdk_put_io_channel(io_ch); 5151 spdk_bdev_close(desc); 5152 free_bdev(bdev); 5153 spdk_bdev_finish(bdev_fini_cb, NULL); 5154 poll_threads(); 5155 } 5156 5157 static void 5158 bdev_unmap(void) 5159 { 5160 struct spdk_bdev *bdev; 5161 struct spdk_bdev_desc *desc = NULL; 5162 struct spdk_io_channel *ioch; 5163 struct spdk_bdev_channel *bdev_ch; 5164 struct ut_expected_io *expected_io; 5165 struct spdk_bdev_opts bdev_opts = {}; 5166 uint32_t i, num_outstanding; 5167 uint64_t offset, num_blocks, max_unmap_blocks, num_children; 5168 int rc; 5169 5170 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 5171 bdev_opts.bdev_io_pool_size = 512; 5172 bdev_opts.bdev_io_cache_size = 64; 5173 rc = spdk_bdev_set_opts(&bdev_opts); 5174 CU_ASSERT(rc == 0); 5175 5176 spdk_bdev_initialize(bdev_init_cb, NULL); 5177 bdev = allocate_bdev("bdev"); 5178 5179 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 5180 CU_ASSERT_EQUAL(rc, 0); 5181 SPDK_CU_ASSERT_FATAL(desc != NULL); 5182 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 5183 ioch = spdk_bdev_get_io_channel(desc); 5184 SPDK_CU_ASSERT_FATAL(ioch != NULL); 5185 bdev_ch = spdk_io_channel_get_ctx(ioch); 5186 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->io_submitted)); 5187 5188 fn_table.submit_request = stub_submit_request; 5189 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 5190 5191 /* Case 1: First test the request won't be split */ 5192 num_blocks = 32; 5193 5194 g_io_done = false; 5195 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, 0, num_blocks, 0); 5196 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5197 rc = spdk_bdev_unmap_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 5198 CU_ASSERT_EQUAL(rc, 0); 5199 CU_ASSERT(g_io_done == false); 5200 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 5201 stub_complete_io(1); 5202 CU_ASSERT(g_io_done == true); 5203 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 5204 5205 /* Case 2: Test the split with 2 children requests */ 5206 bdev->max_unmap = 8; 5207 bdev->max_unmap_segments = 2; 5208 max_unmap_blocks = bdev->max_unmap * bdev->max_unmap_segments; 5209 num_blocks = max_unmap_blocks * 2; 5210 offset = 0; 5211 5212 g_io_done = false; 5213 for (i = 0; i < 2; i++) { 5214 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, offset, max_unmap_blocks, 0); 5215 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5216 offset += max_unmap_blocks; 5217 } 5218 5219 rc = spdk_bdev_unmap_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 5220 CU_ASSERT_EQUAL(rc, 0); 5221 CU_ASSERT(g_io_done == false); 5222 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 5223 stub_complete_io(2); 5224 CU_ASSERT(g_io_done == true); 5225 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 5226 5227 /* Case 3: Test the split with 15 children requests, will finish 8 requests first */ 5228 num_children = 15; 5229 num_blocks = max_unmap_blocks * num_children; 5230 g_io_done = false; 5231 offset = 0; 5232 for (i = 0; i < num_children; i++) { 5233 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_UNMAP, offset, max_unmap_blocks, 0); 5234 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5235 offset += max_unmap_blocks; 5236 } 5237 5238 rc = spdk_bdev_unmap_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 5239 CU_ASSERT_EQUAL(rc, 0); 5240 CU_ASSERT(g_io_done == false); 5241 5242 while (num_children > 0) { 5243 num_outstanding = spdk_min(num_children, SPDK_BDEV_MAX_CHILDREN_UNMAP_WRITE_ZEROES_REQS); 5244 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == num_outstanding); 5245 stub_complete_io(num_outstanding); 5246 num_children -= num_outstanding; 5247 } 5248 CU_ASSERT(g_io_done == true); 5249 5250 spdk_put_io_channel(ioch); 5251 spdk_bdev_close(desc); 5252 free_bdev(bdev); 5253 spdk_bdev_finish(bdev_fini_cb, NULL); 5254 poll_threads(); 5255 } 5256 5257 static void 5258 bdev_write_zeroes_split_test(void) 5259 { 5260 struct spdk_bdev *bdev; 5261 struct spdk_bdev_desc *desc = NULL; 5262 struct spdk_io_channel *ioch; 5263 struct spdk_bdev_channel *bdev_ch; 5264 struct ut_expected_io *expected_io; 5265 struct spdk_bdev_opts bdev_opts = {}; 5266 uint32_t i, num_outstanding; 5267 uint64_t offset, num_blocks, max_write_zeroes_blocks, num_children; 5268 int rc; 5269 5270 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 5271 bdev_opts.bdev_io_pool_size = 512; 5272 bdev_opts.bdev_io_cache_size = 64; 5273 rc = spdk_bdev_set_opts(&bdev_opts); 5274 CU_ASSERT(rc == 0); 5275 5276 spdk_bdev_initialize(bdev_init_cb, NULL); 5277 bdev = allocate_bdev("bdev"); 5278 5279 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 5280 CU_ASSERT_EQUAL(rc, 0); 5281 SPDK_CU_ASSERT_FATAL(desc != NULL); 5282 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 5283 ioch = spdk_bdev_get_io_channel(desc); 5284 SPDK_CU_ASSERT_FATAL(ioch != NULL); 5285 bdev_ch = spdk_io_channel_get_ctx(ioch); 5286 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->io_submitted)); 5287 5288 fn_table.submit_request = stub_submit_request; 5289 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 5290 5291 /* Case 1: First test the request won't be split */ 5292 num_blocks = 32; 5293 5294 g_io_done = false; 5295 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, 0, num_blocks, 0); 5296 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5297 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 5298 CU_ASSERT_EQUAL(rc, 0); 5299 CU_ASSERT(g_io_done == false); 5300 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 5301 stub_complete_io(1); 5302 CU_ASSERT(g_io_done == true); 5303 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 5304 5305 /* Case 2: Test the split with 2 children requests */ 5306 max_write_zeroes_blocks = 8; 5307 bdev->max_write_zeroes = max_write_zeroes_blocks; 5308 num_blocks = max_write_zeroes_blocks * 2; 5309 offset = 0; 5310 5311 g_io_done = false; 5312 for (i = 0; i < 2; i++) { 5313 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, offset, max_write_zeroes_blocks, 5314 0); 5315 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5316 offset += max_write_zeroes_blocks; 5317 } 5318 5319 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 5320 CU_ASSERT_EQUAL(rc, 0); 5321 CU_ASSERT(g_io_done == false); 5322 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 5323 stub_complete_io(2); 5324 CU_ASSERT(g_io_done == true); 5325 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 5326 5327 /* Case 3: Test the split with 15 children requests, will finish 8 requests first */ 5328 num_children = 15; 5329 num_blocks = max_write_zeroes_blocks * num_children; 5330 g_io_done = false; 5331 offset = 0; 5332 for (i = 0; i < num_children; i++) { 5333 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE_ZEROES, offset, max_write_zeroes_blocks, 5334 0); 5335 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5336 offset += max_write_zeroes_blocks; 5337 } 5338 5339 rc = spdk_bdev_write_zeroes_blocks(desc, ioch, 0, num_blocks, io_done, NULL); 5340 CU_ASSERT_EQUAL(rc, 0); 5341 CU_ASSERT(g_io_done == false); 5342 5343 while (num_children > 0) { 5344 num_outstanding = spdk_min(num_children, SPDK_BDEV_MAX_CHILDREN_UNMAP_WRITE_ZEROES_REQS); 5345 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == num_outstanding); 5346 stub_complete_io(num_outstanding); 5347 num_children -= num_outstanding; 5348 } 5349 CU_ASSERT(g_io_done == true); 5350 5351 spdk_put_io_channel(ioch); 5352 spdk_bdev_close(desc); 5353 free_bdev(bdev); 5354 spdk_bdev_finish(bdev_fini_cb, NULL); 5355 poll_threads(); 5356 } 5357 5358 static void 5359 bdev_set_options_test(void) 5360 { 5361 struct spdk_bdev_opts bdev_opts = {}; 5362 int rc; 5363 5364 /* Case1: Do not set opts_size */ 5365 rc = spdk_bdev_set_opts(&bdev_opts); 5366 CU_ASSERT(rc == -1); 5367 5368 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 5369 bdev_opts.bdev_io_pool_size = 4; 5370 bdev_opts.bdev_io_cache_size = 2; 5371 bdev_opts.small_buf_pool_size = 4; 5372 5373 /* Case 2: Do not set valid small_buf_pool_size and large_buf_pool_size */ 5374 rc = spdk_bdev_set_opts(&bdev_opts); 5375 CU_ASSERT(rc == -1); 5376 5377 /* Case 3: Do not set valid large_buf_pool_size */ 5378 bdev_opts.small_buf_pool_size = BUF_SMALL_POOL_SIZE; 5379 bdev_opts.large_buf_pool_size = BUF_LARGE_POOL_SIZE - 1; 5380 rc = spdk_bdev_set_opts(&bdev_opts); 5381 CU_ASSERT(rc == -1); 5382 5383 /* Case4: set valid large buf_pool_size */ 5384 bdev_opts.large_buf_pool_size = BUF_LARGE_POOL_SIZE; 5385 rc = spdk_bdev_set_opts(&bdev_opts); 5386 CU_ASSERT(rc == 0); 5387 5388 /* Case5: Set different valid value for small and large buf pool */ 5389 bdev_opts.large_buf_pool_size = BUF_SMALL_POOL_SIZE + 3; 5390 bdev_opts.large_buf_pool_size = BUF_LARGE_POOL_SIZE + 3; 5391 rc = spdk_bdev_set_opts(&bdev_opts); 5392 CU_ASSERT(rc == 0); 5393 } 5394 5395 static uint64_t 5396 get_ns_time(void) 5397 { 5398 int rc; 5399 struct timespec ts; 5400 5401 rc = clock_gettime(CLOCK_MONOTONIC, &ts); 5402 CU_ASSERT(rc == 0); 5403 return ts.tv_sec * 1000 * 1000 * 1000 + ts.tv_nsec; 5404 } 5405 5406 static int 5407 rb_tree_get_height(struct spdk_bdev_name *bdev_name) 5408 { 5409 int h1, h2; 5410 5411 if (bdev_name == NULL) { 5412 return -1; 5413 } else { 5414 h1 = rb_tree_get_height(RB_LEFT(bdev_name, node)); 5415 h2 = rb_tree_get_height(RB_RIGHT(bdev_name, node)); 5416 5417 return spdk_max(h1, h2) + 1; 5418 } 5419 } 5420 5421 static void 5422 bdev_multi_allocation(void) 5423 { 5424 const int max_bdev_num = 1024 * 16; 5425 char name[max_bdev_num][16]; 5426 char noexist_name[] = "invalid_bdev"; 5427 struct spdk_bdev *bdev[max_bdev_num]; 5428 int i, j; 5429 uint64_t last_time; 5430 int bdev_num; 5431 int height; 5432 5433 for (j = 0; j < max_bdev_num; j++) { 5434 snprintf(name[j], sizeof(name[j]), "bdev%d", j); 5435 } 5436 5437 for (i = 0; i < 16; i++) { 5438 last_time = get_ns_time(); 5439 bdev_num = 1024 * (i + 1); 5440 for (j = 0; j < bdev_num; j++) { 5441 bdev[j] = allocate_bdev(name[j]); 5442 height = rb_tree_get_height(&bdev[j]->internal.bdev_name); 5443 CU_ASSERT(height <= (int)(spdk_u32log2(2 * j + 2))); 5444 } 5445 SPDK_NOTICELOG("alloc bdev num %d takes %" PRIu64 " ms\n", bdev_num, 5446 (get_ns_time() - last_time) / 1000 / 1000); 5447 for (j = 0; j < bdev_num; j++) { 5448 CU_ASSERT(spdk_bdev_get_by_name(name[j]) != NULL); 5449 } 5450 CU_ASSERT(spdk_bdev_get_by_name(noexist_name) == NULL); 5451 5452 for (j = 0; j < bdev_num; j++) { 5453 free_bdev(bdev[j]); 5454 } 5455 for (j = 0; j < bdev_num; j++) { 5456 CU_ASSERT(spdk_bdev_get_by_name(name[j]) == NULL); 5457 } 5458 } 5459 } 5460 5461 static struct spdk_memory_domain *g_bdev_memory_domain = (struct spdk_memory_domain *) 0xf00df00d; 5462 5463 static int 5464 test_bdev_get_supported_dma_device_types_op(void *ctx, struct spdk_memory_domain **domains, 5465 int array_size) 5466 { 5467 if (array_size > 0 && domains) { 5468 domains[0] = g_bdev_memory_domain; 5469 } 5470 5471 return 1; 5472 } 5473 5474 static void 5475 bdev_get_memory_domains(void) 5476 { 5477 struct spdk_bdev_fn_table fn_table = { 5478 .get_memory_domains = test_bdev_get_supported_dma_device_types_op 5479 }; 5480 struct spdk_bdev bdev = { .fn_table = &fn_table }; 5481 struct spdk_memory_domain *domains[2] = {}; 5482 int rc; 5483 5484 /* bdev is NULL */ 5485 rc = spdk_bdev_get_memory_domains(NULL, domains, 2); 5486 CU_ASSERT(rc == -EINVAL); 5487 5488 /* domains is NULL */ 5489 rc = spdk_bdev_get_memory_domains(&bdev, NULL, 2); 5490 CU_ASSERT(rc == 1); 5491 5492 /* array size is 0 */ 5493 rc = spdk_bdev_get_memory_domains(&bdev, domains, 0); 5494 CU_ASSERT(rc == 1); 5495 5496 /* get_supported_dma_device_types op is set */ 5497 rc = spdk_bdev_get_memory_domains(&bdev, domains, 2); 5498 CU_ASSERT(rc == 1); 5499 CU_ASSERT(domains[0] == g_bdev_memory_domain); 5500 5501 /* get_supported_dma_device_types op is not set */ 5502 fn_table.get_memory_domains = NULL; 5503 rc = spdk_bdev_get_memory_domains(&bdev, domains, 2); 5504 CU_ASSERT(rc == 0); 5505 } 5506 5507 static void 5508 _bdev_io_ext(struct spdk_bdev_ext_io_opts *ext_io_opts) 5509 { 5510 struct spdk_bdev *bdev; 5511 struct spdk_bdev_desc *desc = NULL; 5512 struct spdk_io_channel *io_ch; 5513 char io_buf[512]; 5514 struct iovec iov = { .iov_base = io_buf, .iov_len = 512 }; 5515 struct ut_expected_io *expected_io; 5516 int rc; 5517 5518 spdk_bdev_initialize(bdev_init_cb, NULL); 5519 5520 bdev = allocate_bdev("bdev0"); 5521 bdev->md_interleave = false; 5522 bdev->md_len = 8; 5523 5524 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 5525 CU_ASSERT(rc == 0); 5526 SPDK_CU_ASSERT_FATAL(desc != NULL); 5527 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 5528 io_ch = spdk_bdev_get_io_channel(desc); 5529 CU_ASSERT(io_ch != NULL); 5530 5531 /* read */ 5532 g_io_done = false; 5533 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 32, 14, 1); 5534 if (ext_io_opts) { 5535 expected_io->md_buf = ext_io_opts->metadata; 5536 expected_io->ext_io_opts = ext_io_opts; 5537 } 5538 ut_expected_io_set_iov(expected_io, 0, iov.iov_base, iov.iov_len); 5539 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5540 5541 rc = spdk_bdev_readv_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, ext_io_opts); 5542 5543 CU_ASSERT(rc == 0); 5544 CU_ASSERT(g_io_done == false); 5545 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 5546 stub_complete_io(1); 5547 CU_ASSERT(g_io_done == true); 5548 5549 /* write */ 5550 g_io_done = false; 5551 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 14, 1); 5552 if (ext_io_opts) { 5553 expected_io->md_buf = ext_io_opts->metadata; 5554 expected_io->ext_io_opts = ext_io_opts; 5555 } 5556 ut_expected_io_set_iov(expected_io, 0, iov.iov_base, iov.iov_len); 5557 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5558 5559 rc = spdk_bdev_writev_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, ext_io_opts); 5560 5561 CU_ASSERT(rc == 0); 5562 CU_ASSERT(g_io_done == false); 5563 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 5564 stub_complete_io(1); 5565 CU_ASSERT(g_io_done == true); 5566 5567 spdk_put_io_channel(io_ch); 5568 spdk_bdev_close(desc); 5569 free_bdev(bdev); 5570 spdk_bdev_finish(bdev_fini_cb, NULL); 5571 poll_threads(); 5572 5573 } 5574 5575 static void 5576 bdev_io_ext(void) 5577 { 5578 struct spdk_bdev_ext_io_opts ext_io_opts = { 5579 .metadata = (void *)0xFF000000, 5580 .size = sizeof(ext_io_opts) 5581 }; 5582 5583 _bdev_io_ext(&ext_io_opts); 5584 } 5585 5586 static void 5587 bdev_io_ext_no_opts(void) 5588 { 5589 _bdev_io_ext(NULL); 5590 } 5591 5592 static void 5593 bdev_io_ext_invalid_opts(void) 5594 { 5595 struct spdk_bdev *bdev; 5596 struct spdk_bdev_desc *desc = NULL; 5597 struct spdk_io_channel *io_ch; 5598 char io_buf[512]; 5599 struct iovec iov = { .iov_base = io_buf, .iov_len = 512 }; 5600 struct spdk_bdev_ext_io_opts ext_io_opts = { 5601 .metadata = (void *)0xFF000000, 5602 .size = sizeof(ext_io_opts) 5603 }; 5604 int rc; 5605 5606 spdk_bdev_initialize(bdev_init_cb, NULL); 5607 5608 bdev = allocate_bdev("bdev0"); 5609 bdev->md_interleave = false; 5610 bdev->md_len = 8; 5611 5612 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 5613 CU_ASSERT(rc == 0); 5614 SPDK_CU_ASSERT_FATAL(desc != NULL); 5615 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 5616 io_ch = spdk_bdev_get_io_channel(desc); 5617 CU_ASSERT(io_ch != NULL); 5618 5619 /* Test invalid ext_opts size */ 5620 ext_io_opts.size = 0; 5621 rc = spdk_bdev_readv_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); 5622 CU_ASSERT(rc == -EINVAL); 5623 rc = spdk_bdev_writev_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); 5624 CU_ASSERT(rc == -EINVAL); 5625 5626 ext_io_opts.size = sizeof(ext_io_opts) * 2; 5627 rc = spdk_bdev_readv_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); 5628 CU_ASSERT(rc == -EINVAL); 5629 rc = spdk_bdev_writev_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); 5630 CU_ASSERT(rc == -EINVAL); 5631 5632 ext_io_opts.size = offsetof(struct spdk_bdev_ext_io_opts, metadata) + 5633 sizeof(ext_io_opts.metadata) - 1; 5634 rc = spdk_bdev_readv_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); 5635 CU_ASSERT(rc == -EINVAL); 5636 rc = spdk_bdev_writev_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); 5637 CU_ASSERT(rc == -EINVAL); 5638 5639 spdk_put_io_channel(io_ch); 5640 spdk_bdev_close(desc); 5641 free_bdev(bdev); 5642 spdk_bdev_finish(bdev_fini_cb, NULL); 5643 poll_threads(); 5644 } 5645 5646 static void 5647 bdev_io_ext_split(void) 5648 { 5649 struct spdk_bdev *bdev; 5650 struct spdk_bdev_desc *desc = NULL; 5651 struct spdk_io_channel *io_ch; 5652 char io_buf[512]; 5653 struct iovec iov = { .iov_base = io_buf, .iov_len = 512 }; 5654 struct ut_expected_io *expected_io; 5655 struct spdk_bdev_ext_io_opts ext_io_opts = { 5656 .metadata = (void *)0xFF000000, 5657 .size = sizeof(ext_io_opts) 5658 }; 5659 int rc; 5660 5661 spdk_bdev_initialize(bdev_init_cb, NULL); 5662 5663 bdev = allocate_bdev("bdev0"); 5664 bdev->md_interleave = false; 5665 bdev->md_len = 8; 5666 5667 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 5668 CU_ASSERT(rc == 0); 5669 SPDK_CU_ASSERT_FATAL(desc != NULL); 5670 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 5671 io_ch = spdk_bdev_get_io_channel(desc); 5672 CU_ASSERT(io_ch != NULL); 5673 5674 /* Check that IO request with ext_opts and metadata is split correctly 5675 * Offset 14, length 8, payload 0xF000 5676 * Child - Offset 14, length 2, payload 0xF000 5677 * Child - Offset 16, length 6, payload 0xF000 + 2 * 512 5678 */ 5679 bdev->optimal_io_boundary = 16; 5680 bdev->split_on_optimal_io_boundary = true; 5681 bdev->md_interleave = false; 5682 bdev->md_len = 8; 5683 5684 iov.iov_base = (void *)0xF000; 5685 iov.iov_len = 4096; 5686 memset(&ext_io_opts, 0, sizeof(ext_io_opts)); 5687 ext_io_opts.metadata = (void *)0xFF000000; 5688 ext_io_opts.size = sizeof(ext_io_opts); 5689 g_io_done = false; 5690 5691 /* read */ 5692 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 14, 2, 1); 5693 expected_io->md_buf = ext_io_opts.metadata; 5694 expected_io->ext_io_opts = &ext_io_opts; 5695 expected_io->copy_opts = true; 5696 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 2 * 512); 5697 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5698 5699 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 16, 6, 1); 5700 expected_io->md_buf = ext_io_opts.metadata + 2 * 8; 5701 expected_io->ext_io_opts = &ext_io_opts; 5702 expected_io->copy_opts = true; 5703 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 6 * 512); 5704 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5705 5706 rc = spdk_bdev_readv_blocks_ext(desc, io_ch, &iov, 1, 14, 8, io_done, NULL, &ext_io_opts); 5707 CU_ASSERT(rc == 0); 5708 CU_ASSERT(g_io_done == false); 5709 5710 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 5711 stub_complete_io(2); 5712 CU_ASSERT(g_io_done == true); 5713 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 5714 5715 /* write */ 5716 g_io_done = false; 5717 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 14, 2, 1); 5718 expected_io->md_buf = ext_io_opts.metadata; 5719 expected_io->ext_io_opts = &ext_io_opts; 5720 expected_io->copy_opts = true; 5721 ut_expected_io_set_iov(expected_io, 0, (void *)0xF000, 2 * 512); 5722 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5723 5724 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 16, 6, 1); 5725 expected_io->md_buf = ext_io_opts.metadata + 2 * 8; 5726 expected_io->ext_io_opts = &ext_io_opts; 5727 expected_io->copy_opts = true; 5728 ut_expected_io_set_iov(expected_io, 0, (void *)(0xF000 + 2 * 512), 6 * 512); 5729 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5730 5731 rc = spdk_bdev_writev_blocks_ext(desc, io_ch, &iov, 1, 14, 8, io_done, NULL, &ext_io_opts); 5732 CU_ASSERT(rc == 0); 5733 CU_ASSERT(g_io_done == false); 5734 5735 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 2); 5736 stub_complete_io(2); 5737 CU_ASSERT(g_io_done == true); 5738 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 5739 5740 spdk_put_io_channel(io_ch); 5741 spdk_bdev_close(desc); 5742 free_bdev(bdev); 5743 spdk_bdev_finish(bdev_fini_cb, NULL); 5744 poll_threads(); 5745 } 5746 5747 static void 5748 bdev_io_ext_bounce_buffer(void) 5749 { 5750 struct spdk_bdev *bdev; 5751 struct spdk_bdev_desc *desc = NULL; 5752 struct spdk_io_channel *io_ch; 5753 char io_buf[512]; 5754 struct iovec iov = { .iov_base = io_buf, .iov_len = 512 }; 5755 struct ut_expected_io *expected_io; 5756 struct spdk_bdev_ext_io_opts ext_io_opts = { 5757 .metadata = (void *)0xFF000000, 5758 .size = sizeof(ext_io_opts) 5759 }; 5760 int rc; 5761 5762 spdk_bdev_initialize(bdev_init_cb, NULL); 5763 5764 bdev = allocate_bdev("bdev0"); 5765 bdev->md_interleave = false; 5766 bdev->md_len = 8; 5767 5768 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 5769 CU_ASSERT(rc == 0); 5770 SPDK_CU_ASSERT_FATAL(desc != NULL); 5771 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 5772 io_ch = spdk_bdev_get_io_channel(desc); 5773 CU_ASSERT(io_ch != NULL); 5774 5775 /* Verify data pull/push 5776 * bdev doesn't support memory domains, so buffers from bdev memory pool will be used */ 5777 ext_io_opts.memory_domain = (struct spdk_memory_domain *)0xdeadbeef; 5778 5779 /* read */ 5780 g_io_done = false; 5781 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_READ, 32, 14, 1); 5782 ut_expected_io_set_iov(expected_io, 0, iov.iov_base, iov.iov_len); 5783 expected_io->ext_io_opts = &ext_io_opts; 5784 expected_io->copy_opts = true; 5785 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5786 5787 rc = spdk_bdev_readv_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); 5788 5789 CU_ASSERT(rc == 0); 5790 CU_ASSERT(g_io_done == false); 5791 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 5792 stub_complete_io(1); 5793 CU_ASSERT(g_memory_domain_push_data_called == true); 5794 CU_ASSERT(g_io_done == true); 5795 5796 /* write */ 5797 g_io_done = false; 5798 expected_io = ut_alloc_expected_io(SPDK_BDEV_IO_TYPE_WRITE, 32, 14, 1); 5799 ut_expected_io_set_iov(expected_io, 0, iov.iov_base, iov.iov_len); 5800 expected_io->ext_io_opts = &ext_io_opts; 5801 expected_io->copy_opts = true; 5802 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 5803 5804 rc = spdk_bdev_writev_blocks_ext(desc, io_ch, &iov, 1, 32, 14, io_done, NULL, &ext_io_opts); 5805 5806 CU_ASSERT(rc == 0); 5807 CU_ASSERT(g_memory_domain_pull_data_called == true); 5808 CU_ASSERT(g_io_done == false); 5809 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 5810 stub_complete_io(1); 5811 CU_ASSERT(g_io_done == true); 5812 5813 spdk_put_io_channel(io_ch); 5814 spdk_bdev_close(desc); 5815 free_bdev(bdev); 5816 spdk_bdev_finish(bdev_fini_cb, NULL); 5817 poll_threads(); 5818 } 5819 5820 static void 5821 bdev_register_uuid_alias(void) 5822 { 5823 struct spdk_bdev *bdev, *second; 5824 char uuid[SPDK_UUID_STRING_LEN]; 5825 int rc; 5826 5827 spdk_bdev_initialize(bdev_init_cb, NULL); 5828 bdev = allocate_bdev("bdev0"); 5829 5830 /* Make sure an UUID was generated */ 5831 CU_ASSERT_FALSE(spdk_mem_all_zero(&bdev->uuid, sizeof(bdev->uuid))); 5832 5833 /* Check that an UUID alias was registered */ 5834 spdk_uuid_fmt_lower(uuid, sizeof(uuid), &bdev->uuid); 5835 CU_ASSERT_EQUAL(spdk_bdev_get_by_name(uuid), bdev); 5836 5837 /* Unregister the bdev */ 5838 spdk_bdev_unregister(bdev, NULL, NULL); 5839 poll_threads(); 5840 CU_ASSERT_PTR_NULL(spdk_bdev_get_by_name(uuid)); 5841 5842 /* Check the same, but this time register the bdev with non-zero UUID */ 5843 rc = spdk_bdev_register(bdev); 5844 CU_ASSERT_EQUAL(rc, 0); 5845 CU_ASSERT_EQUAL(spdk_bdev_get_by_name(uuid), bdev); 5846 5847 /* Unregister the bdev */ 5848 spdk_bdev_unregister(bdev, NULL, NULL); 5849 poll_threads(); 5850 CU_ASSERT_PTR_NULL(spdk_bdev_get_by_name(uuid)); 5851 5852 /* Regiser the bdev using UUID as the name */ 5853 bdev->name = uuid; 5854 rc = spdk_bdev_register(bdev); 5855 CU_ASSERT_EQUAL(rc, 0); 5856 CU_ASSERT_EQUAL(spdk_bdev_get_by_name(uuid), bdev); 5857 5858 /* Unregister the bdev */ 5859 spdk_bdev_unregister(bdev, NULL, NULL); 5860 poll_threads(); 5861 CU_ASSERT_PTR_NULL(spdk_bdev_get_by_name(uuid)); 5862 5863 /* Check that it's not possible to register two bdevs with the same UUIDs */ 5864 bdev->name = "bdev0"; 5865 second = allocate_bdev("bdev1"); 5866 spdk_uuid_copy(&bdev->uuid, &second->uuid); 5867 rc = spdk_bdev_register(bdev); 5868 CU_ASSERT_EQUAL(rc, -EEXIST); 5869 5870 /* Regenerate the UUID and re-check */ 5871 spdk_uuid_generate(&bdev->uuid); 5872 rc = spdk_bdev_register(bdev); 5873 CU_ASSERT_EQUAL(rc, 0); 5874 5875 /* And check that both bdevs can be retrieved through their UUIDs */ 5876 spdk_uuid_fmt_lower(uuid, sizeof(uuid), &bdev->uuid); 5877 CU_ASSERT_EQUAL(spdk_bdev_get_by_name(uuid), bdev); 5878 spdk_uuid_fmt_lower(uuid, sizeof(uuid), &second->uuid); 5879 CU_ASSERT_EQUAL(spdk_bdev_get_by_name(uuid), second); 5880 5881 free_bdev(second); 5882 free_bdev(bdev); 5883 spdk_bdev_finish(bdev_fini_cb, NULL); 5884 poll_threads(); 5885 } 5886 5887 static void 5888 bdev_unregister_by_name(void) 5889 { 5890 struct spdk_bdev *bdev; 5891 int rc; 5892 5893 bdev = allocate_bdev("bdev"); 5894 5895 g_event_type1 = 0xFF; 5896 g_unregister_arg = NULL; 5897 g_unregister_rc = -1; 5898 5899 rc = spdk_bdev_unregister_by_name("bdev1", &bdev_ut_if, bdev_unregister_cb, (void *)0x12345678); 5900 CU_ASSERT(rc == -ENODEV); 5901 5902 rc = spdk_bdev_unregister_by_name("bdev", &vbdev_ut_if, bdev_unregister_cb, (void *)0x12345678); 5903 CU_ASSERT(rc == -ENODEV); 5904 5905 rc = spdk_bdev_unregister_by_name("bdev", &bdev_ut_if, bdev_unregister_cb, (void *)0x12345678); 5906 CU_ASSERT(rc == 0); 5907 5908 /* Check that unregister callback is delayed */ 5909 CU_ASSERT(g_unregister_arg == NULL); 5910 CU_ASSERT(g_unregister_rc == -1); 5911 5912 poll_threads(); 5913 5914 /* Event callback shall not be issued because device was closed */ 5915 CU_ASSERT(g_event_type1 == 0xFF); 5916 /* Unregister callback is issued */ 5917 CU_ASSERT(g_unregister_arg == (void *)0x12345678); 5918 CU_ASSERT(g_unregister_rc == 0); 5919 5920 free_bdev(bdev); 5921 } 5922 5923 static int 5924 count_bdevs(void *ctx, struct spdk_bdev *bdev) 5925 { 5926 int *count = ctx; 5927 5928 (*count)++; 5929 5930 return 0; 5931 } 5932 5933 static void 5934 for_each_bdev_test(void) 5935 { 5936 struct spdk_bdev *bdev[8]; 5937 int rc, count; 5938 5939 bdev[0] = allocate_bdev("bdev0"); 5940 5941 bdev[1] = allocate_bdev("bdev1"); 5942 rc = spdk_bdev_module_claim_bdev(bdev[1], NULL, &bdev_ut_if); 5943 CU_ASSERT(rc == 0); 5944 5945 bdev[2] = allocate_bdev("bdev2"); 5946 5947 bdev[3] = allocate_bdev("bdev3"); 5948 rc = spdk_bdev_module_claim_bdev(bdev[3], NULL, &bdev_ut_if); 5949 CU_ASSERT(rc == 0); 5950 5951 bdev[4] = allocate_bdev("bdev4"); 5952 5953 bdev[5] = allocate_bdev("bdev5"); 5954 rc = spdk_bdev_module_claim_bdev(bdev[5], NULL, &bdev_ut_if); 5955 CU_ASSERT(rc == 0); 5956 5957 bdev[6] = allocate_bdev("bdev6"); 5958 5959 bdev[7] = allocate_bdev("bdev7"); 5960 5961 count = 0; 5962 rc = spdk_for_each_bdev(&count, count_bdevs); 5963 CU_ASSERT(rc == 0); 5964 CU_ASSERT(count == 8); 5965 5966 count = 0; 5967 rc = spdk_for_each_bdev_leaf(&count, count_bdevs); 5968 CU_ASSERT(rc == 0); 5969 CU_ASSERT(count == 5); 5970 5971 free_bdev(bdev[0]); 5972 free_bdev(bdev[1]); 5973 free_bdev(bdev[2]); 5974 free_bdev(bdev[3]); 5975 free_bdev(bdev[4]); 5976 free_bdev(bdev[5]); 5977 free_bdev(bdev[6]); 5978 free_bdev(bdev[7]); 5979 } 5980 5981 static void 5982 bdev_seek_test(void) 5983 { 5984 struct spdk_bdev *bdev; 5985 struct spdk_bdev_desc *desc = NULL; 5986 struct spdk_io_channel *io_ch; 5987 int rc; 5988 5989 spdk_bdev_initialize(bdev_init_cb, NULL); 5990 poll_threads(); 5991 5992 bdev = allocate_bdev("bdev0"); 5993 5994 rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); 5995 CU_ASSERT(rc == 0); 5996 poll_threads(); 5997 SPDK_CU_ASSERT_FATAL(desc != NULL); 5998 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 5999 io_ch = spdk_bdev_get_io_channel(desc); 6000 CU_ASSERT(io_ch != NULL); 6001 6002 /* Seek data not supported */ 6003 ut_enable_io_type(SPDK_BDEV_IO_TYPE_SEEK_DATA, false); 6004 rc = spdk_bdev_seek_data(desc, io_ch, 0, bdev_seek_cb, NULL); 6005 CU_ASSERT(rc == 0); 6006 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 6007 poll_threads(); 6008 CU_ASSERT(g_seek_offset == 0); 6009 6010 /* Seek hole not supported */ 6011 ut_enable_io_type(SPDK_BDEV_IO_TYPE_SEEK_HOLE, false); 6012 rc = spdk_bdev_seek_hole(desc, io_ch, 0, bdev_seek_cb, NULL); 6013 CU_ASSERT(rc == 0); 6014 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 6015 poll_threads(); 6016 CU_ASSERT(g_seek_offset == UINT64_MAX); 6017 6018 /* Seek data supported */ 6019 g_seek_data_offset = 12345; 6020 ut_enable_io_type(SPDK_BDEV_IO_TYPE_SEEK_DATA, true); 6021 rc = spdk_bdev_seek_data(desc, io_ch, 0, bdev_seek_cb, NULL); 6022 CU_ASSERT(rc == 0); 6023 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 6024 stub_complete_io(1); 6025 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 6026 CU_ASSERT(g_seek_offset == 12345); 6027 6028 /* Seek hole supported */ 6029 g_seek_hole_offset = 67890; 6030 ut_enable_io_type(SPDK_BDEV_IO_TYPE_SEEK_HOLE, true); 6031 rc = spdk_bdev_seek_hole(desc, io_ch, 0, bdev_seek_cb, NULL); 6032 CU_ASSERT(rc == 0); 6033 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 6034 stub_complete_io(1); 6035 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 6036 CU_ASSERT(g_seek_offset == 67890); 6037 6038 spdk_put_io_channel(io_ch); 6039 spdk_bdev_close(desc); 6040 free_bdev(bdev); 6041 spdk_bdev_finish(bdev_fini_cb, NULL); 6042 poll_threads(); 6043 } 6044 6045 static void 6046 bdev_copy(void) 6047 { 6048 struct spdk_bdev *bdev; 6049 struct spdk_bdev_desc *desc = NULL; 6050 struct spdk_io_channel *ioch; 6051 struct ut_expected_io *expected_io; 6052 uint64_t src_offset, num_blocks; 6053 uint32_t num_completed; 6054 int rc; 6055 6056 spdk_bdev_initialize(bdev_init_cb, NULL); 6057 bdev = allocate_bdev("bdev"); 6058 6059 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 6060 CU_ASSERT_EQUAL(rc, 0); 6061 SPDK_CU_ASSERT_FATAL(desc != NULL); 6062 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 6063 ioch = spdk_bdev_get_io_channel(desc); 6064 SPDK_CU_ASSERT_FATAL(ioch != NULL); 6065 6066 fn_table.submit_request = stub_submit_request; 6067 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 6068 6069 /* First test that if the bdev supports copy, the request won't be split */ 6070 bdev->md_len = 0; 6071 bdev->blocklen = 4096; 6072 num_blocks = 512; 6073 src_offset = bdev->blockcnt - num_blocks; 6074 6075 expected_io = ut_alloc_expected_copy_io(SPDK_BDEV_IO_TYPE_COPY, 0, src_offset, num_blocks); 6076 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 6077 rc = spdk_bdev_copy_blocks(desc, ioch, 0, src_offset, num_blocks, io_done, NULL); 6078 CU_ASSERT_EQUAL(rc, 0); 6079 num_completed = stub_complete_io(1); 6080 CU_ASSERT_EQUAL(num_completed, 1); 6081 6082 /* Check that if copy is not supported it'll fail */ 6083 ut_enable_io_type(SPDK_BDEV_IO_TYPE_COPY, false); 6084 6085 rc = spdk_bdev_copy_blocks(desc, ioch, 0, src_offset, num_blocks, io_done, NULL); 6086 CU_ASSERT_EQUAL(rc, -ENOTSUP); 6087 6088 ut_enable_io_type(SPDK_BDEV_IO_TYPE_COPY, true); 6089 spdk_put_io_channel(ioch); 6090 spdk_bdev_close(desc); 6091 free_bdev(bdev); 6092 spdk_bdev_finish(bdev_fini_cb, NULL); 6093 poll_threads(); 6094 } 6095 6096 static void 6097 bdev_copy_split_test(void) 6098 { 6099 struct spdk_bdev *bdev; 6100 struct spdk_bdev_desc *desc = NULL; 6101 struct spdk_io_channel *ioch; 6102 struct spdk_bdev_channel *bdev_ch; 6103 struct ut_expected_io *expected_io; 6104 struct spdk_bdev_opts bdev_opts = {}; 6105 uint32_t i, num_outstanding; 6106 uint64_t offset, src_offset, num_blocks, max_copy_blocks, num_children; 6107 int rc; 6108 6109 spdk_bdev_get_opts(&bdev_opts, sizeof(bdev_opts)); 6110 bdev_opts.bdev_io_pool_size = 512; 6111 bdev_opts.bdev_io_cache_size = 64; 6112 rc = spdk_bdev_set_opts(&bdev_opts); 6113 CU_ASSERT(rc == 0); 6114 6115 spdk_bdev_initialize(bdev_init_cb, NULL); 6116 bdev = allocate_bdev("bdev"); 6117 6118 rc = spdk_bdev_open_ext("bdev", true, bdev_ut_event_cb, NULL, &desc); 6119 CU_ASSERT_EQUAL(rc, 0); 6120 SPDK_CU_ASSERT_FATAL(desc != NULL); 6121 CU_ASSERT(bdev == spdk_bdev_desc_get_bdev(desc)); 6122 ioch = spdk_bdev_get_io_channel(desc); 6123 SPDK_CU_ASSERT_FATAL(ioch != NULL); 6124 bdev_ch = spdk_io_channel_get_ctx(ioch); 6125 CU_ASSERT(TAILQ_EMPTY(&bdev_ch->io_submitted)); 6126 6127 fn_table.submit_request = stub_submit_request; 6128 g_io_exp_status = SPDK_BDEV_IO_STATUS_SUCCESS; 6129 6130 /* Case 1: First test the request won't be split */ 6131 num_blocks = 32; 6132 src_offset = bdev->blockcnt - num_blocks; 6133 6134 g_io_done = false; 6135 expected_io = ut_alloc_expected_copy_io(SPDK_BDEV_IO_TYPE_COPY, 0, src_offset, num_blocks); 6136 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 6137 rc = spdk_bdev_copy_blocks(desc, ioch, 0, src_offset, num_blocks, io_done, NULL); 6138 CU_ASSERT_EQUAL(rc, 0); 6139 CU_ASSERT(g_io_done == false); 6140 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 1); 6141 stub_complete_io(1); 6142 CU_ASSERT(g_io_done == true); 6143 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 6144 6145 /* Case 2: Test the split with 2 children requests */ 6146 max_copy_blocks = 8; 6147 bdev->max_copy = max_copy_blocks; 6148 num_children = 2; 6149 num_blocks = max_copy_blocks * num_children; 6150 offset = 0; 6151 src_offset = bdev->blockcnt - num_blocks; 6152 6153 g_io_done = false; 6154 for (i = 0; i < num_children; i++) { 6155 expected_io = ut_alloc_expected_copy_io(SPDK_BDEV_IO_TYPE_COPY, offset, 6156 src_offset + offset, max_copy_blocks); 6157 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 6158 offset += max_copy_blocks; 6159 } 6160 6161 rc = spdk_bdev_copy_blocks(desc, ioch, 0, src_offset, num_blocks, io_done, NULL); 6162 CU_ASSERT_EQUAL(rc, 0); 6163 CU_ASSERT(g_io_done == false); 6164 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == num_children); 6165 stub_complete_io(num_children); 6166 CU_ASSERT(g_io_done == true); 6167 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == 0); 6168 6169 /* Case 3: Test the split with 15 children requests, will finish 8 requests first */ 6170 num_children = 15; 6171 num_blocks = max_copy_blocks * num_children; 6172 offset = 0; 6173 src_offset = bdev->blockcnt - num_blocks; 6174 6175 g_io_done = false; 6176 for (i = 0; i < num_children; i++) { 6177 expected_io = ut_alloc_expected_copy_io(SPDK_BDEV_IO_TYPE_COPY, offset, 6178 src_offset + offset, max_copy_blocks); 6179 TAILQ_INSERT_TAIL(&g_bdev_ut_channel->expected_io, expected_io, link); 6180 offset += max_copy_blocks; 6181 } 6182 6183 rc = spdk_bdev_copy_blocks(desc, ioch, 0, src_offset, num_blocks, io_done, NULL); 6184 CU_ASSERT_EQUAL(rc, 0); 6185 CU_ASSERT(g_io_done == false); 6186 6187 while (num_children > 0) { 6188 num_outstanding = spdk_min(num_children, SPDK_BDEV_MAX_CHILDREN_COPY_REQS); 6189 CU_ASSERT(g_bdev_ut_channel->outstanding_io_count == num_outstanding); 6190 stub_complete_io(num_outstanding); 6191 num_children -= num_outstanding; 6192 } 6193 CU_ASSERT(g_io_done == true); 6194 6195 spdk_put_io_channel(ioch); 6196 spdk_bdev_close(desc); 6197 free_bdev(bdev); 6198 spdk_bdev_finish(bdev_fini_cb, NULL); 6199 poll_threads(); 6200 } 6201 6202 int 6203 main(int argc, char **argv) 6204 { 6205 CU_pSuite suite = NULL; 6206 unsigned int num_failures; 6207 6208 CU_set_error_action(CUEA_ABORT); 6209 CU_initialize_registry(); 6210 6211 suite = CU_add_suite("bdev", null_init, null_clean); 6212 6213 CU_ADD_TEST(suite, bytes_to_blocks_test); 6214 CU_ADD_TEST(suite, num_blocks_test); 6215 CU_ADD_TEST(suite, io_valid_test); 6216 CU_ADD_TEST(suite, open_write_test); 6217 CU_ADD_TEST(suite, claim_test); 6218 CU_ADD_TEST(suite, alias_add_del_test); 6219 CU_ADD_TEST(suite, get_device_stat_test); 6220 CU_ADD_TEST(suite, bdev_io_types_test); 6221 CU_ADD_TEST(suite, bdev_io_wait_test); 6222 CU_ADD_TEST(suite, bdev_io_spans_split_test); 6223 CU_ADD_TEST(suite, bdev_io_boundary_split_test); 6224 CU_ADD_TEST(suite, bdev_io_max_size_and_segment_split_test); 6225 CU_ADD_TEST(suite, bdev_io_mix_split_test); 6226 CU_ADD_TEST(suite, bdev_io_split_with_io_wait); 6227 CU_ADD_TEST(suite, bdev_io_write_unit_split_test); 6228 CU_ADD_TEST(suite, bdev_io_alignment_with_boundary); 6229 CU_ADD_TEST(suite, bdev_io_alignment); 6230 CU_ADD_TEST(suite, bdev_histograms); 6231 CU_ADD_TEST(suite, bdev_write_zeroes); 6232 CU_ADD_TEST(suite, bdev_compare_and_write); 6233 CU_ADD_TEST(suite, bdev_compare); 6234 CU_ADD_TEST(suite, bdev_compare_emulated); 6235 CU_ADD_TEST(suite, bdev_zcopy_write); 6236 CU_ADD_TEST(suite, bdev_zcopy_read); 6237 CU_ADD_TEST(suite, bdev_open_while_hotremove); 6238 CU_ADD_TEST(suite, bdev_close_while_hotremove); 6239 CU_ADD_TEST(suite, bdev_open_ext); 6240 CU_ADD_TEST(suite, bdev_open_ext_unregister); 6241 CU_ADD_TEST(suite, bdev_set_io_timeout); 6242 CU_ADD_TEST(suite, bdev_set_qd_sampling); 6243 CU_ADD_TEST(suite, lba_range_overlap); 6244 CU_ADD_TEST(suite, lock_lba_range_check_ranges); 6245 CU_ADD_TEST(suite, lock_lba_range_with_io_outstanding); 6246 CU_ADD_TEST(suite, lock_lba_range_overlapped); 6247 CU_ADD_TEST(suite, bdev_io_abort); 6248 CU_ADD_TEST(suite, bdev_unmap); 6249 CU_ADD_TEST(suite, bdev_write_zeroes_split_test); 6250 CU_ADD_TEST(suite, bdev_set_options_test); 6251 CU_ADD_TEST(suite, bdev_multi_allocation); 6252 CU_ADD_TEST(suite, bdev_get_memory_domains); 6253 CU_ADD_TEST(suite, bdev_io_ext); 6254 CU_ADD_TEST(suite, bdev_io_ext_no_opts); 6255 CU_ADD_TEST(suite, bdev_io_ext_invalid_opts); 6256 CU_ADD_TEST(suite, bdev_io_ext_split); 6257 CU_ADD_TEST(suite, bdev_io_ext_bounce_buffer); 6258 CU_ADD_TEST(suite, bdev_register_uuid_alias); 6259 CU_ADD_TEST(suite, bdev_unregister_by_name); 6260 CU_ADD_TEST(suite, for_each_bdev_test); 6261 CU_ADD_TEST(suite, bdev_seek_test); 6262 CU_ADD_TEST(suite, bdev_copy); 6263 CU_ADD_TEST(suite, bdev_copy_split_test); 6264 6265 allocate_cores(1); 6266 allocate_threads(1); 6267 set_thread(0); 6268 6269 CU_basic_set_mode(CU_BRM_VERBOSE); 6270 CU_basic_run_tests(); 6271 num_failures = CU_get_number_of_failures(); 6272 CU_cleanup_registry(); 6273 6274 free_threads(); 6275 free_cores(); 6276 6277 return num_failures; 6278 } 6279