1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk_cunit.h" 8 #include "spdk/env.h" 9 10 #include "common/lib/ut_multithread.c" 11 12 #include "bdev/raid/raid5f.c" 13 #include "../common.c" 14 15 DEFINE_STUB_V(raid_bdev_module_list_add, (struct raid_bdev_module *raid_module)); 16 DEFINE_STUB(spdk_bdev_get_buf_align, size_t, (const struct spdk_bdev *bdev), 0); 17 DEFINE_STUB_V(raid_bdev_module_stop_done, (struct raid_bdev *raid_bdev)); 18 19 void * 20 spdk_bdev_io_get_md_buf(struct spdk_bdev_io *bdev_io) 21 { 22 return bdev_io->u.bdev.md_buf; 23 } 24 25 uint32_t 26 spdk_bdev_get_md_size(const struct spdk_bdev *bdev) 27 { 28 return bdev->md_len; 29 } 30 31 void 32 raid_bdev_io_complete(struct raid_bdev_io *raid_io, enum spdk_bdev_io_status status) 33 { 34 struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(raid_io); 35 36 if (bdev_io->internal.cb) { 37 bdev_io->internal.cb(bdev_io, status == SPDK_BDEV_IO_STATUS_SUCCESS, bdev_io->internal.caller_ctx); 38 } 39 } 40 41 bool 42 raid_bdev_io_complete_part(struct raid_bdev_io *raid_io, uint64_t completed, 43 enum spdk_bdev_io_status status) 44 { 45 assert(raid_io->base_bdev_io_remaining >= completed); 46 raid_io->base_bdev_io_remaining -= completed; 47 48 if (status != SPDK_BDEV_IO_STATUS_SUCCESS) { 49 raid_io->base_bdev_io_status = status; 50 } 51 52 if (raid_io->base_bdev_io_remaining == 0) { 53 raid_bdev_io_complete(raid_io, raid_io->base_bdev_io_status); 54 return true; 55 } else { 56 return false; 57 } 58 } 59 60 static int 61 test_setup(void) 62 { 63 uint8_t num_base_bdevs_values[] = { 3, 4, 5 }; 64 uint64_t base_bdev_blockcnt_values[] = { 1, 1024, 1024 * 1024 }; 65 uint32_t base_bdev_blocklen_values[] = { 512, 4096 }; 66 uint32_t strip_size_kb_values[] = { 1, 4, 128 }; 67 uint32_t md_len_values[] = { 0, 64 }; 68 uint8_t *num_base_bdevs; 69 uint64_t *base_bdev_blockcnt; 70 uint32_t *base_bdev_blocklen; 71 uint32_t *strip_size_kb; 72 uint32_t *md_len; 73 struct raid_params params; 74 uint64_t params_count; 75 int rc; 76 77 params_count = SPDK_COUNTOF(num_base_bdevs_values) * 78 SPDK_COUNTOF(base_bdev_blockcnt_values) * 79 SPDK_COUNTOF(base_bdev_blocklen_values) * 80 SPDK_COUNTOF(strip_size_kb_values) * 81 SPDK_COUNTOF(md_len_values); 82 rc = raid_test_params_alloc(params_count); 83 if (rc) { 84 return rc; 85 } 86 87 ARRAY_FOR_EACH(num_base_bdevs_values, num_base_bdevs) { 88 ARRAY_FOR_EACH(base_bdev_blockcnt_values, base_bdev_blockcnt) { 89 ARRAY_FOR_EACH(base_bdev_blocklen_values, base_bdev_blocklen) { 90 ARRAY_FOR_EACH(strip_size_kb_values, strip_size_kb) { 91 ARRAY_FOR_EACH(md_len_values, md_len) { 92 params.num_base_bdevs = *num_base_bdevs; 93 params.base_bdev_blockcnt = *base_bdev_blockcnt; 94 params.base_bdev_blocklen = *base_bdev_blocklen; 95 params.strip_size = *strip_size_kb * 1024 / *base_bdev_blocklen; 96 params.md_len = *md_len; 97 if (params.strip_size == 0 || 98 params.strip_size > *base_bdev_blockcnt) { 99 continue; 100 } 101 raid_test_params_add(¶ms); 102 } 103 } 104 } 105 } 106 } 107 108 return 0; 109 } 110 111 static int 112 test_cleanup(void) 113 { 114 raid_test_params_free(); 115 return 0; 116 } 117 118 static struct raid5f_info * 119 create_raid5f(struct raid_params *params) 120 { 121 struct raid_bdev *raid_bdev = raid_test_create_raid_bdev(params, &g_raid5f_module); 122 123 SPDK_CU_ASSERT_FATAL(raid5f_start(raid_bdev) == 0); 124 125 return raid_bdev->module_private; 126 } 127 128 static void 129 delete_raid5f(struct raid5f_info *r5f_info) 130 { 131 struct raid_bdev *raid_bdev = r5f_info->raid_bdev; 132 133 raid5f_stop(raid_bdev); 134 135 raid_test_delete_raid_bdev(raid_bdev); 136 } 137 138 static void 139 test_raid5f_start(void) 140 { 141 struct raid_params *params; 142 143 RAID_PARAMS_FOR_EACH(params) { 144 struct raid5f_info *r5f_info; 145 146 r5f_info = create_raid5f(params); 147 148 SPDK_CU_ASSERT_FATAL(r5f_info != NULL); 149 150 CU_ASSERT_EQUAL(r5f_info->stripe_blocks, params->strip_size * (params->num_base_bdevs - 1)); 151 CU_ASSERT_EQUAL(r5f_info->total_stripes, params->base_bdev_blockcnt / params->strip_size); 152 CU_ASSERT_EQUAL(r5f_info->raid_bdev->bdev.blockcnt, 153 (params->base_bdev_blockcnt - params->base_bdev_blockcnt % params->strip_size) * 154 (params->num_base_bdevs - 1)); 155 CU_ASSERT_EQUAL(r5f_info->raid_bdev->bdev.optimal_io_boundary, params->strip_size); 156 CU_ASSERT_TRUE(r5f_info->raid_bdev->bdev.split_on_optimal_io_boundary); 157 CU_ASSERT_EQUAL(r5f_info->raid_bdev->bdev.write_unit_size, r5f_info->stripe_blocks); 158 159 delete_raid5f(r5f_info); 160 } 161 } 162 163 enum test_bdev_error_type { 164 TEST_BDEV_ERROR_NONE, 165 TEST_BDEV_ERROR_SUBMIT, 166 TEST_BDEV_ERROR_COMPLETE, 167 TEST_BDEV_ERROR_NOMEM, 168 }; 169 170 struct raid_io_info { 171 struct raid5f_info *r5f_info; 172 struct raid_bdev_io_channel *raid_ch; 173 enum spdk_bdev_io_type io_type; 174 uint64_t offset_blocks; 175 uint64_t num_blocks; 176 void *src_buf; 177 void *dest_buf; 178 void *src_md_buf; 179 void *dest_md_buf; 180 size_t buf_size; 181 void *parity_buf; 182 void *reference_parity; 183 size_t parity_buf_size; 184 void *parity_md_buf; 185 void *reference_md_parity; 186 size_t parity_md_buf_size; 187 enum spdk_bdev_io_status status; 188 bool failed; 189 int remaining; 190 TAILQ_HEAD(, spdk_bdev_io) bdev_io_queue; 191 TAILQ_HEAD(, spdk_bdev_io_wait_entry) bdev_io_wait_queue; 192 struct { 193 enum test_bdev_error_type type; 194 struct spdk_bdev *bdev; 195 void (*on_enomem_cb)(struct raid_io_info *io_info, void *ctx); 196 void *on_enomem_cb_ctx; 197 } error; 198 }; 199 200 struct test_raid_bdev_io { 201 char bdev_io_buf[sizeof(struct spdk_bdev_io) + sizeof(struct raid_bdev_io)]; 202 struct raid_io_info *io_info; 203 void *buf; 204 void *buf_md; 205 }; 206 207 void 208 raid_bdev_queue_io_wait(struct raid_bdev_io *raid_io, struct spdk_bdev *bdev, 209 struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn) 210 { 211 struct raid_io_info *io_info; 212 213 io_info = ((struct test_raid_bdev_io *)spdk_bdev_io_from_ctx(raid_io))->io_info; 214 215 raid_io->waitq_entry.bdev = bdev; 216 raid_io->waitq_entry.cb_fn = cb_fn; 217 raid_io->waitq_entry.cb_arg = raid_io; 218 TAILQ_INSERT_TAIL(&io_info->bdev_io_wait_queue, &raid_io->waitq_entry, link); 219 } 220 221 static void 222 raid_bdev_io_completion_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 223 { 224 struct raid_io_info *io_info = cb_arg; 225 226 spdk_bdev_free_io(bdev_io); 227 228 if (!success) { 229 io_info->failed = true; 230 } 231 232 if (--io_info->remaining == 0) { 233 if (io_info->failed) { 234 io_info->status = SPDK_BDEV_IO_STATUS_FAILED; 235 } else { 236 io_info->status = SPDK_BDEV_IO_STATUS_SUCCESS; 237 } 238 } 239 } 240 241 static struct raid_bdev_io * 242 get_raid_io(struct raid_io_info *io_info, uint64_t offset_blocks_split, uint64_t num_blocks) 243 { 244 struct spdk_bdev_io *bdev_io; 245 struct raid_bdev_io *raid_io; 246 struct raid_bdev *raid_bdev = io_info->r5f_info->raid_bdev; 247 uint32_t blocklen = raid_bdev->bdev.blocklen; 248 struct test_raid_bdev_io *test_raid_bdev_io; 249 void *src_buf = io_info->src_buf + offset_blocks_split * blocklen; 250 void *dest_buf = io_info->dest_buf + offset_blocks_split * blocklen; 251 252 void *src_md_buf = io_info->src_md_buf + offset_blocks_split * raid_bdev->bdev.md_len; 253 void *dest_md_buf = io_info->dest_md_buf + offset_blocks_split * raid_bdev->bdev.md_len; 254 255 test_raid_bdev_io = calloc(1, sizeof(*test_raid_bdev_io)); 256 SPDK_CU_ASSERT_FATAL(test_raid_bdev_io != NULL); 257 258 SPDK_CU_ASSERT_FATAL(test_raid_bdev_io->bdev_io_buf == (char *)test_raid_bdev_io); 259 bdev_io = (struct spdk_bdev_io *)test_raid_bdev_io->bdev_io_buf; 260 bdev_io->bdev = &raid_bdev->bdev; 261 bdev_io->type = io_info->io_type; 262 bdev_io->u.bdev.offset_blocks = io_info->offset_blocks + offset_blocks_split; 263 bdev_io->u.bdev.num_blocks = num_blocks; 264 bdev_io->internal.cb = raid_bdev_io_completion_cb; 265 bdev_io->internal.caller_ctx = io_info; 266 267 raid_io = (void *)bdev_io->driver_ctx; 268 raid_io->raid_bdev = raid_bdev; 269 raid_io->raid_ch = io_info->raid_ch; 270 raid_io->base_bdev_io_status = SPDK_BDEV_IO_STATUS_SUCCESS; 271 272 test_raid_bdev_io->io_info = io_info; 273 274 if (io_info->io_type == SPDK_BDEV_IO_TYPE_READ) { 275 test_raid_bdev_io->buf = src_buf; 276 test_raid_bdev_io->buf_md = src_md_buf; 277 bdev_io->u.bdev.md_buf = dest_md_buf; 278 bdev_io->iov.iov_base = dest_buf; 279 } else { 280 test_raid_bdev_io->buf = dest_buf; 281 test_raid_bdev_io->buf_md = dest_md_buf; 282 bdev_io->u.bdev.md_buf = src_md_buf; 283 bdev_io->iov.iov_base = src_buf; 284 } 285 286 bdev_io->u.bdev.iovs = &bdev_io->iov; 287 bdev_io->u.bdev.iovcnt = 1; 288 bdev_io->iov.iov_len = num_blocks * blocklen; 289 290 io_info->remaining++; 291 292 return raid_io; 293 } 294 295 void 296 spdk_bdev_free_io(struct spdk_bdev_io *bdev_io) 297 { 298 free(bdev_io); 299 } 300 301 static int 302 submit_io(struct raid_io_info *io_info, struct spdk_bdev_desc *desc, 303 spdk_bdev_io_completion_cb cb, void *cb_arg) 304 { 305 struct spdk_bdev *bdev = desc->bdev; 306 struct spdk_bdev_io *bdev_io; 307 308 if (bdev == io_info->error.bdev) { 309 if (io_info->error.type == TEST_BDEV_ERROR_SUBMIT) { 310 return -EINVAL; 311 } else if (io_info->error.type == TEST_BDEV_ERROR_NOMEM) { 312 return -ENOMEM; 313 } 314 } 315 316 bdev_io = calloc(1, sizeof(*bdev_io)); 317 SPDK_CU_ASSERT_FATAL(bdev_io != NULL); 318 bdev_io->bdev = bdev; 319 bdev_io->internal.cb = cb; 320 bdev_io->internal.caller_ctx = cb_arg; 321 322 TAILQ_INSERT_TAIL(&io_info->bdev_io_queue, bdev_io, internal.link); 323 324 return 0; 325 } 326 327 static void 328 process_io_completions(struct raid_io_info *io_info) 329 { 330 struct spdk_bdev_io *bdev_io; 331 bool success; 332 333 while ((bdev_io = TAILQ_FIRST(&io_info->bdev_io_queue))) { 334 TAILQ_REMOVE(&io_info->bdev_io_queue, bdev_io, internal.link); 335 336 if (io_info->error.type == TEST_BDEV_ERROR_COMPLETE && 337 io_info->error.bdev == bdev_io->bdev) { 338 success = false; 339 } else { 340 success = true; 341 } 342 343 bdev_io->internal.cb(bdev_io, success, bdev_io->internal.caller_ctx); 344 } 345 346 if (io_info->error.type == TEST_BDEV_ERROR_NOMEM) { 347 struct spdk_bdev_io_wait_entry *waitq_entry, *tmp; 348 struct spdk_bdev *enomem_bdev = io_info->error.bdev; 349 350 io_info->error.type = TEST_BDEV_ERROR_NONE; 351 352 if (io_info->error.on_enomem_cb != NULL) { 353 io_info->error.on_enomem_cb(io_info, io_info->error.on_enomem_cb_ctx); 354 } 355 356 TAILQ_FOREACH_SAFE(waitq_entry, &io_info->bdev_io_wait_queue, link, tmp) { 357 TAILQ_REMOVE(&io_info->bdev_io_wait_queue, waitq_entry, link); 358 CU_ASSERT(waitq_entry->bdev == enomem_bdev); 359 waitq_entry->cb_fn(waitq_entry->cb_arg); 360 } 361 362 process_io_completions(io_info); 363 } else { 364 CU_ASSERT(TAILQ_EMPTY(&io_info->bdev_io_wait_queue)); 365 } 366 } 367 368 #define DATA_OFFSET_TO_MD_OFFSET(raid_bdev, data_offset) ((data_offset >> raid_bdev->blocklen_shift) * raid_bdev->bdev.md_len) 369 370 int 371 spdk_bdev_writev_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 372 struct iovec *iov, int iovcnt, void *md_buf, 373 uint64_t offset_blocks, uint64_t num_blocks, 374 spdk_bdev_io_completion_cb cb, void *cb_arg) 375 { 376 struct chunk *chunk = cb_arg; 377 struct stripe_request *stripe_req; 378 struct test_raid_bdev_io *test_raid_bdev_io; 379 struct raid_io_info *io_info; 380 struct raid_bdev *raid_bdev; 381 uint64_t stripe_idx_off; 382 uint8_t data_chunk_idx; 383 uint64_t data_offset; 384 void *dest_buf, *dest_md_buf; 385 386 SPDK_CU_ASSERT_FATAL(cb == raid5f_chunk_write_complete_bdev_io); 387 SPDK_CU_ASSERT_FATAL(iovcnt == 1); 388 389 stripe_req = raid5f_chunk_stripe_req(chunk); 390 test_raid_bdev_io = (struct test_raid_bdev_io *)spdk_bdev_io_from_ctx(stripe_req->raid_io); 391 io_info = test_raid_bdev_io->io_info; 392 393 raid_bdev = io_info->r5f_info->raid_bdev; 394 395 stripe_idx_off = offset_blocks / raid_bdev->strip_size - 396 io_info->offset_blocks / io_info->r5f_info->stripe_blocks; 397 398 if (chunk == stripe_req->parity_chunk) { 399 if (io_info->parity_buf == NULL) { 400 goto submit; 401 } 402 data_offset = stripe_idx_off * raid_bdev->strip_size_kb * 1024; 403 dest_buf = io_info->parity_buf + data_offset; 404 405 if (md_buf != NULL) { 406 data_offset = DATA_OFFSET_TO_MD_OFFSET(raid_bdev, data_offset); 407 dest_md_buf = io_info->parity_md_buf + data_offset; 408 } 409 } else { 410 data_chunk_idx = chunk < stripe_req->parity_chunk ? chunk->index : chunk->index - 1; 411 data_offset = (stripe_idx_off * io_info->r5f_info->stripe_blocks + 412 data_chunk_idx * raid_bdev->strip_size) * 413 raid_bdev->bdev.blocklen; 414 dest_buf = test_raid_bdev_io->buf + data_offset; 415 416 if (md_buf != NULL) { 417 data_offset = DATA_OFFSET_TO_MD_OFFSET(raid_bdev, data_offset); 418 dest_md_buf = test_raid_bdev_io->buf_md + data_offset; 419 } 420 } 421 422 memcpy(dest_buf, iov->iov_base, iov->iov_len); 423 if (md_buf != NULL) { 424 memcpy(dest_md_buf, md_buf, DATA_OFFSET_TO_MD_OFFSET(raid_bdev, iov->iov_len)); 425 } 426 427 submit: 428 return submit_io(io_info, desc, cb, cb_arg); 429 } 430 431 int 432 spdk_bdev_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 433 struct iovec *iov, int iovcnt, 434 uint64_t offset_blocks, uint64_t num_blocks, 435 spdk_bdev_io_completion_cb cb, void *cb_arg) 436 { 437 return spdk_bdev_writev_blocks_with_md(desc, ch, iov, iovcnt, NULL, offset_blocks, num_blocks, cb, 438 cb_arg); 439 } 440 441 int 442 spdk_bdev_writev_blocks_ext(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 443 struct iovec *iov, int iovcnt, uint64_t offset_blocks, 444 uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg, 445 struct spdk_bdev_ext_io_opts *opts) 446 { 447 CU_ASSERT_PTR_NULL(opts->memory_domain); 448 CU_ASSERT_PTR_NULL(opts->memory_domain_ctx); 449 450 return spdk_bdev_writev_blocks_with_md(desc, ch, iov, iovcnt, opts->metadata, offset_blocks, 451 num_blocks, cb, cb_arg); 452 } 453 454 int 455 spdk_bdev_readv_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 456 struct iovec *iov, int iovcnt, void *md_buf, 457 uint64_t offset_blocks, uint64_t num_blocks, 458 spdk_bdev_io_completion_cb cb, void *cb_arg) 459 { 460 struct raid_bdev_io *raid_io = cb_arg; 461 struct test_raid_bdev_io *test_raid_bdev_io; 462 463 SPDK_CU_ASSERT_FATAL(cb == raid5f_chunk_read_complete); 464 SPDK_CU_ASSERT_FATAL(iovcnt == 1); 465 466 test_raid_bdev_io = (struct test_raid_bdev_io *)spdk_bdev_io_from_ctx(raid_io); 467 468 memcpy(iov->iov_base, test_raid_bdev_io->buf, iov->iov_len); 469 if (md_buf != NULL) { 470 memcpy(md_buf, test_raid_bdev_io->buf_md, DATA_OFFSET_TO_MD_OFFSET(raid_io->raid_bdev, 471 iov->iov_len)); 472 } 473 474 return submit_io(test_raid_bdev_io->io_info, desc, cb, cb_arg); 475 } 476 477 int 478 spdk_bdev_readv_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 479 struct iovec *iov, int iovcnt, 480 uint64_t offset_blocks, uint64_t num_blocks, 481 spdk_bdev_io_completion_cb cb, void *cb_arg) 482 { 483 return spdk_bdev_readv_blocks_with_md(desc, ch, iov, iovcnt, NULL, offset_blocks, num_blocks, cb, 484 cb_arg); 485 } 486 487 int 488 spdk_bdev_readv_blocks_ext(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 489 struct iovec *iov, int iovcnt, uint64_t offset_blocks, 490 uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg, 491 struct spdk_bdev_ext_io_opts *opts) 492 { 493 CU_ASSERT_PTR_NULL(opts->memory_domain); 494 CU_ASSERT_PTR_NULL(opts->memory_domain_ctx); 495 496 return spdk_bdev_readv_blocks_with_md(desc, ch, iov, iovcnt, opts->metadata, offset_blocks, 497 num_blocks, cb, cb_arg); 498 } 499 500 static void 501 xor_block(uint8_t *a, uint8_t *b, size_t size) 502 { 503 while (size-- > 0) { 504 a[size] ^= b[size]; 505 } 506 } 507 508 static void 509 test_raid5f_write_request(struct raid_io_info *io_info) 510 { 511 struct raid_bdev_io *raid_io; 512 513 SPDK_CU_ASSERT_FATAL(io_info->num_blocks / io_info->r5f_info->stripe_blocks == 1); 514 515 raid_io = get_raid_io(io_info, 0, io_info->num_blocks); 516 517 raid5f_submit_rw_request(raid_io); 518 519 process_io_completions(io_info); 520 521 if (io_info->status == SPDK_BDEV_IO_STATUS_SUCCESS) { 522 if (io_info->parity_buf) { 523 CU_ASSERT(memcmp(io_info->parity_buf, io_info->reference_parity, 524 io_info->parity_buf_size) == 0); 525 } 526 if (io_info->parity_md_buf) { 527 CU_ASSERT(memcmp(io_info->parity_md_buf, io_info->reference_md_parity, 528 io_info->parity_md_buf_size) == 0); 529 } 530 } 531 } 532 533 static void 534 test_raid5f_read_request(struct raid_io_info *io_info) 535 { 536 uint32_t strip_size = io_info->r5f_info->raid_bdev->strip_size; 537 uint64_t num_blocks = io_info->num_blocks; 538 uint64_t offset_blocks_split = 0; 539 540 while (num_blocks) { 541 uint64_t chunk_offset = offset_blocks_split % strip_size; 542 uint64_t num_blocks_split = spdk_min(num_blocks, strip_size - chunk_offset); 543 struct raid_bdev_io *raid_io; 544 545 raid_io = get_raid_io(io_info, offset_blocks_split, num_blocks_split); 546 547 raid5f_submit_rw_request(raid_io); 548 549 num_blocks -= num_blocks_split; 550 offset_blocks_split += num_blocks_split; 551 } 552 553 process_io_completions(io_info); 554 } 555 556 static void 557 deinit_io_info(struct raid_io_info *io_info) 558 { 559 free(io_info->src_buf); 560 free(io_info->dest_buf); 561 free(io_info->src_md_buf); 562 free(io_info->dest_md_buf); 563 free(io_info->parity_buf); 564 free(io_info->reference_parity); 565 free(io_info->parity_md_buf); 566 free(io_info->reference_md_parity); 567 } 568 569 static void 570 init_io_info(struct raid_io_info *io_info, struct raid5f_info *r5f_info, 571 struct raid_bdev_io_channel *raid_ch, enum spdk_bdev_io_type io_type, 572 uint64_t offset_blocks, uint64_t num_blocks) 573 { 574 struct raid_bdev *raid_bdev = r5f_info->raid_bdev; 575 uint32_t blocklen = raid_bdev->bdev.blocklen; 576 void *src_buf, *dest_buf; 577 void *src_md_buf, *dest_md_buf; 578 size_t buf_size = num_blocks * blocklen; 579 size_t buf_md_size = num_blocks * raid_bdev->bdev.md_len; 580 uint64_t block; 581 uint64_t i; 582 583 memset(io_info, 0, sizeof(*io_info)); 584 585 if (buf_size) { 586 src_buf = spdk_dma_malloc(buf_size, 4096, NULL); 587 SPDK_CU_ASSERT_FATAL(src_buf != NULL); 588 589 dest_buf = spdk_dma_malloc(buf_size, 4096, NULL); 590 SPDK_CU_ASSERT_FATAL(dest_buf != NULL); 591 592 memset(src_buf, 0xff, buf_size); 593 for (block = 0; block < num_blocks; block++) { 594 *((uint64_t *)(src_buf + block * blocklen)) = block; 595 } 596 } else { 597 src_buf = NULL; 598 dest_buf = NULL; 599 } 600 601 if (buf_md_size) { 602 src_md_buf = spdk_dma_malloc(buf_md_size, 4096, NULL); 603 SPDK_CU_ASSERT_FATAL(src_md_buf != NULL); 604 605 dest_md_buf = spdk_dma_malloc(buf_md_size, 4096, NULL); 606 SPDK_CU_ASSERT_FATAL(dest_md_buf != NULL); 607 608 memset(src_md_buf, 0xff, buf_md_size); 609 for (i = 0; i < buf_md_size; i++) { 610 *((uint8_t *)(src_md_buf + i)) = (uint8_t)i; 611 } 612 } else { 613 src_md_buf = NULL; 614 dest_md_buf = NULL; 615 } 616 617 io_info->r5f_info = r5f_info; 618 io_info->raid_ch = raid_ch; 619 io_info->io_type = io_type; 620 io_info->offset_blocks = offset_blocks; 621 io_info->num_blocks = num_blocks; 622 io_info->src_buf = src_buf; 623 io_info->dest_buf = dest_buf; 624 io_info->src_md_buf = src_md_buf; 625 io_info->dest_md_buf = dest_md_buf; 626 io_info->buf_size = buf_size; 627 io_info->status = SPDK_BDEV_IO_STATUS_PENDING; 628 629 TAILQ_INIT(&io_info->bdev_io_queue); 630 TAILQ_INIT(&io_info->bdev_io_wait_queue); 631 } 632 633 static void 634 io_info_setup_parity(struct raid_io_info *io_info) 635 { 636 struct raid5f_info *r5f_info = io_info->r5f_info; 637 struct raid_bdev *raid_bdev = r5f_info->raid_bdev; 638 uint32_t blocklen = raid_bdev->bdev.blocklen; 639 uint64_t num_stripes = io_info->num_blocks / r5f_info->stripe_blocks; 640 size_t strip_len = raid_bdev->strip_size * blocklen; 641 size_t strip_md_len = raid_bdev->strip_size * raid_bdev->bdev.md_len; 642 void *src = io_info->src_buf; 643 void *dest; 644 unsigned i, j; 645 646 io_info->parity_buf_size = num_stripes * strip_len; 647 io_info->parity_buf = calloc(1, io_info->parity_buf_size); 648 SPDK_CU_ASSERT_FATAL(io_info->parity_buf != NULL); 649 650 io_info->reference_parity = calloc(1, io_info->parity_buf_size); 651 SPDK_CU_ASSERT_FATAL(io_info->reference_parity != NULL); 652 653 dest = io_info->reference_parity; 654 for (i = 0; i < num_stripes; i++) { 655 for (j = 0; j < raid5f_stripe_data_chunks_num(raid_bdev); j++) { 656 xor_block(dest, src, strip_len); 657 src += strip_len; 658 } 659 dest += strip_len; 660 } 661 662 io_info->parity_md_buf_size = num_stripes * strip_md_len; 663 io_info->parity_md_buf = calloc(1, io_info->parity_md_buf_size); 664 SPDK_CU_ASSERT_FATAL(io_info->parity_md_buf != NULL); 665 666 io_info->reference_md_parity = calloc(1, io_info->parity_md_buf_size); 667 SPDK_CU_ASSERT_FATAL(io_info->reference_md_parity != NULL); 668 669 src = io_info->src_md_buf; 670 dest = io_info->reference_md_parity; 671 for (i = 0; i < num_stripes; i++) { 672 for (j = 0; j < raid5f_stripe_data_chunks_num(raid_bdev); j++) { 673 xor_block(dest, src, strip_md_len); 674 src += strip_md_len; 675 } 676 dest += strip_md_len; 677 } 678 } 679 680 static void 681 test_raid5f_submit_rw_request(struct raid5f_info *r5f_info, struct raid_bdev_io_channel *raid_ch, 682 enum spdk_bdev_io_type io_type, uint64_t stripe_index, uint64_t stripe_offset_blocks, 683 uint64_t num_blocks) 684 { 685 uint64_t offset_blocks = stripe_index * r5f_info->stripe_blocks + stripe_offset_blocks; 686 struct raid_io_info io_info; 687 688 init_io_info(&io_info, r5f_info, raid_ch, io_type, offset_blocks, num_blocks); 689 690 switch (io_type) { 691 case SPDK_BDEV_IO_TYPE_READ: 692 test_raid5f_read_request(&io_info); 693 break; 694 case SPDK_BDEV_IO_TYPE_WRITE: 695 io_info_setup_parity(&io_info); 696 test_raid5f_write_request(&io_info); 697 break; 698 default: 699 CU_FAIL_FATAL("unsupported io_type"); 700 } 701 702 assert(io_info.status == SPDK_BDEV_IO_STATUS_SUCCESS); 703 assert(memcmp(io_info.src_buf, io_info.dest_buf, io_info.buf_size) == 0); 704 705 CU_ASSERT(io_info.status == SPDK_BDEV_IO_STATUS_SUCCESS); 706 CU_ASSERT(memcmp(io_info.src_buf, io_info.dest_buf, io_info.buf_size) == 0); 707 708 deinit_io_info(&io_info); 709 } 710 711 static void 712 run_for_each_raid5f_config(void (*test_fn)(struct raid_bdev *raid_bdev, 713 struct raid_bdev_io_channel *raid_ch)) 714 { 715 struct raid_params *params; 716 717 RAID_PARAMS_FOR_EACH(params) { 718 struct raid5f_info *r5f_info; 719 struct raid_bdev_io_channel raid_ch = { 0 }; 720 721 r5f_info = create_raid5f(params); 722 723 raid_ch.num_channels = params->num_base_bdevs; 724 raid_ch.base_channel = calloc(params->num_base_bdevs, sizeof(struct spdk_io_channel *)); 725 SPDK_CU_ASSERT_FATAL(raid_ch.base_channel != NULL); 726 727 raid_ch.module_channel = raid5f_get_io_channel(r5f_info->raid_bdev); 728 SPDK_CU_ASSERT_FATAL(raid_ch.module_channel); 729 730 test_fn(r5f_info->raid_bdev, &raid_ch); 731 732 spdk_put_io_channel(raid_ch.module_channel); 733 poll_threads(); 734 735 free(raid_ch.base_channel); 736 737 delete_raid5f(r5f_info); 738 } 739 } 740 741 #define RAID5F_TEST_FOR_EACH_STRIPE(raid_bdev, i) \ 742 for (i = 0; i < spdk_min(raid_bdev->num_base_bdevs, ((struct raid5f_info *)raid_bdev->module_private)->total_stripes); i++) 743 744 struct test_request_conf { 745 uint64_t stripe_offset_blocks; 746 uint64_t num_blocks; 747 }; 748 749 static void 750 __test_raid5f_submit_read_request(struct raid_bdev *raid_bdev, struct raid_bdev_io_channel *raid_ch) 751 { 752 struct raid5f_info *r5f_info = raid_bdev->module_private; 753 uint32_t strip_size = raid_bdev->strip_size; 754 unsigned int i; 755 756 struct test_request_conf test_requests[] = { 757 { 0, 1 }, 758 { 0, strip_size }, 759 { 0, strip_size + 1 }, 760 { 0, r5f_info->stripe_blocks }, 761 { 1, 1 }, 762 { 1, strip_size }, 763 { 1, strip_size + 1 }, 764 { strip_size, 1 }, 765 { strip_size, strip_size }, 766 { strip_size, strip_size + 1 }, 767 { strip_size - 1, 1 }, 768 { strip_size - 1, strip_size }, 769 { strip_size - 1, strip_size + 1 }, 770 { strip_size - 1, 2 }, 771 }; 772 for (i = 0; i < SPDK_COUNTOF(test_requests); i++) { 773 struct test_request_conf *t = &test_requests[i]; 774 uint64_t stripe_index; 775 776 RAID5F_TEST_FOR_EACH_STRIPE(raid_bdev, stripe_index) { 777 test_raid5f_submit_rw_request(r5f_info, raid_ch, SPDK_BDEV_IO_TYPE_READ, 778 stripe_index, t->stripe_offset_blocks, t->num_blocks); 779 } 780 } 781 } 782 static void 783 test_raid5f_submit_read_request(void) 784 { 785 run_for_each_raid5f_config(__test_raid5f_submit_read_request); 786 } 787 788 static void 789 __test_raid5f_stripe_request_map_iovecs(struct raid_bdev *raid_bdev, 790 struct raid_bdev_io_channel *raid_ch) 791 { 792 struct raid5f_info *r5f_info = raid_bdev->module_private; 793 struct raid5f_io_channel *r5ch = spdk_io_channel_get_ctx(raid_ch->module_channel); 794 size_t strip_bytes = raid_bdev->strip_size * raid_bdev->bdev.blocklen; 795 struct raid_io_info io_info; 796 struct raid_bdev_io *raid_io; 797 struct spdk_bdev_io *bdev_io; 798 struct stripe_request *stripe_req; 799 struct chunk *chunk; 800 struct iovec iovs[] = { 801 { .iov_base = (void *)0x0ff0000, .iov_len = strip_bytes }, 802 { .iov_base = (void *)0x1ff0000, .iov_len = strip_bytes / 2 }, 803 { .iov_base = (void *)0x2ff0000, .iov_len = strip_bytes * 2 }, 804 { .iov_base = (void *)0x3ff0000, .iov_len = strip_bytes * raid_bdev->num_base_bdevs }, 805 }; 806 size_t iovcnt = SPDK_COUNTOF(iovs); 807 int ret; 808 809 init_io_info(&io_info, r5f_info, raid_ch, SPDK_BDEV_IO_TYPE_WRITE, 0, 0); 810 811 raid_io = get_raid_io(&io_info, 0, 0); 812 bdev_io = spdk_bdev_io_from_ctx(raid_io); 813 bdev_io->u.bdev.iovs = iovs; 814 bdev_io->u.bdev.iovcnt = iovcnt; 815 816 stripe_req = raid5f_stripe_request_alloc(r5ch); 817 SPDK_CU_ASSERT_FATAL(stripe_req != NULL); 818 819 stripe_req->parity_chunk = &stripe_req->chunks[raid5f_stripe_data_chunks_num(raid_bdev)]; 820 stripe_req->raid_io = raid_io; 821 822 ret = raid5f_stripe_request_map_iovecs(stripe_req); 823 CU_ASSERT(ret == 0); 824 825 chunk = &stripe_req->chunks[0]; 826 CU_ASSERT_EQUAL(chunk->iovcnt, 1); 827 CU_ASSERT_EQUAL(chunk->iovs[0].iov_base, iovs[0].iov_base); 828 CU_ASSERT_EQUAL(chunk->iovs[0].iov_len, iovs[0].iov_len); 829 830 chunk = &stripe_req->chunks[1]; 831 CU_ASSERT_EQUAL(chunk->iovcnt, 2); 832 CU_ASSERT_EQUAL(chunk->iovs[0].iov_base, iovs[1].iov_base); 833 CU_ASSERT_EQUAL(chunk->iovs[0].iov_len, iovs[1].iov_len); 834 CU_ASSERT_EQUAL(chunk->iovs[1].iov_base, iovs[2].iov_base); 835 CU_ASSERT_EQUAL(chunk->iovs[1].iov_len, iovs[2].iov_len / 4); 836 837 if (raid_bdev->num_base_bdevs > 3) { 838 chunk = &stripe_req->chunks[2]; 839 CU_ASSERT_EQUAL(chunk->iovcnt, 1); 840 CU_ASSERT_EQUAL(chunk->iovs[0].iov_base, iovs[2].iov_base + strip_bytes / 2); 841 CU_ASSERT_EQUAL(chunk->iovs[0].iov_len, iovs[2].iov_len / 2); 842 } 843 if (raid_bdev->num_base_bdevs > 4) { 844 chunk = &stripe_req->chunks[3]; 845 CU_ASSERT_EQUAL(chunk->iovcnt, 2); 846 CU_ASSERT_EQUAL(chunk->iovs[0].iov_base, iovs[2].iov_base + (strip_bytes / 2) * 3); 847 CU_ASSERT_EQUAL(chunk->iovs[0].iov_len, iovs[2].iov_len / 4); 848 CU_ASSERT_EQUAL(chunk->iovs[1].iov_base, iovs[3].iov_base); 849 CU_ASSERT_EQUAL(chunk->iovs[1].iov_len, strip_bytes / 2); 850 } 851 852 raid5f_stripe_request_free(stripe_req); 853 spdk_bdev_free_io(bdev_io); 854 deinit_io_info(&io_info); 855 } 856 static void 857 test_raid5f_stripe_request_map_iovecs(void) 858 { 859 run_for_each_raid5f_config(__test_raid5f_stripe_request_map_iovecs); 860 } 861 862 static void 863 __test_raid5f_submit_full_stripe_write_request(struct raid_bdev *raid_bdev, 864 struct raid_bdev_io_channel *raid_ch) 865 { 866 struct raid5f_info *r5f_info = raid_bdev->module_private; 867 uint64_t stripe_index; 868 869 RAID5F_TEST_FOR_EACH_STRIPE(raid_bdev, stripe_index) { 870 test_raid5f_submit_rw_request(r5f_info, raid_ch, SPDK_BDEV_IO_TYPE_WRITE, 871 stripe_index, 0, r5f_info->stripe_blocks); 872 } 873 } 874 static void 875 test_raid5f_submit_full_stripe_write_request(void) 876 { 877 run_for_each_raid5f_config(__test_raid5f_submit_full_stripe_write_request); 878 } 879 880 static void 881 __test_raid5f_chunk_write_error(struct raid_bdev *raid_bdev, struct raid_bdev_io_channel *raid_ch) 882 { 883 struct raid5f_info *r5f_info = raid_bdev->module_private; 884 struct raid_base_bdev_info *base_bdev_info; 885 uint64_t stripe_index; 886 struct raid_io_info io_info; 887 enum test_bdev_error_type error_type; 888 889 for (error_type = TEST_BDEV_ERROR_SUBMIT; error_type <= TEST_BDEV_ERROR_NOMEM; error_type++) { 890 RAID5F_TEST_FOR_EACH_STRIPE(raid_bdev, stripe_index) { 891 RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_bdev_info) { 892 init_io_info(&io_info, r5f_info, raid_ch, SPDK_BDEV_IO_TYPE_WRITE, 893 stripe_index * r5f_info->stripe_blocks, r5f_info->stripe_blocks); 894 895 io_info.error.type = error_type; 896 io_info.error.bdev = base_bdev_info->bdev; 897 898 test_raid5f_write_request(&io_info); 899 900 if (error_type == TEST_BDEV_ERROR_NOMEM) { 901 CU_ASSERT(io_info.status == SPDK_BDEV_IO_STATUS_SUCCESS); 902 } else { 903 CU_ASSERT(io_info.status == SPDK_BDEV_IO_STATUS_FAILED); 904 } 905 906 deinit_io_info(&io_info); 907 } 908 } 909 } 910 } 911 static void 912 test_raid5f_chunk_write_error(void) 913 { 914 run_for_each_raid5f_config(__test_raid5f_chunk_write_error); 915 } 916 917 struct chunk_write_error_with_enomem_ctx { 918 enum test_bdev_error_type error_type; 919 struct spdk_bdev *bdev; 920 }; 921 922 static void 923 chunk_write_error_with_enomem_cb(struct raid_io_info *io_info, void *_ctx) 924 { 925 struct chunk_write_error_with_enomem_ctx *ctx = _ctx; 926 927 io_info->error.type = ctx->error_type; 928 io_info->error.bdev = ctx->bdev; 929 } 930 931 static void 932 __test_raid5f_chunk_write_error_with_enomem(struct raid_bdev *raid_bdev, 933 struct raid_bdev_io_channel *raid_ch) 934 { 935 struct raid5f_info *r5f_info = raid_bdev->module_private; 936 struct raid_base_bdev_info *base_bdev_info; 937 uint64_t stripe_index; 938 struct raid_io_info io_info; 939 enum test_bdev_error_type error_type; 940 struct chunk_write_error_with_enomem_ctx on_enomem_cb_ctx; 941 942 for (error_type = TEST_BDEV_ERROR_SUBMIT; error_type <= TEST_BDEV_ERROR_COMPLETE; error_type++) { 943 RAID5F_TEST_FOR_EACH_STRIPE(raid_bdev, stripe_index) { 944 struct raid_base_bdev_info *base_bdev_info_last = 945 &raid_bdev->base_bdev_info[raid_bdev->num_base_bdevs - 1]; 946 947 RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_bdev_info) { 948 if (base_bdev_info == base_bdev_info_last) { 949 continue; 950 } 951 952 init_io_info(&io_info, r5f_info, raid_ch, SPDK_BDEV_IO_TYPE_WRITE, 953 stripe_index * r5f_info->stripe_blocks, r5f_info->stripe_blocks); 954 955 io_info.error.type = TEST_BDEV_ERROR_NOMEM; 956 io_info.error.bdev = base_bdev_info->bdev; 957 io_info.error.on_enomem_cb = chunk_write_error_with_enomem_cb; 958 io_info.error.on_enomem_cb_ctx = &on_enomem_cb_ctx; 959 on_enomem_cb_ctx.error_type = error_type; 960 on_enomem_cb_ctx.bdev = base_bdev_info_last->bdev; 961 962 test_raid5f_write_request(&io_info); 963 964 CU_ASSERT(io_info.status == SPDK_BDEV_IO_STATUS_FAILED); 965 966 deinit_io_info(&io_info); 967 } 968 } 969 } 970 } 971 static void 972 test_raid5f_chunk_write_error_with_enomem(void) 973 { 974 run_for_each_raid5f_config(__test_raid5f_chunk_write_error_with_enomem); 975 } 976 977 int 978 main(int argc, char **argv) 979 { 980 CU_pSuite suite = NULL; 981 unsigned int num_failures; 982 983 CU_set_error_action(CUEA_ABORT); 984 CU_initialize_registry(); 985 986 suite = CU_add_suite("raid5f", test_setup, test_cleanup); 987 CU_ADD_TEST(suite, test_raid5f_start); 988 CU_ADD_TEST(suite, test_raid5f_submit_read_request); 989 CU_ADD_TEST(suite, test_raid5f_stripe_request_map_iovecs); 990 CU_ADD_TEST(suite, test_raid5f_submit_full_stripe_write_request); 991 CU_ADD_TEST(suite, test_raid5f_chunk_write_error); 992 CU_ADD_TEST(suite, test_raid5f_chunk_write_error_with_enomem); 993 994 allocate_threads(1); 995 set_thread(0); 996 997 CU_basic_set_mode(CU_BRM_VERBOSE); 998 CU_basic_run_tests(); 999 num_failures = CU_get_number_of_failures(); 1000 CU_cleanup_registry(); 1001 1002 free_threads(); 1003 1004 return num_failures; 1005 } 1006