1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/dif.h" 35 #include "spdk/crc16.h" 36 #include "spdk/endian.h" 37 #include "spdk/log.h" 38 #include "spdk/util.h" 39 40 /* Context to iterate a iovec array. */ 41 struct _iov_iter { 42 /* Current iovec in the iteration */ 43 struct iovec *iov; 44 45 /* Remaining count of iovecs in the iteration. */ 46 int iovcnt; 47 48 /* Current offset in the iovec */ 49 uint32_t iov_offset; 50 }; 51 52 static inline void 53 _iov_iter_init(struct _iov_iter *i, struct iovec *iovs, int iovcnt) 54 { 55 i->iov = iovs; 56 i->iovcnt = iovcnt; 57 i->iov_offset = 0; 58 } 59 60 static inline void 61 _iov_iter_advance(struct _iov_iter *i, uint32_t step) 62 { 63 i->iov_offset += step; 64 if (i->iov_offset == i->iov->iov_len) { 65 i->iov++; 66 assert(i->iovcnt > 0); 67 i->iovcnt--; 68 i->iov_offset = 0; 69 } 70 } 71 72 static inline void 73 _iov_iter_get_buf(struct _iov_iter *i, void **_buf, uint32_t *_buf_len) 74 { 75 if (_buf != NULL) { 76 *_buf = i->iov->iov_base + i->iov_offset; 77 } 78 if (_buf_len != NULL) { 79 *_buf_len = i->iov->iov_len - i->iov_offset; 80 } 81 } 82 83 static void 84 _iov_iter_fast_forward(struct _iov_iter *i, uint32_t offset) 85 { 86 i->iov_offset = offset; 87 while (i->iovcnt != 0) { 88 if (i->iov_offset < i->iov->iov_len) { 89 break; 90 } 91 92 i->iov_offset -= i->iov->iov_len; 93 i->iov++; 94 i->iovcnt--; 95 } 96 } 97 98 static bool 99 _are_iovs_bytes_multiple(struct iovec *iovs, int iovcnt, uint32_t bytes) 100 { 101 int i; 102 103 for (i = 0; i < iovcnt; i++) { 104 if (iovs[i].iov_len % bytes) { 105 return false; 106 } 107 } 108 109 return true; 110 } 111 112 static bool 113 _are_iovs_valid(struct iovec *iovs, int iovcnt, uint32_t bytes) 114 { 115 uint64_t total = 0; 116 int i; 117 118 for (i = 0; i < iovcnt; i++) { 119 total += iovs[i].iov_len; 120 } 121 122 return total >= bytes; 123 } 124 125 static bool 126 _dif_type_is_valid(enum spdk_dif_type dif_type, uint32_t dif_flags) 127 { 128 switch (dif_type) { 129 case SPDK_DIF_TYPE1: 130 case SPDK_DIF_TYPE2: 131 case SPDK_DIF_DISABLE: 132 break; 133 case SPDK_DIF_TYPE3: 134 if (dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 135 SPDK_ERRLOG("Reference Tag should not be checked for Type 3\n"); 136 return false; 137 } 138 break; 139 default: 140 SPDK_ERRLOG("Unknown DIF Type: %d\n", dif_type); 141 return false; 142 } 143 144 return true; 145 } 146 147 static bool 148 _dif_is_disabled(enum spdk_dif_type dif_type) 149 { 150 if (dif_type == SPDK_DIF_DISABLE) { 151 return true; 152 } else { 153 return false; 154 } 155 } 156 157 158 static uint32_t 159 _get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave) 160 { 161 if (!dif_loc) { 162 /* For metadata formats with more than 8 bytes, if the DIF is 163 * contained in the last 8 bytes of metadata, then the CRC 164 * covers all metadata up to but excluding these last 8 bytes. 165 */ 166 if (md_interleave) { 167 return block_size - sizeof(struct spdk_dif); 168 } else { 169 return md_size - sizeof(struct spdk_dif); 170 } 171 } else { 172 /* For metadata formats with more than 8 bytes, if the DIF is 173 * contained in the first 8 bytes of metadata, then the CRC 174 * does not cover any metadata. 175 */ 176 if (md_interleave) { 177 return block_size - md_size; 178 } else { 179 return 0; 180 } 181 } 182 } 183 184 int 185 spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, 186 bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, 187 uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag) 188 { 189 if (md_size < sizeof(struct spdk_dif)) { 190 SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); 191 return -EINVAL; 192 } 193 194 if (md_interleave) { 195 if (block_size < md_size) { 196 SPDK_ERRLOG("Block size is smaller than DIF size.\n"); 197 return -EINVAL; 198 } 199 } else { 200 if (block_size == 0 || (block_size % 512) != 0) { 201 SPDK_ERRLOG("Zero block size is not allowed\n"); 202 return -EINVAL; 203 } 204 } 205 206 if (!_dif_type_is_valid(dif_type, dif_flags)) { 207 SPDK_ERRLOG("DIF type is invalid.\n"); 208 return -EINVAL; 209 } 210 211 ctx->block_size = block_size; 212 ctx->md_size = md_size; 213 ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave); 214 ctx->dif_type = dif_type; 215 ctx->dif_flags = dif_flags; 216 ctx->init_ref_tag = init_ref_tag; 217 ctx->apptag_mask = apptag_mask; 218 ctx->app_tag = app_tag; 219 220 return 0; 221 } 222 223 static void 224 _dif_generate(void *_dif, uint16_t guard, uint32_t offset_blocks, 225 const struct spdk_dif_ctx *ctx) 226 { 227 struct spdk_dif *dif = _dif; 228 uint32_t ref_tag; 229 230 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 231 to_be16(&dif->guard, guard); 232 } 233 234 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 235 to_be16(&dif->app_tag, ctx->app_tag); 236 } 237 238 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 239 /* For type 1 and 2, the reference tag is incremented for each 240 * subsequent logical block. For type 3, the reference tag 241 * remains the same as the initial reference tag. 242 */ 243 if (ctx->dif_type != SPDK_DIF_TYPE3) { 244 ref_tag = ctx->init_ref_tag + offset_blocks; 245 } else { 246 ref_tag = ctx->init_ref_tag; 247 } 248 249 to_be32(&dif->ref_tag, ref_tag); 250 } 251 } 252 253 static void 254 dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 255 const struct spdk_dif_ctx *ctx) 256 { 257 struct _iov_iter iter; 258 uint32_t offset_blocks; 259 void *buf; 260 uint16_t guard = 0; 261 262 offset_blocks = 0; 263 _iov_iter_init(&iter, iovs, iovcnt); 264 265 while (offset_blocks < num_blocks) { 266 _iov_iter_get_buf(&iter, &buf, NULL); 267 268 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 269 guard = spdk_crc16_t10dif(0, buf, ctx->guard_interval); 270 } 271 272 _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); 273 274 _iov_iter_advance(&iter, ctx->block_size); 275 offset_blocks++; 276 } 277 } 278 279 static void 280 _dif_generate_split(struct _iov_iter *iter, uint32_t offset_blocks, 281 const struct spdk_dif_ctx *ctx) 282 { 283 uint32_t offset_in_block, offset_in_dif, buf_len; 284 void *buf; 285 uint16_t guard; 286 struct spdk_dif dif = {}; 287 288 guard = 0; 289 offset_in_block = 0; 290 291 while (offset_in_block < ctx->block_size) { 292 _iov_iter_get_buf(iter, &buf, &buf_len); 293 294 if (offset_in_block < ctx->guard_interval) { 295 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 296 297 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 298 /* Compute CRC over split logical block data. */ 299 guard = spdk_crc16_t10dif(guard, buf, buf_len); 300 } 301 302 if (offset_in_block + buf_len == ctx->guard_interval) { 303 /* If a whole logical block data is parsed, generate DIF 304 * and save it to the temporary DIF area. 305 */ 306 _dif_generate(&dif, guard, offset_blocks, ctx); 307 } 308 } else if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) { 309 /* Copy generated DIF to the split DIF field. */ 310 offset_in_dif = offset_in_block - ctx->guard_interval; 311 buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif); 312 313 memcpy(buf, ((uint8_t *)&dif) + offset_in_dif, buf_len); 314 } else { 315 /* Skip metadata field after DIF field. */ 316 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 317 } 318 319 _iov_iter_advance(iter, buf_len); 320 offset_in_block += buf_len; 321 } 322 } 323 324 static void 325 dif_generate_split(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 326 const struct spdk_dif_ctx *ctx) 327 { 328 struct _iov_iter iter; 329 uint32_t offset_blocks; 330 331 offset_blocks = 0; 332 _iov_iter_init(&iter, iovs, iovcnt); 333 334 while (offset_blocks < num_blocks) { 335 _dif_generate_split(&iter, offset_blocks, ctx); 336 offset_blocks++; 337 } 338 } 339 340 int 341 spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 342 const struct spdk_dif_ctx *ctx) 343 { 344 if (!_are_iovs_valid(iovs, iovcnt, ctx->block_size * num_blocks)) { 345 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 346 return -EINVAL; 347 } 348 349 if (_dif_is_disabled(ctx->dif_type)) { 350 return 0; 351 } 352 353 if (_are_iovs_bytes_multiple(iovs, iovcnt, ctx->block_size)) { 354 dif_generate(iovs, iovcnt, num_blocks, ctx); 355 } else { 356 dif_generate_split(iovs, iovcnt, num_blocks, ctx); 357 } 358 359 return 0; 360 } 361 362 static void 363 _dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, 364 uint32_t expected, uint32_t actual, uint32_t err_offset) 365 { 366 if (err_blk) { 367 err_blk->err_type = err_type; 368 err_blk->expected = expected; 369 err_blk->actual = actual; 370 err_blk->err_offset = err_offset; 371 } 372 } 373 374 static int 375 _dif_verify(void *_dif, uint16_t guard, uint32_t offset_blocks, 376 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 377 { 378 struct spdk_dif *dif = _dif; 379 uint16_t _guard; 380 uint16_t _app_tag; 381 uint32_t ref_tag, _ref_tag; 382 383 switch (ctx->dif_type) { 384 case SPDK_DIF_TYPE1: 385 case SPDK_DIF_TYPE2: 386 /* If Type 1 or 2 is used, then all DIF checks are disabled when 387 * the Application Tag is 0xFFFF. 388 */ 389 if (dif->app_tag == 0xFFFF) { 390 return 0; 391 } 392 break; 393 case SPDK_DIF_TYPE3: 394 /* If Type 3 is used, then all DIF checks are disabled when the 395 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF. 396 */ 397 if (dif->app_tag == 0xFFFF && dif->ref_tag == 0xFFFFFFFF) { 398 return 0; 399 } 400 break; 401 default: 402 break; 403 } 404 405 /* For type 1 and 2, the reference tag is incremented for each 406 * subsequent logical block. For type 3, the reference tag 407 * remains the same as the initial reference tag. 408 */ 409 if (ctx->dif_type != SPDK_DIF_TYPE3) { 410 ref_tag = ctx->init_ref_tag + offset_blocks; 411 } else { 412 ref_tag = ctx->init_ref_tag; 413 } 414 415 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 416 /* Compare the DIF Guard field to the CRC computed over the logical 417 * block data. 418 */ 419 _guard = from_be16(&dif->guard); 420 if (_guard != guard) { 421 _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, 422 offset_blocks); 423 SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu32 "," \ 424 " Expected=%x, Actual=%x\n", 425 ref_tag, _guard, guard); 426 return -1; 427 } 428 } 429 430 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 431 /* Compare unmasked bits in the DIF Application Tag field to the 432 * passed Application Tag. 433 */ 434 _app_tag = from_be16(&dif->app_tag); 435 if ((_app_tag & ctx->apptag_mask) != ctx->app_tag) { 436 _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, 437 (_app_tag & ctx->apptag_mask), offset_blocks); 438 SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu32 "," \ 439 " Expected=%x, Actual=%x\n", 440 ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); 441 return -1; 442 } 443 } 444 445 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 446 switch (ctx->dif_type) { 447 case SPDK_DIF_TYPE1: 448 case SPDK_DIF_TYPE2: 449 /* Compare the DIF Reference Tag field to the passed Reference Tag. 450 * The passed Reference Tag will be the least significant 4 bytes 451 * of the LBA when Type 1 is used, and application specific value 452 * if Type 2 is used, 453 */ 454 _ref_tag = from_be32(&dif->ref_tag); 455 if (_ref_tag != ref_tag) { 456 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, ref_tag, 457 _ref_tag, offset_blocks); 458 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu32 "," \ 459 " Expected=%x, Actual=%x\n", 460 ref_tag, ref_tag, _ref_tag); 461 return -1; 462 } 463 break; 464 case SPDK_DIF_TYPE3: 465 /* For Type 3, computed Reference Tag remains unchanged. 466 * Hence ignore the Reference Tag field. 467 */ 468 break; 469 default: 470 break; 471 } 472 } 473 474 return 0; 475 } 476 477 static int 478 dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 479 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 480 { 481 struct _iov_iter iter; 482 uint32_t offset_blocks; 483 int rc; 484 void *buf; 485 uint16_t guard = 0; 486 487 offset_blocks = 0; 488 _iov_iter_init(&iter, iovs, iovcnt); 489 490 while (offset_blocks < num_blocks) { 491 _iov_iter_get_buf(&iter, &buf, NULL); 492 493 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 494 guard = spdk_crc16_t10dif(0, buf, ctx->guard_interval); 495 } 496 497 rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 498 if (rc != 0) { 499 return rc; 500 } 501 502 _iov_iter_advance(&iter, ctx->block_size); 503 offset_blocks++; 504 } 505 506 return 0; 507 } 508 509 static int 510 _dif_verify_split(struct _iov_iter *iter, uint32_t offset_blocks, 511 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 512 { 513 uint32_t offset_in_block, offset_in_dif, buf_len; 514 void *buf; 515 uint16_t guard; 516 struct spdk_dif dif = {}; 517 518 guard = 0; 519 offset_in_block = 0; 520 521 while (offset_in_block < ctx->block_size) { 522 _iov_iter_get_buf(iter, &buf, &buf_len); 523 524 if (offset_in_block < ctx->guard_interval) { 525 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 526 527 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 528 /* Compute CRC over split logical block data. */ 529 guard = spdk_crc16_t10dif(guard, buf, buf_len); 530 } 531 } else if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) { 532 /* Copy the split DIF field to the temporary DIF buffer. */ 533 offset_in_dif = offset_in_block - ctx->guard_interval; 534 buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif); 535 536 memcpy((uint8_t *)&dif + offset_in_dif, buf, buf_len); 537 } else { 538 /* Skip metadata field after DIF field. */ 539 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 540 } 541 542 _iov_iter_advance(iter, buf_len); 543 offset_in_block += buf_len; 544 } 545 546 return _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 547 } 548 549 static int 550 dif_verify_split(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 551 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 552 { 553 struct _iov_iter iter; 554 uint32_t offset_blocks; 555 int rc; 556 557 offset_blocks = 0; 558 _iov_iter_init(&iter, iovs, iovcnt); 559 560 while (offset_blocks < num_blocks) { 561 rc = _dif_verify_split(&iter, offset_blocks, ctx, err_blk); 562 if (rc != 0) { 563 return rc; 564 } 565 offset_blocks++; 566 } 567 568 return 0; 569 } 570 571 int 572 spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 573 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 574 { 575 if (!_are_iovs_valid(iovs, iovcnt, ctx->block_size * num_blocks)) { 576 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 577 return -EINVAL; 578 } 579 580 if (_dif_is_disabled(ctx->dif_type)) { 581 return 0; 582 } 583 584 if (_are_iovs_bytes_multiple(iovs, iovcnt, ctx->block_size)) { 585 return dif_verify(iovs, iovcnt, num_blocks, ctx, err_blk); 586 } else { 587 return dif_verify_split(iovs, iovcnt, num_blocks, ctx, err_blk); 588 } 589 } 590 591 static void 592 dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 593 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 594 { 595 struct _iov_iter src_iter, dst_iter; 596 uint32_t offset_blocks, data_block_size; 597 void *src, *dst; 598 uint16_t guard; 599 600 offset_blocks = 0; 601 _iov_iter_init(&src_iter, iovs, iovcnt); 602 _iov_iter_init(&dst_iter, bounce_iov, 1); 603 604 data_block_size = ctx->block_size - ctx->md_size; 605 606 while (offset_blocks < num_blocks) { 607 608 _iov_iter_get_buf(&src_iter, &src, NULL); 609 _iov_iter_get_buf(&dst_iter, &dst, NULL); 610 611 guard = 0; 612 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 613 guard = spdk_crc16_t10dif_copy(0, dst, src, data_block_size); 614 guard = spdk_crc16_t10dif(guard, dst + data_block_size, 615 ctx->guard_interval - data_block_size); 616 } else { 617 memcpy(dst, src, data_block_size); 618 } 619 620 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 621 622 _iov_iter_advance(&src_iter, data_block_size); 623 _iov_iter_advance(&dst_iter, ctx->block_size); 624 offset_blocks++; 625 } 626 } 627 628 static void 629 _dif_generate_copy_split(struct _iov_iter *src_iter, struct _iov_iter *dst_iter, 630 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 631 { 632 uint32_t offset_in_block, src_len, data_block_size; 633 uint16_t guard; 634 void *src, *dst; 635 636 _iov_iter_get_buf(dst_iter, &dst, NULL); 637 638 data_block_size = ctx->block_size - ctx->md_size; 639 640 guard = 0; 641 offset_in_block = 0; 642 643 while (offset_in_block < data_block_size) { 644 /* Compute CRC over split logical block data and copy 645 * data to bounce buffer. 646 */ 647 _iov_iter_get_buf(src_iter, &src, &src_len); 648 src_len = spdk_min(src_len, data_block_size - offset_in_block); 649 650 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 651 guard = spdk_crc16_t10dif_copy(guard, dst + offset_in_block, 652 src, src_len); 653 } else { 654 memcpy(dst + offset_in_block, src, src_len); 655 } 656 657 _iov_iter_advance(src_iter, src_len); 658 offset_in_block += src_len; 659 } 660 661 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 662 guard = spdk_crc16_t10dif(guard, dst + data_block_size, 663 ctx->guard_interval - data_block_size); 664 } 665 666 _iov_iter_advance(dst_iter, ctx->block_size); 667 668 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 669 } 670 671 static void 672 dif_generate_copy_split(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 673 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 674 { 675 struct _iov_iter src_iter, dst_iter; 676 uint32_t offset_blocks; 677 678 offset_blocks = 0; 679 _iov_iter_init(&src_iter, iovs, iovcnt); 680 _iov_iter_init(&dst_iter, bounce_iov, 1); 681 682 while (offset_blocks < num_blocks) { 683 _dif_generate_copy_split(&src_iter, &dst_iter, offset_blocks, ctx); 684 offset_blocks++; 685 } 686 } 687 688 int 689 spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 690 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 691 { 692 uint32_t data_block_size; 693 694 data_block_size = ctx->block_size - ctx->md_size; 695 696 if (!_are_iovs_valid(iovs, iovcnt, data_block_size * num_blocks) || 697 !_are_iovs_valid(bounce_iov, 1, ctx->block_size * num_blocks)) { 698 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 699 return -EINVAL; 700 } 701 702 if (_dif_is_disabled(ctx->dif_type)) { 703 return 0; 704 } 705 706 if (_are_iovs_bytes_multiple(iovs, iovcnt, data_block_size)) { 707 dif_generate_copy(iovs, iovcnt, bounce_iov, num_blocks, ctx); 708 } else { 709 dif_generate_copy_split(iovs, iovcnt, bounce_iov, num_blocks, ctx); 710 } 711 712 return 0; 713 } 714 715 static int 716 dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 717 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 718 struct spdk_dif_error *err_blk) 719 { 720 struct _iov_iter src_iter, dst_iter; 721 uint32_t offset_blocks, data_block_size; 722 void *src, *dst; 723 int rc; 724 uint16_t guard; 725 726 offset_blocks = 0; 727 _iov_iter_init(&src_iter, bounce_iov, 1); 728 _iov_iter_init(&dst_iter, iovs, iovcnt); 729 730 data_block_size = ctx->block_size - ctx->md_size; 731 732 while (offset_blocks < num_blocks) { 733 734 _iov_iter_get_buf(&src_iter, &src, NULL); 735 _iov_iter_get_buf(&dst_iter, &dst, NULL); 736 737 guard = 0; 738 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 739 guard = spdk_crc16_t10dif_copy(0, dst, src, data_block_size); 740 guard = spdk_crc16_t10dif(guard, src + data_block_size, 741 ctx->guard_interval - data_block_size); 742 } else { 743 memcpy(dst, src, data_block_size); 744 } 745 746 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 747 if (rc != 0) { 748 return rc; 749 } 750 751 _iov_iter_advance(&src_iter, ctx->block_size); 752 _iov_iter_advance(&dst_iter, data_block_size); 753 offset_blocks++; 754 } 755 756 return 0; 757 } 758 759 static int 760 _dif_verify_copy_split(struct _iov_iter *src_iter, struct _iov_iter *dst_iter, 761 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 762 struct spdk_dif_error *err_blk) 763 { 764 uint32_t offset_in_block, dst_len, data_block_size; 765 uint16_t guard; 766 void *src, *dst; 767 768 _iov_iter_get_buf(src_iter, &src, NULL); 769 770 data_block_size = ctx->block_size - ctx->md_size; 771 772 guard = 0; 773 offset_in_block = 0; 774 775 while (offset_in_block < data_block_size) { 776 /* Compute CRC over split logical block data and copy 777 * data to bounce buffer. 778 */ 779 _iov_iter_get_buf(dst_iter, &dst, &dst_len); 780 dst_len = spdk_min(dst_len, data_block_size - offset_in_block); 781 782 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 783 guard = spdk_crc16_t10dif_copy(guard, dst, 784 src + offset_in_block, dst_len); 785 } else { 786 memcpy(dst, src + offset_in_block, dst_len); 787 } 788 789 _iov_iter_advance(dst_iter, dst_len); 790 offset_in_block += dst_len; 791 } 792 793 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 794 guard = spdk_crc16_t10dif(guard, src + data_block_size, 795 ctx->guard_interval - data_block_size); 796 } 797 798 _iov_iter_advance(src_iter, ctx->block_size); 799 800 return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 801 } 802 803 static int 804 dif_verify_copy_split(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 805 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 806 struct spdk_dif_error *err_blk) 807 { 808 struct _iov_iter src_iter, dst_iter; 809 uint32_t offset_blocks; 810 int rc; 811 812 offset_blocks = 0; 813 _iov_iter_init(&src_iter, bounce_iov, 1); 814 _iov_iter_init(&dst_iter, iovs, iovcnt); 815 816 while (offset_blocks < num_blocks) { 817 rc = _dif_verify_copy_split(&src_iter, &dst_iter, offset_blocks, ctx, err_blk); 818 if (rc != 0) { 819 return rc; 820 } 821 offset_blocks++; 822 } 823 824 return 0; 825 } 826 827 int 828 spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 829 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 830 struct spdk_dif_error *err_blk) 831 { 832 uint32_t data_block_size; 833 834 data_block_size = ctx->block_size - ctx->md_size; 835 836 if (!_are_iovs_valid(iovs, iovcnt, data_block_size * num_blocks) || 837 !_are_iovs_valid(bounce_iov, 1, ctx->block_size * num_blocks)) { 838 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 839 return -EINVAL; 840 } 841 842 if (_dif_is_disabled(ctx->dif_type)) { 843 return 0; 844 } 845 846 if (_are_iovs_bytes_multiple(iovs, iovcnt, data_block_size)) { 847 return dif_verify_copy(iovs, iovcnt, bounce_iov, num_blocks, ctx, err_blk); 848 } else { 849 return dif_verify_copy_split(iovs, iovcnt, bounce_iov, num_blocks, ctx, err_blk); 850 } 851 } 852 853 static void 854 _bit_flip(uint8_t *buf, uint32_t flip_bit) 855 { 856 uint8_t byte; 857 858 byte = *buf; 859 byte ^= 1 << flip_bit; 860 *buf = byte; 861 } 862 863 static int 864 _dif_inject_error(struct iovec *iovs, int iovcnt, 865 uint32_t block_size, uint32_t num_blocks, 866 uint32_t inject_offset_blocks, 867 uint32_t inject_offset_bytes, 868 uint32_t inject_offset_bits) 869 { 870 struct _iov_iter iter; 871 uint32_t offset_in_block, buf_len; 872 void *buf; 873 874 _iov_iter_init(&iter, iovs, iovcnt); 875 876 _iov_iter_fast_forward(&iter, block_size * inject_offset_blocks); 877 878 offset_in_block = 0; 879 880 while (offset_in_block < block_size) { 881 _iov_iter_get_buf(&iter, &buf, &buf_len); 882 buf_len = spdk_min(buf_len, block_size - offset_in_block); 883 884 if (inject_offset_bytes >= offset_in_block && 885 inject_offset_bytes < offset_in_block + buf_len) { 886 buf += inject_offset_bytes - offset_in_block; 887 _bit_flip(buf, inject_offset_bits); 888 return 0; 889 } 890 891 _iov_iter_advance(&iter, buf_len); 892 offset_in_block += buf_len; 893 } 894 895 return -1; 896 } 897 898 static int 899 dif_inject_error(struct iovec *iovs, int iovcnt, 900 uint32_t block_size, uint32_t num_blocks, 901 uint32_t start_inject_bytes, uint32_t inject_range_bytes, 902 uint32_t *inject_offset) 903 { 904 uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; 905 uint32_t offset_blocks; 906 int rc; 907 908 srand(time(0)); 909 910 inject_offset_blocks = rand() % num_blocks; 911 inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); 912 inject_offset_bits = rand() % 8; 913 914 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 915 if (offset_blocks == inject_offset_blocks) { 916 rc = _dif_inject_error(iovs, iovcnt, block_size, num_blocks, 917 inject_offset_blocks, 918 inject_offset_bytes, 919 inject_offset_bits); 920 if (rc == 0) { 921 *inject_offset = inject_offset_blocks; 922 } 923 return rc; 924 } 925 } 926 927 return -1; 928 } 929 930 #define _member_size(type, member) sizeof(((type *)0)->member) 931 932 int 933 spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 934 const struct spdk_dif_ctx *ctx, uint32_t inject_flags, 935 uint32_t *inject_offset) 936 { 937 int rc; 938 939 if (!_are_iovs_valid(iovs, iovcnt, ctx->block_size * num_blocks)) { 940 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 941 return -EINVAL; 942 } 943 944 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 945 rc = dif_inject_error(iovs, iovcnt, ctx->block_size, num_blocks, 946 ctx->guard_interval + offsetof(struct spdk_dif, ref_tag), 947 _member_size(struct spdk_dif, ref_tag), 948 inject_offset); 949 if (rc != 0) { 950 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 951 return rc; 952 } 953 } 954 955 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 956 rc = dif_inject_error(iovs, iovcnt, ctx->block_size, num_blocks, 957 ctx->guard_interval + offsetof(struct spdk_dif, app_tag), 958 _member_size(struct spdk_dif, app_tag), 959 inject_offset); 960 if (rc != 0) { 961 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 962 return rc; 963 } 964 } 965 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 966 rc = dif_inject_error(iovs, iovcnt, ctx->block_size, num_blocks, 967 ctx->guard_interval, 968 _member_size(struct spdk_dif, guard), 969 inject_offset); 970 if (rc != 0) { 971 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 972 return rc; 973 } 974 } 975 976 if (inject_flags & SPDK_DIF_DATA_ERROR) { 977 /* If the DIF information is contained within the last 8 bytes of 978 * metadata, then the CRC covers all metadata bytes up to but excluding 979 * the last 8 bytes. But error injection does not cover these metadata 980 * because classification is not determined yet. 981 * 982 * Note: Error injection to data block is expected to be detected as 983 * guard error. 984 */ 985 rc = dif_inject_error(iovs, iovcnt, ctx->block_size, num_blocks, 986 0, 987 ctx->block_size - ctx->md_size, 988 inject_offset); 989 if (rc != 0) { 990 SPDK_ERRLOG("Failed to inject error to data block.\n"); 991 return rc; 992 } 993 } 994 995 return 0; 996 } 997 998 static void 999 dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1000 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1001 { 1002 struct _iov_iter data_iter, md_iter; 1003 uint32_t offset_blocks; 1004 uint16_t guard; 1005 void *data_buf, *md_buf; 1006 1007 offset_blocks = 0; 1008 _iov_iter_init(&data_iter, iovs, iovcnt); 1009 _iov_iter_init(&md_iter, md_iov, 1); 1010 1011 while (offset_blocks < num_blocks) { 1012 1013 _iov_iter_get_buf(&data_iter, &data_buf, NULL); 1014 _iov_iter_get_buf(&md_iter, &md_buf, NULL); 1015 1016 guard = 0; 1017 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1018 guard = spdk_crc16_t10dif(0, data_buf, ctx->block_size); 1019 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1020 } 1021 1022 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1023 1024 _iov_iter_advance(&data_iter, ctx->block_size); 1025 _iov_iter_advance(&md_iter, ctx->md_size); 1026 offset_blocks++; 1027 } 1028 } 1029 1030 static void 1031 _dix_generate_split(struct _iov_iter *data_iter, struct _iov_iter *md_iter, 1032 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1033 { 1034 uint32_t offset_in_block, data_buf_len; 1035 uint16_t guard; 1036 void *data_buf, *md_buf; 1037 1038 _iov_iter_get_buf(md_iter, &md_buf, NULL); 1039 1040 guard = 0; 1041 offset_in_block = 0; 1042 1043 while (offset_in_block < ctx->block_size) { 1044 _iov_iter_get_buf(data_iter, &data_buf, &data_buf_len); 1045 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1046 1047 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1048 guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len); 1049 } 1050 1051 _iov_iter_advance(data_iter, data_buf_len); 1052 offset_in_block += data_buf_len; 1053 } 1054 1055 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1056 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1057 } 1058 1059 _iov_iter_advance(md_iter, ctx->md_size); 1060 1061 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1062 } 1063 1064 static void 1065 dix_generate_split(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1066 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1067 { 1068 struct _iov_iter data_iter, md_iter; 1069 uint32_t offset_blocks; 1070 1071 offset_blocks = 0; 1072 _iov_iter_init(&data_iter, iovs, iovcnt); 1073 _iov_iter_init(&md_iter, md_iov, 1); 1074 1075 while (offset_blocks < num_blocks) { 1076 _dix_generate_split(&data_iter, &md_iter, offset_blocks, ctx); 1077 offset_blocks++; 1078 } 1079 } 1080 1081 int 1082 spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1083 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1084 { 1085 if (!_are_iovs_valid(iovs, iovcnt, ctx->block_size * num_blocks) || 1086 !_are_iovs_valid(md_iov, 1, ctx->md_size * num_blocks)) { 1087 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1088 return -EINVAL; 1089 } 1090 1091 if (_dif_is_disabled(ctx->dif_type)) { 1092 return 0; 1093 } 1094 1095 if (_are_iovs_bytes_multiple(iovs, iovcnt, ctx->block_size)) { 1096 dix_generate(iovs, iovcnt, md_iov, num_blocks, ctx); 1097 } else { 1098 dix_generate_split(iovs, iovcnt, md_iov, num_blocks, ctx); 1099 } 1100 1101 return 0; 1102 } 1103 1104 static int 1105 dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1106 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1107 struct spdk_dif_error *err_blk) 1108 { 1109 struct _iov_iter data_iter, md_iter; 1110 uint32_t offset_blocks; 1111 uint16_t guard; 1112 void *data_buf, *md_buf; 1113 int rc; 1114 1115 offset_blocks = 0; 1116 _iov_iter_init(&data_iter, iovs, iovcnt); 1117 _iov_iter_init(&md_iter, md_iov, 1); 1118 1119 while (offset_blocks < num_blocks) { 1120 1121 _iov_iter_get_buf(&data_iter, &data_buf, NULL); 1122 _iov_iter_get_buf(&md_iter, &md_buf, NULL); 1123 1124 guard = 0; 1125 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1126 guard = spdk_crc16_t10dif(0, data_buf, ctx->block_size); 1127 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1128 } 1129 1130 rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1131 if (rc != 0) { 1132 return rc; 1133 } 1134 1135 _iov_iter_advance(&data_iter, ctx->block_size); 1136 _iov_iter_advance(&md_iter, ctx->md_size); 1137 offset_blocks++; 1138 } 1139 1140 return 0; 1141 } 1142 1143 static int 1144 _dix_verify_split(struct _iov_iter *data_iter, struct _iov_iter *md_iter, 1145 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1146 struct spdk_dif_error *err_blk) 1147 { 1148 uint32_t offset_in_block, data_buf_len; 1149 uint16_t guard; 1150 void *data_buf, *md_buf; 1151 1152 _iov_iter_get_buf(md_iter, &md_buf, NULL); 1153 1154 guard = 0; 1155 offset_in_block = 0; 1156 1157 while (offset_in_block < ctx->block_size) { 1158 _iov_iter_get_buf(data_iter, &data_buf, &data_buf_len); 1159 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1160 1161 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1162 guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len); 1163 } 1164 1165 _iov_iter_advance(data_iter, data_buf_len); 1166 offset_in_block += data_buf_len; 1167 } 1168 1169 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1170 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1171 } 1172 1173 _iov_iter_advance(md_iter, ctx->md_size); 1174 1175 return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1176 } 1177 1178 static int 1179 dix_verify_split(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1180 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1181 struct spdk_dif_error *err_blk) 1182 { 1183 struct _iov_iter data_iter, md_iter; 1184 uint32_t offset_blocks; 1185 int rc; 1186 1187 offset_blocks = 0; 1188 _iov_iter_init(&data_iter, iovs, iovcnt); 1189 _iov_iter_init(&md_iter, md_iov, 1); 1190 1191 while (offset_blocks < num_blocks) { 1192 rc = _dix_verify_split(&data_iter, &md_iter, offset_blocks, ctx, err_blk); 1193 if (rc != 0) { 1194 return rc; 1195 } 1196 offset_blocks++; 1197 } 1198 1199 return 0; 1200 } 1201 1202 int 1203 spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1204 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1205 struct spdk_dif_error *err_blk) 1206 { 1207 if (!_are_iovs_valid(iovs, iovcnt, ctx->block_size * num_blocks) || 1208 !_are_iovs_valid(md_iov, 1, ctx->md_size * num_blocks)) { 1209 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1210 return -EINVAL; 1211 } 1212 1213 if (_dif_is_disabled(ctx->dif_type)) { 1214 return 0; 1215 } 1216 1217 if (_are_iovs_bytes_multiple(iovs, iovcnt, ctx->block_size)) { 1218 return dix_verify(iovs, iovcnt, md_iov, num_blocks, ctx, err_blk); 1219 } else { 1220 return dix_verify_split(iovs, iovcnt, md_iov, num_blocks, ctx, err_blk); 1221 } 1222 } 1223 1224 int 1225 spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1226 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1227 uint32_t inject_flags, uint32_t *inject_offset) 1228 { 1229 int rc; 1230 1231 if (!_are_iovs_valid(iovs, iovcnt, ctx->block_size * num_blocks) || 1232 !_are_iovs_valid(md_iov, 1, ctx->md_size * num_blocks)) { 1233 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1234 return -EINVAL; 1235 } 1236 1237 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1238 rc = dif_inject_error(md_iov, 1, ctx->md_size, num_blocks, 1239 ctx->guard_interval + offsetof(struct spdk_dif, ref_tag), 1240 _member_size(struct spdk_dif, ref_tag), 1241 inject_offset); 1242 if (rc != 0) { 1243 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1244 return rc; 1245 } 1246 } 1247 1248 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1249 rc = dif_inject_error(md_iov, 1, ctx->md_size, num_blocks, 1250 ctx->guard_interval + offsetof(struct spdk_dif, app_tag), 1251 _member_size(struct spdk_dif, app_tag), 1252 inject_offset); 1253 if (rc != 0) { 1254 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1255 return rc; 1256 } 1257 } 1258 1259 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1260 rc = dif_inject_error(md_iov, 1, ctx->md_size, num_blocks, 1261 ctx->guard_interval, 1262 _member_size(struct spdk_dif, guard), 1263 inject_offset); 1264 if (rc != 0) { 1265 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1266 return rc; 1267 } 1268 } 1269 1270 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1271 /* Note: Error injection to data block is expected to be detected 1272 * as guard error. 1273 */ 1274 rc = dif_inject_error(iovs, iovcnt, ctx->block_size, num_blocks, 1275 0, 1276 ctx->block_size, 1277 inject_offset); 1278 if (rc != 0) { 1279 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1280 return rc; 1281 } 1282 } 1283 1284 return 0; 1285 } 1286