1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/dif.h" 35 #include "spdk/crc16.h" 36 #include "spdk/crc32.h" 37 #include "spdk/endian.h" 38 #include "spdk/log.h" 39 #include "spdk/util.h" 40 41 /* Context to iterate or create a iovec array. 42 * Each sgl is either iterated or created at a time. 43 */ 44 struct _dif_sgl { 45 /* Current iovec in the iteration or creation */ 46 struct iovec *iov; 47 48 /* Remaining count of iovecs in the iteration or creation. */ 49 int iovcnt; 50 51 /* Current offset in the iovec */ 52 uint32_t iov_offset; 53 54 /* Size of the created iovec array in bytes */ 55 uint32_t total_size; 56 }; 57 58 static inline void 59 _dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt) 60 { 61 s->iov = iovs; 62 s->iovcnt = iovcnt; 63 s->iov_offset = 0; 64 s->total_size = 0; 65 } 66 67 static void 68 _dif_sgl_advance(struct _dif_sgl *s, uint32_t step) 69 { 70 s->iov_offset += step; 71 while (s->iovcnt != 0) { 72 if (s->iov_offset < s->iov->iov_len) { 73 break; 74 } 75 76 s->iov_offset -= s->iov->iov_len; 77 s->iov++; 78 s->iovcnt--; 79 } 80 } 81 82 static inline void 83 _dif_sgl_get_buf(struct _dif_sgl *s, void **_buf, uint32_t *_buf_len) 84 { 85 if (_buf != NULL) { 86 *_buf = s->iov->iov_base + s->iov_offset; 87 } 88 if (_buf_len != NULL) { 89 *_buf_len = s->iov->iov_len - s->iov_offset; 90 } 91 } 92 93 static inline bool 94 _dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len) 95 { 96 assert(s->iovcnt > 0); 97 s->iov->iov_base = data; 98 s->iov->iov_len = data_len; 99 s->total_size += data_len; 100 s->iov++; 101 s->iovcnt--; 102 103 if (s->iovcnt > 0) { 104 return true; 105 } else { 106 return false; 107 } 108 } 109 110 static inline bool 111 _dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len) 112 { 113 uint8_t *buf; 114 uint32_t buf_len; 115 116 while (data_len != 0) { 117 _dif_sgl_get_buf(src, (void *)&buf, &buf_len); 118 buf_len = spdk_min(buf_len, data_len); 119 120 if (!_dif_sgl_append(dst, buf, buf_len)) { 121 return false; 122 } 123 124 _dif_sgl_advance(src, buf_len); 125 data_len -= buf_len; 126 } 127 128 return true; 129 } 130 131 /* This function must be used before starting iteration. */ 132 static bool 133 _dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes) 134 { 135 int i; 136 137 for (i = 0; i < s->iovcnt; i++) { 138 if (s->iov[i].iov_len % bytes) { 139 return false; 140 } 141 } 142 143 return true; 144 } 145 146 /* This function must be used before starting iteration. */ 147 static bool 148 _dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes) 149 { 150 uint64_t total = 0; 151 int i; 152 153 for (i = 0; i < s->iovcnt; i++) { 154 total += s->iov[i].iov_len; 155 } 156 157 return total >= bytes; 158 } 159 160 static bool 161 _dif_type_is_valid(enum spdk_dif_type dif_type, uint32_t dif_flags) 162 { 163 switch (dif_type) { 164 case SPDK_DIF_TYPE1: 165 case SPDK_DIF_TYPE2: 166 case SPDK_DIF_DISABLE: 167 break; 168 case SPDK_DIF_TYPE3: 169 if (dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 170 SPDK_ERRLOG("Reference Tag should not be checked for Type 3\n"); 171 return false; 172 } 173 break; 174 default: 175 SPDK_ERRLOG("Unknown DIF Type: %d\n", dif_type); 176 return false; 177 } 178 179 return true; 180 } 181 182 static bool 183 _dif_is_disabled(enum spdk_dif_type dif_type) 184 { 185 if (dif_type == SPDK_DIF_DISABLE) { 186 return true; 187 } else { 188 return false; 189 } 190 } 191 192 193 static uint32_t 194 _get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave) 195 { 196 if (!dif_loc) { 197 /* For metadata formats with more than 8 bytes, if the DIF is 198 * contained in the last 8 bytes of metadata, then the CRC 199 * covers all metadata up to but excluding these last 8 bytes. 200 */ 201 if (md_interleave) { 202 return block_size - sizeof(struct spdk_dif); 203 } else { 204 return md_size - sizeof(struct spdk_dif); 205 } 206 } else { 207 /* For metadata formats with more than 8 bytes, if the DIF is 208 * contained in the first 8 bytes of metadata, then the CRC 209 * does not cover any metadata. 210 */ 211 if (md_interleave) { 212 return block_size - md_size; 213 } else { 214 return 0; 215 } 216 } 217 } 218 219 int 220 spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, 221 bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, 222 uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag, 223 uint32_t data_offset, uint16_t guard_seed) 224 { 225 uint32_t data_block_size; 226 227 if (md_size < sizeof(struct spdk_dif)) { 228 SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); 229 return -EINVAL; 230 } 231 232 if (md_interleave) { 233 if (block_size < md_size) { 234 SPDK_ERRLOG("Block size is smaller than DIF size.\n"); 235 return -EINVAL; 236 } 237 data_block_size = block_size - md_size; 238 } else { 239 if (block_size == 0 || (block_size % 512) != 0) { 240 SPDK_ERRLOG("Zero block size is not allowed\n"); 241 return -EINVAL; 242 } 243 data_block_size = block_size; 244 } 245 246 if (!_dif_type_is_valid(dif_type, dif_flags)) { 247 SPDK_ERRLOG("DIF type is invalid.\n"); 248 return -EINVAL; 249 } 250 251 ctx->block_size = block_size; 252 ctx->md_size = md_size; 253 ctx->md_interleave = md_interleave; 254 ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave); 255 ctx->dif_type = dif_type; 256 ctx->dif_flags = dif_flags; 257 ctx->init_ref_tag = init_ref_tag; 258 ctx->apptag_mask = apptag_mask; 259 ctx->app_tag = app_tag; 260 ctx->data_offset = data_offset; 261 ctx->ref_tag_offset = data_offset / data_block_size; 262 ctx->last_guard = guard_seed; 263 ctx->guard_seed = guard_seed; 264 265 return 0; 266 } 267 268 void 269 spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset) 270 { 271 uint32_t data_block_size; 272 273 if (ctx->md_interleave) { 274 data_block_size = ctx->block_size - ctx->md_size; 275 } else { 276 data_block_size = ctx->block_size; 277 } 278 279 ctx->data_offset = data_offset; 280 ctx->ref_tag_offset = data_offset / data_block_size; 281 } 282 283 static void 284 _dif_generate(void *_dif, uint16_t guard, uint32_t offset_blocks, 285 const struct spdk_dif_ctx *ctx) 286 { 287 struct spdk_dif *dif = _dif; 288 uint32_t ref_tag; 289 290 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 291 to_be16(&dif->guard, guard); 292 } 293 294 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 295 to_be16(&dif->app_tag, ctx->app_tag); 296 } 297 298 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 299 /* For type 1 and 2, the reference tag is incremented for each 300 * subsequent logical block. For type 3, the reference tag 301 * remains the same as the initial reference tag. 302 */ 303 if (ctx->dif_type != SPDK_DIF_TYPE3) { 304 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 305 } else { 306 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 307 } 308 309 to_be32(&dif->ref_tag, ref_tag); 310 } 311 } 312 313 static void 314 dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 315 { 316 uint32_t offset_blocks = 0; 317 void *buf; 318 uint16_t guard = 0; 319 320 while (offset_blocks < num_blocks) { 321 _dif_sgl_get_buf(sgl, &buf, NULL); 322 323 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 324 guard = spdk_crc16_t10dif(ctx->guard_seed, buf, ctx->guard_interval); 325 } 326 327 _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); 328 329 _dif_sgl_advance(sgl, ctx->block_size); 330 offset_blocks++; 331 } 332 } 333 334 static uint16_t 335 _dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 336 uint16_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 337 { 338 uint32_t offset_in_dif, buf_len; 339 void *buf; 340 struct spdk_dif dif = {}; 341 342 assert(offset_in_block < ctx->guard_interval); 343 assert(offset_in_block + data_len < ctx->guard_interval || 344 offset_in_block + data_len == ctx->block_size); 345 346 /* Compute CRC over split logical block data. */ 347 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 348 _dif_sgl_get_buf(sgl, &buf, &buf_len); 349 buf_len = spdk_min(buf_len, data_len); 350 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 351 352 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 353 guard = spdk_crc16_t10dif(guard, buf, buf_len); 354 } 355 356 _dif_sgl_advance(sgl, buf_len); 357 offset_in_block += buf_len; 358 data_len -= buf_len; 359 } 360 361 if (offset_in_block < ctx->guard_interval) { 362 return guard; 363 } 364 365 /* If a whole logical block data is parsed, generate DIF 366 * and save it to the temporary DIF area. 367 */ 368 _dif_generate(&dif, guard, offset_blocks, ctx); 369 370 /* Copy generated DIF field to the split DIF field, and then 371 * skip metadata field after DIF field (if any). 372 */ 373 while (offset_in_block < ctx->block_size) { 374 _dif_sgl_get_buf(sgl, &buf, &buf_len); 375 376 if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) { 377 offset_in_dif = offset_in_block - ctx->guard_interval; 378 buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif); 379 380 memcpy(buf, ((uint8_t *)&dif) + offset_in_dif, buf_len); 381 } else { 382 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 383 } 384 385 _dif_sgl_advance(sgl, buf_len); 386 offset_in_block += buf_len; 387 } 388 389 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 390 guard = ctx->guard_seed; 391 } 392 393 return guard; 394 } 395 396 static void 397 dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks, 398 const struct spdk_dif_ctx *ctx) 399 { 400 uint32_t offset_blocks; 401 uint16_t guard = 0; 402 403 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 404 guard = ctx->guard_seed; 405 } 406 407 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 408 _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx); 409 } 410 } 411 412 int 413 spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 414 const struct spdk_dif_ctx *ctx) 415 { 416 struct _dif_sgl sgl; 417 418 _dif_sgl_init(&sgl, iovs, iovcnt); 419 420 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 421 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 422 return -EINVAL; 423 } 424 425 if (_dif_is_disabled(ctx->dif_type)) { 426 return 0; 427 } 428 429 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 430 dif_generate(&sgl, num_blocks, ctx); 431 } else { 432 dif_generate_split(&sgl, num_blocks, ctx); 433 } 434 435 return 0; 436 } 437 438 static void 439 _dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, 440 uint32_t expected, uint32_t actual, uint32_t err_offset) 441 { 442 if (err_blk) { 443 err_blk->err_type = err_type; 444 err_blk->expected = expected; 445 err_blk->actual = actual; 446 err_blk->err_offset = err_offset; 447 } 448 } 449 450 static int 451 _dif_verify(void *_dif, uint16_t guard, uint32_t offset_blocks, 452 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 453 { 454 struct spdk_dif *dif = _dif; 455 uint16_t _guard; 456 uint16_t _app_tag; 457 uint32_t ref_tag, _ref_tag; 458 459 switch (ctx->dif_type) { 460 case SPDK_DIF_TYPE1: 461 case SPDK_DIF_TYPE2: 462 /* If Type 1 or 2 is used, then all DIF checks are disabled when 463 * the Application Tag is 0xFFFF. 464 */ 465 if (dif->app_tag == 0xFFFF) { 466 return 0; 467 } 468 break; 469 case SPDK_DIF_TYPE3: 470 /* If Type 3 is used, then all DIF checks are disabled when the 471 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF. 472 */ 473 if (dif->app_tag == 0xFFFF && dif->ref_tag == 0xFFFFFFFF) { 474 return 0; 475 } 476 break; 477 default: 478 break; 479 } 480 481 /* For type 1 and 2, the reference tag is incremented for each 482 * subsequent logical block. For type 3, the reference tag 483 * remains the same as the initial reference tag. 484 */ 485 if (ctx->dif_type != SPDK_DIF_TYPE3) { 486 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 487 } else { 488 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 489 } 490 491 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 492 /* Compare the DIF Guard field to the CRC computed over the logical 493 * block data. 494 */ 495 _guard = from_be16(&dif->guard); 496 if (_guard != guard) { 497 _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, 498 offset_blocks); 499 SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu32 "," \ 500 " Expected=%x, Actual=%x\n", 501 ref_tag, _guard, guard); 502 return -1; 503 } 504 } 505 506 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 507 /* Compare unmasked bits in the DIF Application Tag field to the 508 * passed Application Tag. 509 */ 510 _app_tag = from_be16(&dif->app_tag); 511 if ((_app_tag & ctx->apptag_mask) != ctx->app_tag) { 512 _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, 513 (_app_tag & ctx->apptag_mask), offset_blocks); 514 SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu32 "," \ 515 " Expected=%x, Actual=%x\n", 516 ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); 517 return -1; 518 } 519 } 520 521 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 522 switch (ctx->dif_type) { 523 case SPDK_DIF_TYPE1: 524 case SPDK_DIF_TYPE2: 525 /* Compare the DIF Reference Tag field to the passed Reference Tag. 526 * The passed Reference Tag will be the least significant 4 bytes 527 * of the LBA when Type 1 is used, and application specific value 528 * if Type 2 is used, 529 */ 530 _ref_tag = from_be32(&dif->ref_tag); 531 if (_ref_tag != ref_tag) { 532 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, ref_tag, 533 _ref_tag, offset_blocks); 534 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu32 "," \ 535 " Expected=%x, Actual=%x\n", 536 ref_tag, ref_tag, _ref_tag); 537 return -1; 538 } 539 break; 540 case SPDK_DIF_TYPE3: 541 /* For Type 3, computed Reference Tag remains unchanged. 542 * Hence ignore the Reference Tag field. 543 */ 544 break; 545 default: 546 break; 547 } 548 } 549 550 return 0; 551 } 552 553 static int 554 dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks, 555 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 556 { 557 uint32_t offset_blocks = 0; 558 int rc; 559 void *buf; 560 uint16_t guard = 0; 561 562 while (offset_blocks < num_blocks) { 563 _dif_sgl_get_buf(sgl, &buf, NULL); 564 565 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 566 guard = spdk_crc16_t10dif(ctx->guard_seed, buf, ctx->guard_interval); 567 } 568 569 rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 570 if (rc != 0) { 571 return rc; 572 } 573 574 _dif_sgl_advance(sgl, ctx->block_size); 575 offset_blocks++; 576 } 577 578 return 0; 579 } 580 581 static int 582 _dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_blocks, 583 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 584 { 585 uint32_t offset_in_block, offset_in_dif, buf_len; 586 void *buf; 587 uint16_t guard = 0; 588 struct spdk_dif dif = {}; 589 590 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 591 guard = ctx->guard_seed; 592 } 593 offset_in_block = 0; 594 595 while (offset_in_block < ctx->block_size) { 596 _dif_sgl_get_buf(sgl, &buf, &buf_len); 597 598 if (offset_in_block < ctx->guard_interval) { 599 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 600 601 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 602 /* Compute CRC over split logical block data. */ 603 guard = spdk_crc16_t10dif(guard, buf, buf_len); 604 } 605 } else if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) { 606 /* Copy the split DIF field to the temporary DIF buffer. */ 607 offset_in_dif = offset_in_block - ctx->guard_interval; 608 buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif); 609 610 memcpy((uint8_t *)&dif + offset_in_dif, buf, buf_len); 611 } else { 612 /* Skip metadata field after DIF field. */ 613 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 614 } 615 616 _dif_sgl_advance(sgl, buf_len); 617 offset_in_block += buf_len; 618 } 619 620 return _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 621 } 622 623 static int 624 dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks, 625 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 626 { 627 uint32_t offset_blocks; 628 int rc; 629 630 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 631 rc = _dif_verify_split(sgl, offset_blocks, ctx, err_blk); 632 if (rc != 0) { 633 return rc; 634 } 635 } 636 637 return 0; 638 } 639 640 int 641 spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 642 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 643 { 644 struct _dif_sgl sgl; 645 646 _dif_sgl_init(&sgl, iovs, iovcnt); 647 648 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 649 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 650 return -EINVAL; 651 } 652 653 if (_dif_is_disabled(ctx->dif_type)) { 654 return 0; 655 } 656 657 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 658 return dif_verify(&sgl, num_blocks, ctx, err_blk); 659 } else { 660 return dif_verify_split(&sgl, num_blocks, ctx, err_blk); 661 } 662 } 663 664 static uint32_t 665 dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks, 666 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 667 { 668 uint32_t offset_blocks; 669 void *buf; 670 671 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 672 _dif_sgl_get_buf(sgl, &buf, NULL); 673 674 crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c); 675 676 _dif_sgl_advance(sgl, ctx->block_size); 677 } 678 679 return crc32c; 680 } 681 682 static uint32_t 683 _dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t crc32c, 684 const struct spdk_dif_ctx *ctx) 685 { 686 uint32_t data_block_size, offset_in_block, buf_len; 687 void *buf; 688 689 data_block_size = ctx->block_size - ctx->md_size; 690 offset_in_block = 0; 691 692 while (offset_in_block < ctx->block_size) { 693 _dif_sgl_get_buf(sgl, &buf, &buf_len); 694 695 if (offset_in_block < data_block_size) { 696 buf_len = spdk_min(buf_len, data_block_size - offset_in_block); 697 crc32c = spdk_crc32c_update(buf, buf_len, crc32c); 698 } else { 699 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 700 } 701 702 _dif_sgl_advance(sgl, buf_len); 703 offset_in_block += buf_len; 704 } 705 706 return crc32c; 707 } 708 709 static uint32_t 710 dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks, 711 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 712 { 713 uint32_t offset_blocks; 714 715 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 716 crc32c = _dif_update_crc32c_split(sgl, crc32c, ctx); 717 } 718 719 return crc32c; 720 } 721 722 int 723 spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 724 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 725 { 726 struct _dif_sgl sgl; 727 728 if (_crc32c == NULL) { 729 return -EINVAL; 730 } 731 732 _dif_sgl_init(&sgl, iovs, iovcnt); 733 734 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 735 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 736 return -EINVAL; 737 } 738 739 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 740 *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx); 741 } else { 742 *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx); 743 } 744 745 return 0; 746 } 747 748 static void 749 dif_generate_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 750 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 751 { 752 uint32_t offset_blocks = 0, data_block_size; 753 void *src, *dst; 754 uint16_t guard; 755 756 data_block_size = ctx->block_size - ctx->md_size; 757 758 while (offset_blocks < num_blocks) { 759 _dif_sgl_get_buf(src_sgl, &src, NULL); 760 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 761 762 guard = 0; 763 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 764 guard = spdk_crc16_t10dif_copy(ctx->guard_seed, dst, src, data_block_size); 765 guard = spdk_crc16_t10dif(guard, dst + data_block_size, 766 ctx->guard_interval - data_block_size); 767 } else { 768 memcpy(dst, src, data_block_size); 769 } 770 771 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 772 773 _dif_sgl_advance(src_sgl, data_block_size); 774 _dif_sgl_advance(dst_sgl, ctx->block_size); 775 offset_blocks++; 776 } 777 } 778 779 static void 780 _dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 781 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 782 { 783 uint32_t offset_in_block, src_len, data_block_size; 784 uint16_t guard = 0; 785 void *src, *dst; 786 787 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 788 789 data_block_size = ctx->block_size - ctx->md_size; 790 791 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 792 guard = ctx->guard_seed; 793 } 794 offset_in_block = 0; 795 796 while (offset_in_block < data_block_size) { 797 /* Compute CRC over split logical block data and copy 798 * data to bounce buffer. 799 */ 800 _dif_sgl_get_buf(src_sgl, &src, &src_len); 801 src_len = spdk_min(src_len, data_block_size - offset_in_block); 802 803 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 804 guard = spdk_crc16_t10dif_copy(guard, dst + offset_in_block, 805 src, src_len); 806 } else { 807 memcpy(dst + offset_in_block, src, src_len); 808 } 809 810 _dif_sgl_advance(src_sgl, src_len); 811 offset_in_block += src_len; 812 } 813 814 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 815 guard = spdk_crc16_t10dif(guard, dst + data_block_size, 816 ctx->guard_interval - data_block_size); 817 } 818 819 _dif_sgl_advance(dst_sgl, ctx->block_size); 820 821 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 822 } 823 824 static void 825 dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 826 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 827 { 828 uint32_t offset_blocks; 829 830 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 831 _dif_generate_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 832 } 833 } 834 835 int 836 spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 837 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 838 { 839 struct _dif_sgl src_sgl, dst_sgl; 840 uint32_t data_block_size; 841 842 _dif_sgl_init(&src_sgl, iovs, iovcnt); 843 _dif_sgl_init(&dst_sgl, bounce_iov, 1); 844 845 data_block_size = ctx->block_size - ctx->md_size; 846 847 if (!_dif_sgl_is_valid(&src_sgl, data_block_size * num_blocks) || 848 !_dif_sgl_is_valid(&dst_sgl, ctx->block_size * num_blocks)) { 849 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 850 return -EINVAL; 851 } 852 853 if (_dif_is_disabled(ctx->dif_type)) { 854 return 0; 855 } 856 857 if (_dif_sgl_is_bytes_multiple(&src_sgl, data_block_size)) { 858 dif_generate_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 859 } else { 860 dif_generate_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx); 861 } 862 863 return 0; 864 } 865 866 static int 867 dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 868 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 869 struct spdk_dif_error *err_blk) 870 { 871 uint32_t offset_blocks = 0, data_block_size; 872 void *src, *dst; 873 int rc; 874 uint16_t guard; 875 876 data_block_size = ctx->block_size - ctx->md_size; 877 878 while (offset_blocks < num_blocks) { 879 _dif_sgl_get_buf(src_sgl, &src, NULL); 880 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 881 882 guard = 0; 883 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 884 guard = spdk_crc16_t10dif_copy(ctx->guard_seed, dst, src, data_block_size); 885 guard = spdk_crc16_t10dif(guard, src + data_block_size, 886 ctx->guard_interval - data_block_size); 887 } else { 888 memcpy(dst, src, data_block_size); 889 } 890 891 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 892 if (rc != 0) { 893 return rc; 894 } 895 896 _dif_sgl_advance(src_sgl, ctx->block_size); 897 _dif_sgl_advance(dst_sgl, data_block_size); 898 offset_blocks++; 899 } 900 901 return 0; 902 } 903 904 static int 905 _dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 906 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 907 struct spdk_dif_error *err_blk) 908 { 909 uint32_t offset_in_block, dst_len, data_block_size; 910 uint16_t guard = 0; 911 void *src, *dst; 912 913 _dif_sgl_get_buf(src_sgl, &src, NULL); 914 915 data_block_size = ctx->block_size - ctx->md_size; 916 917 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 918 guard = ctx->guard_seed; 919 } 920 offset_in_block = 0; 921 922 while (offset_in_block < data_block_size) { 923 /* Compute CRC over split logical block data and copy 924 * data to bounce buffer. 925 */ 926 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 927 dst_len = spdk_min(dst_len, data_block_size - offset_in_block); 928 929 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 930 guard = spdk_crc16_t10dif_copy(guard, dst, 931 src + offset_in_block, dst_len); 932 } else { 933 memcpy(dst, src + offset_in_block, dst_len); 934 } 935 936 _dif_sgl_advance(dst_sgl, dst_len); 937 offset_in_block += dst_len; 938 } 939 940 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 941 guard = spdk_crc16_t10dif(guard, src + data_block_size, 942 ctx->guard_interval - data_block_size); 943 } 944 945 _dif_sgl_advance(src_sgl, ctx->block_size); 946 947 return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 948 } 949 950 static int 951 dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 952 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 953 struct spdk_dif_error *err_blk) 954 { 955 uint32_t offset_blocks; 956 int rc; 957 958 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 959 rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 960 if (rc != 0) { 961 return rc; 962 } 963 } 964 965 return 0; 966 } 967 968 int 969 spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 970 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 971 struct spdk_dif_error *err_blk) 972 { 973 struct _dif_sgl src_sgl, dst_sgl; 974 uint32_t data_block_size; 975 976 _dif_sgl_init(&src_sgl, bounce_iov, 1); 977 _dif_sgl_init(&dst_sgl, iovs, iovcnt); 978 979 data_block_size = ctx->block_size - ctx->md_size; 980 981 if (!_dif_sgl_is_valid(&dst_sgl, data_block_size * num_blocks) || 982 !_dif_sgl_is_valid(&src_sgl, ctx->block_size * num_blocks)) { 983 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 984 return -EINVAL; 985 } 986 987 if (_dif_is_disabled(ctx->dif_type)) { 988 return 0; 989 } 990 991 if (_dif_sgl_is_bytes_multiple(&dst_sgl, data_block_size)) { 992 return dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 993 } else { 994 return dif_verify_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 995 } 996 } 997 998 static void 999 _bit_flip(uint8_t *buf, uint32_t flip_bit) 1000 { 1001 uint8_t byte; 1002 1003 byte = *buf; 1004 byte ^= 1 << flip_bit; 1005 *buf = byte; 1006 } 1007 1008 static int 1009 _dif_inject_error(struct _dif_sgl *sgl, 1010 uint32_t block_size, uint32_t num_blocks, 1011 uint32_t inject_offset_blocks, 1012 uint32_t inject_offset_bytes, 1013 uint32_t inject_offset_bits) 1014 { 1015 uint32_t offset_in_block, buf_len; 1016 void *buf; 1017 1018 _dif_sgl_advance(sgl, block_size * inject_offset_blocks); 1019 1020 offset_in_block = 0; 1021 1022 while (offset_in_block < block_size) { 1023 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1024 buf_len = spdk_min(buf_len, block_size - offset_in_block); 1025 1026 if (inject_offset_bytes >= offset_in_block && 1027 inject_offset_bytes < offset_in_block + buf_len) { 1028 buf += inject_offset_bytes - offset_in_block; 1029 _bit_flip(buf, inject_offset_bits); 1030 return 0; 1031 } 1032 1033 _dif_sgl_advance(sgl, buf_len); 1034 offset_in_block += buf_len; 1035 } 1036 1037 return -1; 1038 } 1039 1040 static int 1041 dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks, 1042 uint32_t start_inject_bytes, uint32_t inject_range_bytes, 1043 uint32_t *inject_offset) 1044 { 1045 uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; 1046 uint32_t offset_blocks; 1047 int rc; 1048 1049 srand(time(0)); 1050 1051 inject_offset_blocks = rand() % num_blocks; 1052 inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); 1053 inject_offset_bits = rand() % 8; 1054 1055 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1056 if (offset_blocks == inject_offset_blocks) { 1057 rc = _dif_inject_error(sgl, block_size, num_blocks, 1058 inject_offset_blocks, 1059 inject_offset_bytes, 1060 inject_offset_bits); 1061 if (rc == 0) { 1062 *inject_offset = inject_offset_blocks; 1063 } 1064 return rc; 1065 } 1066 } 1067 1068 return -1; 1069 } 1070 1071 #define _member_size(type, member) sizeof(((type *)0)->member) 1072 1073 int 1074 spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1075 const struct spdk_dif_ctx *ctx, uint32_t inject_flags, 1076 uint32_t *inject_offset) 1077 { 1078 struct _dif_sgl sgl; 1079 int rc; 1080 1081 _dif_sgl_init(&sgl, iovs, iovcnt); 1082 1083 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1084 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1085 return -EINVAL; 1086 } 1087 1088 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1089 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1090 ctx->guard_interval + offsetof(struct spdk_dif, ref_tag), 1091 _member_size(struct spdk_dif, ref_tag), 1092 inject_offset); 1093 if (rc != 0) { 1094 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1095 return rc; 1096 } 1097 } 1098 1099 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1100 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1101 ctx->guard_interval + offsetof(struct spdk_dif, app_tag), 1102 _member_size(struct spdk_dif, app_tag), 1103 inject_offset); 1104 if (rc != 0) { 1105 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1106 return rc; 1107 } 1108 } 1109 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1110 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1111 ctx->guard_interval, 1112 _member_size(struct spdk_dif, guard), 1113 inject_offset); 1114 if (rc != 0) { 1115 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1116 return rc; 1117 } 1118 } 1119 1120 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1121 /* If the DIF information is contained within the last 8 bytes of 1122 * metadata, then the CRC covers all metadata bytes up to but excluding 1123 * the last 8 bytes. But error injection does not cover these metadata 1124 * because classification is not determined yet. 1125 * 1126 * Note: Error injection to data block is expected to be detected as 1127 * guard error. 1128 */ 1129 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1130 0, 1131 ctx->block_size - ctx->md_size, 1132 inject_offset); 1133 if (rc != 0) { 1134 SPDK_ERRLOG("Failed to inject error to data block.\n"); 1135 return rc; 1136 } 1137 } 1138 1139 return 0; 1140 } 1141 1142 static void 1143 dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1144 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1145 { 1146 uint32_t offset_blocks = 0; 1147 uint16_t guard; 1148 void *data_buf, *md_buf; 1149 1150 while (offset_blocks < num_blocks) { 1151 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1152 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1153 1154 guard = 0; 1155 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1156 guard = spdk_crc16_t10dif(ctx->guard_seed, data_buf, ctx->block_size); 1157 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1158 } 1159 1160 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1161 1162 _dif_sgl_advance(data_sgl, ctx->block_size); 1163 _dif_sgl_advance(md_sgl, ctx->md_size); 1164 offset_blocks++; 1165 } 1166 } 1167 1168 static void 1169 _dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1170 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1171 { 1172 uint32_t offset_in_block, data_buf_len; 1173 uint16_t guard = 0; 1174 void *data_buf, *md_buf; 1175 1176 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1177 1178 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1179 guard = ctx->guard_seed; 1180 } 1181 offset_in_block = 0; 1182 1183 while (offset_in_block < ctx->block_size) { 1184 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1185 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1186 1187 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1188 guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len); 1189 } 1190 1191 _dif_sgl_advance(data_sgl, data_buf_len); 1192 offset_in_block += data_buf_len; 1193 } 1194 1195 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1196 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1197 } 1198 1199 _dif_sgl_advance(md_sgl, ctx->md_size); 1200 1201 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1202 } 1203 1204 static void 1205 dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1206 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1207 { 1208 uint32_t offset_blocks; 1209 1210 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1211 _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx); 1212 } 1213 } 1214 1215 int 1216 spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1217 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1218 { 1219 struct _dif_sgl data_sgl, md_sgl; 1220 1221 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1222 _dif_sgl_init(&md_sgl, md_iov, 1); 1223 1224 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1225 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1226 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1227 return -EINVAL; 1228 } 1229 1230 if (_dif_is_disabled(ctx->dif_type)) { 1231 return 0; 1232 } 1233 1234 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1235 dix_generate(&data_sgl, &md_sgl, num_blocks, ctx); 1236 } else { 1237 dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx); 1238 } 1239 1240 return 0; 1241 } 1242 1243 static int 1244 dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1245 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1246 struct spdk_dif_error *err_blk) 1247 { 1248 uint32_t offset_blocks = 0; 1249 uint16_t guard; 1250 void *data_buf, *md_buf; 1251 int rc; 1252 1253 while (offset_blocks < num_blocks) { 1254 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1255 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1256 1257 guard = 0; 1258 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1259 guard = spdk_crc16_t10dif(ctx->guard_seed, data_buf, ctx->block_size); 1260 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1261 } 1262 1263 rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1264 if (rc != 0) { 1265 return rc; 1266 } 1267 1268 _dif_sgl_advance(data_sgl, ctx->block_size); 1269 _dif_sgl_advance(md_sgl, ctx->md_size); 1270 offset_blocks++; 1271 } 1272 1273 return 0; 1274 } 1275 1276 static int 1277 _dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1278 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1279 struct spdk_dif_error *err_blk) 1280 { 1281 uint32_t offset_in_block, data_buf_len; 1282 uint16_t guard = 0; 1283 void *data_buf, *md_buf; 1284 1285 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1286 1287 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1288 guard = ctx->guard_seed; 1289 } 1290 offset_in_block = 0; 1291 1292 while (offset_in_block < ctx->block_size) { 1293 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1294 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1295 1296 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1297 guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len); 1298 } 1299 1300 _dif_sgl_advance(data_sgl, data_buf_len); 1301 offset_in_block += data_buf_len; 1302 } 1303 1304 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1305 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1306 } 1307 1308 _dif_sgl_advance(md_sgl, ctx->md_size); 1309 1310 return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1311 } 1312 1313 static int 1314 dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1315 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1316 struct spdk_dif_error *err_blk) 1317 { 1318 uint32_t offset_blocks; 1319 int rc; 1320 1321 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1322 rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk); 1323 if (rc != 0) { 1324 return rc; 1325 } 1326 } 1327 1328 return 0; 1329 } 1330 1331 int 1332 spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1333 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1334 struct spdk_dif_error *err_blk) 1335 { 1336 struct _dif_sgl data_sgl, md_sgl; 1337 1338 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1339 _dif_sgl_init(&md_sgl, md_iov, 1); 1340 1341 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1342 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1343 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1344 return -EINVAL; 1345 } 1346 1347 if (_dif_is_disabled(ctx->dif_type)) { 1348 return 0; 1349 } 1350 1351 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1352 return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1353 } else { 1354 return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1355 } 1356 } 1357 1358 int 1359 spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1360 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1361 uint32_t inject_flags, uint32_t *inject_offset) 1362 { 1363 struct _dif_sgl data_sgl, md_sgl; 1364 int rc; 1365 1366 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1367 _dif_sgl_init(&md_sgl, md_iov, 1); 1368 1369 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1370 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1371 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1372 return -EINVAL; 1373 } 1374 1375 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1376 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1377 ctx->guard_interval + offsetof(struct spdk_dif, ref_tag), 1378 _member_size(struct spdk_dif, ref_tag), 1379 inject_offset); 1380 if (rc != 0) { 1381 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1382 return rc; 1383 } 1384 } 1385 1386 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1387 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1388 ctx->guard_interval + offsetof(struct spdk_dif, app_tag), 1389 _member_size(struct spdk_dif, app_tag), 1390 inject_offset); 1391 if (rc != 0) { 1392 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1393 return rc; 1394 } 1395 } 1396 1397 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1398 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1399 ctx->guard_interval, 1400 _member_size(struct spdk_dif, guard), 1401 inject_offset); 1402 if (rc != 0) { 1403 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1404 return rc; 1405 } 1406 } 1407 1408 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1409 /* Note: Error injection to data block is expected to be detected 1410 * as guard error. 1411 */ 1412 rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks, 1413 0, 1414 ctx->block_size, 1415 inject_offset); 1416 if (rc != 0) { 1417 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1418 return rc; 1419 } 1420 } 1421 1422 return 0; 1423 } 1424 1425 static uint32_t 1426 _to_next_boundary(uint32_t offset, uint32_t boundary) 1427 { 1428 return boundary - (offset % boundary); 1429 } 1430 1431 int 1432 spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt, 1433 struct iovec *buf_iovs, int buf_iovcnt, 1434 uint32_t data_offset, uint32_t data_len, 1435 uint32_t *_mapped_len, 1436 const struct spdk_dif_ctx *ctx) 1437 { 1438 uint32_t data_block_size, data_unalign, buf_len, buf_offset, len; 1439 struct _dif_sgl dif_sgl; 1440 struct _dif_sgl buf_sgl; 1441 1442 if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) { 1443 return -EINVAL; 1444 } 1445 1446 data_block_size = ctx->block_size - ctx->md_size; 1447 1448 data_unalign = ctx->data_offset % data_block_size; 1449 1450 buf_len = ((data_unalign + data_offset + data_len) / data_block_size) * ctx->block_size + 1451 ((data_unalign + data_offset + data_len) % data_block_size); 1452 buf_len -= data_unalign; 1453 1454 _dif_sgl_init(&dif_sgl, iovs, iovcnt); 1455 _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt); 1456 1457 if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) { 1458 SPDK_ERRLOG("Buffer overflow will occur.\n"); 1459 return -ERANGE; 1460 } 1461 1462 buf_offset = ((data_unalign + data_offset) / data_block_size) * ctx->block_size + 1463 ((data_unalign + data_offset) % data_block_size); 1464 buf_offset -= data_unalign; 1465 1466 _dif_sgl_advance(&buf_sgl, buf_offset); 1467 1468 while (data_len != 0) { 1469 len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size)); 1470 if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) { 1471 break; 1472 } 1473 _dif_sgl_advance(&buf_sgl, ctx->md_size); 1474 data_offset += len; 1475 data_len -= len; 1476 } 1477 1478 if (_mapped_len != NULL) { 1479 *_mapped_len = dif_sgl.total_size; 1480 } 1481 1482 return iovcnt - dif_sgl.iovcnt; 1483 } 1484 1485 int 1486 spdk_dif_generate_stream(struct iovec *iovs, int iovcnt, 1487 uint32_t data_offset, uint32_t data_len, 1488 struct spdk_dif_ctx *ctx) 1489 { 1490 uint32_t data_block_size, data_unalign, buf_len, buf_offset; 1491 uint32_t len, offset_in_block, offset_blocks; 1492 uint16_t guard = 0; 1493 struct _dif_sgl sgl; 1494 1495 if (iovs == NULL || iovcnt == 0) { 1496 return -EINVAL; 1497 } 1498 1499 data_block_size = ctx->block_size - ctx->md_size; 1500 1501 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1502 guard = ctx->last_guard; 1503 } 1504 1505 data_unalign = ctx->data_offset % data_block_size; 1506 1507 /* If the last data block is complete, DIF of the data block is 1508 * inserted in this function. 1509 */ 1510 buf_len = ((data_unalign + data_offset + data_len) / data_block_size) * ctx->block_size + 1511 ((data_unalign + data_offset + data_len) % data_block_size); 1512 buf_len -= data_unalign; 1513 1514 _dif_sgl_init(&sgl, iovs, iovcnt); 1515 1516 if (!_dif_sgl_is_valid(&sgl, buf_len)) { 1517 return -ERANGE; 1518 } 1519 1520 buf_offset = ((data_unalign + data_offset) / data_block_size) * ctx->block_size + 1521 ((data_unalign + data_offset) % data_block_size); 1522 buf_offset -= data_unalign; 1523 1524 _dif_sgl_advance(&sgl, buf_offset); 1525 buf_len -= buf_offset; 1526 1527 buf_offset += data_unalign; 1528 1529 while (buf_len != 0) { 1530 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1531 offset_in_block = buf_offset % ctx->block_size; 1532 offset_blocks = buf_offset / ctx->block_size; 1533 1534 guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx); 1535 1536 buf_len -= len; 1537 buf_offset += len; 1538 } 1539 1540 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1541 ctx->last_guard = guard; 1542 } 1543 1544 return 0; 1545 } 1546