1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/dif.h" 35 #include "spdk/crc16.h" 36 #include "spdk/crc32.h" 37 #include "spdk/endian.h" 38 #include "spdk/log.h" 39 #include "spdk/util.h" 40 41 /* Context to iterate or create a iovec array. 42 * Each sgl is either iterated or created at a time. 43 */ 44 struct _dif_sgl { 45 /* Current iovec in the iteration or creation */ 46 struct iovec *iov; 47 48 /* Remaining count of iovecs in the iteration or creation. */ 49 int iovcnt; 50 51 /* Current offset in the iovec */ 52 uint32_t iov_offset; 53 54 /* Size of the created iovec array in bytes */ 55 uint32_t total_size; 56 }; 57 58 static inline void 59 _dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt) 60 { 61 s->iov = iovs; 62 s->iovcnt = iovcnt; 63 s->iov_offset = 0; 64 s->total_size = 0; 65 } 66 67 static void 68 _dif_sgl_advance(struct _dif_sgl *s, uint32_t step) 69 { 70 s->iov_offset += step; 71 while (s->iovcnt != 0) { 72 if (s->iov_offset < s->iov->iov_len) { 73 break; 74 } 75 76 s->iov_offset -= s->iov->iov_len; 77 s->iov++; 78 s->iovcnt--; 79 } 80 } 81 82 static inline void 83 _dif_sgl_get_buf(struct _dif_sgl *s, void **_buf, uint32_t *_buf_len) 84 { 85 if (_buf != NULL) { 86 *_buf = s->iov->iov_base + s->iov_offset; 87 } 88 if (_buf_len != NULL) { 89 *_buf_len = s->iov->iov_len - s->iov_offset; 90 } 91 } 92 93 static inline bool 94 _dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len) 95 { 96 assert(s->iovcnt > 0); 97 s->iov->iov_base = data; 98 s->iov->iov_len = data_len; 99 s->total_size += data_len; 100 s->iov++; 101 s->iovcnt--; 102 103 if (s->iovcnt > 0) { 104 return true; 105 } else { 106 return false; 107 } 108 } 109 110 static inline bool 111 _dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len) 112 { 113 uint8_t *buf; 114 uint32_t buf_len; 115 116 while (data_len != 0) { 117 _dif_sgl_get_buf(src, (void *)&buf, &buf_len); 118 buf_len = spdk_min(buf_len, data_len); 119 120 if (!_dif_sgl_append(dst, buf, buf_len)) { 121 return false; 122 } 123 124 _dif_sgl_advance(src, buf_len); 125 data_len -= buf_len; 126 } 127 128 return true; 129 } 130 131 /* This function must be used before starting iteration. */ 132 static bool 133 _dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes) 134 { 135 int i; 136 137 for (i = 0; i < s->iovcnt; i++) { 138 if (s->iov[i].iov_len % bytes) { 139 return false; 140 } 141 } 142 143 return true; 144 } 145 146 /* This function must be used before starting iteration. */ 147 static bool 148 _dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes) 149 { 150 uint64_t total = 0; 151 int i; 152 153 for (i = 0; i < s->iovcnt; i++) { 154 total += s->iov[i].iov_len; 155 } 156 157 return total >= bytes; 158 } 159 160 static bool 161 _dif_type_is_valid(enum spdk_dif_type dif_type, uint32_t dif_flags) 162 { 163 switch (dif_type) { 164 case SPDK_DIF_TYPE1: 165 case SPDK_DIF_TYPE2: 166 case SPDK_DIF_DISABLE: 167 break; 168 case SPDK_DIF_TYPE3: 169 if (dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 170 SPDK_ERRLOG("Reference Tag should not be checked for Type 3\n"); 171 return false; 172 } 173 break; 174 default: 175 SPDK_ERRLOG("Unknown DIF Type: %d\n", dif_type); 176 return false; 177 } 178 179 return true; 180 } 181 182 static bool 183 _dif_is_disabled(enum spdk_dif_type dif_type) 184 { 185 if (dif_type == SPDK_DIF_DISABLE) { 186 return true; 187 } else { 188 return false; 189 } 190 } 191 192 193 static uint32_t 194 _get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave) 195 { 196 if (!dif_loc) { 197 /* For metadata formats with more than 8 bytes, if the DIF is 198 * contained in the last 8 bytes of metadata, then the CRC 199 * covers all metadata up to but excluding these last 8 bytes. 200 */ 201 if (md_interleave) { 202 return block_size - sizeof(struct spdk_dif); 203 } else { 204 return md_size - sizeof(struct spdk_dif); 205 } 206 } else { 207 /* For metadata formats with more than 8 bytes, if the DIF is 208 * contained in the first 8 bytes of metadata, then the CRC 209 * does not cover any metadata. 210 */ 211 if (md_interleave) { 212 return block_size - md_size; 213 } else { 214 return 0; 215 } 216 } 217 } 218 219 int 220 spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, 221 bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, 222 uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag, 223 uint32_t data_offset, uint16_t guard_seed) 224 { 225 uint32_t data_block_size; 226 227 if (md_size < sizeof(struct spdk_dif)) { 228 SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); 229 return -EINVAL; 230 } 231 232 if (md_interleave) { 233 if (block_size < md_size) { 234 SPDK_ERRLOG("Block size is smaller than DIF size.\n"); 235 return -EINVAL; 236 } 237 data_block_size = block_size - md_size; 238 } else { 239 if (block_size == 0 || (block_size % 512) != 0) { 240 SPDK_ERRLOG("Zero block size is not allowed\n"); 241 return -EINVAL; 242 } 243 data_block_size = block_size; 244 } 245 246 if (!_dif_type_is_valid(dif_type, dif_flags)) { 247 SPDK_ERRLOG("DIF type is invalid.\n"); 248 return -EINVAL; 249 } 250 251 ctx->block_size = block_size; 252 ctx->md_size = md_size; 253 ctx->md_interleave = md_interleave; 254 ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave); 255 ctx->dif_type = dif_type; 256 ctx->dif_flags = dif_flags; 257 ctx->init_ref_tag = init_ref_tag; 258 ctx->apptag_mask = apptag_mask; 259 ctx->app_tag = app_tag; 260 ctx->data_offset = data_offset; 261 ctx->ref_tag_offset = data_offset / data_block_size; 262 ctx->last_guard = guard_seed; 263 ctx->guard_seed = guard_seed; 264 265 return 0; 266 } 267 268 void 269 spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset) 270 { 271 uint32_t data_block_size; 272 273 if (ctx->md_interleave) { 274 data_block_size = ctx->block_size - ctx->md_size; 275 } else { 276 data_block_size = ctx->block_size; 277 } 278 279 ctx->data_offset = data_offset; 280 ctx->ref_tag_offset = data_offset / data_block_size; 281 } 282 283 static void 284 _dif_generate(void *_dif, uint16_t guard, uint32_t offset_blocks, 285 const struct spdk_dif_ctx *ctx) 286 { 287 struct spdk_dif *dif = _dif; 288 uint32_t ref_tag; 289 290 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 291 to_be16(&dif->guard, guard); 292 } 293 294 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 295 to_be16(&dif->app_tag, ctx->app_tag); 296 } 297 298 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 299 /* For type 1 and 2, the reference tag is incremented for each 300 * subsequent logical block. For type 3, the reference tag 301 * remains the same as the initial reference tag. 302 */ 303 if (ctx->dif_type != SPDK_DIF_TYPE3) { 304 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 305 } else { 306 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 307 } 308 309 to_be32(&dif->ref_tag, ref_tag); 310 } 311 } 312 313 static void 314 dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 315 { 316 uint32_t offset_blocks = 0; 317 void *buf; 318 uint16_t guard = 0; 319 320 while (offset_blocks < num_blocks) { 321 _dif_sgl_get_buf(sgl, &buf, NULL); 322 323 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 324 guard = spdk_crc16_t10dif(ctx->guard_seed, buf, ctx->guard_interval); 325 } 326 327 _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); 328 329 _dif_sgl_advance(sgl, ctx->block_size); 330 offset_blocks++; 331 } 332 } 333 334 static uint16_t 335 _dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 336 uint16_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 337 { 338 uint32_t offset_in_dif, buf_len; 339 void *buf; 340 struct spdk_dif dif = {}; 341 342 assert(offset_in_block < ctx->guard_interval); 343 assert(offset_in_block + data_len < ctx->guard_interval || 344 offset_in_block + data_len == ctx->block_size); 345 346 /* Compute CRC over split logical block data. */ 347 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 348 _dif_sgl_get_buf(sgl, &buf, &buf_len); 349 buf_len = spdk_min(buf_len, data_len); 350 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 351 352 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 353 guard = spdk_crc16_t10dif(guard, buf, buf_len); 354 } 355 356 _dif_sgl_advance(sgl, buf_len); 357 offset_in_block += buf_len; 358 data_len -= buf_len; 359 } 360 361 if (offset_in_block < ctx->guard_interval) { 362 return guard; 363 } 364 365 /* If a whole logical block data is parsed, generate DIF 366 * and save it to the temporary DIF area. 367 */ 368 _dif_generate(&dif, guard, offset_blocks, ctx); 369 370 /* Copy generated DIF field to the split DIF field, and then 371 * skip metadata field after DIF field (if any). 372 */ 373 while (offset_in_block < ctx->block_size) { 374 _dif_sgl_get_buf(sgl, &buf, &buf_len); 375 376 if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) { 377 offset_in_dif = offset_in_block - ctx->guard_interval; 378 buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif); 379 380 memcpy(buf, ((uint8_t *)&dif) + offset_in_dif, buf_len); 381 } else { 382 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 383 } 384 385 _dif_sgl_advance(sgl, buf_len); 386 offset_in_block += buf_len; 387 } 388 389 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 390 guard = ctx->guard_seed; 391 } 392 393 return guard; 394 } 395 396 static void 397 dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks, 398 const struct spdk_dif_ctx *ctx) 399 { 400 uint32_t offset_blocks; 401 uint16_t guard = 0; 402 403 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 404 guard = ctx->guard_seed; 405 } 406 407 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 408 _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx); 409 } 410 } 411 412 int 413 spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 414 const struct spdk_dif_ctx *ctx) 415 { 416 struct _dif_sgl sgl; 417 418 _dif_sgl_init(&sgl, iovs, iovcnt); 419 420 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 421 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 422 return -EINVAL; 423 } 424 425 if (_dif_is_disabled(ctx->dif_type)) { 426 return 0; 427 } 428 429 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 430 dif_generate(&sgl, num_blocks, ctx); 431 } else { 432 dif_generate_split(&sgl, num_blocks, ctx); 433 } 434 435 return 0; 436 } 437 438 static void 439 _dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, 440 uint32_t expected, uint32_t actual, uint32_t err_offset) 441 { 442 if (err_blk) { 443 err_blk->err_type = err_type; 444 err_blk->expected = expected; 445 err_blk->actual = actual; 446 err_blk->err_offset = err_offset; 447 } 448 } 449 450 static int 451 _dif_verify(void *_dif, uint16_t guard, uint32_t offset_blocks, 452 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 453 { 454 struct spdk_dif *dif = _dif; 455 uint16_t _guard; 456 uint16_t _app_tag; 457 uint32_t ref_tag, _ref_tag; 458 459 switch (ctx->dif_type) { 460 case SPDK_DIF_TYPE1: 461 case SPDK_DIF_TYPE2: 462 /* If Type 1 or 2 is used, then all DIF checks are disabled when 463 * the Application Tag is 0xFFFF. 464 */ 465 if (dif->app_tag == 0xFFFF) { 466 return 0; 467 } 468 break; 469 case SPDK_DIF_TYPE3: 470 /* If Type 3 is used, then all DIF checks are disabled when the 471 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF. 472 */ 473 if (dif->app_tag == 0xFFFF && dif->ref_tag == 0xFFFFFFFF) { 474 return 0; 475 } 476 break; 477 default: 478 break; 479 } 480 481 /* For type 1 and 2, the reference tag is incremented for each 482 * subsequent logical block. For type 3, the reference tag 483 * remains the same as the initial reference tag. 484 */ 485 if (ctx->dif_type != SPDK_DIF_TYPE3) { 486 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 487 } else { 488 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 489 } 490 491 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 492 /* Compare the DIF Guard field to the CRC computed over the logical 493 * block data. 494 */ 495 _guard = from_be16(&dif->guard); 496 if (_guard != guard) { 497 _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, 498 offset_blocks); 499 SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu32 "," \ 500 " Expected=%x, Actual=%x\n", 501 ref_tag, _guard, guard); 502 return -1; 503 } 504 } 505 506 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 507 /* Compare unmasked bits in the DIF Application Tag field to the 508 * passed Application Tag. 509 */ 510 _app_tag = from_be16(&dif->app_tag); 511 if ((_app_tag & ctx->apptag_mask) != ctx->app_tag) { 512 _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, 513 (_app_tag & ctx->apptag_mask), offset_blocks); 514 SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu32 "," \ 515 " Expected=%x, Actual=%x\n", 516 ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); 517 return -1; 518 } 519 } 520 521 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 522 switch (ctx->dif_type) { 523 case SPDK_DIF_TYPE1: 524 case SPDK_DIF_TYPE2: 525 /* Compare the DIF Reference Tag field to the passed Reference Tag. 526 * The passed Reference Tag will be the least significant 4 bytes 527 * of the LBA when Type 1 is used, and application specific value 528 * if Type 2 is used, 529 */ 530 _ref_tag = from_be32(&dif->ref_tag); 531 if (_ref_tag != ref_tag) { 532 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, ref_tag, 533 _ref_tag, offset_blocks); 534 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu32 "," \ 535 " Expected=%x, Actual=%x\n", 536 ref_tag, ref_tag, _ref_tag); 537 return -1; 538 } 539 break; 540 case SPDK_DIF_TYPE3: 541 /* For Type 3, computed Reference Tag remains unchanged. 542 * Hence ignore the Reference Tag field. 543 */ 544 break; 545 default: 546 break; 547 } 548 } 549 550 return 0; 551 } 552 553 static int 554 dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks, 555 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 556 { 557 uint32_t offset_blocks = 0; 558 int rc; 559 void *buf; 560 uint16_t guard = 0; 561 562 while (offset_blocks < num_blocks) { 563 _dif_sgl_get_buf(sgl, &buf, NULL); 564 565 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 566 guard = spdk_crc16_t10dif(ctx->guard_seed, buf, ctx->guard_interval); 567 } 568 569 rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 570 if (rc != 0) { 571 return rc; 572 } 573 574 _dif_sgl_advance(sgl, ctx->block_size); 575 offset_blocks++; 576 } 577 578 return 0; 579 } 580 581 static int 582 _dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 583 uint16_t *_guard, uint32_t offset_blocks, 584 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 585 { 586 uint32_t offset_in_dif, buf_len; 587 void *buf; 588 uint16_t guard; 589 struct spdk_dif dif = {}; 590 int rc; 591 592 assert(_guard != NULL); 593 assert(offset_in_block < ctx->guard_interval); 594 assert(offset_in_block + data_len < ctx->guard_interval || 595 offset_in_block + data_len == ctx->block_size); 596 597 guard = *_guard; 598 599 /* Compute CRC over split logical block data. */ 600 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 601 _dif_sgl_get_buf(sgl, &buf, &buf_len); 602 buf_len = spdk_min(buf_len, data_len); 603 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 604 605 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 606 guard = spdk_crc16_t10dif(guard, buf, buf_len); 607 } 608 609 _dif_sgl_advance(sgl, buf_len); 610 offset_in_block += buf_len; 611 data_len -= buf_len; 612 } 613 614 if (offset_in_block < ctx->guard_interval) { 615 *_guard = guard; 616 return 0; 617 } 618 619 /* Copy the split DIF field to the temporary DIF buffer, and then 620 * skip metadata field after DIF field (if any). */ 621 while (offset_in_block < ctx->block_size) { 622 _dif_sgl_get_buf(sgl, &buf, &buf_len); 623 624 if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) { 625 offset_in_dif = offset_in_block - ctx->guard_interval; 626 buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif); 627 628 memcpy((uint8_t *)&dif + offset_in_dif, buf, buf_len); 629 } else { 630 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 631 } 632 _dif_sgl_advance(sgl, buf_len); 633 offset_in_block += buf_len; 634 } 635 636 rc = _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 637 if (rc != 0) { 638 return rc; 639 } 640 641 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 642 guard = ctx->guard_seed; 643 } 644 645 *_guard = guard; 646 return 0; 647 } 648 649 static int 650 dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks, 651 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 652 { 653 uint32_t offset_blocks; 654 uint16_t guard = 0; 655 int rc; 656 657 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 658 guard = ctx->guard_seed; 659 } 660 661 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 662 rc = _dif_verify_split(sgl, 0, ctx->block_size, &guard, offset_blocks, 663 ctx, err_blk); 664 if (rc != 0) { 665 return rc; 666 } 667 } 668 669 return 0; 670 } 671 672 int 673 spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 674 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 675 { 676 struct _dif_sgl sgl; 677 678 _dif_sgl_init(&sgl, iovs, iovcnt); 679 680 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 681 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 682 return -EINVAL; 683 } 684 685 if (_dif_is_disabled(ctx->dif_type)) { 686 return 0; 687 } 688 689 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 690 return dif_verify(&sgl, num_blocks, ctx, err_blk); 691 } else { 692 return dif_verify_split(&sgl, num_blocks, ctx, err_blk); 693 } 694 } 695 696 static uint32_t 697 dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks, 698 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 699 { 700 uint32_t offset_blocks; 701 void *buf; 702 703 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 704 _dif_sgl_get_buf(sgl, &buf, NULL); 705 706 crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c); 707 708 _dif_sgl_advance(sgl, ctx->block_size); 709 } 710 711 return crc32c; 712 } 713 714 static uint32_t 715 _dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t crc32c, 716 const struct spdk_dif_ctx *ctx) 717 { 718 uint32_t data_block_size, offset_in_block, buf_len; 719 void *buf; 720 721 data_block_size = ctx->block_size - ctx->md_size; 722 offset_in_block = 0; 723 724 while (offset_in_block < ctx->block_size) { 725 _dif_sgl_get_buf(sgl, &buf, &buf_len); 726 727 if (offset_in_block < data_block_size) { 728 buf_len = spdk_min(buf_len, data_block_size - offset_in_block); 729 crc32c = spdk_crc32c_update(buf, buf_len, crc32c); 730 } else { 731 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 732 } 733 734 _dif_sgl_advance(sgl, buf_len); 735 offset_in_block += buf_len; 736 } 737 738 return crc32c; 739 } 740 741 static uint32_t 742 dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks, 743 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 744 { 745 uint32_t offset_blocks; 746 747 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 748 crc32c = _dif_update_crc32c_split(sgl, crc32c, ctx); 749 } 750 751 return crc32c; 752 } 753 754 int 755 spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 756 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 757 { 758 struct _dif_sgl sgl; 759 760 if (_crc32c == NULL) { 761 return -EINVAL; 762 } 763 764 _dif_sgl_init(&sgl, iovs, iovcnt); 765 766 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 767 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 768 return -EINVAL; 769 } 770 771 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 772 *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx); 773 } else { 774 *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx); 775 } 776 777 return 0; 778 } 779 780 static void 781 dif_generate_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 782 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 783 { 784 uint32_t offset_blocks = 0, data_block_size; 785 void *src, *dst; 786 uint16_t guard; 787 788 data_block_size = ctx->block_size - ctx->md_size; 789 790 while (offset_blocks < num_blocks) { 791 _dif_sgl_get_buf(src_sgl, &src, NULL); 792 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 793 794 guard = 0; 795 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 796 guard = spdk_crc16_t10dif_copy(ctx->guard_seed, dst, src, data_block_size); 797 guard = spdk_crc16_t10dif(guard, dst + data_block_size, 798 ctx->guard_interval - data_block_size); 799 } else { 800 memcpy(dst, src, data_block_size); 801 } 802 803 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 804 805 _dif_sgl_advance(src_sgl, data_block_size); 806 _dif_sgl_advance(dst_sgl, ctx->block_size); 807 offset_blocks++; 808 } 809 } 810 811 static void 812 _dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 813 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 814 { 815 uint32_t offset_in_block, src_len, data_block_size; 816 uint16_t guard = 0; 817 void *src, *dst; 818 819 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 820 821 data_block_size = ctx->block_size - ctx->md_size; 822 823 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 824 guard = ctx->guard_seed; 825 } 826 offset_in_block = 0; 827 828 while (offset_in_block < data_block_size) { 829 /* Compute CRC over split logical block data and copy 830 * data to bounce buffer. 831 */ 832 _dif_sgl_get_buf(src_sgl, &src, &src_len); 833 src_len = spdk_min(src_len, data_block_size - offset_in_block); 834 835 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 836 guard = spdk_crc16_t10dif_copy(guard, dst + offset_in_block, 837 src, src_len); 838 } else { 839 memcpy(dst + offset_in_block, src, src_len); 840 } 841 842 _dif_sgl_advance(src_sgl, src_len); 843 offset_in_block += src_len; 844 } 845 846 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 847 guard = spdk_crc16_t10dif(guard, dst + data_block_size, 848 ctx->guard_interval - data_block_size); 849 } 850 851 _dif_sgl_advance(dst_sgl, ctx->block_size); 852 853 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 854 } 855 856 static void 857 dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 858 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 859 { 860 uint32_t offset_blocks; 861 862 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 863 _dif_generate_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 864 } 865 } 866 867 int 868 spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 869 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 870 { 871 struct _dif_sgl src_sgl, dst_sgl; 872 uint32_t data_block_size; 873 874 _dif_sgl_init(&src_sgl, iovs, iovcnt); 875 _dif_sgl_init(&dst_sgl, bounce_iov, 1); 876 877 data_block_size = ctx->block_size - ctx->md_size; 878 879 if (!_dif_sgl_is_valid(&src_sgl, data_block_size * num_blocks) || 880 !_dif_sgl_is_valid(&dst_sgl, ctx->block_size * num_blocks)) { 881 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 882 return -EINVAL; 883 } 884 885 if (_dif_is_disabled(ctx->dif_type)) { 886 return 0; 887 } 888 889 if (_dif_sgl_is_bytes_multiple(&src_sgl, data_block_size)) { 890 dif_generate_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 891 } else { 892 dif_generate_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx); 893 } 894 895 return 0; 896 } 897 898 static int 899 dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 900 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 901 struct spdk_dif_error *err_blk) 902 { 903 uint32_t offset_blocks = 0, data_block_size; 904 void *src, *dst; 905 int rc; 906 uint16_t guard; 907 908 data_block_size = ctx->block_size - ctx->md_size; 909 910 while (offset_blocks < num_blocks) { 911 _dif_sgl_get_buf(src_sgl, &src, NULL); 912 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 913 914 guard = 0; 915 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 916 guard = spdk_crc16_t10dif_copy(ctx->guard_seed, dst, src, data_block_size); 917 guard = spdk_crc16_t10dif(guard, src + data_block_size, 918 ctx->guard_interval - data_block_size); 919 } else { 920 memcpy(dst, src, data_block_size); 921 } 922 923 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 924 if (rc != 0) { 925 return rc; 926 } 927 928 _dif_sgl_advance(src_sgl, ctx->block_size); 929 _dif_sgl_advance(dst_sgl, data_block_size); 930 offset_blocks++; 931 } 932 933 return 0; 934 } 935 936 static int 937 _dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 938 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 939 struct spdk_dif_error *err_blk) 940 { 941 uint32_t offset_in_block, dst_len, data_block_size; 942 uint16_t guard = 0; 943 void *src, *dst; 944 945 _dif_sgl_get_buf(src_sgl, &src, NULL); 946 947 data_block_size = ctx->block_size - ctx->md_size; 948 949 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 950 guard = ctx->guard_seed; 951 } 952 offset_in_block = 0; 953 954 while (offset_in_block < data_block_size) { 955 /* Compute CRC over split logical block data and copy 956 * data to bounce buffer. 957 */ 958 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 959 dst_len = spdk_min(dst_len, data_block_size - offset_in_block); 960 961 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 962 guard = spdk_crc16_t10dif_copy(guard, dst, 963 src + offset_in_block, dst_len); 964 } else { 965 memcpy(dst, src + offset_in_block, dst_len); 966 } 967 968 _dif_sgl_advance(dst_sgl, dst_len); 969 offset_in_block += dst_len; 970 } 971 972 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 973 guard = spdk_crc16_t10dif(guard, src + data_block_size, 974 ctx->guard_interval - data_block_size); 975 } 976 977 _dif_sgl_advance(src_sgl, ctx->block_size); 978 979 return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 980 } 981 982 static int 983 dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 984 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 985 struct spdk_dif_error *err_blk) 986 { 987 uint32_t offset_blocks; 988 int rc; 989 990 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 991 rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 992 if (rc != 0) { 993 return rc; 994 } 995 } 996 997 return 0; 998 } 999 1000 int 1001 spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 1002 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1003 struct spdk_dif_error *err_blk) 1004 { 1005 struct _dif_sgl src_sgl, dst_sgl; 1006 uint32_t data_block_size; 1007 1008 _dif_sgl_init(&src_sgl, bounce_iov, 1); 1009 _dif_sgl_init(&dst_sgl, iovs, iovcnt); 1010 1011 data_block_size = ctx->block_size - ctx->md_size; 1012 1013 if (!_dif_sgl_is_valid(&dst_sgl, data_block_size * num_blocks) || 1014 !_dif_sgl_is_valid(&src_sgl, ctx->block_size * num_blocks)) { 1015 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1016 return -EINVAL; 1017 } 1018 1019 if (_dif_is_disabled(ctx->dif_type)) { 1020 return 0; 1021 } 1022 1023 if (_dif_sgl_is_bytes_multiple(&dst_sgl, data_block_size)) { 1024 return dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1025 } else { 1026 return dif_verify_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1027 } 1028 } 1029 1030 static void 1031 _bit_flip(uint8_t *buf, uint32_t flip_bit) 1032 { 1033 uint8_t byte; 1034 1035 byte = *buf; 1036 byte ^= 1 << flip_bit; 1037 *buf = byte; 1038 } 1039 1040 static int 1041 _dif_inject_error(struct _dif_sgl *sgl, 1042 uint32_t block_size, uint32_t num_blocks, 1043 uint32_t inject_offset_blocks, 1044 uint32_t inject_offset_bytes, 1045 uint32_t inject_offset_bits) 1046 { 1047 uint32_t offset_in_block, buf_len; 1048 void *buf; 1049 1050 _dif_sgl_advance(sgl, block_size * inject_offset_blocks); 1051 1052 offset_in_block = 0; 1053 1054 while (offset_in_block < block_size) { 1055 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1056 buf_len = spdk_min(buf_len, block_size - offset_in_block); 1057 1058 if (inject_offset_bytes >= offset_in_block && 1059 inject_offset_bytes < offset_in_block + buf_len) { 1060 buf += inject_offset_bytes - offset_in_block; 1061 _bit_flip(buf, inject_offset_bits); 1062 return 0; 1063 } 1064 1065 _dif_sgl_advance(sgl, buf_len); 1066 offset_in_block += buf_len; 1067 } 1068 1069 return -1; 1070 } 1071 1072 static int 1073 dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks, 1074 uint32_t start_inject_bytes, uint32_t inject_range_bytes, 1075 uint32_t *inject_offset) 1076 { 1077 uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; 1078 uint32_t offset_blocks; 1079 int rc; 1080 1081 srand(time(0)); 1082 1083 inject_offset_blocks = rand() % num_blocks; 1084 inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); 1085 inject_offset_bits = rand() % 8; 1086 1087 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1088 if (offset_blocks == inject_offset_blocks) { 1089 rc = _dif_inject_error(sgl, block_size, num_blocks, 1090 inject_offset_blocks, 1091 inject_offset_bytes, 1092 inject_offset_bits); 1093 if (rc == 0) { 1094 *inject_offset = inject_offset_blocks; 1095 } 1096 return rc; 1097 } 1098 } 1099 1100 return -1; 1101 } 1102 1103 #define _member_size(type, member) sizeof(((type *)0)->member) 1104 1105 int 1106 spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1107 const struct spdk_dif_ctx *ctx, uint32_t inject_flags, 1108 uint32_t *inject_offset) 1109 { 1110 struct _dif_sgl sgl; 1111 int rc; 1112 1113 _dif_sgl_init(&sgl, iovs, iovcnt); 1114 1115 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1116 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1117 return -EINVAL; 1118 } 1119 1120 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1121 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1122 ctx->guard_interval + offsetof(struct spdk_dif, ref_tag), 1123 _member_size(struct spdk_dif, ref_tag), 1124 inject_offset); 1125 if (rc != 0) { 1126 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1127 return rc; 1128 } 1129 } 1130 1131 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1132 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1133 ctx->guard_interval + offsetof(struct spdk_dif, app_tag), 1134 _member_size(struct spdk_dif, app_tag), 1135 inject_offset); 1136 if (rc != 0) { 1137 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1138 return rc; 1139 } 1140 } 1141 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1142 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1143 ctx->guard_interval, 1144 _member_size(struct spdk_dif, guard), 1145 inject_offset); 1146 if (rc != 0) { 1147 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1148 return rc; 1149 } 1150 } 1151 1152 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1153 /* If the DIF information is contained within the last 8 bytes of 1154 * metadata, then the CRC covers all metadata bytes up to but excluding 1155 * the last 8 bytes. But error injection does not cover these metadata 1156 * because classification is not determined yet. 1157 * 1158 * Note: Error injection to data block is expected to be detected as 1159 * guard error. 1160 */ 1161 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1162 0, 1163 ctx->block_size - ctx->md_size, 1164 inject_offset); 1165 if (rc != 0) { 1166 SPDK_ERRLOG("Failed to inject error to data block.\n"); 1167 return rc; 1168 } 1169 } 1170 1171 return 0; 1172 } 1173 1174 static void 1175 dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1176 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1177 { 1178 uint32_t offset_blocks = 0; 1179 uint16_t guard; 1180 void *data_buf, *md_buf; 1181 1182 while (offset_blocks < num_blocks) { 1183 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1184 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1185 1186 guard = 0; 1187 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1188 guard = spdk_crc16_t10dif(ctx->guard_seed, data_buf, ctx->block_size); 1189 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1190 } 1191 1192 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1193 1194 _dif_sgl_advance(data_sgl, ctx->block_size); 1195 _dif_sgl_advance(md_sgl, ctx->md_size); 1196 offset_blocks++; 1197 } 1198 } 1199 1200 static void 1201 _dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1202 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1203 { 1204 uint32_t offset_in_block, data_buf_len; 1205 uint16_t guard = 0; 1206 void *data_buf, *md_buf; 1207 1208 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1209 1210 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1211 guard = ctx->guard_seed; 1212 } 1213 offset_in_block = 0; 1214 1215 while (offset_in_block < ctx->block_size) { 1216 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1217 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1218 1219 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1220 guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len); 1221 } 1222 1223 _dif_sgl_advance(data_sgl, data_buf_len); 1224 offset_in_block += data_buf_len; 1225 } 1226 1227 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1228 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1229 } 1230 1231 _dif_sgl_advance(md_sgl, ctx->md_size); 1232 1233 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1234 } 1235 1236 static void 1237 dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1238 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1239 { 1240 uint32_t offset_blocks; 1241 1242 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1243 _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx); 1244 } 1245 } 1246 1247 int 1248 spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1249 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1250 { 1251 struct _dif_sgl data_sgl, md_sgl; 1252 1253 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1254 _dif_sgl_init(&md_sgl, md_iov, 1); 1255 1256 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1257 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1258 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1259 return -EINVAL; 1260 } 1261 1262 if (_dif_is_disabled(ctx->dif_type)) { 1263 return 0; 1264 } 1265 1266 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1267 dix_generate(&data_sgl, &md_sgl, num_blocks, ctx); 1268 } else { 1269 dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx); 1270 } 1271 1272 return 0; 1273 } 1274 1275 static int 1276 dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1277 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1278 struct spdk_dif_error *err_blk) 1279 { 1280 uint32_t offset_blocks = 0; 1281 uint16_t guard; 1282 void *data_buf, *md_buf; 1283 int rc; 1284 1285 while (offset_blocks < num_blocks) { 1286 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1287 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1288 1289 guard = 0; 1290 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1291 guard = spdk_crc16_t10dif(ctx->guard_seed, data_buf, ctx->block_size); 1292 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1293 } 1294 1295 rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1296 if (rc != 0) { 1297 return rc; 1298 } 1299 1300 _dif_sgl_advance(data_sgl, ctx->block_size); 1301 _dif_sgl_advance(md_sgl, ctx->md_size); 1302 offset_blocks++; 1303 } 1304 1305 return 0; 1306 } 1307 1308 static int 1309 _dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1310 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1311 struct spdk_dif_error *err_blk) 1312 { 1313 uint32_t offset_in_block, data_buf_len; 1314 uint16_t guard = 0; 1315 void *data_buf, *md_buf; 1316 1317 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1318 1319 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1320 guard = ctx->guard_seed; 1321 } 1322 offset_in_block = 0; 1323 1324 while (offset_in_block < ctx->block_size) { 1325 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1326 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1327 1328 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1329 guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len); 1330 } 1331 1332 _dif_sgl_advance(data_sgl, data_buf_len); 1333 offset_in_block += data_buf_len; 1334 } 1335 1336 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1337 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1338 } 1339 1340 _dif_sgl_advance(md_sgl, ctx->md_size); 1341 1342 return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1343 } 1344 1345 static int 1346 dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1347 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1348 struct spdk_dif_error *err_blk) 1349 { 1350 uint32_t offset_blocks; 1351 int rc; 1352 1353 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1354 rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk); 1355 if (rc != 0) { 1356 return rc; 1357 } 1358 } 1359 1360 return 0; 1361 } 1362 1363 int 1364 spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1365 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1366 struct spdk_dif_error *err_blk) 1367 { 1368 struct _dif_sgl data_sgl, md_sgl; 1369 1370 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1371 _dif_sgl_init(&md_sgl, md_iov, 1); 1372 1373 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1374 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1375 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1376 return -EINVAL; 1377 } 1378 1379 if (_dif_is_disabled(ctx->dif_type)) { 1380 return 0; 1381 } 1382 1383 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1384 return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1385 } else { 1386 return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1387 } 1388 } 1389 1390 int 1391 spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1392 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1393 uint32_t inject_flags, uint32_t *inject_offset) 1394 { 1395 struct _dif_sgl data_sgl, md_sgl; 1396 int rc; 1397 1398 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1399 _dif_sgl_init(&md_sgl, md_iov, 1); 1400 1401 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1402 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1403 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1404 return -EINVAL; 1405 } 1406 1407 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1408 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1409 ctx->guard_interval + offsetof(struct spdk_dif, ref_tag), 1410 _member_size(struct spdk_dif, ref_tag), 1411 inject_offset); 1412 if (rc != 0) { 1413 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1414 return rc; 1415 } 1416 } 1417 1418 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1419 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1420 ctx->guard_interval + offsetof(struct spdk_dif, app_tag), 1421 _member_size(struct spdk_dif, app_tag), 1422 inject_offset); 1423 if (rc != 0) { 1424 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1425 return rc; 1426 } 1427 } 1428 1429 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1430 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1431 ctx->guard_interval, 1432 _member_size(struct spdk_dif, guard), 1433 inject_offset); 1434 if (rc != 0) { 1435 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1436 return rc; 1437 } 1438 } 1439 1440 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1441 /* Note: Error injection to data block is expected to be detected 1442 * as guard error. 1443 */ 1444 rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks, 1445 0, 1446 ctx->block_size, 1447 inject_offset); 1448 if (rc != 0) { 1449 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1450 return rc; 1451 } 1452 } 1453 1454 return 0; 1455 } 1456 1457 static uint32_t 1458 _to_next_boundary(uint32_t offset, uint32_t boundary) 1459 { 1460 return boundary - (offset % boundary); 1461 } 1462 1463 int 1464 spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt, 1465 struct iovec *buf_iovs, int buf_iovcnt, 1466 uint32_t data_offset, uint32_t data_len, 1467 uint32_t *_mapped_len, 1468 const struct spdk_dif_ctx *ctx) 1469 { 1470 uint32_t data_block_size, data_unalign, buf_len, buf_offset, len; 1471 struct _dif_sgl dif_sgl; 1472 struct _dif_sgl buf_sgl; 1473 1474 if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) { 1475 return -EINVAL; 1476 } 1477 1478 data_block_size = ctx->block_size - ctx->md_size; 1479 1480 data_unalign = ctx->data_offset % data_block_size; 1481 1482 buf_len = ((data_unalign + data_offset + data_len) / data_block_size) * ctx->block_size + 1483 ((data_unalign + data_offset + data_len) % data_block_size); 1484 buf_len -= data_unalign; 1485 1486 _dif_sgl_init(&dif_sgl, iovs, iovcnt); 1487 _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt); 1488 1489 if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) { 1490 SPDK_ERRLOG("Buffer overflow will occur.\n"); 1491 return -ERANGE; 1492 } 1493 1494 buf_offset = ((data_unalign + data_offset) / data_block_size) * ctx->block_size + 1495 ((data_unalign + data_offset) % data_block_size); 1496 buf_offset -= data_unalign; 1497 1498 _dif_sgl_advance(&buf_sgl, buf_offset); 1499 1500 while (data_len != 0) { 1501 len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size)); 1502 if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) { 1503 break; 1504 } 1505 _dif_sgl_advance(&buf_sgl, ctx->md_size); 1506 data_offset += len; 1507 data_len -= len; 1508 } 1509 1510 if (_mapped_len != NULL) { 1511 *_mapped_len = dif_sgl.total_size; 1512 } 1513 1514 return iovcnt - dif_sgl.iovcnt; 1515 } 1516 1517 static int 1518 _dif_sgl_setup_stream(struct _dif_sgl *sgl, uint32_t *_buf_offset, uint32_t *_buf_len, 1519 uint32_t data_offset, uint32_t data_len, 1520 const struct spdk_dif_ctx *ctx) 1521 { 1522 uint32_t data_block_size, data_unalign, buf_len, buf_offset; 1523 1524 data_block_size = ctx->block_size - ctx->md_size; 1525 1526 data_unalign = ctx->data_offset % data_block_size; 1527 1528 /* If the last data block is complete, DIF of the data block is 1529 * inserted or verified in this turn. 1530 */ 1531 buf_len = ((data_unalign + data_offset + data_len) / data_block_size) * ctx->block_size + 1532 ((data_unalign + data_offset + data_len) % data_block_size); 1533 buf_len -= data_unalign; 1534 1535 if (!_dif_sgl_is_valid(sgl, buf_len)) { 1536 return -ERANGE; 1537 } 1538 1539 buf_offset = ((data_unalign + data_offset) / data_block_size) * ctx->block_size + 1540 ((data_unalign + data_offset) % data_block_size); 1541 buf_offset -= data_unalign; 1542 1543 _dif_sgl_advance(sgl, buf_offset); 1544 buf_len -= buf_offset; 1545 1546 buf_offset += data_unalign; 1547 1548 *_buf_offset = buf_offset; 1549 *_buf_len = buf_len; 1550 1551 return 0; 1552 } 1553 1554 int 1555 spdk_dif_generate_stream(struct iovec *iovs, int iovcnt, 1556 uint32_t data_offset, uint32_t data_len, 1557 struct spdk_dif_ctx *ctx) 1558 { 1559 uint32_t buf_len = 0, buf_offset = 0; 1560 uint32_t len, offset_in_block, offset_blocks; 1561 uint16_t guard = 0; 1562 struct _dif_sgl sgl; 1563 int rc; 1564 1565 if (iovs == NULL || iovcnt == 0) { 1566 return -EINVAL; 1567 } 1568 1569 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1570 guard = ctx->last_guard; 1571 } 1572 1573 _dif_sgl_init(&sgl, iovs, iovcnt); 1574 1575 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1576 if (rc != 0) { 1577 return rc; 1578 } 1579 1580 while (buf_len != 0) { 1581 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1582 offset_in_block = buf_offset % ctx->block_size; 1583 offset_blocks = buf_offset / ctx->block_size; 1584 1585 guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx); 1586 1587 buf_len -= len; 1588 buf_offset += len; 1589 } 1590 1591 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1592 ctx->last_guard = guard; 1593 } 1594 1595 return 0; 1596 } 1597 1598 int 1599 spdk_dif_verify_stream(struct iovec *iovs, int iovcnt, 1600 uint32_t data_offset, uint32_t data_len, 1601 struct spdk_dif_ctx *ctx, 1602 struct spdk_dif_error *err_blk) 1603 { 1604 uint32_t buf_len = 0, buf_offset = 0; 1605 uint32_t len, offset_in_block, offset_blocks; 1606 uint16_t guard = 0; 1607 struct _dif_sgl sgl; 1608 int rc = 0; 1609 1610 if (iovs == NULL || iovcnt == 0) { 1611 return -EINVAL; 1612 } 1613 1614 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1615 guard = ctx->last_guard; 1616 } 1617 1618 _dif_sgl_init(&sgl, iovs, iovcnt); 1619 1620 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1621 if (rc != 0) { 1622 return rc; 1623 } 1624 1625 while (buf_len != 0) { 1626 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1627 offset_in_block = buf_offset % ctx->block_size; 1628 offset_blocks = buf_offset / ctx->block_size; 1629 1630 rc = _dif_verify_split(&sgl, offset_in_block, len, &guard, offset_blocks, 1631 ctx, err_blk); 1632 if (rc != 0) { 1633 goto error; 1634 } 1635 1636 buf_len -= len; 1637 buf_offset += len; 1638 } 1639 1640 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1641 ctx->last_guard = guard; 1642 } 1643 error: 1644 return rc; 1645 } 1646