1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/dif.h" 35 #include "spdk/crc16.h" 36 #include "spdk/crc32.h" 37 #include "spdk/endian.h" 38 #include "spdk/log.h" 39 #include "spdk/util.h" 40 41 /* Context to iterate or create a iovec array. 42 * Each sgl is either iterated or created at a time. 43 */ 44 struct _dif_sgl { 45 /* Current iovec in the iteration or creation */ 46 struct iovec *iov; 47 48 /* Remaining count of iovecs in the iteration or creation. */ 49 int iovcnt; 50 51 /* Current offset in the iovec */ 52 uint32_t iov_offset; 53 54 /* Size of the created iovec array in bytes */ 55 uint32_t total_size; 56 }; 57 58 static inline void 59 _dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt) 60 { 61 s->iov = iovs; 62 s->iovcnt = iovcnt; 63 s->iov_offset = 0; 64 s->total_size = 0; 65 } 66 67 static void 68 _dif_sgl_advance(struct _dif_sgl *s, uint32_t step) 69 { 70 s->iov_offset += step; 71 while (s->iovcnt != 0) { 72 if (s->iov_offset < s->iov->iov_len) { 73 break; 74 } 75 76 s->iov_offset -= s->iov->iov_len; 77 s->iov++; 78 s->iovcnt--; 79 } 80 } 81 82 static inline void 83 _dif_sgl_get_buf(struct _dif_sgl *s, void **_buf, uint32_t *_buf_len) 84 { 85 if (_buf != NULL) { 86 *_buf = s->iov->iov_base + s->iov_offset; 87 } 88 if (_buf_len != NULL) { 89 *_buf_len = s->iov->iov_len - s->iov_offset; 90 } 91 } 92 93 static inline bool 94 _dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len) 95 { 96 assert(s->iovcnt > 0); 97 s->iov->iov_base = data; 98 s->iov->iov_len = data_len; 99 s->total_size += data_len; 100 s->iov++; 101 s->iovcnt--; 102 103 if (s->iovcnt > 0) { 104 return true; 105 } else { 106 return false; 107 } 108 } 109 110 static inline bool 111 _dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len) 112 { 113 uint8_t *buf; 114 uint32_t buf_len; 115 116 while (data_len != 0) { 117 _dif_sgl_get_buf(src, (void *)&buf, &buf_len); 118 buf_len = spdk_min(buf_len, data_len); 119 120 if (!_dif_sgl_append(dst, buf, buf_len)) { 121 return false; 122 } 123 124 _dif_sgl_advance(src, buf_len); 125 data_len -= buf_len; 126 } 127 128 return true; 129 } 130 131 /* This function must be used before starting iteration. */ 132 static bool 133 _dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes) 134 { 135 int i; 136 137 for (i = 0; i < s->iovcnt; i++) { 138 if (s->iov[i].iov_len % bytes) { 139 return false; 140 } 141 } 142 143 return true; 144 } 145 146 /* This function must be used before starting iteration. */ 147 static bool 148 _dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes) 149 { 150 uint64_t total = 0; 151 int i; 152 153 for (i = 0; i < s->iovcnt; i++) { 154 total += s->iov[i].iov_len; 155 } 156 157 return total >= bytes; 158 } 159 160 static bool 161 _dif_type_is_valid(enum spdk_dif_type dif_type, uint32_t dif_flags) 162 { 163 switch (dif_type) { 164 case SPDK_DIF_TYPE1: 165 case SPDK_DIF_TYPE2: 166 case SPDK_DIF_DISABLE: 167 break; 168 case SPDK_DIF_TYPE3: 169 if (dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 170 SPDK_ERRLOG("Reference Tag should not be checked for Type 3\n"); 171 return false; 172 } 173 break; 174 default: 175 SPDK_ERRLOG("Unknown DIF Type: %d\n", dif_type); 176 return false; 177 } 178 179 return true; 180 } 181 182 static bool 183 _dif_is_disabled(enum spdk_dif_type dif_type) 184 { 185 if (dif_type == SPDK_DIF_DISABLE) { 186 return true; 187 } else { 188 return false; 189 } 190 } 191 192 193 static uint32_t 194 _get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave) 195 { 196 if (!dif_loc) { 197 /* For metadata formats with more than 8 bytes, if the DIF is 198 * contained in the last 8 bytes of metadata, then the CRC 199 * covers all metadata up to but excluding these last 8 bytes. 200 */ 201 if (md_interleave) { 202 return block_size - sizeof(struct spdk_dif); 203 } else { 204 return md_size - sizeof(struct spdk_dif); 205 } 206 } else { 207 /* For metadata formats with more than 8 bytes, if the DIF is 208 * contained in the first 8 bytes of metadata, then the CRC 209 * does not cover any metadata. 210 */ 211 if (md_interleave) { 212 return block_size - md_size; 213 } else { 214 return 0; 215 } 216 } 217 } 218 219 int 220 spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, 221 bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, 222 uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag, 223 uint32_t data_offset, uint16_t guard_seed) 224 { 225 uint32_t data_block_size; 226 227 if (md_size < sizeof(struct spdk_dif)) { 228 SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); 229 return -EINVAL; 230 } 231 232 if (md_interleave) { 233 if (block_size < md_size) { 234 SPDK_ERRLOG("Block size is smaller than DIF size.\n"); 235 return -EINVAL; 236 } 237 data_block_size = block_size - md_size; 238 } else { 239 if (block_size == 0 || (block_size % 512) != 0) { 240 SPDK_ERRLOG("Zero block size is not allowed\n"); 241 return -EINVAL; 242 } 243 data_block_size = block_size; 244 } 245 246 if (!_dif_type_is_valid(dif_type, dif_flags)) { 247 SPDK_ERRLOG("DIF type is invalid.\n"); 248 return -EINVAL; 249 } 250 251 ctx->block_size = block_size; 252 ctx->md_size = md_size; 253 ctx->md_interleave = md_interleave; 254 ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave); 255 ctx->dif_type = dif_type; 256 ctx->dif_flags = dif_flags; 257 ctx->init_ref_tag = init_ref_tag; 258 ctx->apptag_mask = apptag_mask; 259 ctx->app_tag = app_tag; 260 ctx->data_offset = data_offset; 261 ctx->ref_tag_offset = data_offset / data_block_size; 262 ctx->last_guard = guard_seed; 263 ctx->guard_seed = guard_seed; 264 265 return 0; 266 } 267 268 void 269 spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset) 270 { 271 uint32_t data_block_size; 272 273 if (ctx->md_interleave) { 274 data_block_size = ctx->block_size - ctx->md_size; 275 } else { 276 data_block_size = ctx->block_size; 277 } 278 279 ctx->data_offset = data_offset; 280 ctx->ref_tag_offset = data_offset / data_block_size; 281 } 282 283 static void 284 _dif_generate(void *_dif, uint16_t guard, uint32_t offset_blocks, 285 const struct spdk_dif_ctx *ctx) 286 { 287 struct spdk_dif *dif = _dif; 288 uint32_t ref_tag; 289 290 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 291 to_be16(&dif->guard, guard); 292 } 293 294 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 295 to_be16(&dif->app_tag, ctx->app_tag); 296 } 297 298 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 299 /* For type 1 and 2, the reference tag is incremented for each 300 * subsequent logical block. For type 3, the reference tag 301 * remains the same as the initial reference tag. 302 */ 303 if (ctx->dif_type != SPDK_DIF_TYPE3) { 304 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 305 } else { 306 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 307 } 308 309 to_be32(&dif->ref_tag, ref_tag); 310 } 311 } 312 313 static void 314 dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 315 { 316 uint32_t offset_blocks = 0; 317 void *buf; 318 uint16_t guard = 0; 319 320 while (offset_blocks < num_blocks) { 321 _dif_sgl_get_buf(sgl, &buf, NULL); 322 323 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 324 guard = spdk_crc16_t10dif(ctx->guard_seed, buf, ctx->guard_interval); 325 } 326 327 _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); 328 329 _dif_sgl_advance(sgl, ctx->block_size); 330 offset_blocks++; 331 } 332 } 333 334 static uint16_t 335 _dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 336 uint16_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 337 { 338 uint32_t offset_in_dif, buf_len; 339 void *buf; 340 struct spdk_dif dif = {}; 341 342 assert(offset_in_block < ctx->guard_interval); 343 assert(offset_in_block + data_len < ctx->guard_interval || 344 offset_in_block + data_len == ctx->block_size); 345 346 /* Compute CRC over split logical block data. */ 347 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 348 _dif_sgl_get_buf(sgl, &buf, &buf_len); 349 buf_len = spdk_min(buf_len, data_len); 350 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 351 352 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 353 guard = spdk_crc16_t10dif(guard, buf, buf_len); 354 } 355 356 _dif_sgl_advance(sgl, buf_len); 357 offset_in_block += buf_len; 358 data_len -= buf_len; 359 } 360 361 if (offset_in_block < ctx->guard_interval) { 362 return guard; 363 } 364 365 /* If a whole logical block data is parsed, generate DIF 366 * and save it to the temporary DIF area. 367 */ 368 _dif_generate(&dif, guard, offset_blocks, ctx); 369 370 /* Copy generated DIF field to the split DIF field, and then 371 * skip metadata field after DIF field (if any). 372 */ 373 while (offset_in_block < ctx->block_size) { 374 _dif_sgl_get_buf(sgl, &buf, &buf_len); 375 376 if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) { 377 offset_in_dif = offset_in_block - ctx->guard_interval; 378 buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif); 379 380 memcpy(buf, ((uint8_t *)&dif) + offset_in_dif, buf_len); 381 } else { 382 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 383 } 384 385 _dif_sgl_advance(sgl, buf_len); 386 offset_in_block += buf_len; 387 } 388 389 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 390 guard = ctx->guard_seed; 391 } 392 393 return guard; 394 } 395 396 static void 397 dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks, 398 const struct spdk_dif_ctx *ctx) 399 { 400 uint32_t offset_blocks; 401 uint16_t guard = 0; 402 403 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 404 guard = ctx->guard_seed; 405 } 406 407 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 408 _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx); 409 } 410 } 411 412 int 413 spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 414 const struct spdk_dif_ctx *ctx) 415 { 416 struct _dif_sgl sgl; 417 418 _dif_sgl_init(&sgl, iovs, iovcnt); 419 420 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 421 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 422 return -EINVAL; 423 } 424 425 if (_dif_is_disabled(ctx->dif_type)) { 426 return 0; 427 } 428 429 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 430 dif_generate(&sgl, num_blocks, ctx); 431 } else { 432 dif_generate_split(&sgl, num_blocks, ctx); 433 } 434 435 return 0; 436 } 437 438 static void 439 _dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, 440 uint32_t expected, uint32_t actual, uint32_t err_offset) 441 { 442 if (err_blk) { 443 err_blk->err_type = err_type; 444 err_blk->expected = expected; 445 err_blk->actual = actual; 446 err_blk->err_offset = err_offset; 447 } 448 } 449 450 static int 451 _dif_verify(void *_dif, uint16_t guard, uint32_t offset_blocks, 452 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 453 { 454 struct spdk_dif *dif = _dif; 455 uint16_t _guard; 456 uint16_t _app_tag; 457 uint32_t ref_tag, _ref_tag; 458 459 switch (ctx->dif_type) { 460 case SPDK_DIF_TYPE1: 461 case SPDK_DIF_TYPE2: 462 /* If Type 1 or 2 is used, then all DIF checks are disabled when 463 * the Application Tag is 0xFFFF. 464 */ 465 if (dif->app_tag == 0xFFFF) { 466 return 0; 467 } 468 break; 469 case SPDK_DIF_TYPE3: 470 /* If Type 3 is used, then all DIF checks are disabled when the 471 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF. 472 */ 473 if (dif->app_tag == 0xFFFF && dif->ref_tag == 0xFFFFFFFF) { 474 return 0; 475 } 476 break; 477 default: 478 break; 479 } 480 481 /* For type 1 and 2, the reference tag is incremented for each 482 * subsequent logical block. For type 3, the reference tag 483 * remains the same as the initial reference tag. 484 */ 485 if (ctx->dif_type != SPDK_DIF_TYPE3) { 486 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 487 } else { 488 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 489 } 490 491 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 492 /* Compare the DIF Guard field to the CRC computed over the logical 493 * block data. 494 */ 495 _guard = from_be16(&dif->guard); 496 if (_guard != guard) { 497 _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, 498 offset_blocks); 499 SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu32 "," \ 500 " Expected=%x, Actual=%x\n", 501 ref_tag, _guard, guard); 502 return -1; 503 } 504 } 505 506 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 507 /* Compare unmasked bits in the DIF Application Tag field to the 508 * passed Application Tag. 509 */ 510 _app_tag = from_be16(&dif->app_tag); 511 if ((_app_tag & ctx->apptag_mask) != ctx->app_tag) { 512 _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, 513 (_app_tag & ctx->apptag_mask), offset_blocks); 514 SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu32 "," \ 515 " Expected=%x, Actual=%x\n", 516 ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); 517 return -1; 518 } 519 } 520 521 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 522 switch (ctx->dif_type) { 523 case SPDK_DIF_TYPE1: 524 case SPDK_DIF_TYPE2: 525 /* Compare the DIF Reference Tag field to the passed Reference Tag. 526 * The passed Reference Tag will be the least significant 4 bytes 527 * of the LBA when Type 1 is used, and application specific value 528 * if Type 2 is used, 529 */ 530 _ref_tag = from_be32(&dif->ref_tag); 531 if (_ref_tag != ref_tag) { 532 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, ref_tag, 533 _ref_tag, offset_blocks); 534 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu32 "," \ 535 " Expected=%x, Actual=%x\n", 536 ref_tag, ref_tag, _ref_tag); 537 return -1; 538 } 539 break; 540 case SPDK_DIF_TYPE3: 541 /* For Type 3, computed Reference Tag remains unchanged. 542 * Hence ignore the Reference Tag field. 543 */ 544 break; 545 default: 546 break; 547 } 548 } 549 550 return 0; 551 } 552 553 static int 554 dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks, 555 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 556 { 557 uint32_t offset_blocks = 0; 558 int rc; 559 void *buf; 560 uint16_t guard = 0; 561 562 while (offset_blocks < num_blocks) { 563 _dif_sgl_get_buf(sgl, &buf, NULL); 564 565 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 566 guard = spdk_crc16_t10dif(ctx->guard_seed, buf, ctx->guard_interval); 567 } 568 569 rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 570 if (rc != 0) { 571 return rc; 572 } 573 574 _dif_sgl_advance(sgl, ctx->block_size); 575 offset_blocks++; 576 } 577 578 return 0; 579 } 580 581 static int 582 _dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 583 uint16_t *_guard, uint32_t offset_blocks, 584 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 585 { 586 uint32_t offset_in_dif, buf_len; 587 void *buf; 588 uint16_t guard; 589 struct spdk_dif dif = {}; 590 int rc; 591 592 assert(_guard != NULL); 593 assert(offset_in_block < ctx->guard_interval); 594 assert(offset_in_block + data_len < ctx->guard_interval || 595 offset_in_block + data_len == ctx->block_size); 596 597 guard = *_guard; 598 599 /* Compute CRC over split logical block data. */ 600 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 601 _dif_sgl_get_buf(sgl, &buf, &buf_len); 602 buf_len = spdk_min(buf_len, data_len); 603 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 604 605 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 606 guard = spdk_crc16_t10dif(guard, buf, buf_len); 607 } 608 609 _dif_sgl_advance(sgl, buf_len); 610 offset_in_block += buf_len; 611 data_len -= buf_len; 612 } 613 614 if (offset_in_block < ctx->guard_interval) { 615 *_guard = guard; 616 return 0; 617 } 618 619 /* Copy the split DIF field to the temporary DIF buffer, and then 620 * skip metadata field after DIF field (if any). */ 621 while (offset_in_block < ctx->block_size) { 622 _dif_sgl_get_buf(sgl, &buf, &buf_len); 623 624 if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) { 625 offset_in_dif = offset_in_block - ctx->guard_interval; 626 buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif); 627 628 memcpy((uint8_t *)&dif + offset_in_dif, buf, buf_len); 629 } else { 630 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 631 } 632 _dif_sgl_advance(sgl, buf_len); 633 offset_in_block += buf_len; 634 } 635 636 rc = _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 637 if (rc != 0) { 638 return rc; 639 } 640 641 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 642 guard = ctx->guard_seed; 643 } 644 645 *_guard = guard; 646 return 0; 647 } 648 649 static int 650 dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks, 651 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 652 { 653 uint32_t offset_blocks; 654 uint16_t guard = 0; 655 int rc; 656 657 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 658 guard = ctx->guard_seed; 659 } 660 661 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 662 rc = _dif_verify_split(sgl, 0, ctx->block_size, &guard, offset_blocks, 663 ctx, err_blk); 664 if (rc != 0) { 665 return rc; 666 } 667 } 668 669 return 0; 670 } 671 672 int 673 spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 674 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 675 { 676 struct _dif_sgl sgl; 677 678 _dif_sgl_init(&sgl, iovs, iovcnt); 679 680 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 681 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 682 return -EINVAL; 683 } 684 685 if (_dif_is_disabled(ctx->dif_type)) { 686 return 0; 687 } 688 689 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 690 return dif_verify(&sgl, num_blocks, ctx, err_blk); 691 } else { 692 return dif_verify_split(&sgl, num_blocks, ctx, err_blk); 693 } 694 } 695 696 static uint32_t 697 dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks, 698 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 699 { 700 uint32_t offset_blocks; 701 void *buf; 702 703 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 704 _dif_sgl_get_buf(sgl, &buf, NULL); 705 706 crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c); 707 708 _dif_sgl_advance(sgl, ctx->block_size); 709 } 710 711 return crc32c; 712 } 713 714 static uint32_t 715 _dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 716 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 717 { 718 uint32_t data_block_size, buf_len; 719 void *buf; 720 721 data_block_size = ctx->block_size - ctx->md_size; 722 723 assert(offset_in_block + data_len <= ctx->block_size); 724 725 while (data_len != 0) { 726 _dif_sgl_get_buf(sgl, &buf, &buf_len); 727 buf_len = spdk_min(buf_len, data_len); 728 729 if (offset_in_block < data_block_size) { 730 buf_len = spdk_min(buf_len, data_block_size - offset_in_block); 731 crc32c = spdk_crc32c_update(buf, buf_len, crc32c); 732 } 733 734 _dif_sgl_advance(sgl, buf_len); 735 offset_in_block += buf_len; 736 data_len -= buf_len; 737 } 738 739 return crc32c; 740 } 741 742 static uint32_t 743 dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks, 744 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 745 { 746 uint32_t offset_blocks; 747 748 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 749 crc32c = _dif_update_crc32c_split(sgl, 0, ctx->block_size, crc32c, ctx); 750 } 751 752 return crc32c; 753 } 754 755 int 756 spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 757 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 758 { 759 struct _dif_sgl sgl; 760 761 if (_crc32c == NULL) { 762 return -EINVAL; 763 } 764 765 _dif_sgl_init(&sgl, iovs, iovcnt); 766 767 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 768 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 769 return -EINVAL; 770 } 771 772 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 773 *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx); 774 } else { 775 *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx); 776 } 777 778 return 0; 779 } 780 781 static void 782 dif_generate_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 783 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 784 { 785 uint32_t offset_blocks = 0, data_block_size; 786 void *src, *dst; 787 uint16_t guard; 788 789 data_block_size = ctx->block_size - ctx->md_size; 790 791 while (offset_blocks < num_blocks) { 792 _dif_sgl_get_buf(src_sgl, &src, NULL); 793 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 794 795 guard = 0; 796 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 797 guard = spdk_crc16_t10dif_copy(ctx->guard_seed, dst, src, data_block_size); 798 guard = spdk_crc16_t10dif(guard, dst + data_block_size, 799 ctx->guard_interval - data_block_size); 800 } else { 801 memcpy(dst, src, data_block_size); 802 } 803 804 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 805 806 _dif_sgl_advance(src_sgl, data_block_size); 807 _dif_sgl_advance(dst_sgl, ctx->block_size); 808 offset_blocks++; 809 } 810 } 811 812 static void 813 _dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 814 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 815 { 816 uint32_t offset_in_block, src_len, data_block_size; 817 uint16_t guard = 0; 818 void *src, *dst; 819 820 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 821 822 data_block_size = ctx->block_size - ctx->md_size; 823 824 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 825 guard = ctx->guard_seed; 826 } 827 offset_in_block = 0; 828 829 while (offset_in_block < data_block_size) { 830 /* Compute CRC over split logical block data and copy 831 * data to bounce buffer. 832 */ 833 _dif_sgl_get_buf(src_sgl, &src, &src_len); 834 src_len = spdk_min(src_len, data_block_size - offset_in_block); 835 836 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 837 guard = spdk_crc16_t10dif_copy(guard, dst + offset_in_block, 838 src, src_len); 839 } else { 840 memcpy(dst + offset_in_block, src, src_len); 841 } 842 843 _dif_sgl_advance(src_sgl, src_len); 844 offset_in_block += src_len; 845 } 846 847 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 848 guard = spdk_crc16_t10dif(guard, dst + data_block_size, 849 ctx->guard_interval - data_block_size); 850 } 851 852 _dif_sgl_advance(dst_sgl, ctx->block_size); 853 854 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 855 } 856 857 static void 858 dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 859 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 860 { 861 uint32_t offset_blocks; 862 863 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 864 _dif_generate_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 865 } 866 } 867 868 int 869 spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 870 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 871 { 872 struct _dif_sgl src_sgl, dst_sgl; 873 uint32_t data_block_size; 874 875 _dif_sgl_init(&src_sgl, iovs, iovcnt); 876 _dif_sgl_init(&dst_sgl, bounce_iov, 1); 877 878 data_block_size = ctx->block_size - ctx->md_size; 879 880 if (!_dif_sgl_is_valid(&src_sgl, data_block_size * num_blocks) || 881 !_dif_sgl_is_valid(&dst_sgl, ctx->block_size * num_blocks)) { 882 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 883 return -EINVAL; 884 } 885 886 if (_dif_is_disabled(ctx->dif_type)) { 887 return 0; 888 } 889 890 if (_dif_sgl_is_bytes_multiple(&src_sgl, data_block_size)) { 891 dif_generate_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 892 } else { 893 dif_generate_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx); 894 } 895 896 return 0; 897 } 898 899 static int 900 dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 901 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 902 struct spdk_dif_error *err_blk) 903 { 904 uint32_t offset_blocks = 0, data_block_size; 905 void *src, *dst; 906 int rc; 907 uint16_t guard; 908 909 data_block_size = ctx->block_size - ctx->md_size; 910 911 while (offset_blocks < num_blocks) { 912 _dif_sgl_get_buf(src_sgl, &src, NULL); 913 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 914 915 guard = 0; 916 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 917 guard = spdk_crc16_t10dif_copy(ctx->guard_seed, dst, src, data_block_size); 918 guard = spdk_crc16_t10dif(guard, src + data_block_size, 919 ctx->guard_interval - data_block_size); 920 } else { 921 memcpy(dst, src, data_block_size); 922 } 923 924 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 925 if (rc != 0) { 926 return rc; 927 } 928 929 _dif_sgl_advance(src_sgl, ctx->block_size); 930 _dif_sgl_advance(dst_sgl, data_block_size); 931 offset_blocks++; 932 } 933 934 return 0; 935 } 936 937 static int 938 _dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 939 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 940 struct spdk_dif_error *err_blk) 941 { 942 uint32_t offset_in_block, dst_len, data_block_size; 943 uint16_t guard = 0; 944 void *src, *dst; 945 946 _dif_sgl_get_buf(src_sgl, &src, NULL); 947 948 data_block_size = ctx->block_size - ctx->md_size; 949 950 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 951 guard = ctx->guard_seed; 952 } 953 offset_in_block = 0; 954 955 while (offset_in_block < data_block_size) { 956 /* Compute CRC over split logical block data and copy 957 * data to bounce buffer. 958 */ 959 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 960 dst_len = spdk_min(dst_len, data_block_size - offset_in_block); 961 962 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 963 guard = spdk_crc16_t10dif_copy(guard, dst, 964 src + offset_in_block, dst_len); 965 } else { 966 memcpy(dst, src + offset_in_block, dst_len); 967 } 968 969 _dif_sgl_advance(dst_sgl, dst_len); 970 offset_in_block += dst_len; 971 } 972 973 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 974 guard = spdk_crc16_t10dif(guard, src + data_block_size, 975 ctx->guard_interval - data_block_size); 976 } 977 978 _dif_sgl_advance(src_sgl, ctx->block_size); 979 980 return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 981 } 982 983 static int 984 dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 985 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 986 struct spdk_dif_error *err_blk) 987 { 988 uint32_t offset_blocks; 989 int rc; 990 991 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 992 rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 993 if (rc != 0) { 994 return rc; 995 } 996 } 997 998 return 0; 999 } 1000 1001 int 1002 spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, 1003 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1004 struct spdk_dif_error *err_blk) 1005 { 1006 struct _dif_sgl src_sgl, dst_sgl; 1007 uint32_t data_block_size; 1008 1009 _dif_sgl_init(&src_sgl, bounce_iov, 1); 1010 _dif_sgl_init(&dst_sgl, iovs, iovcnt); 1011 1012 data_block_size = ctx->block_size - ctx->md_size; 1013 1014 if (!_dif_sgl_is_valid(&dst_sgl, data_block_size * num_blocks) || 1015 !_dif_sgl_is_valid(&src_sgl, ctx->block_size * num_blocks)) { 1016 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1017 return -EINVAL; 1018 } 1019 1020 if (_dif_is_disabled(ctx->dif_type)) { 1021 return 0; 1022 } 1023 1024 if (_dif_sgl_is_bytes_multiple(&dst_sgl, data_block_size)) { 1025 return dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1026 } else { 1027 return dif_verify_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1028 } 1029 } 1030 1031 static void 1032 _bit_flip(uint8_t *buf, uint32_t flip_bit) 1033 { 1034 uint8_t byte; 1035 1036 byte = *buf; 1037 byte ^= 1 << flip_bit; 1038 *buf = byte; 1039 } 1040 1041 static int 1042 _dif_inject_error(struct _dif_sgl *sgl, 1043 uint32_t block_size, uint32_t num_blocks, 1044 uint32_t inject_offset_blocks, 1045 uint32_t inject_offset_bytes, 1046 uint32_t inject_offset_bits) 1047 { 1048 uint32_t offset_in_block, buf_len; 1049 void *buf; 1050 1051 _dif_sgl_advance(sgl, block_size * inject_offset_blocks); 1052 1053 offset_in_block = 0; 1054 1055 while (offset_in_block < block_size) { 1056 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1057 buf_len = spdk_min(buf_len, block_size - offset_in_block); 1058 1059 if (inject_offset_bytes >= offset_in_block && 1060 inject_offset_bytes < offset_in_block + buf_len) { 1061 buf += inject_offset_bytes - offset_in_block; 1062 _bit_flip(buf, inject_offset_bits); 1063 return 0; 1064 } 1065 1066 _dif_sgl_advance(sgl, buf_len); 1067 offset_in_block += buf_len; 1068 } 1069 1070 return -1; 1071 } 1072 1073 static int 1074 dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks, 1075 uint32_t start_inject_bytes, uint32_t inject_range_bytes, 1076 uint32_t *inject_offset) 1077 { 1078 uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; 1079 uint32_t offset_blocks; 1080 int rc; 1081 1082 srand(time(0)); 1083 1084 inject_offset_blocks = rand() % num_blocks; 1085 inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); 1086 inject_offset_bits = rand() % 8; 1087 1088 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1089 if (offset_blocks == inject_offset_blocks) { 1090 rc = _dif_inject_error(sgl, block_size, num_blocks, 1091 inject_offset_blocks, 1092 inject_offset_bytes, 1093 inject_offset_bits); 1094 if (rc == 0) { 1095 *inject_offset = inject_offset_blocks; 1096 } 1097 return rc; 1098 } 1099 } 1100 1101 return -1; 1102 } 1103 1104 #define _member_size(type, member) sizeof(((type *)0)->member) 1105 1106 int 1107 spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1108 const struct spdk_dif_ctx *ctx, uint32_t inject_flags, 1109 uint32_t *inject_offset) 1110 { 1111 struct _dif_sgl sgl; 1112 int rc; 1113 1114 _dif_sgl_init(&sgl, iovs, iovcnt); 1115 1116 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1117 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1118 return -EINVAL; 1119 } 1120 1121 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1122 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1123 ctx->guard_interval + offsetof(struct spdk_dif, ref_tag), 1124 _member_size(struct spdk_dif, ref_tag), 1125 inject_offset); 1126 if (rc != 0) { 1127 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1128 return rc; 1129 } 1130 } 1131 1132 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1133 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1134 ctx->guard_interval + offsetof(struct spdk_dif, app_tag), 1135 _member_size(struct spdk_dif, app_tag), 1136 inject_offset); 1137 if (rc != 0) { 1138 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1139 return rc; 1140 } 1141 } 1142 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1143 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1144 ctx->guard_interval, 1145 _member_size(struct spdk_dif, guard), 1146 inject_offset); 1147 if (rc != 0) { 1148 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1149 return rc; 1150 } 1151 } 1152 1153 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1154 /* If the DIF information is contained within the last 8 bytes of 1155 * metadata, then the CRC covers all metadata bytes up to but excluding 1156 * the last 8 bytes. But error injection does not cover these metadata 1157 * because classification is not determined yet. 1158 * 1159 * Note: Error injection to data block is expected to be detected as 1160 * guard error. 1161 */ 1162 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1163 0, 1164 ctx->block_size - ctx->md_size, 1165 inject_offset); 1166 if (rc != 0) { 1167 SPDK_ERRLOG("Failed to inject error to data block.\n"); 1168 return rc; 1169 } 1170 } 1171 1172 return 0; 1173 } 1174 1175 static void 1176 dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1177 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1178 { 1179 uint32_t offset_blocks = 0; 1180 uint16_t guard; 1181 void *data_buf, *md_buf; 1182 1183 while (offset_blocks < num_blocks) { 1184 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1185 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1186 1187 guard = 0; 1188 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1189 guard = spdk_crc16_t10dif(ctx->guard_seed, data_buf, ctx->block_size); 1190 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1191 } 1192 1193 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1194 1195 _dif_sgl_advance(data_sgl, ctx->block_size); 1196 _dif_sgl_advance(md_sgl, ctx->md_size); 1197 offset_blocks++; 1198 } 1199 } 1200 1201 static void 1202 _dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1203 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1204 { 1205 uint32_t offset_in_block, data_buf_len; 1206 uint16_t guard = 0; 1207 void *data_buf, *md_buf; 1208 1209 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1210 1211 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1212 guard = ctx->guard_seed; 1213 } 1214 offset_in_block = 0; 1215 1216 while (offset_in_block < ctx->block_size) { 1217 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1218 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1219 1220 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1221 guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len); 1222 } 1223 1224 _dif_sgl_advance(data_sgl, data_buf_len); 1225 offset_in_block += data_buf_len; 1226 } 1227 1228 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1229 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1230 } 1231 1232 _dif_sgl_advance(md_sgl, ctx->md_size); 1233 1234 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1235 } 1236 1237 static void 1238 dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1239 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1240 { 1241 uint32_t offset_blocks; 1242 1243 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1244 _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx); 1245 } 1246 } 1247 1248 int 1249 spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1250 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1251 { 1252 struct _dif_sgl data_sgl, md_sgl; 1253 1254 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1255 _dif_sgl_init(&md_sgl, md_iov, 1); 1256 1257 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1258 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1259 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1260 return -EINVAL; 1261 } 1262 1263 if (_dif_is_disabled(ctx->dif_type)) { 1264 return 0; 1265 } 1266 1267 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1268 dix_generate(&data_sgl, &md_sgl, num_blocks, ctx); 1269 } else { 1270 dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx); 1271 } 1272 1273 return 0; 1274 } 1275 1276 static int 1277 dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1278 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1279 struct spdk_dif_error *err_blk) 1280 { 1281 uint32_t offset_blocks = 0; 1282 uint16_t guard; 1283 void *data_buf, *md_buf; 1284 int rc; 1285 1286 while (offset_blocks < num_blocks) { 1287 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1288 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1289 1290 guard = 0; 1291 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1292 guard = spdk_crc16_t10dif(ctx->guard_seed, data_buf, ctx->block_size); 1293 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1294 } 1295 1296 rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1297 if (rc != 0) { 1298 return rc; 1299 } 1300 1301 _dif_sgl_advance(data_sgl, ctx->block_size); 1302 _dif_sgl_advance(md_sgl, ctx->md_size); 1303 offset_blocks++; 1304 } 1305 1306 return 0; 1307 } 1308 1309 static int 1310 _dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1311 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1312 struct spdk_dif_error *err_blk) 1313 { 1314 uint32_t offset_in_block, data_buf_len; 1315 uint16_t guard = 0; 1316 void *data_buf, *md_buf; 1317 1318 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1319 1320 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1321 guard = ctx->guard_seed; 1322 } 1323 offset_in_block = 0; 1324 1325 while (offset_in_block < ctx->block_size) { 1326 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1327 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1328 1329 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1330 guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len); 1331 } 1332 1333 _dif_sgl_advance(data_sgl, data_buf_len); 1334 offset_in_block += data_buf_len; 1335 } 1336 1337 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1338 guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); 1339 } 1340 1341 _dif_sgl_advance(md_sgl, ctx->md_size); 1342 1343 return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1344 } 1345 1346 static int 1347 dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1348 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1349 struct spdk_dif_error *err_blk) 1350 { 1351 uint32_t offset_blocks; 1352 int rc; 1353 1354 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1355 rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk); 1356 if (rc != 0) { 1357 return rc; 1358 } 1359 } 1360 1361 return 0; 1362 } 1363 1364 int 1365 spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1366 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1367 struct spdk_dif_error *err_blk) 1368 { 1369 struct _dif_sgl data_sgl, md_sgl; 1370 1371 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1372 _dif_sgl_init(&md_sgl, md_iov, 1); 1373 1374 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1375 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1376 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1377 return -EINVAL; 1378 } 1379 1380 if (_dif_is_disabled(ctx->dif_type)) { 1381 return 0; 1382 } 1383 1384 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1385 return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1386 } else { 1387 return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1388 } 1389 } 1390 1391 int 1392 spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1393 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1394 uint32_t inject_flags, uint32_t *inject_offset) 1395 { 1396 struct _dif_sgl data_sgl, md_sgl; 1397 int rc; 1398 1399 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1400 _dif_sgl_init(&md_sgl, md_iov, 1); 1401 1402 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1403 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1404 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1405 return -EINVAL; 1406 } 1407 1408 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1409 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1410 ctx->guard_interval + offsetof(struct spdk_dif, ref_tag), 1411 _member_size(struct spdk_dif, ref_tag), 1412 inject_offset); 1413 if (rc != 0) { 1414 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1415 return rc; 1416 } 1417 } 1418 1419 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1420 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1421 ctx->guard_interval + offsetof(struct spdk_dif, app_tag), 1422 _member_size(struct spdk_dif, app_tag), 1423 inject_offset); 1424 if (rc != 0) { 1425 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1426 return rc; 1427 } 1428 } 1429 1430 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1431 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1432 ctx->guard_interval, 1433 _member_size(struct spdk_dif, guard), 1434 inject_offset); 1435 if (rc != 0) { 1436 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1437 return rc; 1438 } 1439 } 1440 1441 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1442 /* Note: Error injection to data block is expected to be detected 1443 * as guard error. 1444 */ 1445 rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks, 1446 0, 1447 ctx->block_size, 1448 inject_offset); 1449 if (rc != 0) { 1450 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1451 return rc; 1452 } 1453 } 1454 1455 return 0; 1456 } 1457 1458 static uint32_t 1459 _to_next_boundary(uint32_t offset, uint32_t boundary) 1460 { 1461 return boundary - (offset % boundary); 1462 } 1463 1464 static uint32_t 1465 _to_size_with_md(uint32_t size, uint32_t data_block_size, uint32_t block_size) 1466 { 1467 return (size / data_block_size) * block_size + (size % data_block_size); 1468 } 1469 1470 int 1471 spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt, 1472 struct iovec *buf_iovs, int buf_iovcnt, 1473 uint32_t data_offset, uint32_t data_len, 1474 uint32_t *_mapped_len, 1475 const struct spdk_dif_ctx *ctx) 1476 { 1477 uint32_t data_block_size, data_unalign, buf_len, buf_offset, len; 1478 struct _dif_sgl dif_sgl; 1479 struct _dif_sgl buf_sgl; 1480 1481 if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) { 1482 return -EINVAL; 1483 } 1484 1485 data_block_size = ctx->block_size - ctx->md_size; 1486 1487 data_unalign = ctx->data_offset % data_block_size; 1488 1489 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 1490 ctx->block_size); 1491 buf_len -= data_unalign; 1492 1493 _dif_sgl_init(&dif_sgl, iovs, iovcnt); 1494 _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt); 1495 1496 if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) { 1497 SPDK_ERRLOG("Buffer overflow will occur.\n"); 1498 return -ERANGE; 1499 } 1500 1501 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 1502 buf_offset -= data_unalign; 1503 1504 _dif_sgl_advance(&buf_sgl, buf_offset); 1505 1506 while (data_len != 0) { 1507 len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size)); 1508 if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) { 1509 break; 1510 } 1511 _dif_sgl_advance(&buf_sgl, ctx->md_size); 1512 data_offset += len; 1513 data_len -= len; 1514 } 1515 1516 if (_mapped_len != NULL) { 1517 *_mapped_len = dif_sgl.total_size; 1518 } 1519 1520 return iovcnt - dif_sgl.iovcnt; 1521 } 1522 1523 static int 1524 _dif_sgl_setup_stream(struct _dif_sgl *sgl, uint32_t *_buf_offset, uint32_t *_buf_len, 1525 uint32_t data_offset, uint32_t data_len, 1526 const struct spdk_dif_ctx *ctx) 1527 { 1528 uint32_t data_block_size, data_unalign, buf_len, buf_offset; 1529 1530 data_block_size = ctx->block_size - ctx->md_size; 1531 1532 data_unalign = ctx->data_offset % data_block_size; 1533 1534 /* If the last data block is complete, DIF of the data block is 1535 * inserted or verified in this turn. 1536 */ 1537 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 1538 ctx->block_size); 1539 buf_len -= data_unalign; 1540 1541 if (!_dif_sgl_is_valid(sgl, buf_len)) { 1542 return -ERANGE; 1543 } 1544 1545 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 1546 buf_offset -= data_unalign; 1547 1548 _dif_sgl_advance(sgl, buf_offset); 1549 buf_len -= buf_offset; 1550 1551 buf_offset += data_unalign; 1552 1553 *_buf_offset = buf_offset; 1554 *_buf_len = buf_len; 1555 1556 return 0; 1557 } 1558 1559 int 1560 spdk_dif_generate_stream(struct iovec *iovs, int iovcnt, 1561 uint32_t data_offset, uint32_t data_len, 1562 struct spdk_dif_ctx *ctx) 1563 { 1564 uint32_t buf_len = 0, buf_offset = 0; 1565 uint32_t len, offset_in_block, offset_blocks; 1566 uint16_t guard = 0; 1567 struct _dif_sgl sgl; 1568 int rc; 1569 1570 if (iovs == NULL || iovcnt == 0) { 1571 return -EINVAL; 1572 } 1573 1574 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1575 guard = ctx->last_guard; 1576 } 1577 1578 _dif_sgl_init(&sgl, iovs, iovcnt); 1579 1580 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1581 if (rc != 0) { 1582 return rc; 1583 } 1584 1585 while (buf_len != 0) { 1586 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1587 offset_in_block = buf_offset % ctx->block_size; 1588 offset_blocks = buf_offset / ctx->block_size; 1589 1590 guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx); 1591 1592 buf_len -= len; 1593 buf_offset += len; 1594 } 1595 1596 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1597 ctx->last_guard = guard; 1598 } 1599 1600 return 0; 1601 } 1602 1603 int 1604 spdk_dif_verify_stream(struct iovec *iovs, int iovcnt, 1605 uint32_t data_offset, uint32_t data_len, 1606 struct spdk_dif_ctx *ctx, 1607 struct spdk_dif_error *err_blk) 1608 { 1609 uint32_t buf_len = 0, buf_offset = 0; 1610 uint32_t len, offset_in_block, offset_blocks; 1611 uint16_t guard = 0; 1612 struct _dif_sgl sgl; 1613 int rc = 0; 1614 1615 if (iovs == NULL || iovcnt == 0) { 1616 return -EINVAL; 1617 } 1618 1619 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1620 guard = ctx->last_guard; 1621 } 1622 1623 _dif_sgl_init(&sgl, iovs, iovcnt); 1624 1625 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1626 if (rc != 0) { 1627 return rc; 1628 } 1629 1630 while (buf_len != 0) { 1631 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1632 offset_in_block = buf_offset % ctx->block_size; 1633 offset_blocks = buf_offset / ctx->block_size; 1634 1635 rc = _dif_verify_split(&sgl, offset_in_block, len, &guard, offset_blocks, 1636 ctx, err_blk); 1637 if (rc != 0) { 1638 goto error; 1639 } 1640 1641 buf_len -= len; 1642 buf_offset += len; 1643 } 1644 1645 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1646 ctx->last_guard = guard; 1647 } 1648 error: 1649 return rc; 1650 } 1651 1652 int 1653 spdk_dif_update_crc32c_stream(struct iovec *iovs, int iovcnt, 1654 uint32_t data_offset, uint32_t data_len, 1655 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 1656 { 1657 uint32_t buf_len = 0, buf_offset = 0, len, offset_in_block; 1658 uint32_t crc32c; 1659 struct _dif_sgl sgl; 1660 int rc; 1661 1662 if (iovs == NULL || iovcnt == 0) { 1663 return -EINVAL; 1664 } 1665 1666 crc32c = *_crc32c; 1667 _dif_sgl_init(&sgl, iovs, iovcnt); 1668 1669 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1670 if (rc != 0) { 1671 return rc; 1672 } 1673 1674 while (buf_len != 0) { 1675 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1676 offset_in_block = buf_offset % ctx->block_size; 1677 1678 crc32c = _dif_update_crc32c_split(&sgl, offset_in_block, len, crc32c, ctx); 1679 1680 buf_len -= len; 1681 buf_offset += len; 1682 } 1683 1684 *_crc32c = crc32c; 1685 1686 return 0; 1687 } 1688 1689 void 1690 spdk_dif_get_range_with_md(uint32_t data_offset, uint32_t data_len, 1691 uint32_t *_buf_offset, uint32_t *_buf_len, 1692 const struct spdk_dif_ctx *ctx) 1693 { 1694 uint32_t data_block_size, data_unalign, buf_offset, buf_len; 1695 1696 if (!ctx->md_interleave) { 1697 buf_offset = data_offset; 1698 buf_len = data_len; 1699 } else { 1700 data_block_size = ctx->block_size - ctx->md_size; 1701 1702 data_unalign = data_offset % data_block_size; 1703 1704 buf_offset = _to_size_with_md(data_offset, data_block_size, ctx->block_size); 1705 buf_len = _to_size_with_md(data_unalign + data_len, data_block_size, ctx->block_size) - 1706 data_unalign; 1707 } 1708 1709 if (_buf_offset != NULL) { 1710 *_buf_offset = buf_offset; 1711 } 1712 1713 if (_buf_len != NULL) { 1714 *_buf_len = buf_len; 1715 } 1716 } 1717 1718 uint32_t 1719 spdk_dif_get_length_with_md(uint32_t data_len, const struct spdk_dif_ctx *ctx) 1720 { 1721 uint32_t data_block_size; 1722 1723 if (!ctx->md_interleave) { 1724 return data_len; 1725 } else { 1726 data_block_size = ctx->block_size - ctx->md_size; 1727 1728 return _to_size_with_md(data_len, data_block_size, ctx->block_size); 1729 } 1730 } 1731