1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/dif.h" 7 #include "spdk/crc16.h" 8 #include "spdk/crc32.h" 9 #include "spdk/endian.h" 10 #include "spdk/log.h" 11 #include "spdk/util.h" 12 13 #define APPTAG_IGNORE 0xFFFF 14 #define REFTAG_MASK_16 0x00000000FFFFFFFF 15 16 struct spdk_dif { 17 union { 18 struct { 19 uint16_t guard; 20 uint16_t app_tag; 21 uint32_t stor_ref_space; 22 } g16; 23 }; 24 }; 25 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g16) == 8, "Incorrect size"); 26 27 /* Context to iterate or create a iovec array. 28 * Each sgl is either iterated or created at a time. 29 */ 30 struct _dif_sgl { 31 /* Current iovec in the iteration or creation */ 32 struct iovec *iov; 33 34 /* Remaining count of iovecs in the iteration or creation. */ 35 int iovcnt; 36 37 /* Current offset in the iovec */ 38 uint32_t iov_offset; 39 40 /* Size of the created iovec array in bytes */ 41 uint32_t total_size; 42 }; 43 44 static inline void 45 _dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt) 46 { 47 s->iov = iovs; 48 s->iovcnt = iovcnt; 49 s->iov_offset = 0; 50 s->total_size = 0; 51 } 52 53 static void 54 _dif_sgl_advance(struct _dif_sgl *s, uint32_t step) 55 { 56 s->iov_offset += step; 57 while (s->iovcnt != 0) { 58 if (s->iov_offset < s->iov->iov_len) { 59 break; 60 } 61 62 s->iov_offset -= s->iov->iov_len; 63 s->iov++; 64 s->iovcnt--; 65 } 66 } 67 68 static inline void 69 _dif_sgl_get_buf(struct _dif_sgl *s, void **_buf, uint32_t *_buf_len) 70 { 71 if (_buf != NULL) { 72 *_buf = s->iov->iov_base + s->iov_offset; 73 } 74 if (_buf_len != NULL) { 75 *_buf_len = s->iov->iov_len - s->iov_offset; 76 } 77 } 78 79 static inline bool 80 _dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len) 81 { 82 assert(s->iovcnt > 0); 83 s->iov->iov_base = data; 84 s->iov->iov_len = data_len; 85 s->total_size += data_len; 86 s->iov++; 87 s->iovcnt--; 88 89 if (s->iovcnt > 0) { 90 return true; 91 } else { 92 return false; 93 } 94 } 95 96 static inline bool 97 _dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len) 98 { 99 uint8_t *buf; 100 uint32_t buf_len; 101 102 while (data_len != 0) { 103 _dif_sgl_get_buf(src, (void *)&buf, &buf_len); 104 buf_len = spdk_min(buf_len, data_len); 105 106 if (!_dif_sgl_append(dst, buf, buf_len)) { 107 return false; 108 } 109 110 _dif_sgl_advance(src, buf_len); 111 data_len -= buf_len; 112 } 113 114 return true; 115 } 116 117 /* This function must be used before starting iteration. */ 118 static bool 119 _dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes) 120 { 121 int i; 122 123 for (i = 0; i < s->iovcnt; i++) { 124 if (s->iov[i].iov_len % bytes) { 125 return false; 126 } 127 } 128 129 return true; 130 } 131 132 static bool 133 _dif_sgl_is_valid_block_aligned(struct _dif_sgl *s, uint32_t num_blocks, uint32_t block_size) 134 { 135 uint32_t count = 0; 136 int i; 137 138 for (i = 0; i < s->iovcnt; i++) { 139 if (s->iov[i].iov_len % block_size) { 140 return false; 141 } 142 count += s->iov[i].iov_len / block_size; 143 } 144 145 return count >= num_blocks; 146 } 147 148 /* This function must be used before starting iteration. */ 149 static bool 150 _dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes) 151 { 152 uint64_t total = 0; 153 int i; 154 155 for (i = 0; i < s->iovcnt; i++) { 156 total += s->iov[i].iov_len; 157 } 158 159 return total >= bytes; 160 } 161 162 static void 163 _dif_sgl_copy(struct _dif_sgl *to, struct _dif_sgl *from) 164 { 165 memcpy(to, from, sizeof(struct _dif_sgl)); 166 } 167 168 static bool 169 _dif_type_is_valid(enum spdk_dif_type dif_type, uint32_t dif_flags) 170 { 171 switch (dif_type) { 172 case SPDK_DIF_TYPE1: 173 case SPDK_DIF_TYPE2: 174 case SPDK_DIF_DISABLE: 175 break; 176 case SPDK_DIF_TYPE3: 177 if (dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 178 SPDK_ERRLOG("Reference Tag should not be checked for Type 3\n"); 179 return false; 180 } 181 break; 182 default: 183 SPDK_ERRLOG("Unknown DIF Type: %d\n", dif_type); 184 return false; 185 } 186 187 return true; 188 } 189 190 static bool 191 _dif_is_disabled(enum spdk_dif_type dif_type) 192 { 193 if (dif_type == SPDK_DIF_DISABLE) { 194 return true; 195 } else { 196 return false; 197 } 198 } 199 200 static size_t 201 _dif_size(enum spdk_dif_pi_format dif_pi_format) 202 { 203 return SPDK_SIZEOF_MEMBER(struct spdk_dif, g16); 204 } 205 206 static uint32_t 207 _get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave, 208 size_t dif_size) 209 { 210 if (!dif_loc) { 211 /* For metadata formats with more than 8 bytes, if the DIF is 212 * contained in the last 8 bytes of metadata, then the CRC 213 * covers all metadata up to but excluding these last 8 bytes. 214 */ 215 if (md_interleave) { 216 return block_size - dif_size; 217 } else { 218 return md_size - dif_size; 219 } 220 } else { 221 /* For metadata formats with more than 8 bytes, if the DIF is 222 * contained in the first 8 bytes of metadata, then the CRC 223 * does not cover any metadata. 224 */ 225 if (md_interleave) { 226 return block_size - md_size; 227 } else { 228 return 0; 229 } 230 } 231 } 232 233 static uint8_t 234 _dif_sizeof_guard(enum spdk_dif_pi_format dif_pi_format) 235 { 236 return sizeof(((struct spdk_dif *)0)->g16.guard); 237 } 238 239 static uint8_t 240 _dif_offsetof_apptag(enum spdk_dif_pi_format dif_pi_format) 241 { 242 return _dif_sizeof_guard(dif_pi_format); 243 } 244 245 static uint8_t 246 _dif_sizeof_apptag(void) 247 { 248 return sizeof(((struct spdk_dif *)0)->g16.app_tag); 249 } 250 251 static uint8_t 252 _dif_offsetof_reftag(enum spdk_dif_pi_format dif_pi_format) 253 { 254 return _dif_offsetof_apptag(dif_pi_format) + _dif_sizeof_apptag(); 255 } 256 257 static uint8_t 258 _dif_sizeof_reftag(enum spdk_dif_pi_format dif_pi_format) 259 { 260 return sizeof(((struct spdk_dif *)0)->g16.stor_ref_space); 261 } 262 263 static void 264 _dif_set_guard(struct spdk_dif *dif, uint32_t guard, enum spdk_dif_pi_format dif_pi_format) 265 { 266 to_be16(&(dif->g16.guard), (uint16_t)guard); 267 } 268 269 static uint32_t 270 _dif_get_guard(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 271 { 272 return (uint32_t)from_be16(&(dif->g16.guard)); 273 } 274 275 static uint32_t 276 _dif_generate_guard(uint32_t guard_seed, void *buf, size_t buf_len, 277 enum spdk_dif_pi_format dif_pi_format) 278 { 279 return (uint32_t)spdk_crc16_t10dif((uint16_t)guard_seed, buf, buf_len); 280 } 281 282 static uint32_t 283 _dif_generate_guard_copy(uint32_t guard_seed, void *dst, void *src, size_t buf_len, 284 enum spdk_dif_pi_format dif_pi_format) 285 { 286 return (uint32_t)spdk_crc16_t10dif_copy((uint16_t)guard_seed, dst, src, buf_len); 287 } 288 289 static void 290 _dif_set_apptag(struct spdk_dif *dif, uint16_t app_tag, enum spdk_dif_pi_format dif_pi_format) 291 { 292 to_be16(&(dif->g16.app_tag), app_tag); 293 } 294 295 static uint16_t 296 _dif_get_apptag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 297 { 298 return from_be16(&(dif->g16.app_tag)); 299 } 300 301 static bool 302 _dif_apptag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 303 { 304 return _dif_get_apptag(dif, dif_pi_format) == APPTAG_IGNORE; 305 } 306 307 static void 308 _dif_set_reftag(struct spdk_dif *dif, uint64_t ref_tag, enum spdk_dif_pi_format dif_pi_format) 309 { 310 to_be32(&(dif->g16.stor_ref_space), (uint32_t)ref_tag); 311 } 312 313 static uint64_t 314 _dif_get_reftag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 315 { 316 return (uint64_t)from_be32(&(dif->g16.stor_ref_space)); 317 } 318 319 static bool 320 _dif_reftag_match(struct spdk_dif *dif, uint64_t ref_tag, 321 enum spdk_dif_pi_format dif_pi_format) 322 { 323 uint64_t _ref_tag; 324 325 _ref_tag = _dif_get_reftag(dif, dif_pi_format); 326 327 return (_ref_tag == (ref_tag & REFTAG_MASK_16)); 328 } 329 330 static bool 331 _dif_reftag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 332 { 333 return _dif_reftag_match(dif, REFTAG_MASK_16, dif_pi_format); 334 } 335 336 int 337 spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, 338 bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, 339 uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag, 340 uint32_t data_offset, uint32_t guard_seed, struct spdk_dif_ctx_init_ext_opts *opts) 341 { 342 uint32_t data_block_size; 343 enum spdk_dif_pi_format dif_pi_format = SPDK_DIF_PI_FORMAT_16; 344 345 if (opts != NULL) { 346 if (opts->dif_pi_format != SPDK_DIF_PI_FORMAT_16) { 347 SPDK_ERRLOG("No valid DIF PI format provided.\n"); 348 return -EINVAL; 349 } 350 351 dif_pi_format = opts->dif_pi_format; 352 } 353 354 if (md_size < _dif_size(dif_pi_format)) { 355 SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); 356 return -EINVAL; 357 } 358 359 if (md_interleave) { 360 if (block_size < md_size) { 361 SPDK_ERRLOG("Block size is smaller than DIF size.\n"); 362 return -EINVAL; 363 } 364 data_block_size = block_size - md_size; 365 } else { 366 if (block_size == 0 || (block_size % 512) != 0) { 367 SPDK_ERRLOG("Zero block size is not allowed\n"); 368 return -EINVAL; 369 } 370 data_block_size = block_size; 371 } 372 373 if (!_dif_type_is_valid(dif_type, dif_flags)) { 374 SPDK_ERRLOG("DIF type is invalid.\n"); 375 return -EINVAL; 376 } 377 378 ctx->block_size = block_size; 379 ctx->md_size = md_size; 380 ctx->md_interleave = md_interleave; 381 ctx->dif_pi_format = dif_pi_format; 382 ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave, 383 _dif_size(ctx->dif_pi_format)); 384 ctx->dif_type = dif_type; 385 ctx->dif_flags = dif_flags; 386 ctx->init_ref_tag = init_ref_tag; 387 ctx->apptag_mask = apptag_mask; 388 ctx->app_tag = app_tag; 389 ctx->data_offset = data_offset; 390 ctx->ref_tag_offset = data_offset / data_block_size; 391 ctx->last_guard = guard_seed; 392 ctx->guard_seed = guard_seed; 393 ctx->remapped_init_ref_tag = 0; 394 395 return 0; 396 } 397 398 void 399 spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset) 400 { 401 uint32_t data_block_size; 402 403 if (ctx->md_interleave) { 404 data_block_size = ctx->block_size - ctx->md_size; 405 } else { 406 data_block_size = ctx->block_size; 407 } 408 409 ctx->data_offset = data_offset; 410 ctx->ref_tag_offset = data_offset / data_block_size; 411 } 412 413 void 414 spdk_dif_ctx_set_remapped_init_ref_tag(struct spdk_dif_ctx *ctx, 415 uint32_t remapped_init_ref_tag) 416 { 417 ctx->remapped_init_ref_tag = remapped_init_ref_tag; 418 } 419 420 static void 421 _dif_generate(void *_dif, uint32_t guard, uint32_t offset_blocks, 422 const struct spdk_dif_ctx *ctx) 423 { 424 struct spdk_dif *dif = _dif; 425 uint64_t ref_tag; 426 427 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 428 _dif_set_guard(dif, guard, ctx->dif_pi_format); 429 } 430 431 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 432 _dif_set_apptag(dif, ctx->app_tag, ctx->dif_pi_format); 433 } 434 435 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 436 /* For type 1 and 2, the reference tag is incremented for each 437 * subsequent logical block. For type 3, the reference tag 438 * remains the same as the initial reference tag. 439 */ 440 if (ctx->dif_type != SPDK_DIF_TYPE3) { 441 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 442 } else { 443 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 444 } 445 446 _dif_set_reftag(dif, ref_tag, ctx->dif_pi_format); 447 } 448 } 449 450 static void 451 dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 452 { 453 uint32_t offset_blocks = 0; 454 void *buf; 455 uint32_t guard = 0; 456 457 while (offset_blocks < num_blocks) { 458 _dif_sgl_get_buf(sgl, &buf, NULL); 459 460 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 461 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 462 } 463 464 _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); 465 466 _dif_sgl_advance(sgl, ctx->block_size); 467 offset_blocks++; 468 } 469 } 470 471 static uint32_t 472 _dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 473 uint32_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 474 { 475 uint32_t offset_in_dif, buf_len; 476 void *buf; 477 struct spdk_dif dif = {}; 478 479 assert(offset_in_block < ctx->guard_interval); 480 assert(offset_in_block + data_len < ctx->guard_interval || 481 offset_in_block + data_len == ctx->block_size); 482 483 /* Compute CRC over split logical block data. */ 484 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 485 _dif_sgl_get_buf(sgl, &buf, &buf_len); 486 buf_len = spdk_min(buf_len, data_len); 487 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 488 489 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 490 guard = _dif_generate_guard(guard, buf, buf_len, ctx->dif_pi_format); 491 } 492 493 _dif_sgl_advance(sgl, buf_len); 494 offset_in_block += buf_len; 495 data_len -= buf_len; 496 } 497 498 if (offset_in_block < ctx->guard_interval) { 499 return guard; 500 } 501 502 /* If a whole logical block data is parsed, generate DIF 503 * and save it to the temporary DIF area. 504 */ 505 _dif_generate(&dif, guard, offset_blocks, ctx); 506 507 /* Copy generated DIF field to the split DIF field, and then 508 * skip metadata field after DIF field (if any). 509 */ 510 while (offset_in_block < ctx->block_size) { 511 _dif_sgl_get_buf(sgl, &buf, &buf_len); 512 513 if (offset_in_block < ctx->guard_interval + _dif_size(ctx->dif_pi_format)) { 514 offset_in_dif = offset_in_block - ctx->guard_interval; 515 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset_in_dif); 516 517 memcpy(buf, ((uint8_t *)&dif) + offset_in_dif, buf_len); 518 } else { 519 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 520 } 521 522 _dif_sgl_advance(sgl, buf_len); 523 offset_in_block += buf_len; 524 } 525 526 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 527 guard = ctx->guard_seed; 528 } 529 530 return guard; 531 } 532 533 static void 534 dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks, 535 const struct spdk_dif_ctx *ctx) 536 { 537 uint32_t offset_blocks; 538 uint32_t guard = 0; 539 540 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 541 guard = ctx->guard_seed; 542 } 543 544 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 545 _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx); 546 } 547 } 548 549 int 550 spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 551 const struct spdk_dif_ctx *ctx) 552 { 553 struct _dif_sgl sgl; 554 555 _dif_sgl_init(&sgl, iovs, iovcnt); 556 557 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 558 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 559 return -EINVAL; 560 } 561 562 if (_dif_is_disabled(ctx->dif_type)) { 563 return 0; 564 } 565 566 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 567 dif_generate(&sgl, num_blocks, ctx); 568 } else { 569 dif_generate_split(&sgl, num_blocks, ctx); 570 } 571 572 return 0; 573 } 574 575 static void 576 _dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, 577 uint64_t expected, uint64_t actual, uint32_t err_offset) 578 { 579 if (err_blk) { 580 err_blk->err_type = err_type; 581 err_blk->expected = expected; 582 err_blk->actual = actual; 583 err_blk->err_offset = err_offset; 584 } 585 } 586 587 static int 588 _dif_verify(void *_dif, uint32_t guard, uint32_t offset_blocks, 589 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 590 { 591 struct spdk_dif *dif = _dif; 592 uint32_t _guard; 593 uint16_t _app_tag; 594 uint64_t ref_tag, _ref_tag; 595 596 switch (ctx->dif_type) { 597 case SPDK_DIF_TYPE1: 598 case SPDK_DIF_TYPE2: 599 /* If Type 1 or 2 is used, then all DIF checks are disabled when 600 * the Application Tag is 0xFFFF. 601 */ 602 if (_dif_apptag_ignore(dif, ctx->dif_pi_format)) { 603 return 0; 604 } 605 break; 606 case SPDK_DIF_TYPE3: 607 /* If Type 3 is used, then all DIF checks are disabled when the 608 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF 609 * or 0xFFFFFFFFFFFFFFFF depending on the PI format. 610 */ 611 612 if (_dif_apptag_ignore(dif, ctx->dif_pi_format) && 613 _dif_reftag_ignore(dif, ctx->dif_pi_format)) { 614 return 0; 615 } 616 break; 617 default: 618 break; 619 } 620 621 /* For type 1 and 2, the reference tag is incremented for each 622 * subsequent logical block. For type 3, the reference tag 623 * remains the same as the initial reference tag. 624 */ 625 if (ctx->dif_type != SPDK_DIF_TYPE3) { 626 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 627 } else { 628 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 629 } 630 631 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 632 /* Compare the DIF Guard field to the CRC computed over the logical 633 * block data. 634 */ 635 _guard = _dif_get_guard(dif, ctx->dif_pi_format); 636 if (_guard != guard) { 637 _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, 638 offset_blocks); 639 SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu64 "," \ 640 " Expected=%x, Actual=%x\n", 641 ref_tag, _guard, guard); 642 return -1; 643 } 644 } 645 646 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 647 /* Compare unmasked bits in the DIF Application Tag field to the 648 * passed Application Tag. 649 */ 650 _app_tag = _dif_get_apptag(dif, ctx->dif_pi_format); 651 if ((_app_tag & ctx->apptag_mask) != ctx->app_tag) { 652 _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, 653 (_app_tag & ctx->apptag_mask), offset_blocks); 654 SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu64 "," \ 655 " Expected=%x, Actual=%x\n", 656 ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); 657 return -1; 658 } 659 } 660 661 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 662 switch (ctx->dif_type) { 663 case SPDK_DIF_TYPE1: 664 case SPDK_DIF_TYPE2: 665 /* Compare the DIF Reference Tag field to the passed Reference Tag. 666 * The passed Reference Tag will be the least significant 4 bytes 667 * or 8 bytes (depending on the PI format) 668 * of the LBA when Type 1 is used, and application specific value 669 * if Type 2 is used. 670 */ 671 if (!_dif_reftag_match(dif, ref_tag, ctx->dif_pi_format)) { 672 _ref_tag = _dif_get_reftag(dif, ctx->dif_pi_format); 673 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, ref_tag, 674 _ref_tag, offset_blocks); 675 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu64 "," \ 676 " Expected=%lx, Actual=%lx\n", 677 ref_tag, ref_tag, _ref_tag); 678 return -1; 679 } 680 break; 681 case SPDK_DIF_TYPE3: 682 /* For Type 3, computed Reference Tag remains unchanged. 683 * Hence ignore the Reference Tag field. 684 */ 685 break; 686 default: 687 break; 688 } 689 } 690 691 return 0; 692 } 693 694 static int 695 dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks, 696 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 697 { 698 uint32_t offset_blocks = 0; 699 int rc; 700 void *buf; 701 uint32_t guard = 0; 702 703 while (offset_blocks < num_blocks) { 704 _dif_sgl_get_buf(sgl, &buf, NULL); 705 706 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 707 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 708 } 709 710 rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 711 if (rc != 0) { 712 return rc; 713 } 714 715 _dif_sgl_advance(sgl, ctx->block_size); 716 offset_blocks++; 717 } 718 719 return 0; 720 } 721 722 static int 723 _dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 724 uint32_t *_guard, uint32_t offset_blocks, 725 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 726 { 727 uint32_t offset_in_dif, buf_len; 728 void *buf; 729 uint32_t guard; 730 struct spdk_dif dif = {}; 731 int rc; 732 733 assert(_guard != NULL); 734 assert(offset_in_block < ctx->guard_interval); 735 assert(offset_in_block + data_len < ctx->guard_interval || 736 offset_in_block + data_len == ctx->block_size); 737 738 guard = *_guard; 739 740 /* Compute CRC over split logical block data. */ 741 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 742 _dif_sgl_get_buf(sgl, &buf, &buf_len); 743 buf_len = spdk_min(buf_len, data_len); 744 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 745 746 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 747 guard = _dif_generate_guard(guard, buf, buf_len, ctx->dif_pi_format); 748 } 749 750 _dif_sgl_advance(sgl, buf_len); 751 offset_in_block += buf_len; 752 data_len -= buf_len; 753 } 754 755 if (offset_in_block < ctx->guard_interval) { 756 *_guard = guard; 757 return 0; 758 } 759 760 /* Copy the split DIF field to the temporary DIF buffer, and then 761 * skip metadata field after DIF field (if any). */ 762 while (offset_in_block < ctx->block_size) { 763 _dif_sgl_get_buf(sgl, &buf, &buf_len); 764 765 if (offset_in_block < ctx->guard_interval + _dif_size(ctx->dif_pi_format)) { 766 offset_in_dif = offset_in_block - ctx->guard_interval; 767 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset_in_dif); 768 769 memcpy((uint8_t *)&dif + offset_in_dif, buf, buf_len); 770 } else { 771 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 772 } 773 _dif_sgl_advance(sgl, buf_len); 774 offset_in_block += buf_len; 775 } 776 777 rc = _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 778 if (rc != 0) { 779 return rc; 780 } 781 782 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 783 guard = ctx->guard_seed; 784 } 785 786 *_guard = guard; 787 return 0; 788 } 789 790 static int 791 dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks, 792 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 793 { 794 uint32_t offset_blocks; 795 uint32_t guard = 0; 796 int rc; 797 798 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 799 guard = ctx->guard_seed; 800 } 801 802 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 803 rc = _dif_verify_split(sgl, 0, ctx->block_size, &guard, offset_blocks, 804 ctx, err_blk); 805 if (rc != 0) { 806 return rc; 807 } 808 } 809 810 return 0; 811 } 812 813 int 814 spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 815 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 816 { 817 struct _dif_sgl sgl; 818 819 _dif_sgl_init(&sgl, iovs, iovcnt); 820 821 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 822 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 823 return -EINVAL; 824 } 825 826 if (_dif_is_disabled(ctx->dif_type)) { 827 return 0; 828 } 829 830 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 831 return dif_verify(&sgl, num_blocks, ctx, err_blk); 832 } else { 833 return dif_verify_split(&sgl, num_blocks, ctx, err_blk); 834 } 835 } 836 837 static uint32_t 838 dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks, 839 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 840 { 841 uint32_t offset_blocks; 842 void *buf; 843 844 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 845 _dif_sgl_get_buf(sgl, &buf, NULL); 846 847 crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c); 848 849 _dif_sgl_advance(sgl, ctx->block_size); 850 } 851 852 return crc32c; 853 } 854 855 static uint32_t 856 _dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 857 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 858 { 859 uint32_t data_block_size, buf_len; 860 void *buf; 861 862 data_block_size = ctx->block_size - ctx->md_size; 863 864 assert(offset_in_block + data_len <= ctx->block_size); 865 866 while (data_len != 0) { 867 _dif_sgl_get_buf(sgl, &buf, &buf_len); 868 buf_len = spdk_min(buf_len, data_len); 869 870 if (offset_in_block < data_block_size) { 871 buf_len = spdk_min(buf_len, data_block_size - offset_in_block); 872 crc32c = spdk_crc32c_update(buf, buf_len, crc32c); 873 } 874 875 _dif_sgl_advance(sgl, buf_len); 876 offset_in_block += buf_len; 877 data_len -= buf_len; 878 } 879 880 return crc32c; 881 } 882 883 static uint32_t 884 dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks, 885 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 886 { 887 uint32_t offset_blocks; 888 889 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 890 crc32c = _dif_update_crc32c_split(sgl, 0, ctx->block_size, crc32c, ctx); 891 } 892 893 return crc32c; 894 } 895 896 int 897 spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 898 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 899 { 900 struct _dif_sgl sgl; 901 902 if (_crc32c == NULL) { 903 return -EINVAL; 904 } 905 906 _dif_sgl_init(&sgl, iovs, iovcnt); 907 908 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 909 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 910 return -EINVAL; 911 } 912 913 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 914 *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx); 915 } else { 916 *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx); 917 } 918 919 return 0; 920 } 921 922 static void 923 dif_generate_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 924 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 925 { 926 uint32_t offset_blocks = 0, data_block_size; 927 void *src, *dst; 928 uint32_t guard; 929 930 data_block_size = ctx->block_size - ctx->md_size; 931 932 while (offset_blocks < num_blocks) { 933 _dif_sgl_get_buf(src_sgl, &src, NULL); 934 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 935 936 guard = 0; 937 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 938 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 939 ctx->dif_pi_format); 940 guard = _dif_generate_guard(guard, dst + data_block_size, 941 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 942 } else { 943 memcpy(dst, src, data_block_size); 944 } 945 946 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 947 948 _dif_sgl_advance(src_sgl, data_block_size); 949 _dif_sgl_advance(dst_sgl, ctx->block_size); 950 offset_blocks++; 951 } 952 } 953 954 static void 955 _dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 956 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 957 { 958 uint32_t offset_in_block, src_len, data_block_size; 959 uint32_t guard = 0; 960 void *src, *dst; 961 962 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 963 964 data_block_size = ctx->block_size - ctx->md_size; 965 966 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 967 guard = ctx->guard_seed; 968 } 969 offset_in_block = 0; 970 971 while (offset_in_block < data_block_size) { 972 /* Compute CRC over split logical block data and copy 973 * data to bounce buffer. 974 */ 975 _dif_sgl_get_buf(src_sgl, &src, &src_len); 976 src_len = spdk_min(src_len, data_block_size - offset_in_block); 977 978 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 979 guard = _dif_generate_guard_copy(guard, dst + offset_in_block, 980 src, src_len, ctx->dif_pi_format); 981 } else { 982 memcpy(dst + offset_in_block, src, src_len); 983 } 984 985 _dif_sgl_advance(src_sgl, src_len); 986 offset_in_block += src_len; 987 } 988 989 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 990 guard = _dif_generate_guard(guard, dst + data_block_size, 991 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 992 } 993 994 _dif_sgl_advance(dst_sgl, ctx->block_size); 995 996 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 997 } 998 999 static void 1000 dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1001 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1002 { 1003 uint32_t offset_blocks; 1004 1005 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1006 _dif_generate_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 1007 } 1008 } 1009 1010 int 1011 spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1012 int bounce_iovcnt, uint32_t num_blocks, 1013 const struct spdk_dif_ctx *ctx) 1014 { 1015 struct _dif_sgl src_sgl, dst_sgl; 1016 uint32_t data_block_size; 1017 1018 _dif_sgl_init(&src_sgl, iovs, iovcnt); 1019 _dif_sgl_init(&dst_sgl, bounce_iovs, bounce_iovcnt); 1020 1021 data_block_size = ctx->block_size - ctx->md_size; 1022 1023 if (!_dif_sgl_is_valid(&src_sgl, data_block_size * num_blocks)) { 1024 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 1025 return -EINVAL; 1026 } 1027 1028 if (!_dif_sgl_is_valid_block_aligned(&dst_sgl, num_blocks, ctx->block_size)) { 1029 SPDK_ERRLOG("Size of bounce_iovs arrays are not valid or misaligned with block_size.\n"); 1030 return -EINVAL; 1031 } 1032 1033 if (_dif_is_disabled(ctx->dif_type)) { 1034 return 0; 1035 } 1036 1037 if (_dif_sgl_is_bytes_multiple(&src_sgl, data_block_size)) { 1038 dif_generate_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 1039 } else { 1040 dif_generate_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx); 1041 } 1042 1043 return 0; 1044 } 1045 1046 static int 1047 dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1048 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1049 struct spdk_dif_error *err_blk) 1050 { 1051 uint32_t offset_blocks = 0, data_block_size; 1052 void *src, *dst; 1053 int rc; 1054 uint32_t guard; 1055 1056 data_block_size = ctx->block_size - ctx->md_size; 1057 1058 while (offset_blocks < num_blocks) { 1059 _dif_sgl_get_buf(src_sgl, &src, NULL); 1060 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1061 1062 guard = 0; 1063 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1064 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1065 ctx->dif_pi_format); 1066 guard = _dif_generate_guard(guard, src + data_block_size, 1067 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1068 } else { 1069 memcpy(dst, src, data_block_size); 1070 } 1071 1072 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1073 if (rc != 0) { 1074 return rc; 1075 } 1076 1077 _dif_sgl_advance(src_sgl, ctx->block_size); 1078 _dif_sgl_advance(dst_sgl, data_block_size); 1079 offset_blocks++; 1080 } 1081 1082 return 0; 1083 } 1084 1085 static int 1086 _dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1087 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1088 struct spdk_dif_error *err_blk) 1089 { 1090 uint32_t offset_in_block, dst_len, data_block_size; 1091 uint32_t guard = 0; 1092 void *src, *dst; 1093 1094 _dif_sgl_get_buf(src_sgl, &src, NULL); 1095 1096 data_block_size = ctx->block_size - ctx->md_size; 1097 1098 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1099 guard = ctx->guard_seed; 1100 } 1101 offset_in_block = 0; 1102 1103 while (offset_in_block < data_block_size) { 1104 /* Compute CRC over split logical block data and copy 1105 * data to bounce buffer. 1106 */ 1107 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 1108 dst_len = spdk_min(dst_len, data_block_size - offset_in_block); 1109 1110 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1111 guard = _dif_generate_guard_copy(guard, dst, src + offset_in_block, 1112 dst_len, ctx->dif_pi_format); 1113 } else { 1114 memcpy(dst, src + offset_in_block, dst_len); 1115 } 1116 1117 _dif_sgl_advance(dst_sgl, dst_len); 1118 offset_in_block += dst_len; 1119 } 1120 1121 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1122 guard = _dif_generate_guard(guard, src + data_block_size, 1123 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1124 } 1125 1126 _dif_sgl_advance(src_sgl, ctx->block_size); 1127 1128 return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1129 } 1130 1131 static int 1132 dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1133 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1134 struct spdk_dif_error *err_blk) 1135 { 1136 uint32_t offset_blocks; 1137 int rc; 1138 1139 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1140 rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1141 if (rc != 0) { 1142 return rc; 1143 } 1144 } 1145 1146 return 0; 1147 } 1148 1149 int 1150 spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1151 int bounce_iovcnt, uint32_t num_blocks, 1152 const struct spdk_dif_ctx *ctx, 1153 struct spdk_dif_error *err_blk) 1154 { 1155 struct _dif_sgl src_sgl, dst_sgl; 1156 uint32_t data_block_size; 1157 1158 _dif_sgl_init(&src_sgl, bounce_iovs, bounce_iovcnt); 1159 _dif_sgl_init(&dst_sgl, iovs, iovcnt); 1160 1161 data_block_size = ctx->block_size - ctx->md_size; 1162 1163 if (!_dif_sgl_is_valid(&dst_sgl, data_block_size * num_blocks)) { 1164 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1165 return -EINVAL; 1166 } 1167 1168 if (!_dif_sgl_is_valid_block_aligned(&src_sgl, num_blocks, ctx->block_size)) { 1169 SPDK_ERRLOG("Size of bounce_iovs arrays are not valid or misaligned with block_size.\n"); 1170 return -EINVAL; 1171 } 1172 1173 if (_dif_is_disabled(ctx->dif_type)) { 1174 return 0; 1175 } 1176 1177 if (_dif_sgl_is_bytes_multiple(&dst_sgl, data_block_size)) { 1178 return dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1179 } else { 1180 return dif_verify_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1181 } 1182 } 1183 1184 static void 1185 _bit_flip(uint8_t *buf, uint32_t flip_bit) 1186 { 1187 uint8_t byte; 1188 1189 byte = *buf; 1190 byte ^= 1 << flip_bit; 1191 *buf = byte; 1192 } 1193 1194 static int 1195 _dif_inject_error(struct _dif_sgl *sgl, 1196 uint32_t block_size, uint32_t num_blocks, 1197 uint32_t inject_offset_blocks, 1198 uint32_t inject_offset_bytes, 1199 uint32_t inject_offset_bits) 1200 { 1201 uint32_t offset_in_block, buf_len; 1202 void *buf; 1203 1204 _dif_sgl_advance(sgl, block_size * inject_offset_blocks); 1205 1206 offset_in_block = 0; 1207 1208 while (offset_in_block < block_size) { 1209 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1210 buf_len = spdk_min(buf_len, block_size - offset_in_block); 1211 1212 if (inject_offset_bytes >= offset_in_block && 1213 inject_offset_bytes < offset_in_block + buf_len) { 1214 buf += inject_offset_bytes - offset_in_block; 1215 _bit_flip(buf, inject_offset_bits); 1216 return 0; 1217 } 1218 1219 _dif_sgl_advance(sgl, buf_len); 1220 offset_in_block += buf_len; 1221 } 1222 1223 return -1; 1224 } 1225 1226 static int 1227 dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks, 1228 uint32_t start_inject_bytes, uint32_t inject_range_bytes, 1229 uint32_t *inject_offset) 1230 { 1231 uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; 1232 uint32_t offset_blocks; 1233 int rc; 1234 1235 srand(time(0)); 1236 1237 inject_offset_blocks = rand() % num_blocks; 1238 inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); 1239 inject_offset_bits = rand() % 8; 1240 1241 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1242 if (offset_blocks == inject_offset_blocks) { 1243 rc = _dif_inject_error(sgl, block_size, num_blocks, 1244 inject_offset_blocks, 1245 inject_offset_bytes, 1246 inject_offset_bits); 1247 if (rc == 0) { 1248 *inject_offset = inject_offset_blocks; 1249 } 1250 return rc; 1251 } 1252 } 1253 1254 return -1; 1255 } 1256 1257 int 1258 spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1259 const struct spdk_dif_ctx *ctx, uint32_t inject_flags, 1260 uint32_t *inject_offset) 1261 { 1262 struct _dif_sgl sgl; 1263 int rc; 1264 1265 _dif_sgl_init(&sgl, iovs, iovcnt); 1266 1267 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1268 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1269 return -EINVAL; 1270 } 1271 1272 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1273 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1274 ctx->guard_interval + _dif_offsetof_reftag(ctx->dif_pi_format), 1275 _dif_sizeof_reftag(ctx->dif_pi_format), 1276 inject_offset); 1277 if (rc != 0) { 1278 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1279 return rc; 1280 } 1281 } 1282 1283 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1284 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1285 ctx->guard_interval + _dif_offsetof_apptag(ctx->dif_pi_format), 1286 _dif_sizeof_apptag(), 1287 inject_offset); 1288 if (rc != 0) { 1289 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1290 return rc; 1291 } 1292 } 1293 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1294 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1295 ctx->guard_interval, 1296 _dif_sizeof_guard(ctx->dif_pi_format), 1297 inject_offset); 1298 if (rc != 0) { 1299 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1300 return rc; 1301 } 1302 } 1303 1304 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1305 /* If the DIF information is contained within the last 8 bytes of 1306 * metadata, then the CRC covers all metadata bytes up to but excluding 1307 * the last 8 bytes. But error injection does not cover these metadata 1308 * because classification is not determined yet. 1309 * 1310 * Note: Error injection to data block is expected to be detected as 1311 * guard error. 1312 */ 1313 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1314 0, 1315 ctx->block_size - ctx->md_size, 1316 inject_offset); 1317 if (rc != 0) { 1318 SPDK_ERRLOG("Failed to inject error to data block.\n"); 1319 return rc; 1320 } 1321 } 1322 1323 return 0; 1324 } 1325 1326 static void 1327 dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1328 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1329 { 1330 uint32_t offset_blocks = 0; 1331 uint32_t guard; 1332 void *data_buf, *md_buf; 1333 1334 while (offset_blocks < num_blocks) { 1335 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1336 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1337 1338 guard = 0; 1339 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1340 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1341 ctx->dif_pi_format); 1342 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1343 ctx->dif_pi_format); 1344 } 1345 1346 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1347 1348 _dif_sgl_advance(data_sgl, ctx->block_size); 1349 _dif_sgl_advance(md_sgl, ctx->md_size); 1350 offset_blocks++; 1351 } 1352 } 1353 1354 static void 1355 _dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1356 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1357 { 1358 uint32_t offset_in_block, data_buf_len; 1359 uint32_t guard = 0; 1360 void *data_buf, *md_buf; 1361 1362 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1363 1364 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1365 guard = ctx->guard_seed; 1366 } 1367 offset_in_block = 0; 1368 1369 while (offset_in_block < ctx->block_size) { 1370 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1371 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1372 1373 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1374 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 1375 ctx->dif_pi_format); 1376 } 1377 1378 _dif_sgl_advance(data_sgl, data_buf_len); 1379 offset_in_block += data_buf_len; 1380 } 1381 1382 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1383 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1384 ctx->dif_pi_format); 1385 } 1386 1387 _dif_sgl_advance(md_sgl, ctx->md_size); 1388 1389 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1390 } 1391 1392 static void 1393 dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1394 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1395 { 1396 uint32_t offset_blocks; 1397 1398 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1399 _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx); 1400 } 1401 } 1402 1403 int 1404 spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1405 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1406 { 1407 struct _dif_sgl data_sgl, md_sgl; 1408 1409 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1410 _dif_sgl_init(&md_sgl, md_iov, 1); 1411 1412 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1413 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1414 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1415 return -EINVAL; 1416 } 1417 1418 if (_dif_is_disabled(ctx->dif_type)) { 1419 return 0; 1420 } 1421 1422 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1423 dix_generate(&data_sgl, &md_sgl, num_blocks, ctx); 1424 } else { 1425 dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx); 1426 } 1427 1428 return 0; 1429 } 1430 1431 static int 1432 dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1433 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1434 struct spdk_dif_error *err_blk) 1435 { 1436 uint32_t offset_blocks = 0; 1437 uint32_t guard; 1438 void *data_buf, *md_buf; 1439 int rc; 1440 1441 while (offset_blocks < num_blocks) { 1442 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1443 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1444 1445 guard = 0; 1446 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1447 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1448 ctx->dif_pi_format); 1449 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1450 ctx->dif_pi_format); 1451 } 1452 1453 rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1454 if (rc != 0) { 1455 return rc; 1456 } 1457 1458 _dif_sgl_advance(data_sgl, ctx->block_size); 1459 _dif_sgl_advance(md_sgl, ctx->md_size); 1460 offset_blocks++; 1461 } 1462 1463 return 0; 1464 } 1465 1466 static int 1467 _dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1468 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1469 struct spdk_dif_error *err_blk) 1470 { 1471 uint32_t offset_in_block, data_buf_len; 1472 uint32_t guard = 0; 1473 void *data_buf, *md_buf; 1474 1475 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1476 1477 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1478 guard = ctx->guard_seed; 1479 } 1480 offset_in_block = 0; 1481 1482 while (offset_in_block < ctx->block_size) { 1483 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1484 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1485 1486 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1487 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 1488 ctx->dif_pi_format); 1489 } 1490 1491 _dif_sgl_advance(data_sgl, data_buf_len); 1492 offset_in_block += data_buf_len; 1493 } 1494 1495 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1496 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1497 ctx->dif_pi_format); 1498 } 1499 1500 _dif_sgl_advance(md_sgl, ctx->md_size); 1501 1502 return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1503 } 1504 1505 static int 1506 dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1507 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1508 struct spdk_dif_error *err_blk) 1509 { 1510 uint32_t offset_blocks; 1511 int rc; 1512 1513 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1514 rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk); 1515 if (rc != 0) { 1516 return rc; 1517 } 1518 } 1519 1520 return 0; 1521 } 1522 1523 int 1524 spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1525 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1526 struct spdk_dif_error *err_blk) 1527 { 1528 struct _dif_sgl data_sgl, md_sgl; 1529 1530 if (md_iov->iov_base == NULL) { 1531 SPDK_ERRLOG("Metadata buffer is NULL.\n"); 1532 return -EINVAL; 1533 } 1534 1535 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1536 _dif_sgl_init(&md_sgl, md_iov, 1); 1537 1538 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1539 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1540 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1541 return -EINVAL; 1542 } 1543 1544 if (_dif_is_disabled(ctx->dif_type)) { 1545 return 0; 1546 } 1547 1548 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1549 return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1550 } else { 1551 return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1552 } 1553 } 1554 1555 int 1556 spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1557 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1558 uint32_t inject_flags, uint32_t *inject_offset) 1559 { 1560 struct _dif_sgl data_sgl, md_sgl; 1561 int rc; 1562 1563 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1564 _dif_sgl_init(&md_sgl, md_iov, 1); 1565 1566 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1567 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1568 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1569 return -EINVAL; 1570 } 1571 1572 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1573 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1574 ctx->guard_interval + _dif_offsetof_reftag(ctx->dif_pi_format), 1575 _dif_sizeof_reftag(ctx->dif_pi_format), 1576 inject_offset); 1577 if (rc != 0) { 1578 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1579 return rc; 1580 } 1581 } 1582 1583 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1584 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1585 ctx->guard_interval + _dif_offsetof_apptag(ctx->dif_pi_format), 1586 _dif_sizeof_apptag(), 1587 inject_offset); 1588 if (rc != 0) { 1589 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1590 return rc; 1591 } 1592 } 1593 1594 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1595 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1596 ctx->guard_interval, 1597 _dif_sizeof_guard(ctx->dif_pi_format), 1598 inject_offset); 1599 if (rc != 0) { 1600 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1601 return rc; 1602 } 1603 } 1604 1605 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1606 /* Note: Error injection to data block is expected to be detected 1607 * as guard error. 1608 */ 1609 rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks, 1610 0, 1611 ctx->block_size, 1612 inject_offset); 1613 if (rc != 0) { 1614 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1615 return rc; 1616 } 1617 } 1618 1619 return 0; 1620 } 1621 1622 static uint32_t 1623 _to_next_boundary(uint32_t offset, uint32_t boundary) 1624 { 1625 return boundary - (offset % boundary); 1626 } 1627 1628 static uint32_t 1629 _to_size_with_md(uint32_t size, uint32_t data_block_size, uint32_t block_size) 1630 { 1631 return (size / data_block_size) * block_size + (size % data_block_size); 1632 } 1633 1634 int 1635 spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt, 1636 struct iovec *buf_iovs, int buf_iovcnt, 1637 uint32_t data_offset, uint32_t data_len, 1638 uint32_t *_mapped_len, 1639 const struct spdk_dif_ctx *ctx) 1640 { 1641 uint32_t data_block_size, data_unalign, buf_len, buf_offset, len; 1642 struct _dif_sgl dif_sgl; 1643 struct _dif_sgl buf_sgl; 1644 1645 if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) { 1646 return -EINVAL; 1647 } 1648 1649 data_block_size = ctx->block_size - ctx->md_size; 1650 1651 data_unalign = ctx->data_offset % data_block_size; 1652 1653 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 1654 ctx->block_size); 1655 buf_len -= data_unalign; 1656 1657 _dif_sgl_init(&dif_sgl, iovs, iovcnt); 1658 _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt); 1659 1660 if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) { 1661 SPDK_ERRLOG("Buffer overflow will occur.\n"); 1662 return -ERANGE; 1663 } 1664 1665 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 1666 buf_offset -= data_unalign; 1667 1668 _dif_sgl_advance(&buf_sgl, buf_offset); 1669 1670 while (data_len != 0) { 1671 len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size)); 1672 if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) { 1673 break; 1674 } 1675 _dif_sgl_advance(&buf_sgl, ctx->md_size); 1676 data_offset += len; 1677 data_len -= len; 1678 } 1679 1680 if (_mapped_len != NULL) { 1681 *_mapped_len = dif_sgl.total_size; 1682 } 1683 1684 return iovcnt - dif_sgl.iovcnt; 1685 } 1686 1687 static int 1688 _dif_sgl_setup_stream(struct _dif_sgl *sgl, uint32_t *_buf_offset, uint32_t *_buf_len, 1689 uint32_t data_offset, uint32_t data_len, 1690 const struct spdk_dif_ctx *ctx) 1691 { 1692 uint32_t data_block_size, data_unalign, buf_len, buf_offset; 1693 1694 data_block_size = ctx->block_size - ctx->md_size; 1695 1696 data_unalign = ctx->data_offset % data_block_size; 1697 1698 /* If the last data block is complete, DIF of the data block is 1699 * inserted or verified in this turn. 1700 */ 1701 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 1702 ctx->block_size); 1703 buf_len -= data_unalign; 1704 1705 if (!_dif_sgl_is_valid(sgl, buf_len)) { 1706 return -ERANGE; 1707 } 1708 1709 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 1710 buf_offset -= data_unalign; 1711 1712 _dif_sgl_advance(sgl, buf_offset); 1713 buf_len -= buf_offset; 1714 1715 buf_offset += data_unalign; 1716 1717 *_buf_offset = buf_offset; 1718 *_buf_len = buf_len; 1719 1720 return 0; 1721 } 1722 1723 int 1724 spdk_dif_generate_stream(struct iovec *iovs, int iovcnt, 1725 uint32_t data_offset, uint32_t data_len, 1726 struct spdk_dif_ctx *ctx) 1727 { 1728 uint32_t buf_len = 0, buf_offset = 0; 1729 uint32_t len, offset_in_block, offset_blocks; 1730 uint32_t guard = 0; 1731 struct _dif_sgl sgl; 1732 int rc; 1733 1734 if (iovs == NULL || iovcnt == 0) { 1735 return -EINVAL; 1736 } 1737 1738 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1739 guard = ctx->last_guard; 1740 } 1741 1742 _dif_sgl_init(&sgl, iovs, iovcnt); 1743 1744 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1745 if (rc != 0) { 1746 return rc; 1747 } 1748 1749 while (buf_len != 0) { 1750 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1751 offset_in_block = buf_offset % ctx->block_size; 1752 offset_blocks = buf_offset / ctx->block_size; 1753 1754 guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx); 1755 1756 buf_len -= len; 1757 buf_offset += len; 1758 } 1759 1760 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1761 ctx->last_guard = guard; 1762 } 1763 1764 return 0; 1765 } 1766 1767 int 1768 spdk_dif_verify_stream(struct iovec *iovs, int iovcnt, 1769 uint32_t data_offset, uint32_t data_len, 1770 struct spdk_dif_ctx *ctx, 1771 struct spdk_dif_error *err_blk) 1772 { 1773 uint32_t buf_len = 0, buf_offset = 0; 1774 uint32_t len, offset_in_block, offset_blocks; 1775 uint32_t guard = 0; 1776 struct _dif_sgl sgl; 1777 int rc = 0; 1778 1779 if (iovs == NULL || iovcnt == 0) { 1780 return -EINVAL; 1781 } 1782 1783 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1784 guard = ctx->last_guard; 1785 } 1786 1787 _dif_sgl_init(&sgl, iovs, iovcnt); 1788 1789 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1790 if (rc != 0) { 1791 return rc; 1792 } 1793 1794 while (buf_len != 0) { 1795 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1796 offset_in_block = buf_offset % ctx->block_size; 1797 offset_blocks = buf_offset / ctx->block_size; 1798 1799 rc = _dif_verify_split(&sgl, offset_in_block, len, &guard, offset_blocks, 1800 ctx, err_blk); 1801 if (rc != 0) { 1802 goto error; 1803 } 1804 1805 buf_len -= len; 1806 buf_offset += len; 1807 } 1808 1809 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1810 ctx->last_guard = guard; 1811 } 1812 error: 1813 return rc; 1814 } 1815 1816 int 1817 spdk_dif_update_crc32c_stream(struct iovec *iovs, int iovcnt, 1818 uint32_t data_offset, uint32_t data_len, 1819 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 1820 { 1821 uint32_t buf_len = 0, buf_offset = 0, len, offset_in_block; 1822 uint32_t crc32c; 1823 struct _dif_sgl sgl; 1824 int rc; 1825 1826 if (iovs == NULL || iovcnt == 0) { 1827 return -EINVAL; 1828 } 1829 1830 crc32c = *_crc32c; 1831 _dif_sgl_init(&sgl, iovs, iovcnt); 1832 1833 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1834 if (rc != 0) { 1835 return rc; 1836 } 1837 1838 while (buf_len != 0) { 1839 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1840 offset_in_block = buf_offset % ctx->block_size; 1841 1842 crc32c = _dif_update_crc32c_split(&sgl, offset_in_block, len, crc32c, ctx); 1843 1844 buf_len -= len; 1845 buf_offset += len; 1846 } 1847 1848 *_crc32c = crc32c; 1849 1850 return 0; 1851 } 1852 1853 void 1854 spdk_dif_get_range_with_md(uint32_t data_offset, uint32_t data_len, 1855 uint32_t *_buf_offset, uint32_t *_buf_len, 1856 const struct spdk_dif_ctx *ctx) 1857 { 1858 uint32_t data_block_size, data_unalign, buf_offset, buf_len; 1859 1860 if (!ctx->md_interleave) { 1861 buf_offset = data_offset; 1862 buf_len = data_len; 1863 } else { 1864 data_block_size = ctx->block_size - ctx->md_size; 1865 1866 data_unalign = data_offset % data_block_size; 1867 1868 buf_offset = _to_size_with_md(data_offset, data_block_size, ctx->block_size); 1869 buf_len = _to_size_with_md(data_unalign + data_len, data_block_size, ctx->block_size) - 1870 data_unalign; 1871 } 1872 1873 if (_buf_offset != NULL) { 1874 *_buf_offset = buf_offset; 1875 } 1876 1877 if (_buf_len != NULL) { 1878 *_buf_len = buf_len; 1879 } 1880 } 1881 1882 uint32_t 1883 spdk_dif_get_length_with_md(uint32_t data_len, const struct spdk_dif_ctx *ctx) 1884 { 1885 uint32_t data_block_size; 1886 1887 if (!ctx->md_interleave) { 1888 return data_len; 1889 } else { 1890 data_block_size = ctx->block_size - ctx->md_size; 1891 1892 return _to_size_with_md(data_len, data_block_size, ctx->block_size); 1893 } 1894 } 1895 1896 static int 1897 _dif_remap_ref_tag(struct _dif_sgl *sgl, uint32_t offset_blocks, 1898 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 1899 { 1900 uint32_t offset, buf_len; 1901 uint64_t expected = 0, _actual, remapped; 1902 void *buf; 1903 struct _dif_sgl tmp_sgl; 1904 struct spdk_dif dif; 1905 1906 /* Fast forward to DIF field. */ 1907 _dif_sgl_advance(sgl, ctx->guard_interval); 1908 _dif_sgl_copy(&tmp_sgl, sgl); 1909 1910 /* Copy the split DIF field to the temporary DIF buffer */ 1911 offset = 0; 1912 while (offset < _dif_size(ctx->dif_pi_format)) { 1913 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1914 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 1915 1916 memcpy((uint8_t *)&dif + offset, buf, buf_len); 1917 1918 _dif_sgl_advance(sgl, buf_len); 1919 offset += buf_len; 1920 } 1921 1922 switch (ctx->dif_type) { 1923 case SPDK_DIF_TYPE1: 1924 case SPDK_DIF_TYPE2: 1925 /* If Type 1 or 2 is used, then all DIF checks are disabled when 1926 * the Application Tag is 0xFFFF. 1927 */ 1928 if (_dif_apptag_ignore(&dif, ctx->dif_pi_format)) { 1929 goto end; 1930 } 1931 break; 1932 case SPDK_DIF_TYPE3: 1933 /* If Type 3 is used, then all DIF checks are disabled when the 1934 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF 1935 * or 0xFFFFFFFFFFFFFFFF depending on the PI format. 1936 */ 1937 if (_dif_apptag_ignore(&dif, ctx->dif_pi_format) && 1938 _dif_reftag_ignore(&dif, ctx->dif_pi_format)) { 1939 goto end; 1940 } 1941 break; 1942 default: 1943 break; 1944 } 1945 1946 /* For type 1 and 2, the Reference Tag is incremented for each 1947 * subsequent logical block. For type 3, the Reference Tag 1948 * remains the same as the initial Reference Tag. 1949 */ 1950 if (ctx->dif_type != SPDK_DIF_TYPE3) { 1951 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 1952 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 1953 } else { 1954 remapped = ctx->remapped_init_ref_tag; 1955 } 1956 1957 /* Verify the stored Reference Tag. */ 1958 switch (ctx->dif_type) { 1959 case SPDK_DIF_TYPE1: 1960 case SPDK_DIF_TYPE2: 1961 /* Compare the DIF Reference Tag field to the passed Reference Tag. 1962 * The passed Reference Tag will be the least significant 4 bytes 1963 * or 8 bytes (depending on the PI format) 1964 * of the LBA when Type 1 is used, and application specific value 1965 * if Type 2 is used. 1966 */ 1967 if (!_dif_reftag_match(&dif, expected, ctx->dif_pi_format)) { 1968 _actual = _dif_get_reftag(&dif, ctx->dif_pi_format); 1969 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected, 1970 _actual, offset_blocks); 1971 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu64 "," \ 1972 " Expected=%lx, Actual=%lx\n", 1973 expected, expected, _actual); 1974 return -1; 1975 } 1976 break; 1977 case SPDK_DIF_TYPE3: 1978 /* For type 3, the computed Reference Tag remains unchanged. 1979 * Hence ignore the Reference Tag field. 1980 */ 1981 break; 1982 default: 1983 break; 1984 } 1985 1986 /* Update the stored Reference Tag to the remapped one. */ 1987 _dif_set_reftag(&dif, remapped, ctx->dif_pi_format); 1988 1989 offset = 0; 1990 while (offset < _dif_size(ctx->dif_pi_format)) { 1991 _dif_sgl_get_buf(&tmp_sgl, &buf, &buf_len); 1992 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 1993 1994 memcpy(buf, (uint8_t *)&dif + offset, buf_len); 1995 1996 _dif_sgl_advance(&tmp_sgl, buf_len); 1997 offset += buf_len; 1998 } 1999 2000 end: 2001 _dif_sgl_advance(sgl, ctx->block_size - ctx->guard_interval - _dif_size(ctx->dif_pi_format)); 2002 2003 return 0; 2004 } 2005 2006 int 2007 spdk_dif_remap_ref_tag(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 2008 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 2009 { 2010 struct _dif_sgl sgl; 2011 uint32_t offset_blocks; 2012 int rc; 2013 2014 _dif_sgl_init(&sgl, iovs, iovcnt); 2015 2016 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 2017 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2018 return -EINVAL; 2019 } 2020 2021 if (_dif_is_disabled(ctx->dif_type)) { 2022 return 0; 2023 } 2024 2025 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2026 return 0; 2027 } 2028 2029 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2030 rc = _dif_remap_ref_tag(&sgl, offset_blocks, ctx, err_blk); 2031 if (rc != 0) { 2032 return rc; 2033 } 2034 } 2035 2036 return 0; 2037 } 2038 2039 static int 2040 _dix_remap_ref_tag(struct _dif_sgl *md_sgl, uint32_t offset_blocks, 2041 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 2042 { 2043 uint64_t expected = 0, _actual, remapped; 2044 uint8_t *md_buf; 2045 struct spdk_dif *dif; 2046 2047 _dif_sgl_get_buf(md_sgl, (void *)&md_buf, NULL); 2048 2049 dif = (struct spdk_dif *)(md_buf + ctx->guard_interval); 2050 2051 switch (ctx->dif_type) { 2052 case SPDK_DIF_TYPE1: 2053 case SPDK_DIF_TYPE2: 2054 /* If Type 1 or 2 is used, then all DIF checks are disabled when 2055 * the Application Tag is 0xFFFF. 2056 */ 2057 if (_dif_apptag_ignore(dif, ctx->dif_pi_format)) { 2058 goto end; 2059 } 2060 break; 2061 case SPDK_DIF_TYPE3: 2062 /* If Type 3 is used, then all DIF checks are disabled when the 2063 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF 2064 * or 0xFFFFFFFFFFFFFFFF depending on the PI format. 2065 */ 2066 if (_dif_apptag_ignore(dif, ctx->dif_pi_format) && 2067 _dif_reftag_ignore(dif, ctx->dif_pi_format)) { 2068 goto end; 2069 } 2070 break; 2071 default: 2072 break; 2073 } 2074 2075 /* For type 1 and 2, the Reference Tag is incremented for each 2076 * subsequent logical block. For type 3, the Reference Tag 2077 * remains the same as the initialReference Tag. 2078 */ 2079 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2080 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2081 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2082 } else { 2083 remapped = ctx->remapped_init_ref_tag; 2084 } 2085 2086 /* Verify the stored Reference Tag. */ 2087 switch (ctx->dif_type) { 2088 case SPDK_DIF_TYPE1: 2089 case SPDK_DIF_TYPE2: 2090 /* Compare the DIF Reference Tag field to the passed Reference Tag. 2091 * The passed Reference Tag will be the least significant 4 bytes 2092 * or 8 bytes (depending on the PI format) 2093 * of the LBA when Type 1 is used, and application specific value 2094 * if Type 2 is used. 2095 */ 2096 if (!_dif_reftag_match(dif, expected, ctx->dif_pi_format)) { 2097 _actual = _dif_get_reftag(dif, ctx->dif_pi_format); 2098 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected, 2099 _actual, offset_blocks); 2100 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu64 "," \ 2101 " Expected=%lx, Actual=%lx\n", 2102 expected, expected, _actual); 2103 return -1; 2104 } 2105 break; 2106 case SPDK_DIF_TYPE3: 2107 /* For type 3, the computed Reference Tag remains unchanged. 2108 * Hence ignore the Reference Tag field. 2109 */ 2110 break; 2111 default: 2112 break; 2113 } 2114 2115 /* Update the stored Reference Tag to the remapped one. */ 2116 _dif_set_reftag(dif, remapped, ctx->dif_pi_format); 2117 2118 end: 2119 _dif_sgl_advance(md_sgl, ctx->md_size); 2120 2121 return 0; 2122 } 2123 2124 int 2125 spdk_dix_remap_ref_tag(struct iovec *md_iov, uint32_t num_blocks, 2126 const struct spdk_dif_ctx *ctx, 2127 struct spdk_dif_error *err_blk) 2128 { 2129 struct _dif_sgl md_sgl; 2130 uint32_t offset_blocks; 2131 int rc; 2132 2133 _dif_sgl_init(&md_sgl, md_iov, 1); 2134 2135 if (!_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2136 SPDK_ERRLOG("Size of metadata iovec array is not valid.\n"); 2137 return -EINVAL; 2138 } 2139 2140 if (_dif_is_disabled(ctx->dif_type)) { 2141 return 0; 2142 } 2143 2144 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2145 return 0; 2146 } 2147 2148 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2149 rc = _dix_remap_ref_tag(&md_sgl, offset_blocks, ctx, err_blk); 2150 if (rc != 0) { 2151 return rc; 2152 } 2153 } 2154 2155 return 0; 2156 } 2157