1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/dif.h" 7 #include "spdk/crc16.h" 8 #include "spdk/crc32.h" 9 #include "spdk/crc64.h" 10 #include "spdk/endian.h" 11 #include "spdk/log.h" 12 #include "spdk/util.h" 13 14 #define REFTAG_MASK_16 0x00000000FFFFFFFF 15 #define REFTAG_MASK_32 0xFFFFFFFFFFFFFFFF 16 #define REFTAG_MASK_64 0x0000FFFFFFFFFFFF 17 18 /* The variable size Storage Tag and Reference Tag is not supported yet, 19 * so the maximum size of the Reference Tag is assumed. 20 */ 21 struct spdk_dif { 22 union { 23 struct { 24 uint16_t guard; 25 uint16_t app_tag; 26 uint32_t stor_ref_space; 27 } g16; 28 struct { 29 uint32_t guard; 30 uint16_t app_tag; 31 uint16_t stor_ref_space_p1; 32 uint64_t stor_ref_space_p2; 33 } g32; 34 struct { 35 uint64_t guard; 36 uint16_t app_tag; 37 uint16_t stor_ref_space_p1; 38 uint32_t stor_ref_space_p2; 39 } g64; 40 }; 41 }; 42 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g16) == 8, "Incorrect size"); 43 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g32) == 16, "Incorrect size"); 44 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g64) == 16, "Incorrect size"); 45 46 /* Context to iterate or create a iovec array. 47 * Each sgl is either iterated or created at a time. 48 */ 49 struct _dif_sgl { 50 /* Current iovec in the iteration or creation */ 51 struct iovec *iov; 52 53 /* Remaining count of iovecs in the iteration or creation. */ 54 int iovcnt; 55 56 /* Current offset in the iovec */ 57 uint32_t iov_offset; 58 59 /* Size of the created iovec array in bytes */ 60 uint32_t total_size; 61 }; 62 63 static inline void 64 _dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt) 65 { 66 s->iov = iovs; 67 s->iovcnt = iovcnt; 68 s->iov_offset = 0; 69 s->total_size = 0; 70 } 71 72 static void 73 _dif_sgl_advance(struct _dif_sgl *s, uint32_t step) 74 { 75 s->iov_offset += step; 76 while (s->iovcnt != 0) { 77 if (s->iov_offset < s->iov->iov_len) { 78 break; 79 } 80 81 s->iov_offset -= s->iov->iov_len; 82 s->iov++; 83 s->iovcnt--; 84 } 85 } 86 87 static inline void 88 _dif_sgl_get_buf(struct _dif_sgl *s, uint8_t **_buf, uint32_t *_buf_len) 89 { 90 if (_buf != NULL) { 91 *_buf = (uint8_t *)s->iov->iov_base + s->iov_offset; 92 } 93 if (_buf_len != NULL) { 94 *_buf_len = s->iov->iov_len - s->iov_offset; 95 } 96 } 97 98 static inline bool 99 _dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len) 100 { 101 assert(s->iovcnt > 0); 102 s->iov->iov_base = data; 103 s->iov->iov_len = data_len; 104 s->total_size += data_len; 105 s->iov++; 106 s->iovcnt--; 107 108 if (s->iovcnt > 0) { 109 return true; 110 } else { 111 return false; 112 } 113 } 114 115 static inline bool 116 _dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len) 117 { 118 uint8_t *buf; 119 uint32_t buf_len; 120 121 while (data_len != 0) { 122 _dif_sgl_get_buf(src, &buf, &buf_len); 123 buf_len = spdk_min(buf_len, data_len); 124 125 if (!_dif_sgl_append(dst, buf, buf_len)) { 126 return false; 127 } 128 129 _dif_sgl_advance(src, buf_len); 130 data_len -= buf_len; 131 } 132 133 return true; 134 } 135 136 /* This function must be used before starting iteration. */ 137 static bool 138 _dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes) 139 { 140 int i; 141 142 for (i = 0; i < s->iovcnt; i++) { 143 if (s->iov[i].iov_len % bytes) { 144 return false; 145 } 146 } 147 148 return true; 149 } 150 151 /* This function must be used before starting iteration. */ 152 static bool 153 _dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes) 154 { 155 uint64_t total = 0; 156 int i; 157 158 for (i = 0; i < s->iovcnt; i++) { 159 total += s->iov[i].iov_len; 160 } 161 162 return total >= bytes; 163 } 164 165 static void 166 _dif_sgl_copy(struct _dif_sgl *to, struct _dif_sgl *from) 167 { 168 memcpy(to, from, sizeof(struct _dif_sgl)); 169 } 170 171 static bool 172 _dif_is_disabled(enum spdk_dif_type dif_type) 173 { 174 if (dif_type == SPDK_DIF_DISABLE) { 175 return true; 176 } else { 177 return false; 178 } 179 } 180 181 static inline size_t 182 _dif_size(enum spdk_dif_pi_format dif_pi_format) 183 { 184 uint8_t size; 185 186 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 187 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16); 188 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 189 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32); 190 } else { 191 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64); 192 } 193 194 return size; 195 } 196 197 static uint32_t 198 _get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave, 199 size_t dif_size) 200 { 201 if (!dif_loc) { 202 /* For metadata formats with more than 8/16 bytes (depending on 203 * the PI format), if the DIF is contained in the last 8/16 bytes 204 * of metadata, then the CRC covers all metadata up to but excluding 205 * these last 8/16 bytes. 206 */ 207 if (md_interleave) { 208 return block_size - dif_size; 209 } else { 210 return md_size - dif_size; 211 } 212 } else { 213 /* For metadata formats with more than 8/16 bytes (depending on 214 * the PI format), if the DIF is contained in the first 8/16 bytes 215 * of metadata, then the CRC does not cover any metadata. 216 */ 217 if (md_interleave) { 218 return block_size - md_size; 219 } else { 220 return 0; 221 } 222 } 223 } 224 225 static inline uint8_t 226 _dif_guard_size(enum spdk_dif_pi_format dif_pi_format) 227 { 228 uint8_t size; 229 230 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 231 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.guard); 232 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 233 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.guard); 234 } else { 235 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.guard); 236 } 237 238 return size; 239 } 240 241 static inline void 242 _dif_set_guard(struct spdk_dif *dif, uint64_t guard, enum spdk_dif_pi_format dif_pi_format) 243 { 244 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 245 to_be16(&(dif->g16.guard), (uint16_t)guard); 246 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 247 to_be32(&(dif->g32.guard), (uint32_t)guard); 248 } else { 249 to_be64(&(dif->g64.guard), guard); 250 } 251 } 252 253 static inline uint64_t 254 _dif_get_guard(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 255 { 256 uint64_t guard; 257 258 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 259 guard = (uint64_t)from_be16(&(dif->g16.guard)); 260 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 261 guard = (uint64_t)from_be32(&(dif->g32.guard)); 262 } else { 263 guard = from_be64(&(dif->g64.guard)); 264 } 265 266 return guard; 267 } 268 269 static inline uint64_t 270 _dif_generate_guard(uint64_t guard_seed, void *buf, size_t buf_len, 271 enum spdk_dif_pi_format dif_pi_format) 272 { 273 uint64_t guard; 274 275 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 276 guard = (uint64_t)spdk_crc16_t10dif((uint16_t)guard_seed, buf, buf_len); 277 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 278 guard = (uint64_t)spdk_crc32c_nvme(buf, buf_len, guard_seed); 279 } else { 280 guard = spdk_crc64_nvme(buf, buf_len, guard_seed); 281 } 282 283 return guard; 284 } 285 286 static uint64_t 287 dif_generate_guard_split(uint64_t guard_seed, struct _dif_sgl *sgl, uint32_t start, 288 uint32_t len, const struct spdk_dif_ctx *ctx) 289 { 290 uint64_t guard = guard_seed; 291 uint32_t offset, end, buf_len; 292 uint8_t *buf; 293 294 offset = start; 295 end = start + spdk_min(len, ctx->guard_interval - start); 296 297 while (offset < end) { 298 _dif_sgl_get_buf(sgl, &buf, &buf_len); 299 buf_len = spdk_min(buf_len, end - offset); 300 301 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 302 guard = _dif_generate_guard(guard, buf, buf_len, ctx->dif_pi_format); 303 } 304 305 _dif_sgl_advance(sgl, buf_len); 306 offset += buf_len; 307 } 308 309 return guard; 310 } 311 312 static inline uint64_t 313 _dif_generate_guard_copy(uint64_t guard_seed, void *dst, void *src, size_t buf_len, 314 enum spdk_dif_pi_format dif_pi_format) 315 { 316 uint64_t guard; 317 318 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 319 guard = (uint64_t)spdk_crc16_t10dif_copy((uint16_t)guard_seed, dst, src, buf_len); 320 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 321 memcpy(dst, src, buf_len); 322 guard = (uint64_t)spdk_crc32c_nvme(src, buf_len, guard_seed); 323 } else { 324 memcpy(dst, src, buf_len); 325 guard = spdk_crc64_nvme(src, buf_len, guard_seed); 326 } 327 328 return guard; 329 } 330 331 static uint64_t 332 _dif_generate_guard_copy_split(uint64_t guard, struct _dif_sgl *dst_sgl, 333 struct _dif_sgl *src_sgl, uint32_t data_len, 334 enum spdk_dif_pi_format dif_pi_format) 335 { 336 uint32_t offset = 0, src_len, dst_len, buf_len; 337 uint8_t *src, *dst; 338 339 while (offset < data_len) { 340 _dif_sgl_get_buf(src_sgl, &src, &src_len); 341 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 342 buf_len = spdk_min(src_len, dst_len); 343 buf_len = spdk_min(buf_len, data_len - offset); 344 345 guard = _dif_generate_guard_copy(guard, dst, src, buf_len, dif_pi_format); 346 347 _dif_sgl_advance(src_sgl, buf_len); 348 _dif_sgl_advance(dst_sgl, buf_len); 349 offset += buf_len; 350 } 351 352 return guard; 353 } 354 355 static void 356 _data_copy_split(struct _dif_sgl *dst_sgl, struct _dif_sgl *src_sgl, uint32_t data_len) 357 { 358 uint32_t offset = 0, src_len, dst_len, buf_len; 359 uint8_t *src, *dst; 360 361 while (offset < data_len) { 362 _dif_sgl_get_buf(src_sgl, &src, &src_len); 363 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 364 buf_len = spdk_min(src_len, dst_len); 365 buf_len = spdk_min(buf_len, data_len - offset); 366 367 memcpy(dst, src, buf_len); 368 369 _dif_sgl_advance(src_sgl, buf_len); 370 _dif_sgl_advance(dst_sgl, buf_len); 371 offset += buf_len; 372 } 373 } 374 375 static inline uint8_t 376 _dif_apptag_offset(enum spdk_dif_pi_format dif_pi_format) 377 { 378 return _dif_guard_size(dif_pi_format); 379 } 380 381 static inline uint8_t 382 _dif_apptag_size(void) 383 { 384 return SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.app_tag); 385 } 386 387 static inline void 388 _dif_set_apptag(struct spdk_dif *dif, uint16_t app_tag, enum spdk_dif_pi_format dif_pi_format) 389 { 390 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 391 to_be16(&(dif->g16.app_tag), app_tag); 392 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 393 to_be16(&(dif->g32.app_tag), app_tag); 394 } else { 395 to_be16(&(dif->g64.app_tag), app_tag); 396 } 397 } 398 399 static inline uint16_t 400 _dif_get_apptag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 401 { 402 uint16_t app_tag; 403 404 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 405 app_tag = from_be16(&(dif->g16.app_tag)); 406 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 407 app_tag = from_be16(&(dif->g32.app_tag)); 408 } else { 409 app_tag = from_be16(&(dif->g64.app_tag)); 410 } 411 412 return app_tag; 413 } 414 415 static inline bool 416 _dif_apptag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 417 { 418 return _dif_get_apptag(dif, dif_pi_format) == SPDK_DIF_APPTAG_IGNORE; 419 } 420 421 static inline uint8_t 422 _dif_reftag_offset(enum spdk_dif_pi_format dif_pi_format) 423 { 424 uint8_t offset; 425 426 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 427 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size(); 428 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 429 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size() 430 + SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.stor_ref_space_p1); 431 } else { 432 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size(); 433 } 434 435 return offset; 436 } 437 438 static inline uint8_t 439 _dif_reftag_size(enum spdk_dif_pi_format dif_pi_format) 440 { 441 uint8_t size; 442 443 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 444 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.stor_ref_space); 445 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 446 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.stor_ref_space_p2); 447 } else { 448 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.stor_ref_space_p1) + 449 SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.stor_ref_space_p2); 450 } 451 452 return size; 453 } 454 455 static inline void 456 _dif_set_reftag(struct spdk_dif *dif, uint64_t ref_tag, enum spdk_dif_pi_format dif_pi_format) 457 { 458 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 459 to_be32(&(dif->g16.stor_ref_space), (uint32_t)ref_tag); 460 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 461 to_be64(&(dif->g32.stor_ref_space_p2), ref_tag); 462 } else { 463 to_be16(&(dif->g64.stor_ref_space_p1), (uint16_t)(ref_tag >> 32)); 464 to_be32(&(dif->g64.stor_ref_space_p2), (uint32_t)ref_tag); 465 } 466 } 467 468 static inline uint64_t 469 _dif_get_reftag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 470 { 471 uint64_t ref_tag; 472 473 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 474 ref_tag = (uint64_t)from_be32(&(dif->g16.stor_ref_space)); 475 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 476 ref_tag = from_be64(&(dif->g32.stor_ref_space_p2)); 477 } else { 478 ref_tag = (uint64_t)from_be16(&(dif->g64.stor_ref_space_p1)); 479 ref_tag <<= 32; 480 ref_tag |= (uint64_t)from_be32(&(dif->g64.stor_ref_space_p2)); 481 } 482 483 return ref_tag; 484 } 485 486 static inline bool 487 _dif_reftag_match(struct spdk_dif *dif, uint64_t ref_tag, 488 enum spdk_dif_pi_format dif_pi_format) 489 { 490 uint64_t _ref_tag; 491 bool match; 492 493 _ref_tag = _dif_get_reftag(dif, dif_pi_format); 494 495 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 496 match = (_ref_tag == (ref_tag & REFTAG_MASK_16)); 497 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 498 match = (_ref_tag == ref_tag); 499 } else { 500 match = (_ref_tag == (ref_tag & REFTAG_MASK_64)); 501 } 502 503 return match; 504 } 505 506 static inline bool 507 _dif_reftag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 508 { 509 return _dif_reftag_match(dif, REFTAG_MASK_32, dif_pi_format); 510 } 511 512 static bool 513 _dif_ignore(struct spdk_dif *dif, const struct spdk_dif_ctx *ctx) 514 { 515 switch (ctx->dif_type) { 516 case SPDK_DIF_TYPE1: 517 case SPDK_DIF_TYPE2: 518 /* If Type 1 or 2 is used, then all DIF checks are disabled when 519 * the Application Tag is 0xFFFF. 520 */ 521 if (_dif_apptag_ignore(dif, ctx->dif_pi_format)) { 522 return true; 523 } 524 break; 525 case SPDK_DIF_TYPE3: 526 /* If Type 3 is used, then all DIF checks are disabled when the 527 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF 528 * or 0xFFFFFFFFFFFFFFFF depending on the PI format. 529 */ 530 531 if (_dif_apptag_ignore(dif, ctx->dif_pi_format) && 532 _dif_reftag_ignore(dif, ctx->dif_pi_format)) { 533 return true; 534 } 535 break; 536 default: 537 break; 538 } 539 540 return false; 541 } 542 543 static bool 544 _dif_pi_format_is_valid(enum spdk_dif_pi_format dif_pi_format) 545 { 546 switch (dif_pi_format) { 547 case SPDK_DIF_PI_FORMAT_16: 548 case SPDK_DIF_PI_FORMAT_32: 549 case SPDK_DIF_PI_FORMAT_64: 550 return true; 551 default: 552 return false; 553 } 554 } 555 556 static bool 557 _dif_type_is_valid(enum spdk_dif_type dif_type) 558 { 559 switch (dif_type) { 560 case SPDK_DIF_DISABLE: 561 case SPDK_DIF_TYPE1: 562 case SPDK_DIF_TYPE2: 563 case SPDK_DIF_TYPE3: 564 return true; 565 default: 566 return false; 567 } 568 } 569 570 int 571 spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, 572 bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, 573 uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag, 574 uint32_t data_offset, uint64_t guard_seed, struct spdk_dif_ctx_init_ext_opts *opts) 575 { 576 uint32_t data_block_size; 577 enum spdk_dif_pi_format dif_pi_format = SPDK_DIF_PI_FORMAT_16; 578 579 if (opts != NULL) { 580 if (!_dif_pi_format_is_valid(opts->dif_pi_format)) { 581 SPDK_ERRLOG("No valid DIF PI format provided.\n"); 582 return -EINVAL; 583 } 584 585 dif_pi_format = opts->dif_pi_format; 586 } 587 588 if (!_dif_type_is_valid(dif_type)) { 589 SPDK_ERRLOG("No valid DIF type was provided.\n"); 590 return -EINVAL; 591 } 592 593 if (md_size < _dif_size(dif_pi_format)) { 594 SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); 595 return -EINVAL; 596 } 597 598 if (md_interleave) { 599 if (block_size < md_size) { 600 SPDK_ERRLOG("Block size is smaller than DIF size.\n"); 601 return -EINVAL; 602 } 603 data_block_size = block_size - md_size; 604 } else { 605 data_block_size = block_size; 606 } 607 608 if (data_block_size == 0) { 609 SPDK_ERRLOG("Zero data block size is not allowed\n"); 610 return -EINVAL; 611 } 612 613 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 614 if ((data_block_size % 512) != 0) { 615 SPDK_ERRLOG("Data block size should be a multiple of 512B\n"); 616 return -EINVAL; 617 } 618 } else { 619 if ((data_block_size % 4096) != 0) { 620 SPDK_ERRLOG("Data block size should be a multiple of 4kB\n"); 621 return -EINVAL; 622 } 623 } 624 625 ctx->block_size = block_size; 626 ctx->md_size = md_size; 627 ctx->md_interleave = md_interleave; 628 ctx->dif_pi_format = dif_pi_format; 629 ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave, 630 _dif_size(ctx->dif_pi_format)); 631 ctx->dif_type = dif_type; 632 ctx->dif_flags = dif_flags; 633 ctx->init_ref_tag = init_ref_tag; 634 ctx->apptag_mask = apptag_mask; 635 ctx->app_tag = app_tag; 636 ctx->data_offset = data_offset; 637 ctx->ref_tag_offset = data_offset / data_block_size; 638 ctx->last_guard = guard_seed; 639 ctx->guard_seed = guard_seed; 640 ctx->remapped_init_ref_tag = 0; 641 642 return 0; 643 } 644 645 void 646 spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset) 647 { 648 uint32_t data_block_size; 649 650 if (ctx->md_interleave) { 651 data_block_size = ctx->block_size - ctx->md_size; 652 } else { 653 data_block_size = ctx->block_size; 654 } 655 656 ctx->data_offset = data_offset; 657 ctx->ref_tag_offset = data_offset / data_block_size; 658 } 659 660 void 661 spdk_dif_ctx_set_remapped_init_ref_tag(struct spdk_dif_ctx *ctx, 662 uint32_t remapped_init_ref_tag) 663 { 664 ctx->remapped_init_ref_tag = remapped_init_ref_tag; 665 } 666 667 static void 668 _dif_generate(void *_dif, uint64_t guard, uint32_t offset_blocks, 669 const struct spdk_dif_ctx *ctx) 670 { 671 struct spdk_dif *dif = _dif; 672 uint64_t ref_tag; 673 674 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 675 _dif_set_guard(dif, guard, ctx->dif_pi_format); 676 } else { 677 _dif_set_guard(dif, 0, ctx->dif_pi_format); 678 } 679 680 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 681 _dif_set_apptag(dif, ctx->app_tag, ctx->dif_pi_format); 682 } else { 683 _dif_set_apptag(dif, 0, ctx->dif_pi_format); 684 } 685 686 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 687 /* For type 1 and 2, the reference tag is incremented for each 688 * subsequent logical block. For type 3, the reference tag 689 * remains the same as the initial reference tag. 690 */ 691 if (ctx->dif_type != SPDK_DIF_TYPE3) { 692 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 693 } else { 694 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 695 } 696 697 /* Overwrite reference tag if initialization reference tag is SPDK_DIF_REFTAG_IGNORE */ 698 if (ctx->init_ref_tag == SPDK_DIF_REFTAG_IGNORE) { 699 if (ctx->dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 700 ref_tag = REFTAG_MASK_16; 701 } else if (ctx->dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 702 ref_tag = REFTAG_MASK_32; 703 } else { 704 ref_tag = REFTAG_MASK_64; 705 } 706 } 707 708 _dif_set_reftag(dif, ref_tag, ctx->dif_pi_format); 709 } else { 710 _dif_set_reftag(dif, 0, ctx->dif_pi_format); 711 } 712 } 713 714 static void 715 dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 716 { 717 uint32_t offset_blocks; 718 uint8_t *buf; 719 uint64_t guard = 0; 720 721 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 722 _dif_sgl_get_buf(sgl, &buf, NULL); 723 724 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 725 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 726 } 727 728 _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); 729 730 _dif_sgl_advance(sgl, ctx->block_size); 731 } 732 } 733 734 static void 735 dif_store_split(struct _dif_sgl *sgl, struct spdk_dif *dif, 736 const struct spdk_dif_ctx *ctx) 737 { 738 uint32_t offset = 0, rest_md_len, buf_len; 739 uint8_t *buf; 740 741 rest_md_len = ctx->block_size - ctx->guard_interval; 742 743 while (offset < rest_md_len) { 744 _dif_sgl_get_buf(sgl, &buf, &buf_len); 745 746 if (offset < _dif_size(ctx->dif_pi_format)) { 747 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 748 memcpy(buf, (uint8_t *)dif + offset, buf_len); 749 } else { 750 buf_len = spdk_min(buf_len, rest_md_len - offset); 751 } 752 753 _dif_sgl_advance(sgl, buf_len); 754 offset += buf_len; 755 } 756 } 757 758 static uint64_t 759 _dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 760 uint64_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 761 { 762 struct spdk_dif dif = {}; 763 764 assert(offset_in_block < ctx->guard_interval); 765 assert(offset_in_block + data_len < ctx->guard_interval || 766 offset_in_block + data_len == ctx->block_size); 767 768 /* Compute CRC over split logical block data. */ 769 guard = dif_generate_guard_split(guard, sgl, offset_in_block, data_len, ctx); 770 771 if (offset_in_block + data_len < ctx->guard_interval) { 772 return guard; 773 } 774 775 /* If a whole logical block data is parsed, generate DIF 776 * and save it to the temporary DIF area. 777 */ 778 _dif_generate(&dif, guard, offset_blocks, ctx); 779 780 /* Copy generated DIF field to the split DIF field, and then 781 * skip metadata field after DIF field (if any). 782 */ 783 dif_store_split(sgl, &dif, ctx); 784 785 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 786 guard = ctx->guard_seed; 787 } 788 789 return guard; 790 } 791 792 static void 793 dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks, 794 const struct spdk_dif_ctx *ctx) 795 { 796 uint32_t offset_blocks; 797 uint64_t guard = 0; 798 799 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 800 guard = ctx->guard_seed; 801 } 802 803 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 804 _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx); 805 } 806 } 807 808 int 809 spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 810 const struct spdk_dif_ctx *ctx) 811 { 812 struct _dif_sgl sgl; 813 814 _dif_sgl_init(&sgl, iovs, iovcnt); 815 816 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 817 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 818 return -EINVAL; 819 } 820 821 if (_dif_is_disabled(ctx->dif_type)) { 822 return 0; 823 } 824 825 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 826 dif_generate(&sgl, num_blocks, ctx); 827 } else { 828 dif_generate_split(&sgl, num_blocks, ctx); 829 } 830 831 return 0; 832 } 833 834 static void 835 _dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, 836 uint64_t expected, uint64_t actual, uint32_t err_offset) 837 { 838 if (err_blk) { 839 err_blk->err_type = err_type; 840 err_blk->expected = expected; 841 err_blk->actual = actual; 842 err_blk->err_offset = err_offset; 843 } 844 } 845 846 static bool 847 _dif_reftag_check(struct spdk_dif *dif, const struct spdk_dif_ctx *ctx, 848 uint64_t expected_reftag, uint32_t offset_blocks, struct spdk_dif_error *err_blk) 849 { 850 uint64_t reftag; 851 852 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 853 switch (ctx->dif_type) { 854 case SPDK_DIF_TYPE1: 855 case SPDK_DIF_TYPE2: 856 /* Compare the DIF Reference Tag field to the passed Reference Tag. 857 * The passed Reference Tag will be the least significant 4 bytes 858 * or 8 bytes (depending on the PI format) 859 * of the LBA when Type 1 is used, and application specific value 860 * if Type 2 is used. 861 */ 862 if (!_dif_reftag_match(dif, expected_reftag, ctx->dif_pi_format)) { 863 reftag = _dif_get_reftag(dif, ctx->dif_pi_format); 864 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected_reftag, 865 reftag, offset_blocks); 866 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu64 "," \ 867 " Expected=%lx, Actual=%lx\n", 868 expected_reftag, expected_reftag, reftag); 869 return false; 870 } 871 break; 872 case SPDK_DIF_TYPE3: 873 /* For Type 3, computed Reference Tag remains unchanged. 874 * Hence ignore the Reference Tag field. 875 */ 876 break; 877 default: 878 break; 879 } 880 } 881 882 return true; 883 } 884 885 static int 886 _dif_verify(void *_dif, uint64_t guard, uint32_t offset_blocks, 887 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 888 { 889 struct spdk_dif *dif = _dif; 890 uint64_t _guard; 891 uint16_t _app_tag; 892 uint64_t ref_tag; 893 894 if (_dif_ignore(dif, ctx)) { 895 return 0; 896 } 897 898 /* For type 1 and 2, the reference tag is incremented for each 899 * subsequent logical block. For type 3, the reference tag 900 * remains the same as the initial reference tag. 901 */ 902 if (ctx->dif_type != SPDK_DIF_TYPE3) { 903 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 904 } else { 905 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 906 } 907 908 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 909 /* Compare the DIF Guard field to the CRC computed over the logical 910 * block data. 911 */ 912 _guard = _dif_get_guard(dif, ctx->dif_pi_format); 913 if (_guard != guard) { 914 _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, 915 offset_blocks); 916 SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu64 "," \ 917 " Expected=%lx, Actual=%lx\n", 918 ref_tag, _guard, guard); 919 return -1; 920 } 921 } 922 923 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 924 /* Compare unmasked bits in the DIF Application Tag field to the 925 * passed Application Tag. 926 */ 927 _app_tag = _dif_get_apptag(dif, ctx->dif_pi_format); 928 if ((_app_tag & ctx->apptag_mask) != (ctx->app_tag & ctx->apptag_mask)) { 929 _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, 930 (_app_tag & ctx->apptag_mask), offset_blocks); 931 SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu64 "," \ 932 " Expected=%x, Actual=%x\n", 933 ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); 934 return -1; 935 } 936 } 937 938 if (!_dif_reftag_check(dif, ctx, ref_tag, offset_blocks, err_blk)) { 939 return -1; 940 } 941 942 return 0; 943 } 944 945 static int 946 dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks, 947 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 948 { 949 uint32_t offset_blocks; 950 int rc; 951 uint8_t *buf; 952 uint64_t guard = 0; 953 954 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 955 _dif_sgl_get_buf(sgl, &buf, NULL); 956 957 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 958 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 959 } 960 961 rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 962 if (rc != 0) { 963 return rc; 964 } 965 966 _dif_sgl_advance(sgl, ctx->block_size); 967 } 968 969 return 0; 970 } 971 972 static void 973 dif_load_split(struct _dif_sgl *sgl, struct spdk_dif *dif, 974 const struct spdk_dif_ctx *ctx) 975 { 976 uint32_t offset = 0, rest_md_len, buf_len; 977 uint8_t *buf; 978 979 rest_md_len = ctx->block_size - ctx->guard_interval; 980 981 while (offset < rest_md_len) { 982 _dif_sgl_get_buf(sgl, &buf, &buf_len); 983 984 if (offset < _dif_size(ctx->dif_pi_format)) { 985 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 986 memcpy((uint8_t *)dif + offset, buf, buf_len); 987 } else { 988 buf_len = spdk_min(buf_len, rest_md_len - offset); 989 } 990 991 _dif_sgl_advance(sgl, buf_len); 992 offset += buf_len; 993 } 994 } 995 996 static int 997 _dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 998 uint64_t *_guard, uint32_t offset_blocks, 999 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 1000 { 1001 uint64_t guard = *_guard; 1002 struct spdk_dif dif = {}; 1003 int rc; 1004 1005 assert(_guard != NULL); 1006 assert(offset_in_block < ctx->guard_interval); 1007 assert(offset_in_block + data_len < ctx->guard_interval || 1008 offset_in_block + data_len == ctx->block_size); 1009 1010 guard = dif_generate_guard_split(guard, sgl, offset_in_block, data_len, ctx); 1011 1012 if (offset_in_block + data_len < ctx->guard_interval) { 1013 *_guard = guard; 1014 return 0; 1015 } 1016 1017 dif_load_split(sgl, &dif, ctx); 1018 1019 rc = _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 1020 if (rc != 0) { 1021 return rc; 1022 } 1023 1024 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1025 guard = ctx->guard_seed; 1026 } 1027 1028 *_guard = guard; 1029 return 0; 1030 } 1031 1032 static int 1033 dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks, 1034 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 1035 { 1036 uint32_t offset_blocks; 1037 uint64_t guard = 0; 1038 int rc; 1039 1040 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1041 guard = ctx->guard_seed; 1042 } 1043 1044 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1045 rc = _dif_verify_split(sgl, 0, ctx->block_size, &guard, offset_blocks, 1046 ctx, err_blk); 1047 if (rc != 0) { 1048 return rc; 1049 } 1050 } 1051 1052 return 0; 1053 } 1054 1055 int 1056 spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1057 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 1058 { 1059 struct _dif_sgl sgl; 1060 1061 _dif_sgl_init(&sgl, iovs, iovcnt); 1062 1063 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1064 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1065 return -EINVAL; 1066 } 1067 1068 if (_dif_is_disabled(ctx->dif_type)) { 1069 return 0; 1070 } 1071 1072 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 1073 return dif_verify(&sgl, num_blocks, ctx, err_blk); 1074 } else { 1075 return dif_verify_split(&sgl, num_blocks, ctx, err_blk); 1076 } 1077 } 1078 1079 static uint32_t 1080 dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks, 1081 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1082 { 1083 uint32_t offset_blocks; 1084 uint8_t *buf; 1085 1086 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1087 _dif_sgl_get_buf(sgl, &buf, NULL); 1088 1089 crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c); 1090 1091 _dif_sgl_advance(sgl, ctx->block_size); 1092 } 1093 1094 return crc32c; 1095 } 1096 1097 static uint32_t 1098 _dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 1099 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1100 { 1101 uint32_t data_block_size, buf_len; 1102 uint8_t *buf; 1103 1104 data_block_size = ctx->block_size - ctx->md_size; 1105 1106 assert(offset_in_block + data_len <= ctx->block_size); 1107 1108 while (data_len != 0) { 1109 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1110 buf_len = spdk_min(buf_len, data_len); 1111 1112 if (offset_in_block < data_block_size) { 1113 buf_len = spdk_min(buf_len, data_block_size - offset_in_block); 1114 crc32c = spdk_crc32c_update(buf, buf_len, crc32c); 1115 } 1116 1117 _dif_sgl_advance(sgl, buf_len); 1118 offset_in_block += buf_len; 1119 data_len -= buf_len; 1120 } 1121 1122 return crc32c; 1123 } 1124 1125 static uint32_t 1126 dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks, 1127 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1128 { 1129 uint32_t offset_blocks; 1130 1131 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1132 crc32c = _dif_update_crc32c_split(sgl, 0, ctx->block_size, crc32c, ctx); 1133 } 1134 1135 return crc32c; 1136 } 1137 1138 int 1139 spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1140 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 1141 { 1142 struct _dif_sgl sgl; 1143 1144 if (_crc32c == NULL) { 1145 return -EINVAL; 1146 } 1147 1148 _dif_sgl_init(&sgl, iovs, iovcnt); 1149 1150 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1151 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1152 return -EINVAL; 1153 } 1154 1155 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 1156 *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx); 1157 } else { 1158 *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx); 1159 } 1160 1161 return 0; 1162 } 1163 1164 static void 1165 _dif_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1166 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1167 { 1168 uint32_t data_block_size; 1169 uint8_t *src, *dst; 1170 uint64_t guard = 0; 1171 1172 data_block_size = ctx->block_size - ctx->md_size; 1173 1174 _dif_sgl_get_buf(src_sgl, &src, NULL); 1175 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1176 1177 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1178 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1179 ctx->dif_pi_format); 1180 guard = _dif_generate_guard(guard, dst + data_block_size, 1181 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1182 } else { 1183 memcpy(dst, src, data_block_size); 1184 } 1185 1186 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 1187 1188 _dif_sgl_advance(src_sgl, data_block_size); 1189 _dif_sgl_advance(dst_sgl, ctx->block_size); 1190 } 1191 1192 static void 1193 dif_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1194 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1195 { 1196 uint32_t offset_blocks; 1197 1198 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1199 _dif_insert_copy(src_sgl, dst_sgl, offset_blocks, ctx); 1200 } 1201 } 1202 1203 static void 1204 _dif_insert_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1205 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1206 { 1207 uint32_t data_block_size; 1208 uint64_t guard = 0; 1209 struct spdk_dif dif = {}; 1210 1211 data_block_size = ctx->block_size - ctx->md_size; 1212 1213 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1214 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1215 data_block_size, ctx->dif_pi_format); 1216 guard = dif_generate_guard_split(guard, dst_sgl, data_block_size, 1217 ctx->guard_interval - data_block_size, ctx); 1218 } else { 1219 _data_copy_split(dst_sgl, src_sgl, data_block_size); 1220 _dif_sgl_advance(dst_sgl, ctx->guard_interval - data_block_size); 1221 } 1222 1223 _dif_generate(&dif, guard, offset_blocks, ctx); 1224 1225 dif_store_split(dst_sgl, &dif, ctx); 1226 } 1227 1228 static void 1229 dif_insert_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1230 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1231 { 1232 uint32_t offset_blocks; 1233 1234 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1235 _dif_insert_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 1236 } 1237 } 1238 1239 static void 1240 _dif_disable_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1241 const struct spdk_dif_ctx *ctx) 1242 { 1243 uint32_t offset = 0, src_len, dst_len, buf_len, data_block_size; 1244 uint8_t *src, *dst; 1245 1246 data_block_size = ctx->block_size - ctx->md_size; 1247 1248 while (offset < data_block_size) { 1249 _dif_sgl_get_buf(src_sgl, &src, &src_len); 1250 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 1251 buf_len = spdk_min(src_len, dst_len); 1252 buf_len = spdk_min(buf_len, data_block_size - offset); 1253 1254 memcpy(dst, src, buf_len); 1255 1256 _dif_sgl_advance(src_sgl, buf_len); 1257 _dif_sgl_advance(dst_sgl, buf_len); 1258 offset += buf_len; 1259 } 1260 1261 _dif_sgl_advance(dst_sgl, ctx->md_size); 1262 } 1263 1264 static void 1265 dif_disable_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1266 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1267 { 1268 uint32_t offset_blocks; 1269 1270 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1271 _dif_disable_insert_copy(src_sgl, dst_sgl, ctx); 1272 } 1273 } 1274 1275 static int 1276 _spdk_dif_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1277 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1278 { 1279 uint32_t data_block_size; 1280 1281 data_block_size = ctx->block_size - ctx->md_size; 1282 1283 if (!_dif_sgl_is_valid(src_sgl, data_block_size * num_blocks) || 1284 !_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks)) { 1285 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 1286 return -EINVAL; 1287 } 1288 1289 if (_dif_is_disabled(ctx->dif_type)) { 1290 dif_disable_insert_copy(src_sgl, dst_sgl, num_blocks, ctx); 1291 return 0; 1292 } 1293 1294 if (_dif_sgl_is_bytes_multiple(src_sgl, data_block_size) && 1295 _dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size)) { 1296 dif_insert_copy(src_sgl, dst_sgl, num_blocks, ctx); 1297 } else { 1298 dif_insert_copy_split(src_sgl, dst_sgl, num_blocks, ctx); 1299 } 1300 1301 return 0; 1302 } 1303 1304 static void 1305 _dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1306 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1307 { 1308 uint8_t *src, *dst; 1309 uint64_t guard = 0; 1310 1311 _dif_sgl_get_buf(src_sgl, &src, NULL); 1312 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1313 1314 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1315 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, ctx->guard_interval, 1316 ctx->dif_pi_format); 1317 } else { 1318 memcpy(dst, src, ctx->guard_interval); 1319 } 1320 1321 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 1322 1323 _dif_sgl_advance(src_sgl, ctx->block_size); 1324 _dif_sgl_advance(dst_sgl, ctx->block_size); 1325 } 1326 1327 static void 1328 dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1329 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1330 { 1331 uint32_t offset_blocks; 1332 1333 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1334 _dif_overwrite_copy(src_sgl, dst_sgl, offset_blocks, ctx); 1335 } 1336 } 1337 1338 static void 1339 _dif_overwrite_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1340 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1341 { 1342 uint64_t guard = 0; 1343 struct spdk_dif dif = {}; 1344 1345 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1346 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1347 ctx->guard_interval, ctx->dif_pi_format); 1348 } else { 1349 _data_copy_split(dst_sgl, src_sgl, ctx->guard_interval); 1350 } 1351 1352 _dif_sgl_advance(src_sgl, ctx->block_size - ctx->guard_interval); 1353 1354 _dif_generate(&dif, guard, offset_blocks, ctx); 1355 dif_store_split(dst_sgl, &dif, ctx); 1356 } 1357 1358 static void 1359 dif_overwrite_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1360 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1361 { 1362 uint32_t offset_blocks; 1363 1364 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1365 _dif_overwrite_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 1366 } 1367 } 1368 1369 static void 1370 dif_disable_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1371 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1372 { 1373 _data_copy_split(dst_sgl, src_sgl, ctx->block_size * num_blocks); 1374 } 1375 1376 static int 1377 _spdk_dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1378 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1379 { 1380 if (!_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks) || 1381 !_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks)) { 1382 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 1383 return -EINVAL; 1384 } 1385 1386 if (_dif_is_disabled(ctx->dif_type)) { 1387 dif_disable_copy(src_sgl, dst_sgl, num_blocks, ctx); 1388 return 0; 1389 } 1390 1391 if (_dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size) && 1392 _dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size)) { 1393 dif_overwrite_copy(src_sgl, dst_sgl, num_blocks, ctx); 1394 } else { 1395 dif_overwrite_copy_split(src_sgl, dst_sgl, num_blocks, ctx); 1396 } 1397 1398 return 0; 1399 } 1400 1401 int 1402 spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1403 int bounce_iovcnt, uint32_t num_blocks, 1404 const struct spdk_dif_ctx *ctx) 1405 { 1406 struct _dif_sgl src_sgl, dst_sgl; 1407 1408 _dif_sgl_init(&src_sgl, iovs, iovcnt); 1409 _dif_sgl_init(&dst_sgl, bounce_iovs, bounce_iovcnt); 1410 1411 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_NVME_PRACT) || 1412 ctx->md_size == _dif_size(ctx->dif_pi_format)) { 1413 return _spdk_dif_insert_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 1414 } else { 1415 return _spdk_dif_overwrite_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 1416 } 1417 } 1418 1419 static int 1420 _dif_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1421 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1422 struct spdk_dif_error *err_blk) 1423 { 1424 uint32_t data_block_size; 1425 uint8_t *src, *dst; 1426 int rc; 1427 uint64_t guard = 0; 1428 1429 data_block_size = ctx->block_size - ctx->md_size; 1430 1431 _dif_sgl_get_buf(src_sgl, &src, NULL); 1432 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1433 1434 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1435 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1436 ctx->dif_pi_format); 1437 guard = _dif_generate_guard(guard, src + data_block_size, 1438 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1439 } else { 1440 memcpy(dst, src, data_block_size); 1441 } 1442 1443 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1444 if (rc != 0) { 1445 return rc; 1446 } 1447 1448 _dif_sgl_advance(src_sgl, ctx->block_size); 1449 _dif_sgl_advance(dst_sgl, data_block_size); 1450 1451 return 0; 1452 } 1453 1454 static int 1455 dif_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1456 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1457 struct spdk_dif_error *err_blk) 1458 { 1459 uint32_t offset_blocks; 1460 int rc; 1461 1462 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1463 rc = _dif_strip_copy(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1464 if (rc != 0) { 1465 return rc; 1466 } 1467 } 1468 1469 return 0; 1470 } 1471 1472 static int 1473 _dif_strip_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1474 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1475 struct spdk_dif_error *err_blk) 1476 { 1477 uint32_t data_block_size; 1478 uint64_t guard = 0; 1479 struct spdk_dif dif = {}; 1480 1481 data_block_size = ctx->block_size - ctx->md_size; 1482 1483 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1484 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1485 data_block_size, ctx->dif_pi_format); 1486 guard = dif_generate_guard_split(guard, src_sgl, data_block_size, 1487 ctx->guard_interval - data_block_size, ctx); 1488 } else { 1489 _data_copy_split(dst_sgl, src_sgl, data_block_size); 1490 _dif_sgl_advance(src_sgl, ctx->guard_interval - data_block_size); 1491 } 1492 1493 dif_load_split(src_sgl, &dif, ctx); 1494 1495 return _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 1496 } 1497 1498 static int 1499 dif_strip_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1500 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1501 struct spdk_dif_error *err_blk) 1502 { 1503 uint32_t offset_blocks; 1504 int rc; 1505 1506 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1507 rc = _dif_strip_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1508 if (rc != 0) { 1509 return rc; 1510 } 1511 } 1512 1513 return 0; 1514 } 1515 1516 static void 1517 _dif_disable_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1518 const struct spdk_dif_ctx *ctx) 1519 { 1520 uint32_t offset = 0, src_len, dst_len, buf_len, data_block_size; 1521 uint8_t *src, *dst; 1522 1523 data_block_size = ctx->block_size - ctx->md_size; 1524 1525 while (offset < data_block_size) { 1526 _dif_sgl_get_buf(src_sgl, &src, &src_len); 1527 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 1528 buf_len = spdk_min(src_len, dst_len); 1529 buf_len = spdk_min(buf_len, data_block_size - offset); 1530 1531 memcpy(dst, src, buf_len); 1532 1533 _dif_sgl_advance(src_sgl, buf_len); 1534 _dif_sgl_advance(dst_sgl, buf_len); 1535 offset += buf_len; 1536 } 1537 1538 _dif_sgl_advance(src_sgl, ctx->md_size); 1539 } 1540 1541 static void 1542 dif_disable_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1543 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1544 { 1545 uint32_t offset_blocks; 1546 1547 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1548 _dif_disable_strip_copy(src_sgl, dst_sgl, ctx); 1549 } 1550 } 1551 1552 static int 1553 _spdk_dif_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1554 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1555 struct spdk_dif_error *err_blk) 1556 { 1557 uint32_t data_block_size; 1558 1559 data_block_size = ctx->block_size - ctx->md_size; 1560 1561 if (!_dif_sgl_is_valid(dst_sgl, data_block_size * num_blocks) || 1562 !_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks)) { 1563 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1564 return -EINVAL; 1565 } 1566 1567 if (_dif_is_disabled(ctx->dif_type)) { 1568 dif_disable_strip_copy(src_sgl, dst_sgl, num_blocks, ctx); 1569 return 0; 1570 } 1571 1572 if (_dif_sgl_is_bytes_multiple(dst_sgl, data_block_size) && 1573 _dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size)) { 1574 return dif_strip_copy(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1575 } else { 1576 return dif_strip_copy_split(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1577 } 1578 } 1579 1580 static int 1581 _dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1582 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1583 struct spdk_dif_error *err_blk) 1584 { 1585 uint8_t *src, *dst; 1586 int rc; 1587 uint64_t guard = 0; 1588 1589 _dif_sgl_get_buf(src_sgl, &src, NULL); 1590 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1591 1592 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1593 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, ctx->guard_interval, 1594 ctx->dif_pi_format); 1595 } else { 1596 memcpy(dst, src, ctx->guard_interval); 1597 } 1598 1599 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1600 if (rc != 0) { 1601 return rc; 1602 } 1603 1604 _dif_sgl_advance(src_sgl, ctx->block_size); 1605 _dif_sgl_advance(dst_sgl, ctx->block_size); 1606 1607 return 0; 1608 } 1609 1610 static int 1611 dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1612 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1613 struct spdk_dif_error *err_blk) 1614 { 1615 uint32_t offset_blocks; 1616 int rc; 1617 1618 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1619 rc = _dif_verify_copy(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1620 if (rc != 0) { 1621 return rc; 1622 } 1623 } 1624 1625 return 0; 1626 } 1627 1628 static int 1629 _dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1630 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1631 struct spdk_dif_error *err_blk) 1632 { 1633 uint64_t guard = 0; 1634 struct spdk_dif dif = {}; 1635 1636 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1637 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1638 ctx->guard_interval, ctx->dif_pi_format); 1639 } else { 1640 _data_copy_split(dst_sgl, src_sgl, ctx->guard_interval); 1641 } 1642 1643 dif_load_split(src_sgl, &dif, ctx); 1644 _dif_sgl_advance(dst_sgl, ctx->block_size - ctx->guard_interval); 1645 1646 return _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 1647 } 1648 1649 static int 1650 dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1651 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1652 struct spdk_dif_error *err_blk) 1653 { 1654 uint32_t offset_blocks; 1655 int rc; 1656 1657 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1658 rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1659 if (rc != 0) { 1660 return rc; 1661 } 1662 } 1663 1664 return 0; 1665 } 1666 1667 static int 1668 _spdk_dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1669 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1670 struct spdk_dif_error *err_blk) 1671 { 1672 if (!_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks) || 1673 !_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks)) { 1674 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1675 return -EINVAL; 1676 } 1677 1678 if (_dif_is_disabled(ctx->dif_type)) { 1679 dif_disable_copy(src_sgl, dst_sgl, num_blocks, ctx); 1680 return 0; 1681 } 1682 1683 if (_dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size) && 1684 _dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size)) { 1685 return dif_verify_copy(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1686 } else { 1687 return dif_verify_copy_split(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1688 } 1689 } 1690 1691 int 1692 spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1693 int bounce_iovcnt, uint32_t num_blocks, 1694 const struct spdk_dif_ctx *ctx, 1695 struct spdk_dif_error *err_blk) 1696 { 1697 struct _dif_sgl src_sgl, dst_sgl; 1698 1699 _dif_sgl_init(&src_sgl, bounce_iovs, bounce_iovcnt); 1700 _dif_sgl_init(&dst_sgl, iovs, iovcnt); 1701 1702 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_NVME_PRACT) || 1703 ctx->md_size == _dif_size(ctx->dif_pi_format)) { 1704 return _spdk_dif_strip_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1705 } else { 1706 return _spdk_dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1707 } 1708 } 1709 1710 static void 1711 _bit_flip(uint8_t *buf, uint32_t flip_bit) 1712 { 1713 uint8_t byte; 1714 1715 byte = *buf; 1716 byte ^= 1 << flip_bit; 1717 *buf = byte; 1718 } 1719 1720 static int 1721 _dif_inject_error(struct _dif_sgl *sgl, 1722 uint32_t block_size, uint32_t num_blocks, 1723 uint32_t inject_offset_blocks, 1724 uint32_t inject_offset_bytes, 1725 uint32_t inject_offset_bits) 1726 { 1727 uint32_t offset_in_block, buf_len; 1728 uint8_t *buf; 1729 1730 _dif_sgl_advance(sgl, block_size * inject_offset_blocks); 1731 1732 offset_in_block = 0; 1733 1734 while (offset_in_block < block_size) { 1735 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1736 buf_len = spdk_min(buf_len, block_size - offset_in_block); 1737 1738 if (inject_offset_bytes >= offset_in_block && 1739 inject_offset_bytes < offset_in_block + buf_len) { 1740 buf += inject_offset_bytes - offset_in_block; 1741 _bit_flip(buf, inject_offset_bits); 1742 return 0; 1743 } 1744 1745 _dif_sgl_advance(sgl, buf_len); 1746 offset_in_block += buf_len; 1747 } 1748 1749 return -1; 1750 } 1751 1752 static int 1753 dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks, 1754 uint32_t start_inject_bytes, uint32_t inject_range_bytes, 1755 uint32_t *inject_offset) 1756 { 1757 uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; 1758 uint32_t offset_blocks; 1759 int rc; 1760 1761 srand(time(0)); 1762 1763 inject_offset_blocks = rand() % num_blocks; 1764 inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); 1765 inject_offset_bits = rand() % 8; 1766 1767 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1768 if (offset_blocks == inject_offset_blocks) { 1769 rc = _dif_inject_error(sgl, block_size, num_blocks, 1770 inject_offset_blocks, 1771 inject_offset_bytes, 1772 inject_offset_bits); 1773 if (rc == 0) { 1774 *inject_offset = inject_offset_blocks; 1775 } 1776 return rc; 1777 } 1778 } 1779 1780 return -1; 1781 } 1782 1783 int 1784 spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1785 const struct spdk_dif_ctx *ctx, uint32_t inject_flags, 1786 uint32_t *inject_offset) 1787 { 1788 struct _dif_sgl sgl; 1789 int rc; 1790 1791 _dif_sgl_init(&sgl, iovs, iovcnt); 1792 1793 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1794 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1795 return -EINVAL; 1796 } 1797 1798 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1799 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1800 ctx->guard_interval + _dif_reftag_offset(ctx->dif_pi_format), 1801 _dif_reftag_size(ctx->dif_pi_format), 1802 inject_offset); 1803 if (rc != 0) { 1804 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1805 return rc; 1806 } 1807 } 1808 1809 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1810 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1811 ctx->guard_interval + _dif_apptag_offset(ctx->dif_pi_format), 1812 _dif_apptag_size(), 1813 inject_offset); 1814 if (rc != 0) { 1815 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1816 return rc; 1817 } 1818 } 1819 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1820 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1821 ctx->guard_interval, 1822 _dif_guard_size(ctx->dif_pi_format), 1823 inject_offset); 1824 if (rc != 0) { 1825 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1826 return rc; 1827 } 1828 } 1829 1830 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1831 /* If the DIF information is contained within the last 8/16 bytes of 1832 * metadata (depending on the PI format), then the CRC covers all metadata 1833 * bytes up to but excluding the last 8/16 bytes. But error injection does not 1834 * cover these metadata because classification is not determined yet. 1835 * 1836 * Note: Error injection to data block is expected to be detected as 1837 * guard error. 1838 */ 1839 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1840 0, 1841 ctx->block_size - ctx->md_size, 1842 inject_offset); 1843 if (rc != 0) { 1844 SPDK_ERRLOG("Failed to inject error to data block.\n"); 1845 return rc; 1846 } 1847 } 1848 1849 return 0; 1850 } 1851 1852 static void 1853 dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1854 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1855 { 1856 uint32_t offset_blocks = 0; 1857 uint8_t *data_buf, *md_buf; 1858 uint64_t guard; 1859 1860 while (offset_blocks < num_blocks) { 1861 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1862 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1863 1864 guard = 0; 1865 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1866 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1867 ctx->dif_pi_format); 1868 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1869 ctx->dif_pi_format); 1870 } 1871 1872 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1873 1874 _dif_sgl_advance(data_sgl, ctx->block_size); 1875 _dif_sgl_advance(md_sgl, ctx->md_size); 1876 offset_blocks++; 1877 } 1878 } 1879 1880 static void 1881 _dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1882 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1883 { 1884 uint32_t offset_in_block, data_buf_len; 1885 uint8_t *data_buf, *md_buf; 1886 uint64_t guard = 0; 1887 1888 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1889 1890 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1891 guard = ctx->guard_seed; 1892 } 1893 offset_in_block = 0; 1894 1895 while (offset_in_block < ctx->block_size) { 1896 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1897 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1898 1899 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1900 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 1901 ctx->dif_pi_format); 1902 } 1903 1904 _dif_sgl_advance(data_sgl, data_buf_len); 1905 offset_in_block += data_buf_len; 1906 } 1907 1908 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1909 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1910 ctx->dif_pi_format); 1911 } 1912 1913 _dif_sgl_advance(md_sgl, ctx->md_size); 1914 1915 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1916 } 1917 1918 static void 1919 dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1920 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1921 { 1922 uint32_t offset_blocks; 1923 1924 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1925 _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx); 1926 } 1927 } 1928 1929 int 1930 spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1931 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1932 { 1933 struct _dif_sgl data_sgl, md_sgl; 1934 1935 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1936 _dif_sgl_init(&md_sgl, md_iov, 1); 1937 1938 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1939 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1940 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1941 return -EINVAL; 1942 } 1943 1944 if (_dif_is_disabled(ctx->dif_type)) { 1945 return 0; 1946 } 1947 1948 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1949 dix_generate(&data_sgl, &md_sgl, num_blocks, ctx); 1950 } else { 1951 dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx); 1952 } 1953 1954 return 0; 1955 } 1956 1957 static int 1958 dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1959 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1960 struct spdk_dif_error *err_blk) 1961 { 1962 uint32_t offset_blocks = 0; 1963 uint8_t *data_buf, *md_buf; 1964 uint64_t guard; 1965 int rc; 1966 1967 while (offset_blocks < num_blocks) { 1968 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1969 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1970 1971 guard = 0; 1972 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1973 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1974 ctx->dif_pi_format); 1975 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1976 ctx->dif_pi_format); 1977 } 1978 1979 rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1980 if (rc != 0) { 1981 return rc; 1982 } 1983 1984 _dif_sgl_advance(data_sgl, ctx->block_size); 1985 _dif_sgl_advance(md_sgl, ctx->md_size); 1986 offset_blocks++; 1987 } 1988 1989 return 0; 1990 } 1991 1992 static int 1993 _dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1994 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1995 struct spdk_dif_error *err_blk) 1996 { 1997 uint32_t offset_in_block, data_buf_len; 1998 uint8_t *data_buf, *md_buf; 1999 uint64_t guard = 0; 2000 2001 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 2002 2003 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2004 guard = ctx->guard_seed; 2005 } 2006 offset_in_block = 0; 2007 2008 while (offset_in_block < ctx->block_size) { 2009 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 2010 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 2011 2012 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2013 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 2014 ctx->dif_pi_format); 2015 } 2016 2017 _dif_sgl_advance(data_sgl, data_buf_len); 2018 offset_in_block += data_buf_len; 2019 } 2020 2021 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2022 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 2023 ctx->dif_pi_format); 2024 } 2025 2026 _dif_sgl_advance(md_sgl, ctx->md_size); 2027 2028 return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 2029 } 2030 2031 static int 2032 dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 2033 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 2034 struct spdk_dif_error *err_blk) 2035 { 2036 uint32_t offset_blocks; 2037 int rc; 2038 2039 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2040 rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk); 2041 if (rc != 0) { 2042 return rc; 2043 } 2044 } 2045 2046 return 0; 2047 } 2048 2049 int 2050 spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 2051 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 2052 struct spdk_dif_error *err_blk) 2053 { 2054 struct _dif_sgl data_sgl, md_sgl; 2055 2056 if (md_iov->iov_base == NULL) { 2057 SPDK_ERRLOG("Metadata buffer is NULL.\n"); 2058 return -EINVAL; 2059 } 2060 2061 _dif_sgl_init(&data_sgl, iovs, iovcnt); 2062 _dif_sgl_init(&md_sgl, md_iov, 1); 2063 2064 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 2065 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2066 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2067 return -EINVAL; 2068 } 2069 2070 if (_dif_is_disabled(ctx->dif_type)) { 2071 return 0; 2072 } 2073 2074 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 2075 return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 2076 } else { 2077 return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 2078 } 2079 } 2080 2081 int 2082 spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 2083 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 2084 uint32_t inject_flags, uint32_t *inject_offset) 2085 { 2086 struct _dif_sgl data_sgl, md_sgl; 2087 int rc; 2088 2089 _dif_sgl_init(&data_sgl, iovs, iovcnt); 2090 _dif_sgl_init(&md_sgl, md_iov, 1); 2091 2092 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 2093 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2094 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2095 return -EINVAL; 2096 } 2097 2098 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 2099 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 2100 ctx->guard_interval + _dif_reftag_offset(ctx->dif_pi_format), 2101 _dif_reftag_size(ctx->dif_pi_format), 2102 inject_offset); 2103 if (rc != 0) { 2104 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 2105 return rc; 2106 } 2107 } 2108 2109 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 2110 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 2111 ctx->guard_interval + _dif_apptag_offset(ctx->dif_pi_format), 2112 _dif_apptag_size(), 2113 inject_offset); 2114 if (rc != 0) { 2115 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 2116 return rc; 2117 } 2118 } 2119 2120 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 2121 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 2122 ctx->guard_interval, 2123 _dif_guard_size(ctx->dif_pi_format), 2124 inject_offset); 2125 if (rc != 0) { 2126 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 2127 return rc; 2128 } 2129 } 2130 2131 if (inject_flags & SPDK_DIF_DATA_ERROR) { 2132 /* Note: Error injection to data block is expected to be detected 2133 * as guard error. 2134 */ 2135 rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks, 2136 0, 2137 ctx->block_size, 2138 inject_offset); 2139 if (rc != 0) { 2140 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 2141 return rc; 2142 } 2143 } 2144 2145 return 0; 2146 } 2147 2148 static uint32_t 2149 _to_next_boundary(uint32_t offset, uint32_t boundary) 2150 { 2151 return boundary - (offset % boundary); 2152 } 2153 2154 static uint32_t 2155 _to_size_with_md(uint32_t size, uint32_t data_block_size, uint32_t block_size) 2156 { 2157 return (size / data_block_size) * block_size + (size % data_block_size); 2158 } 2159 2160 int 2161 spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt, 2162 struct iovec *buf_iovs, int buf_iovcnt, 2163 uint32_t data_offset, uint32_t data_len, 2164 uint32_t *_mapped_len, 2165 const struct spdk_dif_ctx *ctx) 2166 { 2167 uint32_t data_block_size, data_unalign, buf_len, buf_offset, len; 2168 struct _dif_sgl dif_sgl; 2169 struct _dif_sgl buf_sgl; 2170 2171 if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) { 2172 return -EINVAL; 2173 } 2174 2175 data_block_size = ctx->block_size - ctx->md_size; 2176 2177 data_unalign = ctx->data_offset % data_block_size; 2178 2179 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 2180 ctx->block_size); 2181 buf_len -= data_unalign; 2182 2183 _dif_sgl_init(&dif_sgl, iovs, iovcnt); 2184 _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt); 2185 2186 if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) { 2187 SPDK_ERRLOG("Buffer overflow will occur.\n"); 2188 return -ERANGE; 2189 } 2190 2191 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 2192 buf_offset -= data_unalign; 2193 2194 _dif_sgl_advance(&buf_sgl, buf_offset); 2195 2196 while (data_len != 0) { 2197 len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size)); 2198 if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) { 2199 break; 2200 } 2201 _dif_sgl_advance(&buf_sgl, ctx->md_size); 2202 data_offset += len; 2203 data_len -= len; 2204 } 2205 2206 if (_mapped_len != NULL) { 2207 *_mapped_len = dif_sgl.total_size; 2208 } 2209 2210 return iovcnt - dif_sgl.iovcnt; 2211 } 2212 2213 static int 2214 _dif_sgl_setup_stream(struct _dif_sgl *sgl, uint32_t *_buf_offset, uint32_t *_buf_len, 2215 uint32_t data_offset, uint32_t data_len, 2216 const struct spdk_dif_ctx *ctx) 2217 { 2218 uint32_t data_block_size, data_unalign, buf_len, buf_offset; 2219 2220 data_block_size = ctx->block_size - ctx->md_size; 2221 2222 data_unalign = ctx->data_offset % data_block_size; 2223 2224 /* If the last data block is complete, DIF of the data block is 2225 * inserted or verified in this turn. 2226 */ 2227 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 2228 ctx->block_size); 2229 buf_len -= data_unalign; 2230 2231 if (!_dif_sgl_is_valid(sgl, buf_len)) { 2232 return -ERANGE; 2233 } 2234 2235 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 2236 buf_offset -= data_unalign; 2237 2238 _dif_sgl_advance(sgl, buf_offset); 2239 buf_len -= buf_offset; 2240 2241 buf_offset += data_unalign; 2242 2243 *_buf_offset = buf_offset; 2244 *_buf_len = buf_len; 2245 2246 return 0; 2247 } 2248 2249 int 2250 spdk_dif_generate_stream(struct iovec *iovs, int iovcnt, 2251 uint32_t data_offset, uint32_t data_len, 2252 struct spdk_dif_ctx *ctx) 2253 { 2254 uint32_t buf_len = 0, buf_offset = 0; 2255 uint32_t len, offset_in_block, offset_blocks; 2256 uint64_t guard = 0; 2257 struct _dif_sgl sgl; 2258 int rc; 2259 2260 if (iovs == NULL || iovcnt == 0) { 2261 return -EINVAL; 2262 } 2263 2264 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2265 guard = ctx->last_guard; 2266 } 2267 2268 _dif_sgl_init(&sgl, iovs, iovcnt); 2269 2270 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 2271 if (rc != 0) { 2272 return rc; 2273 } 2274 2275 while (buf_len != 0) { 2276 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 2277 offset_in_block = buf_offset % ctx->block_size; 2278 offset_blocks = buf_offset / ctx->block_size; 2279 2280 guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx); 2281 2282 buf_len -= len; 2283 buf_offset += len; 2284 } 2285 2286 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2287 ctx->last_guard = guard; 2288 } 2289 2290 return 0; 2291 } 2292 2293 int 2294 spdk_dif_verify_stream(struct iovec *iovs, int iovcnt, 2295 uint32_t data_offset, uint32_t data_len, 2296 struct spdk_dif_ctx *ctx, 2297 struct spdk_dif_error *err_blk) 2298 { 2299 uint32_t buf_len = 0, buf_offset = 0; 2300 uint32_t len, offset_in_block, offset_blocks; 2301 uint64_t guard = 0; 2302 struct _dif_sgl sgl; 2303 int rc = 0; 2304 2305 if (iovs == NULL || iovcnt == 0) { 2306 return -EINVAL; 2307 } 2308 2309 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2310 guard = ctx->last_guard; 2311 } 2312 2313 _dif_sgl_init(&sgl, iovs, iovcnt); 2314 2315 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 2316 if (rc != 0) { 2317 return rc; 2318 } 2319 2320 while (buf_len != 0) { 2321 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 2322 offset_in_block = buf_offset % ctx->block_size; 2323 offset_blocks = buf_offset / ctx->block_size; 2324 2325 rc = _dif_verify_split(&sgl, offset_in_block, len, &guard, offset_blocks, 2326 ctx, err_blk); 2327 if (rc != 0) { 2328 goto error; 2329 } 2330 2331 buf_len -= len; 2332 buf_offset += len; 2333 } 2334 2335 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2336 ctx->last_guard = guard; 2337 } 2338 error: 2339 return rc; 2340 } 2341 2342 int 2343 spdk_dif_update_crc32c_stream(struct iovec *iovs, int iovcnt, 2344 uint32_t data_offset, uint32_t data_len, 2345 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 2346 { 2347 uint32_t buf_len = 0, buf_offset = 0, len, offset_in_block; 2348 uint32_t crc32c; 2349 struct _dif_sgl sgl; 2350 int rc; 2351 2352 if (iovs == NULL || iovcnt == 0) { 2353 return -EINVAL; 2354 } 2355 2356 crc32c = *_crc32c; 2357 _dif_sgl_init(&sgl, iovs, iovcnt); 2358 2359 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 2360 if (rc != 0) { 2361 return rc; 2362 } 2363 2364 while (buf_len != 0) { 2365 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 2366 offset_in_block = buf_offset % ctx->block_size; 2367 2368 crc32c = _dif_update_crc32c_split(&sgl, offset_in_block, len, crc32c, ctx); 2369 2370 buf_len -= len; 2371 buf_offset += len; 2372 } 2373 2374 *_crc32c = crc32c; 2375 2376 return 0; 2377 } 2378 2379 void 2380 spdk_dif_get_range_with_md(uint32_t data_offset, uint32_t data_len, 2381 uint32_t *_buf_offset, uint32_t *_buf_len, 2382 const struct spdk_dif_ctx *ctx) 2383 { 2384 uint32_t data_block_size, data_unalign, buf_offset, buf_len; 2385 2386 if (!ctx->md_interleave) { 2387 buf_offset = data_offset; 2388 buf_len = data_len; 2389 } else { 2390 data_block_size = ctx->block_size - ctx->md_size; 2391 2392 data_unalign = data_offset % data_block_size; 2393 2394 buf_offset = _to_size_with_md(data_offset, data_block_size, ctx->block_size); 2395 buf_len = _to_size_with_md(data_unalign + data_len, data_block_size, ctx->block_size) - 2396 data_unalign; 2397 } 2398 2399 if (_buf_offset != NULL) { 2400 *_buf_offset = buf_offset; 2401 } 2402 2403 if (_buf_len != NULL) { 2404 *_buf_len = buf_len; 2405 } 2406 } 2407 2408 uint32_t 2409 spdk_dif_get_length_with_md(uint32_t data_len, const struct spdk_dif_ctx *ctx) 2410 { 2411 uint32_t data_block_size; 2412 2413 if (!ctx->md_interleave) { 2414 return data_len; 2415 } else { 2416 data_block_size = ctx->block_size - ctx->md_size; 2417 2418 return _to_size_with_md(data_len, data_block_size, ctx->block_size); 2419 } 2420 } 2421 2422 static int 2423 _dif_remap_ref_tag(struct _dif_sgl *sgl, uint32_t offset_blocks, 2424 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2425 bool check_ref_tag) 2426 { 2427 uint32_t offset, buf_len; 2428 uint64_t expected = 0, remapped; 2429 uint8_t *buf; 2430 struct _dif_sgl tmp_sgl; 2431 struct spdk_dif dif; 2432 2433 /* Fast forward to DIF field. */ 2434 _dif_sgl_advance(sgl, ctx->guard_interval); 2435 _dif_sgl_copy(&tmp_sgl, sgl); 2436 2437 /* Copy the split DIF field to the temporary DIF buffer */ 2438 offset = 0; 2439 while (offset < _dif_size(ctx->dif_pi_format)) { 2440 _dif_sgl_get_buf(sgl, &buf, &buf_len); 2441 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 2442 2443 memcpy((uint8_t *)&dif + offset, buf, buf_len); 2444 2445 _dif_sgl_advance(sgl, buf_len); 2446 offset += buf_len; 2447 } 2448 2449 if (_dif_ignore(&dif, ctx)) { 2450 goto end; 2451 } 2452 2453 /* For type 1 and 2, the Reference Tag is incremented for each 2454 * subsequent logical block. For type 3, the Reference Tag 2455 * remains the same as the initial Reference Tag. 2456 */ 2457 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2458 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2459 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2460 } else { 2461 remapped = ctx->remapped_init_ref_tag; 2462 } 2463 2464 /* Verify the stored Reference Tag. */ 2465 if (check_ref_tag && !_dif_reftag_check(&dif, ctx, expected, offset_blocks, err_blk)) { 2466 return -1; 2467 } 2468 2469 /* Update the stored Reference Tag to the remapped one. */ 2470 _dif_set_reftag(&dif, remapped, ctx->dif_pi_format); 2471 2472 offset = 0; 2473 while (offset < _dif_size(ctx->dif_pi_format)) { 2474 _dif_sgl_get_buf(&tmp_sgl, &buf, &buf_len); 2475 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 2476 2477 memcpy(buf, (uint8_t *)&dif + offset, buf_len); 2478 2479 _dif_sgl_advance(&tmp_sgl, buf_len); 2480 offset += buf_len; 2481 } 2482 2483 end: 2484 _dif_sgl_advance(sgl, ctx->block_size - ctx->guard_interval - _dif_size(ctx->dif_pi_format)); 2485 2486 return 0; 2487 } 2488 2489 int 2490 spdk_dif_remap_ref_tag(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 2491 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2492 bool check_ref_tag) 2493 { 2494 struct _dif_sgl sgl; 2495 uint32_t offset_blocks; 2496 int rc; 2497 2498 _dif_sgl_init(&sgl, iovs, iovcnt); 2499 2500 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 2501 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2502 return -EINVAL; 2503 } 2504 2505 if (_dif_is_disabled(ctx->dif_type)) { 2506 return 0; 2507 } 2508 2509 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2510 return 0; 2511 } 2512 2513 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2514 rc = _dif_remap_ref_tag(&sgl, offset_blocks, ctx, err_blk, check_ref_tag); 2515 if (rc != 0) { 2516 return rc; 2517 } 2518 } 2519 2520 return 0; 2521 } 2522 2523 static int 2524 _dix_remap_ref_tag(struct _dif_sgl *md_sgl, uint32_t offset_blocks, 2525 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2526 bool check_ref_tag) 2527 { 2528 uint64_t expected = 0, remapped; 2529 uint8_t *md_buf; 2530 struct spdk_dif *dif; 2531 2532 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 2533 2534 dif = (struct spdk_dif *)(md_buf + ctx->guard_interval); 2535 2536 if (_dif_ignore(dif, ctx)) { 2537 goto end; 2538 } 2539 2540 /* For type 1 and 2, the Reference Tag is incremented for each 2541 * subsequent logical block. For type 3, the Reference Tag 2542 * remains the same as the initialReference Tag. 2543 */ 2544 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2545 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2546 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2547 } else { 2548 remapped = ctx->remapped_init_ref_tag; 2549 } 2550 2551 /* Verify the stored Reference Tag. */ 2552 if (check_ref_tag && !_dif_reftag_check(dif, ctx, expected, offset_blocks, err_blk)) { 2553 return -1; 2554 } 2555 2556 /* Update the stored Reference Tag to the remapped one. */ 2557 _dif_set_reftag(dif, remapped, ctx->dif_pi_format); 2558 2559 end: 2560 _dif_sgl_advance(md_sgl, ctx->md_size); 2561 2562 return 0; 2563 } 2564 2565 int 2566 spdk_dix_remap_ref_tag(struct iovec *md_iov, uint32_t num_blocks, 2567 const struct spdk_dif_ctx *ctx, 2568 struct spdk_dif_error *err_blk, 2569 bool check_ref_tag) 2570 { 2571 struct _dif_sgl md_sgl; 2572 uint32_t offset_blocks; 2573 int rc; 2574 2575 _dif_sgl_init(&md_sgl, md_iov, 1); 2576 2577 if (!_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2578 SPDK_ERRLOG("Size of metadata iovec array is not valid.\n"); 2579 return -EINVAL; 2580 } 2581 2582 if (_dif_is_disabled(ctx->dif_type)) { 2583 return 0; 2584 } 2585 2586 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2587 return 0; 2588 } 2589 2590 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2591 rc = _dix_remap_ref_tag(&md_sgl, offset_blocks, ctx, err_blk, check_ref_tag); 2592 if (rc != 0) { 2593 return rc; 2594 } 2595 } 2596 2597 return 0; 2598 } 2599