1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/dif.h" 7 #include "spdk/crc16.h" 8 #include "spdk/crc32.h" 9 #include "spdk/crc64.h" 10 #include "spdk/endian.h" 11 #include "spdk/log.h" 12 #include "spdk/util.h" 13 14 #define REFTAG_MASK_16 0x00000000FFFFFFFF 15 #define REFTAG_MASK_32 0xFFFFFFFFFFFFFFFF 16 #define REFTAG_MASK_64 0x0000FFFFFFFFFFFF 17 18 /* The variable size Storage Tag and Reference Tag is not supported yet, 19 * so the maximum size of the Reference Tag is assumed. 20 */ 21 struct spdk_dif { 22 union { 23 struct { 24 uint16_t guard; 25 uint16_t app_tag; 26 uint32_t stor_ref_space; 27 } g16; 28 struct { 29 uint32_t guard; 30 uint16_t app_tag; 31 uint16_t stor_ref_space_p1; 32 uint64_t stor_ref_space_p2; 33 } g32; 34 struct { 35 uint64_t guard; 36 uint16_t app_tag; 37 uint16_t stor_ref_space_p1; 38 uint32_t stor_ref_space_p2; 39 } g64; 40 }; 41 }; 42 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g16) == 8, "Incorrect size"); 43 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g32) == 16, "Incorrect size"); 44 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g64) == 16, "Incorrect size"); 45 46 /* Context to iterate or create a iovec array. 47 * Each sgl is either iterated or created at a time. 48 */ 49 struct _dif_sgl { 50 /* Current iovec in the iteration or creation */ 51 struct iovec *iov; 52 53 /* Remaining count of iovecs in the iteration or creation. */ 54 int iovcnt; 55 56 /* Current offset in the iovec */ 57 uint32_t iov_offset; 58 59 /* Size of the created iovec array in bytes */ 60 uint32_t total_size; 61 }; 62 63 static inline void 64 _dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt) 65 { 66 s->iov = iovs; 67 s->iovcnt = iovcnt; 68 s->iov_offset = 0; 69 s->total_size = 0; 70 } 71 72 static void 73 _dif_sgl_advance(struct _dif_sgl *s, uint32_t step) 74 { 75 s->iov_offset += step; 76 while (s->iovcnt != 0) { 77 if (s->iov_offset < s->iov->iov_len) { 78 break; 79 } 80 81 s->iov_offset -= s->iov->iov_len; 82 s->iov++; 83 s->iovcnt--; 84 } 85 } 86 87 static inline void 88 _dif_sgl_get_buf(struct _dif_sgl *s, uint8_t **_buf, uint32_t *_buf_len) 89 { 90 if (_buf != NULL) { 91 *_buf = (uint8_t *)s->iov->iov_base + s->iov_offset; 92 } 93 if (_buf_len != NULL) { 94 *_buf_len = s->iov->iov_len - s->iov_offset; 95 } 96 } 97 98 static inline bool 99 _dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len) 100 { 101 assert(s->iovcnt > 0); 102 s->iov->iov_base = data; 103 s->iov->iov_len = data_len; 104 s->total_size += data_len; 105 s->iov++; 106 s->iovcnt--; 107 108 if (s->iovcnt > 0) { 109 return true; 110 } else { 111 return false; 112 } 113 } 114 115 static inline bool 116 _dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len) 117 { 118 uint8_t *buf; 119 uint32_t buf_len; 120 121 while (data_len != 0) { 122 _dif_sgl_get_buf(src, &buf, &buf_len); 123 buf_len = spdk_min(buf_len, data_len); 124 125 if (!_dif_sgl_append(dst, buf, buf_len)) { 126 return false; 127 } 128 129 _dif_sgl_advance(src, buf_len); 130 data_len -= buf_len; 131 } 132 133 return true; 134 } 135 136 /* This function must be used before starting iteration. */ 137 static bool 138 _dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes) 139 { 140 int i; 141 142 for (i = 0; i < s->iovcnt; i++) { 143 if (s->iov[i].iov_len % bytes) { 144 return false; 145 } 146 } 147 148 return true; 149 } 150 151 /* This function must be used before starting iteration. */ 152 static bool 153 _dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes) 154 { 155 uint64_t total = 0; 156 int i; 157 158 for (i = 0; i < s->iovcnt; i++) { 159 total += s->iov[i].iov_len; 160 } 161 162 return total >= bytes; 163 } 164 165 static void 166 _dif_sgl_copy(struct _dif_sgl *to, struct _dif_sgl *from) 167 { 168 memcpy(to, from, sizeof(struct _dif_sgl)); 169 } 170 171 static bool 172 _dif_is_disabled(enum spdk_dif_type dif_type) 173 { 174 if (dif_type == SPDK_DIF_DISABLE) { 175 return true; 176 } else { 177 return false; 178 } 179 } 180 181 static inline size_t 182 _dif_size(enum spdk_dif_pi_format dif_pi_format) 183 { 184 uint8_t size; 185 186 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 187 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16); 188 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 189 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32); 190 } else { 191 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64); 192 } 193 194 return size; 195 } 196 197 static uint32_t 198 _get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave, 199 size_t dif_size) 200 { 201 if (!dif_loc) { 202 /* For metadata formats with more than 8/16 bytes (depending on 203 * the PI format), if the DIF is contained in the last 8/16 bytes 204 * of metadata, then the CRC covers all metadata up to but excluding 205 * these last 8/16 bytes. 206 */ 207 if (md_interleave) { 208 return block_size - dif_size; 209 } else { 210 return md_size - dif_size; 211 } 212 } else { 213 /* For metadata formats with more than 8/16 bytes (depending on 214 * the PI format), if the DIF is contained in the first 8/16 bytes 215 * of metadata, then the CRC does not cover any metadata. 216 */ 217 if (md_interleave) { 218 return block_size - md_size; 219 } else { 220 return 0; 221 } 222 } 223 } 224 225 static inline uint8_t 226 _dif_guard_size(enum spdk_dif_pi_format dif_pi_format) 227 { 228 uint8_t size; 229 230 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 231 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.guard); 232 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 233 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.guard); 234 } else { 235 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.guard); 236 } 237 238 return size; 239 } 240 241 static inline void 242 _dif_set_guard(struct spdk_dif *dif, uint64_t guard, enum spdk_dif_pi_format dif_pi_format) 243 { 244 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 245 to_be16(&(dif->g16.guard), (uint16_t)guard); 246 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 247 to_be32(&(dif->g32.guard), (uint32_t)guard); 248 } else { 249 to_be64(&(dif->g64.guard), guard); 250 } 251 } 252 253 static inline uint64_t 254 _dif_get_guard(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 255 { 256 uint64_t guard; 257 258 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 259 guard = (uint64_t)from_be16(&(dif->g16.guard)); 260 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 261 guard = (uint64_t)from_be32(&(dif->g32.guard)); 262 } else { 263 guard = from_be64(&(dif->g64.guard)); 264 } 265 266 return guard; 267 } 268 269 static inline uint64_t 270 _dif_generate_guard(uint64_t guard_seed, void *buf, size_t buf_len, 271 enum spdk_dif_pi_format dif_pi_format) 272 { 273 uint64_t guard; 274 275 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 276 guard = (uint64_t)spdk_crc16_t10dif((uint16_t)guard_seed, buf, buf_len); 277 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 278 guard = (uint64_t)spdk_crc32c_nvme(buf, buf_len, guard_seed); 279 } else { 280 guard = spdk_crc64_nvme(buf, buf_len, guard_seed); 281 } 282 283 return guard; 284 } 285 286 static uint64_t 287 dif_generate_guard_split(uint64_t guard_seed, struct _dif_sgl *sgl, uint32_t start, 288 uint32_t len, const struct spdk_dif_ctx *ctx) 289 { 290 uint64_t guard = guard_seed; 291 uint32_t offset, end, buf_len; 292 uint8_t *buf; 293 294 offset = start; 295 end = start + spdk_min(len, ctx->guard_interval - start); 296 297 while (offset < end) { 298 _dif_sgl_get_buf(sgl, &buf, &buf_len); 299 buf_len = spdk_min(buf_len, end - offset); 300 301 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 302 guard = _dif_generate_guard(guard, buf, buf_len, ctx->dif_pi_format); 303 } 304 305 _dif_sgl_advance(sgl, buf_len); 306 offset += buf_len; 307 } 308 309 return guard; 310 } 311 312 static inline uint64_t 313 _dif_generate_guard_copy(uint64_t guard_seed, void *dst, void *src, size_t buf_len, 314 enum spdk_dif_pi_format dif_pi_format) 315 { 316 uint64_t guard; 317 318 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 319 guard = (uint64_t)spdk_crc16_t10dif_copy((uint16_t)guard_seed, dst, src, buf_len); 320 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 321 memcpy(dst, src, buf_len); 322 guard = (uint64_t)spdk_crc32c_nvme(src, buf_len, guard_seed); 323 } else { 324 memcpy(dst, src, buf_len); 325 guard = spdk_crc64_nvme(src, buf_len, guard_seed); 326 } 327 328 return guard; 329 } 330 331 static uint64_t 332 _dif_generate_guard_copy_split(uint64_t guard, struct _dif_sgl *dst_sgl, 333 struct _dif_sgl *src_sgl, uint32_t data_len, 334 enum spdk_dif_pi_format dif_pi_format) 335 { 336 uint32_t offset = 0, src_len, dst_len, buf_len; 337 uint8_t *src, *dst; 338 339 while (offset < data_len) { 340 _dif_sgl_get_buf(src_sgl, &src, &src_len); 341 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 342 buf_len = spdk_min(src_len, dst_len); 343 buf_len = spdk_min(buf_len, data_len - offset); 344 345 guard = _dif_generate_guard_copy(guard, dst, src, buf_len, dif_pi_format); 346 347 _dif_sgl_advance(src_sgl, buf_len); 348 _dif_sgl_advance(dst_sgl, buf_len); 349 offset += buf_len; 350 } 351 352 return guard; 353 } 354 355 static void 356 _data_copy_split(struct _dif_sgl *dst_sgl, struct _dif_sgl *src_sgl, uint32_t data_len) 357 { 358 uint32_t offset = 0, src_len, dst_len, buf_len; 359 uint8_t *src, *dst; 360 361 while (offset < data_len) { 362 _dif_sgl_get_buf(src_sgl, &src, &src_len); 363 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 364 buf_len = spdk_min(src_len, dst_len); 365 buf_len = spdk_min(buf_len, data_len - offset); 366 367 memcpy(dst, src, buf_len); 368 369 _dif_sgl_advance(src_sgl, buf_len); 370 _dif_sgl_advance(dst_sgl, buf_len); 371 offset += buf_len; 372 } 373 } 374 375 static inline uint8_t 376 _dif_apptag_offset(enum spdk_dif_pi_format dif_pi_format) 377 { 378 return _dif_guard_size(dif_pi_format); 379 } 380 381 static inline uint8_t 382 _dif_apptag_size(void) 383 { 384 return SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.app_tag); 385 } 386 387 static inline void 388 _dif_set_apptag(struct spdk_dif *dif, uint16_t app_tag, enum spdk_dif_pi_format dif_pi_format) 389 { 390 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 391 to_be16(&(dif->g16.app_tag), app_tag); 392 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 393 to_be16(&(dif->g32.app_tag), app_tag); 394 } else { 395 to_be16(&(dif->g64.app_tag), app_tag); 396 } 397 } 398 399 static inline uint16_t 400 _dif_get_apptag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 401 { 402 uint16_t app_tag; 403 404 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 405 app_tag = from_be16(&(dif->g16.app_tag)); 406 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 407 app_tag = from_be16(&(dif->g32.app_tag)); 408 } else { 409 app_tag = from_be16(&(dif->g64.app_tag)); 410 } 411 412 return app_tag; 413 } 414 415 static inline bool 416 _dif_apptag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 417 { 418 return _dif_get_apptag(dif, dif_pi_format) == SPDK_DIF_APPTAG_IGNORE; 419 } 420 421 static inline uint8_t 422 _dif_reftag_offset(enum spdk_dif_pi_format dif_pi_format) 423 { 424 uint8_t offset; 425 426 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 427 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size(); 428 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 429 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size() 430 + SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.stor_ref_space_p1); 431 } else { 432 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size(); 433 } 434 435 return offset; 436 } 437 438 static inline uint8_t 439 _dif_reftag_size(enum spdk_dif_pi_format dif_pi_format) 440 { 441 uint8_t size; 442 443 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 444 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.stor_ref_space); 445 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 446 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.stor_ref_space_p2); 447 } else { 448 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.stor_ref_space_p1) + 449 SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.stor_ref_space_p2); 450 } 451 452 return size; 453 } 454 455 static inline void 456 _dif_set_reftag(struct spdk_dif *dif, uint64_t ref_tag, enum spdk_dif_pi_format dif_pi_format) 457 { 458 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 459 to_be32(&(dif->g16.stor_ref_space), (uint32_t)ref_tag); 460 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 461 to_be64(&(dif->g32.stor_ref_space_p2), ref_tag); 462 } else { 463 to_be16(&(dif->g64.stor_ref_space_p1), (uint16_t)(ref_tag >> 32)); 464 to_be32(&(dif->g64.stor_ref_space_p2), (uint32_t)ref_tag); 465 } 466 } 467 468 static inline uint64_t 469 _dif_get_reftag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 470 { 471 uint64_t ref_tag; 472 473 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 474 ref_tag = (uint64_t)from_be32(&(dif->g16.stor_ref_space)); 475 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 476 ref_tag = from_be64(&(dif->g32.stor_ref_space_p2)); 477 } else { 478 ref_tag = (uint64_t)from_be16(&(dif->g64.stor_ref_space_p1)); 479 ref_tag <<= 32; 480 ref_tag |= (uint64_t)from_be32(&(dif->g64.stor_ref_space_p2)); 481 } 482 483 return ref_tag; 484 } 485 486 static inline bool 487 _dif_reftag_match(struct spdk_dif *dif, uint64_t ref_tag, 488 enum spdk_dif_pi_format dif_pi_format) 489 { 490 uint64_t _ref_tag; 491 bool match; 492 493 _ref_tag = _dif_get_reftag(dif, dif_pi_format); 494 495 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 496 match = (_ref_tag == (ref_tag & REFTAG_MASK_16)); 497 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 498 match = (_ref_tag == ref_tag); 499 } else { 500 match = (_ref_tag == (ref_tag & REFTAG_MASK_64)); 501 } 502 503 return match; 504 } 505 506 static inline bool 507 _dif_reftag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 508 { 509 return _dif_reftag_match(dif, REFTAG_MASK_32, dif_pi_format); 510 } 511 512 static bool 513 _dif_ignore(struct spdk_dif *dif, const struct spdk_dif_ctx *ctx) 514 { 515 switch (ctx->dif_type) { 516 case SPDK_DIF_TYPE1: 517 case SPDK_DIF_TYPE2: 518 /* If Type 1 or 2 is used, then all DIF checks are disabled when 519 * the Application Tag is 0xFFFF. 520 */ 521 if (_dif_apptag_ignore(dif, ctx->dif_pi_format)) { 522 return true; 523 } 524 break; 525 case SPDK_DIF_TYPE3: 526 /* If Type 3 is used, then all DIF checks are disabled when the 527 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF 528 * or 0xFFFFFFFFFFFFFFFF depending on the PI format. 529 */ 530 531 if (_dif_apptag_ignore(dif, ctx->dif_pi_format) && 532 _dif_reftag_ignore(dif, ctx->dif_pi_format)) { 533 return true; 534 } 535 break; 536 default: 537 break; 538 } 539 540 return false; 541 } 542 543 static bool 544 _dif_pi_format_is_valid(enum spdk_dif_pi_format dif_pi_format) 545 { 546 switch (dif_pi_format) { 547 case SPDK_DIF_PI_FORMAT_16: 548 case SPDK_DIF_PI_FORMAT_32: 549 case SPDK_DIF_PI_FORMAT_64: 550 return true; 551 default: 552 return false; 553 } 554 } 555 556 static bool 557 _dif_type_is_valid(enum spdk_dif_type dif_type) 558 { 559 switch (dif_type) { 560 case SPDK_DIF_DISABLE: 561 case SPDK_DIF_TYPE1: 562 case SPDK_DIF_TYPE2: 563 case SPDK_DIF_TYPE3: 564 return true; 565 default: 566 return false; 567 } 568 } 569 570 int 571 spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, 572 bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, 573 uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag, 574 uint32_t data_offset, uint64_t guard_seed, struct spdk_dif_ctx_init_ext_opts *opts) 575 { 576 uint32_t data_block_size; 577 enum spdk_dif_pi_format dif_pi_format = SPDK_DIF_PI_FORMAT_16; 578 579 if (opts != NULL) { 580 if (!_dif_pi_format_is_valid(opts->dif_pi_format)) { 581 SPDK_ERRLOG("No valid DIF PI format provided.\n"); 582 return -EINVAL; 583 } 584 585 dif_pi_format = opts->dif_pi_format; 586 } 587 588 if (!_dif_type_is_valid(dif_type)) { 589 SPDK_ERRLOG("No valid DIF type was provided.\n"); 590 return -EINVAL; 591 } 592 593 if (md_size < _dif_size(dif_pi_format)) { 594 SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); 595 return -EINVAL; 596 } 597 598 if (md_interleave) { 599 if (block_size < md_size) { 600 SPDK_ERRLOG("Block size is smaller than DIF size.\n"); 601 return -EINVAL; 602 } 603 data_block_size = block_size - md_size; 604 } else { 605 data_block_size = block_size; 606 } 607 608 if (data_block_size == 0) { 609 SPDK_ERRLOG("Zero data block size is not allowed\n"); 610 return -EINVAL; 611 } 612 613 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 614 if ((data_block_size % 512) != 0) { 615 SPDK_ERRLOG("Data block size should be a multiple of 512B\n"); 616 return -EINVAL; 617 } 618 } else { 619 if ((data_block_size % 4096) != 0) { 620 SPDK_ERRLOG("Data block size should be a multiple of 4kB\n"); 621 return -EINVAL; 622 } 623 } 624 625 ctx->block_size = block_size; 626 ctx->md_size = md_size; 627 ctx->md_interleave = md_interleave; 628 ctx->dif_pi_format = dif_pi_format; 629 ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave, 630 _dif_size(ctx->dif_pi_format)); 631 ctx->dif_type = dif_type; 632 ctx->dif_flags = dif_flags; 633 ctx->init_ref_tag = init_ref_tag; 634 ctx->apptag_mask = apptag_mask; 635 ctx->app_tag = app_tag; 636 ctx->data_offset = data_offset; 637 ctx->ref_tag_offset = data_offset / data_block_size; 638 ctx->last_guard = guard_seed; 639 ctx->guard_seed = guard_seed; 640 ctx->remapped_init_ref_tag = 0; 641 642 return 0; 643 } 644 645 void 646 spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset) 647 { 648 uint32_t data_block_size; 649 650 if (ctx->md_interleave) { 651 data_block_size = ctx->block_size - ctx->md_size; 652 } else { 653 data_block_size = ctx->block_size; 654 } 655 656 ctx->data_offset = data_offset; 657 ctx->ref_tag_offset = data_offset / data_block_size; 658 } 659 660 void 661 spdk_dif_ctx_set_remapped_init_ref_tag(struct spdk_dif_ctx *ctx, 662 uint32_t remapped_init_ref_tag) 663 { 664 ctx->remapped_init_ref_tag = remapped_init_ref_tag; 665 } 666 667 static void 668 _dif_generate(void *_dif, uint64_t guard, uint32_t offset_blocks, 669 const struct spdk_dif_ctx *ctx) 670 { 671 struct spdk_dif *dif = _dif; 672 uint64_t ref_tag; 673 674 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 675 _dif_set_guard(dif, guard, ctx->dif_pi_format); 676 } 677 678 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 679 _dif_set_apptag(dif, ctx->app_tag, ctx->dif_pi_format); 680 } 681 682 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 683 /* For type 1 and 2, the reference tag is incremented for each 684 * subsequent logical block. For type 3, the reference tag 685 * remains the same as the initial reference tag. 686 */ 687 if (ctx->dif_type != SPDK_DIF_TYPE3) { 688 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 689 } else { 690 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 691 } 692 693 /* Overwrite reference tag if initialization reference tag is SPDK_DIF_REFTAG_IGNORE */ 694 if (ctx->init_ref_tag == SPDK_DIF_REFTAG_IGNORE) { 695 if (ctx->dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 696 ref_tag = REFTAG_MASK_16; 697 } else if (ctx->dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 698 ref_tag = REFTAG_MASK_32; 699 } else { 700 ref_tag = REFTAG_MASK_64; 701 } 702 } 703 704 _dif_set_reftag(dif, ref_tag, ctx->dif_pi_format); 705 } 706 } 707 708 static void 709 dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 710 { 711 uint32_t offset_blocks; 712 uint8_t *buf; 713 uint64_t guard = 0; 714 715 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 716 _dif_sgl_get_buf(sgl, &buf, NULL); 717 718 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 719 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 720 } 721 722 _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); 723 724 _dif_sgl_advance(sgl, ctx->block_size); 725 } 726 } 727 728 static void 729 dif_store_split(struct _dif_sgl *sgl, struct spdk_dif *dif, 730 const struct spdk_dif_ctx *ctx) 731 { 732 uint32_t offset = 0, rest_md_len, buf_len; 733 uint8_t *buf; 734 735 rest_md_len = ctx->block_size - ctx->guard_interval; 736 737 while (offset < rest_md_len) { 738 _dif_sgl_get_buf(sgl, &buf, &buf_len); 739 740 if (offset < _dif_size(ctx->dif_pi_format)) { 741 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 742 memcpy(buf, (uint8_t *)dif + offset, buf_len); 743 } else { 744 buf_len = spdk_min(buf_len, rest_md_len - offset); 745 } 746 747 _dif_sgl_advance(sgl, buf_len); 748 offset += buf_len; 749 } 750 } 751 752 static uint64_t 753 _dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 754 uint64_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 755 { 756 struct spdk_dif dif = {}; 757 758 assert(offset_in_block < ctx->guard_interval); 759 assert(offset_in_block + data_len < ctx->guard_interval || 760 offset_in_block + data_len == ctx->block_size); 761 762 /* Compute CRC over split logical block data. */ 763 guard = dif_generate_guard_split(guard, sgl, offset_in_block, data_len, ctx); 764 765 if (offset_in_block + data_len < ctx->guard_interval) { 766 return guard; 767 } 768 769 /* If a whole logical block data is parsed, generate DIF 770 * and save it to the temporary DIF area. 771 */ 772 _dif_generate(&dif, guard, offset_blocks, ctx); 773 774 /* Copy generated DIF field to the split DIF field, and then 775 * skip metadata field after DIF field (if any). 776 */ 777 dif_store_split(sgl, &dif, ctx); 778 779 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 780 guard = ctx->guard_seed; 781 } 782 783 return guard; 784 } 785 786 static void 787 dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks, 788 const struct spdk_dif_ctx *ctx) 789 { 790 uint32_t offset_blocks; 791 uint64_t guard = 0; 792 793 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 794 guard = ctx->guard_seed; 795 } 796 797 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 798 _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx); 799 } 800 } 801 802 int 803 spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 804 const struct spdk_dif_ctx *ctx) 805 { 806 struct _dif_sgl sgl; 807 808 _dif_sgl_init(&sgl, iovs, iovcnt); 809 810 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 811 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 812 return -EINVAL; 813 } 814 815 if (_dif_is_disabled(ctx->dif_type)) { 816 return 0; 817 } 818 819 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 820 dif_generate(&sgl, num_blocks, ctx); 821 } else { 822 dif_generate_split(&sgl, num_blocks, ctx); 823 } 824 825 return 0; 826 } 827 828 static void 829 _dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, 830 uint64_t expected, uint64_t actual, uint32_t err_offset) 831 { 832 if (err_blk) { 833 err_blk->err_type = err_type; 834 err_blk->expected = expected; 835 err_blk->actual = actual; 836 err_blk->err_offset = err_offset; 837 } 838 } 839 840 static bool 841 _dif_reftag_check(struct spdk_dif *dif, const struct spdk_dif_ctx *ctx, 842 uint64_t expected_reftag, uint32_t offset_blocks, struct spdk_dif_error *err_blk) 843 { 844 uint64_t reftag; 845 846 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 847 switch (ctx->dif_type) { 848 case SPDK_DIF_TYPE1: 849 case SPDK_DIF_TYPE2: 850 /* Compare the DIF Reference Tag field to the passed Reference Tag. 851 * The passed Reference Tag will be the least significant 4 bytes 852 * or 8 bytes (depending on the PI format) 853 * of the LBA when Type 1 is used, and application specific value 854 * if Type 2 is used. 855 */ 856 if (!_dif_reftag_match(dif, expected_reftag, ctx->dif_pi_format)) { 857 reftag = _dif_get_reftag(dif, ctx->dif_pi_format); 858 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected_reftag, 859 reftag, offset_blocks); 860 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu64 "," \ 861 " Expected=%lx, Actual=%lx\n", 862 expected_reftag, expected_reftag, reftag); 863 return false; 864 } 865 break; 866 case SPDK_DIF_TYPE3: 867 /* For Type 3, computed Reference Tag remains unchanged. 868 * Hence ignore the Reference Tag field. 869 */ 870 break; 871 default: 872 break; 873 } 874 } 875 876 return true; 877 } 878 879 static int 880 _dif_verify(void *_dif, uint64_t guard, uint32_t offset_blocks, 881 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 882 { 883 struct spdk_dif *dif = _dif; 884 uint64_t _guard; 885 uint16_t _app_tag; 886 uint64_t ref_tag; 887 888 if (_dif_ignore(dif, ctx)) { 889 return 0; 890 } 891 892 /* For type 1 and 2, the reference tag is incremented for each 893 * subsequent logical block. For type 3, the reference tag 894 * remains the same as the initial reference tag. 895 */ 896 if (ctx->dif_type != SPDK_DIF_TYPE3) { 897 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 898 } else { 899 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 900 } 901 902 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 903 /* Compare the DIF Guard field to the CRC computed over the logical 904 * block data. 905 */ 906 _guard = _dif_get_guard(dif, ctx->dif_pi_format); 907 if (_guard != guard) { 908 _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, 909 offset_blocks); 910 SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu64 "," \ 911 " Expected=%lx, Actual=%lx\n", 912 ref_tag, _guard, guard); 913 return -1; 914 } 915 } 916 917 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 918 /* Compare unmasked bits in the DIF Application Tag field to the 919 * passed Application Tag. 920 */ 921 _app_tag = _dif_get_apptag(dif, ctx->dif_pi_format); 922 if ((_app_tag & ctx->apptag_mask) != (ctx->app_tag & ctx->apptag_mask)) { 923 _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, 924 (_app_tag & ctx->apptag_mask), offset_blocks); 925 SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu64 "," \ 926 " Expected=%x, Actual=%x\n", 927 ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); 928 return -1; 929 } 930 } 931 932 if (!_dif_reftag_check(dif, ctx, ref_tag, offset_blocks, err_blk)) { 933 return -1; 934 } 935 936 return 0; 937 } 938 939 static int 940 dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks, 941 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 942 { 943 uint32_t offset_blocks; 944 int rc; 945 uint8_t *buf; 946 uint64_t guard = 0; 947 948 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 949 _dif_sgl_get_buf(sgl, &buf, NULL); 950 951 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 952 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 953 } 954 955 rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 956 if (rc != 0) { 957 return rc; 958 } 959 960 _dif_sgl_advance(sgl, ctx->block_size); 961 } 962 963 return 0; 964 } 965 966 static void 967 dif_load_split(struct _dif_sgl *sgl, struct spdk_dif *dif, 968 const struct spdk_dif_ctx *ctx) 969 { 970 uint32_t offset = 0, rest_md_len, buf_len; 971 uint8_t *buf; 972 973 rest_md_len = ctx->block_size - ctx->guard_interval; 974 975 while (offset < rest_md_len) { 976 _dif_sgl_get_buf(sgl, &buf, &buf_len); 977 978 if (offset < _dif_size(ctx->dif_pi_format)) { 979 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 980 memcpy((uint8_t *)dif + offset, buf, buf_len); 981 } else { 982 buf_len = spdk_min(buf_len, rest_md_len - offset); 983 } 984 985 _dif_sgl_advance(sgl, buf_len); 986 offset += buf_len; 987 } 988 } 989 990 static int 991 _dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 992 uint64_t *_guard, uint32_t offset_blocks, 993 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 994 { 995 uint64_t guard = *_guard; 996 struct spdk_dif dif = {}; 997 int rc; 998 999 assert(_guard != NULL); 1000 assert(offset_in_block < ctx->guard_interval); 1001 assert(offset_in_block + data_len < ctx->guard_interval || 1002 offset_in_block + data_len == ctx->block_size); 1003 1004 guard = dif_generate_guard_split(guard, sgl, offset_in_block, data_len, ctx); 1005 1006 if (offset_in_block + data_len < ctx->guard_interval) { 1007 *_guard = guard; 1008 return 0; 1009 } 1010 1011 dif_load_split(sgl, &dif, ctx); 1012 1013 rc = _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 1014 if (rc != 0) { 1015 return rc; 1016 } 1017 1018 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1019 guard = ctx->guard_seed; 1020 } 1021 1022 *_guard = guard; 1023 return 0; 1024 } 1025 1026 static int 1027 dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks, 1028 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 1029 { 1030 uint32_t offset_blocks; 1031 uint64_t guard = 0; 1032 int rc; 1033 1034 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1035 guard = ctx->guard_seed; 1036 } 1037 1038 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1039 rc = _dif_verify_split(sgl, 0, ctx->block_size, &guard, offset_blocks, 1040 ctx, err_blk); 1041 if (rc != 0) { 1042 return rc; 1043 } 1044 } 1045 1046 return 0; 1047 } 1048 1049 int 1050 spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1051 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 1052 { 1053 struct _dif_sgl sgl; 1054 1055 _dif_sgl_init(&sgl, iovs, iovcnt); 1056 1057 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1058 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1059 return -EINVAL; 1060 } 1061 1062 if (_dif_is_disabled(ctx->dif_type)) { 1063 return 0; 1064 } 1065 1066 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 1067 return dif_verify(&sgl, num_blocks, ctx, err_blk); 1068 } else { 1069 return dif_verify_split(&sgl, num_blocks, ctx, err_blk); 1070 } 1071 } 1072 1073 static uint32_t 1074 dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks, 1075 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1076 { 1077 uint32_t offset_blocks; 1078 uint8_t *buf; 1079 1080 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1081 _dif_sgl_get_buf(sgl, &buf, NULL); 1082 1083 crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c); 1084 1085 _dif_sgl_advance(sgl, ctx->block_size); 1086 } 1087 1088 return crc32c; 1089 } 1090 1091 static uint32_t 1092 _dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 1093 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1094 { 1095 uint32_t data_block_size, buf_len; 1096 uint8_t *buf; 1097 1098 data_block_size = ctx->block_size - ctx->md_size; 1099 1100 assert(offset_in_block + data_len <= ctx->block_size); 1101 1102 while (data_len != 0) { 1103 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1104 buf_len = spdk_min(buf_len, data_len); 1105 1106 if (offset_in_block < data_block_size) { 1107 buf_len = spdk_min(buf_len, data_block_size - offset_in_block); 1108 crc32c = spdk_crc32c_update(buf, buf_len, crc32c); 1109 } 1110 1111 _dif_sgl_advance(sgl, buf_len); 1112 offset_in_block += buf_len; 1113 data_len -= buf_len; 1114 } 1115 1116 return crc32c; 1117 } 1118 1119 static uint32_t 1120 dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks, 1121 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1122 { 1123 uint32_t offset_blocks; 1124 1125 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1126 crc32c = _dif_update_crc32c_split(sgl, 0, ctx->block_size, crc32c, ctx); 1127 } 1128 1129 return crc32c; 1130 } 1131 1132 int 1133 spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1134 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 1135 { 1136 struct _dif_sgl sgl; 1137 1138 if (_crc32c == NULL) { 1139 return -EINVAL; 1140 } 1141 1142 _dif_sgl_init(&sgl, iovs, iovcnt); 1143 1144 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1145 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1146 return -EINVAL; 1147 } 1148 1149 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 1150 *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx); 1151 } else { 1152 *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx); 1153 } 1154 1155 return 0; 1156 } 1157 1158 static void 1159 _dif_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1160 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1161 { 1162 uint32_t data_block_size; 1163 uint8_t *src, *dst; 1164 uint64_t guard = 0; 1165 1166 data_block_size = ctx->block_size - ctx->md_size; 1167 1168 _dif_sgl_get_buf(src_sgl, &src, NULL); 1169 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1170 1171 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1172 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1173 ctx->dif_pi_format); 1174 guard = _dif_generate_guard(guard, dst + data_block_size, 1175 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1176 } else { 1177 memcpy(dst, src, data_block_size); 1178 } 1179 1180 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 1181 1182 _dif_sgl_advance(src_sgl, data_block_size); 1183 _dif_sgl_advance(dst_sgl, ctx->block_size); 1184 } 1185 1186 static void 1187 dif_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1188 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1189 { 1190 uint32_t offset_blocks; 1191 1192 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1193 _dif_insert_copy(src_sgl, dst_sgl, offset_blocks, ctx); 1194 } 1195 } 1196 1197 static void 1198 _dif_insert_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1199 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1200 { 1201 uint32_t data_block_size; 1202 uint64_t guard = 0; 1203 struct spdk_dif dif = {}; 1204 1205 data_block_size = ctx->block_size - ctx->md_size; 1206 1207 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1208 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1209 data_block_size, ctx->dif_pi_format); 1210 guard = dif_generate_guard_split(guard, dst_sgl, data_block_size, 1211 ctx->guard_interval - data_block_size, ctx); 1212 } else { 1213 _data_copy_split(dst_sgl, src_sgl, data_block_size); 1214 _dif_sgl_advance(dst_sgl, ctx->guard_interval - data_block_size); 1215 } 1216 1217 _dif_generate(&dif, guard, offset_blocks, ctx); 1218 1219 dif_store_split(dst_sgl, &dif, ctx); 1220 } 1221 1222 static void 1223 dif_insert_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1224 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1225 { 1226 uint32_t offset_blocks; 1227 1228 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1229 _dif_insert_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 1230 } 1231 } 1232 1233 static void 1234 _dif_disable_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1235 const struct spdk_dif_ctx *ctx) 1236 { 1237 uint32_t offset = 0, src_len, dst_len, buf_len, data_block_size; 1238 uint8_t *src, *dst; 1239 1240 data_block_size = ctx->block_size - ctx->md_size; 1241 1242 while (offset < data_block_size) { 1243 _dif_sgl_get_buf(src_sgl, &src, &src_len); 1244 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 1245 buf_len = spdk_min(src_len, dst_len); 1246 buf_len = spdk_min(buf_len, data_block_size - offset); 1247 1248 memcpy(dst, src, buf_len); 1249 1250 _dif_sgl_advance(src_sgl, buf_len); 1251 _dif_sgl_advance(dst_sgl, buf_len); 1252 offset += buf_len; 1253 } 1254 1255 _dif_sgl_advance(dst_sgl, ctx->md_size); 1256 } 1257 1258 static void 1259 dif_disable_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1260 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1261 { 1262 uint32_t offset_blocks; 1263 1264 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1265 _dif_disable_insert_copy(src_sgl, dst_sgl, ctx); 1266 } 1267 } 1268 1269 static int 1270 _spdk_dif_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1271 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1272 { 1273 uint32_t data_block_size; 1274 1275 data_block_size = ctx->block_size - ctx->md_size; 1276 1277 if (!_dif_sgl_is_valid(src_sgl, data_block_size * num_blocks) || 1278 !_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks)) { 1279 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 1280 return -EINVAL; 1281 } 1282 1283 if (_dif_is_disabled(ctx->dif_type)) { 1284 dif_disable_insert_copy(src_sgl, dst_sgl, num_blocks, ctx); 1285 return 0; 1286 } 1287 1288 if (_dif_sgl_is_bytes_multiple(src_sgl, data_block_size) && 1289 _dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size)) { 1290 dif_insert_copy(src_sgl, dst_sgl, num_blocks, ctx); 1291 } else { 1292 dif_insert_copy_split(src_sgl, dst_sgl, num_blocks, ctx); 1293 } 1294 1295 return 0; 1296 } 1297 1298 static void 1299 _dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1300 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1301 { 1302 uint8_t *src, *dst; 1303 uint64_t guard = 0; 1304 1305 _dif_sgl_get_buf(src_sgl, &src, NULL); 1306 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1307 1308 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1309 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, ctx->guard_interval, 1310 ctx->dif_pi_format); 1311 } else { 1312 memcpy(dst, src, ctx->guard_interval); 1313 } 1314 1315 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 1316 1317 _dif_sgl_advance(src_sgl, ctx->block_size); 1318 _dif_sgl_advance(dst_sgl, ctx->block_size); 1319 } 1320 1321 static void 1322 dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1323 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1324 { 1325 uint32_t offset_blocks; 1326 1327 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1328 _dif_overwrite_copy(src_sgl, dst_sgl, offset_blocks, ctx); 1329 } 1330 } 1331 1332 static void 1333 _dif_overwrite_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1334 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1335 { 1336 uint64_t guard = 0; 1337 struct spdk_dif dif = {}; 1338 1339 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1340 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1341 ctx->guard_interval, ctx->dif_pi_format); 1342 } else { 1343 _data_copy_split(dst_sgl, src_sgl, ctx->guard_interval); 1344 } 1345 1346 _dif_sgl_advance(src_sgl, ctx->block_size - ctx->guard_interval); 1347 1348 _dif_generate(&dif, guard, offset_blocks, ctx); 1349 dif_store_split(dst_sgl, &dif, ctx); 1350 } 1351 1352 static void 1353 dif_overwrite_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1354 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1355 { 1356 uint32_t offset_blocks; 1357 1358 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1359 _dif_overwrite_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 1360 } 1361 } 1362 1363 static void 1364 dif_disable_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1365 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1366 { 1367 _data_copy_split(dst_sgl, src_sgl, ctx->block_size * num_blocks); 1368 } 1369 1370 static int 1371 _spdk_dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1372 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1373 { 1374 if (!_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks) || 1375 !_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks)) { 1376 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 1377 return -EINVAL; 1378 } 1379 1380 if (_dif_is_disabled(ctx->dif_type)) { 1381 dif_disable_copy(src_sgl, dst_sgl, num_blocks, ctx); 1382 return 0; 1383 } 1384 1385 if (_dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size) && 1386 _dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size)) { 1387 dif_overwrite_copy(src_sgl, dst_sgl, num_blocks, ctx); 1388 } else { 1389 dif_overwrite_copy_split(src_sgl, dst_sgl, num_blocks, ctx); 1390 } 1391 1392 return 0; 1393 } 1394 1395 int 1396 spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1397 int bounce_iovcnt, uint32_t num_blocks, 1398 const struct spdk_dif_ctx *ctx) 1399 { 1400 struct _dif_sgl src_sgl, dst_sgl; 1401 1402 _dif_sgl_init(&src_sgl, iovs, iovcnt); 1403 _dif_sgl_init(&dst_sgl, bounce_iovs, bounce_iovcnt); 1404 1405 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_NVME_PRACT) || 1406 ctx->md_size == _dif_size(ctx->dif_pi_format)) { 1407 return _spdk_dif_insert_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 1408 } else { 1409 return _spdk_dif_overwrite_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 1410 } 1411 } 1412 1413 static int 1414 _dif_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1415 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1416 struct spdk_dif_error *err_blk) 1417 { 1418 uint32_t data_block_size; 1419 uint8_t *src, *dst; 1420 int rc; 1421 uint64_t guard = 0; 1422 1423 data_block_size = ctx->block_size - ctx->md_size; 1424 1425 _dif_sgl_get_buf(src_sgl, &src, NULL); 1426 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1427 1428 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1429 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1430 ctx->dif_pi_format); 1431 guard = _dif_generate_guard(guard, src + data_block_size, 1432 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1433 } else { 1434 memcpy(dst, src, data_block_size); 1435 } 1436 1437 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1438 if (rc != 0) { 1439 return rc; 1440 } 1441 1442 _dif_sgl_advance(src_sgl, ctx->block_size); 1443 _dif_sgl_advance(dst_sgl, data_block_size); 1444 1445 return 0; 1446 } 1447 1448 static int 1449 dif_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1450 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1451 struct spdk_dif_error *err_blk) 1452 { 1453 uint32_t offset_blocks; 1454 int rc; 1455 1456 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1457 rc = _dif_strip_copy(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1458 if (rc != 0) { 1459 return rc; 1460 } 1461 } 1462 1463 return 0; 1464 } 1465 1466 static int 1467 _dif_strip_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1468 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1469 struct spdk_dif_error *err_blk) 1470 { 1471 uint32_t data_block_size; 1472 uint64_t guard = 0; 1473 struct spdk_dif dif = {}; 1474 1475 data_block_size = ctx->block_size - ctx->md_size; 1476 1477 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1478 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1479 data_block_size, ctx->dif_pi_format); 1480 guard = dif_generate_guard_split(guard, src_sgl, data_block_size, 1481 ctx->guard_interval - data_block_size, ctx); 1482 } else { 1483 _data_copy_split(dst_sgl, src_sgl, data_block_size); 1484 _dif_sgl_advance(src_sgl, ctx->guard_interval - data_block_size); 1485 } 1486 1487 dif_load_split(src_sgl, &dif, ctx); 1488 1489 return _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 1490 } 1491 1492 static int 1493 dif_strip_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1494 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1495 struct spdk_dif_error *err_blk) 1496 { 1497 uint32_t offset_blocks; 1498 int rc; 1499 1500 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1501 rc = _dif_strip_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1502 if (rc != 0) { 1503 return rc; 1504 } 1505 } 1506 1507 return 0; 1508 } 1509 1510 static void 1511 _dif_disable_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1512 const struct spdk_dif_ctx *ctx) 1513 { 1514 uint32_t offset = 0, src_len, dst_len, buf_len, data_block_size; 1515 uint8_t *src, *dst; 1516 1517 data_block_size = ctx->block_size - ctx->md_size; 1518 1519 while (offset < data_block_size) { 1520 _dif_sgl_get_buf(src_sgl, &src, &src_len); 1521 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 1522 buf_len = spdk_min(src_len, dst_len); 1523 buf_len = spdk_min(buf_len, data_block_size - offset); 1524 1525 memcpy(dst, src, buf_len); 1526 1527 _dif_sgl_advance(src_sgl, buf_len); 1528 _dif_sgl_advance(dst_sgl, buf_len); 1529 offset += buf_len; 1530 } 1531 1532 _dif_sgl_advance(src_sgl, ctx->md_size); 1533 } 1534 1535 static void 1536 dif_disable_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1537 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1538 { 1539 uint32_t offset_blocks; 1540 1541 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1542 _dif_disable_strip_copy(src_sgl, dst_sgl, ctx); 1543 } 1544 } 1545 1546 static int 1547 _spdk_dif_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1548 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1549 struct spdk_dif_error *err_blk) 1550 { 1551 uint32_t data_block_size; 1552 1553 data_block_size = ctx->block_size - ctx->md_size; 1554 1555 if (!_dif_sgl_is_valid(dst_sgl, data_block_size * num_blocks) || 1556 !_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks)) { 1557 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1558 return -EINVAL; 1559 } 1560 1561 if (_dif_is_disabled(ctx->dif_type)) { 1562 dif_disable_strip_copy(src_sgl, dst_sgl, num_blocks, ctx); 1563 return 0; 1564 } 1565 1566 if (_dif_sgl_is_bytes_multiple(dst_sgl, data_block_size) && 1567 _dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size)) { 1568 return dif_strip_copy(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1569 } else { 1570 return dif_strip_copy_split(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1571 } 1572 } 1573 1574 static int 1575 _dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1576 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1577 struct spdk_dif_error *err_blk) 1578 { 1579 uint8_t *src, *dst; 1580 int rc; 1581 uint64_t guard = 0; 1582 1583 _dif_sgl_get_buf(src_sgl, &src, NULL); 1584 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1585 1586 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1587 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, ctx->guard_interval, 1588 ctx->dif_pi_format); 1589 } else { 1590 memcpy(dst, src, ctx->guard_interval); 1591 } 1592 1593 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1594 if (rc != 0) { 1595 return rc; 1596 } 1597 1598 _dif_sgl_advance(src_sgl, ctx->block_size); 1599 _dif_sgl_advance(dst_sgl, ctx->block_size); 1600 1601 return 0; 1602 } 1603 1604 static int 1605 dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1606 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1607 struct spdk_dif_error *err_blk) 1608 { 1609 uint32_t offset_blocks; 1610 int rc; 1611 1612 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1613 rc = _dif_verify_copy(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1614 if (rc != 0) { 1615 return rc; 1616 } 1617 } 1618 1619 return 0; 1620 } 1621 1622 static int 1623 _dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1624 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1625 struct spdk_dif_error *err_blk) 1626 { 1627 uint64_t guard = 0; 1628 struct spdk_dif dif = {}; 1629 1630 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1631 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1632 ctx->guard_interval, ctx->dif_pi_format); 1633 } else { 1634 _data_copy_split(dst_sgl, src_sgl, ctx->guard_interval); 1635 } 1636 1637 dif_load_split(src_sgl, &dif, ctx); 1638 _dif_sgl_advance(dst_sgl, ctx->block_size - ctx->guard_interval); 1639 1640 return _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 1641 } 1642 1643 static int 1644 dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1645 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1646 struct spdk_dif_error *err_blk) 1647 { 1648 uint32_t offset_blocks; 1649 int rc; 1650 1651 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1652 rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1653 if (rc != 0) { 1654 return rc; 1655 } 1656 } 1657 1658 return 0; 1659 } 1660 1661 static int 1662 _spdk_dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1663 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1664 struct spdk_dif_error *err_blk) 1665 { 1666 if (!_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks) || 1667 !_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks)) { 1668 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1669 return -EINVAL; 1670 } 1671 1672 if (_dif_is_disabled(ctx->dif_type)) { 1673 dif_disable_copy(src_sgl, dst_sgl, num_blocks, ctx); 1674 return 0; 1675 } 1676 1677 if (_dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size) && 1678 _dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size)) { 1679 return dif_verify_copy(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1680 } else { 1681 return dif_verify_copy_split(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1682 } 1683 } 1684 1685 int 1686 spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1687 int bounce_iovcnt, uint32_t num_blocks, 1688 const struct spdk_dif_ctx *ctx, 1689 struct spdk_dif_error *err_blk) 1690 { 1691 struct _dif_sgl src_sgl, dst_sgl; 1692 1693 _dif_sgl_init(&src_sgl, bounce_iovs, bounce_iovcnt); 1694 _dif_sgl_init(&dst_sgl, iovs, iovcnt); 1695 1696 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_NVME_PRACT) || 1697 ctx->md_size == _dif_size(ctx->dif_pi_format)) { 1698 return _spdk_dif_strip_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1699 } else { 1700 return _spdk_dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1701 } 1702 } 1703 1704 static void 1705 _bit_flip(uint8_t *buf, uint32_t flip_bit) 1706 { 1707 uint8_t byte; 1708 1709 byte = *buf; 1710 byte ^= 1 << flip_bit; 1711 *buf = byte; 1712 } 1713 1714 static int 1715 _dif_inject_error(struct _dif_sgl *sgl, 1716 uint32_t block_size, uint32_t num_blocks, 1717 uint32_t inject_offset_blocks, 1718 uint32_t inject_offset_bytes, 1719 uint32_t inject_offset_bits) 1720 { 1721 uint32_t offset_in_block, buf_len; 1722 uint8_t *buf; 1723 1724 _dif_sgl_advance(sgl, block_size * inject_offset_blocks); 1725 1726 offset_in_block = 0; 1727 1728 while (offset_in_block < block_size) { 1729 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1730 buf_len = spdk_min(buf_len, block_size - offset_in_block); 1731 1732 if (inject_offset_bytes >= offset_in_block && 1733 inject_offset_bytes < offset_in_block + buf_len) { 1734 buf += inject_offset_bytes - offset_in_block; 1735 _bit_flip(buf, inject_offset_bits); 1736 return 0; 1737 } 1738 1739 _dif_sgl_advance(sgl, buf_len); 1740 offset_in_block += buf_len; 1741 } 1742 1743 return -1; 1744 } 1745 1746 static int 1747 dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks, 1748 uint32_t start_inject_bytes, uint32_t inject_range_bytes, 1749 uint32_t *inject_offset) 1750 { 1751 uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; 1752 uint32_t offset_blocks; 1753 int rc; 1754 1755 srand(time(0)); 1756 1757 inject_offset_blocks = rand() % num_blocks; 1758 inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); 1759 inject_offset_bits = rand() % 8; 1760 1761 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1762 if (offset_blocks == inject_offset_blocks) { 1763 rc = _dif_inject_error(sgl, block_size, num_blocks, 1764 inject_offset_blocks, 1765 inject_offset_bytes, 1766 inject_offset_bits); 1767 if (rc == 0) { 1768 *inject_offset = inject_offset_blocks; 1769 } 1770 return rc; 1771 } 1772 } 1773 1774 return -1; 1775 } 1776 1777 int 1778 spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1779 const struct spdk_dif_ctx *ctx, uint32_t inject_flags, 1780 uint32_t *inject_offset) 1781 { 1782 struct _dif_sgl sgl; 1783 int rc; 1784 1785 _dif_sgl_init(&sgl, iovs, iovcnt); 1786 1787 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1788 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1789 return -EINVAL; 1790 } 1791 1792 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1793 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1794 ctx->guard_interval + _dif_reftag_offset(ctx->dif_pi_format), 1795 _dif_reftag_size(ctx->dif_pi_format), 1796 inject_offset); 1797 if (rc != 0) { 1798 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1799 return rc; 1800 } 1801 } 1802 1803 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1804 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1805 ctx->guard_interval + _dif_apptag_offset(ctx->dif_pi_format), 1806 _dif_apptag_size(), 1807 inject_offset); 1808 if (rc != 0) { 1809 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1810 return rc; 1811 } 1812 } 1813 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1814 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1815 ctx->guard_interval, 1816 _dif_guard_size(ctx->dif_pi_format), 1817 inject_offset); 1818 if (rc != 0) { 1819 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1820 return rc; 1821 } 1822 } 1823 1824 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1825 /* If the DIF information is contained within the last 8/16 bytes of 1826 * metadata (depending on the PI format), then the CRC covers all metadata 1827 * bytes up to but excluding the last 8/16 bytes. But error injection does not 1828 * cover these metadata because classification is not determined yet. 1829 * 1830 * Note: Error injection to data block is expected to be detected as 1831 * guard error. 1832 */ 1833 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1834 0, 1835 ctx->block_size - ctx->md_size, 1836 inject_offset); 1837 if (rc != 0) { 1838 SPDK_ERRLOG("Failed to inject error to data block.\n"); 1839 return rc; 1840 } 1841 } 1842 1843 return 0; 1844 } 1845 1846 static void 1847 dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1848 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1849 { 1850 uint32_t offset_blocks = 0; 1851 uint8_t *data_buf, *md_buf; 1852 uint64_t guard; 1853 1854 while (offset_blocks < num_blocks) { 1855 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1856 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1857 1858 guard = 0; 1859 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1860 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1861 ctx->dif_pi_format); 1862 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1863 ctx->dif_pi_format); 1864 } 1865 1866 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1867 1868 _dif_sgl_advance(data_sgl, ctx->block_size); 1869 _dif_sgl_advance(md_sgl, ctx->md_size); 1870 offset_blocks++; 1871 } 1872 } 1873 1874 static void 1875 _dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1876 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1877 { 1878 uint32_t offset_in_block, data_buf_len; 1879 uint8_t *data_buf, *md_buf; 1880 uint64_t guard = 0; 1881 1882 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1883 1884 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1885 guard = ctx->guard_seed; 1886 } 1887 offset_in_block = 0; 1888 1889 while (offset_in_block < ctx->block_size) { 1890 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1891 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1892 1893 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1894 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 1895 ctx->dif_pi_format); 1896 } 1897 1898 _dif_sgl_advance(data_sgl, data_buf_len); 1899 offset_in_block += data_buf_len; 1900 } 1901 1902 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1903 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1904 ctx->dif_pi_format); 1905 } 1906 1907 _dif_sgl_advance(md_sgl, ctx->md_size); 1908 1909 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1910 } 1911 1912 static void 1913 dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1914 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1915 { 1916 uint32_t offset_blocks; 1917 1918 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1919 _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx); 1920 } 1921 } 1922 1923 int 1924 spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1925 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1926 { 1927 struct _dif_sgl data_sgl, md_sgl; 1928 1929 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1930 _dif_sgl_init(&md_sgl, md_iov, 1); 1931 1932 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1933 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1934 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1935 return -EINVAL; 1936 } 1937 1938 if (_dif_is_disabled(ctx->dif_type)) { 1939 return 0; 1940 } 1941 1942 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1943 dix_generate(&data_sgl, &md_sgl, num_blocks, ctx); 1944 } else { 1945 dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx); 1946 } 1947 1948 return 0; 1949 } 1950 1951 static int 1952 dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1953 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1954 struct spdk_dif_error *err_blk) 1955 { 1956 uint32_t offset_blocks = 0; 1957 uint8_t *data_buf, *md_buf; 1958 uint64_t guard; 1959 int rc; 1960 1961 while (offset_blocks < num_blocks) { 1962 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1963 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1964 1965 guard = 0; 1966 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1967 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1968 ctx->dif_pi_format); 1969 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1970 ctx->dif_pi_format); 1971 } 1972 1973 rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1974 if (rc != 0) { 1975 return rc; 1976 } 1977 1978 _dif_sgl_advance(data_sgl, ctx->block_size); 1979 _dif_sgl_advance(md_sgl, ctx->md_size); 1980 offset_blocks++; 1981 } 1982 1983 return 0; 1984 } 1985 1986 static int 1987 _dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1988 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1989 struct spdk_dif_error *err_blk) 1990 { 1991 uint32_t offset_in_block, data_buf_len; 1992 uint8_t *data_buf, *md_buf; 1993 uint64_t guard = 0; 1994 1995 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1996 1997 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1998 guard = ctx->guard_seed; 1999 } 2000 offset_in_block = 0; 2001 2002 while (offset_in_block < ctx->block_size) { 2003 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 2004 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 2005 2006 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2007 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 2008 ctx->dif_pi_format); 2009 } 2010 2011 _dif_sgl_advance(data_sgl, data_buf_len); 2012 offset_in_block += data_buf_len; 2013 } 2014 2015 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2016 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 2017 ctx->dif_pi_format); 2018 } 2019 2020 _dif_sgl_advance(md_sgl, ctx->md_size); 2021 2022 return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 2023 } 2024 2025 static int 2026 dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 2027 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 2028 struct spdk_dif_error *err_blk) 2029 { 2030 uint32_t offset_blocks; 2031 int rc; 2032 2033 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2034 rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk); 2035 if (rc != 0) { 2036 return rc; 2037 } 2038 } 2039 2040 return 0; 2041 } 2042 2043 int 2044 spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 2045 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 2046 struct spdk_dif_error *err_blk) 2047 { 2048 struct _dif_sgl data_sgl, md_sgl; 2049 2050 if (md_iov->iov_base == NULL) { 2051 SPDK_ERRLOG("Metadata buffer is NULL.\n"); 2052 return -EINVAL; 2053 } 2054 2055 _dif_sgl_init(&data_sgl, iovs, iovcnt); 2056 _dif_sgl_init(&md_sgl, md_iov, 1); 2057 2058 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 2059 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2060 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2061 return -EINVAL; 2062 } 2063 2064 if (_dif_is_disabled(ctx->dif_type)) { 2065 return 0; 2066 } 2067 2068 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 2069 return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 2070 } else { 2071 return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 2072 } 2073 } 2074 2075 int 2076 spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 2077 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 2078 uint32_t inject_flags, uint32_t *inject_offset) 2079 { 2080 struct _dif_sgl data_sgl, md_sgl; 2081 int rc; 2082 2083 _dif_sgl_init(&data_sgl, iovs, iovcnt); 2084 _dif_sgl_init(&md_sgl, md_iov, 1); 2085 2086 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 2087 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2088 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2089 return -EINVAL; 2090 } 2091 2092 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 2093 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 2094 ctx->guard_interval + _dif_reftag_offset(ctx->dif_pi_format), 2095 _dif_reftag_size(ctx->dif_pi_format), 2096 inject_offset); 2097 if (rc != 0) { 2098 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 2099 return rc; 2100 } 2101 } 2102 2103 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 2104 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 2105 ctx->guard_interval + _dif_apptag_offset(ctx->dif_pi_format), 2106 _dif_apptag_size(), 2107 inject_offset); 2108 if (rc != 0) { 2109 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 2110 return rc; 2111 } 2112 } 2113 2114 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 2115 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 2116 ctx->guard_interval, 2117 _dif_guard_size(ctx->dif_pi_format), 2118 inject_offset); 2119 if (rc != 0) { 2120 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 2121 return rc; 2122 } 2123 } 2124 2125 if (inject_flags & SPDK_DIF_DATA_ERROR) { 2126 /* Note: Error injection to data block is expected to be detected 2127 * as guard error. 2128 */ 2129 rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks, 2130 0, 2131 ctx->block_size, 2132 inject_offset); 2133 if (rc != 0) { 2134 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 2135 return rc; 2136 } 2137 } 2138 2139 return 0; 2140 } 2141 2142 static uint32_t 2143 _to_next_boundary(uint32_t offset, uint32_t boundary) 2144 { 2145 return boundary - (offset % boundary); 2146 } 2147 2148 static uint32_t 2149 _to_size_with_md(uint32_t size, uint32_t data_block_size, uint32_t block_size) 2150 { 2151 return (size / data_block_size) * block_size + (size % data_block_size); 2152 } 2153 2154 int 2155 spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt, 2156 struct iovec *buf_iovs, int buf_iovcnt, 2157 uint32_t data_offset, uint32_t data_len, 2158 uint32_t *_mapped_len, 2159 const struct spdk_dif_ctx *ctx) 2160 { 2161 uint32_t data_block_size, data_unalign, buf_len, buf_offset, len; 2162 struct _dif_sgl dif_sgl; 2163 struct _dif_sgl buf_sgl; 2164 2165 if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) { 2166 return -EINVAL; 2167 } 2168 2169 data_block_size = ctx->block_size - ctx->md_size; 2170 2171 data_unalign = ctx->data_offset % data_block_size; 2172 2173 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 2174 ctx->block_size); 2175 buf_len -= data_unalign; 2176 2177 _dif_sgl_init(&dif_sgl, iovs, iovcnt); 2178 _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt); 2179 2180 if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) { 2181 SPDK_ERRLOG("Buffer overflow will occur.\n"); 2182 return -ERANGE; 2183 } 2184 2185 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 2186 buf_offset -= data_unalign; 2187 2188 _dif_sgl_advance(&buf_sgl, buf_offset); 2189 2190 while (data_len != 0) { 2191 len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size)); 2192 if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) { 2193 break; 2194 } 2195 _dif_sgl_advance(&buf_sgl, ctx->md_size); 2196 data_offset += len; 2197 data_len -= len; 2198 } 2199 2200 if (_mapped_len != NULL) { 2201 *_mapped_len = dif_sgl.total_size; 2202 } 2203 2204 return iovcnt - dif_sgl.iovcnt; 2205 } 2206 2207 static int 2208 _dif_sgl_setup_stream(struct _dif_sgl *sgl, uint32_t *_buf_offset, uint32_t *_buf_len, 2209 uint32_t data_offset, uint32_t data_len, 2210 const struct spdk_dif_ctx *ctx) 2211 { 2212 uint32_t data_block_size, data_unalign, buf_len, buf_offset; 2213 2214 data_block_size = ctx->block_size - ctx->md_size; 2215 2216 data_unalign = ctx->data_offset % data_block_size; 2217 2218 /* If the last data block is complete, DIF of the data block is 2219 * inserted or verified in this turn. 2220 */ 2221 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 2222 ctx->block_size); 2223 buf_len -= data_unalign; 2224 2225 if (!_dif_sgl_is_valid(sgl, buf_len)) { 2226 return -ERANGE; 2227 } 2228 2229 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 2230 buf_offset -= data_unalign; 2231 2232 _dif_sgl_advance(sgl, buf_offset); 2233 buf_len -= buf_offset; 2234 2235 buf_offset += data_unalign; 2236 2237 *_buf_offset = buf_offset; 2238 *_buf_len = buf_len; 2239 2240 return 0; 2241 } 2242 2243 int 2244 spdk_dif_generate_stream(struct iovec *iovs, int iovcnt, 2245 uint32_t data_offset, uint32_t data_len, 2246 struct spdk_dif_ctx *ctx) 2247 { 2248 uint32_t buf_len = 0, buf_offset = 0; 2249 uint32_t len, offset_in_block, offset_blocks; 2250 uint64_t guard = 0; 2251 struct _dif_sgl sgl; 2252 int rc; 2253 2254 if (iovs == NULL || iovcnt == 0) { 2255 return -EINVAL; 2256 } 2257 2258 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2259 guard = ctx->last_guard; 2260 } 2261 2262 _dif_sgl_init(&sgl, iovs, iovcnt); 2263 2264 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 2265 if (rc != 0) { 2266 return rc; 2267 } 2268 2269 while (buf_len != 0) { 2270 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 2271 offset_in_block = buf_offset % ctx->block_size; 2272 offset_blocks = buf_offset / ctx->block_size; 2273 2274 guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx); 2275 2276 buf_len -= len; 2277 buf_offset += len; 2278 } 2279 2280 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2281 ctx->last_guard = guard; 2282 } 2283 2284 return 0; 2285 } 2286 2287 int 2288 spdk_dif_verify_stream(struct iovec *iovs, int iovcnt, 2289 uint32_t data_offset, uint32_t data_len, 2290 struct spdk_dif_ctx *ctx, 2291 struct spdk_dif_error *err_blk) 2292 { 2293 uint32_t buf_len = 0, buf_offset = 0; 2294 uint32_t len, offset_in_block, offset_blocks; 2295 uint64_t guard = 0; 2296 struct _dif_sgl sgl; 2297 int rc = 0; 2298 2299 if (iovs == NULL || iovcnt == 0) { 2300 return -EINVAL; 2301 } 2302 2303 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2304 guard = ctx->last_guard; 2305 } 2306 2307 _dif_sgl_init(&sgl, iovs, iovcnt); 2308 2309 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 2310 if (rc != 0) { 2311 return rc; 2312 } 2313 2314 while (buf_len != 0) { 2315 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 2316 offset_in_block = buf_offset % ctx->block_size; 2317 offset_blocks = buf_offset / ctx->block_size; 2318 2319 rc = _dif_verify_split(&sgl, offset_in_block, len, &guard, offset_blocks, 2320 ctx, err_blk); 2321 if (rc != 0) { 2322 goto error; 2323 } 2324 2325 buf_len -= len; 2326 buf_offset += len; 2327 } 2328 2329 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2330 ctx->last_guard = guard; 2331 } 2332 error: 2333 return rc; 2334 } 2335 2336 int 2337 spdk_dif_update_crc32c_stream(struct iovec *iovs, int iovcnt, 2338 uint32_t data_offset, uint32_t data_len, 2339 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 2340 { 2341 uint32_t buf_len = 0, buf_offset = 0, len, offset_in_block; 2342 uint32_t crc32c; 2343 struct _dif_sgl sgl; 2344 int rc; 2345 2346 if (iovs == NULL || iovcnt == 0) { 2347 return -EINVAL; 2348 } 2349 2350 crc32c = *_crc32c; 2351 _dif_sgl_init(&sgl, iovs, iovcnt); 2352 2353 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 2354 if (rc != 0) { 2355 return rc; 2356 } 2357 2358 while (buf_len != 0) { 2359 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 2360 offset_in_block = buf_offset % ctx->block_size; 2361 2362 crc32c = _dif_update_crc32c_split(&sgl, offset_in_block, len, crc32c, ctx); 2363 2364 buf_len -= len; 2365 buf_offset += len; 2366 } 2367 2368 *_crc32c = crc32c; 2369 2370 return 0; 2371 } 2372 2373 void 2374 spdk_dif_get_range_with_md(uint32_t data_offset, uint32_t data_len, 2375 uint32_t *_buf_offset, uint32_t *_buf_len, 2376 const struct spdk_dif_ctx *ctx) 2377 { 2378 uint32_t data_block_size, data_unalign, buf_offset, buf_len; 2379 2380 if (!ctx->md_interleave) { 2381 buf_offset = data_offset; 2382 buf_len = data_len; 2383 } else { 2384 data_block_size = ctx->block_size - ctx->md_size; 2385 2386 data_unalign = data_offset % data_block_size; 2387 2388 buf_offset = _to_size_with_md(data_offset, data_block_size, ctx->block_size); 2389 buf_len = _to_size_with_md(data_unalign + data_len, data_block_size, ctx->block_size) - 2390 data_unalign; 2391 } 2392 2393 if (_buf_offset != NULL) { 2394 *_buf_offset = buf_offset; 2395 } 2396 2397 if (_buf_len != NULL) { 2398 *_buf_len = buf_len; 2399 } 2400 } 2401 2402 uint32_t 2403 spdk_dif_get_length_with_md(uint32_t data_len, const struct spdk_dif_ctx *ctx) 2404 { 2405 uint32_t data_block_size; 2406 2407 if (!ctx->md_interleave) { 2408 return data_len; 2409 } else { 2410 data_block_size = ctx->block_size - ctx->md_size; 2411 2412 return _to_size_with_md(data_len, data_block_size, ctx->block_size); 2413 } 2414 } 2415 2416 static int 2417 _dif_remap_ref_tag(struct _dif_sgl *sgl, uint32_t offset_blocks, 2418 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2419 bool check_ref_tag) 2420 { 2421 uint32_t offset, buf_len; 2422 uint64_t expected = 0, remapped; 2423 uint8_t *buf; 2424 struct _dif_sgl tmp_sgl; 2425 struct spdk_dif dif; 2426 2427 /* Fast forward to DIF field. */ 2428 _dif_sgl_advance(sgl, ctx->guard_interval); 2429 _dif_sgl_copy(&tmp_sgl, sgl); 2430 2431 /* Copy the split DIF field to the temporary DIF buffer */ 2432 offset = 0; 2433 while (offset < _dif_size(ctx->dif_pi_format)) { 2434 _dif_sgl_get_buf(sgl, &buf, &buf_len); 2435 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 2436 2437 memcpy((uint8_t *)&dif + offset, buf, buf_len); 2438 2439 _dif_sgl_advance(sgl, buf_len); 2440 offset += buf_len; 2441 } 2442 2443 if (_dif_ignore(&dif, ctx)) { 2444 goto end; 2445 } 2446 2447 /* For type 1 and 2, the Reference Tag is incremented for each 2448 * subsequent logical block. For type 3, the Reference Tag 2449 * remains the same as the initial Reference Tag. 2450 */ 2451 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2452 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2453 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2454 } else { 2455 remapped = ctx->remapped_init_ref_tag; 2456 } 2457 2458 /* Verify the stored Reference Tag. */ 2459 if (check_ref_tag && !_dif_reftag_check(&dif, ctx, expected, offset_blocks, err_blk)) { 2460 return -1; 2461 } 2462 2463 /* Update the stored Reference Tag to the remapped one. */ 2464 _dif_set_reftag(&dif, remapped, ctx->dif_pi_format); 2465 2466 offset = 0; 2467 while (offset < _dif_size(ctx->dif_pi_format)) { 2468 _dif_sgl_get_buf(&tmp_sgl, &buf, &buf_len); 2469 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 2470 2471 memcpy(buf, (uint8_t *)&dif + offset, buf_len); 2472 2473 _dif_sgl_advance(&tmp_sgl, buf_len); 2474 offset += buf_len; 2475 } 2476 2477 end: 2478 _dif_sgl_advance(sgl, ctx->block_size - ctx->guard_interval - _dif_size(ctx->dif_pi_format)); 2479 2480 return 0; 2481 } 2482 2483 int 2484 spdk_dif_remap_ref_tag(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 2485 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2486 bool check_ref_tag) 2487 { 2488 struct _dif_sgl sgl; 2489 uint32_t offset_blocks; 2490 int rc; 2491 2492 _dif_sgl_init(&sgl, iovs, iovcnt); 2493 2494 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 2495 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2496 return -EINVAL; 2497 } 2498 2499 if (_dif_is_disabled(ctx->dif_type)) { 2500 return 0; 2501 } 2502 2503 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2504 return 0; 2505 } 2506 2507 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2508 rc = _dif_remap_ref_tag(&sgl, offset_blocks, ctx, err_blk, check_ref_tag); 2509 if (rc != 0) { 2510 return rc; 2511 } 2512 } 2513 2514 return 0; 2515 } 2516 2517 static int 2518 _dix_remap_ref_tag(struct _dif_sgl *md_sgl, uint32_t offset_blocks, 2519 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2520 bool check_ref_tag) 2521 { 2522 uint64_t expected = 0, remapped; 2523 uint8_t *md_buf; 2524 struct spdk_dif *dif; 2525 2526 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 2527 2528 dif = (struct spdk_dif *)(md_buf + ctx->guard_interval); 2529 2530 if (_dif_ignore(dif, ctx)) { 2531 goto end; 2532 } 2533 2534 /* For type 1 and 2, the Reference Tag is incremented for each 2535 * subsequent logical block. For type 3, the Reference Tag 2536 * remains the same as the initialReference Tag. 2537 */ 2538 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2539 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2540 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2541 } else { 2542 remapped = ctx->remapped_init_ref_tag; 2543 } 2544 2545 /* Verify the stored Reference Tag. */ 2546 if (check_ref_tag && !_dif_reftag_check(dif, ctx, expected, offset_blocks, err_blk)) { 2547 return -1; 2548 } 2549 2550 /* Update the stored Reference Tag to the remapped one. */ 2551 _dif_set_reftag(dif, remapped, ctx->dif_pi_format); 2552 2553 end: 2554 _dif_sgl_advance(md_sgl, ctx->md_size); 2555 2556 return 0; 2557 } 2558 2559 int 2560 spdk_dix_remap_ref_tag(struct iovec *md_iov, uint32_t num_blocks, 2561 const struct spdk_dif_ctx *ctx, 2562 struct spdk_dif_error *err_blk, 2563 bool check_ref_tag) 2564 { 2565 struct _dif_sgl md_sgl; 2566 uint32_t offset_blocks; 2567 int rc; 2568 2569 _dif_sgl_init(&md_sgl, md_iov, 1); 2570 2571 if (!_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2572 SPDK_ERRLOG("Size of metadata iovec array is not valid.\n"); 2573 return -EINVAL; 2574 } 2575 2576 if (_dif_is_disabled(ctx->dif_type)) { 2577 return 0; 2578 } 2579 2580 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2581 return 0; 2582 } 2583 2584 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2585 rc = _dix_remap_ref_tag(&md_sgl, offset_blocks, ctx, err_blk, check_ref_tag); 2586 if (rc != 0) { 2587 return rc; 2588 } 2589 } 2590 2591 return 0; 2592 } 2593