1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/dif.h" 7 #include "spdk/crc16.h" 8 #include "spdk/crc32.h" 9 #include "spdk/crc64.h" 10 #include "spdk/endian.h" 11 #include "spdk/log.h" 12 #include "spdk/util.h" 13 14 #define REFTAG_MASK_16 0x00000000FFFFFFFF 15 #define REFTAG_MASK_32 0xFFFFFFFFFFFFFFFF 16 #define REFTAG_MASK_64 0x0000FFFFFFFFFFFF 17 18 /* The variable size Storage Tag and Reference Tag is not supported yet, 19 * so the maximum size of the Reference Tag is assumed. 20 */ 21 struct spdk_dif { 22 union { 23 struct { 24 uint16_t guard; 25 uint16_t app_tag; 26 uint32_t stor_ref_space; 27 } g16; 28 struct { 29 uint32_t guard; 30 uint16_t app_tag; 31 uint16_t stor_ref_space_p1; 32 uint64_t stor_ref_space_p2; 33 } g32; 34 struct { 35 uint64_t guard; 36 uint16_t app_tag; 37 uint16_t stor_ref_space_p1; 38 uint32_t stor_ref_space_p2; 39 } g64; 40 }; 41 }; 42 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g16) == 8, "Incorrect size"); 43 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g32) == 16, "Incorrect size"); 44 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g64) == 16, "Incorrect size"); 45 46 /* Context to iterate or create a iovec array. 47 * Each sgl is either iterated or created at a time. 48 */ 49 struct _dif_sgl { 50 /* Current iovec in the iteration or creation */ 51 struct iovec *iov; 52 53 /* Remaining count of iovecs in the iteration or creation. */ 54 int iovcnt; 55 56 /* Current offset in the iovec */ 57 uint32_t iov_offset; 58 59 /* Size of the created iovec array in bytes */ 60 uint32_t total_size; 61 }; 62 63 static inline void 64 _dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt) 65 { 66 s->iov = iovs; 67 s->iovcnt = iovcnt; 68 s->iov_offset = 0; 69 s->total_size = 0; 70 } 71 72 static void 73 _dif_sgl_advance(struct _dif_sgl *s, uint32_t step) 74 { 75 s->iov_offset += step; 76 while (s->iovcnt != 0) { 77 if (s->iov_offset < s->iov->iov_len) { 78 break; 79 } 80 81 s->iov_offset -= s->iov->iov_len; 82 s->iov++; 83 s->iovcnt--; 84 } 85 } 86 87 static inline void 88 _dif_sgl_get_buf(struct _dif_sgl *s, uint8_t **_buf, uint32_t *_buf_len) 89 { 90 if (_buf != NULL) { 91 *_buf = (uint8_t *)s->iov->iov_base + s->iov_offset; 92 } 93 if (_buf_len != NULL) { 94 *_buf_len = s->iov->iov_len - s->iov_offset; 95 } 96 } 97 98 static inline bool 99 _dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len) 100 { 101 assert(s->iovcnt > 0); 102 s->iov->iov_base = data; 103 s->iov->iov_len = data_len; 104 s->total_size += data_len; 105 s->iov++; 106 s->iovcnt--; 107 108 if (s->iovcnt > 0) { 109 return true; 110 } else { 111 return false; 112 } 113 } 114 115 static inline bool 116 _dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len) 117 { 118 uint8_t *buf; 119 uint32_t buf_len; 120 121 while (data_len != 0) { 122 _dif_sgl_get_buf(src, &buf, &buf_len); 123 buf_len = spdk_min(buf_len, data_len); 124 125 if (!_dif_sgl_append(dst, buf, buf_len)) { 126 return false; 127 } 128 129 _dif_sgl_advance(src, buf_len); 130 data_len -= buf_len; 131 } 132 133 return true; 134 } 135 136 /* This function must be used before starting iteration. */ 137 static bool 138 _dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes) 139 { 140 int i; 141 142 for (i = 0; i < s->iovcnt; i++) { 143 if (s->iov[i].iov_len % bytes) { 144 return false; 145 } 146 } 147 148 return true; 149 } 150 151 /* This function must be used before starting iteration. */ 152 static bool 153 _dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes) 154 { 155 uint64_t total = 0; 156 int i; 157 158 for (i = 0; i < s->iovcnt; i++) { 159 total += s->iov[i].iov_len; 160 } 161 162 return total >= bytes; 163 } 164 165 static void 166 _dif_sgl_copy(struct _dif_sgl *to, struct _dif_sgl *from) 167 { 168 memcpy(to, from, sizeof(struct _dif_sgl)); 169 } 170 171 static bool 172 _dif_is_disabled(enum spdk_dif_type dif_type) 173 { 174 if (dif_type == SPDK_DIF_DISABLE) { 175 return true; 176 } else { 177 return false; 178 } 179 } 180 181 static inline size_t 182 _dif_size(enum spdk_dif_pi_format dif_pi_format) 183 { 184 uint8_t size; 185 186 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 187 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16); 188 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 189 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32); 190 } else { 191 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64); 192 } 193 194 return size; 195 } 196 197 uint32_t 198 spdk_dif_pi_format_get_size(enum spdk_dif_pi_format dif_pi_format) 199 { 200 return _dif_size(dif_pi_format); 201 } 202 203 static uint32_t 204 _get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave, 205 size_t dif_size) 206 { 207 if (!dif_loc) { 208 /* For metadata formats with more than 8/16 bytes (depending on 209 * the PI format), if the DIF is contained in the last 8/16 bytes 210 * of metadata, then the CRC covers all metadata up to but excluding 211 * these last 8/16 bytes. 212 */ 213 if (md_interleave) { 214 return block_size - dif_size; 215 } else { 216 return md_size - dif_size; 217 } 218 } else { 219 /* For metadata formats with more than 8/16 bytes (depending on 220 * the PI format), if the DIF is contained in the first 8/16 bytes 221 * of metadata, then the CRC does not cover any metadata. 222 */ 223 if (md_interleave) { 224 return block_size - md_size; 225 } else { 226 return 0; 227 } 228 } 229 } 230 231 static inline uint8_t 232 _dif_guard_size(enum spdk_dif_pi_format dif_pi_format) 233 { 234 uint8_t size; 235 236 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 237 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.guard); 238 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 239 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.guard); 240 } else { 241 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.guard); 242 } 243 244 return size; 245 } 246 247 static inline void 248 _dif_set_guard(struct spdk_dif *dif, uint64_t guard, enum spdk_dif_pi_format dif_pi_format) 249 { 250 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 251 to_be16(&(dif->g16.guard), (uint16_t)guard); 252 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 253 to_be32(&(dif->g32.guard), (uint32_t)guard); 254 } else { 255 to_be64(&(dif->g64.guard), guard); 256 } 257 } 258 259 static inline uint64_t 260 _dif_get_guard(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 261 { 262 uint64_t guard; 263 264 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 265 guard = (uint64_t)from_be16(&(dif->g16.guard)); 266 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 267 guard = (uint64_t)from_be32(&(dif->g32.guard)); 268 } else { 269 guard = from_be64(&(dif->g64.guard)); 270 } 271 272 return guard; 273 } 274 275 static inline uint64_t 276 _dif_generate_guard(uint64_t guard_seed, void *buf, size_t buf_len, 277 enum spdk_dif_pi_format dif_pi_format) 278 { 279 uint64_t guard; 280 281 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 282 guard = (uint64_t)spdk_crc16_t10dif((uint16_t)guard_seed, buf, buf_len); 283 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 284 guard = (uint64_t)spdk_crc32c_nvme(buf, buf_len, guard_seed); 285 } else { 286 guard = spdk_crc64_nvme(buf, buf_len, guard_seed); 287 } 288 289 return guard; 290 } 291 292 static uint64_t 293 dif_generate_guard_split(uint64_t guard_seed, struct _dif_sgl *sgl, uint32_t start, 294 uint32_t len, const struct spdk_dif_ctx *ctx) 295 { 296 uint64_t guard = guard_seed; 297 uint32_t offset, end, buf_len; 298 uint8_t *buf; 299 300 offset = start; 301 end = start + spdk_min(len, ctx->guard_interval - start); 302 303 while (offset < end) { 304 _dif_sgl_get_buf(sgl, &buf, &buf_len); 305 buf_len = spdk_min(buf_len, end - offset); 306 307 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 308 guard = _dif_generate_guard(guard, buf, buf_len, ctx->dif_pi_format); 309 } 310 311 _dif_sgl_advance(sgl, buf_len); 312 offset += buf_len; 313 } 314 315 return guard; 316 } 317 318 static inline uint64_t 319 _dif_generate_guard_copy(uint64_t guard_seed, void *dst, void *src, size_t buf_len, 320 enum spdk_dif_pi_format dif_pi_format) 321 { 322 uint64_t guard; 323 324 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 325 guard = (uint64_t)spdk_crc16_t10dif_copy((uint16_t)guard_seed, dst, src, buf_len); 326 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 327 memcpy(dst, src, buf_len); 328 guard = (uint64_t)spdk_crc32c_nvme(src, buf_len, guard_seed); 329 } else { 330 memcpy(dst, src, buf_len); 331 guard = spdk_crc64_nvme(src, buf_len, guard_seed); 332 } 333 334 return guard; 335 } 336 337 static uint64_t 338 _dif_generate_guard_copy_split(uint64_t guard, struct _dif_sgl *dst_sgl, 339 struct _dif_sgl *src_sgl, uint32_t data_len, 340 enum spdk_dif_pi_format dif_pi_format) 341 { 342 uint32_t offset = 0, src_len, dst_len, buf_len; 343 uint8_t *src, *dst; 344 345 while (offset < data_len) { 346 _dif_sgl_get_buf(src_sgl, &src, &src_len); 347 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 348 buf_len = spdk_min(src_len, dst_len); 349 buf_len = spdk_min(buf_len, data_len - offset); 350 351 guard = _dif_generate_guard_copy(guard, dst, src, buf_len, dif_pi_format); 352 353 _dif_sgl_advance(src_sgl, buf_len); 354 _dif_sgl_advance(dst_sgl, buf_len); 355 offset += buf_len; 356 } 357 358 return guard; 359 } 360 361 static void 362 _data_copy_split(struct _dif_sgl *dst_sgl, struct _dif_sgl *src_sgl, uint32_t data_len) 363 { 364 uint32_t offset = 0, src_len, dst_len, buf_len; 365 uint8_t *src, *dst; 366 367 while (offset < data_len) { 368 _dif_sgl_get_buf(src_sgl, &src, &src_len); 369 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 370 buf_len = spdk_min(src_len, dst_len); 371 buf_len = spdk_min(buf_len, data_len - offset); 372 373 memcpy(dst, src, buf_len); 374 375 _dif_sgl_advance(src_sgl, buf_len); 376 _dif_sgl_advance(dst_sgl, buf_len); 377 offset += buf_len; 378 } 379 } 380 381 static inline uint8_t 382 _dif_apptag_offset(enum spdk_dif_pi_format dif_pi_format) 383 { 384 return _dif_guard_size(dif_pi_format); 385 } 386 387 static inline uint8_t 388 _dif_apptag_size(void) 389 { 390 return SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.app_tag); 391 } 392 393 static inline void 394 _dif_set_apptag(struct spdk_dif *dif, uint16_t app_tag, enum spdk_dif_pi_format dif_pi_format) 395 { 396 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 397 to_be16(&(dif->g16.app_tag), app_tag); 398 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 399 to_be16(&(dif->g32.app_tag), app_tag); 400 } else { 401 to_be16(&(dif->g64.app_tag), app_tag); 402 } 403 } 404 405 static inline uint16_t 406 _dif_get_apptag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 407 { 408 uint16_t app_tag; 409 410 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 411 app_tag = from_be16(&(dif->g16.app_tag)); 412 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 413 app_tag = from_be16(&(dif->g32.app_tag)); 414 } else { 415 app_tag = from_be16(&(dif->g64.app_tag)); 416 } 417 418 return app_tag; 419 } 420 421 static inline bool 422 _dif_apptag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 423 { 424 return _dif_get_apptag(dif, dif_pi_format) == SPDK_DIF_APPTAG_IGNORE; 425 } 426 427 static inline uint8_t 428 _dif_reftag_offset(enum spdk_dif_pi_format dif_pi_format) 429 { 430 uint8_t offset; 431 432 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 433 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size(); 434 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 435 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size() 436 + SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.stor_ref_space_p1); 437 } else { 438 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size(); 439 } 440 441 return offset; 442 } 443 444 static inline uint8_t 445 _dif_reftag_size(enum spdk_dif_pi_format dif_pi_format) 446 { 447 uint8_t size; 448 449 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 450 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.stor_ref_space); 451 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 452 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.stor_ref_space_p2); 453 } else { 454 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.stor_ref_space_p1) + 455 SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.stor_ref_space_p2); 456 } 457 458 return size; 459 } 460 461 static inline void 462 _dif_set_reftag(struct spdk_dif *dif, uint64_t ref_tag, enum spdk_dif_pi_format dif_pi_format) 463 { 464 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 465 to_be32(&(dif->g16.stor_ref_space), (uint32_t)ref_tag); 466 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 467 to_be64(&(dif->g32.stor_ref_space_p2), ref_tag); 468 } else { 469 to_be16(&(dif->g64.stor_ref_space_p1), (uint16_t)(ref_tag >> 32)); 470 to_be32(&(dif->g64.stor_ref_space_p2), (uint32_t)ref_tag); 471 } 472 } 473 474 static inline uint64_t 475 _dif_get_reftag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 476 { 477 uint64_t ref_tag; 478 479 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 480 ref_tag = (uint64_t)from_be32(&(dif->g16.stor_ref_space)); 481 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 482 ref_tag = from_be64(&(dif->g32.stor_ref_space_p2)); 483 } else { 484 ref_tag = (uint64_t)from_be16(&(dif->g64.stor_ref_space_p1)); 485 ref_tag <<= 32; 486 ref_tag |= (uint64_t)from_be32(&(dif->g64.stor_ref_space_p2)); 487 } 488 489 return ref_tag; 490 } 491 492 static inline bool 493 _dif_reftag_match(struct spdk_dif *dif, uint64_t ref_tag, 494 enum spdk_dif_pi_format dif_pi_format) 495 { 496 uint64_t _ref_tag; 497 bool match; 498 499 _ref_tag = _dif_get_reftag(dif, dif_pi_format); 500 501 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 502 match = (_ref_tag == (ref_tag & REFTAG_MASK_16)); 503 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 504 match = (_ref_tag == ref_tag); 505 } else { 506 match = (_ref_tag == (ref_tag & REFTAG_MASK_64)); 507 } 508 509 return match; 510 } 511 512 static inline bool 513 _dif_reftag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 514 { 515 return _dif_reftag_match(dif, REFTAG_MASK_32, dif_pi_format); 516 } 517 518 static bool 519 _dif_ignore(struct spdk_dif *dif, const struct spdk_dif_ctx *ctx) 520 { 521 switch (ctx->dif_type) { 522 case SPDK_DIF_TYPE1: 523 case SPDK_DIF_TYPE2: 524 /* If Type 1 or 2 is used, then all DIF checks are disabled when 525 * the Application Tag is 0xFFFF. 526 */ 527 if (_dif_apptag_ignore(dif, ctx->dif_pi_format)) { 528 return true; 529 } 530 break; 531 case SPDK_DIF_TYPE3: 532 /* If Type 3 is used, then all DIF checks are disabled when the 533 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF 534 * or 0xFFFFFFFFFFFFFFFF depending on the PI format. 535 */ 536 537 if (_dif_apptag_ignore(dif, ctx->dif_pi_format) && 538 _dif_reftag_ignore(dif, ctx->dif_pi_format)) { 539 return true; 540 } 541 break; 542 default: 543 break; 544 } 545 546 return false; 547 } 548 549 static bool 550 _dif_pi_format_is_valid(enum spdk_dif_pi_format dif_pi_format) 551 { 552 switch (dif_pi_format) { 553 case SPDK_DIF_PI_FORMAT_16: 554 case SPDK_DIF_PI_FORMAT_32: 555 case SPDK_DIF_PI_FORMAT_64: 556 return true; 557 default: 558 return false; 559 } 560 } 561 562 static bool 563 _dif_type_is_valid(enum spdk_dif_type dif_type) 564 { 565 switch (dif_type) { 566 case SPDK_DIF_DISABLE: 567 case SPDK_DIF_TYPE1: 568 case SPDK_DIF_TYPE2: 569 case SPDK_DIF_TYPE3: 570 return true; 571 default: 572 return false; 573 } 574 } 575 576 int 577 spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, 578 bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, 579 uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag, 580 uint32_t data_offset, uint64_t guard_seed, struct spdk_dif_ctx_init_ext_opts *opts) 581 { 582 uint32_t data_block_size; 583 enum spdk_dif_pi_format dif_pi_format = SPDK_DIF_PI_FORMAT_16; 584 585 if (opts != NULL) { 586 if (!_dif_pi_format_is_valid(opts->dif_pi_format)) { 587 SPDK_ERRLOG("No valid DIF PI format provided.\n"); 588 return -EINVAL; 589 } 590 591 dif_pi_format = opts->dif_pi_format; 592 } 593 594 if (!_dif_type_is_valid(dif_type)) { 595 SPDK_ERRLOG("No valid DIF type was provided.\n"); 596 return -EINVAL; 597 } 598 599 if (md_size < _dif_size(dif_pi_format)) { 600 SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); 601 return -EINVAL; 602 } 603 604 if (md_interleave) { 605 if (block_size < md_size) { 606 SPDK_ERRLOG("Block size is smaller than DIF size.\n"); 607 return -EINVAL; 608 } 609 data_block_size = block_size - md_size; 610 } else { 611 data_block_size = block_size; 612 } 613 614 if (data_block_size == 0) { 615 SPDK_ERRLOG("Zero data block size is not allowed\n"); 616 return -EINVAL; 617 } 618 619 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 620 if ((data_block_size % 512) != 0) { 621 SPDK_ERRLOG("Data block size should be a multiple of 512B\n"); 622 return -EINVAL; 623 } 624 } else { 625 if ((data_block_size % 4096) != 0) { 626 SPDK_ERRLOG("Data block size should be a multiple of 4kB\n"); 627 return -EINVAL; 628 } 629 } 630 631 ctx->block_size = block_size; 632 ctx->md_size = md_size; 633 ctx->md_interleave = md_interleave; 634 ctx->dif_pi_format = dif_pi_format; 635 ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave, 636 _dif_size(ctx->dif_pi_format)); 637 ctx->dif_type = dif_type; 638 ctx->dif_flags = dif_flags; 639 ctx->init_ref_tag = init_ref_tag; 640 ctx->apptag_mask = apptag_mask; 641 ctx->app_tag = app_tag; 642 ctx->data_offset = data_offset; 643 ctx->ref_tag_offset = data_offset / data_block_size; 644 ctx->last_guard = guard_seed; 645 ctx->guard_seed = guard_seed; 646 ctx->remapped_init_ref_tag = 0; 647 648 return 0; 649 } 650 651 void 652 spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset) 653 { 654 uint32_t data_block_size; 655 656 if (ctx->md_interleave) { 657 data_block_size = ctx->block_size - ctx->md_size; 658 } else { 659 data_block_size = ctx->block_size; 660 } 661 662 ctx->data_offset = data_offset; 663 ctx->ref_tag_offset = data_offset / data_block_size; 664 } 665 666 void 667 spdk_dif_ctx_set_remapped_init_ref_tag(struct spdk_dif_ctx *ctx, 668 uint32_t remapped_init_ref_tag) 669 { 670 ctx->remapped_init_ref_tag = remapped_init_ref_tag; 671 } 672 673 static void 674 _dif_generate(void *_dif, uint64_t guard, uint32_t offset_blocks, 675 const struct spdk_dif_ctx *ctx) 676 { 677 struct spdk_dif *dif = _dif; 678 uint64_t ref_tag; 679 680 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 681 _dif_set_guard(dif, guard, ctx->dif_pi_format); 682 } else { 683 _dif_set_guard(dif, 0, ctx->dif_pi_format); 684 } 685 686 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 687 _dif_set_apptag(dif, ctx->app_tag, ctx->dif_pi_format); 688 } else { 689 _dif_set_apptag(dif, 0, ctx->dif_pi_format); 690 } 691 692 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 693 /* For type 1 and 2, the reference tag is incremented for each 694 * subsequent logical block. For type 3, the reference tag 695 * remains the same as the initial reference tag. 696 */ 697 if (ctx->dif_type != SPDK_DIF_TYPE3) { 698 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 699 } else { 700 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 701 } 702 703 /* Overwrite reference tag if initialization reference tag is SPDK_DIF_REFTAG_IGNORE */ 704 if (ctx->init_ref_tag == SPDK_DIF_REFTAG_IGNORE) { 705 if (ctx->dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 706 ref_tag = REFTAG_MASK_16; 707 } else if (ctx->dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 708 ref_tag = REFTAG_MASK_32; 709 } else { 710 ref_tag = REFTAG_MASK_64; 711 } 712 } 713 714 _dif_set_reftag(dif, ref_tag, ctx->dif_pi_format); 715 } else { 716 _dif_set_reftag(dif, 0, ctx->dif_pi_format); 717 } 718 } 719 720 static void 721 dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 722 { 723 uint32_t offset_blocks; 724 uint8_t *buf; 725 uint64_t guard = 0; 726 727 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 728 _dif_sgl_get_buf(sgl, &buf, NULL); 729 730 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 731 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 732 } 733 734 _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); 735 736 _dif_sgl_advance(sgl, ctx->block_size); 737 } 738 } 739 740 static void 741 dif_store_split(struct _dif_sgl *sgl, struct spdk_dif *dif, 742 const struct spdk_dif_ctx *ctx) 743 { 744 uint32_t offset = 0, rest_md_len, buf_len; 745 uint8_t *buf; 746 747 rest_md_len = ctx->block_size - ctx->guard_interval; 748 749 while (offset < rest_md_len) { 750 _dif_sgl_get_buf(sgl, &buf, &buf_len); 751 752 if (offset < _dif_size(ctx->dif_pi_format)) { 753 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 754 memcpy(buf, (uint8_t *)dif + offset, buf_len); 755 } else { 756 buf_len = spdk_min(buf_len, rest_md_len - offset); 757 } 758 759 _dif_sgl_advance(sgl, buf_len); 760 offset += buf_len; 761 } 762 } 763 764 static uint64_t 765 _dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 766 uint64_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 767 { 768 struct spdk_dif dif = {}; 769 770 assert(offset_in_block < ctx->guard_interval); 771 assert(offset_in_block + data_len < ctx->guard_interval || 772 offset_in_block + data_len == ctx->block_size); 773 774 /* Compute CRC over split logical block data. */ 775 guard = dif_generate_guard_split(guard, sgl, offset_in_block, data_len, ctx); 776 777 if (offset_in_block + data_len < ctx->guard_interval) { 778 return guard; 779 } 780 781 /* If a whole logical block data is parsed, generate DIF 782 * and save it to the temporary DIF area. 783 */ 784 _dif_generate(&dif, guard, offset_blocks, ctx); 785 786 /* Copy generated DIF field to the split DIF field, and then 787 * skip metadata field after DIF field (if any). 788 */ 789 dif_store_split(sgl, &dif, ctx); 790 791 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 792 guard = ctx->guard_seed; 793 } 794 795 return guard; 796 } 797 798 static void 799 dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks, 800 const struct spdk_dif_ctx *ctx) 801 { 802 uint32_t offset_blocks; 803 uint64_t guard = 0; 804 805 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 806 guard = ctx->guard_seed; 807 } 808 809 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 810 _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx); 811 } 812 } 813 814 int 815 spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 816 const struct spdk_dif_ctx *ctx) 817 { 818 struct _dif_sgl sgl; 819 820 _dif_sgl_init(&sgl, iovs, iovcnt); 821 822 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 823 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 824 return -EINVAL; 825 } 826 827 if (_dif_is_disabled(ctx->dif_type)) { 828 return 0; 829 } 830 831 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 832 dif_generate(&sgl, num_blocks, ctx); 833 } else { 834 dif_generate_split(&sgl, num_blocks, ctx); 835 } 836 837 return 0; 838 } 839 840 static void 841 _dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, 842 uint64_t expected, uint64_t actual, uint32_t err_offset) 843 { 844 if (err_blk) { 845 err_blk->err_type = err_type; 846 err_blk->expected = expected; 847 err_blk->actual = actual; 848 err_blk->err_offset = err_offset; 849 } 850 } 851 852 static bool 853 _dif_reftag_check(struct spdk_dif *dif, const struct spdk_dif_ctx *ctx, 854 uint64_t expected_reftag, uint32_t offset_blocks, struct spdk_dif_error *err_blk) 855 { 856 uint64_t reftag; 857 858 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 859 switch (ctx->dif_type) { 860 case SPDK_DIF_TYPE1: 861 case SPDK_DIF_TYPE2: 862 /* Compare the DIF Reference Tag field to the passed Reference Tag. 863 * The passed Reference Tag will be the least significant 4 bytes 864 * or 8 bytes (depending on the PI format) 865 * of the LBA when Type 1 is used, and application specific value 866 * if Type 2 is used. 867 */ 868 if (!_dif_reftag_match(dif, expected_reftag, ctx->dif_pi_format)) { 869 reftag = _dif_get_reftag(dif, ctx->dif_pi_format); 870 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected_reftag, 871 reftag, offset_blocks); 872 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu64 "," \ 873 " Expected=%lx, Actual=%lx\n", 874 expected_reftag, expected_reftag, reftag); 875 return false; 876 } 877 break; 878 case SPDK_DIF_TYPE3: 879 /* For Type 3, computed Reference Tag remains unchanged. 880 * Hence ignore the Reference Tag field. 881 */ 882 break; 883 default: 884 break; 885 } 886 } 887 888 return true; 889 } 890 891 static int 892 _dif_verify(void *_dif, uint64_t guard, uint32_t offset_blocks, 893 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 894 { 895 struct spdk_dif *dif = _dif; 896 uint64_t _guard; 897 uint16_t _app_tag; 898 uint64_t ref_tag; 899 900 if (_dif_ignore(dif, ctx)) { 901 return 0; 902 } 903 904 /* For type 1 and 2, the reference tag is incremented for each 905 * subsequent logical block. For type 3, the reference tag 906 * remains the same as the initial reference tag. 907 */ 908 if (ctx->dif_type != SPDK_DIF_TYPE3) { 909 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 910 } else { 911 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 912 } 913 914 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 915 /* Compare the DIF Guard field to the CRC computed over the logical 916 * block data. 917 */ 918 _guard = _dif_get_guard(dif, ctx->dif_pi_format); 919 if (_guard != guard) { 920 _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, 921 offset_blocks); 922 SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu64 "," \ 923 " Expected=%lx, Actual=%lx\n", 924 ref_tag, _guard, guard); 925 return -1; 926 } 927 } 928 929 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 930 /* Compare unmasked bits in the DIF Application Tag field to the 931 * passed Application Tag. 932 */ 933 _app_tag = _dif_get_apptag(dif, ctx->dif_pi_format); 934 if ((_app_tag & ctx->apptag_mask) != (ctx->app_tag & ctx->apptag_mask)) { 935 _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, 936 (_app_tag & ctx->apptag_mask), offset_blocks); 937 SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu64 "," \ 938 " Expected=%x, Actual=%x\n", 939 ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); 940 return -1; 941 } 942 } 943 944 if (!_dif_reftag_check(dif, ctx, ref_tag, offset_blocks, err_blk)) { 945 return -1; 946 } 947 948 return 0; 949 } 950 951 static int 952 dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks, 953 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 954 { 955 uint32_t offset_blocks; 956 int rc; 957 uint8_t *buf; 958 uint64_t guard = 0; 959 960 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 961 _dif_sgl_get_buf(sgl, &buf, NULL); 962 963 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 964 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 965 } 966 967 rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 968 if (rc != 0) { 969 return rc; 970 } 971 972 _dif_sgl_advance(sgl, ctx->block_size); 973 } 974 975 return 0; 976 } 977 978 static void 979 dif_load_split(struct _dif_sgl *sgl, struct spdk_dif *dif, 980 const struct spdk_dif_ctx *ctx) 981 { 982 uint32_t offset = 0, rest_md_len, buf_len; 983 uint8_t *buf; 984 985 rest_md_len = ctx->block_size - ctx->guard_interval; 986 987 while (offset < rest_md_len) { 988 _dif_sgl_get_buf(sgl, &buf, &buf_len); 989 990 if (offset < _dif_size(ctx->dif_pi_format)) { 991 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 992 memcpy((uint8_t *)dif + offset, buf, buf_len); 993 } else { 994 buf_len = spdk_min(buf_len, rest_md_len - offset); 995 } 996 997 _dif_sgl_advance(sgl, buf_len); 998 offset += buf_len; 999 } 1000 } 1001 1002 static int 1003 _dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 1004 uint64_t *_guard, uint32_t offset_blocks, 1005 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 1006 { 1007 uint64_t guard = *_guard; 1008 struct spdk_dif dif = {}; 1009 int rc; 1010 1011 assert(_guard != NULL); 1012 assert(offset_in_block < ctx->guard_interval); 1013 assert(offset_in_block + data_len < ctx->guard_interval || 1014 offset_in_block + data_len == ctx->block_size); 1015 1016 guard = dif_generate_guard_split(guard, sgl, offset_in_block, data_len, ctx); 1017 1018 if (offset_in_block + data_len < ctx->guard_interval) { 1019 *_guard = guard; 1020 return 0; 1021 } 1022 1023 dif_load_split(sgl, &dif, ctx); 1024 1025 rc = _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 1026 if (rc != 0) { 1027 return rc; 1028 } 1029 1030 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1031 guard = ctx->guard_seed; 1032 } 1033 1034 *_guard = guard; 1035 return 0; 1036 } 1037 1038 static int 1039 dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks, 1040 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 1041 { 1042 uint32_t offset_blocks; 1043 uint64_t guard = 0; 1044 int rc; 1045 1046 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1047 guard = ctx->guard_seed; 1048 } 1049 1050 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1051 rc = _dif_verify_split(sgl, 0, ctx->block_size, &guard, offset_blocks, 1052 ctx, err_blk); 1053 if (rc != 0) { 1054 return rc; 1055 } 1056 } 1057 1058 return 0; 1059 } 1060 1061 int 1062 spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1063 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 1064 { 1065 struct _dif_sgl sgl; 1066 1067 _dif_sgl_init(&sgl, iovs, iovcnt); 1068 1069 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1070 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1071 return -EINVAL; 1072 } 1073 1074 if (_dif_is_disabled(ctx->dif_type)) { 1075 return 0; 1076 } 1077 1078 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 1079 return dif_verify(&sgl, num_blocks, ctx, err_blk); 1080 } else { 1081 return dif_verify_split(&sgl, num_blocks, ctx, err_blk); 1082 } 1083 } 1084 1085 static uint32_t 1086 dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks, 1087 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1088 { 1089 uint32_t offset_blocks; 1090 uint8_t *buf; 1091 1092 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1093 _dif_sgl_get_buf(sgl, &buf, NULL); 1094 1095 crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c); 1096 1097 _dif_sgl_advance(sgl, ctx->block_size); 1098 } 1099 1100 return crc32c; 1101 } 1102 1103 static uint32_t 1104 _dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 1105 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1106 { 1107 uint32_t data_block_size, buf_len; 1108 uint8_t *buf; 1109 1110 data_block_size = ctx->block_size - ctx->md_size; 1111 1112 assert(offset_in_block + data_len <= ctx->block_size); 1113 1114 while (data_len != 0) { 1115 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1116 buf_len = spdk_min(buf_len, data_len); 1117 1118 if (offset_in_block < data_block_size) { 1119 buf_len = spdk_min(buf_len, data_block_size - offset_in_block); 1120 crc32c = spdk_crc32c_update(buf, buf_len, crc32c); 1121 } 1122 1123 _dif_sgl_advance(sgl, buf_len); 1124 offset_in_block += buf_len; 1125 data_len -= buf_len; 1126 } 1127 1128 return crc32c; 1129 } 1130 1131 static uint32_t 1132 dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks, 1133 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1134 { 1135 uint32_t offset_blocks; 1136 1137 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1138 crc32c = _dif_update_crc32c_split(sgl, 0, ctx->block_size, crc32c, ctx); 1139 } 1140 1141 return crc32c; 1142 } 1143 1144 int 1145 spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1146 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 1147 { 1148 struct _dif_sgl sgl; 1149 1150 if (_crc32c == NULL) { 1151 return -EINVAL; 1152 } 1153 1154 _dif_sgl_init(&sgl, iovs, iovcnt); 1155 1156 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1157 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1158 return -EINVAL; 1159 } 1160 1161 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 1162 *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx); 1163 } else { 1164 *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx); 1165 } 1166 1167 return 0; 1168 } 1169 1170 static void 1171 _dif_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1172 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1173 { 1174 uint32_t data_block_size; 1175 uint8_t *src, *dst; 1176 uint64_t guard = 0; 1177 1178 data_block_size = ctx->block_size - ctx->md_size; 1179 1180 _dif_sgl_get_buf(src_sgl, &src, NULL); 1181 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1182 1183 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1184 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1185 ctx->dif_pi_format); 1186 guard = _dif_generate_guard(guard, dst + data_block_size, 1187 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1188 } else { 1189 memcpy(dst, src, data_block_size); 1190 } 1191 1192 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 1193 1194 _dif_sgl_advance(src_sgl, data_block_size); 1195 _dif_sgl_advance(dst_sgl, ctx->block_size); 1196 } 1197 1198 static void 1199 dif_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1200 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1201 { 1202 uint32_t offset_blocks; 1203 1204 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1205 _dif_insert_copy(src_sgl, dst_sgl, offset_blocks, ctx); 1206 } 1207 } 1208 1209 static void 1210 _dif_insert_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1211 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1212 { 1213 uint32_t data_block_size; 1214 uint64_t guard = 0; 1215 struct spdk_dif dif = {}; 1216 1217 data_block_size = ctx->block_size - ctx->md_size; 1218 1219 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1220 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1221 data_block_size, ctx->dif_pi_format); 1222 guard = dif_generate_guard_split(guard, dst_sgl, data_block_size, 1223 ctx->guard_interval - data_block_size, ctx); 1224 } else { 1225 _data_copy_split(dst_sgl, src_sgl, data_block_size); 1226 _dif_sgl_advance(dst_sgl, ctx->guard_interval - data_block_size); 1227 } 1228 1229 _dif_generate(&dif, guard, offset_blocks, ctx); 1230 1231 dif_store_split(dst_sgl, &dif, ctx); 1232 } 1233 1234 static void 1235 dif_insert_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1236 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1237 { 1238 uint32_t offset_blocks; 1239 1240 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1241 _dif_insert_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 1242 } 1243 } 1244 1245 static void 1246 _dif_disable_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1247 const struct spdk_dif_ctx *ctx) 1248 { 1249 uint32_t offset = 0, src_len, dst_len, buf_len, data_block_size; 1250 uint8_t *src, *dst; 1251 1252 data_block_size = ctx->block_size - ctx->md_size; 1253 1254 while (offset < data_block_size) { 1255 _dif_sgl_get_buf(src_sgl, &src, &src_len); 1256 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 1257 buf_len = spdk_min(src_len, dst_len); 1258 buf_len = spdk_min(buf_len, data_block_size - offset); 1259 1260 memcpy(dst, src, buf_len); 1261 1262 _dif_sgl_advance(src_sgl, buf_len); 1263 _dif_sgl_advance(dst_sgl, buf_len); 1264 offset += buf_len; 1265 } 1266 1267 _dif_sgl_advance(dst_sgl, ctx->md_size); 1268 } 1269 1270 static void 1271 dif_disable_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1272 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1273 { 1274 uint32_t offset_blocks; 1275 1276 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1277 _dif_disable_insert_copy(src_sgl, dst_sgl, ctx); 1278 } 1279 } 1280 1281 static int 1282 _spdk_dif_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1283 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1284 { 1285 uint32_t data_block_size; 1286 1287 data_block_size = ctx->block_size - ctx->md_size; 1288 1289 if (!_dif_sgl_is_valid(src_sgl, data_block_size * num_blocks) || 1290 !_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks)) { 1291 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 1292 return -EINVAL; 1293 } 1294 1295 if (_dif_is_disabled(ctx->dif_type)) { 1296 dif_disable_insert_copy(src_sgl, dst_sgl, num_blocks, ctx); 1297 return 0; 1298 } 1299 1300 if (_dif_sgl_is_bytes_multiple(src_sgl, data_block_size) && 1301 _dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size)) { 1302 dif_insert_copy(src_sgl, dst_sgl, num_blocks, ctx); 1303 } else { 1304 dif_insert_copy_split(src_sgl, dst_sgl, num_blocks, ctx); 1305 } 1306 1307 return 0; 1308 } 1309 1310 static void 1311 _dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1312 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1313 { 1314 uint8_t *src, *dst; 1315 uint64_t guard = 0; 1316 1317 _dif_sgl_get_buf(src_sgl, &src, NULL); 1318 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1319 1320 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1321 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, ctx->guard_interval, 1322 ctx->dif_pi_format); 1323 } else { 1324 memcpy(dst, src, ctx->guard_interval); 1325 } 1326 1327 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 1328 1329 _dif_sgl_advance(src_sgl, ctx->block_size); 1330 _dif_sgl_advance(dst_sgl, ctx->block_size); 1331 } 1332 1333 static void 1334 dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1335 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1336 { 1337 uint32_t offset_blocks; 1338 1339 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1340 _dif_overwrite_copy(src_sgl, dst_sgl, offset_blocks, ctx); 1341 } 1342 } 1343 1344 static void 1345 _dif_overwrite_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1346 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1347 { 1348 uint64_t guard = 0; 1349 struct spdk_dif dif = {}; 1350 1351 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1352 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1353 ctx->guard_interval, ctx->dif_pi_format); 1354 } else { 1355 _data_copy_split(dst_sgl, src_sgl, ctx->guard_interval); 1356 } 1357 1358 _dif_sgl_advance(src_sgl, ctx->block_size - ctx->guard_interval); 1359 1360 _dif_generate(&dif, guard, offset_blocks, ctx); 1361 dif_store_split(dst_sgl, &dif, ctx); 1362 } 1363 1364 static void 1365 dif_overwrite_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1366 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1367 { 1368 uint32_t offset_blocks; 1369 1370 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1371 _dif_overwrite_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 1372 } 1373 } 1374 1375 static void 1376 dif_disable_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1377 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1378 { 1379 _data_copy_split(dst_sgl, src_sgl, ctx->block_size * num_blocks); 1380 } 1381 1382 static int 1383 _spdk_dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1384 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1385 { 1386 if (!_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks) || 1387 !_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks)) { 1388 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 1389 return -EINVAL; 1390 } 1391 1392 if (_dif_is_disabled(ctx->dif_type)) { 1393 dif_disable_copy(src_sgl, dst_sgl, num_blocks, ctx); 1394 return 0; 1395 } 1396 1397 if (_dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size) && 1398 _dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size)) { 1399 dif_overwrite_copy(src_sgl, dst_sgl, num_blocks, ctx); 1400 } else { 1401 dif_overwrite_copy_split(src_sgl, dst_sgl, num_blocks, ctx); 1402 } 1403 1404 return 0; 1405 } 1406 1407 int 1408 spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1409 int bounce_iovcnt, uint32_t num_blocks, 1410 const struct spdk_dif_ctx *ctx) 1411 { 1412 struct _dif_sgl src_sgl, dst_sgl; 1413 1414 _dif_sgl_init(&src_sgl, iovs, iovcnt); 1415 _dif_sgl_init(&dst_sgl, bounce_iovs, bounce_iovcnt); 1416 1417 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_NVME_PRACT) || 1418 ctx->md_size == _dif_size(ctx->dif_pi_format)) { 1419 return _spdk_dif_insert_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 1420 } else { 1421 return _spdk_dif_overwrite_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 1422 } 1423 } 1424 1425 static int 1426 _dif_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1427 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1428 struct spdk_dif_error *err_blk) 1429 { 1430 uint32_t data_block_size; 1431 uint8_t *src, *dst; 1432 int rc; 1433 uint64_t guard = 0; 1434 1435 data_block_size = ctx->block_size - ctx->md_size; 1436 1437 _dif_sgl_get_buf(src_sgl, &src, NULL); 1438 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1439 1440 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1441 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1442 ctx->dif_pi_format); 1443 guard = _dif_generate_guard(guard, src + data_block_size, 1444 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1445 } else { 1446 memcpy(dst, src, data_block_size); 1447 } 1448 1449 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1450 if (rc != 0) { 1451 return rc; 1452 } 1453 1454 _dif_sgl_advance(src_sgl, ctx->block_size); 1455 _dif_sgl_advance(dst_sgl, data_block_size); 1456 1457 return 0; 1458 } 1459 1460 static int 1461 dif_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1462 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1463 struct spdk_dif_error *err_blk) 1464 { 1465 uint32_t offset_blocks; 1466 int rc; 1467 1468 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1469 rc = _dif_strip_copy(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1470 if (rc != 0) { 1471 return rc; 1472 } 1473 } 1474 1475 return 0; 1476 } 1477 1478 static int 1479 _dif_strip_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1480 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1481 struct spdk_dif_error *err_blk) 1482 { 1483 uint32_t data_block_size; 1484 uint64_t guard = 0; 1485 struct spdk_dif dif = {}; 1486 1487 data_block_size = ctx->block_size - ctx->md_size; 1488 1489 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1490 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1491 data_block_size, ctx->dif_pi_format); 1492 guard = dif_generate_guard_split(guard, src_sgl, data_block_size, 1493 ctx->guard_interval - data_block_size, ctx); 1494 } else { 1495 _data_copy_split(dst_sgl, src_sgl, data_block_size); 1496 _dif_sgl_advance(src_sgl, ctx->guard_interval - data_block_size); 1497 } 1498 1499 dif_load_split(src_sgl, &dif, ctx); 1500 1501 return _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 1502 } 1503 1504 static int 1505 dif_strip_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1506 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1507 struct spdk_dif_error *err_blk) 1508 { 1509 uint32_t offset_blocks; 1510 int rc; 1511 1512 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1513 rc = _dif_strip_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1514 if (rc != 0) { 1515 return rc; 1516 } 1517 } 1518 1519 return 0; 1520 } 1521 1522 static void 1523 _dif_disable_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1524 const struct spdk_dif_ctx *ctx) 1525 { 1526 uint32_t offset = 0, src_len, dst_len, buf_len, data_block_size; 1527 uint8_t *src, *dst; 1528 1529 data_block_size = ctx->block_size - ctx->md_size; 1530 1531 while (offset < data_block_size) { 1532 _dif_sgl_get_buf(src_sgl, &src, &src_len); 1533 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 1534 buf_len = spdk_min(src_len, dst_len); 1535 buf_len = spdk_min(buf_len, data_block_size - offset); 1536 1537 memcpy(dst, src, buf_len); 1538 1539 _dif_sgl_advance(src_sgl, buf_len); 1540 _dif_sgl_advance(dst_sgl, buf_len); 1541 offset += buf_len; 1542 } 1543 1544 _dif_sgl_advance(src_sgl, ctx->md_size); 1545 } 1546 1547 static void 1548 dif_disable_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1549 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1550 { 1551 uint32_t offset_blocks; 1552 1553 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1554 _dif_disable_strip_copy(src_sgl, dst_sgl, ctx); 1555 } 1556 } 1557 1558 static int 1559 _spdk_dif_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1560 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1561 struct spdk_dif_error *err_blk) 1562 { 1563 uint32_t data_block_size; 1564 1565 data_block_size = ctx->block_size - ctx->md_size; 1566 1567 if (!_dif_sgl_is_valid(dst_sgl, data_block_size * num_blocks) || 1568 !_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks)) { 1569 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1570 return -EINVAL; 1571 } 1572 1573 if (_dif_is_disabled(ctx->dif_type)) { 1574 dif_disable_strip_copy(src_sgl, dst_sgl, num_blocks, ctx); 1575 return 0; 1576 } 1577 1578 if (_dif_sgl_is_bytes_multiple(dst_sgl, data_block_size) && 1579 _dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size)) { 1580 return dif_strip_copy(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1581 } else { 1582 return dif_strip_copy_split(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1583 } 1584 } 1585 1586 static int 1587 _dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1588 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1589 struct spdk_dif_error *err_blk) 1590 { 1591 uint8_t *src, *dst; 1592 int rc; 1593 uint64_t guard = 0; 1594 1595 _dif_sgl_get_buf(src_sgl, &src, NULL); 1596 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1597 1598 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1599 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, ctx->guard_interval, 1600 ctx->dif_pi_format); 1601 } else { 1602 memcpy(dst, src, ctx->guard_interval); 1603 } 1604 1605 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1606 if (rc != 0) { 1607 return rc; 1608 } 1609 1610 _dif_sgl_advance(src_sgl, ctx->block_size); 1611 _dif_sgl_advance(dst_sgl, ctx->block_size); 1612 1613 return 0; 1614 } 1615 1616 static int 1617 dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1618 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1619 struct spdk_dif_error *err_blk) 1620 { 1621 uint32_t offset_blocks; 1622 int rc; 1623 1624 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1625 rc = _dif_verify_copy(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1626 if (rc != 0) { 1627 return rc; 1628 } 1629 } 1630 1631 return 0; 1632 } 1633 1634 static int 1635 _dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1636 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1637 struct spdk_dif_error *err_blk) 1638 { 1639 uint64_t guard = 0; 1640 struct spdk_dif dif = {}; 1641 1642 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1643 guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl, 1644 ctx->guard_interval, ctx->dif_pi_format); 1645 } else { 1646 _data_copy_split(dst_sgl, src_sgl, ctx->guard_interval); 1647 } 1648 1649 dif_load_split(src_sgl, &dif, ctx); 1650 _dif_sgl_advance(dst_sgl, ctx->block_size - ctx->guard_interval); 1651 1652 return _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 1653 } 1654 1655 static int 1656 dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1657 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1658 struct spdk_dif_error *err_blk) 1659 { 1660 uint32_t offset_blocks; 1661 int rc; 1662 1663 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1664 rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1665 if (rc != 0) { 1666 return rc; 1667 } 1668 } 1669 1670 return 0; 1671 } 1672 1673 static int 1674 _spdk_dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1675 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1676 struct spdk_dif_error *err_blk) 1677 { 1678 if (!_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks) || 1679 !_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks)) { 1680 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1681 return -EINVAL; 1682 } 1683 1684 if (_dif_is_disabled(ctx->dif_type)) { 1685 dif_disable_copy(src_sgl, dst_sgl, num_blocks, ctx); 1686 return 0; 1687 } 1688 1689 if (_dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size) && 1690 _dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size)) { 1691 return dif_verify_copy(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1692 } else { 1693 return dif_verify_copy_split(src_sgl, dst_sgl, num_blocks, ctx, err_blk); 1694 } 1695 } 1696 1697 int 1698 spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1699 int bounce_iovcnt, uint32_t num_blocks, 1700 const struct spdk_dif_ctx *ctx, 1701 struct spdk_dif_error *err_blk) 1702 { 1703 struct _dif_sgl src_sgl, dst_sgl; 1704 1705 _dif_sgl_init(&src_sgl, bounce_iovs, bounce_iovcnt); 1706 _dif_sgl_init(&dst_sgl, iovs, iovcnt); 1707 1708 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_NVME_PRACT) || 1709 ctx->md_size == _dif_size(ctx->dif_pi_format)) { 1710 return _spdk_dif_strip_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1711 } else { 1712 return _spdk_dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1713 } 1714 } 1715 1716 static void 1717 _bit_flip(uint8_t *buf, uint32_t flip_bit) 1718 { 1719 uint8_t byte; 1720 1721 byte = *buf; 1722 byte ^= 1 << flip_bit; 1723 *buf = byte; 1724 } 1725 1726 static int 1727 _dif_inject_error(struct _dif_sgl *sgl, 1728 uint32_t block_size, uint32_t num_blocks, 1729 uint32_t inject_offset_blocks, 1730 uint32_t inject_offset_bytes, 1731 uint32_t inject_offset_bits) 1732 { 1733 uint32_t offset_in_block, buf_len; 1734 uint8_t *buf; 1735 1736 _dif_sgl_advance(sgl, block_size * inject_offset_blocks); 1737 1738 offset_in_block = 0; 1739 1740 while (offset_in_block < block_size) { 1741 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1742 buf_len = spdk_min(buf_len, block_size - offset_in_block); 1743 1744 if (inject_offset_bytes >= offset_in_block && 1745 inject_offset_bytes < offset_in_block + buf_len) { 1746 buf += inject_offset_bytes - offset_in_block; 1747 _bit_flip(buf, inject_offset_bits); 1748 return 0; 1749 } 1750 1751 _dif_sgl_advance(sgl, buf_len); 1752 offset_in_block += buf_len; 1753 } 1754 1755 return -1; 1756 } 1757 1758 static int 1759 dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks, 1760 uint32_t start_inject_bytes, uint32_t inject_range_bytes, 1761 uint32_t *inject_offset) 1762 { 1763 uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; 1764 uint32_t offset_blocks; 1765 int rc; 1766 1767 srand(time(0)); 1768 1769 inject_offset_blocks = rand() % num_blocks; 1770 inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); 1771 inject_offset_bits = rand() % 8; 1772 1773 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1774 if (offset_blocks == inject_offset_blocks) { 1775 rc = _dif_inject_error(sgl, block_size, num_blocks, 1776 inject_offset_blocks, 1777 inject_offset_bytes, 1778 inject_offset_bits); 1779 if (rc == 0) { 1780 *inject_offset = inject_offset_blocks; 1781 } 1782 return rc; 1783 } 1784 } 1785 1786 return -1; 1787 } 1788 1789 int 1790 spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1791 const struct spdk_dif_ctx *ctx, uint32_t inject_flags, 1792 uint32_t *inject_offset) 1793 { 1794 struct _dif_sgl sgl; 1795 int rc; 1796 1797 _dif_sgl_init(&sgl, iovs, iovcnt); 1798 1799 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1800 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1801 return -EINVAL; 1802 } 1803 1804 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1805 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1806 ctx->guard_interval + _dif_reftag_offset(ctx->dif_pi_format), 1807 _dif_reftag_size(ctx->dif_pi_format), 1808 inject_offset); 1809 if (rc != 0) { 1810 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1811 return rc; 1812 } 1813 } 1814 1815 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1816 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1817 ctx->guard_interval + _dif_apptag_offset(ctx->dif_pi_format), 1818 _dif_apptag_size(), 1819 inject_offset); 1820 if (rc != 0) { 1821 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1822 return rc; 1823 } 1824 } 1825 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1826 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1827 ctx->guard_interval, 1828 _dif_guard_size(ctx->dif_pi_format), 1829 inject_offset); 1830 if (rc != 0) { 1831 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1832 return rc; 1833 } 1834 } 1835 1836 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1837 /* If the DIF information is contained within the last 8/16 bytes of 1838 * metadata (depending on the PI format), then the CRC covers all metadata 1839 * bytes up to but excluding the last 8/16 bytes. But error injection does not 1840 * cover these metadata because classification is not determined yet. 1841 * 1842 * Note: Error injection to data block is expected to be detected as 1843 * guard error. 1844 */ 1845 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1846 0, 1847 ctx->block_size - ctx->md_size, 1848 inject_offset); 1849 if (rc != 0) { 1850 SPDK_ERRLOG("Failed to inject error to data block.\n"); 1851 return rc; 1852 } 1853 } 1854 1855 return 0; 1856 } 1857 1858 static void 1859 dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1860 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1861 { 1862 uint32_t offset_blocks = 0; 1863 uint8_t *data_buf, *md_buf; 1864 uint64_t guard; 1865 1866 while (offset_blocks < num_blocks) { 1867 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1868 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1869 1870 guard = 0; 1871 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1872 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1873 ctx->dif_pi_format); 1874 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1875 ctx->dif_pi_format); 1876 } 1877 1878 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1879 1880 _dif_sgl_advance(data_sgl, ctx->block_size); 1881 _dif_sgl_advance(md_sgl, ctx->md_size); 1882 offset_blocks++; 1883 } 1884 } 1885 1886 static void 1887 _dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1888 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1889 { 1890 uint32_t offset_in_block, data_buf_len; 1891 uint8_t *data_buf, *md_buf; 1892 uint64_t guard = 0; 1893 1894 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1895 1896 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1897 guard = ctx->guard_seed; 1898 } 1899 offset_in_block = 0; 1900 1901 while (offset_in_block < ctx->block_size) { 1902 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1903 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1904 1905 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1906 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 1907 ctx->dif_pi_format); 1908 } 1909 1910 _dif_sgl_advance(data_sgl, data_buf_len); 1911 offset_in_block += data_buf_len; 1912 } 1913 1914 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1915 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1916 ctx->dif_pi_format); 1917 } 1918 1919 _dif_sgl_advance(md_sgl, ctx->md_size); 1920 1921 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1922 } 1923 1924 static void 1925 dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1926 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1927 { 1928 uint32_t offset_blocks; 1929 1930 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1931 _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx); 1932 } 1933 } 1934 1935 int 1936 spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1937 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1938 { 1939 struct _dif_sgl data_sgl, md_sgl; 1940 1941 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1942 _dif_sgl_init(&md_sgl, md_iov, 1); 1943 1944 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1945 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1946 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1947 return -EINVAL; 1948 } 1949 1950 if (_dif_is_disabled(ctx->dif_type)) { 1951 return 0; 1952 } 1953 1954 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1955 dix_generate(&data_sgl, &md_sgl, num_blocks, ctx); 1956 } else { 1957 dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx); 1958 } 1959 1960 return 0; 1961 } 1962 1963 static int 1964 dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1965 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1966 struct spdk_dif_error *err_blk) 1967 { 1968 uint32_t offset_blocks = 0; 1969 uint8_t *data_buf, *md_buf; 1970 uint64_t guard; 1971 int rc; 1972 1973 while (offset_blocks < num_blocks) { 1974 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1975 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1976 1977 guard = 0; 1978 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1979 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1980 ctx->dif_pi_format); 1981 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1982 ctx->dif_pi_format); 1983 } 1984 1985 rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1986 if (rc != 0) { 1987 return rc; 1988 } 1989 1990 _dif_sgl_advance(data_sgl, ctx->block_size); 1991 _dif_sgl_advance(md_sgl, ctx->md_size); 1992 offset_blocks++; 1993 } 1994 1995 return 0; 1996 } 1997 1998 static int 1999 _dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 2000 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 2001 struct spdk_dif_error *err_blk) 2002 { 2003 uint32_t offset_in_block, data_buf_len; 2004 uint8_t *data_buf, *md_buf; 2005 uint64_t guard = 0; 2006 2007 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 2008 2009 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2010 guard = ctx->guard_seed; 2011 } 2012 offset_in_block = 0; 2013 2014 while (offset_in_block < ctx->block_size) { 2015 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 2016 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 2017 2018 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2019 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 2020 ctx->dif_pi_format); 2021 } 2022 2023 _dif_sgl_advance(data_sgl, data_buf_len); 2024 offset_in_block += data_buf_len; 2025 } 2026 2027 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2028 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 2029 ctx->dif_pi_format); 2030 } 2031 2032 _dif_sgl_advance(md_sgl, ctx->md_size); 2033 2034 return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 2035 } 2036 2037 static int 2038 dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 2039 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 2040 struct spdk_dif_error *err_blk) 2041 { 2042 uint32_t offset_blocks; 2043 int rc; 2044 2045 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2046 rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk); 2047 if (rc != 0) { 2048 return rc; 2049 } 2050 } 2051 2052 return 0; 2053 } 2054 2055 int 2056 spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 2057 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 2058 struct spdk_dif_error *err_blk) 2059 { 2060 struct _dif_sgl data_sgl, md_sgl; 2061 2062 if (md_iov->iov_base == NULL) { 2063 SPDK_ERRLOG("Metadata buffer is NULL.\n"); 2064 return -EINVAL; 2065 } 2066 2067 _dif_sgl_init(&data_sgl, iovs, iovcnt); 2068 _dif_sgl_init(&md_sgl, md_iov, 1); 2069 2070 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 2071 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2072 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2073 return -EINVAL; 2074 } 2075 2076 if (_dif_is_disabled(ctx->dif_type)) { 2077 return 0; 2078 } 2079 2080 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 2081 return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 2082 } else { 2083 return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 2084 } 2085 } 2086 2087 int 2088 spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 2089 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 2090 uint32_t inject_flags, uint32_t *inject_offset) 2091 { 2092 struct _dif_sgl data_sgl, md_sgl; 2093 int rc; 2094 2095 _dif_sgl_init(&data_sgl, iovs, iovcnt); 2096 _dif_sgl_init(&md_sgl, md_iov, 1); 2097 2098 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 2099 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2100 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2101 return -EINVAL; 2102 } 2103 2104 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 2105 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 2106 ctx->guard_interval + _dif_reftag_offset(ctx->dif_pi_format), 2107 _dif_reftag_size(ctx->dif_pi_format), 2108 inject_offset); 2109 if (rc != 0) { 2110 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 2111 return rc; 2112 } 2113 } 2114 2115 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 2116 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 2117 ctx->guard_interval + _dif_apptag_offset(ctx->dif_pi_format), 2118 _dif_apptag_size(), 2119 inject_offset); 2120 if (rc != 0) { 2121 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 2122 return rc; 2123 } 2124 } 2125 2126 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 2127 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 2128 ctx->guard_interval, 2129 _dif_guard_size(ctx->dif_pi_format), 2130 inject_offset); 2131 if (rc != 0) { 2132 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 2133 return rc; 2134 } 2135 } 2136 2137 if (inject_flags & SPDK_DIF_DATA_ERROR) { 2138 /* Note: Error injection to data block is expected to be detected 2139 * as guard error. 2140 */ 2141 rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks, 2142 0, 2143 ctx->block_size, 2144 inject_offset); 2145 if (rc != 0) { 2146 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 2147 return rc; 2148 } 2149 } 2150 2151 return 0; 2152 } 2153 2154 static uint32_t 2155 _to_next_boundary(uint32_t offset, uint32_t boundary) 2156 { 2157 return boundary - (offset % boundary); 2158 } 2159 2160 static uint32_t 2161 _to_size_with_md(uint32_t size, uint32_t data_block_size, uint32_t block_size) 2162 { 2163 return (size / data_block_size) * block_size + (size % data_block_size); 2164 } 2165 2166 int 2167 spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt, 2168 struct iovec *buf_iovs, int buf_iovcnt, 2169 uint32_t data_offset, uint32_t data_len, 2170 uint32_t *_mapped_len, 2171 const struct spdk_dif_ctx *ctx) 2172 { 2173 uint32_t data_block_size, data_unalign, buf_len, buf_offset, len; 2174 struct _dif_sgl dif_sgl; 2175 struct _dif_sgl buf_sgl; 2176 2177 if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) { 2178 return -EINVAL; 2179 } 2180 2181 data_block_size = ctx->block_size - ctx->md_size; 2182 2183 data_unalign = ctx->data_offset % data_block_size; 2184 2185 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 2186 ctx->block_size); 2187 buf_len -= data_unalign; 2188 2189 _dif_sgl_init(&dif_sgl, iovs, iovcnt); 2190 _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt); 2191 2192 if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) { 2193 SPDK_ERRLOG("Buffer overflow will occur.\n"); 2194 return -ERANGE; 2195 } 2196 2197 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 2198 buf_offset -= data_unalign; 2199 2200 _dif_sgl_advance(&buf_sgl, buf_offset); 2201 2202 while (data_len != 0) { 2203 len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size)); 2204 if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) { 2205 break; 2206 } 2207 _dif_sgl_advance(&buf_sgl, ctx->md_size); 2208 data_offset += len; 2209 data_len -= len; 2210 } 2211 2212 if (_mapped_len != NULL) { 2213 *_mapped_len = dif_sgl.total_size; 2214 } 2215 2216 return iovcnt - dif_sgl.iovcnt; 2217 } 2218 2219 static int 2220 _dif_sgl_setup_stream(struct _dif_sgl *sgl, uint32_t *_buf_offset, uint32_t *_buf_len, 2221 uint32_t data_offset, uint32_t data_len, 2222 const struct spdk_dif_ctx *ctx) 2223 { 2224 uint32_t data_block_size, data_unalign, buf_len, buf_offset; 2225 2226 data_block_size = ctx->block_size - ctx->md_size; 2227 2228 data_unalign = ctx->data_offset % data_block_size; 2229 2230 /* If the last data block is complete, DIF of the data block is 2231 * inserted or verified in this turn. 2232 */ 2233 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 2234 ctx->block_size); 2235 buf_len -= data_unalign; 2236 2237 if (!_dif_sgl_is_valid(sgl, buf_len)) { 2238 return -ERANGE; 2239 } 2240 2241 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 2242 buf_offset -= data_unalign; 2243 2244 _dif_sgl_advance(sgl, buf_offset); 2245 buf_len -= buf_offset; 2246 2247 buf_offset += data_unalign; 2248 2249 *_buf_offset = buf_offset; 2250 *_buf_len = buf_len; 2251 2252 return 0; 2253 } 2254 2255 int 2256 spdk_dif_generate_stream(struct iovec *iovs, int iovcnt, 2257 uint32_t data_offset, uint32_t data_len, 2258 struct spdk_dif_ctx *ctx) 2259 { 2260 uint32_t buf_len = 0, buf_offset = 0; 2261 uint32_t len, offset_in_block, offset_blocks; 2262 uint64_t guard = 0; 2263 struct _dif_sgl sgl; 2264 int rc; 2265 2266 if (iovs == NULL || iovcnt == 0) { 2267 return -EINVAL; 2268 } 2269 2270 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2271 guard = ctx->last_guard; 2272 } 2273 2274 _dif_sgl_init(&sgl, iovs, iovcnt); 2275 2276 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 2277 if (rc != 0) { 2278 return rc; 2279 } 2280 2281 while (buf_len != 0) { 2282 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 2283 offset_in_block = buf_offset % ctx->block_size; 2284 offset_blocks = buf_offset / ctx->block_size; 2285 2286 guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx); 2287 2288 buf_len -= len; 2289 buf_offset += len; 2290 } 2291 2292 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2293 ctx->last_guard = guard; 2294 } 2295 2296 return 0; 2297 } 2298 2299 int 2300 spdk_dif_verify_stream(struct iovec *iovs, int iovcnt, 2301 uint32_t data_offset, uint32_t data_len, 2302 struct spdk_dif_ctx *ctx, 2303 struct spdk_dif_error *err_blk) 2304 { 2305 uint32_t buf_len = 0, buf_offset = 0; 2306 uint32_t len, offset_in_block, offset_blocks; 2307 uint64_t guard = 0; 2308 struct _dif_sgl sgl; 2309 int rc = 0; 2310 2311 if (iovs == NULL || iovcnt == 0) { 2312 return -EINVAL; 2313 } 2314 2315 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2316 guard = ctx->last_guard; 2317 } 2318 2319 _dif_sgl_init(&sgl, iovs, iovcnt); 2320 2321 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 2322 if (rc != 0) { 2323 return rc; 2324 } 2325 2326 while (buf_len != 0) { 2327 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 2328 offset_in_block = buf_offset % ctx->block_size; 2329 offset_blocks = buf_offset / ctx->block_size; 2330 2331 rc = _dif_verify_split(&sgl, offset_in_block, len, &guard, offset_blocks, 2332 ctx, err_blk); 2333 if (rc != 0) { 2334 goto error; 2335 } 2336 2337 buf_len -= len; 2338 buf_offset += len; 2339 } 2340 2341 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 2342 ctx->last_guard = guard; 2343 } 2344 error: 2345 return rc; 2346 } 2347 2348 int 2349 spdk_dif_update_crc32c_stream(struct iovec *iovs, int iovcnt, 2350 uint32_t data_offset, uint32_t data_len, 2351 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 2352 { 2353 uint32_t buf_len = 0, buf_offset = 0, len, offset_in_block; 2354 uint32_t crc32c; 2355 struct _dif_sgl sgl; 2356 int rc; 2357 2358 if (iovs == NULL || iovcnt == 0) { 2359 return -EINVAL; 2360 } 2361 2362 crc32c = *_crc32c; 2363 _dif_sgl_init(&sgl, iovs, iovcnt); 2364 2365 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 2366 if (rc != 0) { 2367 return rc; 2368 } 2369 2370 while (buf_len != 0) { 2371 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 2372 offset_in_block = buf_offset % ctx->block_size; 2373 2374 crc32c = _dif_update_crc32c_split(&sgl, offset_in_block, len, crc32c, ctx); 2375 2376 buf_len -= len; 2377 buf_offset += len; 2378 } 2379 2380 *_crc32c = crc32c; 2381 2382 return 0; 2383 } 2384 2385 void 2386 spdk_dif_get_range_with_md(uint32_t data_offset, uint32_t data_len, 2387 uint32_t *_buf_offset, uint32_t *_buf_len, 2388 const struct spdk_dif_ctx *ctx) 2389 { 2390 uint32_t data_block_size, data_unalign, buf_offset, buf_len; 2391 2392 if (!ctx->md_interleave) { 2393 buf_offset = data_offset; 2394 buf_len = data_len; 2395 } else { 2396 data_block_size = ctx->block_size - ctx->md_size; 2397 2398 data_unalign = data_offset % data_block_size; 2399 2400 buf_offset = _to_size_with_md(data_offset, data_block_size, ctx->block_size); 2401 buf_len = _to_size_with_md(data_unalign + data_len, data_block_size, ctx->block_size) - 2402 data_unalign; 2403 } 2404 2405 if (_buf_offset != NULL) { 2406 *_buf_offset = buf_offset; 2407 } 2408 2409 if (_buf_len != NULL) { 2410 *_buf_len = buf_len; 2411 } 2412 } 2413 2414 uint32_t 2415 spdk_dif_get_length_with_md(uint32_t data_len, const struct spdk_dif_ctx *ctx) 2416 { 2417 uint32_t data_block_size; 2418 2419 if (!ctx->md_interleave) { 2420 return data_len; 2421 } else { 2422 data_block_size = ctx->block_size - ctx->md_size; 2423 2424 return _to_size_with_md(data_len, data_block_size, ctx->block_size); 2425 } 2426 } 2427 2428 static int 2429 _dif_remap_ref_tag(struct _dif_sgl *sgl, uint32_t offset_blocks, 2430 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2431 bool check_ref_tag) 2432 { 2433 uint32_t offset, buf_len; 2434 uint64_t expected = 0, remapped; 2435 uint8_t *buf; 2436 struct _dif_sgl tmp_sgl; 2437 struct spdk_dif dif; 2438 2439 /* Fast forward to DIF field. */ 2440 _dif_sgl_advance(sgl, ctx->guard_interval); 2441 _dif_sgl_copy(&tmp_sgl, sgl); 2442 2443 /* Copy the split DIF field to the temporary DIF buffer */ 2444 offset = 0; 2445 while (offset < _dif_size(ctx->dif_pi_format)) { 2446 _dif_sgl_get_buf(sgl, &buf, &buf_len); 2447 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 2448 2449 memcpy((uint8_t *)&dif + offset, buf, buf_len); 2450 2451 _dif_sgl_advance(sgl, buf_len); 2452 offset += buf_len; 2453 } 2454 2455 if (_dif_ignore(&dif, ctx)) { 2456 goto end; 2457 } 2458 2459 /* For type 1 and 2, the Reference Tag is incremented for each 2460 * subsequent logical block. For type 3, the Reference Tag 2461 * remains the same as the initial Reference Tag. 2462 */ 2463 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2464 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2465 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2466 } else { 2467 remapped = ctx->remapped_init_ref_tag; 2468 } 2469 2470 /* Verify the stored Reference Tag. */ 2471 if (check_ref_tag && !_dif_reftag_check(&dif, ctx, expected, offset_blocks, err_blk)) { 2472 return -1; 2473 } 2474 2475 /* Update the stored Reference Tag to the remapped one. */ 2476 _dif_set_reftag(&dif, remapped, ctx->dif_pi_format); 2477 2478 offset = 0; 2479 while (offset < _dif_size(ctx->dif_pi_format)) { 2480 _dif_sgl_get_buf(&tmp_sgl, &buf, &buf_len); 2481 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 2482 2483 memcpy(buf, (uint8_t *)&dif + offset, buf_len); 2484 2485 _dif_sgl_advance(&tmp_sgl, buf_len); 2486 offset += buf_len; 2487 } 2488 2489 end: 2490 _dif_sgl_advance(sgl, ctx->block_size - ctx->guard_interval - _dif_size(ctx->dif_pi_format)); 2491 2492 return 0; 2493 } 2494 2495 int 2496 spdk_dif_remap_ref_tag(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 2497 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2498 bool check_ref_tag) 2499 { 2500 struct _dif_sgl sgl; 2501 uint32_t offset_blocks; 2502 int rc; 2503 2504 _dif_sgl_init(&sgl, iovs, iovcnt); 2505 2506 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 2507 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2508 return -EINVAL; 2509 } 2510 2511 if (_dif_is_disabled(ctx->dif_type)) { 2512 return 0; 2513 } 2514 2515 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2516 return 0; 2517 } 2518 2519 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2520 rc = _dif_remap_ref_tag(&sgl, offset_blocks, ctx, err_blk, check_ref_tag); 2521 if (rc != 0) { 2522 return rc; 2523 } 2524 } 2525 2526 return 0; 2527 } 2528 2529 static int 2530 _dix_remap_ref_tag(struct _dif_sgl *md_sgl, uint32_t offset_blocks, 2531 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2532 bool check_ref_tag) 2533 { 2534 uint64_t expected = 0, remapped; 2535 uint8_t *md_buf; 2536 struct spdk_dif *dif; 2537 2538 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 2539 2540 dif = (struct spdk_dif *)(md_buf + ctx->guard_interval); 2541 2542 if (_dif_ignore(dif, ctx)) { 2543 goto end; 2544 } 2545 2546 /* For type 1 and 2, the Reference Tag is incremented for each 2547 * subsequent logical block. For type 3, the Reference Tag 2548 * remains the same as the initialReference Tag. 2549 */ 2550 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2551 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2552 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2553 } else { 2554 remapped = ctx->remapped_init_ref_tag; 2555 } 2556 2557 /* Verify the stored Reference Tag. */ 2558 if (check_ref_tag && !_dif_reftag_check(dif, ctx, expected, offset_blocks, err_blk)) { 2559 return -1; 2560 } 2561 2562 /* Update the stored Reference Tag to the remapped one. */ 2563 _dif_set_reftag(dif, remapped, ctx->dif_pi_format); 2564 2565 end: 2566 _dif_sgl_advance(md_sgl, ctx->md_size); 2567 2568 return 0; 2569 } 2570 2571 int 2572 spdk_dix_remap_ref_tag(struct iovec *md_iov, uint32_t num_blocks, 2573 const struct spdk_dif_ctx *ctx, 2574 struct spdk_dif_error *err_blk, 2575 bool check_ref_tag) 2576 { 2577 struct _dif_sgl md_sgl; 2578 uint32_t offset_blocks; 2579 int rc; 2580 2581 _dif_sgl_init(&md_sgl, md_iov, 1); 2582 2583 if (!_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2584 SPDK_ERRLOG("Size of metadata iovec array is not valid.\n"); 2585 return -EINVAL; 2586 } 2587 2588 if (_dif_is_disabled(ctx->dif_type)) { 2589 return 0; 2590 } 2591 2592 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2593 return 0; 2594 } 2595 2596 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2597 rc = _dix_remap_ref_tag(&md_sgl, offset_blocks, ctx, err_blk, check_ref_tag); 2598 if (rc != 0) { 2599 return rc; 2600 } 2601 } 2602 2603 return 0; 2604 } 2605