1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/dif.h" 7 #include "spdk/crc16.h" 8 #include "spdk/crc32.h" 9 #include "spdk/crc64.h" 10 #include "spdk/endian.h" 11 #include "spdk/log.h" 12 #include "spdk/util.h" 13 14 #define APPTAG_IGNORE 0xFFFF 15 #define REFTAG_MASK_16 0x00000000FFFFFFFF 16 #define REFTAG_MASK_32 0xFFFFFFFFFFFFFFFF 17 #define REFTAG_MASK_64 0x0000FFFFFFFFFFFF 18 19 /* The variable size Storage Tag and Reference Tag is not supported yet, 20 * so the maximum size of the Reference Tag is assumed. 21 */ 22 struct spdk_dif { 23 union { 24 struct { 25 uint16_t guard; 26 uint16_t app_tag; 27 uint32_t stor_ref_space; 28 } g16; 29 struct { 30 uint32_t guard; 31 uint16_t app_tag; 32 uint16_t stor_ref_space_p1; 33 uint64_t stor_ref_space_p2; 34 } g32; 35 struct { 36 uint64_t guard; 37 uint16_t app_tag; 38 uint16_t stor_ref_space_p1; 39 uint32_t stor_ref_space_p2; 40 } g64; 41 }; 42 }; 43 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g16) == 8, "Incorrect size"); 44 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g32) == 16, "Incorrect size"); 45 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g64) == 16, "Incorrect size"); 46 47 /* Context to iterate or create a iovec array. 48 * Each sgl is either iterated or created at a time. 49 */ 50 struct _dif_sgl { 51 /* Current iovec in the iteration or creation */ 52 struct iovec *iov; 53 54 /* Remaining count of iovecs in the iteration or creation. */ 55 int iovcnt; 56 57 /* Current offset in the iovec */ 58 uint32_t iov_offset; 59 60 /* Size of the created iovec array in bytes */ 61 uint32_t total_size; 62 }; 63 64 static inline void 65 _dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt) 66 { 67 s->iov = iovs; 68 s->iovcnt = iovcnt; 69 s->iov_offset = 0; 70 s->total_size = 0; 71 } 72 73 static void 74 _dif_sgl_advance(struct _dif_sgl *s, uint32_t step) 75 { 76 s->iov_offset += step; 77 while (s->iovcnt != 0) { 78 if (s->iov_offset < s->iov->iov_len) { 79 break; 80 } 81 82 s->iov_offset -= s->iov->iov_len; 83 s->iov++; 84 s->iovcnt--; 85 } 86 } 87 88 static inline void 89 _dif_sgl_get_buf(struct _dif_sgl *s, uint8_t **_buf, uint32_t *_buf_len) 90 { 91 if (_buf != NULL) { 92 *_buf = (uint8_t *)s->iov->iov_base + s->iov_offset; 93 } 94 if (_buf_len != NULL) { 95 *_buf_len = s->iov->iov_len - s->iov_offset; 96 } 97 } 98 99 static inline bool 100 _dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len) 101 { 102 assert(s->iovcnt > 0); 103 s->iov->iov_base = data; 104 s->iov->iov_len = data_len; 105 s->total_size += data_len; 106 s->iov++; 107 s->iovcnt--; 108 109 if (s->iovcnt > 0) { 110 return true; 111 } else { 112 return false; 113 } 114 } 115 116 static inline bool 117 _dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len) 118 { 119 uint8_t *buf; 120 uint32_t buf_len; 121 122 while (data_len != 0) { 123 _dif_sgl_get_buf(src, &buf, &buf_len); 124 buf_len = spdk_min(buf_len, data_len); 125 126 if (!_dif_sgl_append(dst, buf, buf_len)) { 127 return false; 128 } 129 130 _dif_sgl_advance(src, buf_len); 131 data_len -= buf_len; 132 } 133 134 return true; 135 } 136 137 /* This function must be used before starting iteration. */ 138 static bool 139 _dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes) 140 { 141 int i; 142 143 for (i = 0; i < s->iovcnt; i++) { 144 if (s->iov[i].iov_len % bytes) { 145 return false; 146 } 147 } 148 149 return true; 150 } 151 152 static bool 153 _dif_sgl_is_valid_block_aligned(struct _dif_sgl *s, uint32_t num_blocks, uint32_t block_size) 154 { 155 uint32_t count = 0; 156 int i; 157 158 for (i = 0; i < s->iovcnt; i++) { 159 if (s->iov[i].iov_len % block_size) { 160 return false; 161 } 162 count += s->iov[i].iov_len / block_size; 163 } 164 165 return count >= num_blocks; 166 } 167 168 /* This function must be used before starting iteration. */ 169 static bool 170 _dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes) 171 { 172 uint64_t total = 0; 173 int i; 174 175 for (i = 0; i < s->iovcnt; i++) { 176 total += s->iov[i].iov_len; 177 } 178 179 return total >= bytes; 180 } 181 182 static void 183 _dif_sgl_copy(struct _dif_sgl *to, struct _dif_sgl *from) 184 { 185 memcpy(to, from, sizeof(struct _dif_sgl)); 186 } 187 188 static bool 189 _dif_type_is_valid(enum spdk_dif_type dif_type, uint32_t dif_flags) 190 { 191 switch (dif_type) { 192 case SPDK_DIF_TYPE1: 193 case SPDK_DIF_TYPE2: 194 case SPDK_DIF_DISABLE: 195 break; 196 case SPDK_DIF_TYPE3: 197 if (dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 198 SPDK_ERRLOG("Reference Tag should not be checked for Type 3\n"); 199 return false; 200 } 201 break; 202 default: 203 SPDK_ERRLOG("Unknown DIF Type: %d\n", dif_type); 204 return false; 205 } 206 207 return true; 208 } 209 210 static bool 211 _dif_is_disabled(enum spdk_dif_type dif_type) 212 { 213 if (dif_type == SPDK_DIF_DISABLE) { 214 return true; 215 } else { 216 return false; 217 } 218 } 219 220 static inline size_t 221 _dif_size(enum spdk_dif_pi_format dif_pi_format) 222 { 223 uint8_t size; 224 225 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 226 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16); 227 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 228 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32); 229 } else { 230 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64); 231 } 232 233 return size; 234 } 235 236 static uint32_t 237 _get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave, 238 size_t dif_size) 239 { 240 if (!dif_loc) { 241 /* For metadata formats with more than 8/16 bytes (depending on 242 * the PI format), if the DIF is contained in the last 8/16 bytes 243 * of metadata, then the CRC covers all metadata up to but excluding 244 * these last 8/16 bytes. 245 */ 246 if (md_interleave) { 247 return block_size - dif_size; 248 } else { 249 return md_size - dif_size; 250 } 251 } else { 252 /* For metadata formats with more than 8/16 bytes (depending on 253 * the PI format), if the DIF is contained in the first 8/16 bytes 254 * of metadata, then the CRC does not cover any metadata. 255 */ 256 if (md_interleave) { 257 return block_size - md_size; 258 } else { 259 return 0; 260 } 261 } 262 } 263 264 static inline uint8_t 265 _dif_guard_size(enum spdk_dif_pi_format dif_pi_format) 266 { 267 uint8_t size; 268 269 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 270 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.guard); 271 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 272 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.guard); 273 } else { 274 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.guard); 275 } 276 277 return size; 278 } 279 280 static inline void 281 _dif_set_guard(struct spdk_dif *dif, uint64_t guard, enum spdk_dif_pi_format dif_pi_format) 282 { 283 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 284 to_be16(&(dif->g16.guard), (uint16_t)guard); 285 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 286 to_be32(&(dif->g32.guard), (uint32_t)guard); 287 } else { 288 to_be64(&(dif->g64.guard), guard); 289 } 290 } 291 292 static inline uint64_t 293 _dif_get_guard(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 294 { 295 uint64_t guard; 296 297 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 298 guard = (uint64_t)from_be16(&(dif->g16.guard)); 299 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 300 guard = (uint64_t)from_be32(&(dif->g32.guard)); 301 } else { 302 guard = from_be64(&(dif->g64.guard)); 303 } 304 305 return guard; 306 } 307 308 static inline uint64_t 309 _dif_generate_guard(uint64_t guard_seed, void *buf, size_t buf_len, 310 enum spdk_dif_pi_format dif_pi_format) 311 { 312 uint64_t guard; 313 314 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 315 guard = (uint64_t)spdk_crc16_t10dif((uint16_t)guard_seed, buf, buf_len); 316 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 317 guard = (uint64_t)spdk_crc32c_nvme(buf, buf_len, guard_seed); 318 } else { 319 guard = spdk_crc64_nvme(buf, buf_len, guard_seed); 320 } 321 322 return guard; 323 } 324 325 static inline uint64_t 326 _dif_generate_guard_copy(uint64_t guard_seed, void *dst, void *src, size_t buf_len, 327 enum spdk_dif_pi_format dif_pi_format) 328 { 329 uint64_t guard; 330 331 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 332 guard = (uint64_t)spdk_crc16_t10dif_copy((uint16_t)guard_seed, dst, src, buf_len); 333 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 334 memcpy(dst, src, buf_len); 335 guard = (uint64_t)spdk_crc32c_nvme(src, buf_len, guard_seed); 336 } else { 337 memcpy(dst, src, buf_len); 338 guard = spdk_crc64_nvme(src, buf_len, guard_seed); 339 } 340 341 return guard; 342 } 343 344 static inline uint8_t 345 _dif_apptag_offset(enum spdk_dif_pi_format dif_pi_format) 346 { 347 return _dif_guard_size(dif_pi_format); 348 } 349 350 static inline uint8_t 351 _dif_apptag_size(void) 352 { 353 return SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.app_tag); 354 } 355 356 static inline void 357 _dif_set_apptag(struct spdk_dif *dif, uint16_t app_tag, enum spdk_dif_pi_format dif_pi_format) 358 { 359 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 360 to_be16(&(dif->g16.app_tag), app_tag); 361 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 362 to_be16(&(dif->g32.app_tag), app_tag); 363 } else { 364 to_be16(&(dif->g64.app_tag), app_tag); 365 } 366 } 367 368 static inline uint16_t 369 _dif_get_apptag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 370 { 371 uint16_t app_tag; 372 373 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 374 app_tag = from_be16(&(dif->g16.app_tag)); 375 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 376 app_tag = from_be16(&(dif->g32.app_tag)); 377 } else { 378 app_tag = from_be16(&(dif->g64.app_tag)); 379 } 380 381 return app_tag; 382 } 383 384 static inline bool 385 _dif_apptag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 386 { 387 return _dif_get_apptag(dif, dif_pi_format) == APPTAG_IGNORE; 388 } 389 390 static inline uint8_t 391 _dif_reftag_offset(enum spdk_dif_pi_format dif_pi_format) 392 { 393 uint8_t offset; 394 395 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 396 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size(); 397 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 398 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size() 399 + SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.stor_ref_space_p1); 400 } else { 401 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size(); 402 } 403 404 return offset; 405 } 406 407 static inline uint8_t 408 _dif_reftag_size(enum spdk_dif_pi_format dif_pi_format) 409 { 410 uint8_t size; 411 412 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 413 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.stor_ref_space); 414 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 415 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.stor_ref_space_p2); 416 } else { 417 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.stor_ref_space_p1) + 418 SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.stor_ref_space_p2); 419 } 420 421 return size; 422 } 423 424 static inline void 425 _dif_set_reftag(struct spdk_dif *dif, uint64_t ref_tag, enum spdk_dif_pi_format dif_pi_format) 426 { 427 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 428 to_be32(&(dif->g16.stor_ref_space), (uint32_t)ref_tag); 429 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 430 to_be64(&(dif->g32.stor_ref_space_p2), ref_tag); 431 } else { 432 to_be16(&(dif->g64.stor_ref_space_p1), (uint16_t)(ref_tag >> 32)); 433 to_be32(&(dif->g64.stor_ref_space_p2), (uint32_t)ref_tag); 434 } 435 } 436 437 static inline uint64_t 438 _dif_get_reftag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 439 { 440 uint64_t ref_tag; 441 442 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 443 ref_tag = (uint64_t)from_be32(&(dif->g16.stor_ref_space)); 444 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 445 ref_tag = from_be64(&(dif->g32.stor_ref_space_p2)); 446 } else { 447 ref_tag = (uint64_t)from_be16(&(dif->g64.stor_ref_space_p1)); 448 ref_tag <<= 32; 449 ref_tag |= (uint64_t)from_be32(&(dif->g64.stor_ref_space_p2)); 450 } 451 452 return ref_tag; 453 } 454 455 static inline bool 456 _dif_reftag_match(struct spdk_dif *dif, uint64_t ref_tag, 457 enum spdk_dif_pi_format dif_pi_format) 458 { 459 uint64_t _ref_tag; 460 bool match; 461 462 _ref_tag = _dif_get_reftag(dif, dif_pi_format); 463 464 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 465 match = (_ref_tag == (ref_tag & REFTAG_MASK_16)); 466 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 467 match = (_ref_tag == ref_tag); 468 } else { 469 match = (_ref_tag == (ref_tag & REFTAG_MASK_64)); 470 } 471 472 return match; 473 } 474 475 static inline bool 476 _dif_reftag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 477 { 478 return _dif_reftag_match(dif, REFTAG_MASK_32, dif_pi_format); 479 } 480 481 int 482 spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, 483 bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, 484 uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag, 485 uint32_t data_offset, uint64_t guard_seed, struct spdk_dif_ctx_init_ext_opts *opts) 486 { 487 uint32_t data_block_size; 488 enum spdk_dif_pi_format dif_pi_format = SPDK_DIF_PI_FORMAT_16; 489 490 if (opts != NULL) { 491 if (opts->dif_pi_format != SPDK_DIF_PI_FORMAT_16 && 492 opts->dif_pi_format != SPDK_DIF_PI_FORMAT_32 && 493 opts->dif_pi_format != SPDK_DIF_PI_FORMAT_64) { 494 SPDK_ERRLOG("No valid DIF PI format provided.\n"); 495 return -EINVAL; 496 } 497 498 dif_pi_format = opts->dif_pi_format; 499 } 500 501 if (md_size < _dif_size(dif_pi_format)) { 502 SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); 503 return -EINVAL; 504 } 505 506 if (md_interleave) { 507 if (block_size < md_size) { 508 SPDK_ERRLOG("Block size is smaller than DIF size.\n"); 509 return -EINVAL; 510 } 511 data_block_size = block_size - md_size; 512 } else { 513 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 514 if (block_size == 0 || (block_size % 512) != 0) { 515 SPDK_ERRLOG("Zero block size is not allowed and should be a multiple of 512B\n"); 516 return -EINVAL; 517 } 518 } else { 519 if (block_size == 0 || (block_size % 4096) != 0) { 520 SPDK_ERRLOG("Zero block size is not allowed and should be a multiple of 4kB\n"); 521 return -EINVAL; 522 } 523 } 524 525 data_block_size = block_size; 526 } 527 528 if (!_dif_type_is_valid(dif_type, dif_flags)) { 529 SPDK_ERRLOG("DIF type is invalid.\n"); 530 return -EINVAL; 531 } 532 533 ctx->block_size = block_size; 534 ctx->md_size = md_size; 535 ctx->md_interleave = md_interleave; 536 ctx->dif_pi_format = dif_pi_format; 537 ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave, 538 _dif_size(ctx->dif_pi_format)); 539 ctx->dif_type = dif_type; 540 ctx->dif_flags = dif_flags; 541 ctx->init_ref_tag = init_ref_tag; 542 ctx->apptag_mask = apptag_mask; 543 ctx->app_tag = app_tag; 544 ctx->data_offset = data_offset; 545 ctx->ref_tag_offset = data_offset / data_block_size; 546 ctx->last_guard = guard_seed; 547 ctx->guard_seed = guard_seed; 548 ctx->remapped_init_ref_tag = 0; 549 550 return 0; 551 } 552 553 void 554 spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset) 555 { 556 uint32_t data_block_size; 557 558 if (ctx->md_interleave) { 559 data_block_size = ctx->block_size - ctx->md_size; 560 } else { 561 data_block_size = ctx->block_size; 562 } 563 564 ctx->data_offset = data_offset; 565 ctx->ref_tag_offset = data_offset / data_block_size; 566 } 567 568 void 569 spdk_dif_ctx_set_remapped_init_ref_tag(struct spdk_dif_ctx *ctx, 570 uint32_t remapped_init_ref_tag) 571 { 572 ctx->remapped_init_ref_tag = remapped_init_ref_tag; 573 } 574 575 static void 576 _dif_generate(void *_dif, uint64_t guard, uint32_t offset_blocks, 577 const struct spdk_dif_ctx *ctx) 578 { 579 struct spdk_dif *dif = _dif; 580 uint64_t ref_tag; 581 582 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 583 _dif_set_guard(dif, guard, ctx->dif_pi_format); 584 } 585 586 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 587 _dif_set_apptag(dif, ctx->app_tag, ctx->dif_pi_format); 588 } 589 590 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 591 /* For type 1 and 2, the reference tag is incremented for each 592 * subsequent logical block. For type 3, the reference tag 593 * remains the same as the initial reference tag. 594 */ 595 if (ctx->dif_type != SPDK_DIF_TYPE3) { 596 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 597 } else { 598 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 599 } 600 601 _dif_set_reftag(dif, ref_tag, ctx->dif_pi_format); 602 } 603 } 604 605 static void 606 dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 607 { 608 uint32_t offset_blocks = 0; 609 uint8_t *buf; 610 uint64_t guard = 0; 611 612 while (offset_blocks < num_blocks) { 613 _dif_sgl_get_buf(sgl, &buf, NULL); 614 615 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 616 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 617 } 618 619 _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); 620 621 _dif_sgl_advance(sgl, ctx->block_size); 622 offset_blocks++; 623 } 624 } 625 626 static uint64_t 627 _dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 628 uint64_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 629 { 630 uint32_t offset_in_dif, buf_len; 631 uint8_t *buf; 632 struct spdk_dif dif = {}; 633 634 assert(offset_in_block < ctx->guard_interval); 635 assert(offset_in_block + data_len < ctx->guard_interval || 636 offset_in_block + data_len == ctx->block_size); 637 638 /* Compute CRC over split logical block data. */ 639 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 640 _dif_sgl_get_buf(sgl, &buf, &buf_len); 641 buf_len = spdk_min(buf_len, data_len); 642 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 643 644 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 645 guard = _dif_generate_guard(guard, buf, buf_len, ctx->dif_pi_format); 646 } 647 648 _dif_sgl_advance(sgl, buf_len); 649 offset_in_block += buf_len; 650 data_len -= buf_len; 651 } 652 653 if (offset_in_block < ctx->guard_interval) { 654 return guard; 655 } 656 657 /* If a whole logical block data is parsed, generate DIF 658 * and save it to the temporary DIF area. 659 */ 660 _dif_generate(&dif, guard, offset_blocks, ctx); 661 662 /* Copy generated DIF field to the split DIF field, and then 663 * skip metadata field after DIF field (if any). 664 */ 665 while (offset_in_block < ctx->block_size) { 666 _dif_sgl_get_buf(sgl, &buf, &buf_len); 667 668 if (offset_in_block < ctx->guard_interval + _dif_size(ctx->dif_pi_format)) { 669 offset_in_dif = offset_in_block - ctx->guard_interval; 670 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset_in_dif); 671 672 memcpy(buf, ((uint8_t *)&dif) + offset_in_dif, buf_len); 673 } else { 674 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 675 } 676 677 _dif_sgl_advance(sgl, buf_len); 678 offset_in_block += buf_len; 679 } 680 681 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 682 guard = ctx->guard_seed; 683 } 684 685 return guard; 686 } 687 688 static void 689 dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks, 690 const struct spdk_dif_ctx *ctx) 691 { 692 uint32_t offset_blocks; 693 uint64_t guard = 0; 694 695 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 696 guard = ctx->guard_seed; 697 } 698 699 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 700 _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx); 701 } 702 } 703 704 int 705 spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 706 const struct spdk_dif_ctx *ctx) 707 { 708 struct _dif_sgl sgl; 709 710 _dif_sgl_init(&sgl, iovs, iovcnt); 711 712 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 713 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 714 return -EINVAL; 715 } 716 717 if (_dif_is_disabled(ctx->dif_type)) { 718 return 0; 719 } 720 721 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 722 dif_generate(&sgl, num_blocks, ctx); 723 } else { 724 dif_generate_split(&sgl, num_blocks, ctx); 725 } 726 727 return 0; 728 } 729 730 static void 731 _dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, 732 uint64_t expected, uint64_t actual, uint32_t err_offset) 733 { 734 if (err_blk) { 735 err_blk->err_type = err_type; 736 err_blk->expected = expected; 737 err_blk->actual = actual; 738 err_blk->err_offset = err_offset; 739 } 740 } 741 742 static int 743 _dif_verify(void *_dif, uint64_t guard, uint32_t offset_blocks, 744 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 745 { 746 struct spdk_dif *dif = _dif; 747 uint64_t _guard; 748 uint16_t _app_tag; 749 uint64_t ref_tag, _ref_tag; 750 751 switch (ctx->dif_type) { 752 case SPDK_DIF_TYPE1: 753 case SPDK_DIF_TYPE2: 754 /* If Type 1 or 2 is used, then all DIF checks are disabled when 755 * the Application Tag is 0xFFFF. 756 */ 757 if (_dif_apptag_ignore(dif, ctx->dif_pi_format)) { 758 return 0; 759 } 760 break; 761 case SPDK_DIF_TYPE3: 762 /* If Type 3 is used, then all DIF checks are disabled when the 763 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF 764 * or 0xFFFFFFFFFFFFFFFF depending on the PI format. 765 */ 766 767 if (_dif_apptag_ignore(dif, ctx->dif_pi_format) && 768 _dif_reftag_ignore(dif, ctx->dif_pi_format)) { 769 return 0; 770 } 771 break; 772 default: 773 break; 774 } 775 776 /* For type 1 and 2, the reference tag is incremented for each 777 * subsequent logical block. For type 3, the reference tag 778 * remains the same as the initial reference tag. 779 */ 780 if (ctx->dif_type != SPDK_DIF_TYPE3) { 781 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 782 } else { 783 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 784 } 785 786 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 787 /* Compare the DIF Guard field to the CRC computed over the logical 788 * block data. 789 */ 790 _guard = _dif_get_guard(dif, ctx->dif_pi_format); 791 if (_guard != guard) { 792 _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, 793 offset_blocks); 794 SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu64 "," \ 795 " Expected=%lx, Actual=%lx\n", 796 ref_tag, _guard, guard); 797 return -1; 798 } 799 } 800 801 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 802 /* Compare unmasked bits in the DIF Application Tag field to the 803 * passed Application Tag. 804 */ 805 _app_tag = _dif_get_apptag(dif, ctx->dif_pi_format); 806 if ((_app_tag & ctx->apptag_mask) != (ctx->app_tag & ctx->apptag_mask)) { 807 _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, 808 (_app_tag & ctx->apptag_mask), offset_blocks); 809 SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu64 "," \ 810 " Expected=%x, Actual=%x\n", 811 ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); 812 return -1; 813 } 814 } 815 816 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 817 switch (ctx->dif_type) { 818 case SPDK_DIF_TYPE1: 819 case SPDK_DIF_TYPE2: 820 /* Compare the DIF Reference Tag field to the passed Reference Tag. 821 * The passed Reference Tag will be the least significant 4 bytes 822 * or 8 bytes (depending on the PI format) 823 * of the LBA when Type 1 is used, and application specific value 824 * if Type 2 is used. 825 */ 826 if (!_dif_reftag_match(dif, ref_tag, ctx->dif_pi_format)) { 827 _ref_tag = _dif_get_reftag(dif, ctx->dif_pi_format); 828 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, ref_tag, 829 _ref_tag, offset_blocks); 830 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu64 "," \ 831 " Expected=%lx, Actual=%lx\n", 832 ref_tag, ref_tag, _ref_tag); 833 return -1; 834 } 835 break; 836 case SPDK_DIF_TYPE3: 837 /* For Type 3, computed Reference Tag remains unchanged. 838 * Hence ignore the Reference Tag field. 839 */ 840 break; 841 default: 842 break; 843 } 844 } 845 846 return 0; 847 } 848 849 static int 850 dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks, 851 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 852 { 853 uint32_t offset_blocks = 0; 854 int rc; 855 uint8_t *buf; 856 uint64_t guard = 0; 857 858 while (offset_blocks < num_blocks) { 859 _dif_sgl_get_buf(sgl, &buf, NULL); 860 861 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 862 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 863 } 864 865 rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 866 if (rc != 0) { 867 return rc; 868 } 869 870 _dif_sgl_advance(sgl, ctx->block_size); 871 offset_blocks++; 872 } 873 874 return 0; 875 } 876 877 static int 878 _dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 879 uint64_t *_guard, uint32_t offset_blocks, 880 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 881 { 882 uint32_t offset_in_dif, buf_len; 883 uint8_t *buf; 884 uint64_t guard; 885 struct spdk_dif dif = {}; 886 int rc; 887 888 assert(_guard != NULL); 889 assert(offset_in_block < ctx->guard_interval); 890 assert(offset_in_block + data_len < ctx->guard_interval || 891 offset_in_block + data_len == ctx->block_size); 892 893 guard = *_guard; 894 895 /* Compute CRC over split logical block data. */ 896 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 897 _dif_sgl_get_buf(sgl, &buf, &buf_len); 898 buf_len = spdk_min(buf_len, data_len); 899 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 900 901 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 902 guard = _dif_generate_guard(guard, buf, buf_len, ctx->dif_pi_format); 903 } 904 905 _dif_sgl_advance(sgl, buf_len); 906 offset_in_block += buf_len; 907 data_len -= buf_len; 908 } 909 910 if (offset_in_block < ctx->guard_interval) { 911 *_guard = guard; 912 return 0; 913 } 914 915 /* Copy the split DIF field to the temporary DIF buffer, and then 916 * skip metadata field after DIF field (if any). */ 917 while (offset_in_block < ctx->block_size) { 918 _dif_sgl_get_buf(sgl, &buf, &buf_len); 919 920 if (offset_in_block < ctx->guard_interval + _dif_size(ctx->dif_pi_format)) { 921 offset_in_dif = offset_in_block - ctx->guard_interval; 922 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset_in_dif); 923 924 memcpy((uint8_t *)&dif + offset_in_dif, buf, buf_len); 925 } else { 926 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 927 } 928 _dif_sgl_advance(sgl, buf_len); 929 offset_in_block += buf_len; 930 } 931 932 rc = _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 933 if (rc != 0) { 934 return rc; 935 } 936 937 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 938 guard = ctx->guard_seed; 939 } 940 941 *_guard = guard; 942 return 0; 943 } 944 945 static int 946 dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks, 947 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 948 { 949 uint32_t offset_blocks; 950 uint64_t guard = 0; 951 int rc; 952 953 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 954 guard = ctx->guard_seed; 955 } 956 957 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 958 rc = _dif_verify_split(sgl, 0, ctx->block_size, &guard, offset_blocks, 959 ctx, err_blk); 960 if (rc != 0) { 961 return rc; 962 } 963 } 964 965 return 0; 966 } 967 968 int 969 spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 970 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 971 { 972 struct _dif_sgl sgl; 973 974 _dif_sgl_init(&sgl, iovs, iovcnt); 975 976 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 977 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 978 return -EINVAL; 979 } 980 981 if (_dif_is_disabled(ctx->dif_type)) { 982 return 0; 983 } 984 985 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 986 return dif_verify(&sgl, num_blocks, ctx, err_blk); 987 } else { 988 return dif_verify_split(&sgl, num_blocks, ctx, err_blk); 989 } 990 } 991 992 static uint32_t 993 dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks, 994 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 995 { 996 uint32_t offset_blocks; 997 uint8_t *buf; 998 999 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1000 _dif_sgl_get_buf(sgl, &buf, NULL); 1001 1002 crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c); 1003 1004 _dif_sgl_advance(sgl, ctx->block_size); 1005 } 1006 1007 return crc32c; 1008 } 1009 1010 static uint32_t 1011 _dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 1012 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1013 { 1014 uint32_t data_block_size, buf_len; 1015 uint8_t *buf; 1016 1017 data_block_size = ctx->block_size - ctx->md_size; 1018 1019 assert(offset_in_block + data_len <= ctx->block_size); 1020 1021 while (data_len != 0) { 1022 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1023 buf_len = spdk_min(buf_len, data_len); 1024 1025 if (offset_in_block < data_block_size) { 1026 buf_len = spdk_min(buf_len, data_block_size - offset_in_block); 1027 crc32c = spdk_crc32c_update(buf, buf_len, crc32c); 1028 } 1029 1030 _dif_sgl_advance(sgl, buf_len); 1031 offset_in_block += buf_len; 1032 data_len -= buf_len; 1033 } 1034 1035 return crc32c; 1036 } 1037 1038 static uint32_t 1039 dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks, 1040 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1041 { 1042 uint32_t offset_blocks; 1043 1044 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1045 crc32c = _dif_update_crc32c_split(sgl, 0, ctx->block_size, crc32c, ctx); 1046 } 1047 1048 return crc32c; 1049 } 1050 1051 int 1052 spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1053 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 1054 { 1055 struct _dif_sgl sgl; 1056 1057 if (_crc32c == NULL) { 1058 return -EINVAL; 1059 } 1060 1061 _dif_sgl_init(&sgl, iovs, iovcnt); 1062 1063 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1064 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1065 return -EINVAL; 1066 } 1067 1068 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 1069 *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx); 1070 } else { 1071 *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx); 1072 } 1073 1074 return 0; 1075 } 1076 1077 static void 1078 dif_generate_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1079 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1080 { 1081 uint32_t offset_blocks = 0, data_block_size; 1082 uint8_t *src, *dst; 1083 uint64_t guard; 1084 1085 data_block_size = ctx->block_size - ctx->md_size; 1086 1087 while (offset_blocks < num_blocks) { 1088 _dif_sgl_get_buf(src_sgl, &src, NULL); 1089 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1090 1091 guard = 0; 1092 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1093 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1094 ctx->dif_pi_format); 1095 guard = _dif_generate_guard(guard, dst + data_block_size, 1096 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1097 } else { 1098 memcpy(dst, src, data_block_size); 1099 } 1100 1101 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 1102 1103 _dif_sgl_advance(src_sgl, data_block_size); 1104 _dif_sgl_advance(dst_sgl, ctx->block_size); 1105 offset_blocks++; 1106 } 1107 } 1108 1109 static void 1110 _dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1111 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1112 { 1113 uint32_t offset_in_block, src_len, data_block_size; 1114 uint8_t *src, *dst; 1115 uint64_t guard = 0; 1116 1117 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1118 1119 data_block_size = ctx->block_size - ctx->md_size; 1120 1121 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1122 guard = ctx->guard_seed; 1123 } 1124 offset_in_block = 0; 1125 1126 while (offset_in_block < data_block_size) { 1127 /* Compute CRC over split logical block data and copy 1128 * data to bounce buffer. 1129 */ 1130 _dif_sgl_get_buf(src_sgl, &src, &src_len); 1131 src_len = spdk_min(src_len, data_block_size - offset_in_block); 1132 1133 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1134 guard = _dif_generate_guard_copy(guard, dst + offset_in_block, 1135 src, src_len, ctx->dif_pi_format); 1136 } else { 1137 memcpy(dst + offset_in_block, src, src_len); 1138 } 1139 1140 _dif_sgl_advance(src_sgl, src_len); 1141 offset_in_block += src_len; 1142 } 1143 1144 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1145 guard = _dif_generate_guard(guard, dst + data_block_size, 1146 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1147 } 1148 1149 _dif_sgl_advance(dst_sgl, ctx->block_size); 1150 1151 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 1152 } 1153 1154 static void 1155 dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1156 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1157 { 1158 uint32_t offset_blocks; 1159 1160 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1161 _dif_generate_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 1162 } 1163 } 1164 1165 int 1166 spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1167 int bounce_iovcnt, uint32_t num_blocks, 1168 const struct spdk_dif_ctx *ctx) 1169 { 1170 struct _dif_sgl src_sgl, dst_sgl; 1171 uint32_t data_block_size; 1172 1173 _dif_sgl_init(&src_sgl, iovs, iovcnt); 1174 _dif_sgl_init(&dst_sgl, bounce_iovs, bounce_iovcnt); 1175 1176 data_block_size = ctx->block_size - ctx->md_size; 1177 1178 if (!_dif_sgl_is_valid(&src_sgl, data_block_size * num_blocks)) { 1179 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 1180 return -EINVAL; 1181 } 1182 1183 if (!_dif_sgl_is_valid_block_aligned(&dst_sgl, num_blocks, ctx->block_size)) { 1184 SPDK_ERRLOG("Size of bounce_iovs arrays are not valid or misaligned with block_size.\n"); 1185 return -EINVAL; 1186 } 1187 1188 if (_dif_is_disabled(ctx->dif_type)) { 1189 return 0; 1190 } 1191 1192 if (_dif_sgl_is_bytes_multiple(&src_sgl, data_block_size)) { 1193 dif_generate_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 1194 } else { 1195 dif_generate_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx); 1196 } 1197 1198 return 0; 1199 } 1200 1201 static int 1202 dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1203 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1204 struct spdk_dif_error *err_blk) 1205 { 1206 uint32_t offset_blocks = 0, data_block_size; 1207 uint8_t *src, *dst; 1208 int rc; 1209 uint64_t guard; 1210 1211 data_block_size = ctx->block_size - ctx->md_size; 1212 1213 while (offset_blocks < num_blocks) { 1214 _dif_sgl_get_buf(src_sgl, &src, NULL); 1215 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1216 1217 guard = 0; 1218 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1219 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1220 ctx->dif_pi_format); 1221 guard = _dif_generate_guard(guard, src + data_block_size, 1222 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1223 } else { 1224 memcpy(dst, src, data_block_size); 1225 } 1226 1227 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1228 if (rc != 0) { 1229 return rc; 1230 } 1231 1232 _dif_sgl_advance(src_sgl, ctx->block_size); 1233 _dif_sgl_advance(dst_sgl, data_block_size); 1234 offset_blocks++; 1235 } 1236 1237 return 0; 1238 } 1239 1240 static int 1241 _dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1242 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1243 struct spdk_dif_error *err_blk) 1244 { 1245 uint32_t offset_in_block, dst_len, data_block_size; 1246 uint8_t *src, *dst; 1247 uint64_t guard = 0; 1248 1249 _dif_sgl_get_buf(src_sgl, &src, NULL); 1250 1251 data_block_size = ctx->block_size - ctx->md_size; 1252 1253 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1254 guard = ctx->guard_seed; 1255 } 1256 offset_in_block = 0; 1257 1258 while (offset_in_block < data_block_size) { 1259 /* Compute CRC over split logical block data and copy 1260 * data to bounce buffer. 1261 */ 1262 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 1263 dst_len = spdk_min(dst_len, data_block_size - offset_in_block); 1264 1265 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1266 guard = _dif_generate_guard_copy(guard, dst, src + offset_in_block, 1267 dst_len, ctx->dif_pi_format); 1268 } else { 1269 memcpy(dst, src + offset_in_block, dst_len); 1270 } 1271 1272 _dif_sgl_advance(dst_sgl, dst_len); 1273 offset_in_block += dst_len; 1274 } 1275 1276 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1277 guard = _dif_generate_guard(guard, src + data_block_size, 1278 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1279 } 1280 1281 _dif_sgl_advance(src_sgl, ctx->block_size); 1282 1283 return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1284 } 1285 1286 static int 1287 dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1288 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1289 struct spdk_dif_error *err_blk) 1290 { 1291 uint32_t offset_blocks; 1292 int rc; 1293 1294 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1295 rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1296 if (rc != 0) { 1297 return rc; 1298 } 1299 } 1300 1301 return 0; 1302 } 1303 1304 int 1305 spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1306 int bounce_iovcnt, uint32_t num_blocks, 1307 const struct spdk_dif_ctx *ctx, 1308 struct spdk_dif_error *err_blk) 1309 { 1310 struct _dif_sgl src_sgl, dst_sgl; 1311 uint32_t data_block_size; 1312 1313 _dif_sgl_init(&src_sgl, bounce_iovs, bounce_iovcnt); 1314 _dif_sgl_init(&dst_sgl, iovs, iovcnt); 1315 1316 data_block_size = ctx->block_size - ctx->md_size; 1317 1318 if (!_dif_sgl_is_valid(&dst_sgl, data_block_size * num_blocks)) { 1319 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1320 return -EINVAL; 1321 } 1322 1323 if (!_dif_sgl_is_valid_block_aligned(&src_sgl, num_blocks, ctx->block_size)) { 1324 SPDK_ERRLOG("Size of bounce_iovs arrays are not valid or misaligned with block_size.\n"); 1325 return -EINVAL; 1326 } 1327 1328 if (_dif_is_disabled(ctx->dif_type)) { 1329 return 0; 1330 } 1331 1332 if (_dif_sgl_is_bytes_multiple(&dst_sgl, data_block_size)) { 1333 return dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1334 } else { 1335 return dif_verify_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1336 } 1337 } 1338 1339 static void 1340 _bit_flip(uint8_t *buf, uint32_t flip_bit) 1341 { 1342 uint8_t byte; 1343 1344 byte = *buf; 1345 byte ^= 1 << flip_bit; 1346 *buf = byte; 1347 } 1348 1349 static int 1350 _dif_inject_error(struct _dif_sgl *sgl, 1351 uint32_t block_size, uint32_t num_blocks, 1352 uint32_t inject_offset_blocks, 1353 uint32_t inject_offset_bytes, 1354 uint32_t inject_offset_bits) 1355 { 1356 uint32_t offset_in_block, buf_len; 1357 uint8_t *buf; 1358 1359 _dif_sgl_advance(sgl, block_size * inject_offset_blocks); 1360 1361 offset_in_block = 0; 1362 1363 while (offset_in_block < block_size) { 1364 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1365 buf_len = spdk_min(buf_len, block_size - offset_in_block); 1366 1367 if (inject_offset_bytes >= offset_in_block && 1368 inject_offset_bytes < offset_in_block + buf_len) { 1369 buf += inject_offset_bytes - offset_in_block; 1370 _bit_flip(buf, inject_offset_bits); 1371 return 0; 1372 } 1373 1374 _dif_sgl_advance(sgl, buf_len); 1375 offset_in_block += buf_len; 1376 } 1377 1378 return -1; 1379 } 1380 1381 static int 1382 dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks, 1383 uint32_t start_inject_bytes, uint32_t inject_range_bytes, 1384 uint32_t *inject_offset) 1385 { 1386 uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; 1387 uint32_t offset_blocks; 1388 int rc; 1389 1390 srand(time(0)); 1391 1392 inject_offset_blocks = rand() % num_blocks; 1393 inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); 1394 inject_offset_bits = rand() % 8; 1395 1396 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1397 if (offset_blocks == inject_offset_blocks) { 1398 rc = _dif_inject_error(sgl, block_size, num_blocks, 1399 inject_offset_blocks, 1400 inject_offset_bytes, 1401 inject_offset_bits); 1402 if (rc == 0) { 1403 *inject_offset = inject_offset_blocks; 1404 } 1405 return rc; 1406 } 1407 } 1408 1409 return -1; 1410 } 1411 1412 int 1413 spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1414 const struct spdk_dif_ctx *ctx, uint32_t inject_flags, 1415 uint32_t *inject_offset) 1416 { 1417 struct _dif_sgl sgl; 1418 int rc; 1419 1420 _dif_sgl_init(&sgl, iovs, iovcnt); 1421 1422 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1423 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1424 return -EINVAL; 1425 } 1426 1427 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1428 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1429 ctx->guard_interval + _dif_reftag_offset(ctx->dif_pi_format), 1430 _dif_reftag_size(ctx->dif_pi_format), 1431 inject_offset); 1432 if (rc != 0) { 1433 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1434 return rc; 1435 } 1436 } 1437 1438 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1439 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1440 ctx->guard_interval + _dif_apptag_offset(ctx->dif_pi_format), 1441 _dif_apptag_size(), 1442 inject_offset); 1443 if (rc != 0) { 1444 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1445 return rc; 1446 } 1447 } 1448 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1449 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1450 ctx->guard_interval, 1451 _dif_guard_size(ctx->dif_pi_format), 1452 inject_offset); 1453 if (rc != 0) { 1454 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1455 return rc; 1456 } 1457 } 1458 1459 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1460 /* If the DIF information is contained within the last 8/16 bytes of 1461 * metadata (depending on the PI format), then the CRC covers all metadata 1462 * bytes up to but excluding the last 8/16 bytes. But error injection does not 1463 * cover these metadata because classification is not determined yet. 1464 * 1465 * Note: Error injection to data block is expected to be detected as 1466 * guard error. 1467 */ 1468 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1469 0, 1470 ctx->block_size - ctx->md_size, 1471 inject_offset); 1472 if (rc != 0) { 1473 SPDK_ERRLOG("Failed to inject error to data block.\n"); 1474 return rc; 1475 } 1476 } 1477 1478 return 0; 1479 } 1480 1481 static void 1482 dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1483 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1484 { 1485 uint32_t offset_blocks = 0; 1486 uint8_t *data_buf, *md_buf; 1487 uint64_t guard; 1488 1489 while (offset_blocks < num_blocks) { 1490 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1491 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1492 1493 guard = 0; 1494 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1495 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1496 ctx->dif_pi_format); 1497 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1498 ctx->dif_pi_format); 1499 } 1500 1501 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1502 1503 _dif_sgl_advance(data_sgl, ctx->block_size); 1504 _dif_sgl_advance(md_sgl, ctx->md_size); 1505 offset_blocks++; 1506 } 1507 } 1508 1509 static void 1510 _dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1511 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1512 { 1513 uint32_t offset_in_block, data_buf_len; 1514 uint8_t *data_buf, *md_buf; 1515 uint64_t guard = 0; 1516 1517 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1518 1519 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1520 guard = ctx->guard_seed; 1521 } 1522 offset_in_block = 0; 1523 1524 while (offset_in_block < ctx->block_size) { 1525 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1526 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1527 1528 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1529 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 1530 ctx->dif_pi_format); 1531 } 1532 1533 _dif_sgl_advance(data_sgl, data_buf_len); 1534 offset_in_block += data_buf_len; 1535 } 1536 1537 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1538 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1539 ctx->dif_pi_format); 1540 } 1541 1542 _dif_sgl_advance(md_sgl, ctx->md_size); 1543 1544 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1545 } 1546 1547 static void 1548 dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1549 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1550 { 1551 uint32_t offset_blocks; 1552 1553 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1554 _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx); 1555 } 1556 } 1557 1558 int 1559 spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1560 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1561 { 1562 struct _dif_sgl data_sgl, md_sgl; 1563 1564 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1565 _dif_sgl_init(&md_sgl, md_iov, 1); 1566 1567 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1568 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1569 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1570 return -EINVAL; 1571 } 1572 1573 if (_dif_is_disabled(ctx->dif_type)) { 1574 return 0; 1575 } 1576 1577 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1578 dix_generate(&data_sgl, &md_sgl, num_blocks, ctx); 1579 } else { 1580 dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx); 1581 } 1582 1583 return 0; 1584 } 1585 1586 static int 1587 dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1588 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1589 struct spdk_dif_error *err_blk) 1590 { 1591 uint32_t offset_blocks = 0; 1592 uint8_t *data_buf, *md_buf; 1593 uint64_t guard; 1594 int rc; 1595 1596 while (offset_blocks < num_blocks) { 1597 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1598 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1599 1600 guard = 0; 1601 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1602 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1603 ctx->dif_pi_format); 1604 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1605 ctx->dif_pi_format); 1606 } 1607 1608 rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1609 if (rc != 0) { 1610 return rc; 1611 } 1612 1613 _dif_sgl_advance(data_sgl, ctx->block_size); 1614 _dif_sgl_advance(md_sgl, ctx->md_size); 1615 offset_blocks++; 1616 } 1617 1618 return 0; 1619 } 1620 1621 static int 1622 _dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1623 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1624 struct spdk_dif_error *err_blk) 1625 { 1626 uint32_t offset_in_block, data_buf_len; 1627 uint8_t *data_buf, *md_buf; 1628 uint64_t guard = 0; 1629 1630 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1631 1632 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1633 guard = ctx->guard_seed; 1634 } 1635 offset_in_block = 0; 1636 1637 while (offset_in_block < ctx->block_size) { 1638 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1639 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1640 1641 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1642 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 1643 ctx->dif_pi_format); 1644 } 1645 1646 _dif_sgl_advance(data_sgl, data_buf_len); 1647 offset_in_block += data_buf_len; 1648 } 1649 1650 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1651 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1652 ctx->dif_pi_format); 1653 } 1654 1655 _dif_sgl_advance(md_sgl, ctx->md_size); 1656 1657 return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1658 } 1659 1660 static int 1661 dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1662 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1663 struct spdk_dif_error *err_blk) 1664 { 1665 uint32_t offset_blocks; 1666 int rc; 1667 1668 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1669 rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk); 1670 if (rc != 0) { 1671 return rc; 1672 } 1673 } 1674 1675 return 0; 1676 } 1677 1678 int 1679 spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1680 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1681 struct spdk_dif_error *err_blk) 1682 { 1683 struct _dif_sgl data_sgl, md_sgl; 1684 1685 if (md_iov->iov_base == NULL) { 1686 SPDK_ERRLOG("Metadata buffer is NULL.\n"); 1687 return -EINVAL; 1688 } 1689 1690 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1691 _dif_sgl_init(&md_sgl, md_iov, 1); 1692 1693 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1694 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1695 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1696 return -EINVAL; 1697 } 1698 1699 if (_dif_is_disabled(ctx->dif_type)) { 1700 return 0; 1701 } 1702 1703 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1704 return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1705 } else { 1706 return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1707 } 1708 } 1709 1710 int 1711 spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1712 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1713 uint32_t inject_flags, uint32_t *inject_offset) 1714 { 1715 struct _dif_sgl data_sgl, md_sgl; 1716 int rc; 1717 1718 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1719 _dif_sgl_init(&md_sgl, md_iov, 1); 1720 1721 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1722 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1723 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1724 return -EINVAL; 1725 } 1726 1727 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1728 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1729 ctx->guard_interval + _dif_reftag_offset(ctx->dif_pi_format), 1730 _dif_reftag_size(ctx->dif_pi_format), 1731 inject_offset); 1732 if (rc != 0) { 1733 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1734 return rc; 1735 } 1736 } 1737 1738 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1739 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1740 ctx->guard_interval + _dif_apptag_offset(ctx->dif_pi_format), 1741 _dif_apptag_size(), 1742 inject_offset); 1743 if (rc != 0) { 1744 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1745 return rc; 1746 } 1747 } 1748 1749 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1750 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1751 ctx->guard_interval, 1752 _dif_guard_size(ctx->dif_pi_format), 1753 inject_offset); 1754 if (rc != 0) { 1755 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1756 return rc; 1757 } 1758 } 1759 1760 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1761 /* Note: Error injection to data block is expected to be detected 1762 * as guard error. 1763 */ 1764 rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks, 1765 0, 1766 ctx->block_size, 1767 inject_offset); 1768 if (rc != 0) { 1769 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1770 return rc; 1771 } 1772 } 1773 1774 return 0; 1775 } 1776 1777 static uint32_t 1778 _to_next_boundary(uint32_t offset, uint32_t boundary) 1779 { 1780 return boundary - (offset % boundary); 1781 } 1782 1783 static uint32_t 1784 _to_size_with_md(uint32_t size, uint32_t data_block_size, uint32_t block_size) 1785 { 1786 return (size / data_block_size) * block_size + (size % data_block_size); 1787 } 1788 1789 int 1790 spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt, 1791 struct iovec *buf_iovs, int buf_iovcnt, 1792 uint32_t data_offset, uint32_t data_len, 1793 uint32_t *_mapped_len, 1794 const struct spdk_dif_ctx *ctx) 1795 { 1796 uint32_t data_block_size, data_unalign, buf_len, buf_offset, len; 1797 struct _dif_sgl dif_sgl; 1798 struct _dif_sgl buf_sgl; 1799 1800 if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) { 1801 return -EINVAL; 1802 } 1803 1804 data_block_size = ctx->block_size - ctx->md_size; 1805 1806 data_unalign = ctx->data_offset % data_block_size; 1807 1808 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 1809 ctx->block_size); 1810 buf_len -= data_unalign; 1811 1812 _dif_sgl_init(&dif_sgl, iovs, iovcnt); 1813 _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt); 1814 1815 if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) { 1816 SPDK_ERRLOG("Buffer overflow will occur.\n"); 1817 return -ERANGE; 1818 } 1819 1820 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 1821 buf_offset -= data_unalign; 1822 1823 _dif_sgl_advance(&buf_sgl, buf_offset); 1824 1825 while (data_len != 0) { 1826 len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size)); 1827 if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) { 1828 break; 1829 } 1830 _dif_sgl_advance(&buf_sgl, ctx->md_size); 1831 data_offset += len; 1832 data_len -= len; 1833 } 1834 1835 if (_mapped_len != NULL) { 1836 *_mapped_len = dif_sgl.total_size; 1837 } 1838 1839 return iovcnt - dif_sgl.iovcnt; 1840 } 1841 1842 static int 1843 _dif_sgl_setup_stream(struct _dif_sgl *sgl, uint32_t *_buf_offset, uint32_t *_buf_len, 1844 uint32_t data_offset, uint32_t data_len, 1845 const struct spdk_dif_ctx *ctx) 1846 { 1847 uint32_t data_block_size, data_unalign, buf_len, buf_offset; 1848 1849 data_block_size = ctx->block_size - ctx->md_size; 1850 1851 data_unalign = ctx->data_offset % data_block_size; 1852 1853 /* If the last data block is complete, DIF of the data block is 1854 * inserted or verified in this turn. 1855 */ 1856 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 1857 ctx->block_size); 1858 buf_len -= data_unalign; 1859 1860 if (!_dif_sgl_is_valid(sgl, buf_len)) { 1861 return -ERANGE; 1862 } 1863 1864 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 1865 buf_offset -= data_unalign; 1866 1867 _dif_sgl_advance(sgl, buf_offset); 1868 buf_len -= buf_offset; 1869 1870 buf_offset += data_unalign; 1871 1872 *_buf_offset = buf_offset; 1873 *_buf_len = buf_len; 1874 1875 return 0; 1876 } 1877 1878 int 1879 spdk_dif_generate_stream(struct iovec *iovs, int iovcnt, 1880 uint32_t data_offset, uint32_t data_len, 1881 struct spdk_dif_ctx *ctx) 1882 { 1883 uint32_t buf_len = 0, buf_offset = 0; 1884 uint32_t len, offset_in_block, offset_blocks; 1885 uint64_t guard = 0; 1886 struct _dif_sgl sgl; 1887 int rc; 1888 1889 if (iovs == NULL || iovcnt == 0) { 1890 return -EINVAL; 1891 } 1892 1893 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1894 guard = ctx->last_guard; 1895 } 1896 1897 _dif_sgl_init(&sgl, iovs, iovcnt); 1898 1899 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1900 if (rc != 0) { 1901 return rc; 1902 } 1903 1904 while (buf_len != 0) { 1905 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1906 offset_in_block = buf_offset % ctx->block_size; 1907 offset_blocks = buf_offset / ctx->block_size; 1908 1909 guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx); 1910 1911 buf_len -= len; 1912 buf_offset += len; 1913 } 1914 1915 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1916 ctx->last_guard = guard; 1917 } 1918 1919 return 0; 1920 } 1921 1922 int 1923 spdk_dif_verify_stream(struct iovec *iovs, int iovcnt, 1924 uint32_t data_offset, uint32_t data_len, 1925 struct spdk_dif_ctx *ctx, 1926 struct spdk_dif_error *err_blk) 1927 { 1928 uint32_t buf_len = 0, buf_offset = 0; 1929 uint32_t len, offset_in_block, offset_blocks; 1930 uint64_t guard = 0; 1931 struct _dif_sgl sgl; 1932 int rc = 0; 1933 1934 if (iovs == NULL || iovcnt == 0) { 1935 return -EINVAL; 1936 } 1937 1938 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1939 guard = ctx->last_guard; 1940 } 1941 1942 _dif_sgl_init(&sgl, iovs, iovcnt); 1943 1944 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1945 if (rc != 0) { 1946 return rc; 1947 } 1948 1949 while (buf_len != 0) { 1950 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1951 offset_in_block = buf_offset % ctx->block_size; 1952 offset_blocks = buf_offset / ctx->block_size; 1953 1954 rc = _dif_verify_split(&sgl, offset_in_block, len, &guard, offset_blocks, 1955 ctx, err_blk); 1956 if (rc != 0) { 1957 goto error; 1958 } 1959 1960 buf_len -= len; 1961 buf_offset += len; 1962 } 1963 1964 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1965 ctx->last_guard = guard; 1966 } 1967 error: 1968 return rc; 1969 } 1970 1971 int 1972 spdk_dif_update_crc32c_stream(struct iovec *iovs, int iovcnt, 1973 uint32_t data_offset, uint32_t data_len, 1974 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 1975 { 1976 uint32_t buf_len = 0, buf_offset = 0, len, offset_in_block; 1977 uint32_t crc32c; 1978 struct _dif_sgl sgl; 1979 int rc; 1980 1981 if (iovs == NULL || iovcnt == 0) { 1982 return -EINVAL; 1983 } 1984 1985 crc32c = *_crc32c; 1986 _dif_sgl_init(&sgl, iovs, iovcnt); 1987 1988 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1989 if (rc != 0) { 1990 return rc; 1991 } 1992 1993 while (buf_len != 0) { 1994 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1995 offset_in_block = buf_offset % ctx->block_size; 1996 1997 crc32c = _dif_update_crc32c_split(&sgl, offset_in_block, len, crc32c, ctx); 1998 1999 buf_len -= len; 2000 buf_offset += len; 2001 } 2002 2003 *_crc32c = crc32c; 2004 2005 return 0; 2006 } 2007 2008 void 2009 spdk_dif_get_range_with_md(uint32_t data_offset, uint32_t data_len, 2010 uint32_t *_buf_offset, uint32_t *_buf_len, 2011 const struct spdk_dif_ctx *ctx) 2012 { 2013 uint32_t data_block_size, data_unalign, buf_offset, buf_len; 2014 2015 if (!ctx->md_interleave) { 2016 buf_offset = data_offset; 2017 buf_len = data_len; 2018 } else { 2019 data_block_size = ctx->block_size - ctx->md_size; 2020 2021 data_unalign = data_offset % data_block_size; 2022 2023 buf_offset = _to_size_with_md(data_offset, data_block_size, ctx->block_size); 2024 buf_len = _to_size_with_md(data_unalign + data_len, data_block_size, ctx->block_size) - 2025 data_unalign; 2026 } 2027 2028 if (_buf_offset != NULL) { 2029 *_buf_offset = buf_offset; 2030 } 2031 2032 if (_buf_len != NULL) { 2033 *_buf_len = buf_len; 2034 } 2035 } 2036 2037 uint32_t 2038 spdk_dif_get_length_with_md(uint32_t data_len, const struct spdk_dif_ctx *ctx) 2039 { 2040 uint32_t data_block_size; 2041 2042 if (!ctx->md_interleave) { 2043 return data_len; 2044 } else { 2045 data_block_size = ctx->block_size - ctx->md_size; 2046 2047 return _to_size_with_md(data_len, data_block_size, ctx->block_size); 2048 } 2049 } 2050 2051 static int 2052 _dif_remap_ref_tag(struct _dif_sgl *sgl, uint32_t offset_blocks, 2053 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 2054 { 2055 uint32_t offset, buf_len; 2056 uint64_t expected = 0, _actual, remapped; 2057 uint8_t *buf; 2058 struct _dif_sgl tmp_sgl; 2059 struct spdk_dif dif; 2060 2061 /* Fast forward to DIF field. */ 2062 _dif_sgl_advance(sgl, ctx->guard_interval); 2063 _dif_sgl_copy(&tmp_sgl, sgl); 2064 2065 /* Copy the split DIF field to the temporary DIF buffer */ 2066 offset = 0; 2067 while (offset < _dif_size(ctx->dif_pi_format)) { 2068 _dif_sgl_get_buf(sgl, &buf, &buf_len); 2069 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 2070 2071 memcpy((uint8_t *)&dif + offset, buf, buf_len); 2072 2073 _dif_sgl_advance(sgl, buf_len); 2074 offset += buf_len; 2075 } 2076 2077 switch (ctx->dif_type) { 2078 case SPDK_DIF_TYPE1: 2079 case SPDK_DIF_TYPE2: 2080 /* If Type 1 or 2 is used, then all DIF checks are disabled when 2081 * the Application Tag is 0xFFFF. 2082 */ 2083 if (_dif_apptag_ignore(&dif, ctx->dif_pi_format)) { 2084 goto end; 2085 } 2086 break; 2087 case SPDK_DIF_TYPE3: 2088 /* If Type 3 is used, then all DIF checks are disabled when the 2089 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF 2090 * or 0xFFFFFFFFFFFFFFFF depending on the PI format. 2091 */ 2092 if (_dif_apptag_ignore(&dif, ctx->dif_pi_format) && 2093 _dif_reftag_ignore(&dif, ctx->dif_pi_format)) { 2094 goto end; 2095 } 2096 break; 2097 default: 2098 break; 2099 } 2100 2101 /* For type 1 and 2, the Reference Tag is incremented for each 2102 * subsequent logical block. For type 3, the Reference Tag 2103 * remains the same as the initial Reference Tag. 2104 */ 2105 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2106 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2107 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2108 } else { 2109 remapped = ctx->remapped_init_ref_tag; 2110 } 2111 2112 /* Verify the stored Reference Tag. */ 2113 switch (ctx->dif_type) { 2114 case SPDK_DIF_TYPE1: 2115 case SPDK_DIF_TYPE2: 2116 /* Compare the DIF Reference Tag field to the passed Reference Tag. 2117 * The passed Reference Tag will be the least significant 4 bytes 2118 * or 8 bytes (depending on the PI format) 2119 * of the LBA when Type 1 is used, and application specific value 2120 * if Type 2 is used. 2121 */ 2122 if (!_dif_reftag_match(&dif, expected, ctx->dif_pi_format)) { 2123 _actual = _dif_get_reftag(&dif, ctx->dif_pi_format); 2124 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected, 2125 _actual, offset_blocks); 2126 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu64 "," \ 2127 " Expected=%lx, Actual=%lx\n", 2128 expected, expected, _actual); 2129 return -1; 2130 } 2131 break; 2132 case SPDK_DIF_TYPE3: 2133 /* For type 3, the computed Reference Tag remains unchanged. 2134 * Hence ignore the Reference Tag field. 2135 */ 2136 break; 2137 default: 2138 break; 2139 } 2140 2141 /* Update the stored Reference Tag to the remapped one. */ 2142 _dif_set_reftag(&dif, remapped, ctx->dif_pi_format); 2143 2144 offset = 0; 2145 while (offset < _dif_size(ctx->dif_pi_format)) { 2146 _dif_sgl_get_buf(&tmp_sgl, &buf, &buf_len); 2147 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 2148 2149 memcpy(buf, (uint8_t *)&dif + offset, buf_len); 2150 2151 _dif_sgl_advance(&tmp_sgl, buf_len); 2152 offset += buf_len; 2153 } 2154 2155 end: 2156 _dif_sgl_advance(sgl, ctx->block_size - ctx->guard_interval - _dif_size(ctx->dif_pi_format)); 2157 2158 return 0; 2159 } 2160 2161 int 2162 spdk_dif_remap_ref_tag(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 2163 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 2164 { 2165 struct _dif_sgl sgl; 2166 uint32_t offset_blocks; 2167 int rc; 2168 2169 _dif_sgl_init(&sgl, iovs, iovcnt); 2170 2171 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 2172 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2173 return -EINVAL; 2174 } 2175 2176 if (_dif_is_disabled(ctx->dif_type)) { 2177 return 0; 2178 } 2179 2180 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2181 return 0; 2182 } 2183 2184 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2185 rc = _dif_remap_ref_tag(&sgl, offset_blocks, ctx, err_blk); 2186 if (rc != 0) { 2187 return rc; 2188 } 2189 } 2190 2191 return 0; 2192 } 2193 2194 static int 2195 _dix_remap_ref_tag(struct _dif_sgl *md_sgl, uint32_t offset_blocks, 2196 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 2197 { 2198 uint64_t expected = 0, _actual, remapped; 2199 uint8_t *md_buf; 2200 struct spdk_dif *dif; 2201 2202 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 2203 2204 dif = (struct spdk_dif *)(md_buf + ctx->guard_interval); 2205 2206 switch (ctx->dif_type) { 2207 case SPDK_DIF_TYPE1: 2208 case SPDK_DIF_TYPE2: 2209 /* If Type 1 or 2 is used, then all DIF checks are disabled when 2210 * the Application Tag is 0xFFFF. 2211 */ 2212 if (_dif_apptag_ignore(dif, ctx->dif_pi_format)) { 2213 goto end; 2214 } 2215 break; 2216 case SPDK_DIF_TYPE3: 2217 /* If Type 3 is used, then all DIF checks are disabled when the 2218 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF 2219 * or 0xFFFFFFFFFFFFFFFF depending on the PI format. 2220 */ 2221 if (_dif_apptag_ignore(dif, ctx->dif_pi_format) && 2222 _dif_reftag_ignore(dif, ctx->dif_pi_format)) { 2223 goto end; 2224 } 2225 break; 2226 default: 2227 break; 2228 } 2229 2230 /* For type 1 and 2, the Reference Tag is incremented for each 2231 * subsequent logical block. For type 3, the Reference Tag 2232 * remains the same as the initialReference Tag. 2233 */ 2234 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2235 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2236 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2237 } else { 2238 remapped = ctx->remapped_init_ref_tag; 2239 } 2240 2241 /* Verify the stored Reference Tag. */ 2242 switch (ctx->dif_type) { 2243 case SPDK_DIF_TYPE1: 2244 case SPDK_DIF_TYPE2: 2245 /* Compare the DIF Reference Tag field to the passed Reference Tag. 2246 * The passed Reference Tag will be the least significant 4 bytes 2247 * or 8 bytes (depending on the PI format) 2248 * of the LBA when Type 1 is used, and application specific value 2249 * if Type 2 is used. 2250 */ 2251 if (!_dif_reftag_match(dif, expected, ctx->dif_pi_format)) { 2252 _actual = _dif_get_reftag(dif, ctx->dif_pi_format); 2253 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected, 2254 _actual, offset_blocks); 2255 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu64 "," \ 2256 " Expected=%lx, Actual=%lx\n", 2257 expected, expected, _actual); 2258 return -1; 2259 } 2260 break; 2261 case SPDK_DIF_TYPE3: 2262 /* For type 3, the computed Reference Tag remains unchanged. 2263 * Hence ignore the Reference Tag field. 2264 */ 2265 break; 2266 default: 2267 break; 2268 } 2269 2270 /* Update the stored Reference Tag to the remapped one. */ 2271 _dif_set_reftag(dif, remapped, ctx->dif_pi_format); 2272 2273 end: 2274 _dif_sgl_advance(md_sgl, ctx->md_size); 2275 2276 return 0; 2277 } 2278 2279 int 2280 spdk_dix_remap_ref_tag(struct iovec *md_iov, uint32_t num_blocks, 2281 const struct spdk_dif_ctx *ctx, 2282 struct spdk_dif_error *err_blk) 2283 { 2284 struct _dif_sgl md_sgl; 2285 uint32_t offset_blocks; 2286 int rc; 2287 2288 _dif_sgl_init(&md_sgl, md_iov, 1); 2289 2290 if (!_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2291 SPDK_ERRLOG("Size of metadata iovec array is not valid.\n"); 2292 return -EINVAL; 2293 } 2294 2295 if (_dif_is_disabled(ctx->dif_type)) { 2296 return 0; 2297 } 2298 2299 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2300 return 0; 2301 } 2302 2303 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2304 rc = _dix_remap_ref_tag(&md_sgl, offset_blocks, ctx, err_blk); 2305 if (rc != 0) { 2306 return rc; 2307 } 2308 } 2309 2310 return 0; 2311 } 2312