1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/dif.h" 7 #include "spdk/crc16.h" 8 #include "spdk/crc32.h" 9 #include "spdk/crc64.h" 10 #include "spdk/endian.h" 11 #include "spdk/log.h" 12 #include "spdk/util.h" 13 14 #define REFTAG_MASK_16 0x00000000FFFFFFFF 15 #define REFTAG_MASK_32 0xFFFFFFFFFFFFFFFF 16 #define REFTAG_MASK_64 0x0000FFFFFFFFFFFF 17 18 /* The variable size Storage Tag and Reference Tag is not supported yet, 19 * so the maximum size of the Reference Tag is assumed. 20 */ 21 struct spdk_dif { 22 union { 23 struct { 24 uint16_t guard; 25 uint16_t app_tag; 26 uint32_t stor_ref_space; 27 } g16; 28 struct { 29 uint32_t guard; 30 uint16_t app_tag; 31 uint16_t stor_ref_space_p1; 32 uint64_t stor_ref_space_p2; 33 } g32; 34 struct { 35 uint64_t guard; 36 uint16_t app_tag; 37 uint16_t stor_ref_space_p1; 38 uint32_t stor_ref_space_p2; 39 } g64; 40 }; 41 }; 42 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g16) == 8, "Incorrect size"); 43 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g32) == 16, "Incorrect size"); 44 SPDK_STATIC_ASSERT(SPDK_SIZEOF_MEMBER(struct spdk_dif, g64) == 16, "Incorrect size"); 45 46 /* Context to iterate or create a iovec array. 47 * Each sgl is either iterated or created at a time. 48 */ 49 struct _dif_sgl { 50 /* Current iovec in the iteration or creation */ 51 struct iovec *iov; 52 53 /* Remaining count of iovecs in the iteration or creation. */ 54 int iovcnt; 55 56 /* Current offset in the iovec */ 57 uint32_t iov_offset; 58 59 /* Size of the created iovec array in bytes */ 60 uint32_t total_size; 61 }; 62 63 static inline void 64 _dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt) 65 { 66 s->iov = iovs; 67 s->iovcnt = iovcnt; 68 s->iov_offset = 0; 69 s->total_size = 0; 70 } 71 72 static void 73 _dif_sgl_advance(struct _dif_sgl *s, uint32_t step) 74 { 75 s->iov_offset += step; 76 while (s->iovcnt != 0) { 77 if (s->iov_offset < s->iov->iov_len) { 78 break; 79 } 80 81 s->iov_offset -= s->iov->iov_len; 82 s->iov++; 83 s->iovcnt--; 84 } 85 } 86 87 static inline void 88 _dif_sgl_get_buf(struct _dif_sgl *s, uint8_t **_buf, uint32_t *_buf_len) 89 { 90 if (_buf != NULL) { 91 *_buf = (uint8_t *)s->iov->iov_base + s->iov_offset; 92 } 93 if (_buf_len != NULL) { 94 *_buf_len = s->iov->iov_len - s->iov_offset; 95 } 96 } 97 98 static inline bool 99 _dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len) 100 { 101 assert(s->iovcnt > 0); 102 s->iov->iov_base = data; 103 s->iov->iov_len = data_len; 104 s->total_size += data_len; 105 s->iov++; 106 s->iovcnt--; 107 108 if (s->iovcnt > 0) { 109 return true; 110 } else { 111 return false; 112 } 113 } 114 115 static inline bool 116 _dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len) 117 { 118 uint8_t *buf; 119 uint32_t buf_len; 120 121 while (data_len != 0) { 122 _dif_sgl_get_buf(src, &buf, &buf_len); 123 buf_len = spdk_min(buf_len, data_len); 124 125 if (!_dif_sgl_append(dst, buf, buf_len)) { 126 return false; 127 } 128 129 _dif_sgl_advance(src, buf_len); 130 data_len -= buf_len; 131 } 132 133 return true; 134 } 135 136 /* This function must be used before starting iteration. */ 137 static bool 138 _dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes) 139 { 140 int i; 141 142 for (i = 0; i < s->iovcnt; i++) { 143 if (s->iov[i].iov_len % bytes) { 144 return false; 145 } 146 } 147 148 return true; 149 } 150 151 static bool 152 _dif_sgl_is_valid_block_aligned(struct _dif_sgl *s, uint32_t num_blocks, uint32_t block_size) 153 { 154 uint32_t count = 0; 155 int i; 156 157 for (i = 0; i < s->iovcnt; i++) { 158 if (s->iov[i].iov_len % block_size) { 159 return false; 160 } 161 count += s->iov[i].iov_len / block_size; 162 } 163 164 return count >= num_blocks; 165 } 166 167 /* This function must be used before starting iteration. */ 168 static bool 169 _dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes) 170 { 171 uint64_t total = 0; 172 int i; 173 174 for (i = 0; i < s->iovcnt; i++) { 175 total += s->iov[i].iov_len; 176 } 177 178 return total >= bytes; 179 } 180 181 static void 182 _dif_sgl_copy(struct _dif_sgl *to, struct _dif_sgl *from) 183 { 184 memcpy(to, from, sizeof(struct _dif_sgl)); 185 } 186 187 static bool 188 _dif_is_disabled(enum spdk_dif_type dif_type) 189 { 190 if (dif_type == SPDK_DIF_DISABLE) { 191 return true; 192 } else { 193 return false; 194 } 195 } 196 197 static inline size_t 198 _dif_size(enum spdk_dif_pi_format dif_pi_format) 199 { 200 uint8_t size; 201 202 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 203 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16); 204 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 205 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32); 206 } else { 207 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64); 208 } 209 210 return size; 211 } 212 213 static uint32_t 214 _get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave, 215 size_t dif_size) 216 { 217 if (!dif_loc) { 218 /* For metadata formats with more than 8/16 bytes (depending on 219 * the PI format), if the DIF is contained in the last 8/16 bytes 220 * of metadata, then the CRC covers all metadata up to but excluding 221 * these last 8/16 bytes. 222 */ 223 if (md_interleave) { 224 return block_size - dif_size; 225 } else { 226 return md_size - dif_size; 227 } 228 } else { 229 /* For metadata formats with more than 8/16 bytes (depending on 230 * the PI format), if the DIF is contained in the first 8/16 bytes 231 * of metadata, then the CRC does not cover any metadata. 232 */ 233 if (md_interleave) { 234 return block_size - md_size; 235 } else { 236 return 0; 237 } 238 } 239 } 240 241 static inline uint8_t 242 _dif_guard_size(enum spdk_dif_pi_format dif_pi_format) 243 { 244 uint8_t size; 245 246 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 247 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.guard); 248 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 249 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.guard); 250 } else { 251 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.guard); 252 } 253 254 return size; 255 } 256 257 static inline void 258 _dif_set_guard(struct spdk_dif *dif, uint64_t guard, enum spdk_dif_pi_format dif_pi_format) 259 { 260 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 261 to_be16(&(dif->g16.guard), (uint16_t)guard); 262 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 263 to_be32(&(dif->g32.guard), (uint32_t)guard); 264 } else { 265 to_be64(&(dif->g64.guard), guard); 266 } 267 } 268 269 static inline uint64_t 270 _dif_get_guard(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 271 { 272 uint64_t guard; 273 274 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 275 guard = (uint64_t)from_be16(&(dif->g16.guard)); 276 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 277 guard = (uint64_t)from_be32(&(dif->g32.guard)); 278 } else { 279 guard = from_be64(&(dif->g64.guard)); 280 } 281 282 return guard; 283 } 284 285 static inline uint64_t 286 _dif_generate_guard(uint64_t guard_seed, void *buf, size_t buf_len, 287 enum spdk_dif_pi_format dif_pi_format) 288 { 289 uint64_t guard; 290 291 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 292 guard = (uint64_t)spdk_crc16_t10dif((uint16_t)guard_seed, buf, buf_len); 293 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 294 guard = (uint64_t)spdk_crc32c_nvme(buf, buf_len, guard_seed); 295 } else { 296 guard = spdk_crc64_nvme(buf, buf_len, guard_seed); 297 } 298 299 return guard; 300 } 301 302 static inline uint64_t 303 _dif_generate_guard_copy(uint64_t guard_seed, void *dst, void *src, size_t buf_len, 304 enum spdk_dif_pi_format dif_pi_format) 305 { 306 uint64_t guard; 307 308 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 309 guard = (uint64_t)spdk_crc16_t10dif_copy((uint16_t)guard_seed, dst, src, buf_len); 310 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 311 memcpy(dst, src, buf_len); 312 guard = (uint64_t)spdk_crc32c_nvme(src, buf_len, guard_seed); 313 } else { 314 memcpy(dst, src, buf_len); 315 guard = spdk_crc64_nvme(src, buf_len, guard_seed); 316 } 317 318 return guard; 319 } 320 321 static inline uint8_t 322 _dif_apptag_offset(enum spdk_dif_pi_format dif_pi_format) 323 { 324 return _dif_guard_size(dif_pi_format); 325 } 326 327 static inline uint8_t 328 _dif_apptag_size(void) 329 { 330 return SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.app_tag); 331 } 332 333 static inline void 334 _dif_set_apptag(struct spdk_dif *dif, uint16_t app_tag, enum spdk_dif_pi_format dif_pi_format) 335 { 336 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 337 to_be16(&(dif->g16.app_tag), app_tag); 338 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 339 to_be16(&(dif->g32.app_tag), app_tag); 340 } else { 341 to_be16(&(dif->g64.app_tag), app_tag); 342 } 343 } 344 345 static inline uint16_t 346 _dif_get_apptag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 347 { 348 uint16_t app_tag; 349 350 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 351 app_tag = from_be16(&(dif->g16.app_tag)); 352 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 353 app_tag = from_be16(&(dif->g32.app_tag)); 354 } else { 355 app_tag = from_be16(&(dif->g64.app_tag)); 356 } 357 358 return app_tag; 359 } 360 361 static inline bool 362 _dif_apptag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 363 { 364 return _dif_get_apptag(dif, dif_pi_format) == SPDK_DIF_APPTAG_IGNORE; 365 } 366 367 static inline uint8_t 368 _dif_reftag_offset(enum spdk_dif_pi_format dif_pi_format) 369 { 370 uint8_t offset; 371 372 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 373 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size(); 374 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 375 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size() 376 + SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.stor_ref_space_p1); 377 } else { 378 offset = _dif_apptag_offset(dif_pi_format) + _dif_apptag_size(); 379 } 380 381 return offset; 382 } 383 384 static inline uint8_t 385 _dif_reftag_size(enum spdk_dif_pi_format dif_pi_format) 386 { 387 uint8_t size; 388 389 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 390 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g16.stor_ref_space); 391 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 392 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g32.stor_ref_space_p2); 393 } else { 394 size = SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.stor_ref_space_p1) + 395 SPDK_SIZEOF_MEMBER(struct spdk_dif, g64.stor_ref_space_p2); 396 } 397 398 return size; 399 } 400 401 static inline void 402 _dif_set_reftag(struct spdk_dif *dif, uint64_t ref_tag, enum spdk_dif_pi_format dif_pi_format) 403 { 404 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 405 to_be32(&(dif->g16.stor_ref_space), (uint32_t)ref_tag); 406 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 407 to_be64(&(dif->g32.stor_ref_space_p2), ref_tag); 408 } else { 409 to_be16(&(dif->g64.stor_ref_space_p1), (uint16_t)(ref_tag >> 32)); 410 to_be32(&(dif->g64.stor_ref_space_p2), (uint32_t)ref_tag); 411 } 412 } 413 414 static inline uint64_t 415 _dif_get_reftag(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 416 { 417 uint64_t ref_tag; 418 419 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 420 ref_tag = (uint64_t)from_be32(&(dif->g16.stor_ref_space)); 421 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 422 ref_tag = from_be64(&(dif->g32.stor_ref_space_p2)); 423 } else { 424 ref_tag = (uint64_t)from_be16(&(dif->g64.stor_ref_space_p1)); 425 ref_tag <<= 32; 426 ref_tag |= (uint64_t)from_be32(&(dif->g64.stor_ref_space_p2)); 427 } 428 429 return ref_tag; 430 } 431 432 static inline bool 433 _dif_reftag_match(struct spdk_dif *dif, uint64_t ref_tag, 434 enum spdk_dif_pi_format dif_pi_format) 435 { 436 uint64_t _ref_tag; 437 bool match; 438 439 _ref_tag = _dif_get_reftag(dif, dif_pi_format); 440 441 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 442 match = (_ref_tag == (ref_tag & REFTAG_MASK_16)); 443 } else if (dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 444 match = (_ref_tag == ref_tag); 445 } else { 446 match = (_ref_tag == (ref_tag & REFTAG_MASK_64)); 447 } 448 449 return match; 450 } 451 452 static inline bool 453 _dif_reftag_ignore(struct spdk_dif *dif, enum spdk_dif_pi_format dif_pi_format) 454 { 455 return _dif_reftag_match(dif, REFTAG_MASK_32, dif_pi_format); 456 } 457 458 static bool 459 _dif_ignore(struct spdk_dif *dif, const struct spdk_dif_ctx *ctx) 460 { 461 switch (ctx->dif_type) { 462 case SPDK_DIF_TYPE1: 463 case SPDK_DIF_TYPE2: 464 /* If Type 1 or 2 is used, then all DIF checks are disabled when 465 * the Application Tag is 0xFFFF. 466 */ 467 if (_dif_apptag_ignore(dif, ctx->dif_pi_format)) { 468 return true; 469 } 470 break; 471 case SPDK_DIF_TYPE3: 472 /* If Type 3 is used, then all DIF checks are disabled when the 473 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF 474 * or 0xFFFFFFFFFFFFFFFF depending on the PI format. 475 */ 476 477 if (_dif_apptag_ignore(dif, ctx->dif_pi_format) && 478 _dif_reftag_ignore(dif, ctx->dif_pi_format)) { 479 return true; 480 } 481 break; 482 default: 483 break; 484 } 485 486 return false; 487 } 488 489 int 490 spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, 491 bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, 492 uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag, 493 uint32_t data_offset, uint64_t guard_seed, struct spdk_dif_ctx_init_ext_opts *opts) 494 { 495 uint32_t data_block_size; 496 enum spdk_dif_pi_format dif_pi_format = SPDK_DIF_PI_FORMAT_16; 497 498 if (opts != NULL) { 499 if (opts->dif_pi_format != SPDK_DIF_PI_FORMAT_16 && 500 opts->dif_pi_format != SPDK_DIF_PI_FORMAT_32 && 501 opts->dif_pi_format != SPDK_DIF_PI_FORMAT_64) { 502 SPDK_ERRLOG("No valid DIF PI format provided.\n"); 503 return -EINVAL; 504 } 505 506 dif_pi_format = opts->dif_pi_format; 507 } 508 509 if (md_size < _dif_size(dif_pi_format)) { 510 SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); 511 return -EINVAL; 512 } 513 514 if (md_interleave) { 515 if (block_size < md_size) { 516 SPDK_ERRLOG("Block size is smaller than DIF size.\n"); 517 return -EINVAL; 518 } 519 data_block_size = block_size - md_size; 520 } else { 521 if (dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 522 if (block_size == 0 || (block_size % 512) != 0) { 523 SPDK_ERRLOG("Zero block size is not allowed and should be a multiple of 512B\n"); 524 return -EINVAL; 525 } 526 } else { 527 if (block_size == 0 || (block_size % 4096) != 0) { 528 SPDK_ERRLOG("Zero block size is not allowed and should be a multiple of 4kB\n"); 529 return -EINVAL; 530 } 531 } 532 533 data_block_size = block_size; 534 } 535 536 ctx->block_size = block_size; 537 ctx->md_size = md_size; 538 ctx->md_interleave = md_interleave; 539 ctx->dif_pi_format = dif_pi_format; 540 ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave, 541 _dif_size(ctx->dif_pi_format)); 542 ctx->dif_type = dif_type; 543 ctx->dif_flags = dif_flags; 544 ctx->init_ref_tag = init_ref_tag; 545 ctx->apptag_mask = apptag_mask; 546 ctx->app_tag = app_tag; 547 ctx->data_offset = data_offset; 548 ctx->ref_tag_offset = data_offset / data_block_size; 549 ctx->last_guard = guard_seed; 550 ctx->guard_seed = guard_seed; 551 ctx->remapped_init_ref_tag = 0; 552 553 return 0; 554 } 555 556 void 557 spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset) 558 { 559 uint32_t data_block_size; 560 561 if (ctx->md_interleave) { 562 data_block_size = ctx->block_size - ctx->md_size; 563 } else { 564 data_block_size = ctx->block_size; 565 } 566 567 ctx->data_offset = data_offset; 568 ctx->ref_tag_offset = data_offset / data_block_size; 569 } 570 571 void 572 spdk_dif_ctx_set_remapped_init_ref_tag(struct spdk_dif_ctx *ctx, 573 uint32_t remapped_init_ref_tag) 574 { 575 ctx->remapped_init_ref_tag = remapped_init_ref_tag; 576 } 577 578 static void 579 _dif_generate(void *_dif, uint64_t guard, uint32_t offset_blocks, 580 const struct spdk_dif_ctx *ctx) 581 { 582 struct spdk_dif *dif = _dif; 583 uint64_t ref_tag; 584 585 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 586 _dif_set_guard(dif, guard, ctx->dif_pi_format); 587 } 588 589 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 590 _dif_set_apptag(dif, ctx->app_tag, ctx->dif_pi_format); 591 } 592 593 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 594 /* For type 1 and 2, the reference tag is incremented for each 595 * subsequent logical block. For type 3, the reference tag 596 * remains the same as the initial reference tag. 597 */ 598 if (ctx->dif_type != SPDK_DIF_TYPE3) { 599 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 600 } else { 601 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 602 } 603 604 /* Overwrite reference tag if initialization reference tag is SPDK_DIF_REFTAG_IGNORE */ 605 if (ctx->init_ref_tag == SPDK_DIF_REFTAG_IGNORE) { 606 if (ctx->dif_pi_format == SPDK_DIF_PI_FORMAT_16) { 607 ref_tag = REFTAG_MASK_16; 608 } else if (ctx->dif_pi_format == SPDK_DIF_PI_FORMAT_32) { 609 ref_tag = REFTAG_MASK_32; 610 } else { 611 ref_tag = REFTAG_MASK_64; 612 } 613 } 614 615 _dif_set_reftag(dif, ref_tag, ctx->dif_pi_format); 616 } 617 } 618 619 static void 620 dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 621 { 622 uint32_t offset_blocks = 0; 623 uint8_t *buf; 624 uint64_t guard = 0; 625 626 while (offset_blocks < num_blocks) { 627 _dif_sgl_get_buf(sgl, &buf, NULL); 628 629 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 630 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 631 } 632 633 _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); 634 635 _dif_sgl_advance(sgl, ctx->block_size); 636 offset_blocks++; 637 } 638 } 639 640 static uint64_t 641 _dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 642 uint64_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 643 { 644 uint32_t offset_in_dif, buf_len; 645 uint8_t *buf; 646 struct spdk_dif dif = {}; 647 648 assert(offset_in_block < ctx->guard_interval); 649 assert(offset_in_block + data_len < ctx->guard_interval || 650 offset_in_block + data_len == ctx->block_size); 651 652 /* Compute CRC over split logical block data. */ 653 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 654 _dif_sgl_get_buf(sgl, &buf, &buf_len); 655 buf_len = spdk_min(buf_len, data_len); 656 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 657 658 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 659 guard = _dif_generate_guard(guard, buf, buf_len, ctx->dif_pi_format); 660 } 661 662 _dif_sgl_advance(sgl, buf_len); 663 offset_in_block += buf_len; 664 data_len -= buf_len; 665 } 666 667 if (offset_in_block < ctx->guard_interval) { 668 return guard; 669 } 670 671 /* If a whole logical block data is parsed, generate DIF 672 * and save it to the temporary DIF area. 673 */ 674 _dif_generate(&dif, guard, offset_blocks, ctx); 675 676 /* Copy generated DIF field to the split DIF field, and then 677 * skip metadata field after DIF field (if any). 678 */ 679 while (offset_in_block < ctx->block_size) { 680 _dif_sgl_get_buf(sgl, &buf, &buf_len); 681 682 if (offset_in_block < ctx->guard_interval + _dif_size(ctx->dif_pi_format)) { 683 offset_in_dif = offset_in_block - ctx->guard_interval; 684 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset_in_dif); 685 686 memcpy(buf, ((uint8_t *)&dif) + offset_in_dif, buf_len); 687 } else { 688 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 689 } 690 691 _dif_sgl_advance(sgl, buf_len); 692 offset_in_block += buf_len; 693 } 694 695 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 696 guard = ctx->guard_seed; 697 } 698 699 return guard; 700 } 701 702 static void 703 dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks, 704 const struct spdk_dif_ctx *ctx) 705 { 706 uint32_t offset_blocks; 707 uint64_t guard = 0; 708 709 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 710 guard = ctx->guard_seed; 711 } 712 713 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 714 _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx); 715 } 716 } 717 718 int 719 spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 720 const struct spdk_dif_ctx *ctx) 721 { 722 struct _dif_sgl sgl; 723 724 _dif_sgl_init(&sgl, iovs, iovcnt); 725 726 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 727 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 728 return -EINVAL; 729 } 730 731 if (_dif_is_disabled(ctx->dif_type)) { 732 return 0; 733 } 734 735 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 736 dif_generate(&sgl, num_blocks, ctx); 737 } else { 738 dif_generate_split(&sgl, num_blocks, ctx); 739 } 740 741 return 0; 742 } 743 744 static void 745 _dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, 746 uint64_t expected, uint64_t actual, uint32_t err_offset) 747 { 748 if (err_blk) { 749 err_blk->err_type = err_type; 750 err_blk->expected = expected; 751 err_blk->actual = actual; 752 err_blk->err_offset = err_offset; 753 } 754 } 755 756 static bool 757 _dif_reftag_check(struct spdk_dif *dif, const struct spdk_dif_ctx *ctx, 758 uint64_t expected_reftag, uint32_t offset_blocks, struct spdk_dif_error *err_blk) 759 { 760 uint64_t reftag; 761 762 if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { 763 switch (ctx->dif_type) { 764 case SPDK_DIF_TYPE1: 765 case SPDK_DIF_TYPE2: 766 /* Compare the DIF Reference Tag field to the passed Reference Tag. 767 * The passed Reference Tag will be the least significant 4 bytes 768 * or 8 bytes (depending on the PI format) 769 * of the LBA when Type 1 is used, and application specific value 770 * if Type 2 is used. 771 */ 772 if (!_dif_reftag_match(dif, expected_reftag, ctx->dif_pi_format)) { 773 reftag = _dif_get_reftag(dif, ctx->dif_pi_format); 774 _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected_reftag, 775 reftag, offset_blocks); 776 SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu64 "," \ 777 " Expected=%lx, Actual=%lx\n", 778 expected_reftag, expected_reftag, reftag); 779 return false; 780 } 781 break; 782 case SPDK_DIF_TYPE3: 783 /* For Type 3, computed Reference Tag remains unchanged. 784 * Hence ignore the Reference Tag field. 785 */ 786 break; 787 default: 788 break; 789 } 790 } 791 792 return true; 793 } 794 795 static int 796 _dif_verify(void *_dif, uint64_t guard, uint32_t offset_blocks, 797 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 798 { 799 struct spdk_dif *dif = _dif; 800 uint64_t _guard; 801 uint16_t _app_tag; 802 uint64_t ref_tag; 803 804 if (_dif_ignore(dif, ctx)) { 805 return 0; 806 } 807 808 /* For type 1 and 2, the reference tag is incremented for each 809 * subsequent logical block. For type 3, the reference tag 810 * remains the same as the initial reference tag. 811 */ 812 if (ctx->dif_type != SPDK_DIF_TYPE3) { 813 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 814 } else { 815 ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; 816 } 817 818 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 819 /* Compare the DIF Guard field to the CRC computed over the logical 820 * block data. 821 */ 822 _guard = _dif_get_guard(dif, ctx->dif_pi_format); 823 if (_guard != guard) { 824 _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, 825 offset_blocks); 826 SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu64 "," \ 827 " Expected=%lx, Actual=%lx\n", 828 ref_tag, _guard, guard); 829 return -1; 830 } 831 } 832 833 if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { 834 /* Compare unmasked bits in the DIF Application Tag field to the 835 * passed Application Tag. 836 */ 837 _app_tag = _dif_get_apptag(dif, ctx->dif_pi_format); 838 if ((_app_tag & ctx->apptag_mask) != (ctx->app_tag & ctx->apptag_mask)) { 839 _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, 840 (_app_tag & ctx->apptag_mask), offset_blocks); 841 SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu64 "," \ 842 " Expected=%x, Actual=%x\n", 843 ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); 844 return -1; 845 } 846 } 847 848 if (!_dif_reftag_check(dif, ctx, ref_tag, offset_blocks, err_blk)) { 849 return -1; 850 } 851 852 return 0; 853 } 854 855 static int 856 dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks, 857 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 858 { 859 uint32_t offset_blocks = 0; 860 int rc; 861 uint8_t *buf; 862 uint64_t guard = 0; 863 864 while (offset_blocks < num_blocks) { 865 _dif_sgl_get_buf(sgl, &buf, NULL); 866 867 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 868 guard = _dif_generate_guard(ctx->guard_seed, buf, ctx->guard_interval, ctx->dif_pi_format); 869 } 870 871 rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 872 if (rc != 0) { 873 return rc; 874 } 875 876 _dif_sgl_advance(sgl, ctx->block_size); 877 offset_blocks++; 878 } 879 880 return 0; 881 } 882 883 static int 884 _dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 885 uint64_t *_guard, uint32_t offset_blocks, 886 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 887 { 888 uint32_t offset_in_dif, buf_len; 889 uint8_t *buf; 890 uint64_t guard; 891 struct spdk_dif dif = {}; 892 int rc; 893 894 assert(_guard != NULL); 895 assert(offset_in_block < ctx->guard_interval); 896 assert(offset_in_block + data_len < ctx->guard_interval || 897 offset_in_block + data_len == ctx->block_size); 898 899 guard = *_guard; 900 901 /* Compute CRC over split logical block data. */ 902 while (data_len != 0 && offset_in_block < ctx->guard_interval) { 903 _dif_sgl_get_buf(sgl, &buf, &buf_len); 904 buf_len = spdk_min(buf_len, data_len); 905 buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); 906 907 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 908 guard = _dif_generate_guard(guard, buf, buf_len, ctx->dif_pi_format); 909 } 910 911 _dif_sgl_advance(sgl, buf_len); 912 offset_in_block += buf_len; 913 data_len -= buf_len; 914 } 915 916 if (offset_in_block < ctx->guard_interval) { 917 *_guard = guard; 918 return 0; 919 } 920 921 /* Copy the split DIF field to the temporary DIF buffer, and then 922 * skip metadata field after DIF field (if any). */ 923 while (offset_in_block < ctx->block_size) { 924 _dif_sgl_get_buf(sgl, &buf, &buf_len); 925 926 if (offset_in_block < ctx->guard_interval + _dif_size(ctx->dif_pi_format)) { 927 offset_in_dif = offset_in_block - ctx->guard_interval; 928 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset_in_dif); 929 930 memcpy((uint8_t *)&dif + offset_in_dif, buf, buf_len); 931 } else { 932 buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); 933 } 934 _dif_sgl_advance(sgl, buf_len); 935 offset_in_block += buf_len; 936 } 937 938 rc = _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); 939 if (rc != 0) { 940 return rc; 941 } 942 943 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 944 guard = ctx->guard_seed; 945 } 946 947 *_guard = guard; 948 return 0; 949 } 950 951 static int 952 dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks, 953 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 954 { 955 uint32_t offset_blocks; 956 uint64_t guard = 0; 957 int rc; 958 959 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 960 guard = ctx->guard_seed; 961 } 962 963 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 964 rc = _dif_verify_split(sgl, 0, ctx->block_size, &guard, offset_blocks, 965 ctx, err_blk); 966 if (rc != 0) { 967 return rc; 968 } 969 } 970 971 return 0; 972 } 973 974 int 975 spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 976 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) 977 { 978 struct _dif_sgl sgl; 979 980 _dif_sgl_init(&sgl, iovs, iovcnt); 981 982 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 983 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 984 return -EINVAL; 985 } 986 987 if (_dif_is_disabled(ctx->dif_type)) { 988 return 0; 989 } 990 991 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 992 return dif_verify(&sgl, num_blocks, ctx, err_blk); 993 } else { 994 return dif_verify_split(&sgl, num_blocks, ctx, err_blk); 995 } 996 } 997 998 static uint32_t 999 dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks, 1000 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1001 { 1002 uint32_t offset_blocks; 1003 uint8_t *buf; 1004 1005 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1006 _dif_sgl_get_buf(sgl, &buf, NULL); 1007 1008 crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c); 1009 1010 _dif_sgl_advance(sgl, ctx->block_size); 1011 } 1012 1013 return crc32c; 1014 } 1015 1016 static uint32_t 1017 _dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, 1018 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1019 { 1020 uint32_t data_block_size, buf_len; 1021 uint8_t *buf; 1022 1023 data_block_size = ctx->block_size - ctx->md_size; 1024 1025 assert(offset_in_block + data_len <= ctx->block_size); 1026 1027 while (data_len != 0) { 1028 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1029 buf_len = spdk_min(buf_len, data_len); 1030 1031 if (offset_in_block < data_block_size) { 1032 buf_len = spdk_min(buf_len, data_block_size - offset_in_block); 1033 crc32c = spdk_crc32c_update(buf, buf_len, crc32c); 1034 } 1035 1036 _dif_sgl_advance(sgl, buf_len); 1037 offset_in_block += buf_len; 1038 data_len -= buf_len; 1039 } 1040 1041 return crc32c; 1042 } 1043 1044 static uint32_t 1045 dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks, 1046 uint32_t crc32c, const struct spdk_dif_ctx *ctx) 1047 { 1048 uint32_t offset_blocks; 1049 1050 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1051 crc32c = _dif_update_crc32c_split(sgl, 0, ctx->block_size, crc32c, ctx); 1052 } 1053 1054 return crc32c; 1055 } 1056 1057 int 1058 spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1059 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 1060 { 1061 struct _dif_sgl sgl; 1062 1063 if (_crc32c == NULL) { 1064 return -EINVAL; 1065 } 1066 1067 _dif_sgl_init(&sgl, iovs, iovcnt); 1068 1069 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1070 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1071 return -EINVAL; 1072 } 1073 1074 if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { 1075 *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx); 1076 } else { 1077 *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx); 1078 } 1079 1080 return 0; 1081 } 1082 1083 static void 1084 dif_generate_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1085 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1086 { 1087 uint32_t offset_blocks = 0, data_block_size; 1088 uint8_t *src, *dst; 1089 uint64_t guard; 1090 1091 data_block_size = ctx->block_size - ctx->md_size; 1092 1093 while (offset_blocks < num_blocks) { 1094 _dif_sgl_get_buf(src_sgl, &src, NULL); 1095 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1096 1097 guard = 0; 1098 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1099 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1100 ctx->dif_pi_format); 1101 guard = _dif_generate_guard(guard, dst + data_block_size, 1102 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1103 } else { 1104 memcpy(dst, src, data_block_size); 1105 } 1106 1107 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 1108 1109 _dif_sgl_advance(src_sgl, data_block_size); 1110 _dif_sgl_advance(dst_sgl, ctx->block_size); 1111 offset_blocks++; 1112 } 1113 } 1114 1115 static void 1116 _dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1117 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1118 { 1119 uint32_t offset_in_block, src_len, data_block_size; 1120 uint8_t *src, *dst; 1121 uint64_t guard = 0; 1122 1123 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1124 1125 data_block_size = ctx->block_size - ctx->md_size; 1126 1127 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1128 guard = ctx->guard_seed; 1129 } 1130 offset_in_block = 0; 1131 1132 while (offset_in_block < data_block_size) { 1133 /* Compute CRC over split logical block data and copy 1134 * data to bounce buffer. 1135 */ 1136 _dif_sgl_get_buf(src_sgl, &src, &src_len); 1137 src_len = spdk_min(src_len, data_block_size - offset_in_block); 1138 1139 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1140 guard = _dif_generate_guard_copy(guard, dst + offset_in_block, 1141 src, src_len, ctx->dif_pi_format); 1142 } else { 1143 memcpy(dst + offset_in_block, src, src_len); 1144 } 1145 1146 _dif_sgl_advance(src_sgl, src_len); 1147 offset_in_block += src_len; 1148 } 1149 1150 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1151 guard = _dif_generate_guard(guard, dst + data_block_size, 1152 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1153 } 1154 1155 _dif_sgl_advance(dst_sgl, ctx->block_size); 1156 1157 _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); 1158 } 1159 1160 static void 1161 dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1162 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1163 { 1164 uint32_t offset_blocks; 1165 1166 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1167 _dif_generate_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); 1168 } 1169 } 1170 1171 int 1172 spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1173 int bounce_iovcnt, uint32_t num_blocks, 1174 const struct spdk_dif_ctx *ctx) 1175 { 1176 struct _dif_sgl src_sgl, dst_sgl; 1177 uint32_t data_block_size; 1178 1179 _dif_sgl_init(&src_sgl, iovs, iovcnt); 1180 _dif_sgl_init(&dst_sgl, bounce_iovs, bounce_iovcnt); 1181 1182 data_block_size = ctx->block_size - ctx->md_size; 1183 1184 if (!_dif_sgl_is_valid(&src_sgl, data_block_size * num_blocks)) { 1185 SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); 1186 return -EINVAL; 1187 } 1188 1189 if (!_dif_sgl_is_valid_block_aligned(&dst_sgl, num_blocks, ctx->block_size)) { 1190 SPDK_ERRLOG("Size of bounce_iovs arrays are not valid or misaligned with block_size.\n"); 1191 return -EINVAL; 1192 } 1193 1194 if (_dif_is_disabled(ctx->dif_type)) { 1195 return 0; 1196 } 1197 1198 if (_dif_sgl_is_bytes_multiple(&src_sgl, data_block_size)) { 1199 dif_generate_copy(&src_sgl, &dst_sgl, num_blocks, ctx); 1200 } else { 1201 dif_generate_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx); 1202 } 1203 1204 return 0; 1205 } 1206 1207 static int 1208 dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1209 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1210 struct spdk_dif_error *err_blk) 1211 { 1212 uint32_t offset_blocks = 0, data_block_size; 1213 uint8_t *src, *dst; 1214 int rc; 1215 uint64_t guard; 1216 1217 data_block_size = ctx->block_size - ctx->md_size; 1218 1219 while (offset_blocks < num_blocks) { 1220 _dif_sgl_get_buf(src_sgl, &src, NULL); 1221 _dif_sgl_get_buf(dst_sgl, &dst, NULL); 1222 1223 guard = 0; 1224 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1225 guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, data_block_size, 1226 ctx->dif_pi_format); 1227 guard = _dif_generate_guard(guard, src + data_block_size, 1228 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1229 } else { 1230 memcpy(dst, src, data_block_size); 1231 } 1232 1233 rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1234 if (rc != 0) { 1235 return rc; 1236 } 1237 1238 _dif_sgl_advance(src_sgl, ctx->block_size); 1239 _dif_sgl_advance(dst_sgl, data_block_size); 1240 offset_blocks++; 1241 } 1242 1243 return 0; 1244 } 1245 1246 static int 1247 _dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1248 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1249 struct spdk_dif_error *err_blk) 1250 { 1251 uint32_t offset_in_block, dst_len, data_block_size; 1252 uint8_t *src, *dst; 1253 uint64_t guard = 0; 1254 1255 _dif_sgl_get_buf(src_sgl, &src, NULL); 1256 1257 data_block_size = ctx->block_size - ctx->md_size; 1258 1259 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1260 guard = ctx->guard_seed; 1261 } 1262 offset_in_block = 0; 1263 1264 while (offset_in_block < data_block_size) { 1265 /* Compute CRC over split logical block data and copy 1266 * data to bounce buffer. 1267 */ 1268 _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); 1269 dst_len = spdk_min(dst_len, data_block_size - offset_in_block); 1270 1271 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1272 guard = _dif_generate_guard_copy(guard, dst, src + offset_in_block, 1273 dst_len, ctx->dif_pi_format); 1274 } else { 1275 memcpy(dst, src + offset_in_block, dst_len); 1276 } 1277 1278 _dif_sgl_advance(dst_sgl, dst_len); 1279 offset_in_block += dst_len; 1280 } 1281 1282 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1283 guard = _dif_generate_guard(guard, src + data_block_size, 1284 ctx->guard_interval - data_block_size, ctx->dif_pi_format); 1285 } 1286 1287 _dif_sgl_advance(src_sgl, ctx->block_size); 1288 1289 return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1290 } 1291 1292 static int 1293 dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 1294 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1295 struct spdk_dif_error *err_blk) 1296 { 1297 uint32_t offset_blocks; 1298 int rc; 1299 1300 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1301 rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); 1302 if (rc != 0) { 1303 return rc; 1304 } 1305 } 1306 1307 return 0; 1308 } 1309 1310 int 1311 spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 1312 int bounce_iovcnt, uint32_t num_blocks, 1313 const struct spdk_dif_ctx *ctx, 1314 struct spdk_dif_error *err_blk) 1315 { 1316 struct _dif_sgl src_sgl, dst_sgl; 1317 uint32_t data_block_size; 1318 1319 _dif_sgl_init(&src_sgl, bounce_iovs, bounce_iovcnt); 1320 _dif_sgl_init(&dst_sgl, iovs, iovcnt); 1321 1322 data_block_size = ctx->block_size - ctx->md_size; 1323 1324 if (!_dif_sgl_is_valid(&dst_sgl, data_block_size * num_blocks)) { 1325 SPDK_ERRLOG("Size of iovec arrays are not valid\n"); 1326 return -EINVAL; 1327 } 1328 1329 if (!_dif_sgl_is_valid_block_aligned(&src_sgl, num_blocks, ctx->block_size)) { 1330 SPDK_ERRLOG("Size of bounce_iovs arrays are not valid or misaligned with block_size.\n"); 1331 return -EINVAL; 1332 } 1333 1334 if (_dif_is_disabled(ctx->dif_type)) { 1335 return 0; 1336 } 1337 1338 if (_dif_sgl_is_bytes_multiple(&dst_sgl, data_block_size)) { 1339 return dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1340 } else { 1341 return dif_verify_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); 1342 } 1343 } 1344 1345 static void 1346 _bit_flip(uint8_t *buf, uint32_t flip_bit) 1347 { 1348 uint8_t byte; 1349 1350 byte = *buf; 1351 byte ^= 1 << flip_bit; 1352 *buf = byte; 1353 } 1354 1355 static int 1356 _dif_inject_error(struct _dif_sgl *sgl, 1357 uint32_t block_size, uint32_t num_blocks, 1358 uint32_t inject_offset_blocks, 1359 uint32_t inject_offset_bytes, 1360 uint32_t inject_offset_bits) 1361 { 1362 uint32_t offset_in_block, buf_len; 1363 uint8_t *buf; 1364 1365 _dif_sgl_advance(sgl, block_size * inject_offset_blocks); 1366 1367 offset_in_block = 0; 1368 1369 while (offset_in_block < block_size) { 1370 _dif_sgl_get_buf(sgl, &buf, &buf_len); 1371 buf_len = spdk_min(buf_len, block_size - offset_in_block); 1372 1373 if (inject_offset_bytes >= offset_in_block && 1374 inject_offset_bytes < offset_in_block + buf_len) { 1375 buf += inject_offset_bytes - offset_in_block; 1376 _bit_flip(buf, inject_offset_bits); 1377 return 0; 1378 } 1379 1380 _dif_sgl_advance(sgl, buf_len); 1381 offset_in_block += buf_len; 1382 } 1383 1384 return -1; 1385 } 1386 1387 static int 1388 dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks, 1389 uint32_t start_inject_bytes, uint32_t inject_range_bytes, 1390 uint32_t *inject_offset) 1391 { 1392 uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; 1393 uint32_t offset_blocks; 1394 int rc; 1395 1396 srand(time(0)); 1397 1398 inject_offset_blocks = rand() % num_blocks; 1399 inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); 1400 inject_offset_bits = rand() % 8; 1401 1402 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1403 if (offset_blocks == inject_offset_blocks) { 1404 rc = _dif_inject_error(sgl, block_size, num_blocks, 1405 inject_offset_blocks, 1406 inject_offset_bytes, 1407 inject_offset_bits); 1408 if (rc == 0) { 1409 *inject_offset = inject_offset_blocks; 1410 } 1411 return rc; 1412 } 1413 } 1414 1415 return -1; 1416 } 1417 1418 int 1419 spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 1420 const struct spdk_dif_ctx *ctx, uint32_t inject_flags, 1421 uint32_t *inject_offset) 1422 { 1423 struct _dif_sgl sgl; 1424 int rc; 1425 1426 _dif_sgl_init(&sgl, iovs, iovcnt); 1427 1428 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 1429 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1430 return -EINVAL; 1431 } 1432 1433 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1434 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1435 ctx->guard_interval + _dif_reftag_offset(ctx->dif_pi_format), 1436 _dif_reftag_size(ctx->dif_pi_format), 1437 inject_offset); 1438 if (rc != 0) { 1439 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1440 return rc; 1441 } 1442 } 1443 1444 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1445 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1446 ctx->guard_interval + _dif_apptag_offset(ctx->dif_pi_format), 1447 _dif_apptag_size(), 1448 inject_offset); 1449 if (rc != 0) { 1450 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1451 return rc; 1452 } 1453 } 1454 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1455 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1456 ctx->guard_interval, 1457 _dif_guard_size(ctx->dif_pi_format), 1458 inject_offset); 1459 if (rc != 0) { 1460 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1461 return rc; 1462 } 1463 } 1464 1465 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1466 /* If the DIF information is contained within the last 8/16 bytes of 1467 * metadata (depending on the PI format), then the CRC covers all metadata 1468 * bytes up to but excluding the last 8/16 bytes. But error injection does not 1469 * cover these metadata because classification is not determined yet. 1470 * 1471 * Note: Error injection to data block is expected to be detected as 1472 * guard error. 1473 */ 1474 rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, 1475 0, 1476 ctx->block_size - ctx->md_size, 1477 inject_offset); 1478 if (rc != 0) { 1479 SPDK_ERRLOG("Failed to inject error to data block.\n"); 1480 return rc; 1481 } 1482 } 1483 1484 return 0; 1485 } 1486 1487 static void 1488 dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1489 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1490 { 1491 uint32_t offset_blocks = 0; 1492 uint8_t *data_buf, *md_buf; 1493 uint64_t guard; 1494 1495 while (offset_blocks < num_blocks) { 1496 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1497 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1498 1499 guard = 0; 1500 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1501 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1502 ctx->dif_pi_format); 1503 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1504 ctx->dif_pi_format); 1505 } 1506 1507 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1508 1509 _dif_sgl_advance(data_sgl, ctx->block_size); 1510 _dif_sgl_advance(md_sgl, ctx->md_size); 1511 offset_blocks++; 1512 } 1513 } 1514 1515 static void 1516 _dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1517 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) 1518 { 1519 uint32_t offset_in_block, data_buf_len; 1520 uint8_t *data_buf, *md_buf; 1521 uint64_t guard = 0; 1522 1523 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1524 1525 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1526 guard = ctx->guard_seed; 1527 } 1528 offset_in_block = 0; 1529 1530 while (offset_in_block < ctx->block_size) { 1531 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1532 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1533 1534 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1535 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 1536 ctx->dif_pi_format); 1537 } 1538 1539 _dif_sgl_advance(data_sgl, data_buf_len); 1540 offset_in_block += data_buf_len; 1541 } 1542 1543 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1544 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1545 ctx->dif_pi_format); 1546 } 1547 1548 _dif_sgl_advance(md_sgl, ctx->md_size); 1549 1550 _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); 1551 } 1552 1553 static void 1554 dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1555 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1556 { 1557 uint32_t offset_blocks; 1558 1559 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1560 _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx); 1561 } 1562 } 1563 1564 int 1565 spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1566 uint32_t num_blocks, const struct spdk_dif_ctx *ctx) 1567 { 1568 struct _dif_sgl data_sgl, md_sgl; 1569 1570 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1571 _dif_sgl_init(&md_sgl, md_iov, 1); 1572 1573 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1574 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1575 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1576 return -EINVAL; 1577 } 1578 1579 if (_dif_is_disabled(ctx->dif_type)) { 1580 return 0; 1581 } 1582 1583 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1584 dix_generate(&data_sgl, &md_sgl, num_blocks, ctx); 1585 } else { 1586 dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx); 1587 } 1588 1589 return 0; 1590 } 1591 1592 static int 1593 dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1594 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1595 struct spdk_dif_error *err_blk) 1596 { 1597 uint32_t offset_blocks = 0; 1598 uint8_t *data_buf, *md_buf; 1599 uint64_t guard; 1600 int rc; 1601 1602 while (offset_blocks < num_blocks) { 1603 _dif_sgl_get_buf(data_sgl, &data_buf, NULL); 1604 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1605 1606 guard = 0; 1607 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1608 guard = _dif_generate_guard(ctx->guard_seed, data_buf, ctx->block_size, 1609 ctx->dif_pi_format); 1610 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1611 ctx->dif_pi_format); 1612 } 1613 1614 rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1615 if (rc != 0) { 1616 return rc; 1617 } 1618 1619 _dif_sgl_advance(data_sgl, ctx->block_size); 1620 _dif_sgl_advance(md_sgl, ctx->md_size); 1621 offset_blocks++; 1622 } 1623 1624 return 0; 1625 } 1626 1627 static int 1628 _dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1629 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, 1630 struct spdk_dif_error *err_blk) 1631 { 1632 uint32_t offset_in_block, data_buf_len; 1633 uint8_t *data_buf, *md_buf; 1634 uint64_t guard = 0; 1635 1636 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 1637 1638 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1639 guard = ctx->guard_seed; 1640 } 1641 offset_in_block = 0; 1642 1643 while (offset_in_block < ctx->block_size) { 1644 _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); 1645 data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); 1646 1647 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1648 guard = _dif_generate_guard(guard, data_buf, data_buf_len, 1649 ctx->dif_pi_format); 1650 } 1651 1652 _dif_sgl_advance(data_sgl, data_buf_len); 1653 offset_in_block += data_buf_len; 1654 } 1655 1656 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1657 guard = _dif_generate_guard(guard, md_buf, ctx->guard_interval, 1658 ctx->dif_pi_format); 1659 } 1660 1661 _dif_sgl_advance(md_sgl, ctx->md_size); 1662 1663 return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); 1664 } 1665 1666 static int 1667 dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, 1668 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1669 struct spdk_dif_error *err_blk) 1670 { 1671 uint32_t offset_blocks; 1672 int rc; 1673 1674 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 1675 rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk); 1676 if (rc != 0) { 1677 return rc; 1678 } 1679 } 1680 1681 return 0; 1682 } 1683 1684 int 1685 spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1686 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1687 struct spdk_dif_error *err_blk) 1688 { 1689 struct _dif_sgl data_sgl, md_sgl; 1690 1691 if (md_iov->iov_base == NULL) { 1692 SPDK_ERRLOG("Metadata buffer is NULL.\n"); 1693 return -EINVAL; 1694 } 1695 1696 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1697 _dif_sgl_init(&md_sgl, md_iov, 1); 1698 1699 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1700 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1701 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1702 return -EINVAL; 1703 } 1704 1705 if (_dif_is_disabled(ctx->dif_type)) { 1706 return 0; 1707 } 1708 1709 if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { 1710 return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1711 } else { 1712 return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); 1713 } 1714 } 1715 1716 int 1717 spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, 1718 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1719 uint32_t inject_flags, uint32_t *inject_offset) 1720 { 1721 struct _dif_sgl data_sgl, md_sgl; 1722 int rc; 1723 1724 _dif_sgl_init(&data_sgl, iovs, iovcnt); 1725 _dif_sgl_init(&md_sgl, md_iov, 1); 1726 1727 if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || 1728 !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 1729 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 1730 return -EINVAL; 1731 } 1732 1733 if (inject_flags & SPDK_DIF_REFTAG_ERROR) { 1734 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1735 ctx->guard_interval + _dif_reftag_offset(ctx->dif_pi_format), 1736 _dif_reftag_size(ctx->dif_pi_format), 1737 inject_offset); 1738 if (rc != 0) { 1739 SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); 1740 return rc; 1741 } 1742 } 1743 1744 if (inject_flags & SPDK_DIF_APPTAG_ERROR) { 1745 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1746 ctx->guard_interval + _dif_apptag_offset(ctx->dif_pi_format), 1747 _dif_apptag_size(), 1748 inject_offset); 1749 if (rc != 0) { 1750 SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); 1751 return rc; 1752 } 1753 } 1754 1755 if (inject_flags & SPDK_DIF_GUARD_ERROR) { 1756 rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, 1757 ctx->guard_interval, 1758 _dif_guard_size(ctx->dif_pi_format), 1759 inject_offset); 1760 if (rc != 0) { 1761 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1762 return rc; 1763 } 1764 } 1765 1766 if (inject_flags & SPDK_DIF_DATA_ERROR) { 1767 /* Note: Error injection to data block is expected to be detected 1768 * as guard error. 1769 */ 1770 rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks, 1771 0, 1772 ctx->block_size, 1773 inject_offset); 1774 if (rc != 0) { 1775 SPDK_ERRLOG("Failed to inject error to Guard.\n"); 1776 return rc; 1777 } 1778 } 1779 1780 return 0; 1781 } 1782 1783 static uint32_t 1784 _to_next_boundary(uint32_t offset, uint32_t boundary) 1785 { 1786 return boundary - (offset % boundary); 1787 } 1788 1789 static uint32_t 1790 _to_size_with_md(uint32_t size, uint32_t data_block_size, uint32_t block_size) 1791 { 1792 return (size / data_block_size) * block_size + (size % data_block_size); 1793 } 1794 1795 int 1796 spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt, 1797 struct iovec *buf_iovs, int buf_iovcnt, 1798 uint32_t data_offset, uint32_t data_len, 1799 uint32_t *_mapped_len, 1800 const struct spdk_dif_ctx *ctx) 1801 { 1802 uint32_t data_block_size, data_unalign, buf_len, buf_offset, len; 1803 struct _dif_sgl dif_sgl; 1804 struct _dif_sgl buf_sgl; 1805 1806 if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) { 1807 return -EINVAL; 1808 } 1809 1810 data_block_size = ctx->block_size - ctx->md_size; 1811 1812 data_unalign = ctx->data_offset % data_block_size; 1813 1814 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 1815 ctx->block_size); 1816 buf_len -= data_unalign; 1817 1818 _dif_sgl_init(&dif_sgl, iovs, iovcnt); 1819 _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt); 1820 1821 if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) { 1822 SPDK_ERRLOG("Buffer overflow will occur.\n"); 1823 return -ERANGE; 1824 } 1825 1826 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 1827 buf_offset -= data_unalign; 1828 1829 _dif_sgl_advance(&buf_sgl, buf_offset); 1830 1831 while (data_len != 0) { 1832 len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size)); 1833 if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) { 1834 break; 1835 } 1836 _dif_sgl_advance(&buf_sgl, ctx->md_size); 1837 data_offset += len; 1838 data_len -= len; 1839 } 1840 1841 if (_mapped_len != NULL) { 1842 *_mapped_len = dif_sgl.total_size; 1843 } 1844 1845 return iovcnt - dif_sgl.iovcnt; 1846 } 1847 1848 static int 1849 _dif_sgl_setup_stream(struct _dif_sgl *sgl, uint32_t *_buf_offset, uint32_t *_buf_len, 1850 uint32_t data_offset, uint32_t data_len, 1851 const struct spdk_dif_ctx *ctx) 1852 { 1853 uint32_t data_block_size, data_unalign, buf_len, buf_offset; 1854 1855 data_block_size = ctx->block_size - ctx->md_size; 1856 1857 data_unalign = ctx->data_offset % data_block_size; 1858 1859 /* If the last data block is complete, DIF of the data block is 1860 * inserted or verified in this turn. 1861 */ 1862 buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, 1863 ctx->block_size); 1864 buf_len -= data_unalign; 1865 1866 if (!_dif_sgl_is_valid(sgl, buf_len)) { 1867 return -ERANGE; 1868 } 1869 1870 buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); 1871 buf_offset -= data_unalign; 1872 1873 _dif_sgl_advance(sgl, buf_offset); 1874 buf_len -= buf_offset; 1875 1876 buf_offset += data_unalign; 1877 1878 *_buf_offset = buf_offset; 1879 *_buf_len = buf_len; 1880 1881 return 0; 1882 } 1883 1884 int 1885 spdk_dif_generate_stream(struct iovec *iovs, int iovcnt, 1886 uint32_t data_offset, uint32_t data_len, 1887 struct spdk_dif_ctx *ctx) 1888 { 1889 uint32_t buf_len = 0, buf_offset = 0; 1890 uint32_t len, offset_in_block, offset_blocks; 1891 uint64_t guard = 0; 1892 struct _dif_sgl sgl; 1893 int rc; 1894 1895 if (iovs == NULL || iovcnt == 0) { 1896 return -EINVAL; 1897 } 1898 1899 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1900 guard = ctx->last_guard; 1901 } 1902 1903 _dif_sgl_init(&sgl, iovs, iovcnt); 1904 1905 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1906 if (rc != 0) { 1907 return rc; 1908 } 1909 1910 while (buf_len != 0) { 1911 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1912 offset_in_block = buf_offset % ctx->block_size; 1913 offset_blocks = buf_offset / ctx->block_size; 1914 1915 guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx); 1916 1917 buf_len -= len; 1918 buf_offset += len; 1919 } 1920 1921 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1922 ctx->last_guard = guard; 1923 } 1924 1925 return 0; 1926 } 1927 1928 int 1929 spdk_dif_verify_stream(struct iovec *iovs, int iovcnt, 1930 uint32_t data_offset, uint32_t data_len, 1931 struct spdk_dif_ctx *ctx, 1932 struct spdk_dif_error *err_blk) 1933 { 1934 uint32_t buf_len = 0, buf_offset = 0; 1935 uint32_t len, offset_in_block, offset_blocks; 1936 uint64_t guard = 0; 1937 struct _dif_sgl sgl; 1938 int rc = 0; 1939 1940 if (iovs == NULL || iovcnt == 0) { 1941 return -EINVAL; 1942 } 1943 1944 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1945 guard = ctx->last_guard; 1946 } 1947 1948 _dif_sgl_init(&sgl, iovs, iovcnt); 1949 1950 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1951 if (rc != 0) { 1952 return rc; 1953 } 1954 1955 while (buf_len != 0) { 1956 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 1957 offset_in_block = buf_offset % ctx->block_size; 1958 offset_blocks = buf_offset / ctx->block_size; 1959 1960 rc = _dif_verify_split(&sgl, offset_in_block, len, &guard, offset_blocks, 1961 ctx, err_blk); 1962 if (rc != 0) { 1963 goto error; 1964 } 1965 1966 buf_len -= len; 1967 buf_offset += len; 1968 } 1969 1970 if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { 1971 ctx->last_guard = guard; 1972 } 1973 error: 1974 return rc; 1975 } 1976 1977 int 1978 spdk_dif_update_crc32c_stream(struct iovec *iovs, int iovcnt, 1979 uint32_t data_offset, uint32_t data_len, 1980 uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) 1981 { 1982 uint32_t buf_len = 0, buf_offset = 0, len, offset_in_block; 1983 uint32_t crc32c; 1984 struct _dif_sgl sgl; 1985 int rc; 1986 1987 if (iovs == NULL || iovcnt == 0) { 1988 return -EINVAL; 1989 } 1990 1991 crc32c = *_crc32c; 1992 _dif_sgl_init(&sgl, iovs, iovcnt); 1993 1994 rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); 1995 if (rc != 0) { 1996 return rc; 1997 } 1998 1999 while (buf_len != 0) { 2000 len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); 2001 offset_in_block = buf_offset % ctx->block_size; 2002 2003 crc32c = _dif_update_crc32c_split(&sgl, offset_in_block, len, crc32c, ctx); 2004 2005 buf_len -= len; 2006 buf_offset += len; 2007 } 2008 2009 *_crc32c = crc32c; 2010 2011 return 0; 2012 } 2013 2014 void 2015 spdk_dif_get_range_with_md(uint32_t data_offset, uint32_t data_len, 2016 uint32_t *_buf_offset, uint32_t *_buf_len, 2017 const struct spdk_dif_ctx *ctx) 2018 { 2019 uint32_t data_block_size, data_unalign, buf_offset, buf_len; 2020 2021 if (!ctx->md_interleave) { 2022 buf_offset = data_offset; 2023 buf_len = data_len; 2024 } else { 2025 data_block_size = ctx->block_size - ctx->md_size; 2026 2027 data_unalign = data_offset % data_block_size; 2028 2029 buf_offset = _to_size_with_md(data_offset, data_block_size, ctx->block_size); 2030 buf_len = _to_size_with_md(data_unalign + data_len, data_block_size, ctx->block_size) - 2031 data_unalign; 2032 } 2033 2034 if (_buf_offset != NULL) { 2035 *_buf_offset = buf_offset; 2036 } 2037 2038 if (_buf_len != NULL) { 2039 *_buf_len = buf_len; 2040 } 2041 } 2042 2043 uint32_t 2044 spdk_dif_get_length_with_md(uint32_t data_len, const struct spdk_dif_ctx *ctx) 2045 { 2046 uint32_t data_block_size; 2047 2048 if (!ctx->md_interleave) { 2049 return data_len; 2050 } else { 2051 data_block_size = ctx->block_size - ctx->md_size; 2052 2053 return _to_size_with_md(data_len, data_block_size, ctx->block_size); 2054 } 2055 } 2056 2057 static int 2058 _dif_remap_ref_tag(struct _dif_sgl *sgl, uint32_t offset_blocks, 2059 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2060 bool check_ref_tag) 2061 { 2062 uint32_t offset, buf_len; 2063 uint64_t expected = 0, remapped; 2064 uint8_t *buf; 2065 struct _dif_sgl tmp_sgl; 2066 struct spdk_dif dif; 2067 2068 /* Fast forward to DIF field. */ 2069 _dif_sgl_advance(sgl, ctx->guard_interval); 2070 _dif_sgl_copy(&tmp_sgl, sgl); 2071 2072 /* Copy the split DIF field to the temporary DIF buffer */ 2073 offset = 0; 2074 while (offset < _dif_size(ctx->dif_pi_format)) { 2075 _dif_sgl_get_buf(sgl, &buf, &buf_len); 2076 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 2077 2078 memcpy((uint8_t *)&dif + offset, buf, buf_len); 2079 2080 _dif_sgl_advance(sgl, buf_len); 2081 offset += buf_len; 2082 } 2083 2084 if (_dif_ignore(&dif, ctx)) { 2085 goto end; 2086 } 2087 2088 /* For type 1 and 2, the Reference Tag is incremented for each 2089 * subsequent logical block. For type 3, the Reference Tag 2090 * remains the same as the initial Reference Tag. 2091 */ 2092 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2093 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2094 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2095 } else { 2096 remapped = ctx->remapped_init_ref_tag; 2097 } 2098 2099 /* Verify the stored Reference Tag. */ 2100 if (check_ref_tag && !_dif_reftag_check(&dif, ctx, expected, offset_blocks, err_blk)) { 2101 return -1; 2102 } 2103 2104 /* Update the stored Reference Tag to the remapped one. */ 2105 _dif_set_reftag(&dif, remapped, ctx->dif_pi_format); 2106 2107 offset = 0; 2108 while (offset < _dif_size(ctx->dif_pi_format)) { 2109 _dif_sgl_get_buf(&tmp_sgl, &buf, &buf_len); 2110 buf_len = spdk_min(buf_len, _dif_size(ctx->dif_pi_format) - offset); 2111 2112 memcpy(buf, (uint8_t *)&dif + offset, buf_len); 2113 2114 _dif_sgl_advance(&tmp_sgl, buf_len); 2115 offset += buf_len; 2116 } 2117 2118 end: 2119 _dif_sgl_advance(sgl, ctx->block_size - ctx->guard_interval - _dif_size(ctx->dif_pi_format)); 2120 2121 return 0; 2122 } 2123 2124 int 2125 spdk_dif_remap_ref_tag(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 2126 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2127 bool check_ref_tag) 2128 { 2129 struct _dif_sgl sgl; 2130 uint32_t offset_blocks; 2131 int rc; 2132 2133 _dif_sgl_init(&sgl, iovs, iovcnt); 2134 2135 if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { 2136 SPDK_ERRLOG("Size of iovec array is not valid.\n"); 2137 return -EINVAL; 2138 } 2139 2140 if (_dif_is_disabled(ctx->dif_type)) { 2141 return 0; 2142 } 2143 2144 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2145 return 0; 2146 } 2147 2148 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2149 rc = _dif_remap_ref_tag(&sgl, offset_blocks, ctx, err_blk, check_ref_tag); 2150 if (rc != 0) { 2151 return rc; 2152 } 2153 } 2154 2155 return 0; 2156 } 2157 2158 static int 2159 _dix_remap_ref_tag(struct _dif_sgl *md_sgl, uint32_t offset_blocks, 2160 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk, 2161 bool check_ref_tag) 2162 { 2163 uint64_t expected = 0, remapped; 2164 uint8_t *md_buf; 2165 struct spdk_dif *dif; 2166 2167 _dif_sgl_get_buf(md_sgl, &md_buf, NULL); 2168 2169 dif = (struct spdk_dif *)(md_buf + ctx->guard_interval); 2170 2171 if (_dif_ignore(dif, ctx)) { 2172 goto end; 2173 } 2174 2175 /* For type 1 and 2, the Reference Tag is incremented for each 2176 * subsequent logical block. For type 3, the Reference Tag 2177 * remains the same as the initialReference Tag. 2178 */ 2179 if (ctx->dif_type != SPDK_DIF_TYPE3) { 2180 expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2181 remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; 2182 } else { 2183 remapped = ctx->remapped_init_ref_tag; 2184 } 2185 2186 /* Verify the stored Reference Tag. */ 2187 if (check_ref_tag && !_dif_reftag_check(dif, ctx, expected, offset_blocks, err_blk)) { 2188 return -1; 2189 } 2190 2191 /* Update the stored Reference Tag to the remapped one. */ 2192 _dif_set_reftag(dif, remapped, ctx->dif_pi_format); 2193 2194 end: 2195 _dif_sgl_advance(md_sgl, ctx->md_size); 2196 2197 return 0; 2198 } 2199 2200 int 2201 spdk_dix_remap_ref_tag(struct iovec *md_iov, uint32_t num_blocks, 2202 const struct spdk_dif_ctx *ctx, 2203 struct spdk_dif_error *err_blk, 2204 bool check_ref_tag) 2205 { 2206 struct _dif_sgl md_sgl; 2207 uint32_t offset_blocks; 2208 int rc; 2209 2210 _dif_sgl_init(&md_sgl, md_iov, 1); 2211 2212 if (!_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { 2213 SPDK_ERRLOG("Size of metadata iovec array is not valid.\n"); 2214 return -EINVAL; 2215 } 2216 2217 if (_dif_is_disabled(ctx->dif_type)) { 2218 return 0; 2219 } 2220 2221 if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { 2222 return 0; 2223 } 2224 2225 for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { 2226 rc = _dix_remap_ref_tag(&md_sgl, offset_blocks, ctx, err_blk, check_ref_tag); 2227 if (rc != 0) { 2228 return rc; 2229 } 2230 } 2231 2232 return 0; 2233 } 2234