1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "util_internal.h" 7 #include "crc_internal.h" 8 #include "spdk/crc32.h" 9 10 #ifdef SPDK_HAVE_ISAL 11 12 uint32_t 13 spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) 14 { 15 return crc32_iscsi((unsigned char *)buf, len, crc); 16 } 17 18 #elif defined(SPDK_HAVE_SSE4_2) 19 20 uint32_t 21 spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) 22 { 23 size_t count_pre, count_post, count_mid; 24 const uint64_t *dword_buf; 25 uint64_t crc_tmp64; 26 27 /* process the head and tail bytes seperately to make the buf address 28 * passed to _mm_crc32_u64 is 8 byte aligned. This can avoid unaligned loads. 29 */ 30 count_pre = ((uint64_t)buf & 7) == 0 ? 0 : 8 - ((uint64_t)buf & 7); 31 count_post = (uint64_t)((uintptr_t)buf + len) & 7; 32 count_mid = (len - count_pre - count_post) / 8; 33 34 while (count_pre--) { 35 crc = _mm_crc32_u8(crc, *(const uint8_t *)buf); 36 buf = (uint8_t *)buf + 1; 37 } 38 39 /* _mm_crc32_u64() needs a 64-bit intermediate value */ 40 crc_tmp64 = crc; 41 dword_buf = (const uint64_t *)buf; 42 43 while (count_mid--) { 44 crc_tmp64 = _mm_crc32_u64(crc_tmp64, *dword_buf); 45 dword_buf++; 46 } 47 48 buf = dword_buf; 49 crc = (uint32_t)crc_tmp64; 50 while (count_post--) { 51 crc = _mm_crc32_u8(crc, *(const uint8_t *)buf); 52 buf = (uint8_t *)buf + 1; 53 } 54 55 return crc; 56 } 57 58 #elif defined(SPDK_HAVE_ARM_CRC) 59 60 uint32_t 61 spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) 62 { 63 size_t count_pre, count_post, count_mid; 64 const uint64_t *dword_buf; 65 66 /* process the head and tail bytes seperately to make the buf address 67 * passed to crc32_cd is 8 byte aligned. This can avoid unaligned loads. 68 */ 69 count_pre = ((uint64_t)buf & 7) == 0 ? 0 : 8 - ((uint64_t)buf & 7); 70 count_post = (uint64_t)(buf + len) & 7; 71 count_mid = (len - count_pre - count_post) / 8; 72 73 while (count_pre--) { 74 crc = __crc32cb(crc, *(const uint8_t *)buf); 75 buf++; 76 } 77 78 dword_buf = (const uint64_t *)buf; 79 while (count_mid--) { 80 crc = __crc32cd(crc, *dword_buf); 81 dword_buf++; 82 } 83 84 buf = dword_buf; 85 while (count_post--) { 86 crc = __crc32cb(crc, *(const uint8_t *)buf); 87 buf++; 88 } 89 90 return crc; 91 } 92 93 #else /* Neither SSE 4.2 nor ARM CRC32 instructions available */ 94 95 static struct spdk_crc32_table g_crc32c_table; 96 97 __attribute__((constructor)) static void 98 crc32c_init(void) 99 { 100 crc32_table_init(&g_crc32c_table, SPDK_CRC32C_POLYNOMIAL_REFLECT); 101 } 102 103 uint32_t 104 spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) 105 { 106 return crc32_update(&g_crc32c_table, buf, len, crc); 107 } 108 109 #endif 110 111 uint32_t 112 spdk_crc32c_iov_update(struct iovec *iov, int iovcnt, uint32_t crc32c) 113 { 114 int i; 115 116 if (iov == NULL) { 117 return crc32c; 118 } 119 120 for (i = 0; i < iovcnt; i++) { 121 assert(iov[i].iov_base != NULL); 122 assert(iov[i].iov_len != 0); 123 crc32c = spdk_crc32c_update(iov[i].iov_base, iov[i].iov_len, crc32c); 124 } 125 126 return crc32c; 127 } 128 129 uint32_t 130 spdk_crc32c_nvme(const void *buf, size_t len, uint32_t crc) 131 { 132 return ~(spdk_crc32c_update(buf, len, ~crc)); 133 } 134