1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "util_internal.h" 7 #include "spdk/crc32.h" 8 9 #ifdef SPDK_CONFIG_ISAL 10 #define SPDK_HAVE_ISAL 11 #include <isa-l/include/crc.h> 12 #elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) 13 #define SPDK_HAVE_ARM_CRC 14 #include <arm_acle.h> 15 #elif defined(__x86_64__) && defined(__SSE4_2__) 16 #define SPDK_HAVE_SSE4_2 17 #include <x86intrin.h> 18 #endif 19 20 #ifdef SPDK_HAVE_ISAL 21 22 uint32_t 23 spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) 24 { 25 return crc32_iscsi((unsigned char *)buf, len, crc); 26 } 27 28 #elif defined(SPDK_HAVE_SSE4_2) 29 30 uint32_t 31 spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) 32 { 33 uint64_t crc_tmp64; 34 size_t count; 35 36 /* _mm_crc32_u64() needs a 64-bit intermediate value */ 37 crc_tmp64 = crc; 38 39 /* Process as much of the buffer as possible in 64-bit blocks. */ 40 count = len / 8; 41 while (count--) { 42 uint64_t block; 43 44 /* 45 * Use memcpy() to avoid unaligned loads, which are undefined behavior in C. 46 * The compiler will optimize out the memcpy() in release builds. 47 */ 48 memcpy(&block, buf, sizeof(block)); 49 crc_tmp64 = _mm_crc32_u64(crc_tmp64, block); 50 buf += sizeof(block); 51 } 52 crc = (uint32_t)crc_tmp64; 53 54 /* Handle any trailing bytes. */ 55 count = len & 7; 56 while (count--) { 57 crc = _mm_crc32_u8(crc, *(const uint8_t *)buf); 58 buf++; 59 } 60 61 return crc; 62 } 63 64 #elif defined(SPDK_HAVE_ARM_CRC) 65 66 uint32_t 67 spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) 68 { 69 size_t count; 70 71 count = len / 8; 72 while (count--) { 73 uint64_t block; 74 75 memcpy(&block, buf, sizeof(block)); 76 crc = __crc32cd(crc, block); 77 buf += sizeof(block); 78 } 79 80 count = len & 7; 81 while (count--) { 82 crc = __crc32cb(crc, *(const uint8_t *)buf); 83 buf++; 84 } 85 86 return crc; 87 } 88 89 #else /* Neither SSE 4.2 nor ARM CRC32 instructions available */ 90 91 static struct spdk_crc32_table g_crc32c_table; 92 93 __attribute__((constructor)) static void 94 crc32c_init(void) 95 { 96 crc32_table_init(&g_crc32c_table, SPDK_CRC32C_POLYNOMIAL_REFLECT); 97 } 98 99 uint32_t 100 spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) 101 { 102 return crc32_update(&g_crc32c_table, buf, len, crc); 103 } 104 105 #endif 106 107 uint32_t 108 spdk_crc32c_iov_update(struct iovec *iov, int iovcnt, uint32_t crc32c) 109 { 110 int i; 111 112 if (iov == NULL) { 113 return crc32c; 114 } 115 116 for (i = 0; i < iovcnt; i++) { 117 assert(iov[i].iov_base != NULL); 118 assert(iov[i].iov_len != 0); 119 crc32c = spdk_crc32c_update(iov[i].iov_base, iov[i].iov_len, crc32c); 120 } 121 122 return crc32c; 123 } 124