1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2018 Intel Corporation 3 */ 4 5 #ifndef __INCLUDE_RTE_TABLE_HASH_FUNC_H__ 6 #define __INCLUDE_RTE_TABLE_HASH_FUNC_H__ 7 8 #include <stdint.h> 9 10 #include <rte_compat.h> 11 #include <rte_common.h> 12 13 #if defined(RTE_ARCH_X86_64) 14 15 #include <x86intrin.h> 16 17 #ifdef __cplusplus 18 extern "C" { 19 #endif 20 21 static inline uint64_t 22 rte_crc32_u64(uint64_t crc, uint64_t v) 23 { 24 return _mm_crc32_u64(crc, v); 25 } 26 27 #ifdef __cplusplus 28 } 29 #endif 30 31 #elif defined(RTE_ARCH_ARM64) && defined(__ARM_FEATURE_CRC32) 32 #include "rte_table_hash_func_arm64.h" 33 #else 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 static inline uint64_t 40 rte_crc32_u64(uint64_t crc, uint64_t v) 41 { 42 int i; 43 44 crc = (crc & 0xFFFFFFFFLLU) ^ v; 45 for (i = 63; i >= 0; i--) { 46 uint64_t mask; 47 48 mask = -(crc & 1LLU); 49 crc = (crc >> 1LLU) ^ (0x82F63B78LLU & mask); 50 } 51 52 return crc; 53 } 54 55 #ifdef __cplusplus 56 } 57 #endif 58 59 #endif 60 61 #ifdef __cplusplus 62 extern "C" { 63 #endif 64 65 __rte_experimental 66 static inline uint64_t 67 rte_table_hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size, 68 uint64_t seed) 69 { 70 uint64_t *k = (uint64_t *)key; 71 uint64_t *m = (uint64_t *)mask; 72 uint64_t crc0; 73 74 crc0 = rte_crc32_u64(seed, k[0] & m[0]); 75 76 return crc0; 77 } 78 79 __rte_experimental 80 static inline uint64_t 81 rte_table_hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size, 82 uint64_t seed) 83 { 84 uint64_t *k = (uint64_t *)key; 85 uint64_t *m = (uint64_t *)mask; 86 uint64_t k0, crc0, crc1; 87 88 k0 = k[0] & m[0]; 89 90 crc0 = rte_crc32_u64(k0, seed); 91 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 92 93 crc0 ^= crc1; 94 95 return crc0; 96 } 97 98 __rte_experimental 99 static inline uint64_t 100 rte_table_hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size, 101 uint64_t seed) 102 { 103 uint64_t *k = (uint64_t *)key; 104 uint64_t *m = (uint64_t *)mask; 105 uint64_t k0, k2, crc0, crc1; 106 107 k0 = k[0] & m[0]; 108 k2 = k[2] & m[2]; 109 110 crc0 = rte_crc32_u64(k0, seed); 111 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 112 113 crc0 = rte_crc32_u64(crc0, k2); 114 115 crc0 ^= crc1; 116 117 return crc0; 118 } 119 120 __rte_experimental 121 static inline uint64_t 122 rte_table_hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size, 123 uint64_t seed) 124 { 125 uint64_t *k = (uint64_t *)key; 126 uint64_t *m = (uint64_t *)mask; 127 uint64_t k0, k2, crc0, crc1, crc2, crc3; 128 129 k0 = k[0] & m[0]; 130 k2 = k[2] & m[2]; 131 132 crc0 = rte_crc32_u64(k0, seed); 133 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 134 135 crc2 = rte_crc32_u64(k2, k[3] & m[3]); 136 crc3 = k2 >> 32; 137 138 crc0 = rte_crc32_u64(crc0, crc1); 139 crc1 = rte_crc32_u64(crc2, crc3); 140 141 crc0 ^= crc1; 142 143 return crc0; 144 } 145 146 __rte_experimental 147 static inline uint64_t 148 rte_table_hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size, 149 uint64_t seed) 150 { 151 uint64_t *k = (uint64_t *)key; 152 uint64_t *m = (uint64_t *)mask; 153 uint64_t k0, k2, crc0, crc1, crc2, crc3; 154 155 k0 = k[0] & m[0]; 156 k2 = k[2] & m[2]; 157 158 crc0 = rte_crc32_u64(k0, seed); 159 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 160 161 crc2 = rte_crc32_u64(k2, k[3] & m[3]); 162 crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]); 163 164 crc0 = rte_crc32_u64(crc0, crc1); 165 crc1 = rte_crc32_u64(crc2, crc3); 166 167 crc0 ^= crc1; 168 169 return crc0; 170 } 171 172 __rte_experimental 173 static inline uint64_t 174 rte_table_hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size, 175 uint64_t seed) 176 { 177 uint64_t *k = (uint64_t *)key; 178 uint64_t *m = (uint64_t *)mask; 179 uint64_t k0, k2, k5, crc0, crc1, crc2, crc3; 180 181 k0 = k[0] & m[0]; 182 k2 = k[2] & m[2]; 183 k5 = k[5] & m[5]; 184 185 crc0 = rte_crc32_u64(k0, seed); 186 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 187 188 crc2 = rte_crc32_u64(k2, k[3] & m[3]); 189 crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]); 190 191 crc0 = rte_crc32_u64(crc0, (crc1 << 32) ^ crc2); 192 crc1 = rte_crc32_u64(crc3, k5); 193 194 crc0 ^= crc1; 195 196 return crc0; 197 } 198 199 __rte_experimental 200 static inline uint64_t 201 rte_table_hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size, 202 uint64_t seed) 203 { 204 uint64_t *k = (uint64_t *)key; 205 uint64_t *m = (uint64_t *)mask; 206 uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5; 207 208 k0 = k[0] & m[0]; 209 k2 = k[2] & m[2]; 210 k5 = k[5] & m[5]; 211 212 crc0 = rte_crc32_u64(k0, seed); 213 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 214 215 crc2 = rte_crc32_u64(k2, k[3] & m[3]); 216 crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]); 217 218 crc4 = rte_crc32_u64(k5, k[6] & m[6]); 219 crc5 = k5 >> 32; 220 221 crc0 = rte_crc32_u64(crc0, (crc1 << 32) ^ crc2); 222 crc1 = rte_crc32_u64(crc3, (crc4 << 32) ^ crc5); 223 224 crc0 ^= crc1; 225 226 return crc0; 227 } 228 229 __rte_experimental 230 static inline uint64_t 231 rte_table_hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size, 232 uint64_t seed) 233 { 234 uint64_t *k = (uint64_t *)key; 235 uint64_t *m = (uint64_t *)mask; 236 uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5; 237 238 k0 = k[0] & m[0]; 239 k2 = k[2] & m[2]; 240 k5 = k[5] & m[5]; 241 242 crc0 = rte_crc32_u64(k0, seed); 243 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 244 245 crc2 = rte_crc32_u64(k2, k[3] & m[3]); 246 crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]); 247 248 crc4 = rte_crc32_u64(k5, k[6] & m[6]); 249 crc5 = rte_crc32_u64(k5 >> 32, k[7] & m[7]); 250 251 crc0 = rte_crc32_u64(crc0, (crc1 << 32) ^ crc2); 252 crc1 = rte_crc32_u64(crc3, (crc4 << 32) ^ crc5); 253 254 crc0 ^= crc1; 255 256 return crc0; 257 } 258 259 #ifdef __cplusplus 260 } 261 #endif 262 263 #endif 264