1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2020 Intel Corporation 3 */ 4 5 #include <stddef.h> 6 #include <string.h> 7 #include <stdint.h> 8 9 #include <rte_cpuflags.h> 10 #include <rte_common.h> 11 #include <rte_net_crc.h> 12 #include <rte_eal.h> 13 #include <rte_log.h> 14 #include <rte_vect.h> 15 16 #include "net_crc.h" 17 18 /** CRC polynomials */ 19 #define CRC32_ETH_POLYNOMIAL 0x04c11db7UL 20 #define CRC16_CCITT_POLYNOMIAL 0x1021U 21 22 #define CRC_LUT_SIZE 256 23 24 /* crc tables */ 25 static uint32_t crc32_eth_lut[CRC_LUT_SIZE]; 26 static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE]; 27 28 static uint32_t 29 rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len); 30 31 static uint32_t 32 rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len); 33 34 static uint32_t 35 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len); 36 37 static uint32_t 38 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len); 39 40 typedef uint32_t 41 (*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len); 42 43 static rte_net_crc_handler handlers_default[] = { 44 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_default_handler, 45 [RTE_NET_CRC32_ETH] = rte_crc32_eth_default_handler, 46 }; 47 48 static const rte_net_crc_handler *handlers = handlers_default; 49 50 static const rte_net_crc_handler handlers_scalar[] = { 51 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler, 52 [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler, 53 }; 54 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT 55 static const rte_net_crc_handler handlers_avx512[] = { 56 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_avx512_handler, 57 [RTE_NET_CRC32_ETH] = rte_crc32_eth_avx512_handler, 58 }; 59 #endif 60 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT 61 static const rte_net_crc_handler handlers_sse42[] = { 62 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler, 63 [RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler, 64 }; 65 #endif 66 #ifdef CC_ARM64_NEON_PMULL_SUPPORT 67 static const rte_net_crc_handler handlers_neon[] = { 68 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler, 69 [RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler, 70 }; 71 #endif 72 73 static uint16_t max_simd_bitwidth; 74 75 #define NET_LOG(level, fmt, args...) \ 76 rte_log(RTE_LOG_ ## level, libnet_logtype, "%s(): " fmt "\n", \ 77 __func__, ## args) 78 79 RTE_LOG_REGISTER_DEFAULT(libnet_logtype, INFO); 80 81 /* Scalar handling */ 82 83 /** 84 * Reflect the bits about the middle 85 * 86 * @param val 87 * value to be reflected 88 * 89 * @return 90 * reflected value 91 */ 92 static uint32_t 93 reflect_32bits(uint32_t val) 94 { 95 uint32_t i, res = 0; 96 97 for (i = 0; i < 32; i++) 98 if ((val & (1U << i)) != 0) 99 res |= (uint32_t)(1U << (31 - i)); 100 101 return res; 102 } 103 104 static void 105 crc32_eth_init_lut(uint32_t poly, 106 uint32_t *lut) 107 { 108 uint32_t i, j; 109 110 for (i = 0; i < CRC_LUT_SIZE; i++) { 111 uint32_t crc = reflect_32bits(i); 112 113 for (j = 0; j < 8; j++) { 114 if (crc & 0x80000000L) 115 crc = (crc << 1) ^ poly; 116 else 117 crc <<= 1; 118 } 119 lut[i] = reflect_32bits(crc); 120 } 121 } 122 123 static __rte_always_inline uint32_t 124 crc32_eth_calc_lut(const uint8_t *data, 125 uint32_t data_len, 126 uint32_t crc, 127 const uint32_t *lut) 128 { 129 while (data_len--) 130 crc = lut[(crc ^ *data++) & 0xffL] ^ (crc >> 8); 131 132 return crc; 133 } 134 135 static void 136 rte_net_crc_scalar_init(void) 137 { 138 /* 32-bit crc init */ 139 crc32_eth_init_lut(CRC32_ETH_POLYNOMIAL, crc32_eth_lut); 140 141 /* 16-bit CRC init */ 142 crc32_eth_init_lut(CRC16_CCITT_POLYNOMIAL << 16, crc16_ccitt_lut); 143 } 144 145 static inline uint32_t 146 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len) 147 { 148 /* return 16-bit CRC value */ 149 return (uint16_t)~crc32_eth_calc_lut(data, 150 data_len, 151 0xffff, 152 crc16_ccitt_lut); 153 } 154 155 static inline uint32_t 156 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len) 157 { 158 /* return 32-bit CRC value */ 159 return ~crc32_eth_calc_lut(data, 160 data_len, 161 0xffffffffUL, 162 crc32_eth_lut); 163 } 164 165 /* AVX512/VPCLMULQDQ handling */ 166 167 #define AVX512_VPCLMULQDQ_CPU_SUPPORTED ( \ 168 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) && \ 169 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) && \ 170 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512DQ) && \ 171 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512VL) && \ 172 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) && \ 173 rte_cpu_get_flag_enabled(RTE_CPUFLAG_VPCLMULQDQ) \ 174 ) 175 176 static const rte_net_crc_handler * 177 avx512_vpclmulqdq_get_handlers(void) 178 { 179 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT 180 if (AVX512_VPCLMULQDQ_CPU_SUPPORTED && 181 max_simd_bitwidth >= RTE_VECT_SIMD_512) 182 return handlers_avx512; 183 #endif 184 NET_LOG(INFO, "Requirements not met, can't use AVX512\n"); 185 return NULL; 186 } 187 188 static void 189 avx512_vpclmulqdq_init(void) 190 { 191 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT 192 if (AVX512_VPCLMULQDQ_CPU_SUPPORTED) 193 rte_net_crc_avx512_init(); 194 #endif 195 } 196 197 /* SSE4.2/PCLMULQDQ handling */ 198 199 #define SSE42_PCLMULQDQ_CPU_SUPPORTED \ 200 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) 201 202 static const rte_net_crc_handler * 203 sse42_pclmulqdq_get_handlers(void) 204 { 205 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT 206 if (SSE42_PCLMULQDQ_CPU_SUPPORTED && 207 max_simd_bitwidth >= RTE_VECT_SIMD_128) 208 return handlers_sse42; 209 #endif 210 NET_LOG(INFO, "Requirements not met, can't use SSE\n"); 211 return NULL; 212 } 213 214 static void 215 sse42_pclmulqdq_init(void) 216 { 217 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT 218 if (SSE42_PCLMULQDQ_CPU_SUPPORTED) 219 rte_net_crc_sse42_init(); 220 #endif 221 } 222 223 /* NEON/PMULL handling */ 224 225 #define NEON_PMULL_CPU_SUPPORTED \ 226 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL) 227 228 static const rte_net_crc_handler * 229 neon_pmull_get_handlers(void) 230 { 231 #ifdef CC_ARM64_NEON_PMULL_SUPPORT 232 if (NEON_PMULL_CPU_SUPPORTED && 233 max_simd_bitwidth >= RTE_VECT_SIMD_128) 234 return handlers_neon; 235 #endif 236 NET_LOG(INFO, "Requirements not met, can't use NEON\n"); 237 return NULL; 238 } 239 240 static void 241 neon_pmull_init(void) 242 { 243 #ifdef CC_ARM64_NEON_PMULL_SUPPORT 244 if (NEON_PMULL_CPU_SUPPORTED) 245 rte_net_crc_neon_init(); 246 #endif 247 } 248 249 /* Default handling */ 250 251 static uint32_t 252 rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len) 253 { 254 handlers = NULL; 255 if (max_simd_bitwidth == 0) 256 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); 257 258 handlers = avx512_vpclmulqdq_get_handlers(); 259 if (handlers != NULL) 260 return handlers[RTE_NET_CRC16_CCITT](data, data_len); 261 handlers = sse42_pclmulqdq_get_handlers(); 262 if (handlers != NULL) 263 return handlers[RTE_NET_CRC16_CCITT](data, data_len); 264 handlers = neon_pmull_get_handlers(); 265 if (handlers != NULL) 266 return handlers[RTE_NET_CRC16_CCITT](data, data_len); 267 handlers = handlers_scalar; 268 return handlers[RTE_NET_CRC16_CCITT](data, data_len); 269 } 270 271 static uint32_t 272 rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len) 273 { 274 handlers = NULL; 275 if (max_simd_bitwidth == 0) 276 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); 277 278 handlers = avx512_vpclmulqdq_get_handlers(); 279 if (handlers != NULL) 280 return handlers[RTE_NET_CRC32_ETH](data, data_len); 281 handlers = sse42_pclmulqdq_get_handlers(); 282 if (handlers != NULL) 283 return handlers[RTE_NET_CRC32_ETH](data, data_len); 284 handlers = neon_pmull_get_handlers(); 285 if (handlers != NULL) 286 return handlers[RTE_NET_CRC32_ETH](data, data_len); 287 handlers = handlers_scalar; 288 return handlers[RTE_NET_CRC32_ETH](data, data_len); 289 } 290 291 /* Public API */ 292 293 void 294 rte_net_crc_set_alg(enum rte_net_crc_alg alg) 295 { 296 handlers = NULL; 297 if (max_simd_bitwidth == 0) 298 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); 299 300 switch (alg) { 301 case RTE_NET_CRC_AVX512: 302 handlers = avx512_vpclmulqdq_get_handlers(); 303 if (handlers != NULL) 304 break; 305 /* fall-through */ 306 case RTE_NET_CRC_SSE42: 307 handlers = sse42_pclmulqdq_get_handlers(); 308 break; /* for x86, always break here */ 309 case RTE_NET_CRC_NEON: 310 handlers = neon_pmull_get_handlers(); 311 /* fall-through */ 312 case RTE_NET_CRC_SCALAR: 313 /* fall-through */ 314 default: 315 break; 316 } 317 318 if (handlers == NULL) 319 handlers = handlers_scalar; 320 } 321 322 uint32_t 323 rte_net_crc_calc(const void *data, 324 uint32_t data_len, 325 enum rte_net_crc_type type) 326 { 327 uint32_t ret; 328 rte_net_crc_handler f_handle; 329 330 f_handle = handlers[type]; 331 ret = f_handle(data, data_len); 332 333 return ret; 334 } 335 336 /* Call initialisation helpers for all crc algorithm handlers */ 337 RTE_INIT(rte_net_crc_init) 338 { 339 rte_net_crc_scalar_init(); 340 sse42_pclmulqdq_init(); 341 avx512_vpclmulqdq_init(); 342 neon_pmull_init(); 343 } 344