1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2020 Intel Corporation 3 */ 4 5 #include <stddef.h> 6 #include <stdint.h> 7 8 #include <rte_cpuflags.h> 9 #include <rte_common.h> 10 #include <rte_net_crc.h> 11 #include <rte_log.h> 12 #include <rte_vect.h> 13 14 #include "net_crc.h" 15 16 /** CRC polynomials */ 17 #define CRC32_ETH_POLYNOMIAL 0x04c11db7UL 18 #define CRC16_CCITT_POLYNOMIAL 0x1021U 19 20 #define CRC_LUT_SIZE 256 21 22 /* crc tables */ 23 static uint32_t crc32_eth_lut[CRC_LUT_SIZE]; 24 static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE]; 25 26 static uint32_t 27 rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len); 28 29 static uint32_t 30 rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len); 31 32 static uint32_t 33 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len); 34 35 static uint32_t 36 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len); 37 38 typedef uint32_t 39 (*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len); 40 41 static rte_net_crc_handler handlers_default[] = { 42 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_default_handler, 43 [RTE_NET_CRC32_ETH] = rte_crc32_eth_default_handler, 44 }; 45 46 static const rte_net_crc_handler *handlers = handlers_default; 47 48 static const rte_net_crc_handler handlers_scalar[] = { 49 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler, 50 [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler, 51 }; 52 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT 53 static const rte_net_crc_handler handlers_avx512[] = { 54 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_avx512_handler, 55 [RTE_NET_CRC32_ETH] = rte_crc32_eth_avx512_handler, 56 }; 57 #endif 58 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT 59 static const rte_net_crc_handler handlers_sse42[] = { 60 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler, 61 [RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler, 62 }; 63 #endif 64 #ifdef CC_ARM64_NEON_PMULL_SUPPORT 65 static const rte_net_crc_handler handlers_neon[] = { 66 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler, 67 [RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler, 68 }; 69 #endif 70 71 static uint16_t max_simd_bitwidth; 72 73 RTE_LOG_REGISTER_DEFAULT(libnet_logtype, INFO); 74 #define RTE_LOGTYPE_NET libnet_logtype 75 76 #define NET_LOG(level, ...) \ 77 RTE_LOG_LINE_PREFIX(level, NET, "%s(): ", __func__, __VA_ARGS__) 78 79 /* Scalar handling */ 80 81 /** 82 * Reflect the bits about the middle 83 * 84 * @param val 85 * value to be reflected 86 * 87 * @return 88 * reflected value 89 */ 90 static uint32_t 91 reflect_32bits(uint32_t val) 92 { 93 uint32_t i, res = 0; 94 95 for (i = 0; i < 32; i++) 96 if ((val & (1U << i)) != 0) 97 res |= (uint32_t)(1U << (31 - i)); 98 99 return res; 100 } 101 102 static void 103 crc32_eth_init_lut(uint32_t poly, 104 uint32_t *lut) 105 { 106 uint32_t i, j; 107 108 for (i = 0; i < CRC_LUT_SIZE; i++) { 109 uint32_t crc = reflect_32bits(i); 110 111 for (j = 0; j < 8; j++) { 112 if (crc & 0x80000000L) 113 crc = (crc << 1) ^ poly; 114 else 115 crc <<= 1; 116 } 117 lut[i] = reflect_32bits(crc); 118 } 119 } 120 121 static __rte_always_inline uint32_t 122 crc32_eth_calc_lut(const uint8_t *data, 123 uint32_t data_len, 124 uint32_t crc, 125 const uint32_t *lut) 126 { 127 while (data_len--) 128 crc = lut[(crc ^ *data++) & 0xffL] ^ (crc >> 8); 129 130 return crc; 131 } 132 133 static void 134 rte_net_crc_scalar_init(void) 135 { 136 /* 32-bit crc init */ 137 crc32_eth_init_lut(CRC32_ETH_POLYNOMIAL, crc32_eth_lut); 138 139 /* 16-bit CRC init */ 140 crc32_eth_init_lut(CRC16_CCITT_POLYNOMIAL << 16, crc16_ccitt_lut); 141 } 142 143 static inline uint32_t 144 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len) 145 { 146 /* return 16-bit CRC value */ 147 return (uint16_t)~crc32_eth_calc_lut(data, 148 data_len, 149 0xffff, 150 crc16_ccitt_lut); 151 } 152 153 static inline uint32_t 154 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len) 155 { 156 /* return 32-bit CRC value */ 157 return ~crc32_eth_calc_lut(data, 158 data_len, 159 0xffffffffUL, 160 crc32_eth_lut); 161 } 162 163 /* AVX512/VPCLMULQDQ handling */ 164 165 #define AVX512_VPCLMULQDQ_CPU_SUPPORTED ( \ 166 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) && \ 167 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) && \ 168 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512DQ) && \ 169 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512VL) && \ 170 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) && \ 171 rte_cpu_get_flag_enabled(RTE_CPUFLAG_VPCLMULQDQ) \ 172 ) 173 174 static const rte_net_crc_handler * 175 avx512_vpclmulqdq_get_handlers(void) 176 { 177 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT 178 if (AVX512_VPCLMULQDQ_CPU_SUPPORTED && 179 max_simd_bitwidth >= RTE_VECT_SIMD_512) 180 return handlers_avx512; 181 #endif 182 NET_LOG(INFO, "Requirements not met, can't use AVX512"); 183 return NULL; 184 } 185 186 static void 187 avx512_vpclmulqdq_init(void) 188 { 189 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT 190 if (AVX512_VPCLMULQDQ_CPU_SUPPORTED) 191 rte_net_crc_avx512_init(); 192 #endif 193 } 194 195 /* SSE4.2/PCLMULQDQ handling */ 196 197 #define SSE42_PCLMULQDQ_CPU_SUPPORTED \ 198 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) 199 200 static const rte_net_crc_handler * 201 sse42_pclmulqdq_get_handlers(void) 202 { 203 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT 204 if (SSE42_PCLMULQDQ_CPU_SUPPORTED && 205 max_simd_bitwidth >= RTE_VECT_SIMD_128) 206 return handlers_sse42; 207 #endif 208 NET_LOG(INFO, "Requirements not met, can't use SSE"); 209 return NULL; 210 } 211 212 static void 213 sse42_pclmulqdq_init(void) 214 { 215 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT 216 if (SSE42_PCLMULQDQ_CPU_SUPPORTED) 217 rte_net_crc_sse42_init(); 218 #endif 219 } 220 221 /* NEON/PMULL handling */ 222 223 #define NEON_PMULL_CPU_SUPPORTED \ 224 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL) 225 226 static const rte_net_crc_handler * 227 neon_pmull_get_handlers(void) 228 { 229 #ifdef CC_ARM64_NEON_PMULL_SUPPORT 230 if (NEON_PMULL_CPU_SUPPORTED && 231 max_simd_bitwidth >= RTE_VECT_SIMD_128) 232 return handlers_neon; 233 #endif 234 NET_LOG(INFO, "Requirements not met, can't use NEON"); 235 return NULL; 236 } 237 238 static void 239 neon_pmull_init(void) 240 { 241 #ifdef CC_ARM64_NEON_PMULL_SUPPORT 242 if (NEON_PMULL_CPU_SUPPORTED) 243 rte_net_crc_neon_init(); 244 #endif 245 } 246 247 /* Default handling */ 248 249 static uint32_t 250 rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len) 251 { 252 handlers = NULL; 253 if (max_simd_bitwidth == 0) 254 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); 255 256 handlers = avx512_vpclmulqdq_get_handlers(); 257 if (handlers != NULL) 258 return handlers[RTE_NET_CRC16_CCITT](data, data_len); 259 handlers = sse42_pclmulqdq_get_handlers(); 260 if (handlers != NULL) 261 return handlers[RTE_NET_CRC16_CCITT](data, data_len); 262 handlers = neon_pmull_get_handlers(); 263 if (handlers != NULL) 264 return handlers[RTE_NET_CRC16_CCITT](data, data_len); 265 handlers = handlers_scalar; 266 return handlers[RTE_NET_CRC16_CCITT](data, data_len); 267 } 268 269 static uint32_t 270 rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len) 271 { 272 handlers = NULL; 273 if (max_simd_bitwidth == 0) 274 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); 275 276 handlers = avx512_vpclmulqdq_get_handlers(); 277 if (handlers != NULL) 278 return handlers[RTE_NET_CRC32_ETH](data, data_len); 279 handlers = sse42_pclmulqdq_get_handlers(); 280 if (handlers != NULL) 281 return handlers[RTE_NET_CRC32_ETH](data, data_len); 282 handlers = neon_pmull_get_handlers(); 283 if (handlers != NULL) 284 return handlers[RTE_NET_CRC32_ETH](data, data_len); 285 handlers = handlers_scalar; 286 return handlers[RTE_NET_CRC32_ETH](data, data_len); 287 } 288 289 /* Public API */ 290 291 void 292 rte_net_crc_set_alg(enum rte_net_crc_alg alg) 293 { 294 handlers = NULL; 295 if (max_simd_bitwidth == 0) 296 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); 297 298 switch (alg) { 299 case RTE_NET_CRC_AVX512: 300 handlers = avx512_vpclmulqdq_get_handlers(); 301 if (handlers != NULL) 302 break; 303 /* fall-through */ 304 case RTE_NET_CRC_SSE42: 305 handlers = sse42_pclmulqdq_get_handlers(); 306 break; /* for x86, always break here */ 307 case RTE_NET_CRC_NEON: 308 handlers = neon_pmull_get_handlers(); 309 /* fall-through */ 310 case RTE_NET_CRC_SCALAR: 311 /* fall-through */ 312 default: 313 break; 314 } 315 316 if (handlers == NULL) 317 handlers = handlers_scalar; 318 } 319 320 uint32_t 321 rte_net_crc_calc(const void *data, 322 uint32_t data_len, 323 enum rte_net_crc_type type) 324 { 325 uint32_t ret; 326 rte_net_crc_handler f_handle; 327 328 f_handle = handlers[type]; 329 ret = f_handle(data, data_len); 330 331 return ret; 332 } 333 334 /* Call initialisation helpers for all crc algorithm handlers */ 335 RTE_INIT(rte_net_crc_init) 336 { 337 rte_net_crc_scalar_init(); 338 sse42_pclmulqdq_init(); 339 avx512_vpclmulqdq_init(); 340 neon_pmull_init(); 341 } 342