1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2020 Intel Corporation
3 */
4
5 #include <stddef.h>
6 #include <stdint.h>
7
8 #include <rte_cpuflags.h>
9 #include <rte_common.h>
10 #include <rte_net_crc.h>
11 #include <rte_log.h>
12 #include <rte_vect.h>
13
14 #include "net_crc.h"
15
16 /** CRC polynomials */
17 #define CRC32_ETH_POLYNOMIAL 0x04c11db7UL
18 #define CRC16_CCITT_POLYNOMIAL 0x1021U
19
20 #define CRC_LUT_SIZE 256
21
22 /* crc tables */
23 static uint32_t crc32_eth_lut[CRC_LUT_SIZE];
24 static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE];
25
26 static uint32_t
27 rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len);
28
29 static uint32_t
30 rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len);
31
32 static uint32_t
33 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len);
34
35 static uint32_t
36 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len);
37
38 typedef uint32_t
39 (*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len);
40
41 static rte_net_crc_handler handlers_default[] = {
42 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_default_handler,
43 [RTE_NET_CRC32_ETH] = rte_crc32_eth_default_handler,
44 };
45
46 static const rte_net_crc_handler *handlers = handlers_default;
47
48 static const rte_net_crc_handler handlers_scalar[] = {
49 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler,
50 [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler,
51 };
52 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
53 static const rte_net_crc_handler handlers_avx512[] = {
54 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_avx512_handler,
55 [RTE_NET_CRC32_ETH] = rte_crc32_eth_avx512_handler,
56 };
57 #endif
58 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
59 static const rte_net_crc_handler handlers_sse42[] = {
60 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler,
61 [RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler,
62 };
63 #endif
64 #ifdef CC_ARM64_NEON_PMULL_SUPPORT
65 static const rte_net_crc_handler handlers_neon[] = {
66 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler,
67 [RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler,
68 };
69 #endif
70
71 static uint16_t max_simd_bitwidth;
72
73 RTE_LOG_REGISTER_DEFAULT(libnet_logtype, INFO);
74 #define RTE_LOGTYPE_NET libnet_logtype
75
76 #define NET_LOG(level, ...) \
77 RTE_LOG_LINE_PREFIX(level, NET, "%s(): ", __func__, __VA_ARGS__)
78
79 /* Scalar handling */
80
81 /**
82 * Reflect the bits about the middle
83 *
84 * @param val
85 * value to be reflected
86 *
87 * @return
88 * reflected value
89 */
90 static uint32_t
reflect_32bits(uint32_t val)91 reflect_32bits(uint32_t val)
92 {
93 uint32_t i, res = 0;
94
95 for (i = 0; i < 32; i++)
96 if ((val & (1U << i)) != 0)
97 res |= (uint32_t)(1U << (31 - i));
98
99 return res;
100 }
101
102 static void
crc32_eth_init_lut(uint32_t poly,uint32_t * lut)103 crc32_eth_init_lut(uint32_t poly,
104 uint32_t *lut)
105 {
106 uint32_t i, j;
107
108 for (i = 0; i < CRC_LUT_SIZE; i++) {
109 uint32_t crc = reflect_32bits(i);
110
111 for (j = 0; j < 8; j++) {
112 if (crc & 0x80000000L)
113 crc = (crc << 1) ^ poly;
114 else
115 crc <<= 1;
116 }
117 lut[i] = reflect_32bits(crc);
118 }
119 }
120
121 static __rte_always_inline uint32_t
crc32_eth_calc_lut(const uint8_t * data,uint32_t data_len,uint32_t crc,const uint32_t * lut)122 crc32_eth_calc_lut(const uint8_t *data,
123 uint32_t data_len,
124 uint32_t crc,
125 const uint32_t *lut)
126 {
127 while (data_len--)
128 crc = lut[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
129
130 return crc;
131 }
132
133 static void
rte_net_crc_scalar_init(void)134 rte_net_crc_scalar_init(void)
135 {
136 /* 32-bit crc init */
137 crc32_eth_init_lut(CRC32_ETH_POLYNOMIAL, crc32_eth_lut);
138
139 /* 16-bit CRC init */
140 crc32_eth_init_lut(CRC16_CCITT_POLYNOMIAL << 16, crc16_ccitt_lut);
141 }
142
143 static inline uint32_t
rte_crc16_ccitt_handler(const uint8_t * data,uint32_t data_len)144 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len)
145 {
146 /* return 16-bit CRC value */
147 return (uint16_t)~crc32_eth_calc_lut(data,
148 data_len,
149 0xffff,
150 crc16_ccitt_lut);
151 }
152
153 static inline uint32_t
rte_crc32_eth_handler(const uint8_t * data,uint32_t data_len)154 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len)
155 {
156 /* return 32-bit CRC value */
157 return ~crc32_eth_calc_lut(data,
158 data_len,
159 0xffffffffUL,
160 crc32_eth_lut);
161 }
162
163 /* AVX512/VPCLMULQDQ handling */
164
165 #define AVX512_VPCLMULQDQ_CPU_SUPPORTED ( \
166 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) && \
167 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) && \
168 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512DQ) && \
169 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512VL) && \
170 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) && \
171 rte_cpu_get_flag_enabled(RTE_CPUFLAG_VPCLMULQDQ) \
172 )
173
174 static const rte_net_crc_handler *
avx512_vpclmulqdq_get_handlers(void)175 avx512_vpclmulqdq_get_handlers(void)
176 {
177 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
178 if (AVX512_VPCLMULQDQ_CPU_SUPPORTED &&
179 max_simd_bitwidth >= RTE_VECT_SIMD_512)
180 return handlers_avx512;
181 #endif
182 NET_LOG(INFO, "Requirements not met, can't use AVX512");
183 return NULL;
184 }
185
186 static void
avx512_vpclmulqdq_init(void)187 avx512_vpclmulqdq_init(void)
188 {
189 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
190 if (AVX512_VPCLMULQDQ_CPU_SUPPORTED)
191 rte_net_crc_avx512_init();
192 #endif
193 }
194
195 /* SSE4.2/PCLMULQDQ handling */
196
197 #define SSE42_PCLMULQDQ_CPU_SUPPORTED \
198 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ)
199
200 static const rte_net_crc_handler *
sse42_pclmulqdq_get_handlers(void)201 sse42_pclmulqdq_get_handlers(void)
202 {
203 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
204 if (SSE42_PCLMULQDQ_CPU_SUPPORTED &&
205 max_simd_bitwidth >= RTE_VECT_SIMD_128)
206 return handlers_sse42;
207 #endif
208 NET_LOG(INFO, "Requirements not met, can't use SSE");
209 return NULL;
210 }
211
212 static void
sse42_pclmulqdq_init(void)213 sse42_pclmulqdq_init(void)
214 {
215 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
216 if (SSE42_PCLMULQDQ_CPU_SUPPORTED)
217 rte_net_crc_sse42_init();
218 #endif
219 }
220
221 /* NEON/PMULL handling */
222
223 #define NEON_PMULL_CPU_SUPPORTED \
224 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)
225
226 static const rte_net_crc_handler *
neon_pmull_get_handlers(void)227 neon_pmull_get_handlers(void)
228 {
229 #ifdef CC_ARM64_NEON_PMULL_SUPPORT
230 if (NEON_PMULL_CPU_SUPPORTED &&
231 max_simd_bitwidth >= RTE_VECT_SIMD_128)
232 return handlers_neon;
233 #endif
234 NET_LOG(INFO, "Requirements not met, can't use NEON");
235 return NULL;
236 }
237
238 static void
neon_pmull_init(void)239 neon_pmull_init(void)
240 {
241 #ifdef CC_ARM64_NEON_PMULL_SUPPORT
242 if (NEON_PMULL_CPU_SUPPORTED)
243 rte_net_crc_neon_init();
244 #endif
245 }
246
247 /* Default handling */
248
249 static uint32_t
rte_crc16_ccitt_default_handler(const uint8_t * data,uint32_t data_len)250 rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len)
251 {
252 handlers = NULL;
253 if (max_simd_bitwidth == 0)
254 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
255
256 handlers = avx512_vpclmulqdq_get_handlers();
257 if (handlers != NULL)
258 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
259 handlers = sse42_pclmulqdq_get_handlers();
260 if (handlers != NULL)
261 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
262 handlers = neon_pmull_get_handlers();
263 if (handlers != NULL)
264 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
265 handlers = handlers_scalar;
266 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
267 }
268
269 static uint32_t
rte_crc32_eth_default_handler(const uint8_t * data,uint32_t data_len)270 rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len)
271 {
272 handlers = NULL;
273 if (max_simd_bitwidth == 0)
274 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
275
276 handlers = avx512_vpclmulqdq_get_handlers();
277 if (handlers != NULL)
278 return handlers[RTE_NET_CRC32_ETH](data, data_len);
279 handlers = sse42_pclmulqdq_get_handlers();
280 if (handlers != NULL)
281 return handlers[RTE_NET_CRC32_ETH](data, data_len);
282 handlers = neon_pmull_get_handlers();
283 if (handlers != NULL)
284 return handlers[RTE_NET_CRC32_ETH](data, data_len);
285 handlers = handlers_scalar;
286 return handlers[RTE_NET_CRC32_ETH](data, data_len);
287 }
288
289 /* Public API */
290
291 void
rte_net_crc_set_alg(enum rte_net_crc_alg alg)292 rte_net_crc_set_alg(enum rte_net_crc_alg alg)
293 {
294 handlers = NULL;
295 if (max_simd_bitwidth == 0)
296 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
297
298 switch (alg) {
299 case RTE_NET_CRC_AVX512:
300 handlers = avx512_vpclmulqdq_get_handlers();
301 if (handlers != NULL)
302 break;
303 /* fall-through */
304 case RTE_NET_CRC_SSE42:
305 handlers = sse42_pclmulqdq_get_handlers();
306 break; /* for x86, always break here */
307 case RTE_NET_CRC_NEON:
308 handlers = neon_pmull_get_handlers();
309 /* fall-through */
310 case RTE_NET_CRC_SCALAR:
311 /* fall-through */
312 default:
313 break;
314 }
315
316 if (handlers == NULL)
317 handlers = handlers_scalar;
318 }
319
320 uint32_t
rte_net_crc_calc(const void * data,uint32_t data_len,enum rte_net_crc_type type)321 rte_net_crc_calc(const void *data,
322 uint32_t data_len,
323 enum rte_net_crc_type type)
324 {
325 uint32_t ret;
326 rte_net_crc_handler f_handle;
327
328 f_handle = handlers[type];
329 ret = f_handle(data, data_len);
330
331 return ret;
332 }
333
334 /* Call initialisation helpers for all crc algorithm handlers */
RTE_INIT(rte_net_crc_init)335 RTE_INIT(rte_net_crc_init)
336 {
337 rte_net_crc_scalar_init();
338 sse42_pclmulqdq_init();
339 avx512_vpclmulqdq_init();
340 neon_pmull_init();
341 }
342