199a2dd95SBruce Richardson /* SPDX-License-Identifier: BSD-3-Clause 299a2dd95SBruce Richardson * Copyright(c) 2010-2018 Intel Corporation 399a2dd95SBruce Richardson */ 499a2dd95SBruce Richardson 599a2dd95SBruce Richardson #ifndef __INCLUDE_RTE_TABLE_HASH_FUNC_H__ 699a2dd95SBruce Richardson #define __INCLUDE_RTE_TABLE_HASH_FUNC_H__ 799a2dd95SBruce Richardson 899a2dd95SBruce Richardson #include <stdint.h> 999a2dd95SBruce Richardson 1099a2dd95SBruce Richardson #include <rte_compat.h> 1199a2dd95SBruce Richardson #include <rte_common.h> 1299a2dd95SBruce Richardson 1399a2dd95SBruce Richardson #if defined(RTE_ARCH_X86_64) 1499a2dd95SBruce Richardson 1599a2dd95SBruce Richardson #include <x86intrin.h> 1699a2dd95SBruce Richardson 17*719834a6SMattias Rönnblom #ifdef __cplusplus 18*719834a6SMattias Rönnblom extern "C" { 19*719834a6SMattias Rönnblom #endif 20*719834a6SMattias Rönnblom 2199a2dd95SBruce Richardson static inline uint64_t 2299a2dd95SBruce Richardson rte_crc32_u64(uint64_t crc, uint64_t v) 2399a2dd95SBruce Richardson { 2499a2dd95SBruce Richardson return _mm_crc32_u64(crc, v); 2599a2dd95SBruce Richardson } 2699a2dd95SBruce Richardson 27*719834a6SMattias Rönnblom #ifdef __cplusplus 28*719834a6SMattias Rönnblom } 29*719834a6SMattias Rönnblom #endif 30*719834a6SMattias Rönnblom 3199a2dd95SBruce Richardson #elif defined(RTE_ARCH_ARM64) && defined(__ARM_FEATURE_CRC32) 3299a2dd95SBruce Richardson #include "rte_table_hash_func_arm64.h" 3399a2dd95SBruce Richardson #else 3499a2dd95SBruce Richardson 35*719834a6SMattias Rönnblom #ifdef __cplusplus 36*719834a6SMattias Rönnblom extern "C" { 37*719834a6SMattias Rönnblom #endif 38*719834a6SMattias Rönnblom 3999a2dd95SBruce Richardson static inline uint64_t 4099a2dd95SBruce Richardson rte_crc32_u64(uint64_t crc, uint64_t v) 4199a2dd95SBruce Richardson { 42b20a2ea5SDavid Marchand int i; 43b20a2ea5SDavid Marchand 44b20a2ea5SDavid Marchand crc = (crc & 0xFFFFFFFFLLU) ^ v; 45b20a2ea5SDavid Marchand for (i = 63; i >= 0; i--) { 46b20a2ea5SDavid Marchand uint64_t mask; 47b20a2ea5SDavid Marchand 48b20a2ea5SDavid Marchand mask = -(crc & 1LLU); 49b20a2ea5SDavid Marchand crc = (crc >> 1LLU) ^ (0x82F63B78LLU & mask); 50b20a2ea5SDavid Marchand } 51b20a2ea5SDavid Marchand 52b20a2ea5SDavid Marchand return crc; 5399a2dd95SBruce Richardson } 5499a2dd95SBruce Richardson 55*719834a6SMattias Rönnblom #ifdef __cplusplus 56*719834a6SMattias Rönnblom } 57*719834a6SMattias Rönnblom #endif 58*719834a6SMattias Rönnblom 59*719834a6SMattias Rönnblom #endif 60*719834a6SMattias Rönnblom 61*719834a6SMattias Rönnblom #ifdef __cplusplus 62*719834a6SMattias Rönnblom extern "C" { 6399a2dd95SBruce Richardson #endif 6499a2dd95SBruce Richardson 6599a2dd95SBruce Richardson __rte_experimental 6699a2dd95SBruce Richardson static inline uint64_t 6799a2dd95SBruce Richardson rte_table_hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size, 6899a2dd95SBruce Richardson uint64_t seed) 6999a2dd95SBruce Richardson { 70097ea8e2SBruce Richardson uint64_t *k = (uint64_t *)key; 71097ea8e2SBruce Richardson uint64_t *m = (uint64_t *)mask; 7299a2dd95SBruce Richardson uint64_t crc0; 7399a2dd95SBruce Richardson 7499a2dd95SBruce Richardson crc0 = rte_crc32_u64(seed, k[0] & m[0]); 7599a2dd95SBruce Richardson 7699a2dd95SBruce Richardson return crc0; 7799a2dd95SBruce Richardson } 7899a2dd95SBruce Richardson 7999a2dd95SBruce Richardson __rte_experimental 8099a2dd95SBruce Richardson static inline uint64_t 8199a2dd95SBruce Richardson rte_table_hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size, 8299a2dd95SBruce Richardson uint64_t seed) 8399a2dd95SBruce Richardson { 84097ea8e2SBruce Richardson uint64_t *k = (uint64_t *)key; 85097ea8e2SBruce Richardson uint64_t *m = (uint64_t *)mask; 8699a2dd95SBruce Richardson uint64_t k0, crc0, crc1; 8799a2dd95SBruce Richardson 8899a2dd95SBruce Richardson k0 = k[0] & m[0]; 8999a2dd95SBruce Richardson 9099a2dd95SBruce Richardson crc0 = rte_crc32_u64(k0, seed); 9199a2dd95SBruce Richardson crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 9299a2dd95SBruce Richardson 9399a2dd95SBruce Richardson crc0 ^= crc1; 9499a2dd95SBruce Richardson 9599a2dd95SBruce Richardson return crc0; 9699a2dd95SBruce Richardson } 9799a2dd95SBruce Richardson 9899a2dd95SBruce Richardson __rte_experimental 9999a2dd95SBruce Richardson static inline uint64_t 10099a2dd95SBruce Richardson rte_table_hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size, 10199a2dd95SBruce Richardson uint64_t seed) 10299a2dd95SBruce Richardson { 103097ea8e2SBruce Richardson uint64_t *k = (uint64_t *)key; 104097ea8e2SBruce Richardson uint64_t *m = (uint64_t *)mask; 10599a2dd95SBruce Richardson uint64_t k0, k2, crc0, crc1; 10699a2dd95SBruce Richardson 10799a2dd95SBruce Richardson k0 = k[0] & m[0]; 10899a2dd95SBruce Richardson k2 = k[2] & m[2]; 10999a2dd95SBruce Richardson 11099a2dd95SBruce Richardson crc0 = rte_crc32_u64(k0, seed); 11199a2dd95SBruce Richardson crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 11299a2dd95SBruce Richardson 11399a2dd95SBruce Richardson crc0 = rte_crc32_u64(crc0, k2); 11499a2dd95SBruce Richardson 11599a2dd95SBruce Richardson crc0 ^= crc1; 11699a2dd95SBruce Richardson 11799a2dd95SBruce Richardson return crc0; 11899a2dd95SBruce Richardson } 11999a2dd95SBruce Richardson 12099a2dd95SBruce Richardson __rte_experimental 12199a2dd95SBruce Richardson static inline uint64_t 12299a2dd95SBruce Richardson rte_table_hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size, 12399a2dd95SBruce Richardson uint64_t seed) 12499a2dd95SBruce Richardson { 125097ea8e2SBruce Richardson uint64_t *k = (uint64_t *)key; 126097ea8e2SBruce Richardson uint64_t *m = (uint64_t *)mask; 12799a2dd95SBruce Richardson uint64_t k0, k2, crc0, crc1, crc2, crc3; 12899a2dd95SBruce Richardson 12999a2dd95SBruce Richardson k0 = k[0] & m[0]; 13099a2dd95SBruce Richardson k2 = k[2] & m[2]; 13199a2dd95SBruce Richardson 13299a2dd95SBruce Richardson crc0 = rte_crc32_u64(k0, seed); 13399a2dd95SBruce Richardson crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 13499a2dd95SBruce Richardson 13599a2dd95SBruce Richardson crc2 = rte_crc32_u64(k2, k[3] & m[3]); 13699a2dd95SBruce Richardson crc3 = k2 >> 32; 13799a2dd95SBruce Richardson 13899a2dd95SBruce Richardson crc0 = rte_crc32_u64(crc0, crc1); 13999a2dd95SBruce Richardson crc1 = rte_crc32_u64(crc2, crc3); 14099a2dd95SBruce Richardson 14199a2dd95SBruce Richardson crc0 ^= crc1; 14299a2dd95SBruce Richardson 14399a2dd95SBruce Richardson return crc0; 14499a2dd95SBruce Richardson } 14599a2dd95SBruce Richardson 14699a2dd95SBruce Richardson __rte_experimental 14799a2dd95SBruce Richardson static inline uint64_t 14899a2dd95SBruce Richardson rte_table_hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size, 14999a2dd95SBruce Richardson uint64_t seed) 15099a2dd95SBruce Richardson { 151097ea8e2SBruce Richardson uint64_t *k = (uint64_t *)key; 152097ea8e2SBruce Richardson uint64_t *m = (uint64_t *)mask; 15399a2dd95SBruce Richardson uint64_t k0, k2, crc0, crc1, crc2, crc3; 15499a2dd95SBruce Richardson 15599a2dd95SBruce Richardson k0 = k[0] & m[0]; 15699a2dd95SBruce Richardson k2 = k[2] & m[2]; 15799a2dd95SBruce Richardson 15899a2dd95SBruce Richardson crc0 = rte_crc32_u64(k0, seed); 15999a2dd95SBruce Richardson crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 16099a2dd95SBruce Richardson 16199a2dd95SBruce Richardson crc2 = rte_crc32_u64(k2, k[3] & m[3]); 16299a2dd95SBruce Richardson crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]); 16399a2dd95SBruce Richardson 16499a2dd95SBruce Richardson crc0 = rte_crc32_u64(crc0, crc1); 16599a2dd95SBruce Richardson crc1 = rte_crc32_u64(crc2, crc3); 16699a2dd95SBruce Richardson 16799a2dd95SBruce Richardson crc0 ^= crc1; 16899a2dd95SBruce Richardson 16999a2dd95SBruce Richardson return crc0; 17099a2dd95SBruce Richardson } 17199a2dd95SBruce Richardson 17299a2dd95SBruce Richardson __rte_experimental 17399a2dd95SBruce Richardson static inline uint64_t 17499a2dd95SBruce Richardson rte_table_hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size, 17599a2dd95SBruce Richardson uint64_t seed) 17699a2dd95SBruce Richardson { 177097ea8e2SBruce Richardson uint64_t *k = (uint64_t *)key; 178097ea8e2SBruce Richardson uint64_t *m = (uint64_t *)mask; 17999a2dd95SBruce Richardson uint64_t k0, k2, k5, crc0, crc1, crc2, crc3; 18099a2dd95SBruce Richardson 18199a2dd95SBruce Richardson k0 = k[0] & m[0]; 18299a2dd95SBruce Richardson k2 = k[2] & m[2]; 18399a2dd95SBruce Richardson k5 = k[5] & m[5]; 18499a2dd95SBruce Richardson 18599a2dd95SBruce Richardson crc0 = rte_crc32_u64(k0, seed); 18699a2dd95SBruce Richardson crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 18799a2dd95SBruce Richardson 18899a2dd95SBruce Richardson crc2 = rte_crc32_u64(k2, k[3] & m[3]); 18999a2dd95SBruce Richardson crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]); 19099a2dd95SBruce Richardson 19199a2dd95SBruce Richardson crc0 = rte_crc32_u64(crc0, (crc1 << 32) ^ crc2); 19299a2dd95SBruce Richardson crc1 = rte_crc32_u64(crc3, k5); 19399a2dd95SBruce Richardson 19499a2dd95SBruce Richardson crc0 ^= crc1; 19599a2dd95SBruce Richardson 19699a2dd95SBruce Richardson return crc0; 19799a2dd95SBruce Richardson } 19899a2dd95SBruce Richardson 19999a2dd95SBruce Richardson __rte_experimental 20099a2dd95SBruce Richardson static inline uint64_t 20199a2dd95SBruce Richardson rte_table_hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size, 20299a2dd95SBruce Richardson uint64_t seed) 20399a2dd95SBruce Richardson { 204097ea8e2SBruce Richardson uint64_t *k = (uint64_t *)key; 205097ea8e2SBruce Richardson uint64_t *m = (uint64_t *)mask; 20699a2dd95SBruce Richardson uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5; 20799a2dd95SBruce Richardson 20899a2dd95SBruce Richardson k0 = k[0] & m[0]; 20999a2dd95SBruce Richardson k2 = k[2] & m[2]; 21099a2dd95SBruce Richardson k5 = k[5] & m[5]; 21199a2dd95SBruce Richardson 21299a2dd95SBruce Richardson crc0 = rte_crc32_u64(k0, seed); 21399a2dd95SBruce Richardson crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 21499a2dd95SBruce Richardson 21599a2dd95SBruce Richardson crc2 = rte_crc32_u64(k2, k[3] & m[3]); 21699a2dd95SBruce Richardson crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]); 21799a2dd95SBruce Richardson 21899a2dd95SBruce Richardson crc4 = rte_crc32_u64(k5, k[6] & m[6]); 21999a2dd95SBruce Richardson crc5 = k5 >> 32; 22099a2dd95SBruce Richardson 22199a2dd95SBruce Richardson crc0 = rte_crc32_u64(crc0, (crc1 << 32) ^ crc2); 22299a2dd95SBruce Richardson crc1 = rte_crc32_u64(crc3, (crc4 << 32) ^ crc5); 22399a2dd95SBruce Richardson 22499a2dd95SBruce Richardson crc0 ^= crc1; 22599a2dd95SBruce Richardson 22699a2dd95SBruce Richardson return crc0; 22799a2dd95SBruce Richardson } 22899a2dd95SBruce Richardson 22999a2dd95SBruce Richardson __rte_experimental 23099a2dd95SBruce Richardson static inline uint64_t 23199a2dd95SBruce Richardson rte_table_hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size, 23299a2dd95SBruce Richardson uint64_t seed) 23399a2dd95SBruce Richardson { 234097ea8e2SBruce Richardson uint64_t *k = (uint64_t *)key; 235097ea8e2SBruce Richardson uint64_t *m = (uint64_t *)mask; 23699a2dd95SBruce Richardson uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5; 23799a2dd95SBruce Richardson 23899a2dd95SBruce Richardson k0 = k[0] & m[0]; 23999a2dd95SBruce Richardson k2 = k[2] & m[2]; 24099a2dd95SBruce Richardson k5 = k[5] & m[5]; 24199a2dd95SBruce Richardson 24299a2dd95SBruce Richardson crc0 = rte_crc32_u64(k0, seed); 24399a2dd95SBruce Richardson crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]); 24499a2dd95SBruce Richardson 24599a2dd95SBruce Richardson crc2 = rte_crc32_u64(k2, k[3] & m[3]); 24699a2dd95SBruce Richardson crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]); 24799a2dd95SBruce Richardson 24899a2dd95SBruce Richardson crc4 = rte_crc32_u64(k5, k[6] & m[6]); 24999a2dd95SBruce Richardson crc5 = rte_crc32_u64(k5 >> 32, k[7] & m[7]); 25099a2dd95SBruce Richardson 25199a2dd95SBruce Richardson crc0 = rte_crc32_u64(crc0, (crc1 << 32) ^ crc2); 25299a2dd95SBruce Richardson crc1 = rte_crc32_u64(crc3, (crc4 << 32) ^ crc5); 25399a2dd95SBruce Richardson 25499a2dd95SBruce Richardson crc0 ^= crc1; 25599a2dd95SBruce Richardson 25699a2dd95SBruce Richardson return crc0; 25799a2dd95SBruce Richardson } 25899a2dd95SBruce Richardson 25999a2dd95SBruce Richardson #ifdef __cplusplus 26099a2dd95SBruce Richardson } 26199a2dd95SBruce Richardson #endif 26299a2dd95SBruce Richardson 26399a2dd95SBruce Richardson #endif 264