1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
3 * Copyright(c) 2018-2024 Arm Limited
4 */
5
6 #ifndef COMPARE_SIGNATURES_ARM_H
7 #define COMPARE_SIGNATURES_ARM_H
8
9 #include <inttypes.h>
10
11 #include <rte_common.h>
12 #include <rte_vect.h>
13
14 #include "rte_cuckoo_hash.h"
15
16 /* Arm's version uses a densely packed hitmask buffer: every bit is in use. */
17 #define DENSE_HASH_BULK_LOOKUP 1
18
19 static inline void
compare_signatures_dense(uint16_t * hitmask_buffer,const uint16_t * prim_bucket_sigs,const uint16_t * sec_bucket_sigs,uint16_t sig,enum rte_hash_sig_compare_function sig_cmp_fn)20 compare_signatures_dense(uint16_t *hitmask_buffer,
21 const uint16_t *prim_bucket_sigs,
22 const uint16_t *sec_bucket_sigs,
23 uint16_t sig,
24 enum rte_hash_sig_compare_function sig_cmp_fn)
25 {
26 static_assert(sizeof(*hitmask_buffer) >= 2 * (RTE_HASH_BUCKET_ENTRIES / 8),
27 "hitmask_buffer must be wide enough to fit a dense hitmask");
28
29 /* For match mask every bits indicates the match */
30 switch (sig_cmp_fn) {
31 #if defined(__ARM_NEON) && RTE_HASH_BUCKET_ENTRIES <= 8
32 case RTE_HASH_COMPARE_NEON: {
33 uint16x8_t vmat, hit1, hit2;
34 const uint16x8_t mask = {0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};
35 const uint16x8_t vsig = vld1q_dup_u16((uint16_t const *)&sig);
36
37 /* Compare all signatures in the primary bucket */
38 vmat = vceqq_u16(vsig, vld1q_u16(prim_bucket_sigs));
39 hit1 = vandq_u16(vmat, mask);
40
41 /* Compare all signatures in the secondary bucket */
42 vmat = vceqq_u16(vsig, vld1q_u16(sec_bucket_sigs));
43 hit2 = vandq_u16(vmat, mask);
44
45 hit2 = vshlq_n_u16(hit2, RTE_HASH_BUCKET_ENTRIES);
46 hit2 = vorrq_u16(hit1, hit2);
47 *hitmask_buffer = vaddvq_u16(hit2);
48 break;
49 }
50 #endif
51 #if defined(RTE_HAS_SVE_ACLE)
52 case RTE_HASH_COMPARE_SVE: {
53 svuint16_t vsign, shift, sv_matches;
54 svbool_t pred, match, bucket_wide_pred;
55 int i = 0;
56 uint64_t vl = svcnth();
57
58 vsign = svdup_u16(sig);
59 shift = svindex_u16(0, 1);
60
61 if (vl >= 2 * RTE_HASH_BUCKET_ENTRIES && RTE_HASH_BUCKET_ENTRIES <= 8) {
62 svuint16_t primary_array_vect, secondary_array_vect;
63 bucket_wide_pred = svwhilelt_b16(0, RTE_HASH_BUCKET_ENTRIES);
64 primary_array_vect = svld1_u16(bucket_wide_pred, prim_bucket_sigs);
65 secondary_array_vect = svld1_u16(bucket_wide_pred, sec_bucket_sigs);
66
67 /* We merged the two vectors so we can do both comparisons at once */
68 primary_array_vect = svsplice_u16(bucket_wide_pred, primary_array_vect,
69 secondary_array_vect);
70 pred = svwhilelt_b16(0, 2*RTE_HASH_BUCKET_ENTRIES);
71
72 /* Compare all signatures in the buckets */
73 match = svcmpeq_u16(pred, vsign, primary_array_vect);
74 if (svptest_any(svptrue_b16(), match)) {
75 sv_matches = svdup_u16(1);
76 sv_matches = svlsl_u16_z(match, sv_matches, shift);
77 *hitmask_buffer = svorv_u16(svptrue_b16(), sv_matches);
78 }
79 } else {
80 do {
81 pred = svwhilelt_b16(i, RTE_HASH_BUCKET_ENTRIES);
82 uint16_t lower_half = 0;
83 uint16_t upper_half = 0;
84 /* Compare all signatures in the primary bucket */
85 match = svcmpeq_u16(pred, vsign, svld1_u16(pred,
86 &prim_bucket_sigs[i]));
87 if (svptest_any(svptrue_b16(), match)) {
88 sv_matches = svdup_u16(1);
89 sv_matches = svlsl_u16_z(match, sv_matches, shift);
90 lower_half = svorv_u16(svptrue_b16(), sv_matches);
91 }
92 /* Compare all signatures in the secondary bucket */
93 match = svcmpeq_u16(pred, vsign, svld1_u16(pred,
94 &sec_bucket_sigs[i]));
95 if (svptest_any(svptrue_b16(), match)) {
96 sv_matches = svdup_u16(1);
97 sv_matches = svlsl_u16_z(match, sv_matches, shift);
98 upper_half = svorv_u16(svptrue_b16(), sv_matches)
99 << RTE_HASH_BUCKET_ENTRIES;
100 }
101 hitmask_buffer[i / 8] = upper_half | lower_half;
102 i += vl;
103 } while (i < RTE_HASH_BUCKET_ENTRIES);
104 }
105 break;
106 }
107 #endif
108 default:
109 for (unsigned int i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
110 *hitmask_buffer |= (sig == prim_bucket_sigs[i]) << i;
111 *hitmask_buffer |=
112 ((sig == sec_bucket_sigs[i]) << i) << RTE_HASH_BUCKET_ENTRIES;
113 }
114 }
115 }
116 #endif /* COMPARE_SIGNATURES_ARM_H */
117