1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
3 */
4
5 #include "rte_xxh64_avx512.h"
6 #include "rte_member_sketch_avx512.h"
7
8 __rte_always_inline void
sketch_update_avx512(const struct rte_member_setsum * ss,const void * key,uint32_t count)9 sketch_update_avx512(const struct rte_member_setsum *ss,
10 const void *key,
11 uint32_t count)
12 {
13 uint64_t *count_array = ss->table;
14 uint32_t num_col = ss->num_col;
15 uint32_t key_len = ss->key_len;
16 __m256i v_row_base;
17 __m256i v_hash_result;
18 __m512i current_sketch;
19 __m512i updated_sketch;
20 __m512i v_count;
21
22 const __m256i v_idx = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
23 const __m256i v_col = _mm256_set1_epi32(num_col);
24
25 /* compute the hash result parallelly */
26 v_hash_result = rte_xxh64_sketch_avx512
27 (key, key_len, *(__m512i *)ss->hash_seeds, num_col);
28 v_row_base = _mm256_mullo_epi32(v_idx, v_col);
29 v_hash_result = _mm256_add_epi32(v_row_base, v_hash_result);
30
31 current_sketch = _mm512_i32gather_epi64
32 (v_hash_result, (void *)count_array, 8);
33 v_count = _mm512_set1_epi64(count);
34 updated_sketch = _mm512_add_epi64(current_sketch, v_count);
35 _mm512_i32scatter_epi64
36 ((void *)count_array, v_hash_result, updated_sketch, 8);
37 }
38
39 uint64_t
sketch_lookup_avx512(const struct rte_member_setsum * ss,const void * key)40 sketch_lookup_avx512(const struct rte_member_setsum *ss, const void *key)
41 {
42 uint32_t col[ss->num_row];
43
44 /* currently only for sketch byte count mode */
45 __m256i v_hash_result = rte_xxh64_sketch_avx512
46 (key, ss->key_len, *(__m512i *)ss->hash_seeds, ss->num_col);
47 _mm256_storeu_si256((__m256i *)col, v_hash_result);
48
49 return count_min(ss, col);
50 }
51
52 void
sketch_delete_avx512(const struct rte_member_setsum * ss,const void * key)53 sketch_delete_avx512(const struct rte_member_setsum *ss, const void *key)
54 {
55 uint32_t col[ss->num_row];
56 uint64_t *count_array = ss->table;
57 uint64_t min = UINT64_MAX;
58 uint32_t cur_row;
59
60 __m256i v_hash_result = rte_xxh64_sketch_avx512
61 (key, ss->key_len, *(__m512i *)ss->hash_seeds,
62 RTE_ALIGN_FLOOR(ss->num_col, 32));
63 _mm256_storeu_si256((__m256i *)col, v_hash_result);
64
65 min = count_min(ss, col);
66
67 /* subtract the min value from all the counters */
68 for (cur_row = 0; cur_row < ss->num_row; cur_row++)
69 count_array[cur_row * ss->num_col + col[cur_row]] -= min;
70 }
71