1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016 Intel Corporation
3 */
4
5 #include <inttypes.h>
6 #include <locale.h>
7
8 #include <rte_cycles.h>
9 #include <rte_hash.h>
10 #include <rte_hash_crc.h>
11 #include <rte_launch.h>
12 #include <rte_malloc.h>
13 #include <rte_random.h>
14 #include <rte_spinlock.h>
15 #include <rte_jhash.h>
16
17 #include "test.h"
18
19 /*
20 * Check condition and return an error if true. Assumes that "handle" is the
21 * name of the hash structure pointer to be freed.
22 */
23 #define RETURN_IF_ERROR(cond, str, ...) do { \
24 if (cond) { \
25 printf("ERROR line %d: " str "\n", __LINE__, \
26 ##__VA_ARGS__); \
27 if (handle) \
28 rte_hash_free(handle); \
29 return -1; \
30 } \
31 } while (0)
32
33 #define RTE_APP_TEST_HASH_MULTIWRITER_FAILED 0
34
35 struct {
36 uint32_t *keys;
37 uint32_t *found;
38 uint32_t nb_tsx_insertion;
39 struct rte_hash *h;
40 } tbl_multiwriter_test_params;
41
42 const uint32_t nb_entries = 5*1024*1024;
43 const uint32_t nb_total_tsx_insertion = 4.5*1024*1024;
44 uint32_t rounded_nb_total_tsx_insertion;
45
46 static RTE_ATOMIC(uint64_t) gcycles;
47 static RTE_ATOMIC(uint64_t) ginsertions;
48
49 static int use_htm;
50
51 static int
test_hash_multiwriter_worker(void * arg)52 test_hash_multiwriter_worker(void *arg)
53 {
54 uint64_t i, offset;
55 uint16_t pos_core;
56 uint32_t lcore_id = rte_lcore_id();
57 uint64_t begin, cycles;
58 uint16_t *enabled_core_ids = (uint16_t *)arg;
59
60 for (pos_core = 0; pos_core < rte_lcore_count(); pos_core++) {
61 if (enabled_core_ids[pos_core] == lcore_id)
62 break;
63 }
64
65 /*
66 * Calculate offset for entries based on the position of the
67 * logical core, from the main core (not counting not enabled cores)
68 */
69 offset = pos_core * tbl_multiwriter_test_params.nb_tsx_insertion;
70
71 printf("Core #%d inserting %d: %'"PRId64" - %'"PRId64"\n",
72 lcore_id, tbl_multiwriter_test_params.nb_tsx_insertion,
73 offset,
74 offset + tbl_multiwriter_test_params.nb_tsx_insertion - 1);
75
76 begin = rte_rdtsc_precise();
77
78 for (i = offset;
79 i < offset + tbl_multiwriter_test_params.nb_tsx_insertion;
80 i++) {
81 if (rte_hash_add_key(tbl_multiwriter_test_params.h,
82 tbl_multiwriter_test_params.keys + i) < 0)
83 break;
84 }
85
86 cycles = rte_rdtsc_precise() - begin;
87 rte_atomic_fetch_add_explicit(&gcycles, cycles, rte_memory_order_relaxed);
88 rte_atomic_fetch_add_explicit(&ginsertions, i - offset, rte_memory_order_relaxed);
89
90 for (; i < offset + tbl_multiwriter_test_params.nb_tsx_insertion; i++)
91 tbl_multiwriter_test_params.keys[i]
92 = RTE_APP_TEST_HASH_MULTIWRITER_FAILED;
93
94 return 0;
95 }
96
97
98 static int
test_hash_multiwriter(void)99 test_hash_multiwriter(void)
100 {
101 unsigned int i, rounded_nb_total_tsx_insertion;
102 static unsigned calledCount = 1;
103 uint16_t enabled_core_ids[RTE_MAX_LCORE];
104 uint16_t core_id;
105
106 uint32_t *keys;
107 uint32_t *found;
108
109 struct rte_hash_parameters hash_params = {
110 .entries = nb_entries,
111 .key_len = sizeof(uint32_t),
112 .hash_func = rte_jhash,
113 .hash_func_init_val = 0,
114 .socket_id = rte_socket_id(),
115 };
116 if (use_htm)
117 hash_params.extra_flag =
118 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT
119 | RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
120 else
121 hash_params.extra_flag =
122 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
123
124 struct rte_hash *handle;
125 char name[RTE_HASH_NAMESIZE];
126
127 const void *next_key;
128 void *next_data;
129 uint32_t iter = 0;
130
131 uint32_t duplicated_keys = 0;
132 uint32_t lost_keys = 0;
133 uint32_t count;
134
135 snprintf(name, 32, "test%u", calledCount++);
136 hash_params.name = name;
137
138 handle = rte_hash_create(&hash_params);
139 RETURN_IF_ERROR(handle == NULL, "hash creation failed");
140
141 tbl_multiwriter_test_params.h = handle;
142 tbl_multiwriter_test_params.nb_tsx_insertion =
143 nb_total_tsx_insertion / rte_lcore_count();
144
145 rounded_nb_total_tsx_insertion = (nb_total_tsx_insertion /
146 tbl_multiwriter_test_params.nb_tsx_insertion)
147 * tbl_multiwriter_test_params.nb_tsx_insertion;
148
149 keys = rte_malloc(NULL, sizeof(uint32_t) * nb_entries, 0);
150
151 if (keys == NULL) {
152 printf("RTE_MALLOC failed\n");
153 goto err1;
154 }
155
156 for (i = 0; i < nb_entries; i++)
157 keys[i] = i;
158
159 tbl_multiwriter_test_params.keys = keys;
160
161 found = rte_zmalloc(NULL, sizeof(uint32_t) * nb_entries, 0);
162 if (found == NULL) {
163 printf("RTE_ZMALLOC failed\n");
164 goto err2;
165 }
166
167 tbl_multiwriter_test_params.found = found;
168
169 rte_atomic_store_explicit(&gcycles, 0, rte_memory_order_relaxed);
170 rte_atomic_store_explicit(&ginsertions, 0, rte_memory_order_relaxed);
171
172 /* Get list of enabled cores */
173 i = 0;
174 for (core_id = 0; core_id < RTE_MAX_LCORE; core_id++) {
175 if (i == rte_lcore_count())
176 break;
177
178 if (rte_lcore_is_enabled(core_id)) {
179 enabled_core_ids[i] = core_id;
180 i++;
181 }
182 }
183
184 if (i != rte_lcore_count()) {
185 printf("Number of enabled cores in list is different from "
186 "number given by rte_lcore_count()\n");
187 goto err3;
188 }
189
190 /* Fire all threads. */
191 rte_eal_mp_remote_launch(test_hash_multiwriter_worker,
192 enabled_core_ids, CALL_MAIN);
193 rte_eal_mp_wait_lcore();
194
195 count = rte_hash_count(handle);
196 if (count != rounded_nb_total_tsx_insertion) {
197 printf("rte_hash_count returned wrong value %u, %d\n",
198 rounded_nb_total_tsx_insertion, count);
199 goto err3;
200 }
201
202 while (rte_hash_iterate(handle, &next_key, &next_data, &iter) >= 0) {
203 /* Search for the key in the list of keys added .*/
204 i = *(const uint32_t *)next_key;
205 tbl_multiwriter_test_params.found[i]++;
206 }
207
208 for (i = 0; i < rounded_nb_total_tsx_insertion; i++) {
209 if (tbl_multiwriter_test_params.keys[i]
210 != RTE_APP_TEST_HASH_MULTIWRITER_FAILED) {
211 if (tbl_multiwriter_test_params.found[i] > 1) {
212 duplicated_keys++;
213 break;
214 }
215 if (tbl_multiwriter_test_params.found[i] == 0) {
216 lost_keys++;
217 printf("key %d is lost\n", i);
218 break;
219 }
220 }
221 }
222
223 if (duplicated_keys > 0) {
224 printf("%d key duplicated\n", duplicated_keys);
225 goto err3;
226 }
227
228 if (lost_keys > 0) {
229 printf("%d key lost\n", lost_keys);
230 goto err3;
231 }
232
233 printf("No key corrupted during multiwriter insertion.\n");
234
235 unsigned long long int cycles_per_insertion =
236 rte_atomic_load_explicit(&gcycles, rte_memory_order_relaxed)/
237 rte_atomic_load_explicit(&ginsertions, rte_memory_order_relaxed);
238
239 printf(" cycles per insertion: %llu\n", cycles_per_insertion);
240
241 rte_free(tbl_multiwriter_test_params.found);
242 rte_free(tbl_multiwriter_test_params.keys);
243 rte_hash_free(handle);
244 return 0;
245
246 err3:
247 rte_free(tbl_multiwriter_test_params.found);
248 err2:
249 rte_free(tbl_multiwriter_test_params.keys);
250 err1:
251 rte_hash_free(handle);
252 return -1;
253 }
254
255 static int
test_hash_multiwriter_main(void)256 test_hash_multiwriter_main(void)
257 {
258 if (rte_lcore_count() < 2) {
259 printf("Not enough cores for distributor_autotest, expecting at least 2\n");
260 return TEST_SKIPPED;
261 }
262
263 setlocale(LC_NUMERIC, "");
264
265
266 if (!rte_tm_supported()) {
267 printf("Hardware transactional memory (lock elision) "
268 "is NOT supported\n");
269 } else {
270 printf("Hardware transactional memory (lock elision) "
271 "is supported\n");
272
273 printf("Test multi-writer with Hardware transactional memory\n");
274
275 use_htm = 1;
276 if (test_hash_multiwriter() < 0)
277 return -1;
278 }
279
280 printf("Test multi-writer without Hardware transactional memory\n");
281 use_htm = 0;
282 if (test_hash_multiwriter() < 0)
283 return -1;
284
285 return 0;
286 }
287
288 REGISTER_PERF_TEST(hash_multiwriter_autotest, test_hash_multiwriter_main);
289