xref: /dpdk/app/test/test_hash_readwrite_lf_perf.c (revision 7adf992fb9bf7162a7edc45b50d10fbb1d57824d)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2018 Arm Limited
3  */
4 
5 #include <inttypes.h>
6 #include <locale.h>
7 
8 #include <rte_cycles.h>
9 #include <rte_hash.h>
10 #include <rte_hash_crc.h>
11 #include <rte_jhash.h>
12 #include <rte_launch.h>
13 #include <rte_malloc.h>
14 #include <rte_random.h>
15 #include <rte_spinlock.h>
16 
17 #include "test.h"
18 
19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
21 #endif
22 
23 #define BULK_LOOKUP_SIZE 32
24 
25 #define RUN_WITH_HTM_DISABLED 0
26 
27 #if (RUN_WITH_HTM_DISABLED)
28 
29 #define TOTAL_ENTRY (5*1024)
30 #define TOTAL_INSERT (5*1024)
31 
32 #else
33 
34 #define TOTAL_ENTRY (4*1024*1024)
35 #define TOTAL_INSERT (4*1024*1024)
36 
37 #endif
38 
39 #define READ_FAIL 1
40 #define READ_PASS_NO_KEY_SHIFTS 2
41 #define READ_PASS_SHIFT_PATH 4
42 #define READ_PASS_NON_SHIFT_PATH 8
43 #define BULK_LOOKUP 16
44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32
45 
46 #define WRITE_NO_KEY_SHIFT 0
47 #define WRITE_KEY_SHIFT 1
48 #define WRITE_EXT_BKT 2
49 
50 #define NUM_TEST 3
51 static unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
52 
53 struct rwc_perf {
54 	uint32_t w_no_ks_r_hit[2][NUM_TEST];
55 	uint32_t w_no_ks_r_miss[2][NUM_TEST];
56 	uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
57 	uint32_t w_ks_r_hit_sp[2][NUM_TEST];
58 	uint32_t w_ks_r_miss[2][NUM_TEST];
59 	uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
60 	uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
61 };
62 
63 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
64 
65 static struct {
66 	uint32_t *keys;
67 	uint32_t *keys_no_ks;
68 	uint32_t *keys_ks;
69 	uint32_t *keys_absent;
70 	uint32_t *keys_shift_path;
71 	uint32_t *keys_non_shift_path;
72 	uint32_t *keys_ext_bkt;
73 	uint32_t *keys_ks_extbkt;
74 	uint32_t count_keys_no_ks;
75 	uint32_t count_keys_ks;
76 	uint32_t count_keys_absent;
77 	uint32_t count_keys_shift_path;
78 	uint32_t count_keys_non_shift_path;
79 	uint32_t count_keys_extbkt;
80 	uint32_t count_keys_ks_extbkt;
81 	uint32_t single_insert;
82 	struct rte_hash *h;
83 } tbl_rwc_test_param;
84 
85 static rte_atomic64_t gread_cycles;
86 static rte_atomic64_t greads;
87 
88 static volatile uint8_t writer_done;
89 
90 static uint16_t enabled_core_ids[RTE_MAX_LCORE];
91 
92 static uint8_t *scanned_bkts;
93 
94 static inline uint16_t
95 get_short_sig(const hash_sig_t hash)
96 {
97 	return hash >> 16;
98 }
99 
100 static inline uint32_t
101 get_prim_bucket_index(__attribute__((unused)) const struct rte_hash *h,
102 		      const hash_sig_t hash)
103 {
104 	uint32_t num_buckets;
105 	uint32_t bucket_bitmask;
106 	num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
107 	bucket_bitmask = num_buckets - 1;
108 	return hash & bucket_bitmask;
109 }
110 
111 static inline uint32_t
112 get_alt_bucket_index(__attribute__((unused)) const struct rte_hash *h,
113 			uint32_t cur_bkt_idx, uint16_t sig)
114 {
115 	uint32_t num_buckets;
116 	uint32_t bucket_bitmask;
117 	num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
118 	bucket_bitmask = num_buckets - 1;
119 	return (cur_bkt_idx ^ sig) & bucket_bitmask;
120 }
121 
122 
123 static inline int
124 get_enabled_cores_list(void)
125 {
126 	uint32_t i = 0;
127 	uint16_t core_id;
128 	uint32_t max_cores = rte_lcore_count();
129 	RTE_LCORE_FOREACH(core_id) {
130 		enabled_core_ids[i] = core_id;
131 		i++;
132 	}
133 
134 	if (i != max_cores) {
135 		printf("Number of enabled cores in list is different from "
136 			"number given by rte_lcore_count()\n");
137 		return -1;
138 	}
139 	return 0;
140 }
141 
142 static int
143 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt)
144 {
145 	struct rte_hash *handle;
146 
147 	struct rte_hash_parameters hash_params = {
148 		.entries = TOTAL_ENTRY,
149 		.key_len = sizeof(uint32_t),
150 		.hash_func_init_val = 0,
151 		.socket_id = rte_socket_id(),
152 	};
153 
154 	if (use_jhash)
155 		hash_params.hash_func = rte_jhash;
156 	else
157 		hash_params.hash_func = rte_hash_crc;
158 
159 	if (rwc_lf)
160 		hash_params.extra_flag =
161 			RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
162 			RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
163 	else if (htm)
164 		hash_params.extra_flag =
165 			RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
166 			RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
167 			RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
168 	else
169 		hash_params.extra_flag =
170 			RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
171 			RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
172 
173 	if (ext_bkt)
174 		hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
175 
176 	hash_params.name = "tests";
177 
178 	handle = rte_hash_create(&hash_params);
179 	if (handle == NULL) {
180 		printf("hash creation failed");
181 		return -1;
182 	}
183 
184 	tbl_rwc_test_param.h = handle;
185 	return 0;
186 }
187 
188 static inline int
189 check_bucket(uint32_t bkt_idx, uint32_t key)
190 {
191 	uint32_t iter;
192 	uint32_t prev_iter;
193 	uint32_t diff;
194 	uint32_t count = 0;
195 	const void *next_key;
196 	void *next_data;
197 
198 	/* Temporary bucket to hold the keys */
199 	uint32_t keys_in_bkt[8];
200 
201 	iter = bkt_idx * 8;
202 	prev_iter = iter;
203 	while (rte_hash_iterate(tbl_rwc_test_param.h,
204 			&next_key, &next_data, &iter) >= 0) {
205 
206 		/* Check for duplicate entries */
207 		if (*(const uint32_t *)next_key == key)
208 			return 1;
209 
210 		/* Identify if there is any free entry in the bucket */
211 		diff = iter - prev_iter;
212 		if (diff > 1)
213 			break;
214 
215 		prev_iter = iter;
216 		keys_in_bkt[count] = *(const uint32_t *)next_key;
217 		count++;
218 
219 		/* All entries in the bucket are occupied */
220 		if (count == 8) {
221 
222 			/*
223 			 * Check if bucket was not scanned before, to avoid
224 			 * duplicate keys.
225 			 */
226 			if (scanned_bkts[bkt_idx] == 0) {
227 				/*
228 				 * Since this bucket (pointed to by bkt_idx) is
229 				 * full, it is likely that key(s) in this
230 				 * bucket will be on the shift path, when
231 				 * collision occurs. Thus, add it to
232 				 * keys_shift_path.
233 				 */
234 				memcpy(tbl_rwc_test_param.keys_shift_path +
235 					tbl_rwc_test_param.count_keys_shift_path
236 					, keys_in_bkt, 32);
237 				tbl_rwc_test_param.count_keys_shift_path += 8;
238 				scanned_bkts[bkt_idx] = 1;
239 			}
240 			return -1;
241 		}
242 	}
243 	return 0;
244 }
245 
246 static int
247 generate_keys(void)
248 {
249 	uint32_t *keys = NULL;
250 	uint32_t *keys_no_ks = NULL;
251 	uint32_t *keys_ks = NULL;
252 	uint32_t *keys_absent = NULL;
253 	uint32_t *keys_non_shift_path = NULL;
254 	uint32_t *keys_ext_bkt = NULL;
255 	uint32_t *keys_ks_extbkt = NULL;
256 	uint32_t *found = NULL;
257 	uint32_t count_keys_no_ks = 0;
258 	uint32_t count_keys_ks = 0;
259 	uint32_t count_keys_extbkt = 0;
260 	uint32_t i;
261 
262 	if (init_params(0, 0, 0, 0) != 0)
263 		return -1;
264 
265 	/*
266 	 * keys will consist of a) keys whose addition to the hash table
267 	 * will result in shifting of the existing keys to their alternate
268 	 * locations b) keys whose addition to the hash table will not result
269 	 * in shifting of the existing keys.
270 	 */
271 	keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
272 	if (keys == NULL) {
273 		printf("RTE_MALLOC failed\n");
274 		goto err;
275 	}
276 
277 	/*
278 	 * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys  that
279 	 * will NOT result in shifting of the existing keys to their alternate
280 	 * locations. Roughly around 900K keys.
281 	 */
282 	keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
283 	if (keys_no_ks == NULL) {
284 		printf("RTE_MALLOC failed\n");
285 		goto err;
286 	}
287 
288 	/*
289 	 * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
290 	 * result in shifting of the existing keys to their alternate locations.
291 	 * Roughly around 146K keys. There might be repeating keys. More code is
292 	 * required to filter out these keys which will complicate the test case
293 	 */
294 	keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
295 	if (keys_ks == NULL) {
296 		printf("RTE_MALLOC failed\n");
297 		goto err;
298 	}
299 
300 	/* Used to identify keys not inserted in the hash table */
301 	found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
302 	if (found == NULL) {
303 		printf("RTE_MALLOC failed\n");
304 		goto err;
305 	}
306 
307 	/*
308 	 * This consist of keys not inserted to the hash table.
309 	 * Used to test perf of lookup on keys that do not exist in the table.
310 	 */
311 	keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
312 	if (keys_absent == NULL) {
313 		printf("RTE_MALLOC failed\n");
314 		goto err;
315 	}
316 
317 	/*
318 	 * This consist of keys which are likely to be on the shift
319 	 * path (i.e. being moved to alternate location), when collision occurs
320 	 * on addition of a key to an already full primary bucket.
321 	 * Used to test perf of lookup on keys that are on the shift path.
322 	 */
323 	tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
324 							TOTAL_INSERT, 0);
325 	if (tbl_rwc_test_param.keys_shift_path == NULL) {
326 		printf("RTE_MALLOC failed\n");
327 		goto err;
328 	}
329 
330 	/*
331 	 * This consist of keys which are never on the shift
332 	 * path (i.e. being moved to alternate location), when collision occurs
333 	 * on addition of a key to an already full primary bucket.
334 	 * Used to test perf of lookup on keys that are not on the shift path.
335 	 */
336 	keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
337 					 0);
338 	if (keys_non_shift_path == NULL) {
339 		printf("RTE_MALLOC failed\n");
340 		goto err;
341 	}
342 
343 	/*
344 	 * This consist of keys which will be stored in extended buckets
345 	 */
346 	keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
347 	if (keys_ext_bkt == NULL) {
348 		printf("RTE_MALLOC failed\n");
349 		goto err;
350 	}
351 
352 	/*
353 	 * This consist of keys which when deleted causes shifting of keys
354 	 * in extended buckets to respective secondary buckets
355 	 */
356 	keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
357 	if (keys_ks_extbkt == NULL) {
358 		printf("RTE_MALLOC failed\n");
359 		goto err;
360 	}
361 
362 	hash_sig_t sig;
363 	uint32_t prim_bucket_idx;
364 	uint32_t sec_bucket_idx;
365 	uint16_t short_sig;
366 	uint32_t num_buckets;
367 	num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
368 	int ret;
369 
370 	/*
371 	 * Used to mark bkts in which at least one key was shifted to its
372 	 * alternate location
373 	 */
374 	scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
375 	if (scanned_bkts == NULL) {
376 		printf("RTE_MALLOC failed\n");
377 		goto err;
378 	}
379 
380 	tbl_rwc_test_param.keys = keys;
381 	tbl_rwc_test_param.keys_no_ks = keys_no_ks;
382 	tbl_rwc_test_param.keys_ks = keys_ks;
383 	tbl_rwc_test_param.keys_absent = keys_absent;
384 	tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
385 	tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt;
386 	tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt;
387 	/* Generate keys by adding previous two keys, neglect overflow */
388 	printf("Generating keys...\n");
389 	keys[0] = 0;
390 	keys[1] = 1;
391 	for (i = 2; i < TOTAL_INSERT; i++)
392 		keys[i] = keys[i-1] + keys[i-2];
393 
394 	/* Segregate keys into keys_no_ks and keys_ks */
395 	for (i = 0; i < TOTAL_INSERT; i++) {
396 		/* Check if primary bucket has space.*/
397 		sig = rte_hash_hash(tbl_rwc_test_param.h,
398 					tbl_rwc_test_param.keys+i);
399 		prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h,
400 							sig);
401 		ret = check_bucket(prim_bucket_idx, keys[i]);
402 		if (ret < 0) {
403 			/*
404 			 * Primary bucket is full, this key will result in
405 			 * shifting of the keys to their alternate locations.
406 			 */
407 			keys_ks[count_keys_ks] = keys[i];
408 			count_keys_ks++;
409 		} else if (ret == 0) {
410 			/*
411 			 * Primary bucket has space, this key will not result in
412 			 * shifting of the keys. Hence, add key to the table.
413 			 */
414 			ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
415 							keys+i,
416 							(void *)((uintptr_t)i));
417 			if (ret < 0) {
418 				printf("writer failed %"PRIu32"\n", i);
419 				break;
420 			}
421 			keys_no_ks[count_keys_no_ks] = keys[i];
422 			count_keys_no_ks++;
423 		}
424 	}
425 
426 	for (i = 0; i < count_keys_no_ks; i++) {
427 		/*
428 		 * Identify keys in keys_no_ks with value less than
429 		 * 4M (HTM enabled) OR 5K (HTM disabled)
430 		 */
431 		if (keys_no_ks[i] < TOTAL_INSERT)
432 			found[keys_no_ks[i]]++;
433 	}
434 
435 	for (i = 0; i < count_keys_ks; i++) {
436 		/*
437 		 * Identify keys in keys_ks with value less than
438 		 * 4M (HTM enabled) OR 5K (HTM disabled)
439 		 */
440 		if (keys_ks[i] < TOTAL_INSERT)
441 			found[keys_ks[i]]++;
442 	}
443 
444 	uint32_t count_keys_absent = 0;
445 	for (i = 0; i < TOTAL_INSERT; i++) {
446 		/*
447 		 * Identify missing keys between 0 and
448 		 * 4M (HTM enabled) OR 5K (HTM disabled)
449 		 */
450 		if (found[i] == 0)
451 			keys_absent[count_keys_absent++] = i;
452 	}
453 
454 	/* Find keys that will not be on the shift path */
455 	uint32_t iter;
456 	const void *next_key;
457 	void *next_data;
458 	uint32_t count = 0;
459 	for (i = 0; i < num_buckets; i++) {
460 		/* Check bucket for no keys shifted to alternate locations */
461 		if (scanned_bkts[i] == 0) {
462 			iter = i * 8;
463 			while (rte_hash_iterate(tbl_rwc_test_param.h,
464 				&next_key, &next_data, &iter) >= 0) {
465 
466 				/* Check if key belongs to the current bucket */
467 				if (i >= (iter-1)/8)
468 					keys_non_shift_path[count++]
469 						= *(const uint32_t *)next_key;
470 				else
471 					break;
472 			}
473 		}
474 	}
475 
476 	tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
477 	tbl_rwc_test_param.count_keys_ks = count_keys_ks;
478 	tbl_rwc_test_param.count_keys_absent = count_keys_absent;
479 	tbl_rwc_test_param.count_keys_non_shift_path = count;
480 
481 	memset(scanned_bkts, 0, num_buckets);
482 	count = 0;
483 	/* Find keys that will be in extended buckets */
484 	for (i = 0; i < count_keys_ks; i++) {
485 		ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i);
486 		if (ret < 0) {
487 			/* Key will be added to ext bkt */
488 			keys_ext_bkt[count_keys_extbkt++] = keys_ks[i];
489 			/* Sec bkt to be added to keys_ks_extbkt */
490 			sig = rte_hash_hash(tbl_rwc_test_param.h,
491 					tbl_rwc_test_param.keys_ks + i);
492 			prim_bucket_idx = get_prim_bucket_index(
493 						tbl_rwc_test_param.h, sig);
494 			short_sig = get_short_sig(sig);
495 			sec_bucket_idx = get_alt_bucket_index(
496 						tbl_rwc_test_param.h,
497 						prim_bucket_idx, short_sig);
498 			if (scanned_bkts[sec_bucket_idx] == 0)
499 				scanned_bkts[sec_bucket_idx] = 1;
500 		}
501 	}
502 
503 	/* Find keys that will shift keys in ext bucket*/
504 	for (i = 0; i < num_buckets; i++) {
505 		if (scanned_bkts[i] == 1) {
506 			iter = i * 8;
507 			while (rte_hash_iterate(tbl_rwc_test_param.h,
508 				&next_key, &next_data, &iter) >= 0) {
509 				/* Check if key belongs to the current bucket */
510 				if (i >= (iter-1)/8)
511 					keys_ks_extbkt[count++]
512 						= *(const uint32_t *)next_key;
513 				else
514 					break;
515 			}
516 		}
517 	}
518 
519 	tbl_rwc_test_param.count_keys_ks_extbkt = count;
520 	tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt;
521 
522 	printf("\nCount of keys NOT causing shifting of existing keys to "
523 	"alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
524 	printf("\nCount of keys causing shifting of existing keys to alternate "
525 		"locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
526 	printf("Count of absent keys that will never be added to the hash "
527 		"table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
528 	printf("Count of keys likely to be on the shift path: %d\n\n",
529 	       tbl_rwc_test_param.count_keys_shift_path);
530 	printf("Count of keys not likely to be on the shift path: %d\n\n",
531 	       tbl_rwc_test_param.count_keys_non_shift_path);
532 	printf("Count of keys in extended buckets: %d\n\n",
533 	       tbl_rwc_test_param.count_keys_extbkt);
534 	printf("Count of keys shifting keys in ext buckets: %d\n\n",
535 	       tbl_rwc_test_param.count_keys_ks_extbkt);
536 
537 	rte_free(found);
538 	rte_free(scanned_bkts);
539 	rte_hash_free(tbl_rwc_test_param.h);
540 	return 0;
541 
542 err:
543 	rte_free(keys);
544 	rte_free(keys_no_ks);
545 	rte_free(keys_ks);
546 	rte_free(keys_absent);
547 	rte_free(found);
548 	rte_free(tbl_rwc_test_param.keys_shift_path);
549 	rte_free(keys_non_shift_path);
550 	rte_free(keys_ext_bkt);
551 	rte_free(keys_ks_extbkt);
552 	rte_free(scanned_bkts);
553 	rte_hash_free(tbl_rwc_test_param.h);
554 	return -1;
555 }
556 
557 static int
558 test_rwc_reader(__attribute__((unused)) void *arg)
559 {
560 	uint32_t i, j;
561 	int ret;
562 	uint64_t begin, cycles;
563 	uint32_t loop_cnt = 0;
564 	uint8_t read_type = (uint8_t)((uintptr_t)arg);
565 	uint32_t read_cnt;
566 	uint32_t *keys;
567 	uint32_t extra_keys;
568 	int32_t pos[BULK_LOOKUP_SIZE];
569 	void *temp_a[BULK_LOOKUP_SIZE];
570 
571 	if (read_type & READ_FAIL) {
572 		keys = tbl_rwc_test_param.keys_absent;
573 		read_cnt = tbl_rwc_test_param.count_keys_absent;
574 	} else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
575 		keys = tbl_rwc_test_param.keys_no_ks;
576 		read_cnt = tbl_rwc_test_param.count_keys_no_ks;
577 	} else if (read_type & READ_PASS_SHIFT_PATH) {
578 		keys = tbl_rwc_test_param.keys_shift_path;
579 		read_cnt = tbl_rwc_test_param.count_keys_shift_path;
580 	} else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) {
581 		keys = tbl_rwc_test_param.keys_ext_bkt;
582 		read_cnt = tbl_rwc_test_param.count_keys_extbkt;
583 	} else {
584 		keys = tbl_rwc_test_param.keys_non_shift_path;
585 		read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
586 	}
587 
588 	extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
589 
590 	begin = rte_rdtsc_precise();
591 	do {
592 		if (read_type & BULK_LOOKUP) {
593 			for (i = 0; i < (read_cnt - extra_keys);
594 			     i += BULK_LOOKUP_SIZE) {
595 				/* Array of  pointer to the list of keys */
596 				for (j = 0; j < BULK_LOOKUP_SIZE; j++)
597 					temp_a[j] = keys + i + j;
598 				rte_hash_lookup_bulk(tbl_rwc_test_param.h,
599 						   (const void **)
600 						   ((uintptr_t)temp_a),
601 						   BULK_LOOKUP_SIZE, pos);
602 				/* Validate lookup result */
603 				for (j = 0; j < BULK_LOOKUP_SIZE; j++)
604 					if ((read_type & READ_FAIL &&
605 					     pos[j] != -ENOENT) ||
606 					    (!(read_type & READ_FAIL) &&
607 					     pos[j] == -ENOENT)) {
608 						printf("lookup failed!"
609 						       "%"PRIu32"\n",
610 						       keys[i + j]);
611 						return -1;
612 					}
613 			}
614 			for (j = 0; j < extra_keys; j++)
615 				temp_a[j] = keys + i + j;
616 
617 			rte_hash_lookup_bulk(tbl_rwc_test_param.h,
618 					   (const void **)
619 					   ((uintptr_t)temp_a),
620 					   extra_keys, pos);
621 			for (j = 0; j < extra_keys; j++)
622 				if ((read_type & READ_FAIL &&
623 				     pos[j] != -ENOENT) ||
624 				    (!(read_type & READ_FAIL) &&
625 				     pos[j] == -ENOENT)) {
626 					printf("lookup failed! %"PRIu32"\n",
627 					       keys[i + j]);
628 					return -1;
629 				}
630 		} else {
631 			for (i = 0; i < read_cnt; i++) {
632 				ret = rte_hash_lookup
633 					(tbl_rwc_test_param.h, keys + i);
634 				if (((read_type & READ_FAIL) &&
635 				     (ret != -ENOENT)) ||
636 				    (!(read_type & READ_FAIL) &&
637 					ret == -ENOENT)) {
638 					printf("lookup failed! %"PRIu32"\n",
639 					       keys[i]);
640 					return -1;
641 				}
642 			}
643 		}
644 		loop_cnt++;
645 	} while (!writer_done);
646 
647 	cycles = rte_rdtsc_precise() - begin;
648 	rte_atomic64_add(&gread_cycles, cycles);
649 	rte_atomic64_add(&greads, read_cnt*loop_cnt);
650 	return 0;
651 }
652 
653 static int
654 write_keys(uint8_t write_type)
655 {
656 	uint32_t i;
657 	int ret;
658 	uint32_t key_cnt = 0;
659 	uint32_t *keys;
660 	if (write_type == WRITE_KEY_SHIFT) {
661 		key_cnt = tbl_rwc_test_param.count_keys_ks;
662 		keys = tbl_rwc_test_param.keys_ks;
663 	} else if (write_type == WRITE_NO_KEY_SHIFT) {
664 		key_cnt = tbl_rwc_test_param.count_keys_no_ks;
665 		keys = tbl_rwc_test_param.keys_no_ks;
666 	} else if (write_type == WRITE_EXT_BKT) {
667 		key_cnt = tbl_rwc_test_param.count_keys_extbkt;
668 		keys = tbl_rwc_test_param.keys_ext_bkt;
669 	}
670 	for (i = 0; i < key_cnt; i++) {
671 		ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
672 		if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) {
673 			printf("writer failed %"PRIu32"\n", i);
674 			return -1;
675 		}
676 	}
677 	return 0;
678 }
679 
680 static int
681 test_rwc_multi_writer(__attribute__((unused)) void *arg)
682 {
683 	uint32_t i, offset;
684 	uint32_t pos_core = (uint32_t)((uintptr_t)arg);
685 	offset = pos_core * tbl_rwc_test_param.single_insert;
686 	for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
687 		rte_hash_add_key(tbl_rwc_test_param.h,
688 				 tbl_rwc_test_param.keys_ks + i);
689 	return 0;
690 }
691 
692 /*
693  * Test lookup perf:
694  * Reader(s) lookup keys present in the table.
695  */
696 static int
697 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
698 				int htm, int ext_bkt)
699 {
700 	unsigned int n, m;
701 	uint64_t i;
702 	int use_jhash = 0;
703 	uint8_t write_type = WRITE_NO_KEY_SHIFT;
704 	uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
705 
706 	rte_atomic64_init(&greads);
707 	rte_atomic64_init(&gread_cycles);
708 
709 	if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
710 		goto err;
711 	printf("\nTest: Hash add - no key-shifts, read - hit\n");
712 	for (m = 0; m < 2; m++) {
713 		if (m == 1) {
714 			printf("\n** With bulk-lookup **\n");
715 			read_type |= BULK_LOOKUP;
716 		}
717 		for (n = 0; n < NUM_TEST; n++) {
718 			unsigned int tot_lcore = rte_lcore_count();
719 			if (tot_lcore < rwc_core_cnt[n] + 1)
720 				goto finish;
721 
722 			printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
723 
724 			rte_atomic64_clear(&greads);
725 			rte_atomic64_clear(&gread_cycles);
726 
727 			rte_hash_reset(tbl_rwc_test_param.h);
728 			writer_done = 0;
729 			if (write_keys(write_type) < 0)
730 				goto err;
731 			writer_done = 1;
732 			for (i = 1; i <= rwc_core_cnt[n]; i++)
733 				rte_eal_remote_launch(test_rwc_reader,
734 						(void *)(uintptr_t)read_type,
735 							enabled_core_ids[i]);
736 
737 			for (i = 1; i <= rwc_core_cnt[n]; i++)
738 				if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
739 					goto err;
740 
741 			unsigned long long cycles_per_lookup =
742 				rte_atomic64_read(&gread_cycles) /
743 				rte_atomic64_read(&greads);
744 			rwc_perf_results->w_no_ks_r_hit[m][n]
745 						= cycles_per_lookup;
746 			printf("Cycles per lookup: %llu\n", cycles_per_lookup);
747 		}
748 	}
749 
750 finish:
751 	rte_hash_free(tbl_rwc_test_param.h);
752 	return 0;
753 
754 err:
755 	rte_eal_mp_wait_lcore();
756 	rte_hash_free(tbl_rwc_test_param.h);
757 	return -1;
758 }
759 
760 /*
761  * Test lookup perf:
762  * Reader(s) lookup keys absent in the table while
763  * 'Main' thread adds with no key-shifts.
764  */
765 static int
766 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
767 				int htm, int ext_bkt)
768 {
769 	unsigned int n, m;
770 	uint64_t i;
771 	int use_jhash = 0;
772 	uint8_t write_type = WRITE_NO_KEY_SHIFT;
773 	uint8_t read_type = READ_FAIL;
774 	int ret;
775 
776 	rte_atomic64_init(&greads);
777 	rte_atomic64_init(&gread_cycles);
778 
779 	if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
780 		goto err;
781 	printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
782 	for (m = 0; m < 2; m++) {
783 		if (m == 1) {
784 			printf("\n** With bulk-lookup **\n");
785 			read_type |= BULK_LOOKUP;
786 		}
787 		for (n = 0; n < NUM_TEST; n++) {
788 			unsigned int tot_lcore = rte_lcore_count();
789 			if (tot_lcore < rwc_core_cnt[n] + 1)
790 				goto finish;
791 
792 			printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
793 
794 			rte_atomic64_clear(&greads);
795 			rte_atomic64_clear(&gread_cycles);
796 
797 			rte_hash_reset(tbl_rwc_test_param.h);
798 			writer_done = 0;
799 
800 			for (i = 1; i <= rwc_core_cnt[n]; i++)
801 				rte_eal_remote_launch(test_rwc_reader,
802 						(void *)(uintptr_t)read_type,
803 							enabled_core_ids[i]);
804 			ret = write_keys(write_type);
805 			writer_done = 1;
806 
807 			if (ret < 0)
808 				goto err;
809 			for (i = 1; i <= rwc_core_cnt[n]; i++)
810 				if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
811 					goto err;
812 
813 			unsigned long long cycles_per_lookup =
814 				rte_atomic64_read(&gread_cycles) /
815 				rte_atomic64_read(&greads);
816 			rwc_perf_results->w_no_ks_r_miss[m][n]
817 						= cycles_per_lookup;
818 			printf("Cycles per lookup: %llu\n", cycles_per_lookup);
819 		}
820 	}
821 
822 finish:
823 	rte_hash_free(tbl_rwc_test_param.h);
824 	return 0;
825 
826 err:
827 	rte_eal_mp_wait_lcore();
828 	rte_hash_free(tbl_rwc_test_param.h);
829 	return -1;
830 }
831 
832 /*
833  * Test lookup perf:
834  * Reader(s) lookup keys present in the table and not likely to be on the
835  * shift path  while 'Main' thread adds keys causing key-shifts.
836  */
837 static int
838 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
839 				    int rwc_lf, int htm, int ext_bkt)
840 {
841 	unsigned int n, m;
842 	uint64_t i;
843 	int use_jhash = 0;
844 	int ret;
845 	uint8_t write_type;
846 	uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
847 
848 	rte_atomic64_init(&greads);
849 	rte_atomic64_init(&gread_cycles);
850 
851 	if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
852 		goto err;
853 	printf("\nTest: Hash add - key shift, Hash lookup - hit"
854 	       " (non-shift-path)\n");
855 	for (m = 0; m < 2; m++) {
856 		if (m == 1) {
857 			printf("\n** With bulk-lookup **\n");
858 			read_type |= BULK_LOOKUP;
859 		}
860 		for (n = 0; n < NUM_TEST; n++) {
861 			unsigned int tot_lcore = rte_lcore_count();
862 			if (tot_lcore < rwc_core_cnt[n] + 1)
863 				goto finish;
864 
865 			printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
866 
867 			rte_atomic64_clear(&greads);
868 			rte_atomic64_clear(&gread_cycles);
869 
870 			rte_hash_reset(tbl_rwc_test_param.h);
871 			writer_done = 0;
872 			write_type = WRITE_NO_KEY_SHIFT;
873 			if (write_keys(write_type) < 0)
874 				goto err;
875 			for (i = 1; i <= rwc_core_cnt[n]; i++)
876 				rte_eal_remote_launch(test_rwc_reader,
877 						(void *)(uintptr_t)read_type,
878 							enabled_core_ids[i]);
879 			write_type = WRITE_KEY_SHIFT;
880 			ret = write_keys(write_type);
881 			writer_done = 1;
882 
883 			if (ret < 0)
884 				goto err;
885 			for (i = 1; i <= rwc_core_cnt[n]; i++)
886 				if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
887 					goto err;
888 
889 			unsigned long long cycles_per_lookup =
890 				rte_atomic64_read(&gread_cycles) /
891 				rte_atomic64_read(&greads);
892 			rwc_perf_results->w_ks_r_hit_nsp[m][n]
893 						= cycles_per_lookup;
894 			printf("Cycles per lookup: %llu\n", cycles_per_lookup);
895 		}
896 	}
897 
898 finish:
899 	rte_hash_free(tbl_rwc_test_param.h);
900 	return 0;
901 
902 err:
903 	rte_eal_mp_wait_lcore();
904 	rte_hash_free(tbl_rwc_test_param.h);
905 	return -1;
906 }
907 
908 /*
909  * Test lookup perf:
910  * Reader(s) lookup keys present in the table and likely on the shift-path while
911  * 'Main' thread adds keys causing key-shifts.
912  */
913 static int
914 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
915 				int htm, int ext_bkt)
916 {
917 	unsigned int n, m;
918 	uint64_t i;
919 	int use_jhash = 0;
920 	int ret;
921 	uint8_t write_type;
922 	uint8_t read_type = READ_PASS_SHIFT_PATH;
923 
924 	rte_atomic64_init(&greads);
925 	rte_atomic64_init(&gread_cycles);
926 
927 	if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
928 		goto err;
929 	printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
930 	       "\n");
931 
932 	for (m = 0; m < 2; m++) {
933 		if (m == 1) {
934 			printf("\n** With bulk-lookup **\n");
935 			read_type |= BULK_LOOKUP;
936 		}
937 		for (n = 0; n < NUM_TEST; n++) {
938 			unsigned int tot_lcore = rte_lcore_count();
939 			if (tot_lcore < rwc_core_cnt[n] + 1)
940 				goto finish;
941 
942 			printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
943 			rte_atomic64_clear(&greads);
944 			rte_atomic64_clear(&gread_cycles);
945 
946 			rte_hash_reset(tbl_rwc_test_param.h);
947 			writer_done = 0;
948 			write_type = WRITE_NO_KEY_SHIFT;
949 			if (write_keys(write_type) < 0)
950 				goto err;
951 			for (i = 1; i <= rwc_core_cnt[n]; i++)
952 				rte_eal_remote_launch(test_rwc_reader,
953 						(void *)(uintptr_t)read_type,
954 						enabled_core_ids[i]);
955 			write_type = WRITE_KEY_SHIFT;
956 			ret = write_keys(write_type);
957 			writer_done = 1;
958 
959 			if (ret < 0)
960 				goto err;
961 			for (i = 1; i <= rwc_core_cnt[n]; i++)
962 				if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
963 					goto err;
964 
965 			unsigned long long cycles_per_lookup =
966 				rte_atomic64_read(&gread_cycles) /
967 				rte_atomic64_read(&greads);
968 			rwc_perf_results->w_ks_r_hit_sp[m][n]
969 						= cycles_per_lookup;
970 			printf("Cycles per lookup: %llu\n", cycles_per_lookup);
971 		}
972 	}
973 
974 finish:
975 	rte_hash_free(tbl_rwc_test_param.h);
976 	return 0;
977 
978 err:
979 	rte_eal_mp_wait_lcore();
980 	rte_hash_free(tbl_rwc_test_param.h);
981 	return -1;
982 }
983 
984 /*
985  * Test lookup perf:
986  * Reader(s) lookup keys absent in the table while
987  * 'Main' thread adds keys causing key-shifts.
988  */
989 static int
990 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
991 			     htm, int ext_bkt)
992 {
993 	unsigned int n, m;
994 	uint64_t i;
995 	int use_jhash = 0;
996 	int ret;
997 	uint8_t write_type;
998 	uint8_t read_type = READ_FAIL;
999 
1000 	rte_atomic64_init(&greads);
1001 	rte_atomic64_init(&gread_cycles);
1002 
1003 	if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1004 		goto err;
1005 	printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
1006 	for (m = 0; m < 2; m++) {
1007 		if (m == 1) {
1008 			printf("\n** With bulk-lookup **\n");
1009 			read_type |= BULK_LOOKUP;
1010 		}
1011 		for (n = 0; n < NUM_TEST; n++) {
1012 			unsigned int tot_lcore = rte_lcore_count();
1013 			if (tot_lcore < rwc_core_cnt[n] + 1)
1014 				goto finish;
1015 
1016 			printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1017 
1018 			rte_atomic64_clear(&greads);
1019 			rte_atomic64_clear(&gread_cycles);
1020 
1021 			rte_hash_reset(tbl_rwc_test_param.h);
1022 			writer_done = 0;
1023 			write_type = WRITE_NO_KEY_SHIFT;
1024 			if (write_keys(write_type) < 0)
1025 				goto err;
1026 			for (i = 1; i <= rwc_core_cnt[n]; i++)
1027 				rte_eal_remote_launch(test_rwc_reader,
1028 						(void *)(uintptr_t)read_type,
1029 							enabled_core_ids[i]);
1030 			write_type = WRITE_KEY_SHIFT;
1031 			ret = write_keys(write_type);
1032 			writer_done = 1;
1033 
1034 			if (ret < 0)
1035 				goto err;
1036 			for (i = 1; i <= rwc_core_cnt[n]; i++)
1037 				if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1038 					goto err;
1039 
1040 			unsigned long long cycles_per_lookup =
1041 				rte_atomic64_read(&gread_cycles) /
1042 				rte_atomic64_read(&greads);
1043 			rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
1044 			printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1045 		}
1046 	}
1047 
1048 finish:
1049 	rte_hash_free(tbl_rwc_test_param.h);
1050 	return 0;
1051 
1052 err:
1053 	rte_eal_mp_wait_lcore();
1054 	rte_hash_free(tbl_rwc_test_param.h);
1055 	return -1;
1056 }
1057 
1058 /*
1059  * Test lookup perf for multi-writer:
1060  * Reader(s) lookup keys present in the table and likely on the shift-path while
1061  * Writers add keys causing key-shiftsi.
1062  * Writers are running in parallel, on different data plane cores.
1063  */
1064 static int
1065 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
1066 			   int htm, int ext_bkt)
1067 {
1068 	unsigned int n, m, k;
1069 	uint64_t i;
1070 	int use_jhash = 0;
1071 	uint8_t write_type;
1072 	uint8_t read_type = READ_PASS_SHIFT_PATH;
1073 
1074 	rte_atomic64_init(&greads);
1075 	rte_atomic64_init(&gread_cycles);
1076 
1077 	if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1078 		goto err;
1079 	printf("\nTest: Multi-add-lookup\n");
1080 	uint8_t pos_core;
1081 	for (m = 1; m < NUM_TEST; m++) {
1082 		/* Calculate keys added by each writer */
1083 		tbl_rwc_test_param.single_insert =
1084 			tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
1085 		for (k = 0; k < 2; k++) {
1086 			if (k == 1) {
1087 				printf("\n** With bulk-lookup **\n");
1088 				read_type |= BULK_LOOKUP;
1089 			}
1090 			for (n = 0; n < NUM_TEST; n++) {
1091 				unsigned int tot_lcore	= rte_lcore_count();
1092 				if (tot_lcore < (rwc_core_cnt[n] +
1093 				     rwc_core_cnt[m] + 1))
1094 					goto finish;
1095 
1096 				printf("\nNumber of writers: %u",
1097 				       rwc_core_cnt[m]);
1098 				printf("\nNumber of readers: %u\n",
1099 				       rwc_core_cnt[n]);
1100 
1101 				rte_atomic64_clear(&greads);
1102 				rte_atomic64_clear(&gread_cycles);
1103 
1104 				rte_hash_reset(tbl_rwc_test_param.h);
1105 				writer_done = 0;
1106 				write_type = WRITE_NO_KEY_SHIFT;
1107 				if (write_keys(write_type) < 0)
1108 					goto err;
1109 
1110 				/* Launch reader(s) */
1111 				for (i = 1; i <= rwc_core_cnt[n]; i++)
1112 					rte_eal_remote_launch(test_rwc_reader,
1113 						(void *)(uintptr_t)read_type,
1114 						enabled_core_ids[i]);
1115 				write_type = WRITE_KEY_SHIFT;
1116 				pos_core = 0;
1117 
1118 				/* Launch writers */
1119 				for (; i <= rwc_core_cnt[m]
1120 				     + rwc_core_cnt[n];	i++) {
1121 					rte_eal_remote_launch
1122 						(test_rwc_multi_writer,
1123 						(void *)(uintptr_t)pos_core,
1124 						enabled_core_ids[i]);
1125 					pos_core++;
1126 				}
1127 
1128 				/* Wait for writers to complete */
1129 				for (i = rwc_core_cnt[n] + 1;
1130 				     i <= rwc_core_cnt[m] + rwc_core_cnt[n];
1131 				     i++)
1132 					rte_eal_wait_lcore(enabled_core_ids[i]);
1133 
1134 				writer_done = 1;
1135 
1136 				for (i = 1; i <= rwc_core_cnt[n]; i++)
1137 					if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1138 						goto err;
1139 
1140 				unsigned long long cycles_per_lookup =
1141 					rte_atomic64_read(&gread_cycles)
1142 					/ rte_atomic64_read(&greads);
1143 				rwc_perf_results->multi_rw[m][k][n]
1144 					= cycles_per_lookup;
1145 				printf("Cycles per lookup: %llu\n",
1146 				       cycles_per_lookup);
1147 			}
1148 		}
1149 	}
1150 
1151 finish:
1152 	rte_hash_free(tbl_rwc_test_param.h);
1153 	return 0;
1154 
1155 err:
1156 	rte_eal_mp_wait_lcore();
1157 	rte_hash_free(tbl_rwc_test_param.h);
1158 	return -1;
1159 }
1160 
1161 /*
1162  * Test lookup perf:
1163  * Reader(s) lookup keys present in the extendable bkt.
1164  */
1165 static int
1166 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
1167 				int rwc_lf, int htm, int ext_bkt)
1168 {
1169 	unsigned int n, m;
1170 	uint64_t i;
1171 	int use_jhash = 0;
1172 	uint8_t write_type;
1173 	uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT;
1174 
1175 	rte_atomic64_init(&greads);
1176 	rte_atomic64_init(&gread_cycles);
1177 
1178 	if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1179 		goto err;
1180 	printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n");
1181 	for (m = 0; m < 2; m++) {
1182 		if (m == 1) {
1183 			printf("\n** With bulk-lookup **\n");
1184 			read_type |= BULK_LOOKUP;
1185 		}
1186 		for (n = 0; n < NUM_TEST; n++) {
1187 			unsigned int tot_lcore = rte_lcore_count();
1188 			if (tot_lcore < rwc_core_cnt[n] + 1)
1189 				goto finish;
1190 
1191 			printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1192 
1193 			rte_atomic64_clear(&greads);
1194 			rte_atomic64_clear(&gread_cycles);
1195 
1196 			rte_hash_reset(tbl_rwc_test_param.h);
1197 			write_type = WRITE_NO_KEY_SHIFT;
1198 			if (write_keys(write_type) < 0)
1199 				goto err;
1200 			write_type = WRITE_KEY_SHIFT;
1201 			if (write_keys(write_type) < 0)
1202 				goto err;
1203 			writer_done = 0;
1204 			for (i = 1; i <= rwc_core_cnt[n]; i++)
1205 				rte_eal_remote_launch(test_rwc_reader,
1206 						(void *)(uintptr_t)read_type,
1207 							enabled_core_ids[i]);
1208 			for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt;
1209 			     i++) {
1210 				if (rte_hash_del_key(tbl_rwc_test_param.h,
1211 					tbl_rwc_test_param.keys_ks_extbkt + i)
1212 							< 0) {
1213 					printf("Delete Failed: %u\n",
1214 					tbl_rwc_test_param.keys_ks_extbkt[i]);
1215 					goto err;
1216 				}
1217 			}
1218 			writer_done = 1;
1219 
1220 			for (i = 1; i <= rwc_core_cnt[n]; i++)
1221 				if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1222 					goto err;
1223 
1224 			unsigned long long cycles_per_lookup =
1225 				rte_atomic64_read(&gread_cycles) /
1226 				rte_atomic64_read(&greads);
1227 			rwc_perf_results->w_ks_r_hit_extbkt[m][n]
1228 						= cycles_per_lookup;
1229 			printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1230 		}
1231 	}
1232 
1233 finish:
1234 	rte_hash_free(tbl_rwc_test_param.h);
1235 	return 0;
1236 
1237 err:
1238 	rte_eal_mp_wait_lcore();
1239 	rte_hash_free(tbl_rwc_test_param.h);
1240 	return -1;
1241 }
1242 
1243 static int
1244 test_hash_readwrite_lf_perf_main(void)
1245 {
1246 	/*
1247 	 * Variables used to choose different tests.
1248 	 * rwc_lf indicates if read-write concurrency lock-free support is
1249 	 * enabled.
1250 	 * htm indicates if Hardware transactional memory support is enabled.
1251 	 */
1252 	int rwc_lf = 0;
1253 	int htm;
1254 	int ext_bkt = 0;
1255 
1256 	if (rte_lcore_count() < 2) {
1257 		printf("Not enough cores for hash_readwrite_lf_perf_autotest, expecting at least 2\n");
1258 		return TEST_SKIPPED;
1259 	}
1260 
1261 	setlocale(LC_NUMERIC, "");
1262 
1263 	/* Reset tbl_rwc_test_param to discard values from previous run */
1264 	memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param));
1265 
1266 	if (rte_tm_supported())
1267 		htm = 1;
1268 	else
1269 		htm = 0;
1270 
1271 	if (generate_keys() != 0)
1272 		return -1;
1273 	if (get_enabled_cores_list() != 0)
1274 		return -1;
1275 
1276 	if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1277 		rwc_lf = 1;
1278 		ext_bkt = 1;
1279 		printf("Test lookup with read-write concurrency lock free support"
1280 		       " enabled\n");
1281 		if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1282 							htm, ext_bkt) < 0)
1283 			return -1;
1284 		if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1285 							htm, ext_bkt) < 0)
1286 			return -1;
1287 		if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1288 							htm, ext_bkt) < 0)
1289 			return -1;
1290 		if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1291 							htm, ext_bkt) < 0)
1292 			return -1;
1293 		if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm,
1294 						 ext_bkt) < 0)
1295 			return -1;
1296 		if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm,
1297 					       ext_bkt) < 0)
1298 			return -1;
1299 		if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
1300 							htm, ext_bkt) < 0)
1301 			return -1;
1302 	}
1303 	printf("\nTest lookup with read-write concurrency lock free support"
1304 	       " disabled\n");
1305 	rwc_lf = 0;
1306 	if (!htm) {
1307 		printf("With HTM Disabled\n");
1308 		if (!RUN_WITH_HTM_DISABLED) {
1309 			printf("Enable RUN_WITH_HTM_DISABLED to test with"
1310 			       " lock-free disabled");
1311 			goto results;
1312 		}
1313 	} else
1314 		printf("With HTM Enabled\n");
1315 	if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm,
1316 					   ext_bkt) < 0)
1317 		return -1;
1318 	if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1319 						ext_bkt) < 0)
1320 		return -1;
1321 	if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1322 						htm, ext_bkt) < 0)
1323 		return -1;
1324 	if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm,
1325 						ext_bkt) < 0)
1326 		return -1;
1327 	if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1328 					 ext_bkt) < 0)
1329 		return -1;
1330 	if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm,
1331 							ext_bkt) < 0)
1332 		return -1;
1333 	if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf,
1334 						htm, ext_bkt) < 0)
1335 		return -1;
1336 results:
1337 	printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1338 	int i, j, k;
1339 	for (j = 0; j < 2; j++) {
1340 		if (j == 1)
1341 			printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1342 			       "**********#######\n\n");
1343 		printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1344 			"\t\t\t\t_________________\n");
1345 		printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1346 		       "\t\t\tCycles per lookup\n");
1347 		printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1348 		       "\t\t\t_________________\n");
1349 		for (i = 0; i < NUM_TEST; i++) {
1350 			printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1351 			printf("Enabled\t\t");
1352 			printf("N/A\t\t");
1353 			printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1354 				"%u\n\t\t\t\t\t\t\t\t",
1355 				rwc_lf_results.w_no_ks_r_hit[j][i]);
1356 			printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1357 				"%u\n\t\t\t\t\t\t\t\t",
1358 				rwc_lf_results.w_no_ks_r_miss[j][i]);
1359 			printf("Hash add - key-shifts, lookup - hit"
1360 			       "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1361 			       rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1362 			printf("Hash add - key-shifts, lookup - hit "
1363 			       "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1364 			       rwc_lf_results.w_ks_r_hit_sp[j][i]);
1365 			printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1366 				"%u\n\t\t\t\t\t\t\t\t",
1367 				rwc_lf_results.w_ks_r_miss[j][i]);
1368 			printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1369 				"%u\n\n\t\t\t\t",
1370 				rwc_lf_results.w_ks_r_hit_extbkt[j][i]);
1371 
1372 			printf("Disabled\t");
1373 			if (htm)
1374 				printf("Enabled\t\t");
1375 			else
1376 				printf("Disabled\t");
1377 			printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1378 				"%u\n\t\t\t\t\t\t\t\t",
1379 				rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1380 			printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1381 				"%u\n\t\t\t\t\t\t\t\t",
1382 				rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1383 			printf("Hash add - key-shifts, lookup - hit "
1384 			       "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1385 			       rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1386 			printf("Hash add - key-shifts, lookup - hit "
1387 			       "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1388 			       rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1389 			printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1390 			       "%u\n\t\t\t\t\t\t\t\t",
1391 			       rwc_non_lf_results.w_ks_r_miss[j][i]);
1392 			printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1393 				"%u\n",
1394 				rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]);
1395 
1396 			printf("_______\t\t_______\t\t_________\t___\t\t"
1397 			       "_________\t\t\t\t\t\t_________________\n");
1398 		}
1399 
1400 		for (i = 1; i < NUM_TEST; i++) {
1401 			for (k = 0; k < NUM_TEST; k++) {
1402 				printf("%u", rwc_core_cnt[i]);
1403 				printf("\t\t%u\t\t", rwc_core_cnt[k]);
1404 				printf("Enabled\t\t");
1405 				printf("N/A\t\t");
1406 				printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1407 				       "\t\t",
1408 				       rwc_lf_results.multi_rw[i][j][k]);
1409 				printf("Disabled\t");
1410 				if (htm)
1411 					printf("Enabled\t\t");
1412 				else
1413 					printf("Disabled\t");
1414 				printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1415 				       rwc_non_lf_results.multi_rw[i][j][k]);
1416 
1417 				printf("_______\t\t_______\t\t_________\t___"
1418 				       "\t\t_________\t\t\t\t\t\t"
1419 				       "_________________\n");
1420 			}
1421 		}
1422 	}
1423 	rte_free(tbl_rwc_test_param.keys);
1424 	rte_free(tbl_rwc_test_param.keys_no_ks);
1425 	rte_free(tbl_rwc_test_param.keys_ks);
1426 	rte_free(tbl_rwc_test_param.keys_absent);
1427 	rte_free(tbl_rwc_test_param.keys_shift_path);
1428 	rte_free(tbl_rwc_test_param.keys_non_shift_path);
1429 	rte_free(tbl_rwc_test_param.keys_ext_bkt);
1430 	rte_free(tbl_rwc_test_param.keys_ks_extbkt);
1431 	return 0;
1432 }
1433 
1434 REGISTER_TEST_COMMAND(hash_readwrite_lf_perf_autotest,
1435 	test_hash_readwrite_lf_perf_main);
1436