1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Arm Limited 3 */ 4 5 #include <inttypes.h> 6 #include <locale.h> 7 8 #include <rte_cycles.h> 9 #include <rte_hash.h> 10 #include <rte_hash_crc.h> 11 #include <rte_jhash.h> 12 #include <rte_launch.h> 13 #include <rte_malloc.h> 14 #include <rte_random.h> 15 #include <rte_spinlock.h> 16 17 #include "test.h" 18 19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0 21 #endif 22 23 #define BULK_LOOKUP_SIZE 32 24 25 #define RUN_WITH_HTM_DISABLED 0 26 27 #if (RUN_WITH_HTM_DISABLED) 28 29 #define TOTAL_ENTRY (5*1024) 30 #define TOTAL_INSERT (5*1024) 31 32 #else 33 34 #define TOTAL_ENTRY (4*1024*1024) 35 #define TOTAL_INSERT (4*1024*1024) 36 37 #endif 38 39 #define READ_FAIL 1 40 #define READ_PASS_NO_KEY_SHIFTS 2 41 #define READ_PASS_SHIFT_PATH 4 42 #define READ_PASS_NON_SHIFT_PATH 8 43 #define BULK_LOOKUP 16 44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32 45 46 #define WRITE_NO_KEY_SHIFT 0 47 #define WRITE_KEY_SHIFT 1 48 #define WRITE_EXT_BKT 2 49 50 #define NUM_TEST 3 51 static unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4}; 52 53 struct rwc_perf { 54 uint32_t w_no_ks_r_hit[2][NUM_TEST]; 55 uint32_t w_no_ks_r_miss[2][NUM_TEST]; 56 uint32_t w_ks_r_hit_nsp[2][NUM_TEST]; 57 uint32_t w_ks_r_hit_sp[2][NUM_TEST]; 58 uint32_t w_ks_r_miss[2][NUM_TEST]; 59 uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST]; 60 uint32_t w_ks_r_hit_extbkt[2][NUM_TEST]; 61 }; 62 63 static struct rwc_perf rwc_lf_results, rwc_non_lf_results; 64 65 static struct { 66 uint32_t *keys; 67 uint32_t *keys_no_ks; 68 uint32_t *keys_ks; 69 uint32_t *keys_absent; 70 uint32_t *keys_shift_path; 71 uint32_t *keys_non_shift_path; 72 uint32_t *keys_ext_bkt; 73 uint32_t *keys_ks_extbkt; 74 uint32_t count_keys_no_ks; 75 uint32_t count_keys_ks; 76 uint32_t count_keys_absent; 77 uint32_t count_keys_shift_path; 78 uint32_t count_keys_non_shift_path; 79 uint32_t count_keys_extbkt; 80 uint32_t count_keys_ks_extbkt; 81 uint32_t single_insert; 82 struct rte_hash *h; 83 } tbl_rwc_test_param; 84 85 static rte_atomic64_t gread_cycles; 86 static rte_atomic64_t greads; 87 88 static volatile uint8_t writer_done; 89 90 static uint16_t enabled_core_ids[RTE_MAX_LCORE]; 91 92 static uint8_t *scanned_bkts; 93 94 static inline uint16_t 95 get_short_sig(const hash_sig_t hash) 96 { 97 return hash >> 16; 98 } 99 100 static inline uint32_t 101 get_prim_bucket_index(__attribute__((unused)) const struct rte_hash *h, 102 const hash_sig_t hash) 103 { 104 uint32_t num_buckets; 105 uint32_t bucket_bitmask; 106 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; 107 bucket_bitmask = num_buckets - 1; 108 return hash & bucket_bitmask; 109 } 110 111 static inline uint32_t 112 get_alt_bucket_index(__attribute__((unused)) const struct rte_hash *h, 113 uint32_t cur_bkt_idx, uint16_t sig) 114 { 115 uint32_t num_buckets; 116 uint32_t bucket_bitmask; 117 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; 118 bucket_bitmask = num_buckets - 1; 119 return (cur_bkt_idx ^ sig) & bucket_bitmask; 120 } 121 122 123 static inline int 124 get_enabled_cores_list(void) 125 { 126 uint32_t i = 0; 127 uint16_t core_id; 128 uint32_t max_cores = rte_lcore_count(); 129 RTE_LCORE_FOREACH(core_id) { 130 enabled_core_ids[i] = core_id; 131 i++; 132 } 133 134 if (i != max_cores) { 135 printf("Number of enabled cores in list is different from " 136 "number given by rte_lcore_count()\n"); 137 return -1; 138 } 139 return 0; 140 } 141 142 static int 143 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt) 144 { 145 struct rte_hash *handle; 146 147 struct rte_hash_parameters hash_params = { 148 .entries = TOTAL_ENTRY, 149 .key_len = sizeof(uint32_t), 150 .hash_func_init_val = 0, 151 .socket_id = rte_socket_id(), 152 }; 153 154 if (use_jhash) 155 hash_params.hash_func = rte_jhash; 156 else 157 hash_params.hash_func = rte_hash_crc; 158 159 if (rwc_lf) 160 hash_params.extra_flag = 161 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF | 162 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 163 else if (htm) 164 hash_params.extra_flag = 165 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT | 166 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | 167 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 168 else 169 hash_params.extra_flag = 170 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | 171 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 172 173 if (ext_bkt) 174 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE; 175 176 hash_params.name = "tests"; 177 178 handle = rte_hash_create(&hash_params); 179 if (handle == NULL) { 180 printf("hash creation failed"); 181 return -1; 182 } 183 184 tbl_rwc_test_param.h = handle; 185 return 0; 186 } 187 188 static inline int 189 check_bucket(uint32_t bkt_idx, uint32_t key) 190 { 191 uint32_t iter; 192 uint32_t prev_iter; 193 uint32_t diff; 194 uint32_t count = 0; 195 const void *next_key; 196 void *next_data; 197 198 /* Temporary bucket to hold the keys */ 199 uint32_t keys_in_bkt[8]; 200 201 iter = bkt_idx * 8; 202 prev_iter = iter; 203 while (rte_hash_iterate(tbl_rwc_test_param.h, 204 &next_key, &next_data, &iter) >= 0) { 205 206 /* Check for duplicate entries */ 207 if (*(const uint32_t *)next_key == key) 208 return 1; 209 210 /* Identify if there is any free entry in the bucket */ 211 diff = iter - prev_iter; 212 if (diff > 1) 213 break; 214 215 prev_iter = iter; 216 keys_in_bkt[count] = *(const uint32_t *)next_key; 217 count++; 218 219 /* All entries in the bucket are occupied */ 220 if (count == 8) { 221 222 /* 223 * Check if bucket was not scanned before, to avoid 224 * duplicate keys. 225 */ 226 if (scanned_bkts[bkt_idx] == 0) { 227 /* 228 * Since this bucket (pointed to by bkt_idx) is 229 * full, it is likely that key(s) in this 230 * bucket will be on the shift path, when 231 * collision occurs. Thus, add it to 232 * keys_shift_path. 233 */ 234 memcpy(tbl_rwc_test_param.keys_shift_path + 235 tbl_rwc_test_param.count_keys_shift_path 236 , keys_in_bkt, 32); 237 tbl_rwc_test_param.count_keys_shift_path += 8; 238 scanned_bkts[bkt_idx] = 1; 239 } 240 return -1; 241 } 242 } 243 return 0; 244 } 245 246 static int 247 generate_keys(void) 248 { 249 uint32_t *keys = NULL; 250 uint32_t *keys_no_ks = NULL; 251 uint32_t *keys_ks = NULL; 252 uint32_t *keys_absent = NULL; 253 uint32_t *keys_non_shift_path = NULL; 254 uint32_t *keys_ext_bkt = NULL; 255 uint32_t *keys_ks_extbkt = NULL; 256 uint32_t *found = NULL; 257 uint32_t count_keys_no_ks = 0; 258 uint32_t count_keys_ks = 0; 259 uint32_t count_keys_extbkt = 0; 260 uint32_t i; 261 262 if (init_params(0, 0, 0, 0) != 0) 263 return -1; 264 265 /* 266 * keys will consist of a) keys whose addition to the hash table 267 * will result in shifting of the existing keys to their alternate 268 * locations b) keys whose addition to the hash table will not result 269 * in shifting of the existing keys. 270 */ 271 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 272 if (keys == NULL) { 273 printf("RTE_MALLOC failed\n"); 274 goto err; 275 } 276 277 /* 278 * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys that 279 * will NOT result in shifting of the existing keys to their alternate 280 * locations. Roughly around 900K keys. 281 */ 282 keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 283 if (keys_no_ks == NULL) { 284 printf("RTE_MALLOC failed\n"); 285 goto err; 286 } 287 288 /* 289 * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will 290 * result in shifting of the existing keys to their alternate locations. 291 * Roughly around 146K keys. There might be repeating keys. More code is 292 * required to filter out these keys which will complicate the test case 293 */ 294 keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 295 if (keys_ks == NULL) { 296 printf("RTE_MALLOC failed\n"); 297 goto err; 298 } 299 300 /* Used to identify keys not inserted in the hash table */ 301 found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 302 if (found == NULL) { 303 printf("RTE_MALLOC failed\n"); 304 goto err; 305 } 306 307 /* 308 * This consist of keys not inserted to the hash table. 309 * Used to test perf of lookup on keys that do not exist in the table. 310 */ 311 keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 312 if (keys_absent == NULL) { 313 printf("RTE_MALLOC failed\n"); 314 goto err; 315 } 316 317 /* 318 * This consist of keys which are likely to be on the shift 319 * path (i.e. being moved to alternate location), when collision occurs 320 * on addition of a key to an already full primary bucket. 321 * Used to test perf of lookup on keys that are on the shift path. 322 */ 323 tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) * 324 TOTAL_INSERT, 0); 325 if (tbl_rwc_test_param.keys_shift_path == NULL) { 326 printf("RTE_MALLOC failed\n"); 327 goto err; 328 } 329 330 /* 331 * This consist of keys which are never on the shift 332 * path (i.e. being moved to alternate location), when collision occurs 333 * on addition of a key to an already full primary bucket. 334 * Used to test perf of lookup on keys that are not on the shift path. 335 */ 336 keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 337 0); 338 if (keys_non_shift_path == NULL) { 339 printf("RTE_MALLOC failed\n"); 340 goto err; 341 } 342 343 /* 344 * This consist of keys which will be stored in extended buckets 345 */ 346 keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 347 if (keys_ext_bkt == NULL) { 348 printf("RTE_MALLOC failed\n"); 349 goto err; 350 } 351 352 /* 353 * This consist of keys which when deleted causes shifting of keys 354 * in extended buckets to respective secondary buckets 355 */ 356 keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 357 if (keys_ks_extbkt == NULL) { 358 printf("RTE_MALLOC failed\n"); 359 goto err; 360 } 361 362 hash_sig_t sig; 363 uint32_t prim_bucket_idx; 364 uint32_t sec_bucket_idx; 365 uint16_t short_sig; 366 uint32_t num_buckets; 367 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; 368 int ret; 369 370 /* 371 * Used to mark bkts in which at least one key was shifted to its 372 * alternate location 373 */ 374 scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0); 375 if (scanned_bkts == NULL) { 376 printf("RTE_MALLOC failed\n"); 377 goto err; 378 } 379 380 tbl_rwc_test_param.keys = keys; 381 tbl_rwc_test_param.keys_no_ks = keys_no_ks; 382 tbl_rwc_test_param.keys_ks = keys_ks; 383 tbl_rwc_test_param.keys_absent = keys_absent; 384 tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path; 385 tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt; 386 tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt; 387 /* Generate keys by adding previous two keys, neglect overflow */ 388 printf("Generating keys...\n"); 389 keys[0] = 0; 390 keys[1] = 1; 391 for (i = 2; i < TOTAL_INSERT; i++) 392 keys[i] = keys[i-1] + keys[i-2]; 393 394 /* Segregate keys into keys_no_ks and keys_ks */ 395 for (i = 0; i < TOTAL_INSERT; i++) { 396 /* Check if primary bucket has space.*/ 397 sig = rte_hash_hash(tbl_rwc_test_param.h, 398 tbl_rwc_test_param.keys+i); 399 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h, 400 sig); 401 ret = check_bucket(prim_bucket_idx, keys[i]); 402 if (ret < 0) { 403 /* 404 * Primary bucket is full, this key will result in 405 * shifting of the keys to their alternate locations. 406 */ 407 keys_ks[count_keys_ks] = keys[i]; 408 count_keys_ks++; 409 } else if (ret == 0) { 410 /* 411 * Primary bucket has space, this key will not result in 412 * shifting of the keys. Hence, add key to the table. 413 */ 414 ret = rte_hash_add_key_data(tbl_rwc_test_param.h, 415 keys+i, 416 (void *)((uintptr_t)i)); 417 if (ret < 0) { 418 printf("writer failed %"PRIu32"\n", i); 419 break; 420 } 421 keys_no_ks[count_keys_no_ks] = keys[i]; 422 count_keys_no_ks++; 423 } 424 } 425 426 for (i = 0; i < count_keys_no_ks; i++) { 427 /* 428 * Identify keys in keys_no_ks with value less than 429 * 4M (HTM enabled) OR 5K (HTM disabled) 430 */ 431 if (keys_no_ks[i] < TOTAL_INSERT) 432 found[keys_no_ks[i]]++; 433 } 434 435 for (i = 0; i < count_keys_ks; i++) { 436 /* 437 * Identify keys in keys_ks with value less than 438 * 4M (HTM enabled) OR 5K (HTM disabled) 439 */ 440 if (keys_ks[i] < TOTAL_INSERT) 441 found[keys_ks[i]]++; 442 } 443 444 uint32_t count_keys_absent = 0; 445 for (i = 0; i < TOTAL_INSERT; i++) { 446 /* 447 * Identify missing keys between 0 and 448 * 4M (HTM enabled) OR 5K (HTM disabled) 449 */ 450 if (found[i] == 0) 451 keys_absent[count_keys_absent++] = i; 452 } 453 454 /* Find keys that will not be on the shift path */ 455 uint32_t iter; 456 const void *next_key; 457 void *next_data; 458 uint32_t count = 0; 459 for (i = 0; i < num_buckets; i++) { 460 /* Check bucket for no keys shifted to alternate locations */ 461 if (scanned_bkts[i] == 0) { 462 iter = i * 8; 463 while (rte_hash_iterate(tbl_rwc_test_param.h, 464 &next_key, &next_data, &iter) >= 0) { 465 466 /* Check if key belongs to the current bucket */ 467 if (i >= (iter-1)/8) 468 keys_non_shift_path[count++] 469 = *(const uint32_t *)next_key; 470 else 471 break; 472 } 473 } 474 } 475 476 tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks; 477 tbl_rwc_test_param.count_keys_ks = count_keys_ks; 478 tbl_rwc_test_param.count_keys_absent = count_keys_absent; 479 tbl_rwc_test_param.count_keys_non_shift_path = count; 480 481 memset(scanned_bkts, 0, num_buckets); 482 count = 0; 483 /* Find keys that will be in extended buckets */ 484 for (i = 0; i < count_keys_ks; i++) { 485 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i); 486 if (ret < 0) { 487 /* Key will be added to ext bkt */ 488 keys_ext_bkt[count_keys_extbkt++] = keys_ks[i]; 489 /* Sec bkt to be added to keys_ks_extbkt */ 490 sig = rte_hash_hash(tbl_rwc_test_param.h, 491 tbl_rwc_test_param.keys_ks + i); 492 prim_bucket_idx = get_prim_bucket_index( 493 tbl_rwc_test_param.h, sig); 494 short_sig = get_short_sig(sig); 495 sec_bucket_idx = get_alt_bucket_index( 496 tbl_rwc_test_param.h, 497 prim_bucket_idx, short_sig); 498 if (scanned_bkts[sec_bucket_idx] == 0) 499 scanned_bkts[sec_bucket_idx] = 1; 500 } 501 } 502 503 /* Find keys that will shift keys in ext bucket*/ 504 for (i = 0; i < num_buckets; i++) { 505 if (scanned_bkts[i] == 1) { 506 iter = i * 8; 507 while (rte_hash_iterate(tbl_rwc_test_param.h, 508 &next_key, &next_data, &iter) >= 0) { 509 /* Check if key belongs to the current bucket */ 510 if (i >= (iter-1)/8) 511 keys_ks_extbkt[count++] 512 = *(const uint32_t *)next_key; 513 else 514 break; 515 } 516 } 517 } 518 519 tbl_rwc_test_param.count_keys_ks_extbkt = count; 520 tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt; 521 522 printf("\nCount of keys NOT causing shifting of existing keys to " 523 "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks); 524 printf("\nCount of keys causing shifting of existing keys to alternate " 525 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks); 526 printf("Count of absent keys that will never be added to the hash " 527 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent); 528 printf("Count of keys likely to be on the shift path: %d\n\n", 529 tbl_rwc_test_param.count_keys_shift_path); 530 printf("Count of keys not likely to be on the shift path: %d\n\n", 531 tbl_rwc_test_param.count_keys_non_shift_path); 532 printf("Count of keys in extended buckets: %d\n\n", 533 tbl_rwc_test_param.count_keys_extbkt); 534 printf("Count of keys shifting keys in ext buckets: %d\n\n", 535 tbl_rwc_test_param.count_keys_ks_extbkt); 536 537 rte_free(found); 538 rte_free(scanned_bkts); 539 rte_hash_free(tbl_rwc_test_param.h); 540 return 0; 541 542 err: 543 rte_free(keys); 544 rte_free(keys_no_ks); 545 rte_free(keys_ks); 546 rte_free(keys_absent); 547 rte_free(found); 548 rte_free(tbl_rwc_test_param.keys_shift_path); 549 rte_free(keys_non_shift_path); 550 rte_free(keys_ext_bkt); 551 rte_free(keys_ks_extbkt); 552 rte_free(scanned_bkts); 553 rte_hash_free(tbl_rwc_test_param.h); 554 return -1; 555 } 556 557 static int 558 test_rwc_reader(__attribute__((unused)) void *arg) 559 { 560 uint32_t i, j; 561 int ret; 562 uint64_t begin, cycles; 563 uint32_t loop_cnt = 0; 564 uint8_t read_type = (uint8_t)((uintptr_t)arg); 565 uint32_t read_cnt; 566 uint32_t *keys; 567 uint32_t extra_keys; 568 int32_t pos[BULK_LOOKUP_SIZE]; 569 void *temp_a[BULK_LOOKUP_SIZE]; 570 571 if (read_type & READ_FAIL) { 572 keys = tbl_rwc_test_param.keys_absent; 573 read_cnt = tbl_rwc_test_param.count_keys_absent; 574 } else if (read_type & READ_PASS_NO_KEY_SHIFTS) { 575 keys = tbl_rwc_test_param.keys_no_ks; 576 read_cnt = tbl_rwc_test_param.count_keys_no_ks; 577 } else if (read_type & READ_PASS_SHIFT_PATH) { 578 keys = tbl_rwc_test_param.keys_shift_path; 579 read_cnt = tbl_rwc_test_param.count_keys_shift_path; 580 } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) { 581 keys = tbl_rwc_test_param.keys_ext_bkt; 582 read_cnt = tbl_rwc_test_param.count_keys_extbkt; 583 } else { 584 keys = tbl_rwc_test_param.keys_non_shift_path; 585 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path; 586 } 587 588 extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1); 589 590 begin = rte_rdtsc_precise(); 591 do { 592 if (read_type & BULK_LOOKUP) { 593 for (i = 0; i < (read_cnt - extra_keys); 594 i += BULK_LOOKUP_SIZE) { 595 /* Array of pointer to the list of keys */ 596 for (j = 0; j < BULK_LOOKUP_SIZE; j++) 597 temp_a[j] = keys + i + j; 598 rte_hash_lookup_bulk(tbl_rwc_test_param.h, 599 (const void **) 600 ((uintptr_t)temp_a), 601 BULK_LOOKUP_SIZE, pos); 602 /* Validate lookup result */ 603 for (j = 0; j < BULK_LOOKUP_SIZE; j++) 604 if ((read_type & READ_FAIL && 605 pos[j] != -ENOENT) || 606 (!(read_type & READ_FAIL) && 607 pos[j] == -ENOENT)) { 608 printf("lookup failed!" 609 "%"PRIu32"\n", 610 keys[i + j]); 611 return -1; 612 } 613 } 614 for (j = 0; j < extra_keys; j++) 615 temp_a[j] = keys + i + j; 616 617 rte_hash_lookup_bulk(tbl_rwc_test_param.h, 618 (const void **) 619 ((uintptr_t)temp_a), 620 extra_keys, pos); 621 for (j = 0; j < extra_keys; j++) 622 if ((read_type & READ_FAIL && 623 pos[j] != -ENOENT) || 624 (!(read_type & READ_FAIL) && 625 pos[j] == -ENOENT)) { 626 printf("lookup failed! %"PRIu32"\n", 627 keys[i + j]); 628 return -1; 629 } 630 } else { 631 for (i = 0; i < read_cnt; i++) { 632 ret = rte_hash_lookup 633 (tbl_rwc_test_param.h, keys + i); 634 if (((read_type & READ_FAIL) && 635 (ret != -ENOENT)) || 636 (!(read_type & READ_FAIL) && 637 ret == -ENOENT)) { 638 printf("lookup failed! %"PRIu32"\n", 639 keys[i]); 640 return -1; 641 } 642 } 643 } 644 loop_cnt++; 645 } while (!writer_done); 646 647 cycles = rte_rdtsc_precise() - begin; 648 rte_atomic64_add(&gread_cycles, cycles); 649 rte_atomic64_add(&greads, read_cnt*loop_cnt); 650 return 0; 651 } 652 653 static int 654 write_keys(uint8_t write_type) 655 { 656 uint32_t i; 657 int ret; 658 uint32_t key_cnt = 0; 659 uint32_t *keys; 660 if (write_type == WRITE_KEY_SHIFT) { 661 key_cnt = tbl_rwc_test_param.count_keys_ks; 662 keys = tbl_rwc_test_param.keys_ks; 663 } else if (write_type == WRITE_NO_KEY_SHIFT) { 664 key_cnt = tbl_rwc_test_param.count_keys_no_ks; 665 keys = tbl_rwc_test_param.keys_no_ks; 666 } else if (write_type == WRITE_EXT_BKT) { 667 key_cnt = tbl_rwc_test_param.count_keys_extbkt; 668 keys = tbl_rwc_test_param.keys_ext_bkt; 669 } 670 for (i = 0; i < key_cnt; i++) { 671 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i); 672 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) { 673 printf("writer failed %"PRIu32"\n", i); 674 return -1; 675 } 676 } 677 return 0; 678 } 679 680 static int 681 test_rwc_multi_writer(__attribute__((unused)) void *arg) 682 { 683 uint32_t i, offset; 684 uint32_t pos_core = (uint32_t)((uintptr_t)arg); 685 offset = pos_core * tbl_rwc_test_param.single_insert; 686 for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++) 687 rte_hash_add_key(tbl_rwc_test_param.h, 688 tbl_rwc_test_param.keys_ks + i); 689 return 0; 690 } 691 692 /* 693 * Test lookup perf: 694 * Reader(s) lookup keys present in the table. 695 */ 696 static int 697 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf, 698 int htm, int ext_bkt) 699 { 700 unsigned int n, m; 701 uint64_t i; 702 int use_jhash = 0; 703 uint8_t write_type = WRITE_NO_KEY_SHIFT; 704 uint8_t read_type = READ_PASS_NO_KEY_SHIFTS; 705 706 rte_atomic64_init(&greads); 707 rte_atomic64_init(&gread_cycles); 708 709 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 710 goto err; 711 printf("\nTest: Hash add - no key-shifts, read - hit\n"); 712 for (m = 0; m < 2; m++) { 713 if (m == 1) { 714 printf("\n** With bulk-lookup **\n"); 715 read_type |= BULK_LOOKUP; 716 } 717 for (n = 0; n < NUM_TEST; n++) { 718 unsigned int tot_lcore = rte_lcore_count(); 719 if (tot_lcore < rwc_core_cnt[n] + 1) 720 goto finish; 721 722 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 723 724 rte_atomic64_clear(&greads); 725 rte_atomic64_clear(&gread_cycles); 726 727 rte_hash_reset(tbl_rwc_test_param.h); 728 writer_done = 0; 729 if (write_keys(write_type) < 0) 730 goto err; 731 writer_done = 1; 732 for (i = 1; i <= rwc_core_cnt[n]; i++) 733 rte_eal_remote_launch(test_rwc_reader, 734 (void *)(uintptr_t)read_type, 735 enabled_core_ids[i]); 736 737 for (i = 1; i <= rwc_core_cnt[n]; i++) 738 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 739 goto err; 740 741 unsigned long long cycles_per_lookup = 742 rte_atomic64_read(&gread_cycles) / 743 rte_atomic64_read(&greads); 744 rwc_perf_results->w_no_ks_r_hit[m][n] 745 = cycles_per_lookup; 746 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 747 } 748 } 749 750 finish: 751 rte_hash_free(tbl_rwc_test_param.h); 752 return 0; 753 754 err: 755 rte_eal_mp_wait_lcore(); 756 rte_hash_free(tbl_rwc_test_param.h); 757 return -1; 758 } 759 760 /* 761 * Test lookup perf: 762 * Reader(s) lookup keys absent in the table while 763 * 'Main' thread adds with no key-shifts. 764 */ 765 static int 766 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, 767 int htm, int ext_bkt) 768 { 769 unsigned int n, m; 770 uint64_t i; 771 int use_jhash = 0; 772 uint8_t write_type = WRITE_NO_KEY_SHIFT; 773 uint8_t read_type = READ_FAIL; 774 int ret; 775 776 rte_atomic64_init(&greads); 777 rte_atomic64_init(&gread_cycles); 778 779 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 780 goto err; 781 printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n"); 782 for (m = 0; m < 2; m++) { 783 if (m == 1) { 784 printf("\n** With bulk-lookup **\n"); 785 read_type |= BULK_LOOKUP; 786 } 787 for (n = 0; n < NUM_TEST; n++) { 788 unsigned int tot_lcore = rte_lcore_count(); 789 if (tot_lcore < rwc_core_cnt[n] + 1) 790 goto finish; 791 792 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 793 794 rte_atomic64_clear(&greads); 795 rte_atomic64_clear(&gread_cycles); 796 797 rte_hash_reset(tbl_rwc_test_param.h); 798 writer_done = 0; 799 800 for (i = 1; i <= rwc_core_cnt[n]; i++) 801 rte_eal_remote_launch(test_rwc_reader, 802 (void *)(uintptr_t)read_type, 803 enabled_core_ids[i]); 804 ret = write_keys(write_type); 805 writer_done = 1; 806 807 if (ret < 0) 808 goto err; 809 for (i = 1; i <= rwc_core_cnt[n]; i++) 810 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 811 goto err; 812 813 unsigned long long cycles_per_lookup = 814 rte_atomic64_read(&gread_cycles) / 815 rte_atomic64_read(&greads); 816 rwc_perf_results->w_no_ks_r_miss[m][n] 817 = cycles_per_lookup; 818 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 819 } 820 } 821 822 finish: 823 rte_hash_free(tbl_rwc_test_param.h); 824 return 0; 825 826 err: 827 rte_eal_mp_wait_lcore(); 828 rte_hash_free(tbl_rwc_test_param.h); 829 return -1; 830 } 831 832 /* 833 * Test lookup perf: 834 * Reader(s) lookup keys present in the table and not likely to be on the 835 * shift path while 'Main' thread adds keys causing key-shifts. 836 */ 837 static int 838 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results, 839 int rwc_lf, int htm, int ext_bkt) 840 { 841 unsigned int n, m; 842 uint64_t i; 843 int use_jhash = 0; 844 int ret; 845 uint8_t write_type; 846 uint8_t read_type = READ_PASS_NON_SHIFT_PATH; 847 848 rte_atomic64_init(&greads); 849 rte_atomic64_init(&gread_cycles); 850 851 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 852 goto err; 853 printf("\nTest: Hash add - key shift, Hash lookup - hit" 854 " (non-shift-path)\n"); 855 for (m = 0; m < 2; m++) { 856 if (m == 1) { 857 printf("\n** With bulk-lookup **\n"); 858 read_type |= BULK_LOOKUP; 859 } 860 for (n = 0; n < NUM_TEST; n++) { 861 unsigned int tot_lcore = rte_lcore_count(); 862 if (tot_lcore < rwc_core_cnt[n] + 1) 863 goto finish; 864 865 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 866 867 rte_atomic64_clear(&greads); 868 rte_atomic64_clear(&gread_cycles); 869 870 rte_hash_reset(tbl_rwc_test_param.h); 871 writer_done = 0; 872 write_type = WRITE_NO_KEY_SHIFT; 873 if (write_keys(write_type) < 0) 874 goto err; 875 for (i = 1; i <= rwc_core_cnt[n]; i++) 876 rte_eal_remote_launch(test_rwc_reader, 877 (void *)(uintptr_t)read_type, 878 enabled_core_ids[i]); 879 write_type = WRITE_KEY_SHIFT; 880 ret = write_keys(write_type); 881 writer_done = 1; 882 883 if (ret < 0) 884 goto err; 885 for (i = 1; i <= rwc_core_cnt[n]; i++) 886 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 887 goto err; 888 889 unsigned long long cycles_per_lookup = 890 rte_atomic64_read(&gread_cycles) / 891 rte_atomic64_read(&greads); 892 rwc_perf_results->w_ks_r_hit_nsp[m][n] 893 = cycles_per_lookup; 894 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 895 } 896 } 897 898 finish: 899 rte_hash_free(tbl_rwc_test_param.h); 900 return 0; 901 902 err: 903 rte_eal_mp_wait_lcore(); 904 rte_hash_free(tbl_rwc_test_param.h); 905 return -1; 906 } 907 908 /* 909 * Test lookup perf: 910 * Reader(s) lookup keys present in the table and likely on the shift-path while 911 * 'Main' thread adds keys causing key-shifts. 912 */ 913 static int 914 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf, 915 int htm, int ext_bkt) 916 { 917 unsigned int n, m; 918 uint64_t i; 919 int use_jhash = 0; 920 int ret; 921 uint8_t write_type; 922 uint8_t read_type = READ_PASS_SHIFT_PATH; 923 924 rte_atomic64_init(&greads); 925 rte_atomic64_init(&gread_cycles); 926 927 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 928 goto err; 929 printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)" 930 "\n"); 931 932 for (m = 0; m < 2; m++) { 933 if (m == 1) { 934 printf("\n** With bulk-lookup **\n"); 935 read_type |= BULK_LOOKUP; 936 } 937 for (n = 0; n < NUM_TEST; n++) { 938 unsigned int tot_lcore = rte_lcore_count(); 939 if (tot_lcore < rwc_core_cnt[n] + 1) 940 goto finish; 941 942 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 943 rte_atomic64_clear(&greads); 944 rte_atomic64_clear(&gread_cycles); 945 946 rte_hash_reset(tbl_rwc_test_param.h); 947 writer_done = 0; 948 write_type = WRITE_NO_KEY_SHIFT; 949 if (write_keys(write_type) < 0) 950 goto err; 951 for (i = 1; i <= rwc_core_cnt[n]; i++) 952 rte_eal_remote_launch(test_rwc_reader, 953 (void *)(uintptr_t)read_type, 954 enabled_core_ids[i]); 955 write_type = WRITE_KEY_SHIFT; 956 ret = write_keys(write_type); 957 writer_done = 1; 958 959 if (ret < 0) 960 goto err; 961 for (i = 1; i <= rwc_core_cnt[n]; i++) 962 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 963 goto err; 964 965 unsigned long long cycles_per_lookup = 966 rte_atomic64_read(&gread_cycles) / 967 rte_atomic64_read(&greads); 968 rwc_perf_results->w_ks_r_hit_sp[m][n] 969 = cycles_per_lookup; 970 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 971 } 972 } 973 974 finish: 975 rte_hash_free(tbl_rwc_test_param.h); 976 return 0; 977 978 err: 979 rte_eal_mp_wait_lcore(); 980 rte_hash_free(tbl_rwc_test_param.h); 981 return -1; 982 } 983 984 /* 985 * Test lookup perf: 986 * Reader(s) lookup keys absent in the table while 987 * 'Main' thread adds keys causing key-shifts. 988 */ 989 static int 990 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int 991 htm, int ext_bkt) 992 { 993 unsigned int n, m; 994 uint64_t i; 995 int use_jhash = 0; 996 int ret; 997 uint8_t write_type; 998 uint8_t read_type = READ_FAIL; 999 1000 rte_atomic64_init(&greads); 1001 rte_atomic64_init(&gread_cycles); 1002 1003 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 1004 goto err; 1005 printf("\nTest: Hash add - key shift, Hash lookup - miss\n"); 1006 for (m = 0; m < 2; m++) { 1007 if (m == 1) { 1008 printf("\n** With bulk-lookup **\n"); 1009 read_type |= BULK_LOOKUP; 1010 } 1011 for (n = 0; n < NUM_TEST; n++) { 1012 unsigned int tot_lcore = rte_lcore_count(); 1013 if (tot_lcore < rwc_core_cnt[n] + 1) 1014 goto finish; 1015 1016 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 1017 1018 rte_atomic64_clear(&greads); 1019 rte_atomic64_clear(&gread_cycles); 1020 1021 rte_hash_reset(tbl_rwc_test_param.h); 1022 writer_done = 0; 1023 write_type = WRITE_NO_KEY_SHIFT; 1024 if (write_keys(write_type) < 0) 1025 goto err; 1026 for (i = 1; i <= rwc_core_cnt[n]; i++) 1027 rte_eal_remote_launch(test_rwc_reader, 1028 (void *)(uintptr_t)read_type, 1029 enabled_core_ids[i]); 1030 write_type = WRITE_KEY_SHIFT; 1031 ret = write_keys(write_type); 1032 writer_done = 1; 1033 1034 if (ret < 0) 1035 goto err; 1036 for (i = 1; i <= rwc_core_cnt[n]; i++) 1037 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 1038 goto err; 1039 1040 unsigned long long cycles_per_lookup = 1041 rte_atomic64_read(&gread_cycles) / 1042 rte_atomic64_read(&greads); 1043 rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup; 1044 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 1045 } 1046 } 1047 1048 finish: 1049 rte_hash_free(tbl_rwc_test_param.h); 1050 return 0; 1051 1052 err: 1053 rte_eal_mp_wait_lcore(); 1054 rte_hash_free(tbl_rwc_test_param.h); 1055 return -1; 1056 } 1057 1058 /* 1059 * Test lookup perf for multi-writer: 1060 * Reader(s) lookup keys present in the table and likely on the shift-path while 1061 * Writers add keys causing key-shiftsi. 1062 * Writers are running in parallel, on different data plane cores. 1063 */ 1064 static int 1065 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf, 1066 int htm, int ext_bkt) 1067 { 1068 unsigned int n, m, k; 1069 uint64_t i; 1070 int use_jhash = 0; 1071 uint8_t write_type; 1072 uint8_t read_type = READ_PASS_SHIFT_PATH; 1073 1074 rte_atomic64_init(&greads); 1075 rte_atomic64_init(&gread_cycles); 1076 1077 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 1078 goto err; 1079 printf("\nTest: Multi-add-lookup\n"); 1080 uint8_t pos_core; 1081 for (m = 1; m < NUM_TEST; m++) { 1082 /* Calculate keys added by each writer */ 1083 tbl_rwc_test_param.single_insert = 1084 tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m]; 1085 for (k = 0; k < 2; k++) { 1086 if (k == 1) { 1087 printf("\n** With bulk-lookup **\n"); 1088 read_type |= BULK_LOOKUP; 1089 } 1090 for (n = 0; n < NUM_TEST; n++) { 1091 unsigned int tot_lcore = rte_lcore_count(); 1092 if (tot_lcore < (rwc_core_cnt[n] + 1093 rwc_core_cnt[m] + 1)) 1094 goto finish; 1095 1096 printf("\nNumber of writers: %u", 1097 rwc_core_cnt[m]); 1098 printf("\nNumber of readers: %u\n", 1099 rwc_core_cnt[n]); 1100 1101 rte_atomic64_clear(&greads); 1102 rte_atomic64_clear(&gread_cycles); 1103 1104 rte_hash_reset(tbl_rwc_test_param.h); 1105 writer_done = 0; 1106 write_type = WRITE_NO_KEY_SHIFT; 1107 if (write_keys(write_type) < 0) 1108 goto err; 1109 1110 /* Launch reader(s) */ 1111 for (i = 1; i <= rwc_core_cnt[n]; i++) 1112 rte_eal_remote_launch(test_rwc_reader, 1113 (void *)(uintptr_t)read_type, 1114 enabled_core_ids[i]); 1115 write_type = WRITE_KEY_SHIFT; 1116 pos_core = 0; 1117 1118 /* Launch writers */ 1119 for (; i <= rwc_core_cnt[m] 1120 + rwc_core_cnt[n]; i++) { 1121 rte_eal_remote_launch 1122 (test_rwc_multi_writer, 1123 (void *)(uintptr_t)pos_core, 1124 enabled_core_ids[i]); 1125 pos_core++; 1126 } 1127 1128 /* Wait for writers to complete */ 1129 for (i = rwc_core_cnt[n] + 1; 1130 i <= rwc_core_cnt[m] + rwc_core_cnt[n]; 1131 i++) 1132 rte_eal_wait_lcore(enabled_core_ids[i]); 1133 1134 writer_done = 1; 1135 1136 for (i = 1; i <= rwc_core_cnt[n]; i++) 1137 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 1138 goto err; 1139 1140 unsigned long long cycles_per_lookup = 1141 rte_atomic64_read(&gread_cycles) 1142 / rte_atomic64_read(&greads); 1143 rwc_perf_results->multi_rw[m][k][n] 1144 = cycles_per_lookup; 1145 printf("Cycles per lookup: %llu\n", 1146 cycles_per_lookup); 1147 } 1148 } 1149 } 1150 1151 finish: 1152 rte_hash_free(tbl_rwc_test_param.h); 1153 return 0; 1154 1155 err: 1156 rte_eal_mp_wait_lcore(); 1157 rte_hash_free(tbl_rwc_test_param.h); 1158 return -1; 1159 } 1160 1161 /* 1162 * Test lookup perf: 1163 * Reader(s) lookup keys present in the extendable bkt. 1164 */ 1165 static int 1166 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results, 1167 int rwc_lf, int htm, int ext_bkt) 1168 { 1169 unsigned int n, m; 1170 uint64_t i; 1171 int use_jhash = 0; 1172 uint8_t write_type; 1173 uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT; 1174 1175 rte_atomic64_init(&greads); 1176 rte_atomic64_init(&gread_cycles); 1177 1178 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 1179 goto err; 1180 printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n"); 1181 for (m = 0; m < 2; m++) { 1182 if (m == 1) { 1183 printf("\n** With bulk-lookup **\n"); 1184 read_type |= BULK_LOOKUP; 1185 } 1186 for (n = 0; n < NUM_TEST; n++) { 1187 unsigned int tot_lcore = rte_lcore_count(); 1188 if (tot_lcore < rwc_core_cnt[n] + 1) 1189 goto finish; 1190 1191 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 1192 1193 rte_atomic64_clear(&greads); 1194 rte_atomic64_clear(&gread_cycles); 1195 1196 rte_hash_reset(tbl_rwc_test_param.h); 1197 write_type = WRITE_NO_KEY_SHIFT; 1198 if (write_keys(write_type) < 0) 1199 goto err; 1200 write_type = WRITE_KEY_SHIFT; 1201 if (write_keys(write_type) < 0) 1202 goto err; 1203 writer_done = 0; 1204 for (i = 1; i <= rwc_core_cnt[n]; i++) 1205 rte_eal_remote_launch(test_rwc_reader, 1206 (void *)(uintptr_t)read_type, 1207 enabled_core_ids[i]); 1208 for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt; 1209 i++) { 1210 if (rte_hash_del_key(tbl_rwc_test_param.h, 1211 tbl_rwc_test_param.keys_ks_extbkt + i) 1212 < 0) { 1213 printf("Delete Failed: %u\n", 1214 tbl_rwc_test_param.keys_ks_extbkt[i]); 1215 goto err; 1216 } 1217 } 1218 writer_done = 1; 1219 1220 for (i = 1; i <= rwc_core_cnt[n]; i++) 1221 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 1222 goto err; 1223 1224 unsigned long long cycles_per_lookup = 1225 rte_atomic64_read(&gread_cycles) / 1226 rte_atomic64_read(&greads); 1227 rwc_perf_results->w_ks_r_hit_extbkt[m][n] 1228 = cycles_per_lookup; 1229 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 1230 } 1231 } 1232 1233 finish: 1234 rte_hash_free(tbl_rwc_test_param.h); 1235 return 0; 1236 1237 err: 1238 rte_eal_mp_wait_lcore(); 1239 rte_hash_free(tbl_rwc_test_param.h); 1240 return -1; 1241 } 1242 1243 static int 1244 test_hash_readwrite_lf_perf_main(void) 1245 { 1246 /* 1247 * Variables used to choose different tests. 1248 * rwc_lf indicates if read-write concurrency lock-free support is 1249 * enabled. 1250 * htm indicates if Hardware transactional memory support is enabled. 1251 */ 1252 int rwc_lf = 0; 1253 int htm; 1254 int ext_bkt = 0; 1255 1256 if (rte_lcore_count() < 2) { 1257 printf("Not enough cores for hash_readwrite_lf_perf_autotest, expecting at least 2\n"); 1258 return TEST_SKIPPED; 1259 } 1260 1261 setlocale(LC_NUMERIC, ""); 1262 1263 /* Reset tbl_rwc_test_param to discard values from previous run */ 1264 memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param)); 1265 1266 if (rte_tm_supported()) 1267 htm = 1; 1268 else 1269 htm = 0; 1270 1271 if (generate_keys() != 0) 1272 return -1; 1273 if (get_enabled_cores_list() != 0) 1274 return -1; 1275 1276 if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) { 1277 rwc_lf = 1; 1278 ext_bkt = 1; 1279 printf("Test lookup with read-write concurrency lock free support" 1280 " enabled\n"); 1281 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf, 1282 htm, ext_bkt) < 0) 1283 return -1; 1284 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf, 1285 htm, ext_bkt) < 0) 1286 return -1; 1287 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf, 1288 htm, ext_bkt) < 0) 1289 return -1; 1290 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf, 1291 htm, ext_bkt) < 0) 1292 return -1; 1293 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm, 1294 ext_bkt) < 0) 1295 return -1; 1296 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm, 1297 ext_bkt) < 0) 1298 return -1; 1299 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf, 1300 htm, ext_bkt) < 0) 1301 return -1; 1302 } 1303 printf("\nTest lookup with read-write concurrency lock free support" 1304 " disabled\n"); 1305 rwc_lf = 0; 1306 if (!htm) { 1307 printf("With HTM Disabled\n"); 1308 if (!RUN_WITH_HTM_DISABLED) { 1309 printf("Enable RUN_WITH_HTM_DISABLED to test with" 1310 " lock-free disabled"); 1311 goto results; 1312 } 1313 } else 1314 printf("With HTM Enabled\n"); 1315 if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm, 1316 ext_bkt) < 0) 1317 return -1; 1318 if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm, 1319 ext_bkt) < 0) 1320 return -1; 1321 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf, 1322 htm, ext_bkt) < 0) 1323 return -1; 1324 if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm, 1325 ext_bkt) < 0) 1326 return -1; 1327 if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm, 1328 ext_bkt) < 0) 1329 return -1; 1330 if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm, 1331 ext_bkt) < 0) 1332 return -1; 1333 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf, 1334 htm, ext_bkt) < 0) 1335 return -1; 1336 results: 1337 printf("\n\t\t\t\t\t\t********** Results summary **********\n\n"); 1338 int i, j, k; 1339 for (j = 0; j < 2; j++) { 1340 if (j == 1) 1341 printf("\n\t\t\t\t\t#######********** Bulk Lookup " 1342 "**********#######\n\n"); 1343 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t" 1344 "\t\t\t\t_________________\n"); 1345 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t" 1346 "\t\t\tCycles per lookup\n"); 1347 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t" 1348 "\t\t\t_________________\n"); 1349 for (i = 0; i < NUM_TEST; i++) { 1350 printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]); 1351 printf("Enabled\t\t"); 1352 printf("N/A\t\t"); 1353 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t" 1354 "%u\n\t\t\t\t\t\t\t\t", 1355 rwc_lf_results.w_no_ks_r_hit[j][i]); 1356 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t" 1357 "%u\n\t\t\t\t\t\t\t\t", 1358 rwc_lf_results.w_no_ks_r_miss[j][i]); 1359 printf("Hash add - key-shifts, lookup - hit" 1360 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1361 rwc_lf_results.w_ks_r_hit_nsp[j][i]); 1362 printf("Hash add - key-shifts, lookup - hit " 1363 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1364 rwc_lf_results.w_ks_r_hit_sp[j][i]); 1365 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t" 1366 "%u\n\t\t\t\t\t\t\t\t", 1367 rwc_lf_results.w_ks_r_miss[j][i]); 1368 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t" 1369 "%u\n\n\t\t\t\t", 1370 rwc_lf_results.w_ks_r_hit_extbkt[j][i]); 1371 1372 printf("Disabled\t"); 1373 if (htm) 1374 printf("Enabled\t\t"); 1375 else 1376 printf("Disabled\t"); 1377 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t" 1378 "%u\n\t\t\t\t\t\t\t\t", 1379 rwc_non_lf_results.w_no_ks_r_hit[j][i]); 1380 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t" 1381 "%u\n\t\t\t\t\t\t\t\t", 1382 rwc_non_lf_results.w_no_ks_r_miss[j][i]); 1383 printf("Hash add - key-shifts, lookup - hit " 1384 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1385 rwc_non_lf_results.w_ks_r_hit_nsp[j][i]); 1386 printf("Hash add - key-shifts, lookup - hit " 1387 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1388 rwc_non_lf_results.w_ks_r_hit_sp[j][i]); 1389 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t" 1390 "%u\n\t\t\t\t\t\t\t\t", 1391 rwc_non_lf_results.w_ks_r_miss[j][i]); 1392 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t" 1393 "%u\n", 1394 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]); 1395 1396 printf("_______\t\t_______\t\t_________\t___\t\t" 1397 "_________\t\t\t\t\t\t_________________\n"); 1398 } 1399 1400 for (i = 1; i < NUM_TEST; i++) { 1401 for (k = 0; k < NUM_TEST; k++) { 1402 printf("%u", rwc_core_cnt[i]); 1403 printf("\t\t%u\t\t", rwc_core_cnt[k]); 1404 printf("Enabled\t\t"); 1405 printf("N/A\t\t"); 1406 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t" 1407 "\t\t", 1408 rwc_lf_results.multi_rw[i][j][k]); 1409 printf("Disabled\t"); 1410 if (htm) 1411 printf("Enabled\t\t"); 1412 else 1413 printf("Disabled\t"); 1414 printf("Multi-add-lookup\t\t\t\t\t\t%u\n", 1415 rwc_non_lf_results.multi_rw[i][j][k]); 1416 1417 printf("_______\t\t_______\t\t_________\t___" 1418 "\t\t_________\t\t\t\t\t\t" 1419 "_________________\n"); 1420 } 1421 } 1422 } 1423 rte_free(tbl_rwc_test_param.keys); 1424 rte_free(tbl_rwc_test_param.keys_no_ks); 1425 rte_free(tbl_rwc_test_param.keys_ks); 1426 rte_free(tbl_rwc_test_param.keys_absent); 1427 rte_free(tbl_rwc_test_param.keys_shift_path); 1428 rte_free(tbl_rwc_test_param.keys_non_shift_path); 1429 rte_free(tbl_rwc_test_param.keys_ext_bkt); 1430 rte_free(tbl_rwc_test_param.keys_ks_extbkt); 1431 return 0; 1432 } 1433 1434 REGISTER_TEST_COMMAND(hash_readwrite_lf_perf_autotest, 1435 test_hash_readwrite_lf_perf_main); 1436