1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Arm Limited 3 */ 4 5 #include <inttypes.h> 6 #include <locale.h> 7 8 #include <rte_cycles.h> 9 #include <rte_hash.h> 10 #include <rte_hash_crc.h> 11 #include <rte_jhash.h> 12 #include <rte_launch.h> 13 #include <rte_malloc.h> 14 #include <rte_random.h> 15 #include <rte_spinlock.h> 16 17 #include "test.h" 18 19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0 21 #endif 22 23 #define BULK_LOOKUP_SIZE 32 24 25 #define RUN_WITH_HTM_DISABLED 0 26 27 #if (RUN_WITH_HTM_DISABLED) 28 29 #define TOTAL_ENTRY (5*1024) 30 #define TOTAL_INSERT (5*1024) 31 32 #else 33 34 #define TOTAL_ENTRY (4*1024*1024) 35 #define TOTAL_INSERT (4*1024*1024) 36 37 #endif 38 39 #define READ_FAIL 1 40 #define READ_PASS_NO_KEY_SHIFTS 2 41 #define READ_PASS_SHIFT_PATH 4 42 #define READ_PASS_NON_SHIFT_PATH 8 43 #define BULK_LOOKUP 16 44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32 45 46 #define WRITE_NO_KEY_SHIFT 0 47 #define WRITE_KEY_SHIFT 1 48 #define WRITE_EXT_BKT 2 49 50 #define NUM_TEST 3 51 52 #define QSBR_REPORTING_INTERVAL 1024 53 54 static unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4}; 55 56 struct rwc_perf { 57 uint32_t w_no_ks_r_hit[2][NUM_TEST]; 58 uint32_t w_no_ks_r_miss[2][NUM_TEST]; 59 uint32_t w_ks_r_hit_nsp[2][NUM_TEST]; 60 uint32_t w_ks_r_hit_sp[2][NUM_TEST]; 61 uint32_t w_ks_r_miss[2][NUM_TEST]; 62 uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST]; 63 uint32_t w_ks_r_hit_extbkt[2][NUM_TEST]; 64 uint32_t writer_add_del[NUM_TEST]; 65 }; 66 67 static struct rwc_perf rwc_lf_results, rwc_non_lf_results; 68 69 static struct { 70 uint32_t *keys; 71 uint32_t *keys_no_ks; 72 uint32_t *keys_ks; 73 uint32_t *keys_absent; 74 uint32_t *keys_shift_path; 75 uint32_t *keys_non_shift_path; 76 uint32_t *keys_ext_bkt; 77 uint32_t *keys_ks_extbkt; 78 uint32_t count_keys_no_ks; 79 uint32_t count_keys_ks; 80 uint32_t count_keys_absent; 81 uint32_t count_keys_shift_path; 82 uint32_t count_keys_non_shift_path; 83 uint32_t count_keys_extbkt; 84 uint32_t count_keys_ks_extbkt; 85 uint32_t single_insert; 86 struct rte_hash *h; 87 } tbl_rwc_test_param; 88 89 static uint64_t gread_cycles; 90 static uint64_t greads; 91 static uint64_t gwrite_cycles; 92 static uint64_t gwrites; 93 94 static volatile uint8_t writer_done; 95 96 static uint16_t enabled_core_ids[RTE_MAX_LCORE]; 97 98 static uint8_t *scanned_bkts; 99 100 static inline uint16_t 101 get_short_sig(const hash_sig_t hash) 102 { 103 return hash >> 16; 104 } 105 106 static inline uint32_t 107 get_prim_bucket_index(__rte_unused const struct rte_hash *h, 108 const hash_sig_t hash) 109 { 110 uint32_t num_buckets; 111 uint32_t bucket_bitmask; 112 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; 113 bucket_bitmask = num_buckets - 1; 114 return hash & bucket_bitmask; 115 } 116 117 static inline uint32_t 118 get_alt_bucket_index(__rte_unused const struct rte_hash *h, 119 uint32_t cur_bkt_idx, uint16_t sig) 120 { 121 uint32_t num_buckets; 122 uint32_t bucket_bitmask; 123 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; 124 bucket_bitmask = num_buckets - 1; 125 return (cur_bkt_idx ^ sig) & bucket_bitmask; 126 } 127 128 129 static inline int 130 get_enabled_cores_list(void) 131 { 132 uint32_t i = 0; 133 uint16_t core_id; 134 uint32_t max_cores = rte_lcore_count(); 135 RTE_LCORE_FOREACH(core_id) { 136 enabled_core_ids[i] = core_id; 137 i++; 138 } 139 140 if (i != max_cores) { 141 printf("Number of enabled cores in list is different from " 142 "number given by rte_lcore_count()\n"); 143 return -1; 144 } 145 return 0; 146 } 147 148 static int 149 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt) 150 { 151 struct rte_hash *handle; 152 153 struct rte_hash_parameters hash_params = { 154 .entries = TOTAL_ENTRY, 155 .key_len = sizeof(uint32_t), 156 .hash_func_init_val = 0, 157 .socket_id = rte_socket_id(), 158 }; 159 160 if (use_jhash) 161 hash_params.hash_func = rte_jhash; 162 else 163 hash_params.hash_func = rte_hash_crc; 164 165 if (rwc_lf) 166 hash_params.extra_flag = 167 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF | 168 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 169 else if (htm) 170 hash_params.extra_flag = 171 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT | 172 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | 173 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 174 else 175 hash_params.extra_flag = 176 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | 177 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 178 179 if (ext_bkt) 180 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE; 181 182 hash_params.name = "tests"; 183 184 handle = rte_hash_create(&hash_params); 185 if (handle == NULL) { 186 printf("hash creation failed"); 187 return -1; 188 } 189 190 tbl_rwc_test_param.h = handle; 191 return 0; 192 } 193 194 static inline int 195 check_bucket(uint32_t bkt_idx, uint32_t key) 196 { 197 uint32_t iter; 198 uint32_t prev_iter; 199 uint32_t diff; 200 uint32_t count = 0; 201 const void *next_key; 202 void *next_data; 203 204 /* Temporary bucket to hold the keys */ 205 uint32_t keys_in_bkt[8]; 206 207 iter = bkt_idx * 8; 208 prev_iter = iter; 209 while (rte_hash_iterate(tbl_rwc_test_param.h, 210 &next_key, &next_data, &iter) >= 0) { 211 212 /* Check for duplicate entries */ 213 if (*(const uint32_t *)next_key == key) 214 return 1; 215 216 /* Identify if there is any free entry in the bucket */ 217 diff = iter - prev_iter; 218 if (diff > 1) 219 break; 220 221 prev_iter = iter; 222 keys_in_bkt[count] = *(const uint32_t *)next_key; 223 count++; 224 225 /* All entries in the bucket are occupied */ 226 if (count == 8) { 227 228 /* 229 * Check if bucket was not scanned before, to avoid 230 * duplicate keys. 231 */ 232 if (scanned_bkts[bkt_idx] == 0) { 233 /* 234 * Since this bucket (pointed to by bkt_idx) is 235 * full, it is likely that key(s) in this 236 * bucket will be on the shift path, when 237 * collision occurs. Thus, add it to 238 * keys_shift_path. 239 */ 240 memcpy(tbl_rwc_test_param.keys_shift_path + 241 tbl_rwc_test_param.count_keys_shift_path 242 , keys_in_bkt, 32); 243 tbl_rwc_test_param.count_keys_shift_path += 8; 244 scanned_bkts[bkt_idx] = 1; 245 } 246 return -1; 247 } 248 } 249 return 0; 250 } 251 252 static int 253 generate_keys(void) 254 { 255 uint32_t *keys = NULL; 256 uint32_t *keys_no_ks = NULL; 257 uint32_t *keys_ks = NULL; 258 uint32_t *keys_absent = NULL; 259 uint32_t *keys_non_shift_path = NULL; 260 uint32_t *keys_ext_bkt = NULL; 261 uint32_t *keys_ks_extbkt = NULL; 262 uint32_t *found = NULL; 263 uint32_t count_keys_no_ks = 0; 264 uint32_t count_keys_ks = 0; 265 uint32_t count_keys_extbkt = 0; 266 uint32_t i; 267 268 if (init_params(0, 0, 0, 0) != 0) 269 return -1; 270 271 /* 272 * keys will consist of a) keys whose addition to the hash table 273 * will result in shifting of the existing keys to their alternate 274 * locations b) keys whose addition to the hash table will not result 275 * in shifting of the existing keys. 276 */ 277 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 278 if (keys == NULL) { 279 printf("RTE_MALLOC failed\n"); 280 goto err; 281 } 282 283 /* 284 * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys that 285 * will NOT result in shifting of the existing keys to their alternate 286 * locations. Roughly around 900K keys. 287 */ 288 keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 289 if (keys_no_ks == NULL) { 290 printf("RTE_MALLOC failed\n"); 291 goto err; 292 } 293 294 /* 295 * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will 296 * result in shifting of the existing keys to their alternate locations. 297 * Roughly around 146K keys. There might be repeating keys. More code is 298 * required to filter out these keys which will complicate the test case 299 */ 300 keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 301 if (keys_ks == NULL) { 302 printf("RTE_MALLOC failed\n"); 303 goto err; 304 } 305 306 /* Used to identify keys not inserted in the hash table */ 307 found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 308 if (found == NULL) { 309 printf("RTE_MALLOC failed\n"); 310 goto err; 311 } 312 313 /* 314 * This consist of keys not inserted to the hash table. 315 * Used to test perf of lookup on keys that do not exist in the table. 316 */ 317 keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 318 if (keys_absent == NULL) { 319 printf("RTE_MALLOC failed\n"); 320 goto err; 321 } 322 323 /* 324 * This consist of keys which are likely to be on the shift 325 * path (i.e. being moved to alternate location), when collision occurs 326 * on addition of a key to an already full primary bucket. 327 * Used to test perf of lookup on keys that are on the shift path. 328 */ 329 tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) * 330 TOTAL_INSERT, 0); 331 if (tbl_rwc_test_param.keys_shift_path == NULL) { 332 printf("RTE_MALLOC failed\n"); 333 goto err; 334 } 335 336 /* 337 * This consist of keys which are never on the shift 338 * path (i.e. being moved to alternate location), when collision occurs 339 * on addition of a key to an already full primary bucket. 340 * Used to test perf of lookup on keys that are not on the shift path. 341 */ 342 keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 343 0); 344 if (keys_non_shift_path == NULL) { 345 printf("RTE_MALLOC failed\n"); 346 goto err; 347 } 348 349 /* 350 * This consist of keys which will be stored in extended buckets 351 */ 352 keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 353 if (keys_ext_bkt == NULL) { 354 printf("RTE_MALLOC failed\n"); 355 goto err; 356 } 357 358 /* 359 * This consist of keys which when deleted causes shifting of keys 360 * in extended buckets to respective secondary buckets 361 */ 362 keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 363 if (keys_ks_extbkt == NULL) { 364 printf("RTE_MALLOC failed\n"); 365 goto err; 366 } 367 368 hash_sig_t sig; 369 uint32_t prim_bucket_idx; 370 uint32_t sec_bucket_idx; 371 uint16_t short_sig; 372 uint32_t num_buckets; 373 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; 374 int ret; 375 376 /* 377 * Used to mark bkts in which at least one key was shifted to its 378 * alternate location 379 */ 380 scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0); 381 if (scanned_bkts == NULL) { 382 printf("RTE_MALLOC failed\n"); 383 goto err; 384 } 385 386 tbl_rwc_test_param.keys = keys; 387 tbl_rwc_test_param.keys_no_ks = keys_no_ks; 388 tbl_rwc_test_param.keys_ks = keys_ks; 389 tbl_rwc_test_param.keys_absent = keys_absent; 390 tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path; 391 tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt; 392 tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt; 393 /* Generate keys by adding previous two keys, neglect overflow */ 394 printf("Generating keys...\n"); 395 keys[0] = 0; 396 keys[1] = 1; 397 for (i = 2; i < TOTAL_INSERT; i++) 398 keys[i] = keys[i-1] + keys[i-2]; 399 400 /* Segregate keys into keys_no_ks and keys_ks */ 401 for (i = 0; i < TOTAL_INSERT; i++) { 402 /* Check if primary bucket has space.*/ 403 sig = rte_hash_hash(tbl_rwc_test_param.h, 404 tbl_rwc_test_param.keys+i); 405 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h, 406 sig); 407 ret = check_bucket(prim_bucket_idx, keys[i]); 408 if (ret < 0) { 409 /* 410 * Primary bucket is full, this key will result in 411 * shifting of the keys to their alternate locations. 412 */ 413 keys_ks[count_keys_ks] = keys[i]; 414 count_keys_ks++; 415 } else if (ret == 0) { 416 /* 417 * Primary bucket has space, this key will not result in 418 * shifting of the keys. Hence, add key to the table. 419 */ 420 ret = rte_hash_add_key_data(tbl_rwc_test_param.h, 421 keys+i, 422 (void *)((uintptr_t)i)); 423 if (ret < 0) { 424 printf("writer failed %"PRIu32"\n", i); 425 break; 426 } 427 keys_no_ks[count_keys_no_ks] = keys[i]; 428 count_keys_no_ks++; 429 } 430 } 431 432 for (i = 0; i < count_keys_no_ks; i++) { 433 /* 434 * Identify keys in keys_no_ks with value less than 435 * 4M (HTM enabled) OR 5K (HTM disabled) 436 */ 437 if (keys_no_ks[i] < TOTAL_INSERT) 438 found[keys_no_ks[i]]++; 439 } 440 441 for (i = 0; i < count_keys_ks; i++) { 442 /* 443 * Identify keys in keys_ks with value less than 444 * 4M (HTM enabled) OR 5K (HTM disabled) 445 */ 446 if (keys_ks[i] < TOTAL_INSERT) 447 found[keys_ks[i]]++; 448 } 449 450 uint32_t count_keys_absent = 0; 451 for (i = 0; i < TOTAL_INSERT; i++) { 452 /* 453 * Identify missing keys between 0 and 454 * 4M (HTM enabled) OR 5K (HTM disabled) 455 */ 456 if (found[i] == 0) 457 keys_absent[count_keys_absent++] = i; 458 } 459 460 /* Find keys that will not be on the shift path */ 461 uint32_t iter; 462 const void *next_key; 463 void *next_data; 464 uint32_t count = 0; 465 for (i = 0; i < num_buckets; i++) { 466 /* Check bucket for no keys shifted to alternate locations */ 467 if (scanned_bkts[i] == 0) { 468 iter = i * 8; 469 while (rte_hash_iterate(tbl_rwc_test_param.h, 470 &next_key, &next_data, &iter) >= 0) { 471 472 /* Check if key belongs to the current bucket */ 473 if (i >= (iter-1)/8) 474 keys_non_shift_path[count++] 475 = *(const uint32_t *)next_key; 476 else 477 break; 478 } 479 } 480 } 481 482 tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks; 483 tbl_rwc_test_param.count_keys_ks = count_keys_ks; 484 tbl_rwc_test_param.count_keys_absent = count_keys_absent; 485 tbl_rwc_test_param.count_keys_non_shift_path = count; 486 487 memset(scanned_bkts, 0, num_buckets); 488 count = 0; 489 /* Find keys that will be in extended buckets */ 490 for (i = 0; i < count_keys_ks; i++) { 491 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i); 492 if (ret < 0) { 493 /* Key will be added to ext bkt */ 494 keys_ext_bkt[count_keys_extbkt++] = keys_ks[i]; 495 /* Sec bkt to be added to keys_ks_extbkt */ 496 sig = rte_hash_hash(tbl_rwc_test_param.h, 497 tbl_rwc_test_param.keys_ks + i); 498 prim_bucket_idx = get_prim_bucket_index( 499 tbl_rwc_test_param.h, sig); 500 short_sig = get_short_sig(sig); 501 sec_bucket_idx = get_alt_bucket_index( 502 tbl_rwc_test_param.h, 503 prim_bucket_idx, short_sig); 504 if (scanned_bkts[sec_bucket_idx] == 0) 505 scanned_bkts[sec_bucket_idx] = 1; 506 } 507 } 508 509 /* Find keys that will shift keys in ext bucket*/ 510 for (i = 0; i < num_buckets; i++) { 511 if (scanned_bkts[i] == 1) { 512 iter = i * 8; 513 while (rte_hash_iterate(tbl_rwc_test_param.h, 514 &next_key, &next_data, &iter) >= 0) { 515 /* Check if key belongs to the current bucket */ 516 if (i >= (iter-1)/8) 517 keys_ks_extbkt[count++] 518 = *(const uint32_t *)next_key; 519 else 520 break; 521 } 522 } 523 } 524 525 tbl_rwc_test_param.count_keys_ks_extbkt = count; 526 tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt; 527 528 printf("\nCount of keys NOT causing shifting of existing keys to " 529 "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks); 530 printf("\nCount of keys causing shifting of existing keys to alternate " 531 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks); 532 printf("Count of absent keys that will never be added to the hash " 533 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent); 534 printf("Count of keys likely to be on the shift path: %d\n\n", 535 tbl_rwc_test_param.count_keys_shift_path); 536 printf("Count of keys not likely to be on the shift path: %d\n\n", 537 tbl_rwc_test_param.count_keys_non_shift_path); 538 printf("Count of keys in extended buckets: %d\n\n", 539 tbl_rwc_test_param.count_keys_extbkt); 540 printf("Count of keys shifting keys in ext buckets: %d\n\n", 541 tbl_rwc_test_param.count_keys_ks_extbkt); 542 543 rte_free(found); 544 rte_free(scanned_bkts); 545 rte_hash_free(tbl_rwc_test_param.h); 546 return 0; 547 548 err: 549 rte_free(keys); 550 rte_free(keys_no_ks); 551 rte_free(keys_ks); 552 rte_free(keys_absent); 553 rte_free(found); 554 rte_free(tbl_rwc_test_param.keys_shift_path); 555 rte_free(keys_non_shift_path); 556 rte_free(keys_ext_bkt); 557 rte_free(keys_ks_extbkt); 558 rte_free(scanned_bkts); 559 rte_hash_free(tbl_rwc_test_param.h); 560 return -1; 561 } 562 563 static int 564 test_rwc_reader(__rte_unused void *arg) 565 { 566 uint32_t i, j; 567 int ret; 568 uint64_t begin, cycles; 569 uint32_t loop_cnt = 0; 570 uint8_t read_type = (uint8_t)((uintptr_t)arg); 571 uint32_t read_cnt; 572 uint32_t *keys; 573 uint32_t extra_keys; 574 int32_t pos[BULK_LOOKUP_SIZE]; 575 void *temp_a[BULK_LOOKUP_SIZE]; 576 577 if (read_type & READ_FAIL) { 578 keys = tbl_rwc_test_param.keys_absent; 579 read_cnt = tbl_rwc_test_param.count_keys_absent; 580 } else if (read_type & READ_PASS_NO_KEY_SHIFTS) { 581 keys = tbl_rwc_test_param.keys_no_ks; 582 read_cnt = tbl_rwc_test_param.count_keys_no_ks; 583 } else if (read_type & READ_PASS_SHIFT_PATH) { 584 keys = tbl_rwc_test_param.keys_shift_path; 585 read_cnt = tbl_rwc_test_param.count_keys_shift_path; 586 } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) { 587 keys = tbl_rwc_test_param.keys_ext_bkt; 588 read_cnt = tbl_rwc_test_param.count_keys_extbkt; 589 } else { 590 keys = tbl_rwc_test_param.keys_non_shift_path; 591 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path; 592 } 593 594 extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1); 595 596 begin = rte_rdtsc_precise(); 597 do { 598 if (read_type & BULK_LOOKUP) { 599 for (i = 0; i < (read_cnt - extra_keys); 600 i += BULK_LOOKUP_SIZE) { 601 /* Array of pointer to the list of keys */ 602 for (j = 0; j < BULK_LOOKUP_SIZE; j++) 603 temp_a[j] = keys + i + j; 604 rte_hash_lookup_bulk(tbl_rwc_test_param.h, 605 (const void **) 606 ((uintptr_t)temp_a), 607 BULK_LOOKUP_SIZE, pos); 608 /* Validate lookup result */ 609 for (j = 0; j < BULK_LOOKUP_SIZE; j++) 610 if ((read_type & READ_FAIL && 611 pos[j] != -ENOENT) || 612 (!(read_type & READ_FAIL) && 613 pos[j] == -ENOENT)) { 614 printf("lookup failed!" 615 "%"PRIu32"\n", 616 keys[i + j]); 617 return -1; 618 } 619 } 620 for (j = 0; j < extra_keys; j++) 621 temp_a[j] = keys + i + j; 622 623 rte_hash_lookup_bulk(tbl_rwc_test_param.h, 624 (const void **) 625 ((uintptr_t)temp_a), 626 extra_keys, pos); 627 for (j = 0; j < extra_keys; j++) 628 if ((read_type & READ_FAIL && 629 pos[j] != -ENOENT) || 630 (!(read_type & READ_FAIL) && 631 pos[j] == -ENOENT)) { 632 printf("lookup failed! %"PRIu32"\n", 633 keys[i + j]); 634 return -1; 635 } 636 } else { 637 for (i = 0; i < read_cnt; i++) { 638 ret = rte_hash_lookup 639 (tbl_rwc_test_param.h, keys + i); 640 if (((read_type & READ_FAIL) && 641 (ret != -ENOENT)) || 642 (!(read_type & READ_FAIL) && 643 ret == -ENOENT)) { 644 printf("lookup failed! %"PRIu32"\n", 645 keys[i]); 646 return -1; 647 } 648 } 649 } 650 loop_cnt++; 651 } while (!writer_done); 652 653 cycles = rte_rdtsc_precise() - begin; 654 __atomic_fetch_add(&gread_cycles, cycles, __ATOMIC_RELAXED); 655 __atomic_fetch_add(&greads, read_cnt*loop_cnt, __ATOMIC_RELAXED); 656 return 0; 657 } 658 659 static int 660 write_keys(uint8_t write_type) 661 { 662 uint32_t i; 663 int ret; 664 uint32_t key_cnt = 0; 665 uint32_t *keys; 666 if (write_type == WRITE_KEY_SHIFT) { 667 key_cnt = tbl_rwc_test_param.count_keys_ks; 668 keys = tbl_rwc_test_param.keys_ks; 669 } else if (write_type == WRITE_NO_KEY_SHIFT) { 670 key_cnt = tbl_rwc_test_param.count_keys_no_ks; 671 keys = tbl_rwc_test_param.keys_no_ks; 672 } else if (write_type == WRITE_EXT_BKT) { 673 key_cnt = tbl_rwc_test_param.count_keys_extbkt; 674 keys = tbl_rwc_test_param.keys_ext_bkt; 675 } 676 for (i = 0; i < key_cnt; i++) { 677 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i); 678 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) { 679 printf("writer failed %"PRIu32"\n", i); 680 return -1; 681 } 682 } 683 return 0; 684 } 685 686 static int 687 test_rwc_multi_writer(__rte_unused void *arg) 688 { 689 uint32_t i, offset; 690 uint32_t pos_core = (uint32_t)((uintptr_t)arg); 691 offset = pos_core * tbl_rwc_test_param.single_insert; 692 for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++) 693 rte_hash_add_key(tbl_rwc_test_param.h, 694 tbl_rwc_test_param.keys_ks + i); 695 return 0; 696 } 697 698 /* 699 * Test lookup perf: 700 * Reader(s) lookup keys present in the table. 701 */ 702 static int 703 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf, 704 int htm, int ext_bkt) 705 { 706 unsigned int n, m; 707 uint64_t i; 708 int use_jhash = 0; 709 uint8_t write_type = WRITE_NO_KEY_SHIFT; 710 uint8_t read_type = READ_PASS_NO_KEY_SHIFTS; 711 712 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 713 goto err; 714 printf("\nTest: Hash add - no key-shifts, read - hit\n"); 715 for (m = 0; m < 2; m++) { 716 if (m == 1) { 717 printf("\n** With bulk-lookup **\n"); 718 read_type |= BULK_LOOKUP; 719 } 720 for (n = 0; n < NUM_TEST; n++) { 721 unsigned int tot_lcore = rte_lcore_count(); 722 if (tot_lcore < rwc_core_cnt[n] + 1) 723 goto finish; 724 725 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 726 727 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED); 728 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED); 729 730 rte_hash_reset(tbl_rwc_test_param.h); 731 writer_done = 0; 732 if (write_keys(write_type) < 0) 733 goto err; 734 writer_done = 1; 735 for (i = 1; i <= rwc_core_cnt[n]; i++) 736 rte_eal_remote_launch(test_rwc_reader, 737 (void *)(uintptr_t)read_type, 738 enabled_core_ids[i]); 739 740 for (i = 1; i <= rwc_core_cnt[n]; i++) 741 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 742 goto err; 743 744 unsigned long long cycles_per_lookup = 745 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED) 746 / __atomic_load_n(&greads, __ATOMIC_RELAXED); 747 rwc_perf_results->w_no_ks_r_hit[m][n] 748 = cycles_per_lookup; 749 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 750 } 751 } 752 753 finish: 754 rte_hash_free(tbl_rwc_test_param.h); 755 return 0; 756 757 err: 758 rte_eal_mp_wait_lcore(); 759 rte_hash_free(tbl_rwc_test_param.h); 760 return -1; 761 } 762 763 /* 764 * Test lookup perf: 765 * Reader(s) lookup keys absent in the table while 766 * 'Main' thread adds with no key-shifts. 767 */ 768 static int 769 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, 770 int htm, int ext_bkt) 771 { 772 unsigned int n, m; 773 uint64_t i; 774 int use_jhash = 0; 775 uint8_t write_type = WRITE_NO_KEY_SHIFT; 776 uint8_t read_type = READ_FAIL; 777 int ret; 778 779 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 780 goto err; 781 printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n"); 782 for (m = 0; m < 2; m++) { 783 if (m == 1) { 784 printf("\n** With bulk-lookup **\n"); 785 read_type |= BULK_LOOKUP; 786 } 787 for (n = 0; n < NUM_TEST; n++) { 788 unsigned int tot_lcore = rte_lcore_count(); 789 if (tot_lcore < rwc_core_cnt[n] + 1) 790 goto finish; 791 792 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 793 794 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED); 795 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED); 796 797 rte_hash_reset(tbl_rwc_test_param.h); 798 writer_done = 0; 799 800 for (i = 1; i <= rwc_core_cnt[n]; i++) 801 rte_eal_remote_launch(test_rwc_reader, 802 (void *)(uintptr_t)read_type, 803 enabled_core_ids[i]); 804 ret = write_keys(write_type); 805 writer_done = 1; 806 807 if (ret < 0) 808 goto err; 809 for (i = 1; i <= rwc_core_cnt[n]; i++) 810 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 811 goto err; 812 813 unsigned long long cycles_per_lookup = 814 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED) 815 / __atomic_load_n(&greads, __ATOMIC_RELAXED); 816 rwc_perf_results->w_no_ks_r_miss[m][n] 817 = cycles_per_lookup; 818 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 819 } 820 } 821 822 finish: 823 rte_hash_free(tbl_rwc_test_param.h); 824 return 0; 825 826 err: 827 rte_eal_mp_wait_lcore(); 828 rte_hash_free(tbl_rwc_test_param.h); 829 return -1; 830 } 831 832 /* 833 * Test lookup perf: 834 * Reader(s) lookup keys present in the table and not likely to be on the 835 * shift path while 'Main' thread adds keys causing key-shifts. 836 */ 837 static int 838 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results, 839 int rwc_lf, int htm, int ext_bkt) 840 { 841 unsigned int n, m; 842 uint64_t i; 843 int use_jhash = 0; 844 int ret; 845 uint8_t write_type; 846 uint8_t read_type = READ_PASS_NON_SHIFT_PATH; 847 848 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 849 goto err; 850 printf("\nTest: Hash add - key shift, Hash lookup - hit" 851 " (non-shift-path)\n"); 852 for (m = 0; m < 2; m++) { 853 if (m == 1) { 854 printf("\n** With bulk-lookup **\n"); 855 read_type |= BULK_LOOKUP; 856 } 857 for (n = 0; n < NUM_TEST; n++) { 858 unsigned int tot_lcore = rte_lcore_count(); 859 if (tot_lcore < rwc_core_cnt[n] + 1) 860 goto finish; 861 862 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 863 864 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED); 865 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED); 866 867 rte_hash_reset(tbl_rwc_test_param.h); 868 writer_done = 0; 869 write_type = WRITE_NO_KEY_SHIFT; 870 if (write_keys(write_type) < 0) 871 goto err; 872 for (i = 1; i <= rwc_core_cnt[n]; i++) 873 rte_eal_remote_launch(test_rwc_reader, 874 (void *)(uintptr_t)read_type, 875 enabled_core_ids[i]); 876 write_type = WRITE_KEY_SHIFT; 877 ret = write_keys(write_type); 878 writer_done = 1; 879 880 if (ret < 0) 881 goto err; 882 for (i = 1; i <= rwc_core_cnt[n]; i++) 883 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 884 goto err; 885 886 unsigned long long cycles_per_lookup = 887 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED) 888 / __atomic_load_n(&greads, __ATOMIC_RELAXED); 889 rwc_perf_results->w_ks_r_hit_nsp[m][n] 890 = cycles_per_lookup; 891 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 892 } 893 } 894 895 finish: 896 rte_hash_free(tbl_rwc_test_param.h); 897 return 0; 898 899 err: 900 rte_eal_mp_wait_lcore(); 901 rte_hash_free(tbl_rwc_test_param.h); 902 return -1; 903 } 904 905 /* 906 * Test lookup perf: 907 * Reader(s) lookup keys present in the table and likely on the shift-path while 908 * 'Main' thread adds keys causing key-shifts. 909 */ 910 static int 911 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf, 912 int htm, int ext_bkt) 913 { 914 unsigned int n, m; 915 uint64_t i; 916 int use_jhash = 0; 917 int ret; 918 uint8_t write_type; 919 uint8_t read_type = READ_PASS_SHIFT_PATH; 920 921 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 922 goto err; 923 printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)" 924 "\n"); 925 926 for (m = 0; m < 2; m++) { 927 if (m == 1) { 928 printf("\n** With bulk-lookup **\n"); 929 read_type |= BULK_LOOKUP; 930 } 931 for (n = 0; n < NUM_TEST; n++) { 932 unsigned int tot_lcore = rte_lcore_count(); 933 if (tot_lcore < rwc_core_cnt[n] + 1) 934 goto finish; 935 936 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 937 938 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED); 939 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED); 940 941 rte_hash_reset(tbl_rwc_test_param.h); 942 writer_done = 0; 943 write_type = WRITE_NO_KEY_SHIFT; 944 if (write_keys(write_type) < 0) 945 goto err; 946 for (i = 1; i <= rwc_core_cnt[n]; i++) 947 rte_eal_remote_launch(test_rwc_reader, 948 (void *)(uintptr_t)read_type, 949 enabled_core_ids[i]); 950 write_type = WRITE_KEY_SHIFT; 951 ret = write_keys(write_type); 952 writer_done = 1; 953 954 if (ret < 0) 955 goto err; 956 for (i = 1; i <= rwc_core_cnt[n]; i++) 957 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 958 goto err; 959 960 unsigned long long cycles_per_lookup = 961 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED) 962 / __atomic_load_n(&greads, __ATOMIC_RELAXED); 963 rwc_perf_results->w_ks_r_hit_sp[m][n] 964 = cycles_per_lookup; 965 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 966 } 967 } 968 969 finish: 970 rte_hash_free(tbl_rwc_test_param.h); 971 return 0; 972 973 err: 974 rte_eal_mp_wait_lcore(); 975 rte_hash_free(tbl_rwc_test_param.h); 976 return -1; 977 } 978 979 /* 980 * Test lookup perf: 981 * Reader(s) lookup keys absent in the table while 982 * 'Main' thread adds keys causing key-shifts. 983 */ 984 static int 985 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int 986 htm, int ext_bkt) 987 { 988 unsigned int n, m; 989 uint64_t i; 990 int use_jhash = 0; 991 int ret; 992 uint8_t write_type; 993 uint8_t read_type = READ_FAIL; 994 995 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 996 goto err; 997 printf("\nTest: Hash add - key shift, Hash lookup - miss\n"); 998 for (m = 0; m < 2; m++) { 999 if (m == 1) { 1000 printf("\n** With bulk-lookup **\n"); 1001 read_type |= BULK_LOOKUP; 1002 } 1003 for (n = 0; n < NUM_TEST; n++) { 1004 unsigned int tot_lcore = rte_lcore_count(); 1005 if (tot_lcore < rwc_core_cnt[n] + 1) 1006 goto finish; 1007 1008 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 1009 1010 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED); 1011 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED); 1012 1013 rte_hash_reset(tbl_rwc_test_param.h); 1014 writer_done = 0; 1015 write_type = WRITE_NO_KEY_SHIFT; 1016 if (write_keys(write_type) < 0) 1017 goto err; 1018 for (i = 1; i <= rwc_core_cnt[n]; i++) 1019 rte_eal_remote_launch(test_rwc_reader, 1020 (void *)(uintptr_t)read_type, 1021 enabled_core_ids[i]); 1022 write_type = WRITE_KEY_SHIFT; 1023 ret = write_keys(write_type); 1024 writer_done = 1; 1025 1026 if (ret < 0) 1027 goto err; 1028 for (i = 1; i <= rwc_core_cnt[n]; i++) 1029 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 1030 goto err; 1031 1032 unsigned long long cycles_per_lookup = 1033 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED) 1034 / __atomic_load_n(&greads, __ATOMIC_RELAXED); 1035 rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup; 1036 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 1037 } 1038 } 1039 1040 finish: 1041 rte_hash_free(tbl_rwc_test_param.h); 1042 return 0; 1043 1044 err: 1045 rte_eal_mp_wait_lcore(); 1046 rte_hash_free(tbl_rwc_test_param.h); 1047 return -1; 1048 } 1049 1050 /* 1051 * Test lookup perf for multi-writer: 1052 * Reader(s) lookup keys present in the table and likely on the shift-path while 1053 * Writers add keys causing key-shiftsi. 1054 * Writers are running in parallel, on different data plane cores. 1055 */ 1056 static int 1057 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf, 1058 int htm, int ext_bkt) 1059 { 1060 unsigned int n, m, k; 1061 uint64_t i; 1062 int use_jhash = 0; 1063 uint8_t write_type; 1064 uint8_t read_type = READ_PASS_SHIFT_PATH; 1065 1066 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 1067 goto err; 1068 printf("\nTest: Multi-add-lookup\n"); 1069 uint8_t pos_core; 1070 for (m = 1; m < NUM_TEST; m++) { 1071 /* Calculate keys added by each writer */ 1072 tbl_rwc_test_param.single_insert = 1073 tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m]; 1074 for (k = 0; k < 2; k++) { 1075 if (k == 1) { 1076 printf("\n** With bulk-lookup **\n"); 1077 read_type |= BULK_LOOKUP; 1078 } 1079 for (n = 0; n < NUM_TEST; n++) { 1080 unsigned int tot_lcore = rte_lcore_count(); 1081 if (tot_lcore < (rwc_core_cnt[n] + 1082 rwc_core_cnt[m] + 1)) 1083 goto finish; 1084 1085 printf("\nNumber of writers: %u", 1086 rwc_core_cnt[m]); 1087 printf("\nNumber of readers: %u\n", 1088 rwc_core_cnt[n]); 1089 1090 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED); 1091 __atomic_store_n(&gread_cycles, 0, 1092 __ATOMIC_RELAXED); 1093 1094 rte_hash_reset(tbl_rwc_test_param.h); 1095 writer_done = 0; 1096 write_type = WRITE_NO_KEY_SHIFT; 1097 if (write_keys(write_type) < 0) 1098 goto err; 1099 1100 /* Launch reader(s) */ 1101 for (i = 1; i <= rwc_core_cnt[n]; i++) 1102 rte_eal_remote_launch(test_rwc_reader, 1103 (void *)(uintptr_t)read_type, 1104 enabled_core_ids[i]); 1105 write_type = WRITE_KEY_SHIFT; 1106 pos_core = 0; 1107 1108 /* Launch writers */ 1109 for (; i <= rwc_core_cnt[m] 1110 + rwc_core_cnt[n]; i++) { 1111 rte_eal_remote_launch 1112 (test_rwc_multi_writer, 1113 (void *)(uintptr_t)pos_core, 1114 enabled_core_ids[i]); 1115 pos_core++; 1116 } 1117 1118 /* Wait for writers to complete */ 1119 for (i = rwc_core_cnt[n] + 1; 1120 i <= rwc_core_cnt[m] + rwc_core_cnt[n]; 1121 i++) 1122 rte_eal_wait_lcore(enabled_core_ids[i]); 1123 1124 writer_done = 1; 1125 1126 for (i = 1; i <= rwc_core_cnt[n]; i++) 1127 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 1128 goto err; 1129 1130 unsigned long long cycles_per_lookup = 1131 __atomic_load_n(&gread_cycles, 1132 __ATOMIC_RELAXED) / 1133 __atomic_load_n(&greads, 1134 __ATOMIC_RELAXED); 1135 rwc_perf_results->multi_rw[m][k][n] 1136 = cycles_per_lookup; 1137 printf("Cycles per lookup: %llu\n", 1138 cycles_per_lookup); 1139 } 1140 } 1141 } 1142 1143 finish: 1144 rte_hash_free(tbl_rwc_test_param.h); 1145 return 0; 1146 1147 err: 1148 rte_eal_mp_wait_lcore(); 1149 rte_hash_free(tbl_rwc_test_param.h); 1150 return -1; 1151 } 1152 1153 /* 1154 * Test lookup perf: 1155 * Reader(s) lookup keys present in the extendable bkt. 1156 */ 1157 static int 1158 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results, 1159 int rwc_lf, int htm, int ext_bkt) 1160 { 1161 unsigned int n, m; 1162 uint64_t i; 1163 int use_jhash = 0; 1164 uint8_t write_type; 1165 uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT; 1166 1167 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 1168 goto err; 1169 printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n"); 1170 for (m = 0; m < 2; m++) { 1171 if (m == 1) { 1172 printf("\n** With bulk-lookup **\n"); 1173 read_type |= BULK_LOOKUP; 1174 } 1175 for (n = 0; n < NUM_TEST; n++) { 1176 unsigned int tot_lcore = rte_lcore_count(); 1177 if (tot_lcore < rwc_core_cnt[n] + 1) 1178 goto finish; 1179 1180 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 1181 1182 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED); 1183 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED); 1184 1185 rte_hash_reset(tbl_rwc_test_param.h); 1186 write_type = WRITE_NO_KEY_SHIFT; 1187 if (write_keys(write_type) < 0) 1188 goto err; 1189 write_type = WRITE_KEY_SHIFT; 1190 if (write_keys(write_type) < 0) 1191 goto err; 1192 writer_done = 0; 1193 for (i = 1; i <= rwc_core_cnt[n]; i++) 1194 rte_eal_remote_launch(test_rwc_reader, 1195 (void *)(uintptr_t)read_type, 1196 enabled_core_ids[i]); 1197 for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt; 1198 i++) { 1199 if (rte_hash_del_key(tbl_rwc_test_param.h, 1200 tbl_rwc_test_param.keys_ks_extbkt + i) 1201 < 0) { 1202 printf("Delete Failed: %u\n", 1203 tbl_rwc_test_param.keys_ks_extbkt[i]); 1204 goto err; 1205 } 1206 } 1207 writer_done = 1; 1208 1209 for (i = 1; i <= rwc_core_cnt[n]; i++) 1210 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 1211 goto err; 1212 1213 unsigned long long cycles_per_lookup = 1214 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED) 1215 / __atomic_load_n(&greads, __ATOMIC_RELAXED); 1216 rwc_perf_results->w_ks_r_hit_extbkt[m][n] 1217 = cycles_per_lookup; 1218 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 1219 } 1220 } 1221 1222 finish: 1223 rte_hash_free(tbl_rwc_test_param.h); 1224 return 0; 1225 1226 err: 1227 rte_eal_mp_wait_lcore(); 1228 rte_hash_free(tbl_rwc_test_param.h); 1229 return -1; 1230 } 1231 1232 static struct rte_rcu_qsbr *rv; 1233 1234 /* 1235 * Reader thread using rte_hash data structure with RCU 1236 */ 1237 static int 1238 test_hash_rcu_qsbr_reader(void *arg) 1239 { 1240 unsigned int i, j; 1241 uint32_t num_keys = tbl_rwc_test_param.count_keys_no_ks 1242 - QSBR_REPORTING_INTERVAL; 1243 uint32_t *keys = tbl_rwc_test_param.keys_no_ks; 1244 uint32_t lcore_id = rte_lcore_id(); 1245 RTE_SET_USED(arg); 1246 1247 (void)rte_rcu_qsbr_thread_register(rv, lcore_id); 1248 rte_rcu_qsbr_thread_online(rv, lcore_id); 1249 do { 1250 for (i = 0; i < num_keys; i += j) { 1251 for (j = 0; j < QSBR_REPORTING_INTERVAL; j++) 1252 rte_hash_lookup(tbl_rwc_test_param.h, 1253 keys + i + j); 1254 /* Update quiescent state counter */ 1255 rte_rcu_qsbr_quiescent(rv, lcore_id); 1256 } 1257 } while (!writer_done); 1258 rte_rcu_qsbr_thread_offline(rv, lcore_id); 1259 (void)rte_rcu_qsbr_thread_unregister(rv, lcore_id); 1260 1261 return 0; 1262 } 1263 1264 /* 1265 * Writer thread using rte_hash data structure with RCU 1266 */ 1267 static int 1268 test_hash_rcu_qsbr_writer(void *arg) 1269 { 1270 uint32_t i, offset; 1271 uint64_t begin, cycles; 1272 uint8_t pos_core = (uint32_t)((uintptr_t)arg); 1273 offset = pos_core * tbl_rwc_test_param.single_insert; 1274 1275 begin = rte_rdtsc_precise(); 1276 for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++) { 1277 /* Delete element from the shared data structure */ 1278 rte_hash_del_key(tbl_rwc_test_param.h, 1279 tbl_rwc_test_param.keys_no_ks + i); 1280 rte_hash_add_key(tbl_rwc_test_param.h, 1281 tbl_rwc_test_param.keys_no_ks + i); 1282 } 1283 cycles = rte_rdtsc_precise() - begin; 1284 __atomic_fetch_add(&gwrite_cycles, cycles, __ATOMIC_RELAXED); 1285 __atomic_fetch_add(&gwrites, tbl_rwc_test_param.single_insert, 1286 __ATOMIC_RELAXED); 1287 return 0; 1288 } 1289 1290 /* 1291 * Writer perf test with RCU QSBR in DQ mode: 1292 * Writer(s) delete and add keys in the table. 1293 * Readers lookup keys in the hash table 1294 */ 1295 static int 1296 test_hash_rcu_qsbr_writer_perf(struct rwc_perf *rwc_perf_results, int rwc_lf, 1297 int htm, int ext_bkt) 1298 { 1299 unsigned int n; 1300 uint64_t i; 1301 uint8_t write_type; 1302 int use_jhash = 0; 1303 struct rte_hash_rcu_config rcu_config = {0}; 1304 uint32_t sz; 1305 uint8_t pos_core; 1306 1307 printf("\nTest: Writer perf with integrated RCU\n"); 1308 1309 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 1310 goto err; 1311 1312 sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE); 1313 rv = (struct rte_rcu_qsbr *)rte_zmalloc(NULL, sz, RTE_CACHE_LINE_SIZE); 1314 rcu_config.v = rv; 1315 1316 if (rte_hash_rcu_qsbr_add(tbl_rwc_test_param.h, &rcu_config) < 0) { 1317 printf("RCU init in hash failed\n"); 1318 goto err; 1319 } 1320 1321 for (n = 0; n < NUM_TEST; n++) { 1322 unsigned int tot_lcore = rte_lcore_count(); 1323 if (tot_lcore < rwc_core_cnt[n] + 3) 1324 goto finish; 1325 1326 /* Calculate keys added by each writer */ 1327 tbl_rwc_test_param.single_insert = 1328 tbl_rwc_test_param.count_keys_no_ks / 1329 rwc_core_cnt[n]; 1330 printf("\nNumber of writers: %u\n", rwc_core_cnt[n]); 1331 1332 __atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED); 1333 __atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED); 1334 1335 rte_hash_reset(tbl_rwc_test_param.h); 1336 rte_rcu_qsbr_init(rv, RTE_MAX_LCORE); 1337 1338 write_type = WRITE_NO_KEY_SHIFT; 1339 if (write_keys(write_type) < 0) 1340 goto err; 1341 write_type = WRITE_KEY_SHIFT; 1342 if (write_keys(write_type) < 0) 1343 goto err; 1344 1345 /* Launch 2 readers */ 1346 for (i = 1; i <= 2; i++) 1347 rte_eal_remote_launch(test_hash_rcu_qsbr_reader, NULL, 1348 enabled_core_ids[i]); 1349 pos_core = 0; 1350 /* Launch writer(s) */ 1351 for (; i <= rwc_core_cnt[n] + 2; i++) { 1352 rte_eal_remote_launch(test_hash_rcu_qsbr_writer, 1353 (void *)(uintptr_t)pos_core, 1354 enabled_core_ids[i]); 1355 pos_core++; 1356 } 1357 1358 /* Wait for writers to complete */ 1359 for (i = 3; i <= rwc_core_cnt[n] + 2; i++) 1360 rte_eal_wait_lcore(enabled_core_ids[i]); 1361 1362 writer_done = 1; 1363 1364 /* Wait for readers to complete */ 1365 rte_eal_mp_wait_lcore(); 1366 1367 unsigned long long cycles_per_write_operation = 1368 __atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) / 1369 __atomic_load_n(&gwrites, __ATOMIC_RELAXED); 1370 rwc_perf_results->writer_add_del[n] 1371 = cycles_per_write_operation; 1372 printf("Cycles per write operation: %llu\n", 1373 cycles_per_write_operation); 1374 } 1375 1376 finish: 1377 rte_hash_free(tbl_rwc_test_param.h); 1378 rte_free(rv); 1379 return 0; 1380 1381 err: 1382 writer_done = 1; 1383 rte_eal_mp_wait_lcore(); 1384 rte_hash_free(tbl_rwc_test_param.h); 1385 rte_free(rv); 1386 return -1; 1387 } 1388 1389 static int 1390 test_hash_readwrite_lf_perf_main(void) 1391 { 1392 /* 1393 * Variables used to choose different tests. 1394 * rwc_lf indicates if read-write concurrency lock-free support is 1395 * enabled. 1396 * htm indicates if Hardware transactional memory support is enabled. 1397 */ 1398 int rwc_lf = 0; 1399 int htm; 1400 int ext_bkt = 0; 1401 1402 if (rte_lcore_count() < 2) { 1403 printf("Not enough cores for hash_readwrite_lf_perf_autotest, expecting at least 2\n"); 1404 return TEST_SKIPPED; 1405 } 1406 1407 setlocale(LC_NUMERIC, ""); 1408 1409 /* Reset tbl_rwc_test_param to discard values from previous run */ 1410 memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param)); 1411 1412 if (rte_tm_supported()) 1413 htm = 1; 1414 else 1415 htm = 0; 1416 1417 if (generate_keys() != 0) 1418 return -1; 1419 if (get_enabled_cores_list() != 0) 1420 return -1; 1421 1422 if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) { 1423 rwc_lf = 1; 1424 ext_bkt = 1; 1425 printf("Test lookup with read-write concurrency lock free support" 1426 " enabled\n"); 1427 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf, 1428 htm, ext_bkt) < 0) 1429 return -1; 1430 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf, 1431 htm, ext_bkt) < 0) 1432 return -1; 1433 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf, 1434 htm, ext_bkt) < 0) 1435 return -1; 1436 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf, 1437 htm, ext_bkt) < 0) 1438 return -1; 1439 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm, 1440 ext_bkt) < 0) 1441 return -1; 1442 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm, 1443 ext_bkt) < 0) 1444 return -1; 1445 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf, 1446 htm, ext_bkt) < 0) 1447 return -1; 1448 if (test_hash_rcu_qsbr_writer_perf(&rwc_lf_results, rwc_lf, 1449 htm, ext_bkt) < 0) 1450 return -1; 1451 } 1452 printf("\nTest lookup with read-write concurrency lock free support" 1453 " disabled\n"); 1454 rwc_lf = 0; 1455 if (!htm) { 1456 printf("With HTM Disabled\n"); 1457 if (!RUN_WITH_HTM_DISABLED) { 1458 printf("Enable RUN_WITH_HTM_DISABLED to test with" 1459 " lock-free disabled"); 1460 goto results; 1461 } 1462 } else 1463 printf("With HTM Enabled\n"); 1464 if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm, 1465 ext_bkt) < 0) 1466 return -1; 1467 if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm, 1468 ext_bkt) < 0) 1469 return -1; 1470 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf, 1471 htm, ext_bkt) < 0) 1472 return -1; 1473 if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm, 1474 ext_bkt) < 0) 1475 return -1; 1476 if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm, 1477 ext_bkt) < 0) 1478 return -1; 1479 if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm, 1480 ext_bkt) < 0) 1481 return -1; 1482 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf, 1483 htm, ext_bkt) < 0) 1484 return -1; 1485 results: 1486 printf("\n\t\t\t\t\t\t********** Results summary **********\n\n"); 1487 int i, j, k; 1488 for (j = 0; j < 2; j++) { 1489 if (j == 1) 1490 printf("\n\t\t\t\t\t#######********** Bulk Lookup " 1491 "**********#######\n\n"); 1492 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t" 1493 "\t\t\t\t_________________\n"); 1494 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t" 1495 "\t\t\tCycles per lookup\n"); 1496 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t" 1497 "\t\t\t_________________\n"); 1498 for (i = 0; i < NUM_TEST; i++) { 1499 printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]); 1500 printf("Enabled\t\t"); 1501 printf("N/A\t\t"); 1502 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t" 1503 "%u\n\t\t\t\t\t\t\t\t", 1504 rwc_lf_results.w_no_ks_r_hit[j][i]); 1505 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t" 1506 "%u\n\t\t\t\t\t\t\t\t", 1507 rwc_lf_results.w_no_ks_r_miss[j][i]); 1508 printf("Hash add - key-shifts, lookup - hit" 1509 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1510 rwc_lf_results.w_ks_r_hit_nsp[j][i]); 1511 printf("Hash add - key-shifts, lookup - hit " 1512 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1513 rwc_lf_results.w_ks_r_hit_sp[j][i]); 1514 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t" 1515 "%u\n\t\t\t\t\t\t\t\t", 1516 rwc_lf_results.w_ks_r_miss[j][i]); 1517 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t" 1518 "%u\n\n\t\t\t\t", 1519 rwc_lf_results.w_ks_r_hit_extbkt[j][i]); 1520 1521 printf("Disabled\t"); 1522 if (htm) 1523 printf("Enabled\t\t"); 1524 else 1525 printf("Disabled\t"); 1526 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t" 1527 "%u\n\t\t\t\t\t\t\t\t", 1528 rwc_non_lf_results.w_no_ks_r_hit[j][i]); 1529 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t" 1530 "%u\n\t\t\t\t\t\t\t\t", 1531 rwc_non_lf_results.w_no_ks_r_miss[j][i]); 1532 printf("Hash add - key-shifts, lookup - hit " 1533 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1534 rwc_non_lf_results.w_ks_r_hit_nsp[j][i]); 1535 printf("Hash add - key-shifts, lookup - hit " 1536 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1537 rwc_non_lf_results.w_ks_r_hit_sp[j][i]); 1538 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t" 1539 "%u\n\t\t\t\t\t\t\t\t", 1540 rwc_non_lf_results.w_ks_r_miss[j][i]); 1541 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t" 1542 "%u\n", 1543 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]); 1544 1545 printf("_______\t\t_______\t\t_________\t___\t\t" 1546 "_________\t\t\t\t\t\t_________________\n"); 1547 } 1548 1549 for (i = 1; i < NUM_TEST; i++) { 1550 for (k = 0; k < NUM_TEST; k++) { 1551 printf("%u", rwc_core_cnt[i]); 1552 printf("\t\t%u\t\t", rwc_core_cnt[k]); 1553 printf("Enabled\t\t"); 1554 printf("N/A\t\t"); 1555 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t" 1556 "\t\t", 1557 rwc_lf_results.multi_rw[i][j][k]); 1558 printf("Disabled\t"); 1559 if (htm) 1560 printf("Enabled\t\t"); 1561 else 1562 printf("Disabled\t"); 1563 printf("Multi-add-lookup\t\t\t\t\t\t%u\n", 1564 rwc_non_lf_results.multi_rw[i][j][k]); 1565 1566 printf("_______\t\t_______\t\t_________\t___" 1567 "\t\t_________\t\t\t\t\t\t" 1568 "_________________\n"); 1569 } 1570 } 1571 } 1572 rte_free(tbl_rwc_test_param.keys); 1573 rte_free(tbl_rwc_test_param.keys_no_ks); 1574 rte_free(tbl_rwc_test_param.keys_ks); 1575 rte_free(tbl_rwc_test_param.keys_absent); 1576 rte_free(tbl_rwc_test_param.keys_shift_path); 1577 rte_free(tbl_rwc_test_param.keys_non_shift_path); 1578 rte_free(tbl_rwc_test_param.keys_ext_bkt); 1579 rte_free(tbl_rwc_test_param.keys_ks_extbkt); 1580 return 0; 1581 } 1582 1583 REGISTER_TEST_COMMAND(hash_readwrite_lf_perf_autotest, 1584 test_hash_readwrite_lf_perf_main); 1585