1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Arm Limited 3 */ 4 5 #include <inttypes.h> 6 #include <locale.h> 7 8 #include <rte_cycles.h> 9 #include <rte_hash.h> 10 #include <rte_hash_crc.h> 11 #include <rte_jhash.h> 12 #include <rte_launch.h> 13 #include <rte_malloc.h> 14 #include <rte_random.h> 15 #include <rte_spinlock.h> 16 17 #include "test.h" 18 19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0 21 #endif 22 23 #define BULK_LOOKUP_SIZE 32 24 25 #define RUN_WITH_HTM_DISABLED 0 26 27 #if (RUN_WITH_HTM_DISABLED) 28 29 #define TOTAL_ENTRY (5*1024) 30 #define TOTAL_INSERT (5*1024) 31 32 #else 33 34 #define TOTAL_ENTRY (4*1024*1024) 35 #define TOTAL_INSERT (4*1024*1024) 36 37 #endif 38 39 #define READ_FAIL 1 40 #define READ_PASS_NO_KEY_SHIFTS 2 41 #define READ_PASS_SHIFT_PATH 4 42 #define READ_PASS_NON_SHIFT_PATH 8 43 #define BULK_LOOKUP 16 44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32 45 46 #define WRITE_NO_KEY_SHIFT 0 47 #define WRITE_KEY_SHIFT 1 48 #define WRITE_EXT_BKT 2 49 50 #define NUM_TEST 3 51 52 #define QSBR_REPORTING_INTERVAL 1024 53 54 static unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4}; 55 56 struct rwc_perf { 57 uint32_t w_no_ks_r_hit[2][NUM_TEST]; 58 uint32_t w_no_ks_r_miss[2][NUM_TEST]; 59 uint32_t w_ks_r_hit_nsp[2][NUM_TEST]; 60 uint32_t w_ks_r_hit_sp[2][NUM_TEST]; 61 uint32_t w_ks_r_miss[2][NUM_TEST]; 62 uint32_t multi_rw[NUM_TEST][2][NUM_TEST]; 63 uint32_t w_ks_r_hit_extbkt[2][NUM_TEST]; 64 uint32_t writer_add_del[NUM_TEST]; 65 }; 66 67 static struct rwc_perf rwc_lf_results, rwc_non_lf_results; 68 69 static struct { 70 uint32_t *keys; 71 uint32_t *keys_no_ks; 72 uint32_t *keys_ks; 73 uint32_t *keys_absent; 74 uint32_t *keys_shift_path; 75 uint32_t *keys_non_shift_path; 76 uint32_t *keys_ext_bkt; 77 uint32_t *keys_ks_extbkt; 78 uint32_t count_keys_no_ks; 79 uint32_t count_keys_ks; 80 uint32_t count_keys_absent; 81 uint32_t count_keys_shift_path; 82 uint32_t count_keys_non_shift_path; 83 uint32_t count_keys_extbkt; 84 uint32_t count_keys_ks_extbkt; 85 uint32_t single_insert; 86 struct rte_hash *h; 87 } tbl_rwc_test_param; 88 89 static RTE_ATOMIC(uint64_t) gread_cycles; 90 static RTE_ATOMIC(uint64_t) greads; 91 static RTE_ATOMIC(uint64_t) gwrite_cycles; 92 static RTE_ATOMIC(uint64_t) gwrites; 93 94 static volatile uint8_t writer_done; 95 96 static uint16_t enabled_core_ids[RTE_MAX_LCORE]; 97 98 static uint8_t *scanned_bkts; 99 100 static inline uint16_t 101 get_short_sig(const hash_sig_t hash) 102 { 103 return hash >> 16; 104 } 105 106 static inline uint32_t 107 get_prim_bucket_index(__rte_unused const struct rte_hash *h, 108 const hash_sig_t hash) 109 { 110 uint32_t num_buckets; 111 uint32_t bucket_bitmask; 112 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; 113 bucket_bitmask = num_buckets - 1; 114 return hash & bucket_bitmask; 115 } 116 117 static inline uint32_t 118 get_alt_bucket_index(__rte_unused const struct rte_hash *h, 119 uint32_t cur_bkt_idx, uint16_t sig) 120 { 121 uint32_t num_buckets; 122 uint32_t bucket_bitmask; 123 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; 124 bucket_bitmask = num_buckets - 1; 125 return (cur_bkt_idx ^ sig) & bucket_bitmask; 126 } 127 128 129 static inline int 130 get_enabled_cores_list(void) 131 { 132 uint32_t i = 0; 133 uint16_t core_id; 134 uint32_t max_cores = rte_lcore_count(); 135 RTE_LCORE_FOREACH(core_id) { 136 enabled_core_ids[i] = core_id; 137 i++; 138 } 139 140 if (i != max_cores) { 141 printf("Number of enabled cores in list is different from " 142 "number given by rte_lcore_count()\n"); 143 return -1; 144 } 145 return 0; 146 } 147 148 static int 149 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt) 150 { 151 struct rte_hash *handle; 152 153 struct rte_hash_parameters hash_params = { 154 .entries = TOTAL_ENTRY, 155 .key_len = sizeof(uint32_t), 156 .hash_func_init_val = 0, 157 .socket_id = rte_socket_id(), 158 }; 159 160 if (use_jhash) 161 hash_params.hash_func = rte_jhash; 162 else 163 hash_params.hash_func = rte_hash_crc; 164 165 if (rwc_lf) 166 hash_params.extra_flag = 167 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF | 168 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 169 else if (htm) 170 hash_params.extra_flag = 171 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT | 172 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | 173 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 174 else 175 hash_params.extra_flag = 176 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | 177 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 178 179 if (ext_bkt) 180 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE; 181 182 hash_params.name = "tests"; 183 184 handle = rte_hash_create(&hash_params); 185 if (handle == NULL) { 186 printf("hash creation failed"); 187 return -1; 188 } 189 190 tbl_rwc_test_param.h = handle; 191 return 0; 192 } 193 194 static inline int 195 check_bucket(uint32_t bkt_idx, uint32_t key) 196 { 197 uint32_t iter; 198 uint32_t prev_iter; 199 uint32_t diff; 200 uint32_t count = 0; 201 const void *next_key; 202 void *next_data; 203 204 /* Temporary bucket to hold the keys */ 205 uint32_t keys_in_bkt[8]; 206 207 iter = bkt_idx * 8; 208 prev_iter = iter; 209 while (rte_hash_iterate(tbl_rwc_test_param.h, 210 &next_key, &next_data, &iter) >= 0) { 211 212 /* Check for duplicate entries */ 213 if (*(const uint32_t *)next_key == key) 214 return 1; 215 216 /* Identify if there is any free entry in the bucket */ 217 diff = iter - prev_iter; 218 if (diff > 1) 219 break; 220 221 prev_iter = iter; 222 keys_in_bkt[count] = *(const uint32_t *)next_key; 223 count++; 224 225 /* All entries in the bucket are occupied */ 226 if (count == 8) { 227 228 /* 229 * Check if bucket was not scanned before, to avoid 230 * duplicate keys. 231 */ 232 if (scanned_bkts[bkt_idx] == 0) { 233 /* 234 * Since this bucket (pointed to by bkt_idx) is 235 * full, it is likely that key(s) in this 236 * bucket will be on the shift path, when 237 * collision occurs. Thus, add it to 238 * keys_shift_path. 239 */ 240 memcpy(tbl_rwc_test_param.keys_shift_path + 241 tbl_rwc_test_param.count_keys_shift_path 242 , keys_in_bkt, 32); 243 tbl_rwc_test_param.count_keys_shift_path += 8; 244 scanned_bkts[bkt_idx] = 1; 245 } 246 return -1; 247 } 248 } 249 return 0; 250 } 251 252 static int 253 generate_keys(void) 254 { 255 uint32_t *keys = NULL; 256 uint32_t *keys_no_ks = NULL; 257 uint32_t *keys_ks = NULL; 258 uint32_t *keys_absent = NULL; 259 uint32_t *keys_non_shift_path = NULL; 260 uint32_t *keys_ext_bkt = NULL; 261 uint32_t *keys_ks_extbkt = NULL; 262 uint32_t *found = NULL; 263 uint32_t count_keys_no_ks = 0; 264 uint32_t count_keys_ks = 0; 265 uint32_t count_keys_extbkt = 0; 266 uint32_t i; 267 268 if (init_params(0, 0, 0, 0) != 0) 269 return -1; 270 271 /* 272 * keys will consist of a) keys whose addition to the hash table 273 * will result in shifting of the existing keys to their alternate 274 * locations b) keys whose addition to the hash table will not result 275 * in shifting of the existing keys. 276 */ 277 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 278 if (keys == NULL) { 279 printf("RTE_MALLOC failed\n"); 280 goto err; 281 } 282 283 /* 284 * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys that 285 * will NOT result in shifting of the existing keys to their alternate 286 * locations. Roughly around 900K keys. 287 */ 288 keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 289 if (keys_no_ks == NULL) { 290 printf("RTE_MALLOC failed\n"); 291 goto err; 292 } 293 294 /* 295 * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will 296 * result in shifting of the existing keys to their alternate locations. 297 * Roughly around 146K keys. There might be repeating keys. More code is 298 * required to filter out these keys which will complicate the test case 299 */ 300 keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 301 if (keys_ks == NULL) { 302 printf("RTE_MALLOC failed\n"); 303 goto err; 304 } 305 306 /* Used to identify keys not inserted in the hash table */ 307 found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 308 if (found == NULL) { 309 printf("RTE_MALLOC failed\n"); 310 goto err; 311 } 312 313 /* 314 * This consist of keys not inserted to the hash table. 315 * Used to test perf of lookup on keys that do not exist in the table. 316 */ 317 keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 318 if (keys_absent == NULL) { 319 printf("RTE_MALLOC failed\n"); 320 goto err; 321 } 322 323 /* 324 * This consist of keys which are likely to be on the shift 325 * path (i.e. being moved to alternate location), when collision occurs 326 * on addition of a key to an already full primary bucket. 327 * Used to test perf of lookup on keys that are on the shift path. 328 */ 329 tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) * 330 TOTAL_INSERT, 0); 331 if (tbl_rwc_test_param.keys_shift_path == NULL) { 332 printf("RTE_MALLOC failed\n"); 333 goto err; 334 } 335 336 /* 337 * This consist of keys which are never on the shift 338 * path (i.e. being moved to alternate location), when collision occurs 339 * on addition of a key to an already full primary bucket. 340 * Used to test perf of lookup on keys that are not on the shift path. 341 */ 342 keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 343 0); 344 if (keys_non_shift_path == NULL) { 345 printf("RTE_MALLOC failed\n"); 346 goto err; 347 } 348 349 /* 350 * This consist of keys which will be stored in extended buckets 351 */ 352 keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 353 if (keys_ext_bkt == NULL) { 354 printf("RTE_MALLOC failed\n"); 355 goto err; 356 } 357 358 /* 359 * This consist of keys which when deleted causes shifting of keys 360 * in extended buckets to respective secondary buckets 361 */ 362 keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); 363 if (keys_ks_extbkt == NULL) { 364 printf("RTE_MALLOC failed\n"); 365 goto err; 366 } 367 368 hash_sig_t sig; 369 uint32_t prim_bucket_idx; 370 uint32_t sec_bucket_idx; 371 uint16_t short_sig; 372 uint32_t num_buckets; 373 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; 374 int ret; 375 376 /* 377 * Used to mark bkts in which at least one key was shifted to its 378 * alternate location 379 */ 380 scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0); 381 if (scanned_bkts == NULL) { 382 printf("RTE_MALLOC failed\n"); 383 goto err; 384 } 385 386 tbl_rwc_test_param.keys = keys; 387 tbl_rwc_test_param.keys_no_ks = keys_no_ks; 388 tbl_rwc_test_param.keys_ks = keys_ks; 389 tbl_rwc_test_param.keys_absent = keys_absent; 390 tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path; 391 tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt; 392 tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt; 393 /* Generate keys by adding previous two keys, neglect overflow */ 394 printf("Generating keys...\n"); 395 keys[0] = 0; 396 keys[1] = 1; 397 for (i = 2; i < TOTAL_INSERT; i++) 398 keys[i] = keys[i-1] + keys[i-2]; 399 400 /* Separate keys into keys_no_ks and keys_ks */ 401 for (i = 0; i < TOTAL_INSERT; i++) { 402 /* Check if primary bucket has space.*/ 403 sig = rte_hash_hash(tbl_rwc_test_param.h, 404 tbl_rwc_test_param.keys+i); 405 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h, 406 sig); 407 ret = check_bucket(prim_bucket_idx, keys[i]); 408 if (ret < 0) { 409 /* 410 * Primary bucket is full, this key will result in 411 * shifting of the keys to their alternate locations. 412 */ 413 keys_ks[count_keys_ks] = keys[i]; 414 count_keys_ks++; 415 } else if (ret == 0) { 416 /* 417 * Primary bucket has space, this key will not result in 418 * shifting of the keys. Hence, add key to the table. 419 */ 420 ret = rte_hash_add_key_data(tbl_rwc_test_param.h, 421 keys+i, 422 (void *)((uintptr_t)i)); 423 if (ret < 0) { 424 printf("writer failed %"PRIu32"\n", i); 425 break; 426 } 427 keys_no_ks[count_keys_no_ks] = keys[i]; 428 count_keys_no_ks++; 429 } 430 } 431 432 for (i = 0; i < count_keys_no_ks; i++) { 433 /* 434 * Identify keys in keys_no_ks with value less than 435 * 4M (HTM enabled) OR 5K (HTM disabled) 436 */ 437 if (keys_no_ks[i] < TOTAL_INSERT) 438 found[keys_no_ks[i]]++; 439 } 440 441 for (i = 0; i < count_keys_ks; i++) { 442 /* 443 * Identify keys in keys_ks with value less than 444 * 4M (HTM enabled) OR 5K (HTM disabled) 445 */ 446 if (keys_ks[i] < TOTAL_INSERT) 447 found[keys_ks[i]]++; 448 } 449 450 uint32_t count_keys_absent = 0; 451 for (i = 0; i < TOTAL_INSERT; i++) { 452 /* 453 * Identify missing keys between 0 and 454 * 4M (HTM enabled) OR 5K (HTM disabled) 455 */ 456 if (found[i] == 0) 457 keys_absent[count_keys_absent++] = i; 458 } 459 460 /* Find keys that will not be on the shift path */ 461 uint32_t iter; 462 const void *next_key; 463 void *next_data; 464 uint32_t count = 0; 465 for (i = 0; i < num_buckets; i++) { 466 /* Check bucket for no keys shifted to alternate locations */ 467 if (scanned_bkts[i] == 0) { 468 iter = i * 8; 469 while (rte_hash_iterate(tbl_rwc_test_param.h, 470 &next_key, &next_data, &iter) >= 0) { 471 472 /* Check if key belongs to the current bucket */ 473 if (i >= (iter-1)/8) 474 keys_non_shift_path[count++] 475 = *(const uint32_t *)next_key; 476 else 477 break; 478 } 479 } 480 } 481 482 tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks; 483 tbl_rwc_test_param.count_keys_ks = count_keys_ks; 484 tbl_rwc_test_param.count_keys_absent = count_keys_absent; 485 tbl_rwc_test_param.count_keys_non_shift_path = count; 486 487 memset(scanned_bkts, 0, num_buckets); 488 count = 0; 489 /* Find keys that will be in extended buckets */ 490 for (i = 0; i < count_keys_ks; i++) { 491 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i); 492 if (ret < 0) { 493 /* Key will be added to ext bkt */ 494 keys_ext_bkt[count_keys_extbkt++] = keys_ks[i]; 495 /* Sec bkt to be added to keys_ks_extbkt */ 496 sig = rte_hash_hash(tbl_rwc_test_param.h, 497 tbl_rwc_test_param.keys_ks + i); 498 prim_bucket_idx = get_prim_bucket_index( 499 tbl_rwc_test_param.h, sig); 500 short_sig = get_short_sig(sig); 501 sec_bucket_idx = get_alt_bucket_index( 502 tbl_rwc_test_param.h, 503 prim_bucket_idx, short_sig); 504 if (scanned_bkts[sec_bucket_idx] == 0) 505 scanned_bkts[sec_bucket_idx] = 1; 506 } 507 } 508 509 /* Find keys that will shift keys in ext bucket*/ 510 for (i = 0; i < num_buckets; i++) { 511 if (scanned_bkts[i] == 1) { 512 iter = i * 8; 513 while (rte_hash_iterate(tbl_rwc_test_param.h, 514 &next_key, &next_data, &iter) >= 0) { 515 /* Check if key belongs to the current bucket */ 516 if (i >= (iter-1)/8) 517 keys_ks_extbkt[count++] 518 = *(const uint32_t *)next_key; 519 else 520 break; 521 } 522 } 523 } 524 525 tbl_rwc_test_param.count_keys_ks_extbkt = count; 526 tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt; 527 528 printf("\nCount of keys NOT causing shifting of existing keys to " 529 "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks); 530 printf("\nCount of keys causing shifting of existing keys to alternate " 531 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks); 532 printf("Count of absent keys that will never be added to the hash " 533 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent); 534 printf("Count of keys likely to be on the shift path: %d\n\n", 535 tbl_rwc_test_param.count_keys_shift_path); 536 printf("Count of keys not likely to be on the shift path: %d\n\n", 537 tbl_rwc_test_param.count_keys_non_shift_path); 538 printf("Count of keys in extended buckets: %d\n\n", 539 tbl_rwc_test_param.count_keys_extbkt); 540 printf("Count of keys shifting keys in ext buckets: %d\n\n", 541 tbl_rwc_test_param.count_keys_ks_extbkt); 542 543 rte_free(found); 544 rte_free(scanned_bkts); 545 rte_hash_free(tbl_rwc_test_param.h); 546 return 0; 547 548 err: 549 rte_free(keys); 550 rte_free(keys_no_ks); 551 rte_free(keys_ks); 552 rte_free(keys_absent); 553 rte_free(found); 554 rte_free(tbl_rwc_test_param.keys_shift_path); 555 rte_free(keys_non_shift_path); 556 rte_free(keys_ext_bkt); 557 rte_free(keys_ks_extbkt); 558 rte_free(scanned_bkts); 559 rte_hash_free(tbl_rwc_test_param.h); 560 return -1; 561 } 562 563 static int 564 test_rwc_reader(__rte_unused void *arg) 565 { 566 uint32_t i, j; 567 int ret; 568 uint64_t begin, cycles; 569 uint32_t loop_cnt = 0; 570 uint8_t read_type = (uint8_t)((uintptr_t)arg); 571 uint32_t read_cnt; 572 uint32_t *keys; 573 uint32_t extra_keys; 574 int32_t pos[BULK_LOOKUP_SIZE]; 575 void *temp_a[BULK_LOOKUP_SIZE]; 576 577 if (read_type & READ_FAIL) { 578 keys = tbl_rwc_test_param.keys_absent; 579 read_cnt = tbl_rwc_test_param.count_keys_absent; 580 } else if (read_type & READ_PASS_NO_KEY_SHIFTS) { 581 keys = tbl_rwc_test_param.keys_no_ks; 582 read_cnt = tbl_rwc_test_param.count_keys_no_ks; 583 } else if (read_type & READ_PASS_SHIFT_PATH) { 584 keys = tbl_rwc_test_param.keys_shift_path; 585 read_cnt = tbl_rwc_test_param.count_keys_shift_path; 586 } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) { 587 keys = tbl_rwc_test_param.keys_ext_bkt; 588 read_cnt = tbl_rwc_test_param.count_keys_extbkt; 589 } else { 590 keys = tbl_rwc_test_param.keys_non_shift_path; 591 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path; 592 } 593 594 extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1); 595 596 begin = rte_rdtsc_precise(); 597 do { 598 if (read_type & BULK_LOOKUP) { 599 for (i = 0; i < (read_cnt - extra_keys); 600 i += BULK_LOOKUP_SIZE) { 601 /* Array of pointer to the list of keys */ 602 for (j = 0; j < BULK_LOOKUP_SIZE; j++) 603 temp_a[j] = keys + i + j; 604 rte_hash_lookup_bulk(tbl_rwc_test_param.h, 605 (const void **) 606 ((uintptr_t)temp_a), 607 BULK_LOOKUP_SIZE, pos); 608 /* Validate lookup result */ 609 for (j = 0; j < BULK_LOOKUP_SIZE; j++) 610 if ((read_type & READ_FAIL && 611 pos[j] != -ENOENT) || 612 (!(read_type & READ_FAIL) && 613 pos[j] == -ENOENT)) { 614 printf("lookup failed!" 615 "%"PRIu32"\n", 616 keys[i + j]); 617 return -1; 618 } 619 } 620 for (j = 0; j < extra_keys; j++) 621 temp_a[j] = keys + i + j; 622 623 rte_hash_lookup_bulk(tbl_rwc_test_param.h, 624 (const void **) 625 ((uintptr_t)temp_a), 626 extra_keys, pos); 627 for (j = 0; j < extra_keys; j++) 628 if ((read_type & READ_FAIL && 629 pos[j] != -ENOENT) || 630 (!(read_type & READ_FAIL) && 631 pos[j] == -ENOENT)) { 632 printf("lookup failed! %"PRIu32"\n", 633 keys[i + j]); 634 return -1; 635 } 636 } else { 637 for (i = 0; i < read_cnt; i++) { 638 ret = rte_hash_lookup 639 (tbl_rwc_test_param.h, keys + i); 640 if (((read_type & READ_FAIL) && 641 (ret != -ENOENT)) || 642 (!(read_type & READ_FAIL) && 643 ret == -ENOENT)) { 644 printf("lookup failed! %"PRIu32"\n", 645 keys[i]); 646 return -1; 647 } 648 } 649 } 650 loop_cnt++; 651 } while (!writer_done); 652 653 cycles = rte_rdtsc_precise() - begin; 654 rte_atomic_fetch_add_explicit(&gread_cycles, cycles, rte_memory_order_relaxed); 655 rte_atomic_fetch_add_explicit(&greads, read_cnt*loop_cnt, rte_memory_order_relaxed); 656 return 0; 657 } 658 659 static int 660 write_keys(uint8_t write_type) 661 { 662 uint32_t i; 663 int ret; 664 uint32_t key_cnt = 0; 665 uint32_t *keys; 666 if (write_type == WRITE_KEY_SHIFT) { 667 key_cnt = tbl_rwc_test_param.count_keys_ks; 668 keys = tbl_rwc_test_param.keys_ks; 669 } else if (write_type == WRITE_NO_KEY_SHIFT) { 670 key_cnt = tbl_rwc_test_param.count_keys_no_ks; 671 keys = tbl_rwc_test_param.keys_no_ks; 672 } else if (write_type == WRITE_EXT_BKT) { 673 key_cnt = tbl_rwc_test_param.count_keys_extbkt; 674 keys = tbl_rwc_test_param.keys_ext_bkt; 675 } 676 for (i = 0; i < key_cnt; i++) { 677 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i); 678 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) { 679 printf("writer failed %"PRIu32"\n", i); 680 return -1; 681 } 682 } 683 return 0; 684 } 685 686 static int 687 test_rwc_multi_writer(__rte_unused void *arg) 688 { 689 uint32_t i, offset; 690 uint32_t pos_core = (uint32_t)((uintptr_t)arg); 691 offset = pos_core * tbl_rwc_test_param.single_insert; 692 for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++) 693 rte_hash_add_key(tbl_rwc_test_param.h, 694 tbl_rwc_test_param.keys_ks + i); 695 return 0; 696 } 697 698 /* 699 * Test lookup perf: 700 * Reader(s) lookup keys present in the table. 701 */ 702 static int 703 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf, 704 int htm, int ext_bkt) 705 { 706 unsigned int n, m; 707 uint64_t i; 708 int use_jhash = 0; 709 uint8_t write_type = WRITE_NO_KEY_SHIFT; 710 uint8_t read_type = READ_PASS_NO_KEY_SHIFTS; 711 712 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 713 goto err; 714 printf("\nTest: Hash add - no key-shifts, read - hit\n"); 715 for (m = 0; m < 2; m++) { 716 if (m == 1) { 717 printf("\n** With bulk-lookup **\n"); 718 read_type |= BULK_LOOKUP; 719 } 720 for (n = 0; n < NUM_TEST; n++) { 721 unsigned int tot_lcore = rte_lcore_count(); 722 if (tot_lcore < rwc_core_cnt[n] + 1) 723 goto finish; 724 725 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 726 727 rte_atomic_store_explicit(&greads, 0, rte_memory_order_relaxed); 728 rte_atomic_store_explicit(&gread_cycles, 0, rte_memory_order_relaxed); 729 730 rte_hash_reset(tbl_rwc_test_param.h); 731 writer_done = 0; 732 if (write_keys(write_type) < 0) 733 goto err; 734 writer_done = 1; 735 for (i = 1; i <= rwc_core_cnt[n]; i++) 736 rte_eal_remote_launch(test_rwc_reader, 737 (void *)(uintptr_t)read_type, 738 enabled_core_ids[i]); 739 740 for (i = 1; i <= rwc_core_cnt[n]; i++) 741 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 742 goto err; 743 744 unsigned long long cycles_per_lookup = 745 rte_atomic_load_explicit(&gread_cycles, rte_memory_order_relaxed) 746 / rte_atomic_load_explicit(&greads, rte_memory_order_relaxed); 747 rwc_perf_results->w_no_ks_r_hit[m][n] 748 = cycles_per_lookup; 749 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 750 } 751 } 752 753 finish: 754 rte_hash_free(tbl_rwc_test_param.h); 755 return 0; 756 757 err: 758 rte_eal_mp_wait_lcore(); 759 rte_hash_free(tbl_rwc_test_param.h); 760 return -1; 761 } 762 763 /* 764 * Test lookup perf: 765 * Reader(s) lookup keys absent in the table while 766 * 'Main' thread adds with no key-shifts. 767 */ 768 static int 769 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, 770 int htm, int ext_bkt) 771 { 772 unsigned int n, m; 773 uint64_t i; 774 int use_jhash = 0; 775 uint8_t write_type = WRITE_NO_KEY_SHIFT; 776 uint8_t read_type = READ_FAIL; 777 int ret; 778 779 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 780 goto err; 781 printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n"); 782 for (m = 0; m < 2; m++) { 783 if (m == 1) { 784 printf("\n** With bulk-lookup **\n"); 785 read_type |= BULK_LOOKUP; 786 } 787 for (n = 0; n < NUM_TEST; n++) { 788 unsigned int tot_lcore = rte_lcore_count(); 789 if (tot_lcore < rwc_core_cnt[n] + 1) 790 goto finish; 791 792 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 793 794 rte_atomic_store_explicit(&greads, 0, rte_memory_order_relaxed); 795 rte_atomic_store_explicit(&gread_cycles, 0, rte_memory_order_relaxed); 796 797 rte_hash_reset(tbl_rwc_test_param.h); 798 writer_done = 0; 799 800 for (i = 1; i <= rwc_core_cnt[n]; i++) 801 rte_eal_remote_launch(test_rwc_reader, 802 (void *)(uintptr_t)read_type, 803 enabled_core_ids[i]); 804 ret = write_keys(write_type); 805 writer_done = 1; 806 807 if (ret < 0) 808 goto err; 809 for (i = 1; i <= rwc_core_cnt[n]; i++) 810 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 811 goto err; 812 813 unsigned long long cycles_per_lookup = 814 rte_atomic_load_explicit(&gread_cycles, rte_memory_order_relaxed) 815 / rte_atomic_load_explicit(&greads, rte_memory_order_relaxed); 816 rwc_perf_results->w_no_ks_r_miss[m][n] 817 = cycles_per_lookup; 818 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 819 } 820 } 821 822 finish: 823 rte_hash_free(tbl_rwc_test_param.h); 824 return 0; 825 826 err: 827 rte_eal_mp_wait_lcore(); 828 rte_hash_free(tbl_rwc_test_param.h); 829 return -1; 830 } 831 832 /* 833 * Test lookup perf: 834 * Reader(s) lookup keys present in the table and not likely to be on the 835 * shift path while 'Main' thread adds keys causing key-shifts. 836 */ 837 static int 838 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results, 839 int rwc_lf, int htm, int ext_bkt) 840 { 841 unsigned int n, m; 842 uint64_t i; 843 int use_jhash = 0; 844 int ret; 845 uint8_t write_type; 846 uint8_t read_type = READ_PASS_NON_SHIFT_PATH; 847 848 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 849 goto err; 850 printf("\nTest: Hash add - key shift, Hash lookup - hit" 851 " (non-shift-path)\n"); 852 for (m = 0; m < 2; m++) { 853 if (m == 1) { 854 printf("\n** With bulk-lookup **\n"); 855 read_type |= BULK_LOOKUP; 856 } 857 for (n = 0; n < NUM_TEST; n++) { 858 unsigned int tot_lcore = rte_lcore_count(); 859 if (tot_lcore < rwc_core_cnt[n] + 1) 860 goto finish; 861 862 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 863 864 rte_atomic_store_explicit(&greads, 0, rte_memory_order_relaxed); 865 rte_atomic_store_explicit(&gread_cycles, 0, rte_memory_order_relaxed); 866 867 rte_hash_reset(tbl_rwc_test_param.h); 868 writer_done = 0; 869 write_type = WRITE_NO_KEY_SHIFT; 870 if (write_keys(write_type) < 0) 871 goto err; 872 for (i = 1; i <= rwc_core_cnt[n]; i++) 873 rte_eal_remote_launch(test_rwc_reader, 874 (void *)(uintptr_t)read_type, 875 enabled_core_ids[i]); 876 write_type = WRITE_KEY_SHIFT; 877 ret = write_keys(write_type); 878 writer_done = 1; 879 880 if (ret < 0) 881 goto err; 882 for (i = 1; i <= rwc_core_cnt[n]; i++) 883 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 884 goto err; 885 886 unsigned long long cycles_per_lookup = 887 rte_atomic_load_explicit(&gread_cycles, rte_memory_order_relaxed) 888 / rte_atomic_load_explicit(&greads, rte_memory_order_relaxed); 889 rwc_perf_results->w_ks_r_hit_nsp[m][n] 890 = cycles_per_lookup; 891 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 892 } 893 } 894 895 finish: 896 rte_hash_free(tbl_rwc_test_param.h); 897 return 0; 898 899 err: 900 rte_eal_mp_wait_lcore(); 901 rte_hash_free(tbl_rwc_test_param.h); 902 return -1; 903 } 904 905 /* 906 * Test lookup perf: 907 * Reader(s) lookup keys present in the table and likely on the shift-path while 908 * 'Main' thread adds keys causing key-shifts. 909 */ 910 static int 911 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf, 912 int htm, int ext_bkt) 913 { 914 unsigned int n, m; 915 uint64_t i; 916 int use_jhash = 0; 917 int ret; 918 uint8_t write_type; 919 uint8_t read_type = READ_PASS_SHIFT_PATH; 920 921 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 922 goto err; 923 printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)" 924 "\n"); 925 926 for (m = 0; m < 2; m++) { 927 if (m == 1) { 928 printf("\n** With bulk-lookup **\n"); 929 read_type |= BULK_LOOKUP; 930 } 931 for (n = 0; n < NUM_TEST; n++) { 932 unsigned int tot_lcore = rte_lcore_count(); 933 if (tot_lcore < rwc_core_cnt[n] + 1) 934 goto finish; 935 936 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 937 938 rte_atomic_store_explicit(&greads, 0, rte_memory_order_relaxed); 939 rte_atomic_store_explicit(&gread_cycles, 0, rte_memory_order_relaxed); 940 941 rte_hash_reset(tbl_rwc_test_param.h); 942 writer_done = 0; 943 write_type = WRITE_NO_KEY_SHIFT; 944 if (write_keys(write_type) < 0) 945 goto err; 946 for (i = 1; i <= rwc_core_cnt[n]; i++) 947 rte_eal_remote_launch(test_rwc_reader, 948 (void *)(uintptr_t)read_type, 949 enabled_core_ids[i]); 950 write_type = WRITE_KEY_SHIFT; 951 ret = write_keys(write_type); 952 writer_done = 1; 953 954 if (ret < 0) 955 goto err; 956 for (i = 1; i <= rwc_core_cnt[n]; i++) 957 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 958 goto err; 959 960 unsigned long long cycles_per_lookup = 961 rte_atomic_load_explicit(&gread_cycles, rte_memory_order_relaxed) 962 / rte_atomic_load_explicit(&greads, rte_memory_order_relaxed); 963 rwc_perf_results->w_ks_r_hit_sp[m][n] 964 = cycles_per_lookup; 965 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 966 } 967 } 968 969 finish: 970 rte_hash_free(tbl_rwc_test_param.h); 971 return 0; 972 973 err: 974 rte_eal_mp_wait_lcore(); 975 rte_hash_free(tbl_rwc_test_param.h); 976 return -1; 977 } 978 979 /* 980 * Test lookup perf: 981 * Reader(s) lookup keys absent in the table while 982 * 'Main' thread adds keys causing key-shifts. 983 */ 984 static int 985 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int 986 htm, int ext_bkt) 987 { 988 unsigned int n, m; 989 uint64_t i; 990 int use_jhash = 0; 991 int ret; 992 uint8_t write_type; 993 uint8_t read_type = READ_FAIL; 994 995 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 996 goto err; 997 printf("\nTest: Hash add - key shift, Hash lookup - miss\n"); 998 for (m = 0; m < 2; m++) { 999 if (m == 1) { 1000 printf("\n** With bulk-lookup **\n"); 1001 read_type |= BULK_LOOKUP; 1002 } 1003 for (n = 0; n < NUM_TEST; n++) { 1004 unsigned int tot_lcore = rte_lcore_count(); 1005 if (tot_lcore < rwc_core_cnt[n] + 1) 1006 goto finish; 1007 1008 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 1009 1010 rte_atomic_store_explicit(&greads, 0, rte_memory_order_relaxed); 1011 rte_atomic_store_explicit(&gread_cycles, 0, rte_memory_order_relaxed); 1012 1013 rte_hash_reset(tbl_rwc_test_param.h); 1014 writer_done = 0; 1015 write_type = WRITE_NO_KEY_SHIFT; 1016 if (write_keys(write_type) < 0) 1017 goto err; 1018 for (i = 1; i <= rwc_core_cnt[n]; i++) 1019 rte_eal_remote_launch(test_rwc_reader, 1020 (void *)(uintptr_t)read_type, 1021 enabled_core_ids[i]); 1022 write_type = WRITE_KEY_SHIFT; 1023 ret = write_keys(write_type); 1024 writer_done = 1; 1025 1026 if (ret < 0) 1027 goto err; 1028 for (i = 1; i <= rwc_core_cnt[n]; i++) 1029 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 1030 goto err; 1031 1032 unsigned long long cycles_per_lookup = 1033 rte_atomic_load_explicit(&gread_cycles, rte_memory_order_relaxed) 1034 / rte_atomic_load_explicit(&greads, rte_memory_order_relaxed); 1035 rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup; 1036 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 1037 } 1038 } 1039 1040 finish: 1041 rte_hash_free(tbl_rwc_test_param.h); 1042 return 0; 1043 1044 err: 1045 rte_eal_mp_wait_lcore(); 1046 rte_hash_free(tbl_rwc_test_param.h); 1047 return -1; 1048 } 1049 1050 /* 1051 * Test lookup perf for multi-writer: 1052 * Reader(s) lookup keys present in the table and likely on the shift-path while 1053 * Writers add keys causing key-shiftsi. 1054 * Writers are running in parallel, on different data plane cores. 1055 */ 1056 static int 1057 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf, 1058 int htm, int ext_bkt) 1059 { 1060 unsigned int n, m, k; 1061 uint64_t i; 1062 int use_jhash = 0; 1063 uint8_t write_type; 1064 uint8_t read_type = READ_PASS_SHIFT_PATH; 1065 1066 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 1067 goto err; 1068 printf("\nTest: Multi-add-lookup\n"); 1069 uint8_t pos_core; 1070 for (m = 1; m < NUM_TEST; m++) { 1071 /* Calculate keys added by each writer */ 1072 tbl_rwc_test_param.single_insert = 1073 tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m]; 1074 for (k = 0; k < 2; k++) { 1075 if (k == 1) { 1076 printf("\n** With bulk-lookup **\n"); 1077 read_type |= BULK_LOOKUP; 1078 } 1079 for (n = 0; n < NUM_TEST; n++) { 1080 unsigned int tot_lcore = rte_lcore_count(); 1081 if (tot_lcore < (rwc_core_cnt[n] + 1082 rwc_core_cnt[m] + 1)) 1083 goto finish; 1084 1085 printf("\nNumber of writers: %u", 1086 rwc_core_cnt[m]); 1087 printf("\nNumber of readers: %u\n", 1088 rwc_core_cnt[n]); 1089 1090 rte_atomic_store_explicit(&greads, 0, rte_memory_order_relaxed); 1091 rte_atomic_store_explicit(&gread_cycles, 0, 1092 rte_memory_order_relaxed); 1093 1094 rte_hash_reset(tbl_rwc_test_param.h); 1095 writer_done = 0; 1096 write_type = WRITE_NO_KEY_SHIFT; 1097 if (write_keys(write_type) < 0) 1098 goto err; 1099 1100 /* Launch reader(s) */ 1101 for (i = 1; i <= rwc_core_cnt[n]; i++) 1102 rte_eal_remote_launch(test_rwc_reader, 1103 (void *)(uintptr_t)read_type, 1104 enabled_core_ids[i]); 1105 pos_core = 0; 1106 1107 /* Launch writers */ 1108 for (; i <= rwc_core_cnt[m] 1109 + rwc_core_cnt[n]; i++) { 1110 rte_eal_remote_launch 1111 (test_rwc_multi_writer, 1112 (void *)(uintptr_t)pos_core, 1113 enabled_core_ids[i]); 1114 pos_core++; 1115 } 1116 1117 /* Wait for writers to complete */ 1118 for (i = rwc_core_cnt[n] + 1; 1119 i <= rwc_core_cnt[m] + rwc_core_cnt[n]; 1120 i++) 1121 rte_eal_wait_lcore(enabled_core_ids[i]); 1122 1123 writer_done = 1; 1124 1125 for (i = 1; i <= rwc_core_cnt[n]; i++) 1126 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 1127 goto err; 1128 1129 unsigned long long cycles_per_lookup = 1130 rte_atomic_load_explicit(&gread_cycles, 1131 rte_memory_order_relaxed) / 1132 rte_atomic_load_explicit(&greads, 1133 rte_memory_order_relaxed); 1134 rwc_perf_results->multi_rw[m][k][n] 1135 = cycles_per_lookup; 1136 printf("Cycles per lookup: %llu\n", 1137 cycles_per_lookup); 1138 } 1139 } 1140 } 1141 1142 finish: 1143 rte_hash_free(tbl_rwc_test_param.h); 1144 return 0; 1145 1146 err: 1147 rte_eal_mp_wait_lcore(); 1148 rte_hash_free(tbl_rwc_test_param.h); 1149 return -1; 1150 } 1151 1152 /* 1153 * Test lookup perf: 1154 * Reader(s) lookup keys present in the extendable bkt. 1155 */ 1156 static int 1157 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results, 1158 int rwc_lf, int htm, int ext_bkt) 1159 { 1160 unsigned int n, m; 1161 uint64_t i; 1162 int use_jhash = 0; 1163 uint8_t write_type; 1164 uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT; 1165 1166 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 1167 goto err; 1168 printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n"); 1169 for (m = 0; m < 2; m++) { 1170 if (m == 1) { 1171 printf("\n** With bulk-lookup **\n"); 1172 read_type |= BULK_LOOKUP; 1173 } 1174 for (n = 0; n < NUM_TEST; n++) { 1175 unsigned int tot_lcore = rte_lcore_count(); 1176 if (tot_lcore < rwc_core_cnt[n] + 1) 1177 goto finish; 1178 1179 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); 1180 1181 rte_atomic_store_explicit(&greads, 0, rte_memory_order_relaxed); 1182 rte_atomic_store_explicit(&gread_cycles, 0, rte_memory_order_relaxed); 1183 1184 rte_hash_reset(tbl_rwc_test_param.h); 1185 write_type = WRITE_NO_KEY_SHIFT; 1186 if (write_keys(write_type) < 0) 1187 goto err; 1188 write_type = WRITE_KEY_SHIFT; 1189 if (write_keys(write_type) < 0) 1190 goto err; 1191 writer_done = 0; 1192 for (i = 1; i <= rwc_core_cnt[n]; i++) 1193 rte_eal_remote_launch(test_rwc_reader, 1194 (void *)(uintptr_t)read_type, 1195 enabled_core_ids[i]); 1196 for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt; 1197 i++) { 1198 if (rte_hash_del_key(tbl_rwc_test_param.h, 1199 tbl_rwc_test_param.keys_ks_extbkt + i) 1200 < 0) { 1201 printf("Delete Failed: %u\n", 1202 tbl_rwc_test_param.keys_ks_extbkt[i]); 1203 goto err; 1204 } 1205 } 1206 writer_done = 1; 1207 1208 for (i = 1; i <= rwc_core_cnt[n]; i++) 1209 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) 1210 goto err; 1211 1212 unsigned long long cycles_per_lookup = 1213 rte_atomic_load_explicit(&gread_cycles, rte_memory_order_relaxed) 1214 / rte_atomic_load_explicit(&greads, rte_memory_order_relaxed); 1215 rwc_perf_results->w_ks_r_hit_extbkt[m][n] 1216 = cycles_per_lookup; 1217 printf("Cycles per lookup: %llu\n", cycles_per_lookup); 1218 } 1219 } 1220 1221 finish: 1222 rte_hash_free(tbl_rwc_test_param.h); 1223 return 0; 1224 1225 err: 1226 rte_eal_mp_wait_lcore(); 1227 rte_hash_free(tbl_rwc_test_param.h); 1228 return -1; 1229 } 1230 1231 static struct rte_rcu_qsbr *rv; 1232 1233 /* 1234 * Reader thread using rte_hash data structure with RCU 1235 */ 1236 static int 1237 test_hash_rcu_qsbr_reader(void *arg) 1238 { 1239 unsigned int i, j; 1240 uint32_t num_keys = tbl_rwc_test_param.count_keys_no_ks 1241 - QSBR_REPORTING_INTERVAL; 1242 uint32_t *keys = tbl_rwc_test_param.keys_no_ks; 1243 uint32_t lcore_id = rte_lcore_id(); 1244 RTE_SET_USED(arg); 1245 1246 (void)rte_rcu_qsbr_thread_register(rv, lcore_id); 1247 rte_rcu_qsbr_thread_online(rv, lcore_id); 1248 do { 1249 for (i = 0; i < num_keys; i += j) { 1250 for (j = 0; j < QSBR_REPORTING_INTERVAL; j++) 1251 rte_hash_lookup(tbl_rwc_test_param.h, 1252 keys + i + j); 1253 /* Update quiescent state counter */ 1254 rte_rcu_qsbr_quiescent(rv, lcore_id); 1255 } 1256 } while (!writer_done); 1257 rte_rcu_qsbr_thread_offline(rv, lcore_id); 1258 (void)rte_rcu_qsbr_thread_unregister(rv, lcore_id); 1259 1260 return 0; 1261 } 1262 1263 /* 1264 * Writer thread using rte_hash data structure with RCU 1265 */ 1266 static int 1267 test_hash_rcu_qsbr_writer(void *arg) 1268 { 1269 uint32_t i, offset; 1270 uint64_t begin, cycles; 1271 uint8_t pos_core = (uint32_t)((uintptr_t)arg); 1272 offset = pos_core * tbl_rwc_test_param.single_insert; 1273 1274 begin = rte_rdtsc_precise(); 1275 for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++) { 1276 /* Delete element from the shared data structure */ 1277 rte_hash_del_key(tbl_rwc_test_param.h, 1278 tbl_rwc_test_param.keys_no_ks + i); 1279 rte_hash_add_key(tbl_rwc_test_param.h, 1280 tbl_rwc_test_param.keys_no_ks + i); 1281 } 1282 cycles = rte_rdtsc_precise() - begin; 1283 rte_atomic_fetch_add_explicit(&gwrite_cycles, cycles, rte_memory_order_relaxed); 1284 rte_atomic_fetch_add_explicit(&gwrites, tbl_rwc_test_param.single_insert, 1285 rte_memory_order_relaxed); 1286 return 0; 1287 } 1288 1289 /* 1290 * Writer perf test with RCU QSBR in DQ mode: 1291 * Writer(s) delete and add keys in the table. 1292 * Readers lookup keys in the hash table 1293 */ 1294 static int 1295 test_hash_rcu_qsbr_writer_perf(struct rwc_perf *rwc_perf_results, int rwc_lf, 1296 int htm, int ext_bkt) 1297 { 1298 unsigned int n; 1299 uint64_t i; 1300 uint8_t write_type; 1301 int use_jhash = 0; 1302 struct rte_hash_rcu_config rcu_config = {0}; 1303 uint32_t sz; 1304 uint8_t pos_core; 1305 1306 printf("\nTest: Writer perf with integrated RCU\n"); 1307 1308 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) 1309 goto err; 1310 1311 sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE); 1312 rv = (struct rte_rcu_qsbr *)rte_zmalloc(NULL, sz, RTE_CACHE_LINE_SIZE); 1313 rcu_config.v = rv; 1314 1315 if (rte_hash_rcu_qsbr_add(tbl_rwc_test_param.h, &rcu_config) < 0) { 1316 printf("RCU init in hash failed\n"); 1317 goto err; 1318 } 1319 1320 for (n = 0; n < NUM_TEST; n++) { 1321 unsigned int tot_lcore = rte_lcore_count(); 1322 if (tot_lcore < rwc_core_cnt[n] + 3) 1323 goto finish; 1324 1325 /* Calculate keys added by each writer */ 1326 tbl_rwc_test_param.single_insert = 1327 tbl_rwc_test_param.count_keys_no_ks / 1328 rwc_core_cnt[n]; 1329 printf("\nNumber of writers: %u\n", rwc_core_cnt[n]); 1330 1331 rte_atomic_store_explicit(&gwrites, 0, rte_memory_order_relaxed); 1332 rte_atomic_store_explicit(&gwrite_cycles, 0, rte_memory_order_relaxed); 1333 1334 rte_hash_reset(tbl_rwc_test_param.h); 1335 rte_rcu_qsbr_init(rv, RTE_MAX_LCORE); 1336 1337 write_type = WRITE_NO_KEY_SHIFT; 1338 if (write_keys(write_type) < 0) 1339 goto err; 1340 write_type = WRITE_KEY_SHIFT; 1341 if (write_keys(write_type) < 0) 1342 goto err; 1343 1344 /* Launch 2 readers */ 1345 for (i = 1; i <= 2; i++) 1346 rte_eal_remote_launch(test_hash_rcu_qsbr_reader, NULL, 1347 enabled_core_ids[i]); 1348 pos_core = 0; 1349 /* Launch writer(s) */ 1350 for (; i <= rwc_core_cnt[n] + 2; i++) { 1351 rte_eal_remote_launch(test_hash_rcu_qsbr_writer, 1352 (void *)(uintptr_t)pos_core, 1353 enabled_core_ids[i]); 1354 pos_core++; 1355 } 1356 1357 /* Wait for writers to complete */ 1358 for (i = 3; i <= rwc_core_cnt[n] + 2; i++) 1359 rte_eal_wait_lcore(enabled_core_ids[i]); 1360 1361 writer_done = 1; 1362 1363 /* Wait for readers to complete */ 1364 rte_eal_mp_wait_lcore(); 1365 1366 unsigned long long cycles_per_write_operation = 1367 rte_atomic_load_explicit(&gwrite_cycles, rte_memory_order_relaxed) / 1368 rte_atomic_load_explicit(&gwrites, rte_memory_order_relaxed); 1369 rwc_perf_results->writer_add_del[n] 1370 = cycles_per_write_operation; 1371 printf("Cycles per write operation: %llu\n", 1372 cycles_per_write_operation); 1373 } 1374 1375 finish: 1376 rte_hash_free(tbl_rwc_test_param.h); 1377 rte_free(rv); 1378 return 0; 1379 1380 err: 1381 writer_done = 1; 1382 rte_eal_mp_wait_lcore(); 1383 rte_hash_free(tbl_rwc_test_param.h); 1384 rte_free(rv); 1385 return -1; 1386 } 1387 1388 static int 1389 test_hash_readwrite_lf_perf_main(void) 1390 { 1391 /* 1392 * Variables used to choose different tests. 1393 * rwc_lf indicates if read-write concurrency lock-free support is 1394 * enabled. 1395 * htm indicates if Hardware transactional memory support is enabled. 1396 */ 1397 int rwc_lf = 0; 1398 int htm; 1399 int ext_bkt = 0; 1400 1401 if (rte_lcore_count() < 2) { 1402 printf("Not enough cores for hash_readwrite_lf_perf_autotest, expecting at least 2\n"); 1403 return TEST_SKIPPED; 1404 } 1405 1406 setlocale(LC_NUMERIC, ""); 1407 1408 /* Reset tbl_rwc_test_param to discard values from previous run */ 1409 memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param)); 1410 1411 if (rte_tm_supported()) 1412 htm = 1; 1413 else 1414 htm = 0; 1415 1416 if (generate_keys() != 0) 1417 return -1; 1418 if (get_enabled_cores_list() != 0) 1419 return -1; 1420 1421 if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) { 1422 rwc_lf = 1; 1423 ext_bkt = 1; 1424 printf("Test lookup with read-write concurrency lock free support" 1425 " enabled\n"); 1426 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf, 1427 htm, ext_bkt) < 0) 1428 return -1; 1429 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf, 1430 htm, ext_bkt) < 0) 1431 return -1; 1432 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf, 1433 htm, ext_bkt) < 0) 1434 return -1; 1435 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf, 1436 htm, ext_bkt) < 0) 1437 return -1; 1438 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm, 1439 ext_bkt) < 0) 1440 return -1; 1441 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm, 1442 ext_bkt) < 0) 1443 return -1; 1444 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf, 1445 htm, ext_bkt) < 0) 1446 return -1; 1447 if (test_hash_rcu_qsbr_writer_perf(&rwc_lf_results, rwc_lf, 1448 htm, ext_bkt) < 0) 1449 return -1; 1450 } 1451 printf("\nTest lookup with read-write concurrency lock free support" 1452 " disabled\n"); 1453 rwc_lf = 0; 1454 if (!htm) { 1455 printf("With HTM Disabled\n"); 1456 if (!RUN_WITH_HTM_DISABLED) { 1457 printf("Enable RUN_WITH_HTM_DISABLED to test with" 1458 " lock-free disabled"); 1459 goto results; 1460 } 1461 } else 1462 printf("With HTM Enabled\n"); 1463 if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm, 1464 ext_bkt) < 0) 1465 return -1; 1466 if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm, 1467 ext_bkt) < 0) 1468 return -1; 1469 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf, 1470 htm, ext_bkt) < 0) 1471 return -1; 1472 if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm, 1473 ext_bkt) < 0) 1474 return -1; 1475 if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm, 1476 ext_bkt) < 0) 1477 return -1; 1478 if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm, 1479 ext_bkt) < 0) 1480 return -1; 1481 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf, 1482 htm, ext_bkt) < 0) 1483 return -1; 1484 results: 1485 printf("\n\t\t\t\t\t\t********** Results summary **********\n\n"); 1486 int i, j, k; 1487 for (j = 0; j < 2; j++) { 1488 if (j == 1) 1489 printf("\n\t\t\t\t\t#######********** Bulk Lookup " 1490 "**********#######\n\n"); 1491 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t" 1492 "\t\t\t\t_________________\n"); 1493 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t" 1494 "\t\t\tCycles per lookup\n"); 1495 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t" 1496 "\t\t\t_________________\n"); 1497 for (i = 0; i < NUM_TEST; i++) { 1498 printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]); 1499 printf("Enabled\t\t"); 1500 printf("N/A\t\t"); 1501 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t" 1502 "%u\n\t\t\t\t\t\t\t\t", 1503 rwc_lf_results.w_no_ks_r_hit[j][i]); 1504 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t" 1505 "%u\n\t\t\t\t\t\t\t\t", 1506 rwc_lf_results.w_no_ks_r_miss[j][i]); 1507 printf("Hash add - key-shifts, lookup - hit" 1508 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1509 rwc_lf_results.w_ks_r_hit_nsp[j][i]); 1510 printf("Hash add - key-shifts, lookup - hit " 1511 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1512 rwc_lf_results.w_ks_r_hit_sp[j][i]); 1513 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t" 1514 "%u\n\t\t\t\t\t\t\t\t", 1515 rwc_lf_results.w_ks_r_miss[j][i]); 1516 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t" 1517 "%u\n\n\t\t\t\t", 1518 rwc_lf_results.w_ks_r_hit_extbkt[j][i]); 1519 1520 printf("Disabled\t"); 1521 if (htm) 1522 printf("Enabled\t\t"); 1523 else 1524 printf("Disabled\t"); 1525 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t" 1526 "%u\n\t\t\t\t\t\t\t\t", 1527 rwc_non_lf_results.w_no_ks_r_hit[j][i]); 1528 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t" 1529 "%u\n\t\t\t\t\t\t\t\t", 1530 rwc_non_lf_results.w_no_ks_r_miss[j][i]); 1531 printf("Hash add - key-shifts, lookup - hit " 1532 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1533 rwc_non_lf_results.w_ks_r_hit_nsp[j][i]); 1534 printf("Hash add - key-shifts, lookup - hit " 1535 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", 1536 rwc_non_lf_results.w_ks_r_hit_sp[j][i]); 1537 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t" 1538 "%u\n\t\t\t\t\t\t\t\t", 1539 rwc_non_lf_results.w_ks_r_miss[j][i]); 1540 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t" 1541 "%u\n", 1542 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]); 1543 1544 printf("_______\t\t_______\t\t_________\t___\t\t" 1545 "_________\t\t\t\t\t\t_________________\n"); 1546 } 1547 1548 for (i = 1; i < NUM_TEST; i++) { 1549 for (k = 0; k < NUM_TEST; k++) { 1550 printf("%u", rwc_core_cnt[i]); 1551 printf("\t\t%u\t\t", rwc_core_cnt[k]); 1552 printf("Enabled\t\t"); 1553 printf("N/A\t\t"); 1554 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t" 1555 "\t\t", 1556 rwc_lf_results.multi_rw[i][j][k]); 1557 printf("Disabled\t"); 1558 if (htm) 1559 printf("Enabled\t\t"); 1560 else 1561 printf("Disabled\t"); 1562 printf("Multi-add-lookup\t\t\t\t\t\t%u\n", 1563 rwc_non_lf_results.multi_rw[i][j][k]); 1564 1565 printf("_______\t\t_______\t\t_________\t___" 1566 "\t\t_________\t\t\t\t\t\t" 1567 "_________________\n"); 1568 } 1569 } 1570 } 1571 rte_free(tbl_rwc_test_param.keys); 1572 rte_free(tbl_rwc_test_param.keys_no_ks); 1573 rte_free(tbl_rwc_test_param.keys_ks); 1574 rte_free(tbl_rwc_test_param.keys_absent); 1575 rte_free(tbl_rwc_test_param.keys_shift_path); 1576 rte_free(tbl_rwc_test_param.keys_non_shift_path); 1577 rte_free(tbl_rwc_test_param.keys_ext_bkt); 1578 rte_free(tbl_rwc_test_param.keys_ks_extbkt); 1579 return 0; 1580 } 1581 1582 REGISTER_PERF_TEST(hash_readwrite_lf_perf_autotest, 1583 test_hash_readwrite_lf_perf_main); 1584