1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 #include <inttypes.h> 6 #include <locale.h> 7 8 #include <rte_cycles.h> 9 #include <rte_hash.h> 10 #include <rte_hash_crc.h> 11 #include <rte_jhash.h> 12 #include <rte_launch.h> 13 #include <rte_malloc.h> 14 #include <rte_random.h> 15 #include <rte_spinlock.h> 16 17 #include "test.h" 18 19 #define RTE_RWTEST_FAIL 0 20 21 #define TOTAL_ENTRY (5*1024*1024) 22 #define TOTAL_INSERT (4.5*1024*1024) 23 #define TOTAL_INSERT_EXT (5*1024*1024) 24 25 #define NUM_TEST 3 26 unsigned int core_cnt[NUM_TEST] = {2, 4, 8}; 27 28 unsigned int slave_core_ids[RTE_MAX_LCORE]; 29 struct perf { 30 uint32_t single_read; 31 uint32_t single_write; 32 uint32_t read_only[NUM_TEST]; 33 uint32_t write_only[NUM_TEST]; 34 uint32_t read_write_r[NUM_TEST]; 35 uint32_t read_write_w[NUM_TEST]; 36 }; 37 38 static struct perf htm_results, non_htm_results; 39 40 struct { 41 uint32_t *keys; 42 uint8_t *found; 43 uint32_t num_insert; 44 uint32_t rounded_tot_insert; 45 struct rte_hash *h; 46 } tbl_rw_test_param; 47 48 static rte_atomic64_t gcycles; 49 static rte_atomic64_t ginsertions; 50 51 static rte_atomic64_t gread_cycles; 52 static rte_atomic64_t gwrite_cycles; 53 54 static rte_atomic64_t greads; 55 static rte_atomic64_t gwrites; 56 57 static int 58 test_hash_readwrite_worker(__attribute__((unused)) void *arg) 59 { 60 uint64_t i, offset; 61 uint32_t lcore_id = rte_lcore_id(); 62 uint64_t begin, cycles; 63 int *ret; 64 65 ret = rte_malloc(NULL, sizeof(int) * 66 tbl_rw_test_param.num_insert, 0); 67 for (i = 0; i < rte_lcore_count(); i++) { 68 if (slave_core_ids[i] == lcore_id) 69 break; 70 } 71 offset = tbl_rw_test_param.num_insert * i; 72 73 printf("Core #%d inserting and reading %d: %'"PRId64" - %'"PRId64"\n", 74 lcore_id, tbl_rw_test_param.num_insert, 75 offset, offset + tbl_rw_test_param.num_insert - 1); 76 77 begin = rte_rdtsc_precise(); 78 79 for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) { 80 81 if (rte_hash_lookup(tbl_rw_test_param.h, 82 tbl_rw_test_param.keys + i) > 0) 83 break; 84 85 ret[i - offset] = rte_hash_add_key(tbl_rw_test_param.h, 86 tbl_rw_test_param.keys + i); 87 if (ret[i - offset] < 0) 88 break; 89 90 /* lookup a random key */ 91 uint32_t rand = rte_rand() % (i + 1 - offset); 92 93 if (rte_hash_lookup(tbl_rw_test_param.h, 94 tbl_rw_test_param.keys + rand) != ret[rand]) 95 break; 96 97 98 if (rte_hash_del_key(tbl_rw_test_param.h, 99 tbl_rw_test_param.keys + rand) != ret[rand]) 100 break; 101 102 ret[rand] = rte_hash_add_key(tbl_rw_test_param.h, 103 tbl_rw_test_param.keys + rand); 104 if (ret[rand] < 0) 105 break; 106 107 if (rte_hash_lookup(tbl_rw_test_param.h, 108 tbl_rw_test_param.keys + rand) != ret[rand]) 109 break; 110 } 111 112 cycles = rte_rdtsc_precise() - begin; 113 rte_atomic64_add(&gcycles, cycles); 114 rte_atomic64_add(&ginsertions, i - offset); 115 116 for (; i < offset + tbl_rw_test_param.num_insert; i++) 117 tbl_rw_test_param.keys[i] = RTE_RWTEST_FAIL; 118 119 rte_free(ret); 120 return 0; 121 } 122 123 static int 124 init_params(int use_ext, int use_htm, int use_jhash) 125 { 126 unsigned int i; 127 128 uint32_t *keys = NULL; 129 uint8_t *found = NULL; 130 struct rte_hash *handle; 131 132 struct rte_hash_parameters hash_params = { 133 .entries = TOTAL_ENTRY, 134 .key_len = sizeof(uint32_t), 135 .hash_func_init_val = 0, 136 .socket_id = rte_socket_id(), 137 }; 138 if (use_jhash) 139 hash_params.hash_func = rte_jhash; 140 else 141 hash_params.hash_func = rte_hash_crc; 142 143 if (use_htm) 144 hash_params.extra_flag = 145 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT | 146 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | 147 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 148 else 149 hash_params.extra_flag = 150 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | 151 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; 152 153 if (use_ext) 154 hash_params.extra_flag |= 155 RTE_HASH_EXTRA_FLAGS_EXT_TABLE; 156 else 157 hash_params.extra_flag &= 158 ~RTE_HASH_EXTRA_FLAGS_EXT_TABLE; 159 160 hash_params.name = "tests"; 161 162 handle = rte_hash_create(&hash_params); 163 if (handle == NULL) { 164 printf("hash creation failed"); 165 return -1; 166 } 167 168 tbl_rw_test_param.h = handle; 169 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0); 170 171 if (keys == NULL) { 172 printf("RTE_MALLOC failed\n"); 173 goto err; 174 } 175 176 found = rte_zmalloc(NULL, sizeof(uint8_t) * TOTAL_ENTRY, 0); 177 if (found == NULL) { 178 printf("RTE_ZMALLOC failed\n"); 179 goto err; 180 } 181 182 tbl_rw_test_param.keys = keys; 183 tbl_rw_test_param.found = found; 184 185 for (i = 0; i < TOTAL_ENTRY; i++) 186 keys[i] = i; 187 188 return 0; 189 190 err: 191 rte_free(keys); 192 rte_hash_free(handle); 193 194 return -1; 195 } 196 197 static int 198 test_hash_readwrite_functional(int use_ext, int use_htm) 199 { 200 unsigned int i; 201 const void *next_key; 202 void *next_data; 203 uint32_t iter = 0; 204 205 uint32_t duplicated_keys = 0; 206 uint32_t lost_keys = 0; 207 int use_jhash = 1; 208 int slave_cnt = rte_lcore_count() - 1; 209 uint32_t tot_insert = 0; 210 211 rte_atomic64_init(&gcycles); 212 rte_atomic64_clear(&gcycles); 213 214 rte_atomic64_init(&ginsertions); 215 rte_atomic64_clear(&ginsertions); 216 217 if (init_params(use_ext, use_htm, use_jhash) != 0) 218 goto err; 219 220 if (use_ext) 221 tot_insert = TOTAL_INSERT_EXT; 222 else 223 tot_insert = TOTAL_INSERT; 224 225 tbl_rw_test_param.num_insert = 226 tot_insert / slave_cnt; 227 228 tbl_rw_test_param.rounded_tot_insert = 229 tbl_rw_test_param.num_insert 230 * slave_cnt; 231 232 printf("++++++++Start function tests:+++++++++\n"); 233 234 /* Fire all threads. */ 235 rte_eal_mp_remote_launch(test_hash_readwrite_worker, 236 NULL, SKIP_MASTER); 237 rte_eal_mp_wait_lcore(); 238 239 while (rte_hash_iterate(tbl_rw_test_param.h, &next_key, 240 &next_data, &iter) >= 0) { 241 /* Search for the key in the list of keys added .*/ 242 i = *(const uint32_t *)next_key; 243 tbl_rw_test_param.found[i]++; 244 } 245 246 for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) { 247 if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) { 248 if (tbl_rw_test_param.found[i] > 1) { 249 duplicated_keys++; 250 break; 251 } 252 if (tbl_rw_test_param.found[i] == 0) { 253 lost_keys++; 254 printf("key %d is lost\n", i); 255 break; 256 } 257 } 258 } 259 260 if (duplicated_keys > 0) { 261 printf("%d key duplicated\n", duplicated_keys); 262 goto err_free; 263 } 264 265 if (lost_keys > 0) { 266 printf("%d key lost\n", lost_keys); 267 goto err_free; 268 } 269 270 printf("No key corrupted during read-write test.\n"); 271 272 unsigned long long int cycles_per_insertion = 273 rte_atomic64_read(&gcycles) / 274 rte_atomic64_read(&ginsertions); 275 276 printf("cycles per insertion and lookup: %llu\n", cycles_per_insertion); 277 278 rte_free(tbl_rw_test_param.found); 279 rte_free(tbl_rw_test_param.keys); 280 rte_hash_free(tbl_rw_test_param.h); 281 printf("+++++++++Complete function tests+++++++++\n"); 282 return 0; 283 284 err_free: 285 rte_free(tbl_rw_test_param.found); 286 rte_free(tbl_rw_test_param.keys); 287 rte_hash_free(tbl_rw_test_param.h); 288 err: 289 return -1; 290 } 291 292 static int 293 test_rw_reader(void *arg) 294 { 295 uint64_t i; 296 uint64_t begin, cycles; 297 uint64_t read_cnt = (uint64_t)((uintptr_t)arg); 298 299 begin = rte_rdtsc_precise(); 300 for (i = 0; i < read_cnt; i++) { 301 void *data; 302 rte_hash_lookup_data(tbl_rw_test_param.h, 303 tbl_rw_test_param.keys + i, 304 &data); 305 if (i != (uint64_t)(uintptr_t)data) { 306 printf("lookup find wrong value %"PRIu64"," 307 "%"PRIu64"\n", i, 308 (uint64_t)(uintptr_t)data); 309 break; 310 } 311 } 312 313 cycles = rte_rdtsc_precise() - begin; 314 rte_atomic64_add(&gread_cycles, cycles); 315 rte_atomic64_add(&greads, i); 316 return 0; 317 } 318 319 static int 320 test_rw_writer(void *arg) 321 { 322 uint64_t i; 323 uint32_t lcore_id = rte_lcore_id(); 324 uint64_t begin, cycles; 325 int ret; 326 uint64_t start_coreid = (uint64_t)(uintptr_t)arg; 327 uint64_t offset; 328 329 for (i = 0; i < rte_lcore_count(); i++) { 330 if (slave_core_ids[i] == lcore_id) 331 break; 332 } 333 334 offset = TOTAL_INSERT / 2 + (i - (start_coreid)) * 335 tbl_rw_test_param.num_insert; 336 begin = rte_rdtsc_precise(); 337 for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) { 338 ret = rte_hash_add_key_data(tbl_rw_test_param.h, 339 tbl_rw_test_param.keys + i, 340 (void *)((uintptr_t)i)); 341 if (ret < 0) { 342 printf("writer failed %"PRIu64"\n", i); 343 break; 344 } 345 } 346 347 cycles = rte_rdtsc_precise() - begin; 348 rte_atomic64_add(&gwrite_cycles, cycles); 349 rte_atomic64_add(&gwrites, tbl_rw_test_param.num_insert); 350 return 0; 351 } 352 353 static int 354 test_hash_readwrite_perf(struct perf *perf_results, int use_htm, 355 int reader_faster) 356 { 357 unsigned int n; 358 int ret; 359 int start_coreid; 360 uint64_t i, read_cnt; 361 362 const void *next_key; 363 void *next_data; 364 uint32_t iter; 365 int use_jhash = 0; 366 367 uint32_t duplicated_keys = 0; 368 uint32_t lost_keys = 0; 369 370 uint64_t start = 0, end = 0; 371 372 rte_atomic64_init(&greads); 373 rte_atomic64_init(&gwrites); 374 rte_atomic64_clear(&gwrites); 375 rte_atomic64_clear(&greads); 376 377 rte_atomic64_init(&gread_cycles); 378 rte_atomic64_clear(&gread_cycles); 379 rte_atomic64_init(&gwrite_cycles); 380 rte_atomic64_clear(&gwrite_cycles); 381 382 if (init_params(0, use_htm, use_jhash) != 0) 383 goto err; 384 385 /* 386 * Do a readers finish faster or writers finish faster test. 387 * When readers finish faster, we timing the readers, and when writers 388 * finish faster, we timing the writers. 389 * Divided by 10 or 2 is just experimental values to vary the workload 390 * of readers. 391 */ 392 if (reader_faster) { 393 printf("++++++Start perf test: reader++++++++\n"); 394 read_cnt = TOTAL_INSERT / 10; 395 } else { 396 printf("++++++Start perf test: writer++++++++\n"); 397 read_cnt = TOTAL_INSERT / 2; 398 } 399 400 /* We first test single thread performance */ 401 start = rte_rdtsc_precise(); 402 /* Insert half of the keys */ 403 for (i = 0; i < TOTAL_INSERT / 2; i++) { 404 ret = rte_hash_add_key_data(tbl_rw_test_param.h, 405 tbl_rw_test_param.keys + i, 406 (void *)((uintptr_t)i)); 407 if (ret < 0) { 408 printf("Failed to insert half of keys\n"); 409 goto err_free; 410 } 411 } 412 end = rte_rdtsc_precise() - start; 413 perf_results->single_write = end / i; 414 415 start = rte_rdtsc_precise(); 416 417 for (i = 0; i < read_cnt; i++) { 418 void *data; 419 rte_hash_lookup_data(tbl_rw_test_param.h, 420 tbl_rw_test_param.keys + i, 421 &data); 422 if (i != (uint64_t)(uintptr_t)data) { 423 printf("lookup find wrong value" 424 " %"PRIu64",%"PRIu64"\n", i, 425 (uint64_t)(uintptr_t)data); 426 break; 427 } 428 } 429 end = rte_rdtsc_precise() - start; 430 perf_results->single_read = end / i; 431 432 for (n = 0; n < NUM_TEST; n++) { 433 unsigned int tot_slave_lcore = rte_lcore_count() - 1; 434 if (tot_slave_lcore < core_cnt[n] * 2) 435 goto finish; 436 437 rte_atomic64_clear(&greads); 438 rte_atomic64_clear(&gread_cycles); 439 rte_atomic64_clear(&gwrites); 440 rte_atomic64_clear(&gwrite_cycles); 441 442 rte_hash_reset(tbl_rw_test_param.h); 443 444 tbl_rw_test_param.num_insert = TOTAL_INSERT / 2 / core_cnt[n]; 445 tbl_rw_test_param.rounded_tot_insert = TOTAL_INSERT / 2 + 446 tbl_rw_test_param.num_insert * 447 core_cnt[n]; 448 449 for (i = 0; i < TOTAL_INSERT / 2; i++) { 450 ret = rte_hash_add_key_data(tbl_rw_test_param.h, 451 tbl_rw_test_param.keys + i, 452 (void *)((uintptr_t)i)); 453 if (ret < 0) { 454 printf("Failed to insert half of keys\n"); 455 goto err_free; 456 } 457 } 458 459 /* Then test multiple thread case but only all reads or 460 * all writes 461 */ 462 463 /* Test only reader cases */ 464 for (i = 0; i < core_cnt[n]; i++) 465 rte_eal_remote_launch(test_rw_reader, 466 (void *)(uintptr_t)read_cnt, 467 slave_core_ids[i]); 468 469 rte_eal_mp_wait_lcore(); 470 471 start_coreid = i; 472 /* Test only writer cases */ 473 for (; i < core_cnt[n] * 2; i++) 474 rte_eal_remote_launch(test_rw_writer, 475 (void *)((uintptr_t)start_coreid), 476 slave_core_ids[i]); 477 478 rte_eal_mp_wait_lcore(); 479 480 if (reader_faster) { 481 unsigned long long int cycles_per_insertion = 482 rte_atomic64_read(&gread_cycles) / 483 rte_atomic64_read(&greads); 484 perf_results->read_only[n] = cycles_per_insertion; 485 printf("Reader only: cycles per lookup: %llu\n", 486 cycles_per_insertion); 487 } 488 489 else { 490 unsigned long long int cycles_per_insertion = 491 rte_atomic64_read(&gwrite_cycles) / 492 rte_atomic64_read(&gwrites); 493 perf_results->write_only[n] = cycles_per_insertion; 494 printf("Writer only: cycles per writes: %llu\n", 495 cycles_per_insertion); 496 } 497 498 rte_atomic64_clear(&greads); 499 rte_atomic64_clear(&gread_cycles); 500 rte_atomic64_clear(&gwrites); 501 rte_atomic64_clear(&gwrite_cycles); 502 503 rte_hash_reset(tbl_rw_test_param.h); 504 505 for (i = 0; i < TOTAL_INSERT / 2; i++) { 506 ret = rte_hash_add_key_data(tbl_rw_test_param.h, 507 tbl_rw_test_param.keys + i, 508 (void *)((uintptr_t)i)); 509 if (ret < 0) { 510 printf("Failed to insert half of keys\n"); 511 goto err_free; 512 } 513 } 514 515 start_coreid = core_cnt[n]; 516 517 if (reader_faster) { 518 for (i = core_cnt[n]; i < core_cnt[n] * 2; i++) 519 rte_eal_remote_launch(test_rw_writer, 520 (void *)((uintptr_t)start_coreid), 521 slave_core_ids[i]); 522 for (i = 0; i < core_cnt[n]; i++) 523 rte_eal_remote_launch(test_rw_reader, 524 (void *)(uintptr_t)read_cnt, 525 slave_core_ids[i]); 526 } else { 527 for (i = 0; i < core_cnt[n]; i++) 528 rte_eal_remote_launch(test_rw_reader, 529 (void *)(uintptr_t)read_cnt, 530 slave_core_ids[i]); 531 for (; i < core_cnt[n] * 2; i++) 532 rte_eal_remote_launch(test_rw_writer, 533 (void *)((uintptr_t)start_coreid), 534 slave_core_ids[i]); 535 } 536 537 rte_eal_mp_wait_lcore(); 538 539 iter = 0; 540 memset(tbl_rw_test_param.found, 0, TOTAL_ENTRY); 541 while (rte_hash_iterate(tbl_rw_test_param.h, 542 &next_key, &next_data, &iter) >= 0) { 543 /* Search for the key in the list of keys added .*/ 544 i = *(const uint32_t *)next_key; 545 tbl_rw_test_param.found[i]++; 546 } 547 548 for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) { 549 if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) { 550 if (tbl_rw_test_param.found[i] > 1) { 551 duplicated_keys++; 552 break; 553 } 554 if (tbl_rw_test_param.found[i] == 0) { 555 lost_keys++; 556 printf("key %"PRIu64" is lost\n", i); 557 break; 558 } 559 } 560 } 561 562 if (duplicated_keys > 0) { 563 printf("%d key duplicated\n", duplicated_keys); 564 goto err_free; 565 } 566 567 if (lost_keys > 0) { 568 printf("%d key lost\n", lost_keys); 569 goto err_free; 570 } 571 572 printf("No key corrupted during read-write test.\n"); 573 574 if (reader_faster) { 575 unsigned long long int cycles_per_insertion = 576 rte_atomic64_read(&gread_cycles) / 577 rte_atomic64_read(&greads); 578 perf_results->read_write_r[n] = cycles_per_insertion; 579 printf("Read-write cycles per lookup: %llu\n", 580 cycles_per_insertion); 581 } 582 583 else { 584 unsigned long long int cycles_per_insertion = 585 rte_atomic64_read(&gwrite_cycles) / 586 rte_atomic64_read(&gwrites); 587 perf_results->read_write_w[n] = cycles_per_insertion; 588 printf("Read-write cycles per writes: %llu\n", 589 cycles_per_insertion); 590 } 591 } 592 593 finish: 594 rte_free(tbl_rw_test_param.found); 595 rte_free(tbl_rw_test_param.keys); 596 rte_hash_free(tbl_rw_test_param.h); 597 return 0; 598 599 err_free: 600 rte_free(tbl_rw_test_param.found); 601 rte_free(tbl_rw_test_param.keys); 602 rte_hash_free(tbl_rw_test_param.h); 603 604 err: 605 return -1; 606 } 607 608 static int 609 test_hash_readwrite_main(void) 610 { 611 /* 612 * Variables used to choose different tests. 613 * use_htm indicates if hardware transactional memory should be used. 614 * reader_faster indicates if the reader threads should finish earlier 615 * than writer threads. This is to timing either reader threads or 616 * writer threads for performance numbers. 617 */ 618 int use_htm, use_ext, reader_faster; 619 unsigned int i = 0, core_id = 0; 620 621 if (rte_lcore_count() < 3) { 622 printf("Not enough cores for hash_readwrite_autotest, expecting at least 3\n"); 623 return TEST_SKIPPED; 624 } 625 626 RTE_LCORE_FOREACH_SLAVE(core_id) { 627 slave_core_ids[i] = core_id; 628 i++; 629 } 630 631 setlocale(LC_NUMERIC, ""); 632 633 if (rte_tm_supported()) { 634 printf("Hardware transactional memory (lock elision) " 635 "is supported\n"); 636 637 printf("Test read-write with Hardware transactional memory\n"); 638 639 use_htm = 1; 640 use_ext = 0; 641 642 if (test_hash_readwrite_functional(use_ext, use_htm) < 0) 643 return -1; 644 645 use_ext = 1; 646 if (test_hash_readwrite_functional(use_ext, use_htm) < 0) 647 return -1; 648 649 reader_faster = 1; 650 if (test_hash_readwrite_perf(&htm_results, use_htm, 651 reader_faster) < 0) 652 return -1; 653 654 reader_faster = 0; 655 if (test_hash_readwrite_perf(&htm_results, use_htm, 656 reader_faster) < 0) 657 return -1; 658 } else { 659 printf("Hardware transactional memory (lock elision) " 660 "is NOT supported\n"); 661 } 662 663 printf("Test read-write without Hardware transactional memory\n"); 664 use_htm = 0; 665 use_ext = 0; 666 if (test_hash_readwrite_functional(use_ext, use_htm) < 0) 667 return -1; 668 669 use_ext = 1; 670 if (test_hash_readwrite_functional(use_ext, use_htm) < 0) 671 return -1; 672 673 reader_faster = 1; 674 if (test_hash_readwrite_perf(&non_htm_results, use_htm, 675 reader_faster) < 0) 676 return -1; 677 reader_faster = 0; 678 if (test_hash_readwrite_perf(&non_htm_results, use_htm, 679 reader_faster) < 0) 680 return -1; 681 682 printf("================\n"); 683 printf("Results summary:\n"); 684 printf("================\n"); 685 686 printf("single read: %u\n", htm_results.single_read); 687 printf("single write: %u\n", htm_results.single_write); 688 for (i = 0; i < NUM_TEST; i++) { 689 printf("+++ core_cnt: %u +++\n", core_cnt[i]); 690 printf("HTM:\n"); 691 printf(" read only: %u\n", htm_results.read_only[i]); 692 printf(" write only: %u\n", htm_results.write_only[i]); 693 printf(" read-write read: %u\n", htm_results.read_write_r[i]); 694 printf(" read-write write: %u\n", htm_results.read_write_w[i]); 695 696 printf("non HTM:\n"); 697 printf(" read only: %u\n", non_htm_results.read_only[i]); 698 printf(" write only: %u\n", non_htm_results.write_only[i]); 699 printf(" read-write read: %u\n", 700 non_htm_results.read_write_r[i]); 701 printf(" read-write write: %u\n", 702 non_htm_results.read_write_w[i]); 703 } 704 705 return 0; 706 } 707 708 REGISTER_TEST_COMMAND(hash_readwrite_autotest, test_hash_readwrite_main); 709