1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdint.h> 7 #include <inttypes.h> 8 #include <unistd.h> 9 #include <sys/queue.h> 10 #include <string.h> 11 12 #include <rte_common.h> 13 #include <rte_memory.h> 14 #include <rte_per_lcore.h> 15 #include <rte_launch.h> 16 #include <rte_atomic.h> 17 #include <rte_rwlock.h> 18 #include <rte_eal.h> 19 #include <rte_lcore.h> 20 #include <rte_cycles.h> 21 22 #include "test.h" 23 24 /* 25 * rwlock test 26 * =========== 27 * Provides UT for rte_rwlock API. 28 * Main concern is on functional testing, but also provides some 29 * performance measurements. 30 * Obviously for proper testing need to be executed with more than one lcore. 31 */ 32 33 #define ITER_NUM 0x80 34 35 #define TEST_SEC 5 36 37 static rte_rwlock_t sl; 38 static rte_rwlock_t sl_tab[RTE_MAX_LCORE]; 39 static rte_atomic32_t synchro; 40 41 enum { 42 LC_TYPE_RDLOCK, 43 LC_TYPE_WRLOCK, 44 }; 45 46 static struct { 47 rte_rwlock_t lock; 48 uint64_t tick; 49 50 volatile union { 51 uint8_t u8[RTE_CACHE_LINE_SIZE]; 52 uint64_t u64[RTE_CACHE_LINE_SIZE / sizeof(uint64_t)]; 53 } data; 54 } __rte_cache_aligned try_rwlock_data; 55 56 struct try_rwlock_lcore { 57 int32_t rc; 58 int32_t type; 59 struct { 60 uint64_t tick; 61 uint64_t fail; 62 uint64_t success; 63 } stat; 64 } __rte_cache_aligned; 65 66 static struct try_rwlock_lcore try_lcore_data[RTE_MAX_LCORE]; 67 68 static int 69 test_rwlock_per_core(__rte_unused void *arg) 70 { 71 rte_rwlock_write_lock(&sl); 72 printf("Global write lock taken on core %u\n", rte_lcore_id()); 73 rte_rwlock_write_unlock(&sl); 74 75 rte_rwlock_write_lock(&sl_tab[rte_lcore_id()]); 76 printf("Hello from core %u !\n", rte_lcore_id()); 77 rte_rwlock_write_unlock(&sl_tab[rte_lcore_id()]); 78 79 rte_rwlock_read_lock(&sl); 80 printf("Global read lock taken on core %u\n", rte_lcore_id()); 81 rte_delay_ms(100); 82 printf("Release global read lock on core %u\n", rte_lcore_id()); 83 rte_rwlock_read_unlock(&sl); 84 85 return 0; 86 } 87 88 static rte_rwlock_t lk = RTE_RWLOCK_INITIALIZER; 89 static volatile uint64_t rwlock_data; 90 static uint64_t time_count[RTE_MAX_LCORE] = {0}; 91 92 #define MAX_LOOP 10000 93 #define TEST_RWLOCK_DEBUG 0 94 95 static int 96 load_loop_fn(__rte_unused void *arg) 97 { 98 uint64_t time_diff = 0, begin; 99 uint64_t hz = rte_get_timer_hz(); 100 uint64_t lcount = 0; 101 const unsigned int lcore = rte_lcore_id(); 102 103 /* wait synchro for workers */ 104 if (lcore != rte_get_main_lcore()) 105 while (rte_atomic32_read(&synchro) == 0) 106 ; 107 108 begin = rte_rdtsc_precise(); 109 while (lcount < MAX_LOOP) { 110 rte_rwlock_write_lock(&lk); 111 ++rwlock_data; 112 rte_rwlock_write_unlock(&lk); 113 114 rte_rwlock_read_lock(&lk); 115 if (TEST_RWLOCK_DEBUG && !(lcount % 100)) 116 printf("Core [%u] rwlock_data = %"PRIu64"\n", 117 lcore, rwlock_data); 118 rte_rwlock_read_unlock(&lk); 119 120 lcount++; 121 /* delay to make lock duty cycle slightly realistic */ 122 rte_pause(); 123 } 124 125 time_diff = rte_rdtsc_precise() - begin; 126 time_count[lcore] = time_diff * 1000000 / hz; 127 return 0; 128 } 129 130 static int 131 test_rwlock_perf(void) 132 { 133 unsigned int i; 134 uint64_t total = 0; 135 136 printf("\nRwlock Perf Test on %u cores...\n", rte_lcore_count()); 137 138 /* clear synchro and start workers */ 139 rte_atomic32_set(&synchro, 0); 140 if (rte_eal_mp_remote_launch(load_loop_fn, NULL, SKIP_MAIN) < 0) 141 return -1; 142 143 /* start synchro and launch test on main */ 144 rte_atomic32_set(&synchro, 1); 145 load_loop_fn(NULL); 146 147 rte_eal_mp_wait_lcore(); 148 149 RTE_LCORE_FOREACH(i) { 150 printf("Core [%u] cost time = %"PRIu64" us\n", 151 i, time_count[i]); 152 total += time_count[i]; 153 } 154 155 printf("Total cost time = %"PRIu64" us\n", total); 156 memset(time_count, 0, sizeof(time_count)); 157 158 return 0; 159 } 160 161 /* 162 * - There is a global rwlock and a table of rwlocks (one per lcore). 163 * 164 * - The test function takes all of these locks and launches the 165 * ``test_rwlock_per_core()`` function on each core (except the main). 166 * 167 * - The function takes the global write lock, display something, 168 * then releases the global lock. 169 * - Then, it takes the per-lcore write lock, display something, and 170 * releases the per-core lock. 171 * - Finally, a read lock is taken during 100 ms, then released. 172 * 173 * - The main function unlocks the per-lcore locks sequentially and 174 * waits between each lock. This triggers the display of a message 175 * for each core, in the correct order. 176 * 177 * Then, it tries to take the global write lock and display the last 178 * message. The autotest script checks that the message order is correct. 179 */ 180 static int 181 rwlock_test1(void) 182 { 183 int i; 184 185 rte_rwlock_init(&sl); 186 for (i = 0; i < RTE_MAX_LCORE; i++) 187 rte_rwlock_init(&sl_tab[i]); 188 189 rte_rwlock_write_lock(&sl); 190 191 RTE_LCORE_FOREACH_WORKER(i) { 192 rte_rwlock_write_lock(&sl_tab[i]); 193 rte_eal_remote_launch(test_rwlock_per_core, NULL, i); 194 } 195 196 rte_rwlock_write_unlock(&sl); 197 198 RTE_LCORE_FOREACH_WORKER(i) { 199 rte_rwlock_write_unlock(&sl_tab[i]); 200 rte_delay_ms(100); 201 } 202 203 rte_rwlock_write_lock(&sl); 204 /* this message should be the last message of test */ 205 printf("Global write lock taken on main core %u\n", rte_lcore_id()); 206 rte_rwlock_write_unlock(&sl); 207 208 rte_eal_mp_wait_lcore(); 209 210 if (test_rwlock_perf() < 0) 211 return -1; 212 213 return 0; 214 } 215 216 static int 217 try_read(uint32_t lc) 218 { 219 int32_t rc; 220 uint32_t i; 221 222 rc = rte_rwlock_read_trylock(&try_rwlock_data.lock); 223 if (rc != 0) 224 return rc; 225 226 for (i = 0; i != RTE_DIM(try_rwlock_data.data.u64); i++) { 227 228 /* race condition occurred, lock doesn't work properly */ 229 if (try_rwlock_data.data.u64[i] != 0) { 230 printf("%s(%u) error: unexpected data pattern\n", 231 __func__, lc); 232 rte_memdump(stdout, NULL, 233 (void *)(uintptr_t)&try_rwlock_data.data, 234 sizeof(try_rwlock_data.data)); 235 rc = -EFAULT; 236 break; 237 } 238 } 239 240 rte_rwlock_read_unlock(&try_rwlock_data.lock); 241 return rc; 242 } 243 244 static int 245 try_write(uint32_t lc) 246 { 247 int32_t rc; 248 uint32_t i, v; 249 250 v = RTE_MAX(lc % UINT8_MAX, 1U); 251 252 rc = rte_rwlock_write_trylock(&try_rwlock_data.lock); 253 if (rc != 0) 254 return rc; 255 256 /* update by bytes in reverse order */ 257 for (i = RTE_DIM(try_rwlock_data.data.u8); i-- != 0; ) { 258 259 /* race condition occurred, lock doesn't work properly */ 260 if (try_rwlock_data.data.u8[i] != 0) { 261 printf("%s:%d(%u) error: unexpected data pattern\n", 262 __func__, __LINE__, lc); 263 rte_memdump(stdout, NULL, 264 (void *)(uintptr_t)&try_rwlock_data.data, 265 sizeof(try_rwlock_data.data)); 266 rc = -EFAULT; 267 break; 268 } 269 270 try_rwlock_data.data.u8[i] = v; 271 } 272 273 /* restore by bytes in reverse order */ 274 for (i = RTE_DIM(try_rwlock_data.data.u8); i-- != 0; ) { 275 276 /* race condition occurred, lock doesn't work properly */ 277 if (try_rwlock_data.data.u8[i] != v) { 278 printf("%s:%d(%u) error: unexpected data pattern\n", 279 __func__, __LINE__, lc); 280 rte_memdump(stdout, NULL, 281 (void *)(uintptr_t)&try_rwlock_data.data, 282 sizeof(try_rwlock_data.data)); 283 rc = -EFAULT; 284 break; 285 } 286 287 try_rwlock_data.data.u8[i] = 0; 288 } 289 290 rte_rwlock_write_unlock(&try_rwlock_data.lock); 291 return rc; 292 } 293 294 static int 295 try_read_lcore(__rte_unused void *data) 296 { 297 int32_t rc; 298 uint32_t i, lc; 299 uint64_t ftm, stm, tm; 300 struct try_rwlock_lcore *lcd; 301 302 lc = rte_lcore_id(); 303 lcd = try_lcore_data + lc; 304 lcd->type = LC_TYPE_RDLOCK; 305 306 ftm = try_rwlock_data.tick; 307 stm = rte_get_timer_cycles(); 308 309 do { 310 for (i = 0; i != ITER_NUM; i++) { 311 rc = try_read(lc); 312 if (rc == 0) 313 lcd->stat.success++; 314 else if (rc == -EBUSY) 315 lcd->stat.fail++; 316 else 317 break; 318 rc = 0; 319 } 320 tm = rte_get_timer_cycles() - stm; 321 } while (tm < ftm && rc == 0); 322 323 lcd->rc = rc; 324 lcd->stat.tick = tm; 325 return rc; 326 } 327 328 static int 329 try_write_lcore(__rte_unused void *data) 330 { 331 int32_t rc; 332 uint32_t i, lc; 333 uint64_t ftm, stm, tm; 334 struct try_rwlock_lcore *lcd; 335 336 lc = rte_lcore_id(); 337 lcd = try_lcore_data + lc; 338 lcd->type = LC_TYPE_WRLOCK; 339 340 ftm = try_rwlock_data.tick; 341 stm = rte_get_timer_cycles(); 342 343 do { 344 for (i = 0; i != ITER_NUM; i++) { 345 rc = try_write(lc); 346 if (rc == 0) 347 lcd->stat.success++; 348 else if (rc == -EBUSY) 349 lcd->stat.fail++; 350 else 351 break; 352 rc = 0; 353 } 354 tm = rte_get_timer_cycles() - stm; 355 } while (tm < ftm && rc == 0); 356 357 lcd->rc = rc; 358 lcd->stat.tick = tm; 359 return rc; 360 } 361 362 static void 363 print_try_lcore_stats(const struct try_rwlock_lcore *tlc, uint32_t lc) 364 { 365 uint64_t f, s; 366 367 f = RTE_MAX(tlc->stat.fail, 1ULL); 368 s = RTE_MAX(tlc->stat.success, 1ULL); 369 370 printf("try_lcore_data[%u]={\n" 371 "\trc=%d,\n" 372 "\ttype=%s,\n" 373 "\tfail=%" PRIu64 ",\n" 374 "\tsuccess=%" PRIu64 ",\n" 375 "\tcycles=%" PRIu64 ",\n" 376 "\tcycles/op=%#Lf,\n" 377 "\tcycles/success=%#Lf,\n" 378 "\tsuccess/fail=%#Lf,\n" 379 "};\n", 380 lc, 381 tlc->rc, 382 tlc->type == LC_TYPE_RDLOCK ? "RDLOCK" : "WRLOCK", 383 tlc->stat.fail, 384 tlc->stat.success, 385 tlc->stat.tick, 386 (long double)tlc->stat.tick / 387 (tlc->stat.fail + tlc->stat.success), 388 (long double)tlc->stat.tick / s, 389 (long double)tlc->stat.success / f); 390 } 391 392 static void 393 collect_try_lcore_stats(struct try_rwlock_lcore *tlc, 394 const struct try_rwlock_lcore *lc) 395 { 396 tlc->stat.tick += lc->stat.tick; 397 tlc->stat.fail += lc->stat.fail; 398 tlc->stat.success += lc->stat.success; 399 } 400 401 /* 402 * Process collected results: 403 * - check status 404 * - collect and print statistics 405 */ 406 static int 407 process_try_lcore_stats(void) 408 { 409 int32_t rc; 410 uint32_t lc, rd, wr; 411 struct try_rwlock_lcore rlc, wlc; 412 413 memset(&rlc, 0, sizeof(rlc)); 414 memset(&wlc, 0, sizeof(wlc)); 415 416 rlc.type = LC_TYPE_RDLOCK; 417 wlc.type = LC_TYPE_WRLOCK; 418 rd = 0; 419 wr = 0; 420 421 rc = 0; 422 RTE_LCORE_FOREACH(lc) { 423 rc |= try_lcore_data[lc].rc; 424 if (try_lcore_data[lc].type == LC_TYPE_RDLOCK) { 425 collect_try_lcore_stats(&rlc, try_lcore_data + lc); 426 rd++; 427 } else { 428 collect_try_lcore_stats(&wlc, try_lcore_data + lc); 429 wr++; 430 } 431 } 432 433 if (rc == 0) { 434 RTE_LCORE_FOREACH(lc) 435 print_try_lcore_stats(try_lcore_data + lc, lc); 436 437 if (rd != 0) { 438 printf("aggregated stats for %u RDLOCK cores:\n", rd); 439 print_try_lcore_stats(&rlc, rd); 440 } 441 442 if (wr != 0) { 443 printf("aggregated stats for %u WRLOCK cores:\n", wr); 444 print_try_lcore_stats(&wlc, wr); 445 } 446 } 447 448 return rc; 449 } 450 451 static void 452 try_test_reset(void) 453 { 454 memset(&try_lcore_data, 0, sizeof(try_lcore_data)); 455 memset(&try_rwlock_data, 0, sizeof(try_rwlock_data)); 456 try_rwlock_data.tick = TEST_SEC * rte_get_tsc_hz(); 457 } 458 459 /* all lcores grab RDLOCK */ 460 static int 461 try_rwlock_test_rda(void) 462 { 463 try_test_reset(); 464 465 /* start read test on all available lcores */ 466 rte_eal_mp_remote_launch(try_read_lcore, NULL, CALL_MAIN); 467 rte_eal_mp_wait_lcore(); 468 469 return process_try_lcore_stats(); 470 } 471 472 /* all worker lcores grab RDLOCK, main one grabs WRLOCK */ 473 static int 474 try_rwlock_test_rds_wrm(void) 475 { 476 try_test_reset(); 477 478 rte_eal_mp_remote_launch(try_read_lcore, NULL, SKIP_MAIN); 479 try_write_lcore(NULL); 480 rte_eal_mp_wait_lcore(); 481 482 return process_try_lcore_stats(); 483 } 484 485 /* main and even worker lcores grab RDLOCK, odd lcores grab WRLOCK */ 486 static int 487 try_rwlock_test_rde_wro(void) 488 { 489 uint32_t lc, mlc; 490 491 try_test_reset(); 492 493 mlc = rte_get_main_lcore(); 494 495 RTE_LCORE_FOREACH(lc) { 496 if (lc != mlc) { 497 if ((lc & 1) == 0) 498 rte_eal_remote_launch(try_read_lcore, 499 NULL, lc); 500 else 501 rte_eal_remote_launch(try_write_lcore, 502 NULL, lc); 503 } 504 } 505 try_read_lcore(NULL); 506 rte_eal_mp_wait_lcore(); 507 508 return process_try_lcore_stats(); 509 } 510 511 static int 512 test_rwlock(void) 513 { 514 uint32_t i; 515 int32_t rc, ret; 516 517 static const struct { 518 const char *name; 519 int (*ftst)(void); 520 } test[] = { 521 { 522 .name = "rwlock_test1", 523 .ftst = rwlock_test1, 524 }, 525 { 526 .name = "try_rwlock_test_rda", 527 .ftst = try_rwlock_test_rda, 528 }, 529 { 530 .name = "try_rwlock_test_rds_wrm", 531 .ftst = try_rwlock_test_rds_wrm, 532 }, 533 { 534 .name = "try_rwlock_test_rde_wro", 535 .ftst = try_rwlock_test_rde_wro, 536 }, 537 }; 538 539 ret = 0; 540 for (i = 0; i != RTE_DIM(test); i++) { 541 printf("starting test %s;\n", test[i].name); 542 rc = test[i].ftst(); 543 printf("test %s completed with status %d\n", test[i].name, rc); 544 ret |= rc; 545 } 546 547 return ret; 548 } 549 550 REGISTER_TEST_COMMAND(rwlock_autotest, test_rwlock); 551 552 /* subtests used in meson for CI */ 553 REGISTER_TEST_COMMAND(rwlock_test1_autotest, rwlock_test1); 554 REGISTER_TEST_COMMAND(rwlock_rda_autotest, try_rwlock_test_rda); 555 REGISTER_TEST_COMMAND(rwlock_rds_wrm_autotest, try_rwlock_test_rds_wrm); 556 REGISTER_TEST_COMMAND(rwlock_rde_wro_autotest, try_rwlock_test_rde_wro); 557