1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdint.h> 7 #include <inttypes.h> 8 #include <unistd.h> 9 #include <sys/queue.h> 10 #include <string.h> 11 12 #include <rte_common.h> 13 #include <rte_memory.h> 14 #include <rte_per_lcore.h> 15 #include <rte_launch.h> 16 #include <rte_atomic.h> 17 #include <rte_rwlock.h> 18 #include <rte_eal.h> 19 #include <rte_lcore.h> 20 #include <rte_cycles.h> 21 22 #include "test.h" 23 24 /* 25 * rwlock test 26 * =========== 27 * Provides UT for rte_rwlock API. 28 * Main concern is on functional testing, but also provides some 29 * performance measurements. 30 * Obviously for proper testing need to be executed with more than one lcore. 31 */ 32 33 #define ITER_NUM 0x80 34 35 #define TEST_SEC 5 36 37 static rte_rwlock_t sl; 38 static rte_rwlock_t sl_tab[RTE_MAX_LCORE]; 39 static rte_atomic32_t synchro; 40 41 enum { 42 LC_TYPE_RDLOCK, 43 LC_TYPE_WRLOCK, 44 }; 45 46 static struct { 47 rte_rwlock_t lock; 48 uint64_t tick; 49 volatile union { 50 uint8_t u8[RTE_CACHE_LINE_SIZE]; 51 uint64_t u64[RTE_CACHE_LINE_SIZE / sizeof(uint64_t)]; 52 } data; 53 } __rte_cache_aligned try_rwlock_data; 54 55 struct try_rwlock_lcore { 56 int32_t rc; 57 int32_t type; 58 struct { 59 uint64_t tick; 60 uint64_t fail; 61 uint64_t success; 62 } stat; 63 } __rte_cache_aligned; 64 65 static struct try_rwlock_lcore try_lcore_data[RTE_MAX_LCORE]; 66 67 static int 68 test_rwlock_per_core(__attribute__((unused)) void *arg) 69 { 70 rte_rwlock_write_lock(&sl); 71 printf("Global write lock taken on core %u\n", rte_lcore_id()); 72 rte_rwlock_write_unlock(&sl); 73 74 rte_rwlock_write_lock(&sl_tab[rte_lcore_id()]); 75 printf("Hello from core %u !\n", rte_lcore_id()); 76 rte_rwlock_write_unlock(&sl_tab[rte_lcore_id()]); 77 78 rte_rwlock_read_lock(&sl); 79 printf("Global read lock taken on core %u\n", rte_lcore_id()); 80 rte_delay_ms(100); 81 printf("Release global read lock on core %u\n", rte_lcore_id()); 82 rte_rwlock_read_unlock(&sl); 83 84 return 0; 85 } 86 87 static rte_rwlock_t lk = RTE_RWLOCK_INITIALIZER; 88 static volatile uint64_t rwlock_data; 89 static uint64_t time_count[RTE_MAX_LCORE] = {0}; 90 91 #define MAX_LOOP 10000 92 #define TEST_RWLOCK_DEBUG 0 93 94 static int 95 load_loop_fn(__attribute__((unused)) void *arg) 96 { 97 uint64_t time_diff = 0, begin; 98 uint64_t hz = rte_get_timer_hz(); 99 uint64_t lcount = 0; 100 const unsigned int lcore = rte_lcore_id(); 101 102 /* wait synchro for slaves */ 103 if (lcore != rte_get_master_lcore()) 104 while (rte_atomic32_read(&synchro) == 0) 105 ; 106 107 begin = rte_rdtsc_precise(); 108 while (lcount < MAX_LOOP) { 109 rte_rwlock_write_lock(&lk); 110 ++rwlock_data; 111 rte_rwlock_write_unlock(&lk); 112 113 rte_rwlock_read_lock(&lk); 114 if (TEST_RWLOCK_DEBUG && !(lcount % 100)) 115 printf("Core [%u] rwlock_data = %"PRIu64"\n", 116 lcore, rwlock_data); 117 rte_rwlock_read_unlock(&lk); 118 119 lcount++; 120 /* delay to make lock duty cycle slightly realistic */ 121 rte_pause(); 122 } 123 124 time_diff = rte_rdtsc_precise() - begin; 125 time_count[lcore] = time_diff * 1000000 / hz; 126 return 0; 127 } 128 129 static int 130 test_rwlock_perf(void) 131 { 132 unsigned int i; 133 uint64_t total = 0; 134 135 printf("\nRwlock Perf Test on %u cores...\n", rte_lcore_count()); 136 137 /* clear synchro and start slaves */ 138 rte_atomic32_set(&synchro, 0); 139 if (rte_eal_mp_remote_launch(load_loop_fn, NULL, SKIP_MASTER) < 0) 140 return -1; 141 142 /* start synchro and launch test on master */ 143 rte_atomic32_set(&synchro, 1); 144 load_loop_fn(NULL); 145 146 rte_eal_mp_wait_lcore(); 147 148 RTE_LCORE_FOREACH(i) { 149 printf("Core [%u] cost time = %"PRIu64" us\n", 150 i, time_count[i]); 151 total += time_count[i]; 152 } 153 154 printf("Total cost time = %"PRIu64" us\n", total); 155 memset(time_count, 0, sizeof(time_count)); 156 157 return 0; 158 } 159 160 /* 161 * - There is a global rwlock and a table of rwlocks (one per lcore). 162 * 163 * - The test function takes all of these locks and launches the 164 * ``test_rwlock_per_core()`` function on each core (except the master). 165 * 166 * - The function takes the global write lock, display something, 167 * then releases the global lock. 168 * - Then, it takes the per-lcore write lock, display something, and 169 * releases the per-core lock. 170 * - Finally, a read lock is taken during 100 ms, then released. 171 * 172 * - The main function unlocks the per-lcore locks sequentially and 173 * waits between each lock. This triggers the display of a message 174 * for each core, in the correct order. 175 * 176 * Then, it tries to take the global write lock and display the last 177 * message. The autotest script checks that the message order is correct. 178 */ 179 static int 180 rwlock_test1(void) 181 { 182 int i; 183 184 rte_rwlock_init(&sl); 185 for (i=0; i<RTE_MAX_LCORE; i++) 186 rte_rwlock_init(&sl_tab[i]); 187 188 rte_rwlock_write_lock(&sl); 189 190 RTE_LCORE_FOREACH_SLAVE(i) { 191 rte_rwlock_write_lock(&sl_tab[i]); 192 rte_eal_remote_launch(test_rwlock_per_core, NULL, i); 193 } 194 195 rte_rwlock_write_unlock(&sl); 196 197 RTE_LCORE_FOREACH_SLAVE(i) { 198 rte_rwlock_write_unlock(&sl_tab[i]); 199 rte_delay_ms(100); 200 } 201 202 rte_rwlock_write_lock(&sl); 203 /* this message should be the last message of test */ 204 printf("Global write lock taken on master core %u\n", rte_lcore_id()); 205 rte_rwlock_write_unlock(&sl); 206 207 rte_eal_mp_wait_lcore(); 208 209 if (test_rwlock_perf() < 0) 210 return -1; 211 212 return 0; 213 } 214 215 static int 216 try_read(uint32_t lc) 217 { 218 int32_t rc; 219 uint32_t i; 220 221 rc = rte_rwlock_read_trylock(&try_rwlock_data.lock); 222 if (rc != 0) 223 return rc; 224 225 for (i = 0; i != RTE_DIM(try_rwlock_data.data.u64); i++) { 226 227 /* race condition occurred, lock doesn't work properly */ 228 if (try_rwlock_data.data.u64[i] != 0) { 229 printf("%s(%u) error: unexpected data pattern\n", 230 __func__, lc); 231 rte_memdump(stdout, NULL, 232 (void *)(uintptr_t)&try_rwlock_data.data, 233 sizeof(try_rwlock_data.data)); 234 rc = -EFAULT; 235 break; 236 } 237 } 238 239 rte_rwlock_read_unlock(&try_rwlock_data.lock); 240 return rc; 241 } 242 243 static int 244 try_write(uint32_t lc) 245 { 246 int32_t rc; 247 uint32_t i, v; 248 249 v = RTE_MAX(lc % UINT8_MAX, 1U); 250 251 rc = rte_rwlock_write_trylock(&try_rwlock_data.lock); 252 if (rc != 0) 253 return rc; 254 255 /* update by bytes in reverese order */ 256 for (i = RTE_DIM(try_rwlock_data.data.u8); i-- != 0; ) { 257 258 /* race condition occurred, lock doesn't work properly */ 259 if (try_rwlock_data.data.u8[i] != 0) { 260 printf("%s:%d(%u) error: unexpected data pattern\n", 261 __func__, __LINE__, lc); 262 rte_memdump(stdout, NULL, 263 (void *)(uintptr_t)&try_rwlock_data.data, 264 sizeof(try_rwlock_data.data)); 265 rc = -EFAULT; 266 break; 267 } 268 269 try_rwlock_data.data.u8[i] = v; 270 } 271 272 /* restore by bytes in reverese order */ 273 for (i = RTE_DIM(try_rwlock_data.data.u8); i-- != 0; ) { 274 275 /* race condition occurred, lock doesn't work properly */ 276 if (try_rwlock_data.data.u8[i] != v) { 277 printf("%s:%d(%u) error: unexpected data pattern\n", 278 __func__, __LINE__, lc); 279 rte_memdump(stdout, NULL, 280 (void *)(uintptr_t)&try_rwlock_data.data, 281 sizeof(try_rwlock_data.data)); 282 rc = -EFAULT; 283 break; 284 } 285 286 try_rwlock_data.data.u8[i] = 0; 287 } 288 289 rte_rwlock_write_unlock(&try_rwlock_data.lock); 290 return rc; 291 } 292 293 static int 294 try_read_lcore(__rte_unused void *data) 295 { 296 int32_t rc; 297 uint32_t i, lc; 298 uint64_t ftm, stm, tm; 299 struct try_rwlock_lcore *lcd; 300 301 lc = rte_lcore_id(); 302 lcd = try_lcore_data + lc; 303 lcd->type = LC_TYPE_RDLOCK; 304 305 ftm = try_rwlock_data.tick; 306 stm = rte_get_timer_cycles(); 307 308 do { 309 for (i = 0; i != ITER_NUM; i++) { 310 rc = try_read(lc); 311 if (rc == 0) 312 lcd->stat.success++; 313 else if (rc == -EBUSY) 314 lcd->stat.fail++; 315 else 316 break; 317 rc = 0; 318 } 319 tm = rte_get_timer_cycles() - stm; 320 } while (tm < ftm && rc == 0); 321 322 lcd->rc = rc; 323 lcd->stat.tick = tm; 324 return rc; 325 } 326 327 static int 328 try_write_lcore(__rte_unused void *data) 329 { 330 int32_t rc; 331 uint32_t i, lc; 332 uint64_t ftm, stm, tm; 333 struct try_rwlock_lcore *lcd; 334 335 lc = rte_lcore_id(); 336 lcd = try_lcore_data + lc; 337 lcd->type = LC_TYPE_WRLOCK; 338 339 ftm = try_rwlock_data.tick; 340 stm = rte_get_timer_cycles(); 341 342 do { 343 for (i = 0; i != ITER_NUM; i++) { 344 rc = try_write(lc); 345 if (rc == 0) 346 lcd->stat.success++; 347 else if (rc == -EBUSY) 348 lcd->stat.fail++; 349 else 350 break; 351 rc = 0; 352 } 353 tm = rte_get_timer_cycles() - stm; 354 } while (tm < ftm && rc == 0); 355 356 lcd->rc = rc; 357 lcd->stat.tick = tm; 358 return rc; 359 } 360 361 static void 362 print_try_lcore_stats(const struct try_rwlock_lcore *tlc, uint32_t lc) 363 { 364 uint64_t f, s; 365 366 f = RTE_MAX(tlc->stat.fail, 1ULL); 367 s = RTE_MAX(tlc->stat.success, 1ULL); 368 369 printf("try_lcore_data[%u]={\n" 370 "\trc=%d,\n" 371 "\ttype=%s,\n" 372 "\tfail=%" PRIu64 ",\n" 373 "\tsuccess=%" PRIu64 ",\n" 374 "\tcycles=%" PRIu64 ",\n" 375 "\tcycles/op=%#Lf,\n" 376 "\tcycles/success=%#Lf,\n" 377 "\tsuccess/fail=%#Lf,\n" 378 "};\n", 379 lc, 380 tlc->rc, 381 tlc->type == LC_TYPE_RDLOCK ? "RDLOCK" : "WRLOCK", 382 tlc->stat.fail, 383 tlc->stat.success, 384 tlc->stat.tick, 385 (long double)tlc->stat.tick / 386 (tlc->stat.fail + tlc->stat.success), 387 (long double)tlc->stat.tick / s, 388 (long double)tlc->stat.success / f); 389 } 390 391 static void 392 collect_try_lcore_stats(struct try_rwlock_lcore *tlc, 393 const struct try_rwlock_lcore *lc) 394 { 395 tlc->stat.tick += lc->stat.tick; 396 tlc->stat.fail += lc->stat.fail; 397 tlc->stat.success += lc->stat.success; 398 } 399 400 /* 401 * Process collected results: 402 * - check status 403 * - collect and print statistics 404 */ 405 static int 406 process_try_lcore_stats(void) 407 { 408 int32_t rc; 409 uint32_t lc, rd, wr; 410 struct try_rwlock_lcore rlc, wlc; 411 412 memset(&rlc, 0, sizeof(rlc)); 413 memset(&wlc, 0, sizeof(wlc)); 414 415 rlc.type = LC_TYPE_RDLOCK; 416 wlc.type = LC_TYPE_WRLOCK; 417 rd = 0; 418 wr = 0; 419 420 rc = 0; 421 RTE_LCORE_FOREACH(lc) { 422 rc |= try_lcore_data[lc].rc; 423 if (try_lcore_data[lc].type == LC_TYPE_RDLOCK) { 424 collect_try_lcore_stats(&rlc, try_lcore_data + lc); 425 rd++; 426 } else { 427 collect_try_lcore_stats(&wlc, try_lcore_data + lc); 428 wr++; 429 } 430 } 431 432 if (rc == 0) { 433 RTE_LCORE_FOREACH(lc) 434 print_try_lcore_stats(try_lcore_data + lc, lc); 435 436 if (rd != 0) { 437 printf("aggregated stats for %u RDLOCK cores:\n", rd); 438 print_try_lcore_stats(&rlc, rd); 439 } 440 441 if (wr != 0) { 442 printf("aggregated stats for %u WRLOCK cores:\n", wr); 443 print_try_lcore_stats(&wlc, wr); 444 } 445 } 446 447 return rc; 448 } 449 450 static void 451 try_test_reset(void) 452 { 453 memset(&try_lcore_data, 0, sizeof(try_lcore_data)); 454 memset(&try_rwlock_data, 0, sizeof(try_rwlock_data)); 455 try_rwlock_data.tick = TEST_SEC * rte_get_tsc_hz(); 456 } 457 458 /* all lcores grab RDLOCK */ 459 static int 460 try_rwlock_test_rda(void) 461 { 462 try_test_reset(); 463 464 /* start read test on all avaialble lcores */ 465 rte_eal_mp_remote_launch(try_read_lcore, NULL, CALL_MASTER); 466 rte_eal_mp_wait_lcore(); 467 468 return process_try_lcore_stats(); 469 } 470 471 /* all slave lcores grab RDLOCK, master one grabs WRLOCK */ 472 static int 473 try_rwlock_test_rds_wrm(void) 474 { 475 try_test_reset(); 476 477 rte_eal_mp_remote_launch(try_read_lcore, NULL, SKIP_MASTER); 478 try_write_lcore(NULL); 479 rte_eal_mp_wait_lcore(); 480 481 return process_try_lcore_stats(); 482 } 483 484 /* master and even slave lcores grab RDLOCK, odd lcores grab WRLOCK */ 485 static int 486 try_rwlock_test_rde_wro(void) 487 { 488 uint32_t lc, mlc; 489 490 try_test_reset(); 491 492 mlc = rte_get_master_lcore(); 493 494 RTE_LCORE_FOREACH(lc) { 495 if (lc != mlc) { 496 if ((lc & 1) == 0) 497 rte_eal_remote_launch(try_read_lcore, 498 NULL, lc); 499 else 500 rte_eal_remote_launch(try_write_lcore, 501 NULL, lc); 502 } 503 } 504 try_read_lcore(NULL); 505 rte_eal_mp_wait_lcore(); 506 507 return process_try_lcore_stats(); 508 } 509 510 static int 511 test_rwlock(void) 512 { 513 uint32_t i; 514 int32_t rc, ret; 515 516 static const struct { 517 const char *name; 518 int (*ftst)(void); 519 } test[] = { 520 { 521 .name = "rwlock_test1", 522 .ftst = rwlock_test1, 523 }, 524 { 525 .name = "try_rwlock_test_rda", 526 .ftst = try_rwlock_test_rda, 527 }, 528 { 529 .name = "try_rwlock_test_rds_wrm", 530 .ftst = try_rwlock_test_rds_wrm, 531 }, 532 { 533 .name = "try_rwlock_test_rde_wro", 534 .ftst = try_rwlock_test_rde_wro, 535 }, 536 }; 537 538 ret = 0; 539 for (i = 0; i != RTE_DIM(test); i++) { 540 printf("starting test %s;\n", test[i].name); 541 rc = test[i].ftst(); 542 printf("test %s completed with status %d\n", test[i].name, rc); 543 ret |= rc; 544 } 545 546 return ret; 547 } 548 549 REGISTER_TEST_COMMAND(rwlock_autotest, test_rwlock); 550 551 /* subtests used in meson for CI */ 552 REGISTER_TEST_COMMAND(rwlock_test1_autotest, rwlock_test1); 553 REGISTER_TEST_COMMAND(rwlock_rda_autotest, try_rwlock_test_rda); 554 REGISTER_TEST_COMMAND(rwlock_rds_wrm_autotest, try_rwlock_test_rds_wrm); 555 REGISTER_TEST_COMMAND(rwlock_rde_wro_autotest, try_rwlock_test_rde_wro); 556