1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Arm Limited 3 * Copyright(c) 2024 Ericsson AB 4 */ 5 6 #include <inttypes.h> 7 #include <stdbool.h> 8 9 #include <rte_bitops.h> 10 #include <rte_cycles.h> 11 #include <rte_launch.h> 12 #include <rte_lcore.h> 13 #include <rte_random.h> 14 #include "test.h" 15 16 #define GEN_TEST_BIT_ACCESS(test_name, set_fun, clear_fun, assign_fun, flip_fun, test_fun, size, \ 17 mod) \ 18 static int \ 19 test_name(void) \ 20 { \ 21 uint ## size ## _t reference = (uint ## size ## _t)rte_rand(); \ 22 unsigned int bit_nr; \ 23 mod uint ## size ## _t word = (uint ## size ## _t)rte_rand(); \ 24 for (bit_nr = 0; bit_nr < size; bit_nr++) { \ 25 bool reference_bit = (reference >> bit_nr) & 1; \ 26 bool assign = rte_rand() & 1; \ 27 if (assign) { \ 28 assign_fun(&word, bit_nr, reference_bit); \ 29 } else { \ 30 if (reference_bit) \ 31 set_fun(&word, bit_nr); \ 32 else \ 33 clear_fun(&word, bit_nr); \ 34 } \ 35 TEST_ASSERT(test_fun(&word, bit_nr) == reference_bit, \ 36 "Bit %d had unexpected value", bit_nr); \ 37 flip_fun(&word, bit_nr); \ 38 TEST_ASSERT(test_fun(&word, bit_nr) != reference_bit, \ 39 "Bit %d had unflipped value", bit_nr); \ 40 flip_fun(&word, bit_nr); \ 41 const mod uint ## size ## _t *const_ptr = &word; \ 42 TEST_ASSERT(test_fun(const_ptr, bit_nr) == reference_bit, \ 43 "Bit %d had unexpected value", bit_nr); \ 44 } \ 45 for (bit_nr = 0; bit_nr < size; bit_nr++) { \ 46 bool reference_bit = (reference >> bit_nr) & 1; \ 47 TEST_ASSERT(test_fun(&word, bit_nr) == reference_bit, \ 48 "Bit %d had unexpected value", bit_nr); \ 49 } \ 50 TEST_ASSERT(reference == word, "Word had unexpected value"); \ 51 return TEST_SUCCESS; \ 52 } 53 54 GEN_TEST_BIT_ACCESS(test_bit_access32, rte_bit_set, rte_bit_clear, rte_bit_assign, rte_bit_flip, 55 rte_bit_test, 32,) 56 57 GEN_TEST_BIT_ACCESS(test_bit_access64, rte_bit_set, rte_bit_clear, rte_bit_assign, rte_bit_flip, 58 rte_bit_test, 64,) 59 60 GEN_TEST_BIT_ACCESS(test_bit_v_access32, rte_bit_set, rte_bit_clear, rte_bit_assign, rte_bit_flip, 61 rte_bit_test, 32, volatile) 62 63 GEN_TEST_BIT_ACCESS(test_bit_v_access64, rte_bit_set, rte_bit_clear, rte_bit_assign, rte_bit_flip, 64 rte_bit_test, 64, volatile) 65 66 #define bit_atomic_set(addr, nr) \ 67 rte_bit_atomic_set(addr, nr, rte_memory_order_relaxed) 68 69 #define bit_atomic_clear(addr, nr) \ 70 rte_bit_atomic_clear(addr, nr, rte_memory_order_relaxed) 71 72 #define bit_atomic_assign(addr, nr, value) \ 73 rte_bit_atomic_assign(addr, nr, value, rte_memory_order_relaxed) 74 75 #define bit_atomic_flip(addr, nr) \ 76 rte_bit_atomic_flip(addr, nr, rte_memory_order_relaxed) 77 78 #define bit_atomic_test(addr, nr) \ 79 rte_bit_atomic_test(addr, nr, rte_memory_order_relaxed) 80 81 GEN_TEST_BIT_ACCESS(test_bit_atomic_access32, bit_atomic_set, bit_atomic_clear, bit_atomic_assign, 82 bit_atomic_flip, bit_atomic_test, 32,) 83 84 GEN_TEST_BIT_ACCESS(test_bit_atomic_access64, bit_atomic_set, bit_atomic_clear, bit_atomic_assign, 85 bit_atomic_flip, bit_atomic_test, 64,) 86 87 GEN_TEST_BIT_ACCESS(test_bit_atomic_v_access32, bit_atomic_set, bit_atomic_clear, bit_atomic_assign, 88 bit_atomic_flip, bit_atomic_test, 32, volatile) 89 90 GEN_TEST_BIT_ACCESS(test_bit_atomic_v_access64, bit_atomic_set, bit_atomic_clear, bit_atomic_assign, 91 bit_atomic_flip, bit_atomic_test, 64, volatile) 92 93 #define PARALLEL_TEST_RUNTIME 0.25 94 95 #define GEN_TEST_BIT_PARALLEL_ASSIGN(size) \ 96 struct parallel_access_lcore ## size \ 97 { \ 98 unsigned int bit; \ 99 uint ## size ##_t *word; \ 100 bool failed; \ 101 }; \ 102 static int \ 103 run_parallel_assign ## size(void *arg) \ 104 { \ 105 struct parallel_access_lcore ## size *lcore = arg; \ 106 uint64_t deadline = rte_get_timer_cycles() + PARALLEL_TEST_RUNTIME * rte_get_timer_hz(); \ 107 bool value = false; \ 108 do { \ 109 bool new_value = rte_rand() & 1; \ 110 bool use_test_and_modify = rte_rand() & 1; \ 111 bool use_assign = rte_rand() & 1; \ 112 if (rte_bit_atomic_test(lcore->word, lcore->bit, \ 113 rte_memory_order_relaxed) != value) { \ 114 lcore->failed = true; \ 115 break; \ 116 } \ 117 if (use_test_and_modify) { \ 118 bool old_value; \ 119 if (use_assign) { \ 120 old_value = rte_bit_atomic_test_and_assign(lcore->word, \ 121 lcore->bit, new_value, rte_memory_order_relaxed); \ 122 } else { \ 123 old_value = new_value ? \ 124 rte_bit_atomic_test_and_set(lcore->word, lcore->bit, \ 125 rte_memory_order_relaxed) : \ 126 rte_bit_atomic_test_and_clear(lcore->word, lcore->bit, \ 127 rte_memory_order_relaxed); \ 128 } \ 129 if (old_value != value) { \ 130 lcore->failed = true; \ 131 break; \ 132 } \ 133 } else { \ 134 if (use_assign) { \ 135 rte_bit_atomic_assign(lcore->word, lcore->bit, new_value, \ 136 rte_memory_order_relaxed); \ 137 } else { \ 138 if (new_value) \ 139 rte_bit_atomic_set(lcore->word, lcore->bit, \ 140 rte_memory_order_relaxed); \ 141 else \ 142 rte_bit_atomic_clear(lcore->word, lcore->bit, \ 143 rte_memory_order_relaxed); \ 144 } \ 145 } \ 146 value = new_value; \ 147 } while (rte_get_timer_cycles() < deadline); \ 148 return 0; \ 149 } \ 150 static int \ 151 test_bit_atomic_parallel_assign ## size(void) \ 152 { \ 153 unsigned int worker_lcore_id; \ 154 uint ## size ## _t word = 0; \ 155 struct parallel_access_lcore ## size lmain = { .word = &word }; \ 156 struct parallel_access_lcore ## size lworker = { .word = &word }; \ 157 if (rte_lcore_count() < 2) { \ 158 printf("Need multiple cores to run parallel test.\n"); \ 159 return TEST_SKIPPED; \ 160 } \ 161 worker_lcore_id = rte_get_next_lcore(-1, 1, 0); \ 162 lmain.bit = rte_rand_max(size); \ 163 do { \ 164 lworker.bit = rte_rand_max(size); \ 165 } while (lworker.bit == lmain.bit); \ 166 int rc = rte_eal_remote_launch(run_parallel_assign ## size, &lworker, worker_lcore_id); \ 167 TEST_ASSERT(rc == 0, "Worker thread launch failed"); \ 168 run_parallel_assign ## size(&lmain); \ 169 rte_eal_mp_wait_lcore(); \ 170 TEST_ASSERT(!lmain.failed, "Main lcore atomic access failed"); \ 171 TEST_ASSERT(!lworker.failed, "Worker lcore atomic access failed"); \ 172 return TEST_SUCCESS; \ 173 } 174 175 GEN_TEST_BIT_PARALLEL_ASSIGN(32) 176 GEN_TEST_BIT_PARALLEL_ASSIGN(64) 177 178 #define GEN_TEST_BIT_PARALLEL_TEST_AND_MODIFY(size) \ 179 struct parallel_test_and_set_lcore ## size \ 180 { \ 181 uint ## size ##_t *word; \ 182 unsigned int bit; \ 183 uint64_t flips; \ 184 }; \ 185 static int \ 186 run_parallel_test_and_modify ## size(void *arg) \ 187 { \ 188 struct parallel_test_and_set_lcore ## size *lcore = arg; \ 189 uint64_t deadline = rte_get_timer_cycles() + PARALLEL_TEST_RUNTIME * rte_get_timer_hz(); \ 190 do { \ 191 bool old_value; \ 192 bool new_value = rte_rand() & 1; \ 193 bool use_assign = rte_rand() & 1; \ 194 if (use_assign) \ 195 old_value = rte_bit_atomic_test_and_assign(lcore->word, lcore->bit, \ 196 new_value, rte_memory_order_relaxed); \ 197 else \ 198 old_value = new_value ? \ 199 rte_bit_atomic_test_and_set(lcore->word, lcore->bit, \ 200 rte_memory_order_relaxed) : \ 201 rte_bit_atomic_test_and_clear(lcore->word, lcore->bit, \ 202 rte_memory_order_relaxed); \ 203 if (old_value != new_value) \ 204 lcore->flips++; \ 205 } while (rte_get_timer_cycles() < deadline); \ 206 return 0; \ 207 } \ 208 static int \ 209 test_bit_atomic_parallel_test_and_modify ## size(void) \ 210 { \ 211 unsigned int worker_lcore_id; \ 212 uint ## size ## _t word = 0; \ 213 unsigned int bit = rte_rand_max(size); \ 214 struct parallel_test_and_set_lcore ## size lmain = { .word = &word, .bit = bit }; \ 215 struct parallel_test_and_set_lcore ## size lworker = { .word = &word, .bit = bit }; \ 216 if (rte_lcore_count() < 2) { \ 217 printf("Need multiple cores to run parallel test.\n"); \ 218 return TEST_SKIPPED; \ 219 } \ 220 worker_lcore_id = rte_get_next_lcore(-1, 1, 0); \ 221 int rc = rte_eal_remote_launch(run_parallel_test_and_modify ## size, &lworker, \ 222 worker_lcore_id); \ 223 TEST_ASSERT(rc == 0, "Worker thread launch failed"); \ 224 run_parallel_test_and_modify ## size(&lmain); \ 225 rte_eal_mp_wait_lcore(); \ 226 uint64_t total_flips = lmain.flips + lworker.flips; \ 227 bool expected_value = total_flips % 2; \ 228 TEST_ASSERT(expected_value == rte_bit_test(&word, bit), \ 229 "After %"PRId64" flips, the bit value should be %d", total_flips, expected_value); \ 230 uint ## size ## _t expected_word = 0; \ 231 rte_bit_assign(&expected_word, bit, expected_value); \ 232 TEST_ASSERT(expected_word == word, "Untouched bits have changed value"); \ 233 return TEST_SUCCESS; \ 234 } 235 236 GEN_TEST_BIT_PARALLEL_TEST_AND_MODIFY(32) 237 GEN_TEST_BIT_PARALLEL_TEST_AND_MODIFY(64) 238 239 #define GEN_TEST_BIT_PARALLEL_FLIP(size) \ 240 struct parallel_flip_lcore ## size \ 241 { \ 242 uint ## size ##_t *word; \ 243 unsigned int bit; \ 244 uint64_t flips; \ 245 }; \ 246 static int \ 247 run_parallel_flip ## size(void *arg) \ 248 { \ 249 struct parallel_flip_lcore ## size *lcore = arg; \ 250 uint64_t deadline = rte_get_timer_cycles() + PARALLEL_TEST_RUNTIME * rte_get_timer_hz(); \ 251 do { \ 252 rte_bit_atomic_flip(lcore->word, lcore->bit, rte_memory_order_relaxed); \ 253 lcore->flips++; \ 254 } while (rte_get_timer_cycles() < deadline); \ 255 return 0; \ 256 } \ 257 static int \ 258 test_bit_atomic_parallel_flip ## size(void) \ 259 { \ 260 unsigned int worker_lcore_id; \ 261 uint ## size ## _t word = 0; \ 262 unsigned int bit = rte_rand_max(size); \ 263 struct parallel_flip_lcore ## size lmain = { .word = &word, .bit = bit }; \ 264 struct parallel_flip_lcore ## size lworker = { .word = &word, .bit = bit }; \ 265 if (rte_lcore_count() < 2) { \ 266 printf("Need multiple cores to run parallel test.\n"); \ 267 return TEST_SKIPPED; \ 268 } \ 269 worker_lcore_id = rte_get_next_lcore(-1, 1, 0); \ 270 int rc = rte_eal_remote_launch(run_parallel_flip ## size, &lworker, worker_lcore_id); \ 271 TEST_ASSERT(rc == 0, "Worker thread launch failed"); \ 272 run_parallel_flip ## size(&lmain); \ 273 rte_eal_mp_wait_lcore(); \ 274 uint64_t total_flips = lmain.flips + lworker.flips; \ 275 bool expected_value = total_flips % 2; \ 276 TEST_ASSERT(expected_value == rte_bit_test(&word, bit), \ 277 "After %"PRId64" flips, the bit value should be %d", total_flips, expected_value); \ 278 uint ## size ## _t expected_word = 0; \ 279 rte_bit_assign(&expected_word, bit, expected_value); \ 280 TEST_ASSERT(expected_word == word, "Untouched bits have changed value"); \ 281 return TEST_SUCCESS; \ 282 } 283 284 GEN_TEST_BIT_PARALLEL_FLIP(32) 285 GEN_TEST_BIT_PARALLEL_FLIP(64) 286 287 static uint32_t val32; 288 static uint64_t val64; 289 290 #define MAX_BITS_32 32 291 #define MAX_BITS_64 64 292 293 /* 294 * Bitops functions 295 * ================ 296 * 297 * - The main test function performs several subtests. 298 * - Check bit operations on one core. 299 * - Initialize valXX to specified values, then set each bit of valXX 300 * to 1 one by one in "test_bit_relaxed_set". 301 * 302 * - Clear each bit of valXX to 0 one by one in "test_bit_relaxed_clear". 303 * 304 * - Function "test_bit_relaxed_test_set_clear" checks whether each bit 305 * of valXX can do "test and set" and "test and clear" correctly. 306 */ 307 308 static int 309 test_bit_relaxed_set(void) 310 { 311 unsigned int i; 312 313 for (i = 0; i < MAX_BITS_32; i++) 314 rte_bit_relaxed_set32(i, &val32); 315 316 for (i = 0; i < MAX_BITS_32; i++) 317 if (!rte_bit_relaxed_get32(i, &val32)) { 318 printf("Failed to set bit in relaxed version.\n"); 319 return TEST_FAILED; 320 } 321 322 for (i = 0; i < MAX_BITS_64; i++) 323 rte_bit_relaxed_set64(i, &val64); 324 325 for (i = 0; i < MAX_BITS_64; i++) 326 if (!rte_bit_relaxed_get64(i, &val64)) { 327 printf("Failed to set bit in relaxed version.\n"); 328 return TEST_FAILED; 329 } 330 331 return TEST_SUCCESS; 332 } 333 334 static int 335 test_bit_relaxed_clear(void) 336 { 337 unsigned int i; 338 339 for (i = 0; i < MAX_BITS_32; i++) 340 rte_bit_relaxed_clear32(i, &val32); 341 342 for (i = 0; i < MAX_BITS_32; i++) 343 if (rte_bit_relaxed_get32(i, &val32)) { 344 printf("Failed to clear bit in relaxed version.\n"); 345 return TEST_FAILED; 346 } 347 348 for (i = 0; i < MAX_BITS_64; i++) 349 rte_bit_relaxed_clear64(i, &val64); 350 351 for (i = 0; i < MAX_BITS_64; i++) 352 if (rte_bit_relaxed_get64(i, &val64)) { 353 printf("Failed to clear bit in relaxed version.\n"); 354 return TEST_FAILED; 355 } 356 357 return TEST_SUCCESS; 358 } 359 360 static int 361 test_bit_relaxed_test_set_clear(void) 362 { 363 unsigned int i; 364 365 for (i = 0; i < MAX_BITS_32; i++) 366 rte_bit_relaxed_test_and_set32(i, &val32); 367 368 for (i = 0; i < MAX_BITS_32; i++) 369 if (!rte_bit_relaxed_test_and_clear32(i, &val32)) { 370 printf("Failed to set and test bit in relaxed version.\n"); 371 return TEST_FAILED; 372 } 373 374 for (i = 0; i < MAX_BITS_32; i++) 375 if (rte_bit_relaxed_get32(i, &val32)) { 376 printf("Failed to test and clear bit in relaxed version.\n"); 377 return TEST_FAILED; 378 } 379 380 for (i = 0; i < MAX_BITS_64; i++) 381 rte_bit_relaxed_test_and_set64(i, &val64); 382 383 for (i = 0; i < MAX_BITS_64; i++) 384 if (!rte_bit_relaxed_test_and_clear64(i, &val64)) { 385 printf("Failed to set and test bit in relaxed version.\n"); 386 return TEST_FAILED; 387 } 388 389 for (i = 0; i < MAX_BITS_64; i++) 390 if (rte_bit_relaxed_get64(i, &val64)) { 391 printf("Failed to test and clear bit in relaxed version.\n"); 392 return TEST_FAILED; 393 } 394 395 return TEST_SUCCESS; 396 } 397 398 static struct unit_test_suite test_suite = { 399 .suite_name = "Bitops test suite", 400 .unit_test_cases = { 401 TEST_CASE(test_bit_access32), 402 TEST_CASE(test_bit_access64), 403 TEST_CASE(test_bit_access32), 404 TEST_CASE(test_bit_access64), 405 TEST_CASE(test_bit_v_access32), 406 TEST_CASE(test_bit_v_access64), 407 TEST_CASE(test_bit_atomic_access32), 408 TEST_CASE(test_bit_atomic_access64), 409 TEST_CASE(test_bit_atomic_v_access32), 410 TEST_CASE(test_bit_atomic_v_access64), 411 TEST_CASE(test_bit_atomic_parallel_assign32), 412 TEST_CASE(test_bit_atomic_parallel_assign64), 413 TEST_CASE(test_bit_atomic_parallel_test_and_modify32), 414 TEST_CASE(test_bit_atomic_parallel_test_and_modify64), 415 TEST_CASE(test_bit_atomic_parallel_flip32), 416 TEST_CASE(test_bit_atomic_parallel_flip64), 417 TEST_CASE(test_bit_relaxed_set), 418 TEST_CASE(test_bit_relaxed_clear), 419 TEST_CASE(test_bit_relaxed_test_set_clear), 420 TEST_CASES_END() 421 } 422 }; 423 424 static int 425 test_bitops(void) 426 { 427 return unit_test_suite_runner(&test_suite); 428 } 429 430 REGISTER_FAST_TEST(bitops_autotest, true, true, test_bitops); 431