1 //===----------------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 // UNSUPPORTED: c++03, c++11, c++14, c++17 10 11 #include "atomic_wait_helper.h" 12 13 #include <atomic> 14 #include <cstdint> 15 #include <numeric> 16 #include <stop_token> 17 #include <pthread.h> 18 #include <sched.h> 19 #include <thread> 20 #include <chrono> 21 #include <array> 22 23 #include "benchmark/benchmark.h" 24 #include "make_test_thread.h" 25 26 using namespace std::chrono_literals; 27 28 template <class NotifyPolicy, class NumberOfAtomics, class NumPrioTasks> 29 void BM_N_atomics_N_waiter_N_notifier(benchmark::State& state) { 30 [[maybe_unused]] std::array<HighPrioTask, NumPrioTasks::value> tasks{}; 31 const std::uint64_t total_loop_test_param = state.range(0); 32 constexpr std::uint64_t num_atomics = NumberOfAtomics::value; 33 std::vector<std::atomic<std::uint64_t>> atomics(num_atomics); 34 35 auto notify_func = [&](std::stop_token st, size_t idx) { 36 while (!st.stop_requested()) { 37 NotifyPolicy::notify(atomics[idx], st); 38 } 39 }; 40 41 std::atomic<std::uint64_t> start_flag = 0; 42 std::atomic<std::uint64_t> done_count = 0; 43 44 auto wait_func = [&, total_loop_test_param](std::stop_token st, size_t idx) { 45 auto old_start = 0; 46 while (!st.stop_requested()) { 47 start_flag.wait(old_start); 48 old_start = start_flag.load(); 49 for (std::uint64_t i = 0; i < total_loop_test_param; ++i) { 50 auto old = atomics[idx].load(std::memory_order_relaxed); 51 atomics[idx].wait(old); 52 } 53 done_count.fetch_add(1); 54 } 55 }; 56 57 std::vector<std::jthread> notify_threads; 58 notify_threads.reserve(num_atomics); 59 60 std::vector<std::jthread> wait_threads; 61 wait_threads.reserve(num_atomics); 62 63 for (size_t i = 0; i < num_atomics; ++i) { 64 notify_threads.emplace_back(support::make_test_jthread(notify_func, i)); 65 } 66 67 for (size_t i = 0; i < num_atomics; ++i) { 68 wait_threads.emplace_back(support::make_test_jthread(wait_func, i)); 69 } 70 71 for (auto _ : state) { 72 done_count = 0; 73 start_flag.fetch_add(1); 74 start_flag.notify_all(); 75 while (done_count < num_atomics) { 76 std::this_thread::yield(); 77 } 78 } 79 for (auto& t : wait_threads) { 80 t.request_stop(); 81 } 82 start_flag.fetch_add(1); 83 start_flag.notify_all(); 84 for (auto& t : wait_threads) { 85 t.join(); 86 } 87 } 88 89 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<2>, NumHighPrioTasks<0>>) 90 ->RangeMultiplier(2) 91 ->Range(1 << 12, 1 << 14); 92 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<3>, NumHighPrioTasks<0>>) 93 ->RangeMultiplier(2) 94 ->Range(1 << 10, 1 << 12); 95 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<5>, NumHighPrioTasks<0>>) 96 ->RangeMultiplier(2) 97 ->Range(1 << 10, 1 << 12); 98 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<7>, NumHighPrioTasks<0>>) 99 ->RangeMultiplier(2) 100 ->Range(1 << 8, 1 << 10); 101 102 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<2>, NumHighPrioTasks<0>>) 103 ->RangeMultiplier(2) 104 ->Range(1 << 10, 1 << 12); 105 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<3>, NumHighPrioTasks<0>>) 106 ->RangeMultiplier(2) 107 ->Range(1 << 8, 1 << 10); 108 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<5>, NumHighPrioTasks<0>>) 109 ->RangeMultiplier(2) 110 ->Range(1 << 8, 1 << 10); 111 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<7>, NumHighPrioTasks<0>>) 112 ->RangeMultiplier(2) 113 ->Range(1 << 6, 1 << 8); 114 115 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<2>, NumHighPrioTasks<0>>) 116 ->RangeMultiplier(2) 117 ->Range(1 << 8, 1 << 10); 118 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<3>, NumHighPrioTasks<0>>) 119 ->RangeMultiplier(2) 120 ->Range(1 << 8, 1 << 10); 121 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<5>, NumHighPrioTasks<0>>) 122 ->RangeMultiplier(2) 123 ->Range(1 << 7, 1 << 9); 124 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<7>, NumHighPrioTasks<0>>) 125 ->RangeMultiplier(2) 126 ->Range(1 << 6, 1 << 8); 127 128 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<2>, NumHighPrioTasks<4>>) 129 ->RangeMultiplier(2) 130 ->Range(1 << 7, 1 << 9); 131 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<3>, NumHighPrioTasks<4>>) 132 ->RangeMultiplier(2) 133 ->Range(1 << 7, 1 << 9); 134 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<5>, NumHighPrioTasks<4>>) 135 ->RangeMultiplier(2) 136 ->Range(1 << 6, 1 << 8); 137 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<7>, NumHighPrioTasks<4>>) 138 ->RangeMultiplier(2) 139 ->Range(1 << 4, 1 << 6); 140 141 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<2>, NumHighPrioTasks<4>>) 142 ->RangeMultiplier(2) 143 ->Range(1 << 7, 1 << 9); 144 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<3>, NumHighPrioTasks<4>>) 145 ->RangeMultiplier(2) 146 ->Range(1 << 7, 1 << 9); 147 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<5>, NumHighPrioTasks<4>>) 148 ->RangeMultiplier(2) 149 ->Range(1 << 5, 1 << 7); 150 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<7>, NumHighPrioTasks<4>>) 151 ->RangeMultiplier(2) 152 ->Range(1 << 3, 1 << 5); 153 154 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<2>, NumHighPrioTasks<4>>) 155 ->RangeMultiplier(2) 156 ->Range(1 << 6, 1 << 8); 157 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<3>, NumHighPrioTasks<4>>) 158 ->RangeMultiplier(2) 159 ->Range(1 << 6, 1 << 8); 160 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<5>, NumHighPrioTasks<4>>) 161 ->RangeMultiplier(2) 162 ->Range(1 << 5, 1 << 7); 163 BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<7>, NumHighPrioTasks<4>>) 164 ->RangeMultiplier(2) 165 ->Range(1 << 3, 1 << 5); 166 167 BENCHMARK_MAIN(); 168