1 #include "LibcBenchmark.h"
2 #include "LibcMemoryBenchmark.h"
3 #include "MemorySizeDistributions.h"
4 #include "benchmark/benchmark.h"
5 #include "llvm/ADT/ArrayRef.h"
6 #include "llvm/ADT/Twine.h"
7 #include <chrono>
8 #include <cstdint>
9 #include <random>
10 #include <vector>
11
12 using llvm::Align;
13 using llvm::ArrayRef;
14 using llvm::Twine;
15 using llvm::libc_benchmarks::BzeroConfiguration;
16 using llvm::libc_benchmarks::ComparisonSetup;
17 using llvm::libc_benchmarks::CopySetup;
18 using llvm::libc_benchmarks::MemcmpOrBcmpConfiguration;
19 using llvm::libc_benchmarks::MemcpyConfiguration;
20 using llvm::libc_benchmarks::MemmoveConfiguration;
21 using llvm::libc_benchmarks::MemorySizeDistribution;
22 using llvm::libc_benchmarks::MemsetConfiguration;
23 using llvm::libc_benchmarks::MoveSetup;
24 using llvm::libc_benchmarks::OffsetDistribution;
25 using llvm::libc_benchmarks::SetSetup;
26
27 // Alignment to use for when accessing the buffers.
28 static constexpr Align kBenchmarkAlignment = Align::Constant<1>();
29
getGenerator()30 static std::mt19937_64 &getGenerator() {
31 static std::mt19937_64 Generator(
32 std::chrono::system_clock::now().time_since_epoch().count());
33 return Generator;
34 }
35
36 template <typename SetupType, typename ConfigurationType> struct Runner {
RunnerRunner37 Runner(benchmark::State &S, llvm::ArrayRef<ConfigurationType> Configurations)
38 : State(S), Distribution(SetupType::getDistributions()[State.range(0)]),
39 Probabilities(Distribution.Probabilities),
40 SizeSampler(Probabilities.begin(), Probabilities.end()),
41 OffsetSampler(Setup.BufferSize, Probabilities.size() - 1,
42 kBenchmarkAlignment),
43 Configuration(Configurations[State.range(1)]) {
44 for (auto &P : Setup.Parameters) {
45 P.OffsetBytes = OffsetSampler(getGenerator());
46 P.SizeBytes = SizeSampler(getGenerator());
47 Setup.checkValid(P);
48 }
49 }
50
~RunnerRunner51 ~Runner() {
52 const size_t TotalBytes =
53 (State.iterations() * Setup.getBatchBytes()) / Setup.BatchSize;
54 State.SetBytesProcessed(TotalBytes);
55 State.SetItemsProcessed(State.iterations());
56 State.SetLabel((Twine(Configuration.Name) + "," + Distribution.Name).str());
57 State.counters["bytes_per_cycle"] = benchmark::Counter(
58 TotalBytes / benchmark::CPUInfo::Get().cycles_per_second,
59 benchmark::Counter::kIsRate);
60 }
61
runBatchRunner62 inline void runBatch() {
63 for (const auto &P : Setup.Parameters)
64 benchmark::DoNotOptimize(Setup.Call(P, Configuration.Function));
65 }
66
getBatchSizeRunner67 size_t getBatchSize() const { return Setup.BatchSize; }
68
69 private:
70 SetupType Setup;
71 benchmark::State &State;
72 MemorySizeDistribution Distribution;
73 ArrayRef<double> Probabilities;
74 std::discrete_distribution<unsigned> SizeSampler;
75 OffsetDistribution OffsetSampler;
76 ConfigurationType Configuration;
77 };
78
79 #define BENCHMARK_MEMORY_FUNCTION(BM_NAME, SETUP, CONFIGURATION_TYPE, \
80 CONFIGURATION_ARRAY_REF) \
81 void BM_NAME(benchmark::State &State) { \
82 Runner<SETUP, CONFIGURATION_TYPE> Setup(State, CONFIGURATION_ARRAY_REF); \
83 const size_t BatchSize = Setup.getBatchSize(); \
84 while (State.KeepRunningBatch(BatchSize)) \
85 Setup.runBatch(); \
86 } \
87 BENCHMARK(BM_NAME)->Apply([](benchmark::internal::Benchmark *benchmark) { \
88 const int64_t DistributionSize = SETUP::getDistributions().size(); \
89 const int64_t ConfigurationSize = CONFIGURATION_ARRAY_REF.size(); \
90 for (int64_t DistIndex = 0; DistIndex < DistributionSize; ++DistIndex) \
91 for (int64_t ConfIndex = 0; ConfIndex < ConfigurationSize; ++ConfIndex) \
92 benchmark->Args({DistIndex, ConfIndex}); \
93 })
94
95 extern llvm::ArrayRef<MemcpyConfiguration> getMemcpyConfigurations();
96 BENCHMARK_MEMORY_FUNCTION(BM_Memcpy, CopySetup, MemcpyConfiguration,
97 getMemcpyConfigurations());
98
99 extern llvm::ArrayRef<MemmoveConfiguration> getMemmoveConfigurations();
100 BENCHMARK_MEMORY_FUNCTION(BM_Memmove, MoveSetup, MemmoveConfiguration,
101 getMemmoveConfigurations());
102
103 extern llvm::ArrayRef<MemcmpOrBcmpConfiguration> getMemcmpConfigurations();
104 BENCHMARK_MEMORY_FUNCTION(BM_Memcmp, ComparisonSetup, MemcmpOrBcmpConfiguration,
105 getMemcmpConfigurations());
106
107 extern llvm::ArrayRef<MemcmpOrBcmpConfiguration> getBcmpConfigurations();
108 BENCHMARK_MEMORY_FUNCTION(BM_Bcmp, ComparisonSetup, MemcmpOrBcmpConfiguration,
109 getBcmpConfigurations());
110
111 extern llvm::ArrayRef<MemsetConfiguration> getMemsetConfigurations();
112 BENCHMARK_MEMORY_FUNCTION(BM_Memset, SetSetup, MemsetConfiguration,
113 getMemsetConfigurations());
114
115 extern llvm::ArrayRef<BzeroConfiguration> getBzeroConfigurations();
116 BENCHMARK_MEMORY_FUNCTION(BM_Bzero, SetSetup, BzeroConfiguration,
117 getBzeroConfigurations());
118