1 //===-- Benchmark ---------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "JSON.h" 10 #include "LibcBenchmark.h" 11 #include "LibcMemoryBenchmark.h" 12 #include "MemorySizeDistributions.h" 13 #include "llvm/Support/CommandLine.h" 14 #include "llvm/Support/ErrorHandling.h" 15 #include "llvm/Support/FileSystem.h" 16 #include "llvm/Support/JSON.h" 17 #include "llvm/Support/MathExtras.h" 18 #include "llvm/Support/MemoryBuffer.h" 19 #include "llvm/Support/raw_ostream.h" 20 21 #include <cstring> 22 #include <unistd.h> 23 24 namespace __llvm_libc { 25 26 extern void *memcpy(void *__restrict, const void *__restrict, size_t); 27 extern void *memset(void *, int, size_t); 28 extern void bzero(void *, size_t); 29 extern int memcmp(const void *, const void *, size_t); 30 31 } // namespace __llvm_libc 32 33 namespace llvm { 34 namespace libc_benchmarks { 35 36 static cl::opt<std::string> 37 StudyName("study-name", cl::desc("The name for this study"), cl::Required); 38 39 static cl::opt<std::string> 40 SizeDistributionName("size-distribution-name", 41 cl::desc("The name of the distribution to use")); 42 43 static cl::opt<bool> 44 SweepMode("sweep-mode", 45 cl::desc("If set, benchmark all sizes from 0 to sweep-max-size")); 46 47 static cl::opt<uint32_t> 48 SweepMaxSize("sweep-max-size", 49 cl::desc("The maximum size to use in sweep-mode"), 50 cl::init(256)); 51 52 static cl::opt<uint32_t> 53 AlignedAccess("aligned-access", 54 cl::desc("The alignment to use when accessing the buffers\n" 55 "Default is unaligned\n" 56 "Use 0 to disable address randomization"), 57 cl::init(1)); 58 59 static cl::opt<std::string> Output("output", 60 cl::desc("Specify output filename"), 61 cl::value_desc("filename"), cl::init("-")); 62 63 static cl::opt<uint32_t> 64 NumTrials("num-trials", cl::desc("The number of benchmarks run to perform"), 65 cl::init(1)); 66 67 #if defined(LIBC_BENCHMARK_FUNCTION_MEMCPY) 68 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCPY 69 using BenchmarkSetup = CopySetup; 70 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMSET) 71 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMSET 72 using BenchmarkSetup = SetSetup; 73 #elif defined(LIBC_BENCHMARK_FUNCTION_BZERO) 74 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BZERO 75 using BenchmarkSetup = SetSetup; 76 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMCMP) 77 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCMP 78 using BenchmarkSetup = ComparisonSetup; 79 #else 80 #error "Missing LIBC_BENCHMARK_FUNCTION_XXX definition" 81 #endif 82 83 struct MemfunctionBenchmarkBase : public BenchmarkSetup { 84 MemfunctionBenchmarkBase() : ReportProgress(isatty(fileno(stdout))) {} 85 virtual ~MemfunctionBenchmarkBase() {} 86 87 virtual Study run() = 0; 88 89 CircularArrayRef<ParameterBatch::ParameterType> 90 generateBatch(size_t Iterations) { 91 randomize(); 92 return cycle(makeArrayRef(Parameters), Iterations); 93 } 94 95 protected: 96 Study createStudy() { 97 Study Study; 98 // Setup study. 99 Study.StudyName = StudyName; 100 Runtime &RI = Study.Runtime; 101 RI.Host = HostState::get(); 102 RI.BufferSize = BufferSize; 103 RI.BatchParameterCount = BatchSize; 104 105 BenchmarkOptions &BO = RI.BenchmarkOptions; 106 BO.MinDuration = std::chrono::milliseconds(1); 107 BO.MaxDuration = std::chrono::seconds(1); 108 BO.MaxIterations = 10'000'000U; 109 BO.MinSamples = 4; 110 BO.MaxSamples = 1000; 111 BO.Epsilon = 0.01; // 1% 112 BO.ScalingFactor = 1.4; 113 114 StudyConfiguration &SC = Study.Configuration; 115 SC.NumTrials = NumTrials; 116 SC.IsSweepMode = SweepMode; 117 SC.AccessAlignment = MaybeAlign(AlignedAccess); 118 SC.Function = LIBC_BENCHMARK_FUNCTION_NAME; 119 return Study; 120 } 121 122 void runTrials(const BenchmarkOptions &Options, 123 std::vector<Duration> &Measurements) { 124 for (size_t i = 0; i < NumTrials; ++i) { 125 const BenchmarkResult Result = benchmark( 126 Options, *this, [this](ParameterBatch::ParameterType Parameter) { 127 return Call(Parameter, LIBC_BENCHMARK_FUNCTION); 128 }); 129 Measurements.push_back(Result.BestGuess); 130 reportProgress(Measurements); 131 } 132 } 133 134 virtual void randomize() = 0; 135 136 private: 137 bool ReportProgress; 138 139 void reportProgress(const std::vector<Duration> &Measurements) { 140 if (!ReportProgress) 141 return; 142 static size_t LastPercent = -1; 143 const size_t TotalSteps = Measurements.capacity(); 144 const size_t Steps = Measurements.size(); 145 const size_t Percent = 100 * Steps / TotalSteps; 146 if (Percent == LastPercent) 147 return; 148 LastPercent = Percent; 149 size_t I = 0; 150 errs() << '['; 151 for (; I <= Percent; ++I) 152 errs() << '#'; 153 for (; I <= 100; ++I) 154 errs() << '_'; 155 errs() << "] " << Percent << '%' << '\r'; 156 } 157 }; 158 159 struct MemfunctionBenchmarkSweep final : public MemfunctionBenchmarkBase { 160 MemfunctionBenchmarkSweep() 161 : OffsetSampler(MemfunctionBenchmarkBase::BufferSize, SweepMaxSize, 162 MaybeAlign(AlignedAccess)) {} 163 164 virtual void randomize() override { 165 for (auto &P : Parameters) { 166 P.OffsetBytes = OffsetSampler(Gen); 167 P.SizeBytes = CurrentSweepSize; 168 checkValid(P); 169 } 170 } 171 172 virtual Study run() override { 173 Study Study = createStudy(); 174 Study.Configuration.SweepModeMaxSize = SweepMaxSize; 175 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions; 176 BO.MinDuration = std::chrono::milliseconds(1); 177 BO.InitialIterations = 100; 178 auto &Measurements = Study.Measurements; 179 Measurements.reserve(NumTrials * SweepMaxSize); 180 for (size_t Size = 0; Size <= SweepMaxSize; ++Size) { 181 CurrentSweepSize = Size; 182 runTrials(BO, Measurements); 183 } 184 return Study; 185 } 186 187 private: 188 size_t CurrentSweepSize = 0; 189 OffsetDistribution OffsetSampler; 190 std::mt19937_64 Gen; 191 }; 192 193 struct MemfunctionBenchmarkDistribution final 194 : public MemfunctionBenchmarkBase { 195 MemfunctionBenchmarkDistribution(MemorySizeDistribution Distribution) 196 : Distribution(Distribution), Probabilities(Distribution.Probabilities), 197 SizeSampler(Probabilities.begin(), Probabilities.end()), 198 OffsetSampler(MemfunctionBenchmarkBase::BufferSize, 199 Probabilities.size() - 1, MaybeAlign(AlignedAccess)) {} 200 201 virtual void randomize() override { 202 for (auto &P : Parameters) { 203 P.OffsetBytes = OffsetSampler(Gen); 204 P.SizeBytes = SizeSampler(Gen); 205 checkValid(P); 206 } 207 } 208 209 virtual Study run() override { 210 Study Study = createStudy(); 211 Study.Configuration.SizeDistributionName = Distribution.Name.str(); 212 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions; 213 BO.MinDuration = std::chrono::milliseconds(10); 214 BO.InitialIterations = BatchSize * 10; 215 auto &Measurements = Study.Measurements; 216 Measurements.reserve(NumTrials); 217 runTrials(BO, Measurements); 218 return Study; 219 } 220 221 private: 222 MemorySizeDistribution Distribution; 223 ArrayRef<double> Probabilities; 224 std::discrete_distribution<unsigned> SizeSampler; 225 OffsetDistribution OffsetSampler; 226 std::mt19937_64 Gen; 227 }; 228 229 void writeStudy(const Study &S) { 230 std::error_code EC; 231 raw_fd_ostream FOS(Output, EC); 232 if (EC) 233 report_fatal_error(Twine("Could not open file: ") 234 .concat(EC.message()) 235 .concat(", ") 236 .concat(Output)); 237 json::OStream JOS(FOS); 238 serializeToJson(S, JOS); 239 FOS << "\n"; 240 } 241 242 void main() { 243 checkRequirements(); 244 if (!isPowerOf2_32(AlignedAccess)) 245 report_fatal_error(AlignedAccess.ArgStr + 246 Twine(" must be a power of two or zero")); 247 248 const bool HasDistributionName = !SizeDistributionName.empty(); 249 if (SweepMode && HasDistributionName) 250 report_fatal_error("Select only one of `--" + Twine(SweepMode.ArgStr) + 251 "` or `--" + Twine(SizeDistributionName.ArgStr) + "`"); 252 253 std::unique_ptr<MemfunctionBenchmarkBase> Benchmark; 254 if (SweepMode) 255 Benchmark.reset(new MemfunctionBenchmarkSweep()); 256 else 257 Benchmark.reset(new MemfunctionBenchmarkDistribution(getDistributionOrDie( 258 BenchmarkSetup::getDistributions(), SizeDistributionName))); 259 writeStudy(Benchmark->run()); 260 } 261 262 } // namespace libc_benchmarks 263 } // namespace llvm 264 265 #ifndef NDEBUG 266 #error For reproducibility benchmarks should not be compiled in DEBUG mode. 267 #endif 268 269 int main(int argc, char **argv) { 270 llvm::cl::ParseCommandLineOptions(argc, argv); 271 llvm::libc_benchmarks::main(); 272 return EXIT_SUCCESS; 273 } 274