1 //===-- Benchmark ---------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "JSON.h" 10 #include "LibcBenchmark.h" 11 #include "LibcMemoryBenchmark.h" 12 #include "MemorySizeDistributions.h" 13 #include "llvm/Support/CommandLine.h" 14 #include "llvm/Support/ErrorHandling.h" 15 #include "llvm/Support/FileSystem.h" 16 #include "llvm/Support/JSON.h" 17 #include "llvm/Support/MathExtras.h" 18 #include "llvm/Support/MemoryBuffer.h" 19 #include "llvm/Support/raw_ostream.h" 20 21 #include <cstring> 22 #include <unistd.h> 23 24 namespace LIBC_NAMESPACE { 25 26 extern void *memcpy(void *__restrict, const void *__restrict, size_t); 27 extern void *memmove(void *, const void *, size_t); 28 extern void *memset(void *, int, size_t); 29 extern void bzero(void *, size_t); 30 extern int memcmp(const void *, const void *, size_t); 31 extern int bcmp(const void *, const void *, size_t); 32 33 } // namespace LIBC_NAMESPACE 34 35 namespace llvm { 36 namespace libc_benchmarks { 37 38 static cl::opt<std::string> 39 StudyName("study-name", cl::desc("The name for this study"), cl::Required); 40 41 static cl::opt<std::string> 42 SizeDistributionName("size-distribution-name", 43 cl::desc("The name of the distribution to use")); 44 45 static cl::opt<bool> 46 SweepMode("sweep-mode", 47 cl::desc("If set, benchmark all sizes from 0 to sweep-max-size")); 48 49 static cl::opt<uint32_t> 50 SweepMaxSize("sweep-max-size", 51 cl::desc("The maximum size to use in sweep-mode"), 52 cl::init(256)); 53 54 static cl::opt<uint32_t> 55 AlignedAccess("aligned-access", 56 cl::desc("The alignment to use when accessing the buffers\n" 57 "Default is unaligned\n" 58 "Use 0 to disable address randomization"), 59 cl::init(1)); 60 61 static cl::opt<std::string> Output("output", 62 cl::desc("Specify output filename"), 63 cl::value_desc("filename"), cl::init("-")); 64 65 static cl::opt<uint32_t> 66 NumTrials("num-trials", cl::desc("The number of benchmarks run to perform"), 67 cl::init(1)); 68 69 #if defined(LIBC_BENCHMARK_FUNCTION_MEMCPY) 70 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCPY 71 using BenchmarkSetup = CopySetup; 72 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMMOVE) 73 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMMOVE 74 using BenchmarkSetup = MoveSetup; 75 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMSET) 76 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMSET 77 using BenchmarkSetup = SetSetup; 78 #elif defined(LIBC_BENCHMARK_FUNCTION_BZERO) 79 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BZERO 80 using BenchmarkSetup = SetSetup; 81 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMCMP) 82 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCMP 83 using BenchmarkSetup = ComparisonSetup; 84 #elif defined(LIBC_BENCHMARK_FUNCTION_BCMP) 85 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BCMP 86 using BenchmarkSetup = ComparisonSetup; 87 #else 88 #error "Missing LIBC_BENCHMARK_FUNCTION_XXX definition" 89 #endif 90 91 struct MemfunctionBenchmarkBase : public BenchmarkSetup { 92 MemfunctionBenchmarkBase() : ReportProgress(isatty(fileno(stdout))) {} 93 virtual ~MemfunctionBenchmarkBase() {} 94 95 virtual Study run() = 0; 96 97 CircularArrayRef<ParameterBatch::ParameterType> 98 generateBatch(size_t Iterations) { 99 randomize(); 100 return cycle(ArrayRef(Parameters), Iterations); 101 } 102 103 protected: 104 Study createStudy() { 105 Study Study; 106 // Setup study. 107 Study.StudyName = StudyName; 108 Runtime &RI = Study.Runtime; 109 RI.Host = HostState::get(); 110 RI.BufferSize = BufferSize; 111 RI.BatchParameterCount = BatchSize; 112 113 BenchmarkOptions &BO = RI.BenchmarkOptions; 114 BO.MinDuration = std::chrono::milliseconds(1); 115 BO.MaxDuration = std::chrono::seconds(1); 116 BO.MaxIterations = 10'000'000U; 117 BO.MinSamples = 4; 118 BO.MaxSamples = 1000; 119 BO.Epsilon = 0.01; // 1% 120 BO.ScalingFactor = 1.4; 121 122 StudyConfiguration &SC = Study.Configuration; 123 SC.NumTrials = NumTrials; 124 SC.IsSweepMode = SweepMode; 125 SC.AccessAlignment = MaybeAlign(AlignedAccess); 126 SC.Function = LIBC_BENCHMARK_FUNCTION_NAME; 127 return Study; 128 } 129 130 void runTrials(const BenchmarkOptions &Options, 131 std::vector<Duration> &Measurements) { 132 for (size_t i = 0; i < NumTrials; ++i) { 133 const BenchmarkResult Result = benchmark( 134 Options, *this, [this](ParameterBatch::ParameterType Parameter) { 135 return Call(Parameter, LIBC_BENCHMARK_FUNCTION); 136 }); 137 Measurements.push_back(Result.BestGuess); 138 reportProgress(Measurements); 139 } 140 } 141 142 virtual void randomize() = 0; 143 144 private: 145 bool ReportProgress; 146 147 void reportProgress(const std::vector<Duration> &Measurements) { 148 if (!ReportProgress) 149 return; 150 static size_t LastPercent = -1; 151 const size_t TotalSteps = Measurements.capacity(); 152 const size_t Steps = Measurements.size(); 153 const size_t Percent = 100 * Steps / TotalSteps; 154 if (Percent == LastPercent) 155 return; 156 LastPercent = Percent; 157 size_t I = 0; 158 errs() << '['; 159 for (; I <= Percent; ++I) 160 errs() << '#'; 161 for (; I <= 100; ++I) 162 errs() << '_'; 163 errs() << "] " << Percent << '%' << '\r'; 164 } 165 }; 166 167 struct MemfunctionBenchmarkSweep final : public MemfunctionBenchmarkBase { 168 MemfunctionBenchmarkSweep() 169 : OffsetSampler(MemfunctionBenchmarkBase::BufferSize, SweepMaxSize, 170 MaybeAlign(AlignedAccess)) {} 171 172 virtual void randomize() override { 173 for (auto &P : Parameters) { 174 P.OffsetBytes = OffsetSampler(Gen); 175 P.SizeBytes = CurrentSweepSize; 176 checkValid(P); 177 } 178 } 179 180 virtual Study run() override { 181 Study Study = createStudy(); 182 Study.Configuration.SweepModeMaxSize = SweepMaxSize; 183 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions; 184 BO.MinDuration = std::chrono::milliseconds(1); 185 BO.InitialIterations = 100; 186 auto &Measurements = Study.Measurements; 187 Measurements.reserve(NumTrials * SweepMaxSize); 188 for (size_t Size = 0; Size <= SweepMaxSize; ++Size) { 189 CurrentSweepSize = Size; 190 runTrials(BO, Measurements); 191 } 192 return Study; 193 } 194 195 private: 196 size_t CurrentSweepSize = 0; 197 OffsetDistribution OffsetSampler; 198 std::mt19937_64 Gen; 199 }; 200 201 struct MemfunctionBenchmarkDistribution final 202 : public MemfunctionBenchmarkBase { 203 MemfunctionBenchmarkDistribution(MemorySizeDistribution Distribution) 204 : Distribution(Distribution), Probabilities(Distribution.Probabilities), 205 SizeSampler(Probabilities.begin(), Probabilities.end()), 206 OffsetSampler(MemfunctionBenchmarkBase::BufferSize, 207 Probabilities.size() - 1, MaybeAlign(AlignedAccess)) {} 208 209 virtual void randomize() override { 210 for (auto &P : Parameters) { 211 P.OffsetBytes = OffsetSampler(Gen); 212 P.SizeBytes = SizeSampler(Gen); 213 checkValid(P); 214 } 215 } 216 217 virtual Study run() override { 218 Study Study = createStudy(); 219 Study.Configuration.SizeDistributionName = Distribution.Name.str(); 220 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions; 221 BO.MinDuration = std::chrono::milliseconds(10); 222 BO.InitialIterations = BatchSize * 10; 223 auto &Measurements = Study.Measurements; 224 Measurements.reserve(NumTrials); 225 runTrials(BO, Measurements); 226 return Study; 227 } 228 229 private: 230 MemorySizeDistribution Distribution; 231 ArrayRef<double> Probabilities; 232 std::discrete_distribution<unsigned> SizeSampler; 233 OffsetDistribution OffsetSampler; 234 std::mt19937_64 Gen; 235 }; 236 237 void writeStudy(const Study &S) { 238 std::error_code EC; 239 raw_fd_ostream FOS(Output, EC); 240 if (EC) 241 report_fatal_error(Twine("Could not open file: ") 242 .concat(EC.message()) 243 .concat(", ") 244 .concat(Output)); 245 json::OStream JOS(FOS); 246 serializeToJson(S, JOS); 247 FOS << "\n"; 248 } 249 250 void main() { 251 checkRequirements(); 252 if (!isPowerOf2_32(AlignedAccess)) 253 report_fatal_error(AlignedAccess.ArgStr + 254 Twine(" must be a power of two or zero")); 255 256 const bool HasDistributionName = !SizeDistributionName.empty(); 257 if (SweepMode && HasDistributionName) 258 report_fatal_error("Select only one of `--" + Twine(SweepMode.ArgStr) + 259 "` or `--" + Twine(SizeDistributionName.ArgStr) + "`"); 260 261 std::unique_ptr<MemfunctionBenchmarkBase> Benchmark; 262 if (SweepMode) 263 Benchmark.reset(new MemfunctionBenchmarkSweep()); 264 else 265 Benchmark.reset(new MemfunctionBenchmarkDistribution(getDistributionOrDie( 266 BenchmarkSetup::getDistributions(), SizeDistributionName))); 267 writeStudy(Benchmark->run()); 268 } 269 270 } // namespace libc_benchmarks 271 } // namespace llvm 272 273 #ifndef NDEBUG 274 #error For reproducibility benchmarks should not be compiled in DEBUG mode. 275 #endif 276 277 int main(int argc, char **argv) { 278 llvm::cl::ParseCommandLineOptions(argc, argv); 279 llvm::libc_benchmarks::main(); 280 return EXIT_SUCCESS; 281 } 282