1 //===-- Benchmark ---------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "JSON.h" 10 #include "LibcBenchmark.h" 11 #include "LibcMemoryBenchmark.h" 12 #include "MemorySizeDistributions.h" 13 #include "llvm/Support/CommandLine.h" 14 #include "llvm/Support/ErrorHandling.h" 15 #include "llvm/Support/FileSystem.h" 16 #include "llvm/Support/JSON.h" 17 #include "llvm/Support/MathExtras.h" 18 #include "llvm/Support/MemoryBuffer.h" 19 #include "llvm/Support/raw_ostream.h" 20 21 #include <cstring> 22 #include <unistd.h> 23 24 namespace LIBC_NAMESPACE { 25 26 extern void *memcpy(void *__restrict, const void *__restrict, size_t); 27 extern void *memmove(void *, const void *, size_t); 28 extern void *memset(void *, int, size_t); 29 extern void bzero(void *, size_t); 30 extern int memcmp(const void *, const void *, size_t); 31 extern int bcmp(const void *, const void *, size_t); 32 33 } // namespace LIBC_NAMESPACE 34 35 namespace llvm { 36 namespace libc_benchmarks { 37 38 static cl::opt<std::string> 39 StudyName("study-name", cl::desc("The name for this study"), cl::Required); 40 41 static cl::opt<std::string> 42 SizeDistributionName("size-distribution-name", 43 cl::desc("The name of the distribution to use")); 44 45 static cl::opt<bool> SweepMode( 46 "sweep-mode", 47 cl::desc( 48 "If set, benchmark all sizes from sweep-min-size to sweep-max-size")); 49 50 static cl::opt<uint32_t> 51 SweepMinSize("sweep-min-size", 52 cl::desc("The minimum size to use in sweep-mode"), 53 cl::init(0)); 54 55 static cl::opt<uint32_t> 56 SweepMaxSize("sweep-max-size", 57 cl::desc("The maximum size to use in sweep-mode"), 58 cl::init(256)); 59 60 static cl::opt<uint32_t> 61 AlignedAccess("aligned-access", 62 cl::desc("The alignment to use when accessing the buffers\n" 63 "Default is unaligned\n" 64 "Use 0 to disable address randomization"), 65 cl::init(1)); 66 67 static cl::opt<std::string> Output("output", 68 cl::desc("Specify output filename"), 69 cl::value_desc("filename"), cl::init("-")); 70 71 static cl::opt<uint32_t> 72 NumTrials("num-trials", cl::desc("The number of benchmarks run to perform"), 73 cl::init(1)); 74 75 #if defined(LIBC_BENCHMARK_FUNCTION_MEMCPY) 76 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCPY 77 using BenchmarkSetup = CopySetup; 78 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMMOVE) 79 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMMOVE 80 using BenchmarkSetup = MoveSetup; 81 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMSET) 82 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMSET 83 using BenchmarkSetup = SetSetup; 84 #elif defined(LIBC_BENCHMARK_FUNCTION_BZERO) 85 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BZERO 86 using BenchmarkSetup = SetSetup; 87 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMCMP) 88 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCMP 89 using BenchmarkSetup = ComparisonSetup; 90 #elif defined(LIBC_BENCHMARK_FUNCTION_BCMP) 91 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BCMP 92 using BenchmarkSetup = ComparisonSetup; 93 #else 94 #error "Missing LIBC_BENCHMARK_FUNCTION_XXX definition" 95 #endif 96 97 struct MemfunctionBenchmarkBase : public BenchmarkSetup { 98 MemfunctionBenchmarkBase() : ReportProgress(isatty(fileno(stdout))) {} 99 virtual ~MemfunctionBenchmarkBase() {} 100 101 virtual Study run() = 0; 102 103 CircularArrayRef<ParameterBatch::ParameterType> 104 generateBatch(size_t Iterations) { 105 randomize(); 106 return cycle(ArrayRef(Parameters), Iterations); 107 } 108 109 protected: 110 Study createStudy() { 111 Study Study; 112 // Setup study. 113 Study.StudyName = StudyName; 114 Runtime &RI = Study.Runtime; 115 RI.Host = HostState::get(); 116 RI.BufferSize = BufferSize; 117 RI.BatchParameterCount = BatchSize; 118 119 BenchmarkOptions &BO = RI.BenchmarkOptions; 120 BO.MinDuration = std::chrono::milliseconds(1); 121 BO.MaxDuration = std::chrono::seconds(1); 122 BO.MaxIterations = 10'000'000U; 123 BO.MinSamples = 4; 124 BO.MaxSamples = 1000; 125 BO.Epsilon = 0.01; // 1% 126 BO.ScalingFactor = 1.4; 127 128 StudyConfiguration &SC = Study.Configuration; 129 SC.NumTrials = NumTrials; 130 SC.IsSweepMode = SweepMode; 131 SC.AccessAlignment = MaybeAlign(AlignedAccess); 132 SC.Function = LIBC_BENCHMARK_FUNCTION_NAME; 133 return Study; 134 } 135 136 void runTrials(const BenchmarkOptions &Options, 137 std::vector<Duration> &Measurements) { 138 for (size_t i = 0; i < NumTrials; ++i) { 139 const BenchmarkResult Result = benchmark( 140 Options, *this, [this](ParameterBatch::ParameterType Parameter) { 141 return Call(Parameter, LIBC_BENCHMARK_FUNCTION); 142 }); 143 Measurements.push_back(Result.BestGuess); 144 reportProgress(Measurements); 145 } 146 } 147 148 virtual void randomize() = 0; 149 150 private: 151 bool ReportProgress; 152 153 void reportProgress(const std::vector<Duration> &Measurements) { 154 if (!ReportProgress) 155 return; 156 static size_t LastPercent = -1; 157 const size_t TotalSteps = Measurements.capacity(); 158 const size_t Steps = Measurements.size(); 159 const size_t Percent = 100 * Steps / TotalSteps; 160 if (Percent == LastPercent) 161 return; 162 LastPercent = Percent; 163 size_t I = 0; 164 errs() << '['; 165 for (; I <= Percent; ++I) 166 errs() << '#'; 167 for (; I <= 100; ++I) 168 errs() << '_'; 169 errs() << "] " << Percent << '%' << '\r'; 170 } 171 }; 172 173 struct MemfunctionBenchmarkSweep final : public MemfunctionBenchmarkBase { 174 MemfunctionBenchmarkSweep() 175 : OffsetSampler(MemfunctionBenchmarkBase::BufferSize, SweepMaxSize, 176 MaybeAlign(AlignedAccess)) {} 177 178 virtual void randomize() override { 179 for (auto &P : Parameters) { 180 P.OffsetBytes = OffsetSampler(Gen); 181 P.SizeBytes = CurrentSweepSize; 182 checkValid(P); 183 } 184 } 185 186 virtual Study run() override { 187 Study Study = createStudy(); 188 Study.Configuration.SweepModeMaxSize = SweepMaxSize; 189 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions; 190 BO.MinDuration = std::chrono::milliseconds(1); 191 BO.InitialIterations = 100; 192 auto &Measurements = Study.Measurements; 193 Measurements.reserve(NumTrials * SweepMaxSize); 194 for (size_t Size = SweepMinSize; Size <= SweepMaxSize; ++Size) { 195 CurrentSweepSize = Size; 196 runTrials(BO, Measurements); 197 } 198 return Study; 199 } 200 201 private: 202 size_t CurrentSweepSize = 0; 203 OffsetDistribution OffsetSampler; 204 std::mt19937_64 Gen; 205 }; 206 207 struct MemfunctionBenchmarkDistribution final 208 : public MemfunctionBenchmarkBase { 209 MemfunctionBenchmarkDistribution(MemorySizeDistribution Distribution) 210 : Distribution(Distribution), Probabilities(Distribution.Probabilities), 211 SizeSampler(Probabilities.begin(), Probabilities.end()), 212 OffsetSampler(MemfunctionBenchmarkBase::BufferSize, 213 Probabilities.size() - 1, MaybeAlign(AlignedAccess)) {} 214 215 virtual void randomize() override { 216 for (auto &P : Parameters) { 217 P.OffsetBytes = OffsetSampler(Gen); 218 P.SizeBytes = SizeSampler(Gen); 219 checkValid(P); 220 } 221 } 222 223 virtual Study run() override { 224 Study Study = createStudy(); 225 Study.Configuration.SizeDistributionName = Distribution.Name.str(); 226 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions; 227 BO.MinDuration = std::chrono::milliseconds(10); 228 BO.InitialIterations = BatchSize * 10; 229 auto &Measurements = Study.Measurements; 230 Measurements.reserve(NumTrials); 231 runTrials(BO, Measurements); 232 return Study; 233 } 234 235 private: 236 MemorySizeDistribution Distribution; 237 ArrayRef<double> Probabilities; 238 std::discrete_distribution<unsigned> SizeSampler; 239 OffsetDistribution OffsetSampler; 240 std::mt19937_64 Gen; 241 }; 242 243 void writeStudy(const Study &S) { 244 std::error_code EC; 245 raw_fd_ostream FOS(Output, EC); 246 if (EC) 247 report_fatal_error(Twine("Could not open file: ") 248 .concat(EC.message()) 249 .concat(", ") 250 .concat(Output)); 251 json::OStream JOS(FOS); 252 serializeToJson(S, JOS); 253 FOS << "\n"; 254 } 255 256 void main() { 257 checkRequirements(); 258 if (!isPowerOf2_32(AlignedAccess)) 259 report_fatal_error(AlignedAccess.ArgStr + 260 Twine(" must be a power of two or zero")); 261 262 const bool HasDistributionName = !SizeDistributionName.empty(); 263 if (SweepMode && HasDistributionName) 264 report_fatal_error("Select only one of `--" + Twine(SweepMode.ArgStr) + 265 "` or `--" + Twine(SizeDistributionName.ArgStr) + "`"); 266 267 std::unique_ptr<MemfunctionBenchmarkBase> Benchmark; 268 if (SweepMode) 269 Benchmark.reset(new MemfunctionBenchmarkSweep()); 270 else 271 Benchmark.reset(new MemfunctionBenchmarkDistribution(getDistributionOrDie( 272 BenchmarkSetup::getDistributions(), SizeDistributionName))); 273 writeStudy(Benchmark->run()); 274 } 275 276 } // namespace libc_benchmarks 277 } // namespace llvm 278 279 #ifndef NDEBUG 280 #error For reproducibility benchmarks should not be compiled in DEBUG mode. 281 #endif 282 283 int main(int argc, char **argv) { 284 llvm::cl::ParseCommandLineOptions(argc, argv); 285 llvm::libc_benchmarks::main(); 286 return EXIT_SUCCESS; 287 } 288