1 //===-- Benchmark ---------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "JSON.h" 10 #include "LibcBenchmark.h" 11 #include "LibcMemoryBenchmark.h" 12 #include "MemorySizeDistributions.h" 13 #include "src/__support/macros/config.h" 14 #include "llvm/Support/CommandLine.h" 15 #include "llvm/Support/ErrorHandling.h" 16 #include "llvm/Support/FileSystem.h" 17 #include "llvm/Support/JSON.h" 18 #include "llvm/Support/MathExtras.h" 19 #include "llvm/Support/MemoryBuffer.h" 20 #include "llvm/Support/raw_ostream.h" 21 22 #include <cstring> 23 #include <unistd.h> 24 25 namespace LIBC_NAMESPACE_DECL { 26 27 extern void *memcpy(void *__restrict, const void *__restrict, size_t); 28 extern void *memmove(void *, const void *, size_t); 29 extern void *memset(void *, int, size_t); 30 extern void bzero(void *, size_t); 31 extern int memcmp(const void *, const void *, size_t); 32 extern int bcmp(const void *, const void *, size_t); 33 34 } // namespace LIBC_NAMESPACE_DECL 35 36 namespace llvm { 37 namespace libc_benchmarks { 38 39 static cl::opt<std::string> 40 StudyName("study-name", cl::desc("The name for this study"), cl::Required); 41 42 static cl::opt<std::string> 43 SizeDistributionName("size-distribution-name", 44 cl::desc("The name of the distribution to use")); 45 46 static cl::opt<bool> SweepMode( 47 "sweep-mode", 48 cl::desc( 49 "If set, benchmark all sizes from sweep-min-size to sweep-max-size")); 50 51 static cl::opt<uint32_t> 52 SweepMinSize("sweep-min-size", 53 cl::desc("The minimum size to use in sweep-mode"), 54 cl::init(0)); 55 56 static cl::opt<uint32_t> 57 SweepMaxSize("sweep-max-size", 58 cl::desc("The maximum size to use in sweep-mode"), 59 cl::init(256)); 60 61 static cl::opt<uint32_t> 62 AlignedAccess("aligned-access", 63 cl::desc("The alignment to use when accessing the buffers\n" 64 "Default is unaligned\n" 65 "Use 0 to disable address randomization"), 66 cl::init(1)); 67 68 static cl::opt<std::string> Output("output", 69 cl::desc("Specify output filename"), 70 cl::value_desc("filename"), cl::init("-")); 71 72 static cl::opt<uint32_t> 73 NumTrials("num-trials", cl::desc("The number of benchmarks run to perform"), 74 cl::init(1)); 75 76 #if defined(LIBC_BENCHMARK_FUNCTION_MEMCPY) 77 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCPY 78 using BenchmarkSetup = CopySetup; 79 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMMOVE) 80 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMMOVE 81 using BenchmarkSetup = MoveSetup; 82 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMSET) 83 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMSET 84 using BenchmarkSetup = SetSetup; 85 #elif defined(LIBC_BENCHMARK_FUNCTION_BZERO) 86 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BZERO 87 using BenchmarkSetup = SetSetup; 88 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMCMP) 89 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCMP 90 using BenchmarkSetup = ComparisonSetup; 91 #elif defined(LIBC_BENCHMARK_FUNCTION_BCMP) 92 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BCMP 93 using BenchmarkSetup = ComparisonSetup; 94 #else 95 #error "Missing LIBC_BENCHMARK_FUNCTION_XXX definition" 96 #endif 97 98 struct MemfunctionBenchmarkBase : public BenchmarkSetup { 99 MemfunctionBenchmarkBase() : ReportProgress(isatty(fileno(stdout))) {} 100 virtual ~MemfunctionBenchmarkBase() {} 101 102 virtual Study run() = 0; 103 104 CircularArrayRef<ParameterBatch::ParameterType> 105 generateBatch(size_t Iterations) { 106 randomize(); 107 return cycle(ArrayRef(Parameters), Iterations); 108 } 109 110 protected: 111 Study createStudy() { 112 Study Study; 113 // Setup study. 114 Study.StudyName = StudyName; 115 Runtime &RI = Study.Runtime; 116 RI.Host = HostState::get(); 117 RI.BufferSize = BufferSize; 118 RI.BatchParameterCount = BatchSize; 119 120 BenchmarkOptions &BO = RI.BenchmarkOptions; 121 BO.MinDuration = std::chrono::milliseconds(1); 122 BO.MaxDuration = std::chrono::seconds(1); 123 BO.MaxIterations = 10'000'000U; 124 BO.MinSamples = 4; 125 BO.MaxSamples = 1000; 126 BO.Epsilon = 0.01; // 1% 127 BO.ScalingFactor = 1.4; 128 129 StudyConfiguration &SC = Study.Configuration; 130 SC.NumTrials = NumTrials; 131 SC.IsSweepMode = SweepMode; 132 SC.AccessAlignment = MaybeAlign(AlignedAccess); 133 SC.Function = LIBC_BENCHMARK_FUNCTION_NAME; 134 return Study; 135 } 136 137 void runTrials(const BenchmarkOptions &Options, 138 std::vector<Duration> &Measurements) { 139 for (size_t i = 0; i < NumTrials; ++i) { 140 const BenchmarkResult Result = benchmark( 141 Options, *this, [this](ParameterBatch::ParameterType Parameter) { 142 return Call(Parameter, LIBC_BENCHMARK_FUNCTION); 143 }); 144 Measurements.push_back(Result.BestGuess); 145 reportProgress(Measurements); 146 } 147 } 148 149 virtual void randomize() = 0; 150 151 private: 152 bool ReportProgress; 153 154 void reportProgress(const std::vector<Duration> &Measurements) { 155 if (!ReportProgress) 156 return; 157 static size_t LastPercent = -1; 158 const size_t TotalSteps = Measurements.capacity(); 159 const size_t Steps = Measurements.size(); 160 const size_t Percent = 100 * Steps / TotalSteps; 161 if (Percent == LastPercent) 162 return; 163 LastPercent = Percent; 164 size_t I = 0; 165 errs() << '['; 166 for (; I <= Percent; ++I) 167 errs() << '#'; 168 for (; I <= 100; ++I) 169 errs() << '_'; 170 errs() << "] " << Percent << '%' << '\r'; 171 } 172 }; 173 174 struct MemfunctionBenchmarkSweep final : public MemfunctionBenchmarkBase { 175 MemfunctionBenchmarkSweep() 176 : OffsetSampler(MemfunctionBenchmarkBase::BufferSize, SweepMaxSize, 177 MaybeAlign(AlignedAccess)) {} 178 179 virtual void randomize() override { 180 for (auto &P : Parameters) { 181 P.OffsetBytes = OffsetSampler(Gen); 182 P.SizeBytes = CurrentSweepSize; 183 checkValid(P); 184 } 185 } 186 187 virtual Study run() override { 188 Study Study = createStudy(); 189 Study.Configuration.SweepModeMaxSize = SweepMaxSize; 190 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions; 191 BO.MinDuration = std::chrono::milliseconds(1); 192 BO.InitialIterations = 100; 193 auto &Measurements = Study.Measurements; 194 Measurements.reserve(NumTrials * SweepMaxSize); 195 for (size_t Size = SweepMinSize; Size <= SweepMaxSize; ++Size) { 196 CurrentSweepSize = Size; 197 runTrials(BO, Measurements); 198 } 199 return Study; 200 } 201 202 private: 203 size_t CurrentSweepSize = 0; 204 OffsetDistribution OffsetSampler; 205 std::mt19937_64 Gen; 206 }; 207 208 struct MemfunctionBenchmarkDistribution final 209 : public MemfunctionBenchmarkBase { 210 MemfunctionBenchmarkDistribution(MemorySizeDistribution Distribution) 211 : Distribution(Distribution), Probabilities(Distribution.Probabilities), 212 SizeSampler(Probabilities.begin(), Probabilities.end()), 213 OffsetSampler(MemfunctionBenchmarkBase::BufferSize, 214 Probabilities.size() - 1, MaybeAlign(AlignedAccess)) {} 215 216 virtual void randomize() override { 217 for (auto &P : Parameters) { 218 P.OffsetBytes = OffsetSampler(Gen); 219 P.SizeBytes = SizeSampler(Gen); 220 checkValid(P); 221 } 222 } 223 224 virtual Study run() override { 225 Study Study = createStudy(); 226 Study.Configuration.SizeDistributionName = Distribution.Name.str(); 227 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions; 228 BO.MinDuration = std::chrono::milliseconds(10); 229 BO.InitialIterations = BatchSize * 10; 230 auto &Measurements = Study.Measurements; 231 Measurements.reserve(NumTrials); 232 runTrials(BO, Measurements); 233 return Study; 234 } 235 236 private: 237 MemorySizeDistribution Distribution; 238 ArrayRef<double> Probabilities; 239 std::discrete_distribution<unsigned> SizeSampler; 240 OffsetDistribution OffsetSampler; 241 std::mt19937_64 Gen; 242 }; 243 244 void writeStudy(const Study &S) { 245 std::error_code EC; 246 raw_fd_ostream FOS(Output, EC); 247 if (EC) 248 report_fatal_error(Twine("Could not open file: ") 249 .concat(EC.message()) 250 .concat(", ") 251 .concat(Output)); 252 json::OStream JOS(FOS); 253 serializeToJson(S, JOS); 254 FOS << "\n"; 255 } 256 257 void main() { 258 checkRequirements(); 259 if (!isPowerOf2_32(AlignedAccess)) 260 report_fatal_error(AlignedAccess.ArgStr + 261 Twine(" must be a power of two or zero")); 262 263 const bool HasDistributionName = !SizeDistributionName.empty(); 264 if (SweepMode && HasDistributionName) 265 report_fatal_error("Select only one of `--" + Twine(SweepMode.ArgStr) + 266 "` or `--" + Twine(SizeDistributionName.ArgStr) + "`"); 267 268 std::unique_ptr<MemfunctionBenchmarkBase> Benchmark; 269 if (SweepMode) 270 Benchmark.reset(new MemfunctionBenchmarkSweep()); 271 else 272 Benchmark.reset(new MemfunctionBenchmarkDistribution(getDistributionOrDie( 273 BenchmarkSetup::getDistributions(), SizeDistributionName))); 274 writeStudy(Benchmark->run()); 275 } 276 277 } // namespace libc_benchmarks 278 } // namespace llvm 279 280 #ifndef NDEBUG 281 #error For reproducibility benchmarks should not be compiled in DEBUG mode. 282 #endif 283 284 int main(int argc, char **argv) { 285 llvm::cl::ParseCommandLineOptions(argc, argv); 286 llvm::libc_benchmarks::main(); 287 return EXIT_SUCCESS; 288 } 289