10b57cec5SDimitry Andric //===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // Instrumentation-based profile-guided optimization 100b57cec5SDimitry Andric // 110b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 120b57cec5SDimitry Andric 130b57cec5SDimitry Andric #ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H 140b57cec5SDimitry Andric #define LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H 150b57cec5SDimitry Andric 160b57cec5SDimitry Andric #include "CGBuilder.h" 170b57cec5SDimitry Andric #include "CodeGenModule.h" 180b57cec5SDimitry Andric #include "CodeGenTypes.h" 19*0fca6ea1SDimitry Andric #include "MCDCState.h" 200b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfReader.h" 210b57cec5SDimitry Andric #include <array> 220b57cec5SDimitry Andric #include <memory> 23bdd1243dSDimitry Andric #include <optional> 240b57cec5SDimitry Andric 250b57cec5SDimitry Andric namespace clang { 260b57cec5SDimitry Andric namespace CodeGen { 270b57cec5SDimitry Andric 280b57cec5SDimitry Andric /// Per-function PGO state. 290b57cec5SDimitry Andric class CodeGenPGO { 300b57cec5SDimitry Andric private: 310b57cec5SDimitry Andric CodeGenModule &CGM; 320b57cec5SDimitry Andric std::string FuncName; 330b57cec5SDimitry Andric llvm::GlobalVariable *FuncNameVar; 340b57cec5SDimitry Andric 350b57cec5SDimitry Andric std::array <unsigned, llvm::IPVK_Last + 1> NumValueSites; 360b57cec5SDimitry Andric unsigned NumRegionCounters; 370b57cec5SDimitry Andric uint64_t FunctionHash; 380b57cec5SDimitry Andric std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap; 390b57cec5SDimitry Andric std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap; 400b57cec5SDimitry Andric std::unique_ptr<llvm::InstrProfRecord> ProfRecord; 41*0fca6ea1SDimitry Andric std::unique_ptr<MCDC::State> RegionMCDCState; 420b57cec5SDimitry Andric std::vector<uint64_t> RegionCounts; 430b57cec5SDimitry Andric uint64_t CurrentRegionCount; 440b57cec5SDimitry Andric 450b57cec5SDimitry Andric public: 465ffd83dbSDimitry Andric CodeGenPGO(CodeGenModule &CGModule) 475ffd83dbSDimitry Andric : CGM(CGModule), FuncNameVar(nullptr), NumValueSites({{0}}), 48*0fca6ea1SDimitry Andric NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0) {} 490b57cec5SDimitry Andric 500b57cec5SDimitry Andric /// Whether or not we have PGO region data for the current function. This is 510b57cec5SDimitry Andric /// false both when we have no data at all and when our data has been 520b57cec5SDimitry Andric /// discarded. 530b57cec5SDimitry Andric bool haveRegionCounts() const { return !RegionCounts.empty(); } 540b57cec5SDimitry Andric 550b57cec5SDimitry Andric /// Return the counter value of the current region. 560b57cec5SDimitry Andric uint64_t getCurrentRegionCount() const { return CurrentRegionCount; } 570b57cec5SDimitry Andric 580b57cec5SDimitry Andric /// Set the counter value for the current region. This is used to keep track 590b57cec5SDimitry Andric /// of changes to the most recent counter from control flow and non-local 600b57cec5SDimitry Andric /// exits. 610b57cec5SDimitry Andric void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; } 620b57cec5SDimitry Andric 630b57cec5SDimitry Andric /// Check if an execution count is known for a given statement. If so, return 640b57cec5SDimitry Andric /// true and put the value in Count; else return false. 65bdd1243dSDimitry Andric std::optional<uint64_t> getStmtCount(const Stmt *S) const { 660b57cec5SDimitry Andric if (!StmtCountMap) 67bdd1243dSDimitry Andric return std::nullopt; 680b57cec5SDimitry Andric auto I = StmtCountMap->find(S); 690b57cec5SDimitry Andric if (I == StmtCountMap->end()) 70bdd1243dSDimitry Andric return std::nullopt; 710b57cec5SDimitry Andric return I->second; 720b57cec5SDimitry Andric } 730b57cec5SDimitry Andric 740b57cec5SDimitry Andric /// If the execution count for the current statement is known, record that 750b57cec5SDimitry Andric /// as the current count. 760b57cec5SDimitry Andric void setCurrentStmt(const Stmt *S) { 770b57cec5SDimitry Andric if (auto Count = getStmtCount(S)) 780b57cec5SDimitry Andric setCurrentRegionCount(*Count); 790b57cec5SDimitry Andric } 800b57cec5SDimitry Andric 810b57cec5SDimitry Andric /// Assign counters to regions and configure them for PGO of a given 820b57cec5SDimitry Andric /// function. Does nothing if instrumentation is not enabled and either 830b57cec5SDimitry Andric /// generates global variables or associates PGO data with each of the 840b57cec5SDimitry Andric /// counters depending on whether we are generating or using instrumentation. 850b57cec5SDimitry Andric void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn); 860b57cec5SDimitry Andric /// Emit a coverage mapping range with a counter zero 870b57cec5SDimitry Andric /// for an unused declaration. 880b57cec5SDimitry Andric void emitEmptyCounterMapping(const Decl *D, StringRef FuncName, 890b57cec5SDimitry Andric llvm::GlobalValue::LinkageTypes Linkage); 900b57cec5SDimitry Andric // Insert instrumentation or attach profile metadata at value sites 910b57cec5SDimitry Andric void valueProfile(CGBuilderTy &Builder, uint32_t ValueKind, 920b57cec5SDimitry Andric llvm::Instruction *ValueSite, llvm::Value *ValuePtr); 93fe6060f1SDimitry Andric 94fe6060f1SDimitry Andric // Set a module flag indicating if value profiling is enabled. 95fe6060f1SDimitry Andric void setValueProfilingFlag(llvm::Module &M); 96fe6060f1SDimitry Andric 97*0fca6ea1SDimitry Andric void setProfileVersion(llvm::Module &M); 98*0fca6ea1SDimitry Andric 990b57cec5SDimitry Andric private: 1000b57cec5SDimitry Andric void setFuncName(llvm::Function *Fn); 1010b57cec5SDimitry Andric void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage); 1020b57cec5SDimitry Andric void mapRegionCounters(const Decl *D); 1030b57cec5SDimitry Andric void computeRegionCounts(const Decl *D); 1040b57cec5SDimitry Andric void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, 1050b57cec5SDimitry Andric llvm::Function *Fn); 1060b57cec5SDimitry Andric void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader, 1070b57cec5SDimitry Andric bool IsInMainFile); 1080b57cec5SDimitry Andric bool skipRegionMappingForDecl(const Decl *D); 1090b57cec5SDimitry Andric void emitCounterRegionMapping(const Decl *D); 1101db9f3b2SDimitry Andric bool canEmitMCDCCoverage(const CGBuilderTy &Builder); 1110b57cec5SDimitry Andric 1120b57cec5SDimitry Andric public: 113*0fca6ea1SDimitry Andric void emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S, 1140b57cec5SDimitry Andric llvm::Value *StepV); 1151db9f3b2SDimitry Andric void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, 116*0fca6ea1SDimitry Andric Address MCDCCondBitmapAddr, 117*0fca6ea1SDimitry Andric CodeGenFunction &CGF); 1181db9f3b2SDimitry Andric void emitMCDCParameters(CGBuilderTy &Builder); 1191db9f3b2SDimitry Andric void emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, 1201db9f3b2SDimitry Andric Address MCDCCondBitmapAddr); 1211db9f3b2SDimitry Andric void emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, 122*0fca6ea1SDimitry Andric Address MCDCCondBitmapAddr, llvm::Value *Val, 123*0fca6ea1SDimitry Andric CodeGenFunction &CGF); 1240b57cec5SDimitry Andric 1250b57cec5SDimitry Andric /// Return the region count for the counter at the given index. 1260b57cec5SDimitry Andric uint64_t getRegionCount(const Stmt *S) { 1270b57cec5SDimitry Andric if (!RegionCounterMap) 1280b57cec5SDimitry Andric return 0; 1290b57cec5SDimitry Andric if (!haveRegionCounts()) 1300b57cec5SDimitry Andric return 0; 13106c3fb27SDimitry Andric // With profiles from a differing version of clang we can have mismatched 13206c3fb27SDimitry Andric // decl counts. Don't crash in such a case. 13306c3fb27SDimitry Andric auto Index = (*RegionCounterMap)[S]; 13406c3fb27SDimitry Andric if (Index >= RegionCounts.size()) 13506c3fb27SDimitry Andric return 0; 13606c3fb27SDimitry Andric return RegionCounts[Index]; 1370b57cec5SDimitry Andric } 1380b57cec5SDimitry Andric }; 1390b57cec5SDimitry Andric 1400b57cec5SDimitry Andric } // end namespace CodeGen 1410b57cec5SDimitry Andric } // end namespace clang 1420b57cec5SDimitry Andric 1430b57cec5SDimitry Andric #endif 144