Lines Matching +full:fetch +full:- +full:depth
1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
13 //===----------------------------------------------------------------------===//
68 // ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
70 // TODO: Remove -debug-info-correlate in next LLVM release, in favor of
71 // -profile-correlate=debug-info.
73 "debug-info-correlate",
75 "-profile-correlate=debug-info)"),
79 "profile-correlate",
84 clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info",
93 "hash-based-counter-split",
98 RuntimeCounterRelocation("runtime-counter-relocation",
103 "vp-static-alloc",
108 "vp-counters-per-site",
112 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
113 // For those sites with non-zero profile, the average number of targets
118 "instrprof-atomic-counter-update-all",
123 "atomic-counter-update-promoted",
124 cl::desc("Do counter update using atomic fetch add "
129 "atomic-first-counter",
130 cl::desc("Use atomic fetch add for first counter in a function (usually "
135 "conditional-counter-update",
144 cl::opt<bool> DoCounterPromotion("do-counter-promotion",
148 "max-counter-promotions-per-loop", cl::init(20),
154 MaxNumOfPromotions("max-counter-promotions", cl::init(-1),
158 "speculative-counter-promotion-max-exiting", cl::init(3),
163 "speculative-counter-promotion-to-loop",
170 "iterative-counter-promotion", cl::init(true),
174 "skip-ret-exit-block", cl::init(true),
177 static cl::opt<bool> SampledInstr("sampled-instrumentation", cl::ZeroOrMore,
182 "sampled-instr-period",
186 "by 'sampled-instr-burst-duration' flag. The default sample "
193 "sampled-instr-burst-duration",
195 "from 1 to the value of 'sampled-instr-period' (0 is invalid). "
197 "'sampled-instr-period' count update. Setting to 1 enables simple "
199 "'sampled-instr-period' to a prime number."),
237 // in 64-bits.
238 return cast<ConstantInt>(MD->getValue())->getZExtValue();
265 // Is this lowering for the context-sensitive instrumentation.
306 /// Register-promote counter loads and stores in loops.
341 /// Get the Bias value for data to access mmap-ed area.
373 /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with
449 Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
450 Type *Ty = LiveInValue->getType();
459 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
460 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
461 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
463 PointerType::getUnqual(Ty->getContext()));
519 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
530 // This is to prevent dumping of incomplete profile -- if the
536 if (isa<ReturnInst>(BB->getTerminator()))
549 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
553 auto *BB = Cand.first->getParent();
554 auto InstrCount = BFI->getBlockProfileCount(BB);
557 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
573 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
577 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
601 return isa<CatchSwitchInst>(Exit->getTerminator());
605 if (!LP->hasDedicatedExits())
608 BasicBlock *PH = LP->getLoopPreheader();
618 LP->getExitBlocks(LoopExitBlocks);
623 LP->getExitingBlocks(ExitingBlocks);
627 return (unsigned)-1;
649 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
678 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
695 // if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
703 // "__llvm_profile_sampling__" is a thread-local global shared by all PGO
704 // counters (value-instrumentation and edge instrumentation).
711 // if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
756 MDBuilder MDB(I->getContext());
767 // For the burst-sampling, create the conditional update.
769 LoadSamplingVar, GetConstant(CondBuilder, config.BurstDuration - 1));
771 config.BurstDuration, config.Period - config.BurstDuration);
778 I->moveBefore(ThenTerm->getIterator());
789 BranchWeight = MDB.createBranchWeights(1, config.Period - 1);
795 I->moveBefore(ThenTerm->getIterator());
799 SamplingVarIncr->moveBefore(ElseTerm->getIterator());
835 IPMP->eraseFromParent();
851 // Mach-O don't support weak external references.
893 BasicBlock *BB = CounterLoad->getParent();
902 // Do a post-order traversal of the loops so that counter updates can be
922 return !F->use_empty();
998 // not eliminated by the front-end, e.g. unused functions with internal
1035 GlobalVariable *Name = Ind->getName();
1036 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1037 uint64_t Index = Ind->getIndex()->getZExtValue();
1050 GlobalVariable *Name = Ind->getName();
1052 assert(It != ProfileDataMap.end() && It->second.DataVar &&
1055 GlobalVariable *DataVar = It->second.DataVar;
1056 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1057 uint64_t Index = Ind->getIndex()->getZExtValue();
1059 Index += It->second.NumValueSites[Kind];
1062 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
1065 auto *TLI = &GetTLI(*Ind->getFunction());
1074 Ind->getOperandBundlesAsDefs(OpBundles);
1076 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1081 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1087 if (auto AK = TLI->getExtAttrForI32Param(false))
1088 Call->addParamAttr(2, AK);
1089 Ind->replaceAllUsesWith(Call);
1090 Ind->eraseFromParent();
1105 Bias->setVisibility(GlobalVariable::HiddenVisibility);
1111 Bias->setComdat(M.getOrInsertComdat(VarName));
1121 Counters->setAlignment(Align(8));
1124 Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());
1130 Function *Fn = I->getParent()->getParent();
1133 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1137 BiasLI->setMetadata(LLVMContext::MD_invariant_load,
1141 return Builder.CreateIntToPtr(Add, Addr->getType());
1151 Function *Fn = I->getFunction();
1152 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1156 BiasLI->setMetadata(LLVMContext::MD_invariant_load,
1168 Instruction *SplitBefore = CoverInstruction->getNextNode();
1169 auto &Ctx = CoverInstruction->getParent()->getContext();
1180 CoverInstruction->eraseFromParent();
1185 assert(TimestampInstruction->getIndex()->isZeroValue() &&
1191 FunctionType::get(Type::getVoidTy(Ctx), TimestampAddr->getType(), false);
1195 TimestampInstruction->eraseFromParent();
1203 (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
1204 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
1207 Value *IncStep = Inc->getStep();
1208 Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
1209 auto *Count = Builder.CreateAdd(Load, Inc->getStep());
1214 Inc->eraseFromParent();
1219 cast<ConstantArray>(CoverageNamesVar->getInitializer());
1220 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
1221 Constant *NC = Names->getOperand(I);
1222 Value *V = NC->stripPointerCasts();
1226 Name->setLinkage(GlobalValue::PrivateLinkage);
1229 NC->dropAllReferences();
1231 CoverageNamesVar->eraseFromParent();
1240 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
1247 Update->getBitmapIndex());
1273 // Note, just-loaded Bitmap might not be up-to-date. Use it just for
1297 Update->eraseFromParent();
1304 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
1305 Function *F = Inc->getParent()->getParent();
1306 Module *M = F->getParent();
1313 uint64_t FuncHash = Inc->getHash()->getZExtValue();
1325 if (!profDataReferencedByCode(*F->getParent()))
1329 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
1330 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
1338 F->hasFnAttribute(Attribute::AlwaysInline))
1344 if (F->hasLocalLinkage() && F->hasComdat())
1354 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
1359 if (Fn->isDeclarationForLinker())
1364 if (Fn->hasLocalLinkage())
1373 // aliases to the same module as the jump-table they refer to will be defined.
1374 if (Fn->hasMetadata(LLVMContext::MD_type))
1380 if (Fn->hasComdat() &&
1381 (Fn->getVisibility() == GlobalValue::VisibilityTypes::HiddenVisibility))
1389 auto *Int8PtrTy = PointerType::getUnqual(Fn->getContext());
1401 Fn->getName() + ".local", Fn);
1413 if (Fn->hasComdat()) {
1414 GA->setLinkage(Fn->getLinkage());
1415 GA->setVisibility(GlobalValue::VisibilityTypes::HiddenVisibility);
1418 // appendToCompilerUsed(*Fn->getParent(), {GA});
1424 // compiler-rt uses linker support to get data/counters/name start/end for
1425 // ELF, COFF, Mach-O, XCOFF, and Wasm.
1451 // If the data variable is referenced by code, non-counter variables (notably
1457 ? GV->getName()
1465 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
1466 // allows -z start-stop-gc to discard the entire group when the function is
1468 C->setSelectionKind(Comdat::NoDeduplicate);
1470 GV->setComdat(C);
1474 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
1475 GV->setLinkage(GlobalValue::InternalLinkage);
1479 if (!profDataReferencedByCode(*GV->getParent()))
1482 if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
1483 !GV->hasAvailableExternallyLinkage())
1488 if (GV->hasLocalLinkage() && GV->hasComdat())
1497 auto *Int8PtrTy = PointerType::getUnqual(GV->getContext());
1509 if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
1513 if (GV->getName().starts_with("llvm.") ||
1514 GV->getName().starts_with("__llvm") ||
1515 GV->getName().starts_with("__prof"))
1520 if (It != VTableDataMap.end() && It->second)
1523 GlobalValue::LinkageTypes Linkage = GV->getLinkage();
1524 GlobalValue::VisibilityTypes Visibility = GV->getVisibility();
1526 // This is to keep consistent with per-function profile data
1548 M.getDataLayout().getTypeAllocSize(GV->getValueType());
1561 Data->setVisibility(Visibility);
1562 Data->setSection(getInstrProfSectionName(IPSK_vtab, TT.getObjectFormat()));
1563 Data->setAlignment(Align(8));
1565 maybeSetComdat(Data, GV, Data->getName());
1578 GlobalVariable *NamePtr = Inc->getName();
1581 Function *Fn = Inc->getParent()->getParent();
1582 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
1583 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1621 Ptr->setVisibility(Visibility);
1624 Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat()));
1625 Ptr->setLinkage(Linkage);
1634 uint64_t NumBytes = Inc->getNumBitmapBytes();
1638 GV->setAlignment(Align(1));
1644 GlobalVariable *NamePtr = Inc->getName();
1653 PD.NumBitmapBytes = Inc->getNumBitmapBytes();
1660 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1672 GV->setAlignment(Align(1));
1677 GV->setAlignment(Align(8));
1684 GlobalVariable *NamePtr = Inc->getName();
1697 Function *Fn = Inc->getParent()->getParent();
1698 if (auto *SP = Fn->getSubprogram()) {
1699 DIBuilder DB(M, true, SP->getUnit());
1706 ConstantAsMetadata::get(Inc->getHash()),
1710 ConstantAsMetadata::get(Inc->getNumCounters()),
1718 SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
1720 CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
1723 CounterPtr->addDebugInfo(DICounter);
1743 GlobalVariable *NamePtr = Inc->getName();
1752 Function *Fn = Inc->getParent()->getParent();
1753 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
1754 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1788 ValuesVar->setVisibility(Visibility);
1790 ValuesVar->setSection(
1792 ValuesVar->setAlignment(Align(8));
1795 ValuesVar, PointerType::get(Fn->getContext(), 0));
1798 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1854 // Reference the counter variable with a label difference (link-time
1870 Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1872 Data->setVisibility(Visibility);
1873 Data->setSection(
1875 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
1885 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
1911 // The default value of vp-counters-per-site is chosen based on
1933 VNodesVar->setSection(
1935 VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(VNodesTy));
1956 NamesVar = new GlobalVariable(M, NamesVal->getType(), true,
1962 NamesVar->setLinkage(GlobalValue::ExternalLinkage);
1963 NamesVar->setVisibility(GlobalValue::VisibilityTypes::ProtectedVisibility);
1968 NamesVar->setSection(
1975 NamesVar->setAlignment(Align(1));
1981 NamePtr->eraseFromParent();
1999 new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */,
2002 VTableNamesVar->setSection(
2004 VTableNamesVar->setAlignment(Align(1));
2020 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2022 RegisterF->addFnAttr(Attribute::NoRedZone);
2056 // We expect the linker to be invoked with -u<hook_var> flag for Linux
2071 Var->setVisibility(GlobalValue::ProtectedVisibility);
2073 Var->setVisibility(GlobalValue::HiddenVisibility);
2083 User->addFnAttr(Attribute::NoInline);
2085 User->addFnAttr(Attribute::NoRedZone);
2086 User->setVisibility(GlobalValue::HiddenVisibility);
2088 User->setComdat(M.getOrInsertComdat(User->getName()));
2105 // On ELF and Mach-O, the linker can guarantee the associated sections will be
2124 // context-sensitive instrumentation lowering: This lowering is after
2138 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2139 F->addFnAttr(Attribute::NoInline);
2141 F->addFnAttr(Attribute::NoRedZone);
2166 SamplingVar->setVisibility(GlobalValue::DefaultVisibility);
2167 SamplingVar->setThreadLocal(true);
2170 SamplingVar->setLinkage(GlobalValue::ExternalLinkage);
2171 SamplingVar->setComdat(M.getOrInsertComdat(VarName));