1 //===---- GlobalMergeFunctions.cpp - Global merge functions -------*- C++ -===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass implements the global merge function pass. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/CodeGen/GlobalMergeFunctions.h" 14 #include "llvm/ADT/Statistic.h" 15 #include "llvm/Analysis/ModuleSummaryAnalysis.h" 16 #include "llvm/CGData/CodeGenData.h" 17 #include "llvm/IR/IRBuilder.h" 18 #include "llvm/IR/StructuralHash.h" 19 #include "llvm/InitializePasses.h" 20 #include "llvm/Support/CommandLine.h" 21 #include "llvm/Transforms/Utils/ModuleUtils.h" 22 23 #define DEBUG_TYPE "global-merge-func" 24 25 using namespace llvm; 26 using namespace llvm::support; 27 28 static cl::opt<bool> DisableCGDataForMerging( 29 "disable-cgdata-for-merging", cl::Hidden, 30 cl::desc("Disable codegen data for function merging. Local " 31 "merging is still enabled within a module."), 32 cl::init(false)); 33 34 STATISTIC(NumMismatchedFunctionHash, 35 "Number of mismatched function hash for global merge function"); 36 STATISTIC(NumMismatchedInstCount, 37 "Number of mismatched instruction count for global merge function"); 38 STATISTIC(NumMismatchedConstHash, 39 "Number of mismatched const hash for global merge function"); 40 STATISTIC(NumMismatchedModuleId, 41 "Number of mismatched Module Id for global merge function"); 42 STATISTIC(NumMergedFunctions, 43 "Number of functions that are actually merged using function hash"); 44 STATISTIC(NumAnalyzedModues, "Number of modules that are analyzed"); 45 STATISTIC(NumAnalyzedFunctions, "Number of functions that are analyzed"); 46 STATISTIC(NumEligibleFunctions, "Number of functions that are eligible"); 47 48 /// Returns true if the \OpIdx operand of \p CI is the callee operand. 49 static bool isCalleeOperand(const CallBase *CI, unsigned OpIdx) { 50 return &CI->getCalledOperandUse() == &CI->getOperandUse(OpIdx); 51 } 52 53 static bool canParameterizeCallOperand(const CallBase *CI, unsigned OpIdx) { 54 if (CI->isInlineAsm()) 55 return false; 56 Function *Callee = CI->getCalledOperand() 57 ? dyn_cast_or_null<Function>( 58 CI->getCalledOperand()->stripPointerCasts()) 59 : nullptr; 60 if (Callee) { 61 if (Callee->isIntrinsic()) 62 return false; 63 auto Name = Callee->getName(); 64 // objc_msgSend stubs must be called, and can't have their address taken. 65 if (Name.starts_with("objc_msgSend$")) 66 return false; 67 // Calls to dtrace probes must generate unique patchpoints. 68 if (Name.starts_with("__dtrace")) 69 return false; 70 } 71 if (isCalleeOperand(CI, OpIdx) && 72 CI->getOperandBundle(LLVMContext::OB_ptrauth).has_value()) { 73 // The operand is the callee and it has already been signed. Ignore this 74 // because we cannot add another ptrauth bundle to the call instruction. 75 return false; 76 } 77 return true; 78 } 79 80 /// Returns true if function \p F is eligible for merging. 81 bool isEligibleFunction(Function *F) { 82 if (F->isDeclaration()) 83 return false; 84 85 if (F->hasFnAttribute(llvm::Attribute::NoMerge) || 86 F->hasFnAttribute(llvm::Attribute::AlwaysInline)) 87 return false; 88 89 if (F->hasAvailableExternallyLinkage()) 90 return false; 91 92 if (F->getFunctionType()->isVarArg()) 93 return false; 94 95 if (F->getCallingConv() == CallingConv::SwiftTail) 96 return false; 97 98 // If function contains callsites with musttail, if we merge 99 // it, the merged function will have the musttail callsite, but 100 // the number of parameters can change, thus the parameter count 101 // of the callsite will mismatch with the function itself. 102 for (const BasicBlock &BB : *F) { 103 for (const Instruction &I : BB) { 104 const auto *CB = dyn_cast<CallBase>(&I); 105 if (CB && CB->isMustTailCall()) 106 return false; 107 } 108 } 109 110 return true; 111 } 112 113 static bool isEligibleInstrunctionForConstantSharing(const Instruction *I) { 114 switch (I->getOpcode()) { 115 case Instruction::Load: 116 case Instruction::Store: 117 case Instruction::Call: 118 case Instruction::Invoke: 119 return true; 120 default: 121 return false; 122 } 123 } 124 125 static bool ignoreOp(const Instruction *I, unsigned OpIdx) { 126 assert(OpIdx < I->getNumOperands() && "Invalid operand index"); 127 128 if (!isEligibleInstrunctionForConstantSharing(I)) 129 return false; 130 131 if (!isa<Constant>(I->getOperand(OpIdx))) 132 return false; 133 134 if (const auto *CI = dyn_cast<CallBase>(I)) 135 return canParameterizeCallOperand(CI, OpIdx); 136 137 return true; 138 } 139 140 static Value *createCast(IRBuilder<> &Builder, Value *V, Type *DestTy) { 141 Type *SrcTy = V->getType(); 142 if (SrcTy->isStructTy()) { 143 assert(DestTy->isStructTy()); 144 assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements()); 145 Value *Result = PoisonValue::get(DestTy); 146 for (unsigned int I = 0, E = SrcTy->getStructNumElements(); I < E; ++I) { 147 Value *Element = 148 createCast(Builder, Builder.CreateExtractValue(V, ArrayRef(I)), 149 DestTy->getStructElementType(I)); 150 151 Result = Builder.CreateInsertValue(Result, Element, ArrayRef(I)); 152 } 153 return Result; 154 } 155 assert(!DestTy->isStructTy()); 156 if (auto *SrcAT = dyn_cast<ArrayType>(SrcTy)) { 157 auto *DestAT = dyn_cast<ArrayType>(DestTy); 158 assert(DestAT); 159 assert(SrcAT->getNumElements() == DestAT->getNumElements()); 160 Value *Result = UndefValue::get(DestTy); 161 for (unsigned int I = 0, E = SrcAT->getNumElements(); I < E; ++I) { 162 Value *Element = 163 createCast(Builder, Builder.CreateExtractValue(V, ArrayRef(I)), 164 DestAT->getElementType()); 165 166 Result = Builder.CreateInsertValue(Result, Element, ArrayRef(I)); 167 } 168 return Result; 169 } 170 assert(!DestTy->isArrayTy()); 171 if (SrcTy->isIntegerTy() && DestTy->isPointerTy()) 172 return Builder.CreateIntToPtr(V, DestTy); 173 if (SrcTy->isPointerTy() && DestTy->isIntegerTy()) 174 return Builder.CreatePtrToInt(V, DestTy); 175 return Builder.CreateBitCast(V, DestTy); 176 } 177 178 void GlobalMergeFunc::analyze(Module &M) { 179 ++NumAnalyzedModues; 180 for (Function &Func : M) { 181 ++NumAnalyzedFunctions; 182 if (isEligibleFunction(&Func)) { 183 ++NumEligibleFunctions; 184 185 auto FI = llvm::StructuralHashWithDifferences(Func, ignoreOp); 186 187 // Convert the operand map to a vector for a serialization-friendly 188 // format. 189 IndexOperandHashVecType IndexOperandHashes; 190 for (auto &Pair : *FI.IndexOperandHashMap) 191 IndexOperandHashes.emplace_back(Pair); 192 193 StableFunction SF(FI.FunctionHash, get_stable_name(Func.getName()).str(), 194 M.getModuleIdentifier(), FI.IndexInstruction->size(), 195 std::move(IndexOperandHashes)); 196 197 LocalFunctionMap->insert(SF); 198 } 199 } 200 } 201 202 /// Tuple to hold function info to process merging. 203 struct FuncMergeInfo { 204 StableFunctionMap::StableFunctionEntry *SF; 205 Function *F; 206 std::unique_ptr<IndexInstrMap> IndexInstruction; 207 FuncMergeInfo(StableFunctionMap::StableFunctionEntry *SF, Function *F, 208 std::unique_ptr<IndexInstrMap> IndexInstruction) 209 : SF(SF), F(F), IndexInstruction(std::move(IndexInstruction)) {} 210 }; 211 212 // Given the func info, and the parameterized locations, create and return 213 // a new merged function by replacing the original constants with the new 214 // parameters. 215 static Function *createMergedFunction(FuncMergeInfo &FI, 216 ArrayRef<Type *> ConstParamTypes, 217 const ParamLocsVecTy &ParamLocsVec) { 218 // Synthesize a new merged function name by appending ".Tgm" to the root 219 // function's name. 220 auto *MergedFunc = FI.F; 221 std::string NewFunctionName = 222 MergedFunc->getName().str() + GlobalMergeFunc::MergingInstanceSuffix; 223 auto *M = MergedFunc->getParent(); 224 assert(!M->getFunction(NewFunctionName)); 225 226 FunctionType *OrigTy = MergedFunc->getFunctionType(); 227 // Get the original params' types. 228 SmallVector<Type *> ParamTypes(OrigTy->param_begin(), OrigTy->param_end()); 229 // Append const parameter types that are passed in. 230 ParamTypes.append(ConstParamTypes.begin(), ConstParamTypes.end()); 231 FunctionType *FuncType = FunctionType::get(OrigTy->getReturnType(), 232 ParamTypes, /*isVarArg=*/false); 233 234 // Declare a new function 235 Function *NewFunction = 236 Function::Create(FuncType, MergedFunc->getLinkage(), NewFunctionName); 237 if (auto *SP = MergedFunc->getSubprogram()) 238 NewFunction->setSubprogram(SP); 239 NewFunction->copyAttributesFrom(MergedFunc); 240 NewFunction->setDLLStorageClass(GlobalValue::DefaultStorageClass); 241 242 NewFunction->setLinkage(GlobalValue::InternalLinkage); 243 NewFunction->addFnAttr(Attribute::NoInline); 244 245 // Add the new function before the root function. 246 M->getFunctionList().insert(MergedFunc->getIterator(), NewFunction); 247 248 // Move the body of MergedFunc into the NewFunction. 249 NewFunction->splice(NewFunction->begin(), MergedFunc); 250 251 // Update the original args by the new args. 252 auto NewArgIter = NewFunction->arg_begin(); 253 for (Argument &OrigArg : MergedFunc->args()) { 254 Argument &NewArg = *NewArgIter++; 255 OrigArg.replaceAllUsesWith(&NewArg); 256 } 257 258 // Replace the original Constants by the new args. 259 unsigned NumOrigArgs = MergedFunc->arg_size(); 260 for (unsigned ParamIdx = 0; ParamIdx < ParamLocsVec.size(); ++ParamIdx) { 261 Argument *NewArg = NewFunction->getArg(NumOrigArgs + ParamIdx); 262 for (auto [InstIndex, OpndIndex] : ParamLocsVec[ParamIdx]) { 263 auto *Inst = FI.IndexInstruction->lookup(InstIndex); 264 auto *OrigC = Inst->getOperand(OpndIndex); 265 if (OrigC->getType() != NewArg->getType()) { 266 IRBuilder<> Builder(Inst->getParent(), Inst->getIterator()); 267 Inst->setOperand(OpndIndex, 268 createCast(Builder, NewArg, OrigC->getType())); 269 } else { 270 Inst->setOperand(OpndIndex, NewArg); 271 } 272 } 273 } 274 275 return NewFunction; 276 } 277 278 // Given the original function (Thunk) and the merged function (ToFunc), create 279 // a thunk to the merged function. 280 static void createThunk(FuncMergeInfo &FI, ArrayRef<Constant *> Params, 281 Function *ToFunc) { 282 auto *Thunk = FI.F; 283 284 assert(Thunk->arg_size() + Params.size() == 285 ToFunc->getFunctionType()->getNumParams()); 286 Thunk->dropAllReferences(); 287 288 BasicBlock *BB = BasicBlock::Create(Thunk->getContext(), "", Thunk); 289 IRBuilder<> Builder(BB); 290 291 SmallVector<Value *> Args; 292 unsigned ParamIdx = 0; 293 FunctionType *ToFuncTy = ToFunc->getFunctionType(); 294 295 // Add arguments which are passed through Thunk. 296 for (Argument &AI : Thunk->args()) { 297 Args.push_back(createCast(Builder, &AI, ToFuncTy->getParamType(ParamIdx))); 298 ++ParamIdx; 299 } 300 301 // Add new arguments defined by Params. 302 for (auto *Param : Params) { 303 assert(ParamIdx < ToFuncTy->getNumParams()); 304 Args.push_back( 305 createCast(Builder, Param, ToFuncTy->getParamType(ParamIdx))); 306 ++ParamIdx; 307 } 308 309 CallInst *CI = Builder.CreateCall(ToFunc, Args); 310 bool isSwiftTailCall = ToFunc->getCallingConv() == CallingConv::SwiftTail && 311 Thunk->getCallingConv() == CallingConv::SwiftTail; 312 CI->setTailCallKind(isSwiftTailCall ? llvm::CallInst::TCK_MustTail 313 : llvm::CallInst::TCK_Tail); 314 CI->setCallingConv(ToFunc->getCallingConv()); 315 CI->setAttributes(ToFunc->getAttributes()); 316 if (Thunk->getReturnType()->isVoidTy()) 317 Builder.CreateRetVoid(); 318 else 319 Builder.CreateRet(createCast(Builder, CI, Thunk->getReturnType())); 320 } 321 322 // Check if the old merged/optimized IndexOperandHashMap is compatible with 323 // the current IndexOperandHashMap. An operand hash may not be stable across 324 // different builds due to varying modules combined. To address this, we relax 325 // the hash check condition by comparing Const hash patterns instead of absolute 326 // hash values. For example, let's assume we have three Consts located at idx1, 327 // idx3, and idx6, where their corresponding hashes are hash1, hash2, and hash1 328 // in the old merged map below: 329 // Old (Merged): [(idx1, hash1), (idx3, hash2), (idx6, hash1)] 330 // Current: [(idx1, hash1'), (idx3, hash2'), (idx6, hash1')] 331 // If the current function also has three Consts in the same locations, 332 // with hash sequences hash1', hash2', and hash1' where the first and third 333 // are the same as the old hash sequences, we consider them matched. 334 static bool checkConstHashCompatible( 335 const DenseMap<IndexPair, stable_hash> &OldInstOpndIndexToConstHash, 336 const DenseMap<IndexPair, stable_hash> &CurrInstOpndIndexToConstHash) { 337 338 DenseMap<stable_hash, stable_hash> OldHashToCurrHash; 339 for (const auto &[Index, OldHash] : OldInstOpndIndexToConstHash) { 340 auto It = CurrInstOpndIndexToConstHash.find(Index); 341 if (It == CurrInstOpndIndexToConstHash.end()) 342 return false; 343 344 auto CurrHash = It->second; 345 auto J = OldHashToCurrHash.find(OldHash); 346 if (J == OldHashToCurrHash.end()) 347 OldHashToCurrHash.insert({OldHash, CurrHash}); 348 else if (J->second != CurrHash) 349 return false; 350 } 351 352 return true; 353 } 354 355 // Validate the locations pointed by a param has the same hash and Constant. 356 static bool 357 checkConstLocationCompatible(const StableFunctionMap::StableFunctionEntry &SF, 358 const IndexInstrMap &IndexInstruction, 359 const ParamLocsVecTy &ParamLocsVec) { 360 for (auto &ParamLocs : ParamLocsVec) { 361 std::optional<stable_hash> OldHash; 362 std::optional<Constant *> OldConst; 363 for (auto &Loc : ParamLocs) { 364 assert(SF.IndexOperandHashMap->count(Loc)); 365 auto CurrHash = SF.IndexOperandHashMap.get()->at(Loc); 366 auto [InstIndex, OpndIndex] = Loc; 367 assert(InstIndex < IndexInstruction.size()); 368 const auto *Inst = IndexInstruction.lookup(InstIndex); 369 auto *CurrConst = cast<Constant>(Inst->getOperand(OpndIndex)); 370 if (!OldHash) { 371 OldHash = CurrHash; 372 OldConst = CurrConst; 373 } else if (CurrConst != *OldConst || CurrHash != *OldHash) { 374 return false; 375 } 376 } 377 } 378 return true; 379 } 380 381 static ParamLocsVecTy computeParamInfo( 382 const SmallVector<std::unique_ptr<StableFunctionMap::StableFunctionEntry>> 383 &SFS) { 384 std::map<std::vector<stable_hash>, ParamLocs> HashSeqToLocs; 385 auto &RSF = *SFS[0]; 386 unsigned StableFunctionCount = SFS.size(); 387 388 for (auto &[IndexPair, Hash] : *RSF.IndexOperandHashMap) { 389 // Const hash sequence across stable functions. 390 // We will allocate a parameter per unique hash squence. 391 // can't use SmallVector as key 392 std::vector<stable_hash> ConstHashSeq; 393 ConstHashSeq.push_back(Hash); 394 bool Identical = true; 395 for (unsigned J = 1; J < StableFunctionCount; ++J) { 396 auto &SF = SFS[J]; 397 auto SHash = SF->IndexOperandHashMap->at(IndexPair); 398 if (Hash != SHash) 399 Identical = false; 400 ConstHashSeq.push_back(SHash); 401 } 402 403 if (Identical) 404 continue; 405 406 // For each unique Const hash sequence (parameter), add the locations. 407 HashSeqToLocs[ConstHashSeq].push_back(IndexPair); 408 } 409 410 ParamLocsVecTy ParamLocsVec; 411 for (auto &[HashSeq, Locs] : HashSeqToLocs) { 412 ParamLocsVec.push_back(std::move(Locs)); 413 llvm::sort(ParamLocsVec, [&](const ParamLocs &L, const ParamLocs &R) { 414 return L[0] < R[0]; 415 }); 416 } 417 return ParamLocsVec; 418 } 419 420 bool GlobalMergeFunc::merge(Module &M, const StableFunctionMap *FunctionMap) { 421 bool Changed = false; 422 423 // Build a map from stable function name to function. 424 StringMap<Function *> StableNameToFuncMap; 425 for (auto &F : M) 426 StableNameToFuncMap[get_stable_name(F.getName())] = &F; 427 // Track merged functions 428 DenseSet<Function *> MergedFunctions; 429 430 auto ModId = M.getModuleIdentifier(); 431 for (auto &[Hash, SFS] : FunctionMap->getFunctionMap()) { 432 // Parameter locations based on the unique hash sequences 433 // across the candidates. 434 std::optional<ParamLocsVecTy> ParamLocsVec; 435 Function *MergedFunc = nullptr; 436 std::string MergedModId; 437 SmallVector<FuncMergeInfo> FuncMergeInfos; 438 for (auto &SF : SFS) { 439 // Get the function from the stable name. 440 auto I = StableNameToFuncMap.find( 441 *FunctionMap->getNameForId(SF->FunctionNameId)); 442 if (I == StableNameToFuncMap.end()) 443 continue; 444 Function *F = I->second; 445 assert(F); 446 // Skip if the function has been merged before. 447 if (MergedFunctions.count(F)) 448 continue; 449 // Consider the function if it is eligible for merging. 450 if (!isEligibleFunction(F)) 451 continue; 452 453 auto FI = llvm::StructuralHashWithDifferences(*F, ignoreOp); 454 uint64_t FuncHash = FI.FunctionHash; 455 if (Hash != FuncHash) { 456 ++NumMismatchedFunctionHash; 457 continue; 458 } 459 460 if (SF->InstCount != FI.IndexInstruction->size()) { 461 ++NumMismatchedInstCount; 462 continue; 463 } 464 bool HasValidSharedConst = true; 465 for (auto &[Index, Hash] : *SF->IndexOperandHashMap) { 466 auto [InstIndex, OpndIndex] = Index; 467 assert(InstIndex < FI.IndexInstruction->size()); 468 auto *Inst = FI.IndexInstruction->lookup(InstIndex); 469 if (!ignoreOp(Inst, OpndIndex)) { 470 HasValidSharedConst = false; 471 break; 472 } 473 } 474 if (!HasValidSharedConst) { 475 ++NumMismatchedConstHash; 476 continue; 477 } 478 if (!checkConstHashCompatible(*SF->IndexOperandHashMap, 479 *FI.IndexOperandHashMap)) { 480 ++NumMismatchedConstHash; 481 continue; 482 } 483 if (!ParamLocsVec.has_value()) { 484 ParamLocsVec = computeParamInfo(SFS); 485 LLVM_DEBUG(dbgs() << "[GlobalMergeFunc] Merging hash: " << Hash 486 << " with Params " << ParamLocsVec->size() << "\n"); 487 } 488 if (!checkConstLocationCompatible(*SF, *FI.IndexInstruction, 489 *ParamLocsVec)) { 490 ++NumMismatchedConstHash; 491 continue; 492 } 493 494 if (MergedFunc) { 495 // Check if the matched functions fall into the same (first) module. 496 // This module check is not strictly necessary as the functions can move 497 // around. We just want to avoid merging functions from different 498 // modules than the first one in the function map, as they may not end 499 // up with being ICFed by the linker. 500 if (MergedModId != *FunctionMap->getNameForId(SF->ModuleNameId)) { 501 ++NumMismatchedModuleId; 502 continue; 503 } 504 } else { 505 MergedFunc = F; 506 MergedModId = *FunctionMap->getNameForId(SF->ModuleNameId); 507 } 508 509 FuncMergeInfos.emplace_back(SF.get(), F, std::move(FI.IndexInstruction)); 510 MergedFunctions.insert(F); 511 } 512 unsigned FuncMergeInfoSize = FuncMergeInfos.size(); 513 if (FuncMergeInfoSize == 0) 514 continue; 515 516 LLVM_DEBUG(dbgs() << "[GlobalMergeFunc] Merging function count " 517 << FuncMergeInfoSize << " in " << ModId << "\n"); 518 519 for (auto &FMI : FuncMergeInfos) { 520 Changed = true; 521 522 // We've already validated all locations of constant operands pointed by 523 // the parameters. Populate parameters pointing to the original constants. 524 SmallVector<Constant *> Params; 525 SmallVector<Type *> ParamTypes; 526 for (auto &ParamLocs : *ParamLocsVec) { 527 assert(!ParamLocs.empty()); 528 auto &[InstIndex, OpndIndex] = ParamLocs[0]; 529 auto *Inst = FMI.IndexInstruction->lookup(InstIndex); 530 auto *Opnd = cast<Constant>(Inst->getOperand(OpndIndex)); 531 Params.push_back(Opnd); 532 ParamTypes.push_back(Opnd->getType()); 533 } 534 535 // Create a merged function derived from the current function. 536 Function *MergedFunc = 537 createMergedFunction(FMI, ParamTypes, *ParamLocsVec); 538 539 LLVM_DEBUG({ 540 dbgs() << "[GlobalMergeFunc] Merged function (hash:" << FMI.SF->Hash 541 << ") " << MergedFunc->getName() << " generated from " 542 << FMI.F->getName() << ":\n"; 543 MergedFunc->dump(); 544 }); 545 546 // Transform the current function into a thunk that calls the merged 547 // function. 548 createThunk(FMI, Params, MergedFunc); 549 LLVM_DEBUG({ 550 dbgs() << "[GlobalMergeFunc] Thunk generated: \n"; 551 FMI.F->dump(); 552 }); 553 ++NumMergedFunctions; 554 } 555 } 556 557 return Changed; 558 } 559 560 void GlobalMergeFunc::initializeMergerMode(const Module &M) { 561 // Initialize the local function map regardless of the merger mode. 562 LocalFunctionMap = std::make_unique<StableFunctionMap>(); 563 564 // Disable codegen data for merging. The local merge is still enabled. 565 if (DisableCGDataForMerging) 566 return; 567 568 // (Full)LTO module does not have functions added to the index. 569 // In this case, we run a local merger without using codegen data. 570 if (Index && !Index->hasExportedFunctions(M)) 571 return; 572 573 if (cgdata::emitCGData()) 574 MergerMode = HashFunctionMode::BuildingHashFuncion; 575 else if (cgdata::hasStableFunctionMap()) 576 MergerMode = HashFunctionMode::UsingHashFunction; 577 } 578 579 void GlobalMergeFunc::emitFunctionMap(Module &M) { 580 LLVM_DEBUG(dbgs() << "Emit function map. Size: " << LocalFunctionMap->size() 581 << "\n"); 582 // No need to emit the function map if it is empty. 583 if (LocalFunctionMap->empty()) 584 return; 585 SmallVector<char> Buf; 586 raw_svector_ostream OS(Buf); 587 588 StableFunctionMapRecord::serialize(OS, LocalFunctionMap.get()); 589 590 std::unique_ptr<MemoryBuffer> Buffer = MemoryBuffer::getMemBuffer( 591 OS.str(), "in-memory stable function map", false); 592 593 Triple TT(M.getTargetTriple()); 594 embedBufferInModule(M, *Buffer.get(), 595 getCodeGenDataSectionName(CG_merge, TT.getObjectFormat()), 596 Align(4)); 597 } 598 599 bool GlobalMergeFunc::run(Module &M) { 600 initializeMergerMode(M); 601 602 const StableFunctionMap *FuncMap; 603 if (MergerMode == HashFunctionMode::UsingHashFunction) { 604 // Use the prior CG data to optimistically create global merge candidates. 605 FuncMap = cgdata::getStableFunctionMap(); 606 } else { 607 analyze(M); 608 // Emit the local function map to the custom section, __llvm_merge before 609 // finalizing it. 610 if (MergerMode == HashFunctionMode::BuildingHashFuncion) 611 emitFunctionMap(M); 612 LocalFunctionMap->finalize(); 613 FuncMap = LocalFunctionMap.get(); 614 } 615 616 return merge(M, FuncMap); 617 } 618 619 namespace { 620 621 class GlobalMergeFuncPassWrapper : public ModulePass { 622 623 public: 624 static char ID; 625 626 GlobalMergeFuncPassWrapper(); 627 628 void getAnalysisUsage(AnalysisUsage &AU) const override { 629 AU.addUsedIfAvailable<ImmutableModuleSummaryIndexWrapperPass>(); 630 AU.setPreservesAll(); 631 ModulePass::getAnalysisUsage(AU); 632 } 633 634 StringRef getPassName() const override { return "Global Merge Functions"; } 635 636 bool runOnModule(Module &M) override; 637 }; 638 639 } // namespace 640 641 char GlobalMergeFuncPassWrapper::ID = 0; 642 INITIALIZE_PASS_BEGIN(GlobalMergeFuncPassWrapper, "global-merge-func", 643 "Global merge function pass", false, false) 644 INITIALIZE_PASS_END(GlobalMergeFuncPassWrapper, "global-merge-func", 645 "Global merge function pass", false, false) 646 647 namespace llvm { 648 ModulePass *createGlobalMergeFuncPass() { 649 return new GlobalMergeFuncPassWrapper(); 650 } 651 } // namespace llvm 652 653 GlobalMergeFuncPassWrapper::GlobalMergeFuncPassWrapper() : ModulePass(ID) { 654 initializeGlobalMergeFuncPassWrapperPass( 655 *llvm::PassRegistry::getPassRegistry()); 656 } 657 658 bool GlobalMergeFuncPassWrapper::runOnModule(Module &M) { 659 const ModuleSummaryIndex *Index = nullptr; 660 if (auto *IndexWrapperPass = 661 getAnalysisIfAvailable<ImmutableModuleSummaryIndexWrapperPass>()) 662 Index = IndexWrapperPass->getIndex(); 663 664 return GlobalMergeFunc(Index).run(M); 665 } 666 667 PreservedAnalyses GlobalMergeFuncPass::run(Module &M, 668 AnalysisManager<Module> &AM) { 669 ModuleSummaryIndex *Index = &(AM.getResult<ModuleSummaryIndexAnalysis>(M)); 670 bool Changed = GlobalMergeFunc(Index).run(M); 671 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 672 } 673