xref: /llvm-project/llvm/lib/CodeGen/GlobalMergeFunctions.cpp (revision d23c5c2d6566fce4380cfa31d438422db19fbce9)
1 //===---- GlobalMergeFunctions.cpp - Global merge functions -------*- C++ -===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements the global merge function pass.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/CodeGen/GlobalMergeFunctions.h"
14 #include "llvm/ADT/Statistic.h"
15 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
16 #include "llvm/CGData/CodeGenData.h"
17 #include "llvm/IR/IRBuilder.h"
18 #include "llvm/IR/StructuralHash.h"
19 #include "llvm/InitializePasses.h"
20 #include "llvm/Support/CommandLine.h"
21 #include "llvm/Transforms/Utils/ModuleUtils.h"
22 
23 #define DEBUG_TYPE "global-merge-func"
24 
25 using namespace llvm;
26 using namespace llvm::support;
27 
28 static cl::opt<bool> DisableCGDataForMerging(
29     "disable-cgdata-for-merging", cl::Hidden,
30     cl::desc("Disable codegen data for function merging. Local "
31              "merging is still enabled within a module."),
32     cl::init(false));
33 
34 STATISTIC(NumMismatchedFunctionHash,
35           "Number of mismatched function hash for global merge function");
36 STATISTIC(NumMismatchedInstCount,
37           "Number of mismatched instruction count for global merge function");
38 STATISTIC(NumMismatchedConstHash,
39           "Number of mismatched const hash for global merge function");
40 STATISTIC(NumMismatchedModuleId,
41           "Number of mismatched Module Id for global merge function");
42 STATISTIC(NumMergedFunctions,
43           "Number of functions that are actually merged using function hash");
44 STATISTIC(NumAnalyzedModues, "Number of modules that are analyzed");
45 STATISTIC(NumAnalyzedFunctions, "Number of functions that are analyzed");
46 STATISTIC(NumEligibleFunctions, "Number of functions that are eligible");
47 
48 /// Returns true if the \OpIdx operand of \p CI is the callee operand.
49 static bool isCalleeOperand(const CallBase *CI, unsigned OpIdx) {
50   return &CI->getCalledOperandUse() == &CI->getOperandUse(OpIdx);
51 }
52 
53 static bool canParameterizeCallOperand(const CallBase *CI, unsigned OpIdx) {
54   if (CI->isInlineAsm())
55     return false;
56   Function *Callee = CI->getCalledOperand()
57                          ? dyn_cast_or_null<Function>(
58                                CI->getCalledOperand()->stripPointerCasts())
59                          : nullptr;
60   if (Callee) {
61     if (Callee->isIntrinsic())
62       return false;
63     auto Name = Callee->getName();
64     // objc_msgSend stubs must be called, and can't have their address taken.
65     if (Name.starts_with("objc_msgSend$"))
66       return false;
67     // Calls to dtrace probes must generate unique patchpoints.
68     if (Name.starts_with("__dtrace"))
69       return false;
70   }
71   if (isCalleeOperand(CI, OpIdx) &&
72       CI->getOperandBundle(LLVMContext::OB_ptrauth).has_value()) {
73     // The operand is the callee and it has already been signed. Ignore this
74     // because we cannot add another ptrauth bundle to the call instruction.
75     return false;
76   }
77   return true;
78 }
79 
80 /// Returns true if function \p F is eligible for merging.
81 bool isEligibleFunction(Function *F) {
82   if (F->isDeclaration())
83     return false;
84 
85   if (F->hasFnAttribute(llvm::Attribute::NoMerge) ||
86       F->hasFnAttribute(llvm::Attribute::AlwaysInline))
87     return false;
88 
89   if (F->hasAvailableExternallyLinkage())
90     return false;
91 
92   if (F->getFunctionType()->isVarArg())
93     return false;
94 
95   if (F->getCallingConv() == CallingConv::SwiftTail)
96     return false;
97 
98   // If function contains callsites with musttail, if we merge
99   // it, the merged function will have the musttail callsite, but
100   // the number of parameters can change, thus the parameter count
101   // of the callsite will mismatch with the function itself.
102   for (const BasicBlock &BB : *F) {
103     for (const Instruction &I : BB) {
104       const auto *CB = dyn_cast<CallBase>(&I);
105       if (CB && CB->isMustTailCall())
106         return false;
107     }
108   }
109 
110   return true;
111 }
112 
113 static bool isEligibleInstrunctionForConstantSharing(const Instruction *I) {
114   switch (I->getOpcode()) {
115   case Instruction::Load:
116   case Instruction::Store:
117   case Instruction::Call:
118   case Instruction::Invoke:
119     return true;
120   default:
121     return false;
122   }
123 }
124 
125 static bool ignoreOp(const Instruction *I, unsigned OpIdx) {
126   assert(OpIdx < I->getNumOperands() && "Invalid operand index");
127 
128   if (!isEligibleInstrunctionForConstantSharing(I))
129     return false;
130 
131   if (!isa<Constant>(I->getOperand(OpIdx)))
132     return false;
133 
134   if (const auto *CI = dyn_cast<CallBase>(I))
135     return canParameterizeCallOperand(CI, OpIdx);
136 
137   return true;
138 }
139 
140 static Value *createCast(IRBuilder<> &Builder, Value *V, Type *DestTy) {
141   Type *SrcTy = V->getType();
142   if (SrcTy->isStructTy()) {
143     assert(DestTy->isStructTy());
144     assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements());
145     Value *Result = PoisonValue::get(DestTy);
146     for (unsigned int I = 0, E = SrcTy->getStructNumElements(); I < E; ++I) {
147       Value *Element =
148           createCast(Builder, Builder.CreateExtractValue(V, ArrayRef(I)),
149                      DestTy->getStructElementType(I));
150 
151       Result = Builder.CreateInsertValue(Result, Element, ArrayRef(I));
152     }
153     return Result;
154   }
155   assert(!DestTy->isStructTy());
156   if (auto *SrcAT = dyn_cast<ArrayType>(SrcTy)) {
157     auto *DestAT = dyn_cast<ArrayType>(DestTy);
158     assert(DestAT);
159     assert(SrcAT->getNumElements() == DestAT->getNumElements());
160     Value *Result = UndefValue::get(DestTy);
161     for (unsigned int I = 0, E = SrcAT->getNumElements(); I < E; ++I) {
162       Value *Element =
163           createCast(Builder, Builder.CreateExtractValue(V, ArrayRef(I)),
164                      DestAT->getElementType());
165 
166       Result = Builder.CreateInsertValue(Result, Element, ArrayRef(I));
167     }
168     return Result;
169   }
170   assert(!DestTy->isArrayTy());
171   if (SrcTy->isIntegerTy() && DestTy->isPointerTy())
172     return Builder.CreateIntToPtr(V, DestTy);
173   if (SrcTy->isPointerTy() && DestTy->isIntegerTy())
174     return Builder.CreatePtrToInt(V, DestTy);
175   return Builder.CreateBitCast(V, DestTy);
176 }
177 
178 void GlobalMergeFunc::analyze(Module &M) {
179   ++NumAnalyzedModues;
180   for (Function &Func : M) {
181     ++NumAnalyzedFunctions;
182     if (isEligibleFunction(&Func)) {
183       ++NumEligibleFunctions;
184 
185       auto FI = llvm::StructuralHashWithDifferences(Func, ignoreOp);
186 
187       // Convert the operand map to a vector for a serialization-friendly
188       // format.
189       IndexOperandHashVecType IndexOperandHashes;
190       for (auto &Pair : *FI.IndexOperandHashMap)
191         IndexOperandHashes.emplace_back(Pair);
192 
193       StableFunction SF(FI.FunctionHash, get_stable_name(Func.getName()).str(),
194                         M.getModuleIdentifier(), FI.IndexInstruction->size(),
195                         std::move(IndexOperandHashes));
196 
197       LocalFunctionMap->insert(SF);
198     }
199   }
200 }
201 
202 /// Tuple to hold function info to process merging.
203 struct FuncMergeInfo {
204   StableFunctionMap::StableFunctionEntry *SF;
205   Function *F;
206   std::unique_ptr<IndexInstrMap> IndexInstruction;
207   FuncMergeInfo(StableFunctionMap::StableFunctionEntry *SF, Function *F,
208                 std::unique_ptr<IndexInstrMap> IndexInstruction)
209       : SF(SF), F(F), IndexInstruction(std::move(IndexInstruction)) {}
210 };
211 
212 // Given the func info, and the parameterized locations, create and return
213 // a new merged function by replacing the original constants with the new
214 // parameters.
215 static Function *createMergedFunction(FuncMergeInfo &FI,
216                                       ArrayRef<Type *> ConstParamTypes,
217                                       const ParamLocsVecTy &ParamLocsVec) {
218   // Synthesize a new merged function name by appending ".Tgm" to the root
219   // function's name.
220   auto *MergedFunc = FI.F;
221   std::string NewFunctionName =
222       MergedFunc->getName().str() + GlobalMergeFunc::MergingInstanceSuffix;
223   auto *M = MergedFunc->getParent();
224   assert(!M->getFunction(NewFunctionName));
225 
226   FunctionType *OrigTy = MergedFunc->getFunctionType();
227   // Get the original params' types.
228   SmallVector<Type *> ParamTypes(OrigTy->param_begin(), OrigTy->param_end());
229   // Append const parameter types that are passed in.
230   ParamTypes.append(ConstParamTypes.begin(), ConstParamTypes.end());
231   FunctionType *FuncType = FunctionType::get(OrigTy->getReturnType(),
232                                              ParamTypes, /*isVarArg=*/false);
233 
234   // Declare a new function
235   Function *NewFunction =
236       Function::Create(FuncType, MergedFunc->getLinkage(), NewFunctionName);
237   if (auto *SP = MergedFunc->getSubprogram())
238     NewFunction->setSubprogram(SP);
239   NewFunction->copyAttributesFrom(MergedFunc);
240   NewFunction->setDLLStorageClass(GlobalValue::DefaultStorageClass);
241 
242   NewFunction->setLinkage(GlobalValue::InternalLinkage);
243   NewFunction->addFnAttr(Attribute::NoInline);
244 
245   // Add the new function before the root function.
246   M->getFunctionList().insert(MergedFunc->getIterator(), NewFunction);
247 
248   // Move the body of MergedFunc into the NewFunction.
249   NewFunction->splice(NewFunction->begin(), MergedFunc);
250 
251   // Update the original args by the new args.
252   auto NewArgIter = NewFunction->arg_begin();
253   for (Argument &OrigArg : MergedFunc->args()) {
254     Argument &NewArg = *NewArgIter++;
255     OrigArg.replaceAllUsesWith(&NewArg);
256   }
257 
258   // Replace the original Constants by the new args.
259   unsigned NumOrigArgs = MergedFunc->arg_size();
260   for (unsigned ParamIdx = 0; ParamIdx < ParamLocsVec.size(); ++ParamIdx) {
261     Argument *NewArg = NewFunction->getArg(NumOrigArgs + ParamIdx);
262     for (auto [InstIndex, OpndIndex] : ParamLocsVec[ParamIdx]) {
263       auto *Inst = FI.IndexInstruction->lookup(InstIndex);
264       auto *OrigC = Inst->getOperand(OpndIndex);
265       if (OrigC->getType() != NewArg->getType()) {
266         IRBuilder<> Builder(Inst->getParent(), Inst->getIterator());
267         Inst->setOperand(OpndIndex,
268                          createCast(Builder, NewArg, OrigC->getType()));
269       } else {
270         Inst->setOperand(OpndIndex, NewArg);
271       }
272     }
273   }
274 
275   return NewFunction;
276 }
277 
278 // Given the original function (Thunk) and the merged function (ToFunc), create
279 // a thunk to the merged function.
280 static void createThunk(FuncMergeInfo &FI, ArrayRef<Constant *> Params,
281                         Function *ToFunc) {
282   auto *Thunk = FI.F;
283 
284   assert(Thunk->arg_size() + Params.size() ==
285          ToFunc->getFunctionType()->getNumParams());
286   Thunk->dropAllReferences();
287 
288   BasicBlock *BB = BasicBlock::Create(Thunk->getContext(), "", Thunk);
289   IRBuilder<> Builder(BB);
290 
291   SmallVector<Value *> Args;
292   unsigned ParamIdx = 0;
293   FunctionType *ToFuncTy = ToFunc->getFunctionType();
294 
295   // Add arguments which are passed through Thunk.
296   for (Argument &AI : Thunk->args()) {
297     Args.push_back(createCast(Builder, &AI, ToFuncTy->getParamType(ParamIdx)));
298     ++ParamIdx;
299   }
300 
301   // Add new arguments defined by Params.
302   for (auto *Param : Params) {
303     assert(ParamIdx < ToFuncTy->getNumParams());
304     Args.push_back(
305         createCast(Builder, Param, ToFuncTy->getParamType(ParamIdx)));
306     ++ParamIdx;
307   }
308 
309   CallInst *CI = Builder.CreateCall(ToFunc, Args);
310   bool isSwiftTailCall = ToFunc->getCallingConv() == CallingConv::SwiftTail &&
311                          Thunk->getCallingConv() == CallingConv::SwiftTail;
312   CI->setTailCallKind(isSwiftTailCall ? llvm::CallInst::TCK_MustTail
313                                       : llvm::CallInst::TCK_Tail);
314   CI->setCallingConv(ToFunc->getCallingConv());
315   CI->setAttributes(ToFunc->getAttributes());
316   if (Thunk->getReturnType()->isVoidTy())
317     Builder.CreateRetVoid();
318   else
319     Builder.CreateRet(createCast(Builder, CI, Thunk->getReturnType()));
320 }
321 
322 // Check if the old merged/optimized IndexOperandHashMap is compatible with
323 // the current IndexOperandHashMap. An operand hash may not be stable across
324 // different builds due to varying modules combined. To address this, we relax
325 // the hash check condition by comparing Const hash patterns instead of absolute
326 // hash values. For example, let's assume we have three Consts located at idx1,
327 // idx3, and idx6, where their corresponding hashes are hash1, hash2, and hash1
328 // in the old merged map below:
329 //   Old (Merged): [(idx1, hash1), (idx3, hash2), (idx6, hash1)]
330 //   Current: [(idx1, hash1'), (idx3, hash2'), (idx6, hash1')]
331 // If the current function also has three Consts in the same locations,
332 // with hash sequences hash1', hash2', and hash1' where the first and third
333 // are the same as the old hash sequences, we consider them matched.
334 static bool checkConstHashCompatible(
335     const DenseMap<IndexPair, stable_hash> &OldInstOpndIndexToConstHash,
336     const DenseMap<IndexPair, stable_hash> &CurrInstOpndIndexToConstHash) {
337 
338   DenseMap<stable_hash, stable_hash> OldHashToCurrHash;
339   for (const auto &[Index, OldHash] : OldInstOpndIndexToConstHash) {
340     auto It = CurrInstOpndIndexToConstHash.find(Index);
341     if (It == CurrInstOpndIndexToConstHash.end())
342       return false;
343 
344     auto CurrHash = It->second;
345     auto J = OldHashToCurrHash.find(OldHash);
346     if (J == OldHashToCurrHash.end())
347       OldHashToCurrHash.insert({OldHash, CurrHash});
348     else if (J->second != CurrHash)
349       return false;
350   }
351 
352   return true;
353 }
354 
355 // Validate the locations pointed by a param has the same hash and Constant.
356 static bool
357 checkConstLocationCompatible(const StableFunctionMap::StableFunctionEntry &SF,
358                              const IndexInstrMap &IndexInstruction,
359                              const ParamLocsVecTy &ParamLocsVec) {
360   for (auto &ParamLocs : ParamLocsVec) {
361     std::optional<stable_hash> OldHash;
362     std::optional<Constant *> OldConst;
363     for (auto &Loc : ParamLocs) {
364       assert(SF.IndexOperandHashMap->count(Loc));
365       auto CurrHash = SF.IndexOperandHashMap.get()->at(Loc);
366       auto [InstIndex, OpndIndex] = Loc;
367       assert(InstIndex < IndexInstruction.size());
368       const auto *Inst = IndexInstruction.lookup(InstIndex);
369       auto *CurrConst = cast<Constant>(Inst->getOperand(OpndIndex));
370       if (!OldHash) {
371         OldHash = CurrHash;
372         OldConst = CurrConst;
373       } else if (CurrConst != *OldConst || CurrHash != *OldHash) {
374         return false;
375       }
376     }
377   }
378   return true;
379 }
380 
381 static ParamLocsVecTy computeParamInfo(
382     const SmallVector<std::unique_ptr<StableFunctionMap::StableFunctionEntry>>
383         &SFS) {
384   std::map<std::vector<stable_hash>, ParamLocs> HashSeqToLocs;
385   auto &RSF = *SFS[0];
386   unsigned StableFunctionCount = SFS.size();
387 
388   for (auto &[IndexPair, Hash] : *RSF.IndexOperandHashMap) {
389     // Const hash sequence across stable functions.
390     // We will allocate a parameter per unique hash squence.
391     // can't use SmallVector as key
392     std::vector<stable_hash> ConstHashSeq;
393     ConstHashSeq.push_back(Hash);
394     bool Identical = true;
395     for (unsigned J = 1; J < StableFunctionCount; ++J) {
396       auto &SF = SFS[J];
397       auto SHash = SF->IndexOperandHashMap->at(IndexPair);
398       if (Hash != SHash)
399         Identical = false;
400       ConstHashSeq.push_back(SHash);
401     }
402 
403     if (Identical)
404       continue;
405 
406     // For each unique Const hash sequence (parameter), add the locations.
407     HashSeqToLocs[ConstHashSeq].push_back(IndexPair);
408   }
409 
410   ParamLocsVecTy ParamLocsVec;
411   for (auto &[HashSeq, Locs] : HashSeqToLocs) {
412     ParamLocsVec.push_back(std::move(Locs));
413     llvm::sort(ParamLocsVec, [&](const ParamLocs &L, const ParamLocs &R) {
414       return L[0] < R[0];
415     });
416   }
417   return ParamLocsVec;
418 }
419 
420 bool GlobalMergeFunc::merge(Module &M, const StableFunctionMap *FunctionMap) {
421   bool Changed = false;
422 
423   // Build a map from stable function name to function.
424   StringMap<Function *> StableNameToFuncMap;
425   for (auto &F : M)
426     StableNameToFuncMap[get_stable_name(F.getName())] = &F;
427   // Track merged functions
428   DenseSet<Function *> MergedFunctions;
429 
430   auto ModId = M.getModuleIdentifier();
431   for (auto &[Hash, SFS] : FunctionMap->getFunctionMap()) {
432     // Parameter locations based on the unique hash sequences
433     // across the candidates.
434     std::optional<ParamLocsVecTy> ParamLocsVec;
435     Function *MergedFunc = nullptr;
436     std::string MergedModId;
437     SmallVector<FuncMergeInfo> FuncMergeInfos;
438     for (auto &SF : SFS) {
439       // Get the function from the stable name.
440       auto I = StableNameToFuncMap.find(
441           *FunctionMap->getNameForId(SF->FunctionNameId));
442       if (I == StableNameToFuncMap.end())
443         continue;
444       Function *F = I->second;
445       assert(F);
446       // Skip if the function has been merged before.
447       if (MergedFunctions.count(F))
448         continue;
449       // Consider the function if it is eligible for merging.
450       if (!isEligibleFunction(F))
451         continue;
452 
453       auto FI = llvm::StructuralHashWithDifferences(*F, ignoreOp);
454       uint64_t FuncHash = FI.FunctionHash;
455       if (Hash != FuncHash) {
456         ++NumMismatchedFunctionHash;
457         continue;
458       }
459 
460       if (SF->InstCount != FI.IndexInstruction->size()) {
461         ++NumMismatchedInstCount;
462         continue;
463       }
464       bool HasValidSharedConst = true;
465       for (auto &[Index, Hash] : *SF->IndexOperandHashMap) {
466         auto [InstIndex, OpndIndex] = Index;
467         assert(InstIndex < FI.IndexInstruction->size());
468         auto *Inst = FI.IndexInstruction->lookup(InstIndex);
469         if (!ignoreOp(Inst, OpndIndex)) {
470           HasValidSharedConst = false;
471           break;
472         }
473       }
474       if (!HasValidSharedConst) {
475         ++NumMismatchedConstHash;
476         continue;
477       }
478       if (!checkConstHashCompatible(*SF->IndexOperandHashMap,
479                                     *FI.IndexOperandHashMap)) {
480         ++NumMismatchedConstHash;
481         continue;
482       }
483       if (!ParamLocsVec.has_value()) {
484         ParamLocsVec = computeParamInfo(SFS);
485         LLVM_DEBUG(dbgs() << "[GlobalMergeFunc] Merging hash: " << Hash
486                           << " with Params " << ParamLocsVec->size() << "\n");
487       }
488       if (!checkConstLocationCompatible(*SF, *FI.IndexInstruction,
489                                         *ParamLocsVec)) {
490         ++NumMismatchedConstHash;
491         continue;
492       }
493 
494       if (MergedFunc) {
495         // Check if the matched functions fall into the same (first) module.
496         // This module check is not strictly necessary as the functions can move
497         // around. We just want to avoid merging functions from different
498         // modules than the first one in the function map, as they may not end
499         // up with being ICFed by the linker.
500         if (MergedModId != *FunctionMap->getNameForId(SF->ModuleNameId)) {
501           ++NumMismatchedModuleId;
502           continue;
503         }
504       } else {
505         MergedFunc = F;
506         MergedModId = *FunctionMap->getNameForId(SF->ModuleNameId);
507       }
508 
509       FuncMergeInfos.emplace_back(SF.get(), F, std::move(FI.IndexInstruction));
510       MergedFunctions.insert(F);
511     }
512     unsigned FuncMergeInfoSize = FuncMergeInfos.size();
513     if (FuncMergeInfoSize == 0)
514       continue;
515 
516     LLVM_DEBUG(dbgs() << "[GlobalMergeFunc] Merging function count "
517                       << FuncMergeInfoSize << " in  " << ModId << "\n");
518 
519     for (auto &FMI : FuncMergeInfos) {
520       Changed = true;
521 
522       // We've already validated all locations of constant operands pointed by
523       // the parameters. Populate parameters pointing to the original constants.
524       SmallVector<Constant *> Params;
525       SmallVector<Type *> ParamTypes;
526       for (auto &ParamLocs : *ParamLocsVec) {
527         assert(!ParamLocs.empty());
528         auto &[InstIndex, OpndIndex] = ParamLocs[0];
529         auto *Inst = FMI.IndexInstruction->lookup(InstIndex);
530         auto *Opnd = cast<Constant>(Inst->getOperand(OpndIndex));
531         Params.push_back(Opnd);
532         ParamTypes.push_back(Opnd->getType());
533       }
534 
535       // Create a merged function derived from the current function.
536       Function *MergedFunc =
537           createMergedFunction(FMI, ParamTypes, *ParamLocsVec);
538 
539       LLVM_DEBUG({
540         dbgs() << "[GlobalMergeFunc] Merged function (hash:" << FMI.SF->Hash
541                << ") " << MergedFunc->getName() << " generated from "
542                << FMI.F->getName() << ":\n";
543         MergedFunc->dump();
544       });
545 
546       // Transform the current function into a thunk that calls the merged
547       // function.
548       createThunk(FMI, Params, MergedFunc);
549       LLVM_DEBUG({
550         dbgs() << "[GlobalMergeFunc] Thunk generated: \n";
551         FMI.F->dump();
552       });
553       ++NumMergedFunctions;
554     }
555   }
556 
557   return Changed;
558 }
559 
560 void GlobalMergeFunc::initializeMergerMode(const Module &M) {
561   // Initialize the local function map regardless of the merger mode.
562   LocalFunctionMap = std::make_unique<StableFunctionMap>();
563 
564   // Disable codegen data for merging. The local merge is still enabled.
565   if (DisableCGDataForMerging)
566     return;
567 
568   // (Full)LTO module does not have functions added to the index.
569   // In this case, we run a local merger without using codegen data.
570   if (Index && !Index->hasExportedFunctions(M))
571     return;
572 
573   if (cgdata::emitCGData())
574     MergerMode = HashFunctionMode::BuildingHashFuncion;
575   else if (cgdata::hasStableFunctionMap())
576     MergerMode = HashFunctionMode::UsingHashFunction;
577 }
578 
579 void GlobalMergeFunc::emitFunctionMap(Module &M) {
580   LLVM_DEBUG(dbgs() << "Emit function map. Size: " << LocalFunctionMap->size()
581                     << "\n");
582   // No need to emit the function map if it is empty.
583   if (LocalFunctionMap->empty())
584     return;
585   SmallVector<char> Buf;
586   raw_svector_ostream OS(Buf);
587 
588   StableFunctionMapRecord::serialize(OS, LocalFunctionMap.get());
589 
590   std::unique_ptr<MemoryBuffer> Buffer = MemoryBuffer::getMemBuffer(
591       OS.str(), "in-memory stable function map", false);
592 
593   Triple TT(M.getTargetTriple());
594   embedBufferInModule(M, *Buffer.get(),
595                       getCodeGenDataSectionName(CG_merge, TT.getObjectFormat()),
596                       Align(4));
597 }
598 
599 bool GlobalMergeFunc::run(Module &M) {
600   initializeMergerMode(M);
601 
602   const StableFunctionMap *FuncMap;
603   if (MergerMode == HashFunctionMode::UsingHashFunction) {
604     // Use the prior CG data to optimistically create global merge candidates.
605     FuncMap = cgdata::getStableFunctionMap();
606   } else {
607     analyze(M);
608     // Emit the local function map to the custom section, __llvm_merge before
609     // finalizing it.
610     if (MergerMode == HashFunctionMode::BuildingHashFuncion)
611       emitFunctionMap(M);
612     LocalFunctionMap->finalize();
613     FuncMap = LocalFunctionMap.get();
614   }
615 
616   return merge(M, FuncMap);
617 }
618 
619 namespace {
620 
621 class GlobalMergeFuncPassWrapper : public ModulePass {
622 
623 public:
624   static char ID;
625 
626   GlobalMergeFuncPassWrapper();
627 
628   void getAnalysisUsage(AnalysisUsage &AU) const override {
629     AU.addUsedIfAvailable<ImmutableModuleSummaryIndexWrapperPass>();
630     AU.setPreservesAll();
631     ModulePass::getAnalysisUsage(AU);
632   }
633 
634   StringRef getPassName() const override { return "Global Merge Functions"; }
635 
636   bool runOnModule(Module &M) override;
637 };
638 
639 } // namespace
640 
641 char GlobalMergeFuncPassWrapper::ID = 0;
642 INITIALIZE_PASS_BEGIN(GlobalMergeFuncPassWrapper, "global-merge-func",
643                       "Global merge function pass", false, false)
644 INITIALIZE_PASS_END(GlobalMergeFuncPassWrapper, "global-merge-func",
645                     "Global merge function pass", false, false)
646 
647 namespace llvm {
648 ModulePass *createGlobalMergeFuncPass() {
649   return new GlobalMergeFuncPassWrapper();
650 }
651 } // namespace llvm
652 
653 GlobalMergeFuncPassWrapper::GlobalMergeFuncPassWrapper() : ModulePass(ID) {
654   initializeGlobalMergeFuncPassWrapperPass(
655       *llvm::PassRegistry::getPassRegistry());
656 }
657 
658 bool GlobalMergeFuncPassWrapper::runOnModule(Module &M) {
659   const ModuleSummaryIndex *Index = nullptr;
660   if (auto *IndexWrapperPass =
661           getAnalysisIfAvailable<ImmutableModuleSummaryIndexWrapperPass>())
662     Index = IndexWrapperPass->getIndex();
663 
664   return GlobalMergeFunc(Index).run(M);
665 }
666 
667 PreservedAnalyses GlobalMergeFuncPass::run(Module &M,
668                                            AnalysisManager<Module> &AM) {
669   ModuleSummaryIndex *Index = &(AM.getResult<ModuleSummaryIndexAnalysis>(M));
670   bool Changed = GlobalMergeFunc(Index).run(M);
671   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
672 }
673