1 //===- SplitModule.cpp - Split a module into partitions -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the function llvm::SplitModule, which splits a module
10 // into multiple linkable partitions. It can be used to implement parallel code
11 // generation for link-time optimization.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "llvm/Transforms/Utils/SplitModule.h"
16 #include "llvm/ADT/DenseMap.h"
17 #include "llvm/ADT/EquivalenceClasses.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/IR/Comdat.h"
22 #include "llvm/IR/Constant.h"
23 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/Function.h"
25 #include "llvm/IR/GlobalAlias.h"
26 #include "llvm/IR/GlobalObject.h"
27 #include "llvm/IR/GlobalIndirectSymbol.h"
28 #include "llvm/IR/GlobalValue.h"
29 #include "llvm/IR/GlobalVariable.h"
30 #include "llvm/IR/Instruction.h"
31 #include "llvm/IR/Module.h"
32 #include "llvm/IR/User.h"
33 #include "llvm/IR/Value.h"
34 #include "llvm/Support/Casting.h"
35 #include "llvm/Support/Debug.h"
36 #include "llvm/Support/ErrorHandling.h"
37 #include "llvm/Support/MD5.h"
38 #include "llvm/Support/raw_ostream.h"
39 #include "llvm/Transforms/Utils/Cloning.h"
40 #include "llvm/Transforms/Utils/ValueMapper.h"
41 #include <algorithm>
42 #include <cassert>
43 #include <iterator>
44 #include <memory>
45 #include <queue>
46 #include <utility>
47 #include <vector>
48
49 using namespace llvm;
50
51 #define DEBUG_TYPE "split-module"
52
53 namespace {
54
55 using ClusterMapType = EquivalenceClasses<const GlobalValue *>;
56 using ComdatMembersType = DenseMap<const Comdat *, const GlobalValue *>;
57 using ClusterIDMapType = DenseMap<const GlobalValue *, unsigned>;
58
59 } // end anonymous namespace
60
addNonConstUser(ClusterMapType & GVtoClusterMap,const GlobalValue * GV,const User * U)61 static void addNonConstUser(ClusterMapType &GVtoClusterMap,
62 const GlobalValue *GV, const User *U) {
63 assert((!isa<Constant>(U) || isa<GlobalValue>(U)) && "Bad user");
64
65 if (const Instruction *I = dyn_cast<Instruction>(U)) {
66 const GlobalValue *F = I->getParent()->getParent();
67 GVtoClusterMap.unionSets(GV, F);
68 } else if (isa<GlobalIndirectSymbol>(U) || isa<Function>(U) ||
69 isa<GlobalVariable>(U)) {
70 GVtoClusterMap.unionSets(GV, cast<GlobalValue>(U));
71 } else {
72 llvm_unreachable("Underimplemented use case");
73 }
74 }
75
76 // Adds all GlobalValue users of V to the same cluster as GV.
addAllGlobalValueUsers(ClusterMapType & GVtoClusterMap,const GlobalValue * GV,const Value * V)77 static void addAllGlobalValueUsers(ClusterMapType &GVtoClusterMap,
78 const GlobalValue *GV, const Value *V) {
79 for (auto *U : V->users()) {
80 SmallVector<const User *, 4> Worklist;
81 Worklist.push_back(U);
82 while (!Worklist.empty()) {
83 const User *UU = Worklist.pop_back_val();
84 // For each constant that is not a GV (a pure const) recurse.
85 if (isa<Constant>(UU) && !isa<GlobalValue>(UU)) {
86 Worklist.append(UU->user_begin(), UU->user_end());
87 continue;
88 }
89 addNonConstUser(GVtoClusterMap, GV, UU);
90 }
91 }
92 }
93
94 // Find partitions for module in the way that no locals need to be
95 // globalized.
96 // Try to balance pack those partitions into N files since this roughly equals
97 // thread balancing for the backend codegen step.
findPartitions(Module & M,ClusterIDMapType & ClusterIDMap,unsigned N)98 static void findPartitions(Module &M, ClusterIDMapType &ClusterIDMap,
99 unsigned N) {
100 // At this point module should have the proper mix of globals and locals.
101 // As we attempt to partition this module, we must not change any
102 // locals to globals.
103 LLVM_DEBUG(dbgs() << "Partition module with (" << M.size() << ")functions\n");
104 ClusterMapType GVtoClusterMap;
105 ComdatMembersType ComdatMembers;
106
107 auto recordGVSet = [&GVtoClusterMap, &ComdatMembers](GlobalValue &GV) {
108 if (GV.isDeclaration())
109 return;
110
111 if (!GV.hasName())
112 GV.setName("__llvmsplit_unnamed");
113
114 // Comdat groups must not be partitioned. For comdat groups that contain
115 // locals, record all their members here so we can keep them together.
116 // Comdat groups that only contain external globals are already handled by
117 // the MD5-based partitioning.
118 if (const Comdat *C = GV.getComdat()) {
119 auto &Member = ComdatMembers[C];
120 if (Member)
121 GVtoClusterMap.unionSets(Member, &GV);
122 else
123 Member = &GV;
124 }
125
126 // For aliases we should not separate them from their aliasees regardless
127 // of linkage.
128 if (auto *GIS = dyn_cast<GlobalIndirectSymbol>(&GV)) {
129 if (const GlobalObject *Base = GIS->getBaseObject())
130 GVtoClusterMap.unionSets(&GV, Base);
131 }
132
133 if (const Function *F = dyn_cast<Function>(&GV)) {
134 for (const BasicBlock &BB : *F) {
135 BlockAddress *BA = BlockAddress::lookup(&BB);
136 if (!BA || !BA->isConstantUsed())
137 continue;
138 addAllGlobalValueUsers(GVtoClusterMap, F, BA);
139 }
140 }
141
142 if (GV.hasLocalLinkage())
143 addAllGlobalValueUsers(GVtoClusterMap, &GV, &GV);
144 };
145
146 llvm::for_each(M.functions(), recordGVSet);
147 llvm::for_each(M.globals(), recordGVSet);
148 llvm::for_each(M.aliases(), recordGVSet);
149
150 // Assigned all GVs to merged clusters while balancing number of objects in
151 // each.
152 auto CompareClusters = [](const std::pair<unsigned, unsigned> &a,
153 const std::pair<unsigned, unsigned> &b) {
154 if (a.second || b.second)
155 return a.second > b.second;
156 else
157 return a.first > b.first;
158 };
159
160 std::priority_queue<std::pair<unsigned, unsigned>,
161 std::vector<std::pair<unsigned, unsigned>>,
162 decltype(CompareClusters)>
163 BalancinQueue(CompareClusters);
164 // Pre-populate priority queue with N slot blanks.
165 for (unsigned i = 0; i < N; ++i)
166 BalancinQueue.push(std::make_pair(i, 0));
167
168 using SortType = std::pair<unsigned, ClusterMapType::iterator>;
169
170 SmallVector<SortType, 64> Sets;
171 SmallPtrSet<const GlobalValue *, 32> Visited;
172
173 // To guarantee determinism, we have to sort SCC according to size.
174 // When size is the same, use leader's name.
175 for (ClusterMapType::iterator I = GVtoClusterMap.begin(),
176 E = GVtoClusterMap.end(); I != E; ++I)
177 if (I->isLeader())
178 Sets.push_back(
179 std::make_pair(std::distance(GVtoClusterMap.member_begin(I),
180 GVtoClusterMap.member_end()), I));
181
182 llvm::sort(Sets, [](const SortType &a, const SortType &b) {
183 if (a.first == b.first)
184 return a.second->getData()->getName() > b.second->getData()->getName();
185 else
186 return a.first > b.first;
187 });
188
189 for (auto &I : Sets) {
190 unsigned CurrentClusterID = BalancinQueue.top().first;
191 unsigned CurrentClusterSize = BalancinQueue.top().second;
192 BalancinQueue.pop();
193
194 LLVM_DEBUG(dbgs() << "Root[" << CurrentClusterID << "] cluster_size("
195 << I.first << ") ----> " << I.second->getData()->getName()
196 << "\n");
197
198 for (ClusterMapType::member_iterator MI =
199 GVtoClusterMap.findLeader(I.second);
200 MI != GVtoClusterMap.member_end(); ++MI) {
201 if (!Visited.insert(*MI).second)
202 continue;
203 LLVM_DEBUG(dbgs() << "----> " << (*MI)->getName()
204 << ((*MI)->hasLocalLinkage() ? " l " : " e ") << "\n");
205 Visited.insert(*MI);
206 ClusterIDMap[*MI] = CurrentClusterID;
207 CurrentClusterSize++;
208 }
209 // Add this set size to the number of entries in this cluster.
210 BalancinQueue.push(std::make_pair(CurrentClusterID, CurrentClusterSize));
211 }
212 }
213
externalize(GlobalValue * GV)214 static void externalize(GlobalValue *GV) {
215 if (GV->hasLocalLinkage()) {
216 GV->setLinkage(GlobalValue::ExternalLinkage);
217 GV->setVisibility(GlobalValue::HiddenVisibility);
218 }
219
220 // Unnamed entities must be named consistently between modules. setName will
221 // give a distinct name to each such entity.
222 if (!GV->hasName())
223 GV->setName("__llvmsplit_unnamed");
224 }
225
226 // Returns whether GV should be in partition (0-based) I of N.
isInPartition(const GlobalValue * GV,unsigned I,unsigned N)227 static bool isInPartition(const GlobalValue *GV, unsigned I, unsigned N) {
228 if (auto *GIS = dyn_cast<GlobalIndirectSymbol>(GV))
229 if (const GlobalObject *Base = GIS->getBaseObject())
230 GV = Base;
231
232 StringRef Name;
233 if (const Comdat *C = GV->getComdat())
234 Name = C->getName();
235 else
236 Name = GV->getName();
237
238 // Partition by MD5 hash. We only need a few bits for evenness as the number
239 // of partitions will generally be in the 1-2 figure range; the low 16 bits
240 // are enough.
241 MD5 H;
242 MD5::MD5Result R;
243 H.update(Name);
244 H.final(R);
245 return (R[0] | (R[1] << 8)) % N == I;
246 }
247
SplitModule(Module & M,unsigned N,function_ref<void (std::unique_ptr<Module> MPart)> ModuleCallback,bool PreserveLocals)248 void llvm::SplitModule(
249 Module &M, unsigned N,
250 function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback,
251 bool PreserveLocals) {
252 if (!PreserveLocals) {
253 for (Function &F : M)
254 externalize(&F);
255 for (GlobalVariable &GV : M.globals())
256 externalize(&GV);
257 for (GlobalAlias &GA : M.aliases())
258 externalize(&GA);
259 for (GlobalIFunc &GIF : M.ifuncs())
260 externalize(&GIF);
261 }
262
263 // This performs splitting without a need for externalization, which might not
264 // always be possible.
265 ClusterIDMapType ClusterIDMap;
266 findPartitions(M, ClusterIDMap, N);
267
268 // FIXME: We should be able to reuse M as the last partition instead of
269 // cloning it. Note that the callers at the moment expect the module to
270 // be preserved, so will need some adjustments as well.
271 for (unsigned I = 0; I < N; ++I) {
272 ValueToValueMapTy VMap;
273 std::unique_ptr<Module> MPart(
274 CloneModule(M, VMap, [&](const GlobalValue *GV) {
275 if (ClusterIDMap.count(GV))
276 return (ClusterIDMap[GV] == I);
277 else
278 return isInPartition(GV, I, N);
279 }));
280 if (I != 0)
281 MPart->setModuleInlineAsm("");
282 ModuleCallback(std::move(MPart));
283 }
284 }
285