1097a140dSpatrick //===-- IPO/OpenMPOpt.cpp - Collection of OpenMP specific optimizations ---===//
2097a140dSpatrick //
3097a140dSpatrick // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4097a140dSpatrick // See https://llvm.org/LICENSE.txt for license information.
5097a140dSpatrick // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6097a140dSpatrick //
7097a140dSpatrick //===----------------------------------------------------------------------===//
8097a140dSpatrick //
9097a140dSpatrick // OpenMP specific optimizations:
10097a140dSpatrick //
11097a140dSpatrick // - Deduplication of runtime calls, e.g., omp_get_thread_num.
1273471bf0Spatrick // - Replacing globalized device memory with stack memory.
1373471bf0Spatrick // - Replacing globalized device memory with shared memory.
1473471bf0Spatrick // - Parallel region merging.
1573471bf0Spatrick // - Transforming generic-mode device kernels to SPMD mode.
1673471bf0Spatrick // - Specializing the state machine for generic-mode device kernels.
17097a140dSpatrick //
18097a140dSpatrick //===----------------------------------------------------------------------===//
19097a140dSpatrick
20097a140dSpatrick #include "llvm/Transforms/IPO/OpenMPOpt.h"
21097a140dSpatrick
22097a140dSpatrick #include "llvm/ADT/EnumeratedArray.h"
2373471bf0Spatrick #include "llvm/ADT/PostOrderIterator.h"
24*d415bd75Srobert #include "llvm/ADT/SetVector.h"
25*d415bd75Srobert #include "llvm/ADT/SmallVector.h"
26097a140dSpatrick #include "llvm/ADT/Statistic.h"
27*d415bd75Srobert #include "llvm/ADT/StringRef.h"
28097a140dSpatrick #include "llvm/Analysis/CallGraph.h"
29097a140dSpatrick #include "llvm/Analysis/CallGraphSCCPass.h"
30*d415bd75Srobert #include "llvm/Analysis/MemoryLocation.h"
31097a140dSpatrick #include "llvm/Analysis/OptimizationRemarkEmitter.h"
3273471bf0Spatrick #include "llvm/Analysis/ValueTracking.h"
33097a140dSpatrick #include "llvm/Frontend/OpenMP/OMPConstants.h"
34097a140dSpatrick #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
3573471bf0Spatrick #include "llvm/IR/Assumptions.h"
36*d415bd75Srobert #include "llvm/IR/BasicBlock.h"
37*d415bd75Srobert #include "llvm/IR/Constants.h"
3873471bf0Spatrick #include "llvm/IR/DiagnosticInfo.h"
3973471bf0Spatrick #include "llvm/IR/GlobalValue.h"
40*d415bd75Srobert #include "llvm/IR/GlobalVariable.h"
4173471bf0Spatrick #include "llvm/IR/Instruction.h"
42*d415bd75Srobert #include "llvm/IR/Instructions.h"
4373471bf0Spatrick #include "llvm/IR/IntrinsicInst.h"
44*d415bd75Srobert #include "llvm/IR/IntrinsicsAMDGPU.h"
45*d415bd75Srobert #include "llvm/IR/IntrinsicsNVPTX.h"
46*d415bd75Srobert #include "llvm/IR/LLVMContext.h"
47097a140dSpatrick #include "llvm/InitializePasses.h"
48097a140dSpatrick #include "llvm/Support/CommandLine.h"
49*d415bd75Srobert #include "llvm/Support/Debug.h"
50097a140dSpatrick #include "llvm/Transforms/IPO/Attributor.h"
5173471bf0Spatrick #include "llvm/Transforms/Utils/BasicBlockUtils.h"
52097a140dSpatrick #include "llvm/Transforms/Utils/CallGraphUpdater.h"
53*d415bd75Srobert
54*d415bd75Srobert #include <algorithm>
55*d415bd75Srobert #include <optional>
56*d415bd75Srobert #include <string>
57097a140dSpatrick
58097a140dSpatrick using namespace llvm;
59097a140dSpatrick using namespace omp;
60097a140dSpatrick
61097a140dSpatrick #define DEBUG_TYPE "openmp-opt"
62097a140dSpatrick
63097a140dSpatrick static cl::opt<bool> DisableOpenMPOptimizations(
64*d415bd75Srobert "openmp-opt-disable", cl::desc("Disable OpenMP specific optimizations."),
65*d415bd75Srobert cl::Hidden, cl::init(false));
66097a140dSpatrick
6773471bf0Spatrick static cl::opt<bool> EnableParallelRegionMerging(
68*d415bd75Srobert "openmp-opt-enable-merging",
6973471bf0Spatrick cl::desc("Enable the OpenMP region merging optimization."), cl::Hidden,
7073471bf0Spatrick cl::init(false));
7173471bf0Spatrick
7273471bf0Spatrick static cl::opt<bool>
73*d415bd75Srobert DisableInternalization("openmp-opt-disable-internalization",
7473471bf0Spatrick cl::desc("Disable function internalization."),
7573471bf0Spatrick cl::Hidden, cl::init(false));
7673471bf0Spatrick
77*d415bd75Srobert static cl::opt<bool> DeduceICVValues("openmp-deduce-icv-values",
78*d415bd75Srobert cl::init(false), cl::Hidden);
79097a140dSpatrick static cl::opt<bool> PrintICVValues("openmp-print-icv-values", cl::init(false),
80097a140dSpatrick cl::Hidden);
81097a140dSpatrick static cl::opt<bool> PrintOpenMPKernels("openmp-print-gpu-kernels",
82097a140dSpatrick cl::init(false), cl::Hidden);
83097a140dSpatrick
8473471bf0Spatrick static cl::opt<bool> HideMemoryTransferLatency(
8573471bf0Spatrick "openmp-hide-memory-transfer-latency",
8673471bf0Spatrick cl::desc("[WIP] Tries to hide the latency of host to device memory"
8773471bf0Spatrick " transfers"),
8873471bf0Spatrick cl::Hidden, cl::init(false));
8973471bf0Spatrick
90*d415bd75Srobert static cl::opt<bool> DisableOpenMPOptDeglobalization(
91*d415bd75Srobert "openmp-opt-disable-deglobalization",
92*d415bd75Srobert cl::desc("Disable OpenMP optimizations involving deglobalization."),
93*d415bd75Srobert cl::Hidden, cl::init(false));
94*d415bd75Srobert
95*d415bd75Srobert static cl::opt<bool> DisableOpenMPOptSPMDization(
96*d415bd75Srobert "openmp-opt-disable-spmdization",
97*d415bd75Srobert cl::desc("Disable OpenMP optimizations involving SPMD-ization."),
98*d415bd75Srobert cl::Hidden, cl::init(false));
99*d415bd75Srobert
100*d415bd75Srobert static cl::opt<bool> DisableOpenMPOptFolding(
101*d415bd75Srobert "openmp-opt-disable-folding",
102*d415bd75Srobert cl::desc("Disable OpenMP optimizations involving folding."), cl::Hidden,
103*d415bd75Srobert cl::init(false));
104*d415bd75Srobert
105*d415bd75Srobert static cl::opt<bool> DisableOpenMPOptStateMachineRewrite(
106*d415bd75Srobert "openmp-opt-disable-state-machine-rewrite",
107*d415bd75Srobert cl::desc("Disable OpenMP optimizations that replace the state machine."),
108*d415bd75Srobert cl::Hidden, cl::init(false));
109*d415bd75Srobert
110*d415bd75Srobert static cl::opt<bool> DisableOpenMPOptBarrierElimination(
111*d415bd75Srobert "openmp-opt-disable-barrier-elimination",
112*d415bd75Srobert cl::desc("Disable OpenMP optimizations that eliminate barriers."),
113*d415bd75Srobert cl::Hidden, cl::init(false));
114*d415bd75Srobert
115*d415bd75Srobert static cl::opt<bool> PrintModuleAfterOptimizations(
116*d415bd75Srobert "openmp-opt-print-module-after",
117*d415bd75Srobert cl::desc("Print the current module after OpenMP optimizations."),
118*d415bd75Srobert cl::Hidden, cl::init(false));
119*d415bd75Srobert
120*d415bd75Srobert static cl::opt<bool> PrintModuleBeforeOptimizations(
121*d415bd75Srobert "openmp-opt-print-module-before",
122*d415bd75Srobert cl::desc("Print the current module before OpenMP optimizations."),
123*d415bd75Srobert cl::Hidden, cl::init(false));
124*d415bd75Srobert
125*d415bd75Srobert static cl::opt<bool> AlwaysInlineDeviceFunctions(
126*d415bd75Srobert "openmp-opt-inline-device",
127*d415bd75Srobert cl::desc("Inline all applicible functions on the device."), cl::Hidden,
128*d415bd75Srobert cl::init(false));
129*d415bd75Srobert
130*d415bd75Srobert static cl::opt<bool>
131*d415bd75Srobert EnableVerboseRemarks("openmp-opt-verbose-remarks",
132*d415bd75Srobert cl::desc("Enables more verbose remarks."), cl::Hidden,
133*d415bd75Srobert cl::init(false));
134*d415bd75Srobert
135*d415bd75Srobert static cl::opt<unsigned>
136*d415bd75Srobert SetFixpointIterations("openmp-opt-max-iterations", cl::Hidden,
137*d415bd75Srobert cl::desc("Maximal number of attributor iterations."),
138*d415bd75Srobert cl::init(256));
139*d415bd75Srobert
140*d415bd75Srobert static cl::opt<unsigned>
141*d415bd75Srobert SharedMemoryLimit("openmp-opt-shared-limit", cl::Hidden,
142*d415bd75Srobert cl::desc("Maximum amount of shared memory to use."),
143*d415bd75Srobert cl::init(std::numeric_limits<unsigned>::max()));
144*d415bd75Srobert
145097a140dSpatrick STATISTIC(NumOpenMPRuntimeCallsDeduplicated,
146097a140dSpatrick "Number of OpenMP runtime calls deduplicated");
147097a140dSpatrick STATISTIC(NumOpenMPParallelRegionsDeleted,
148097a140dSpatrick "Number of OpenMP parallel regions deleted");
149097a140dSpatrick STATISTIC(NumOpenMPRuntimeFunctionsIdentified,
150097a140dSpatrick "Number of OpenMP runtime functions identified");
151097a140dSpatrick STATISTIC(NumOpenMPRuntimeFunctionUsesIdentified,
152097a140dSpatrick "Number of OpenMP runtime function uses identified");
153097a140dSpatrick STATISTIC(NumOpenMPTargetRegionKernels,
154097a140dSpatrick "Number of OpenMP target region entry points (=kernels) identified");
15573471bf0Spatrick STATISTIC(NumOpenMPTargetRegionKernelsSPMD,
15673471bf0Spatrick "Number of OpenMP target region entry points (=kernels) executed in "
15773471bf0Spatrick "SPMD-mode instead of generic-mode");
15873471bf0Spatrick STATISTIC(NumOpenMPTargetRegionKernelsWithoutStateMachine,
15973471bf0Spatrick "Number of OpenMP target region entry points (=kernels) executed in "
16073471bf0Spatrick "generic-mode without a state machines");
16173471bf0Spatrick STATISTIC(NumOpenMPTargetRegionKernelsCustomStateMachineWithFallback,
16273471bf0Spatrick "Number of OpenMP target region entry points (=kernels) executed in "
16373471bf0Spatrick "generic-mode with customized state machines with fallback");
16473471bf0Spatrick STATISTIC(NumOpenMPTargetRegionKernelsCustomStateMachineWithoutFallback,
16573471bf0Spatrick "Number of OpenMP target region entry points (=kernels) executed in "
16673471bf0Spatrick "generic-mode with customized state machines without fallback");
167097a140dSpatrick STATISTIC(
168097a140dSpatrick NumOpenMPParallelRegionsReplacedInGPUStateMachine,
169097a140dSpatrick "Number of OpenMP parallel regions replaced with ID in GPU state machines");
17073471bf0Spatrick STATISTIC(NumOpenMPParallelRegionsMerged,
17173471bf0Spatrick "Number of OpenMP parallel regions merged");
17273471bf0Spatrick STATISTIC(NumBytesMovedToSharedMemory,
17373471bf0Spatrick "Amount of memory pushed to shared memory");
174*d415bd75Srobert STATISTIC(NumBarriersEliminated, "Number of redundant barriers eliminated");
175097a140dSpatrick
176097a140dSpatrick #if !defined(NDEBUG)
177097a140dSpatrick static constexpr auto TAG = "[" DEBUG_TYPE "]";
178097a140dSpatrick #endif
179097a140dSpatrick
180097a140dSpatrick namespace {
181097a140dSpatrick
18273471bf0Spatrick struct AAHeapToShared;
18373471bf0Spatrick
184097a140dSpatrick struct AAICVTracker;
185097a140dSpatrick
186097a140dSpatrick /// OpenMP specific information. For now, stores RFIs and ICVs also needed for
187097a140dSpatrick /// Attributor runs.
188097a140dSpatrick struct OMPInformationCache : public InformationCache {
OMPInformationCache__anon4b47cb5e0111::OMPInformationCache189097a140dSpatrick OMPInformationCache(Module &M, AnalysisGetter &AG,
190*d415bd75Srobert BumpPtrAllocator &Allocator, SetVector<Function *> *CGSCC,
191*d415bd75Srobert KernelSet &Kernels, bool OpenMPPostLink)
192*d415bd75Srobert : InformationCache(M, AG, Allocator, CGSCC), OMPBuilder(M),
193*d415bd75Srobert Kernels(Kernels), OpenMPPostLink(OpenMPPostLink) {
194097a140dSpatrick
195097a140dSpatrick OMPBuilder.initialize();
196*d415bd75Srobert initializeRuntimeFunctions(M);
197097a140dSpatrick initializeInternalControlVars();
198097a140dSpatrick }
199097a140dSpatrick
200097a140dSpatrick /// Generic information that describes an internal control variable.
201097a140dSpatrick struct InternalControlVarInfo {
202097a140dSpatrick /// The kind, as described by InternalControlVar enum.
203097a140dSpatrick InternalControlVar Kind;
204097a140dSpatrick
205097a140dSpatrick /// The name of the ICV.
206097a140dSpatrick StringRef Name;
207097a140dSpatrick
208097a140dSpatrick /// Environment variable associated with this ICV.
209097a140dSpatrick StringRef EnvVarName;
210097a140dSpatrick
211097a140dSpatrick /// Initial value kind.
212097a140dSpatrick ICVInitValue InitKind;
213097a140dSpatrick
214097a140dSpatrick /// Initial value.
215097a140dSpatrick ConstantInt *InitValue;
216097a140dSpatrick
217097a140dSpatrick /// Setter RTL function associated with this ICV.
218097a140dSpatrick RuntimeFunction Setter;
219097a140dSpatrick
220097a140dSpatrick /// Getter RTL function associated with this ICV.
221097a140dSpatrick RuntimeFunction Getter;
222097a140dSpatrick
223097a140dSpatrick /// RTL Function corresponding to the override clause of this ICV
224097a140dSpatrick RuntimeFunction Clause;
225097a140dSpatrick };
226097a140dSpatrick
227097a140dSpatrick /// Generic information that describes a runtime function
228097a140dSpatrick struct RuntimeFunctionInfo {
229097a140dSpatrick
230097a140dSpatrick /// The kind, as described by the RuntimeFunction enum.
231097a140dSpatrick RuntimeFunction Kind;
232097a140dSpatrick
233097a140dSpatrick /// The name of the function.
234097a140dSpatrick StringRef Name;
235097a140dSpatrick
236097a140dSpatrick /// Flag to indicate a variadic function.
237097a140dSpatrick bool IsVarArg;
238097a140dSpatrick
239097a140dSpatrick /// The return type of the function.
240097a140dSpatrick Type *ReturnType;
241097a140dSpatrick
242097a140dSpatrick /// The argument types of the function.
243097a140dSpatrick SmallVector<Type *, 8> ArgumentTypes;
244097a140dSpatrick
245097a140dSpatrick /// The declaration if available.
246097a140dSpatrick Function *Declaration = nullptr;
247097a140dSpatrick
248097a140dSpatrick /// Uses of this runtime function per function containing the use.
249097a140dSpatrick using UseVector = SmallVector<Use *, 16>;
250097a140dSpatrick
251097a140dSpatrick /// Clear UsesMap for runtime function.
clearUsesMap__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo252097a140dSpatrick void clearUsesMap() { UsesMap.clear(); }
253097a140dSpatrick
254097a140dSpatrick /// Boolean conversion that is true if the runtime function was found.
operator bool__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo255097a140dSpatrick operator bool() const { return Declaration; }
256097a140dSpatrick
257097a140dSpatrick /// Return the vector of uses in function \p F.
getOrCreateUseVector__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo258097a140dSpatrick UseVector &getOrCreateUseVector(Function *F) {
259097a140dSpatrick std::shared_ptr<UseVector> &UV = UsesMap[F];
260097a140dSpatrick if (!UV)
261097a140dSpatrick UV = std::make_shared<UseVector>();
262097a140dSpatrick return *UV;
263097a140dSpatrick }
264097a140dSpatrick
265097a140dSpatrick /// Return the vector of uses in function \p F or `nullptr` if there are
266097a140dSpatrick /// none.
getUseVector__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo267097a140dSpatrick const UseVector *getUseVector(Function &F) const {
268097a140dSpatrick auto I = UsesMap.find(&F);
269097a140dSpatrick if (I != UsesMap.end())
270097a140dSpatrick return I->second.get();
271097a140dSpatrick return nullptr;
272097a140dSpatrick }
273097a140dSpatrick
274097a140dSpatrick /// Return how many functions contain uses of this runtime function.
getNumFunctionsWithUses__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo275097a140dSpatrick size_t getNumFunctionsWithUses() const { return UsesMap.size(); }
276097a140dSpatrick
277097a140dSpatrick /// Return the number of arguments (or the minimal number for variadic
278097a140dSpatrick /// functions).
getNumArgs__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo279097a140dSpatrick size_t getNumArgs() const { return ArgumentTypes.size(); }
280097a140dSpatrick
281097a140dSpatrick /// Run the callback \p CB on each use and forget the use if the result is
282097a140dSpatrick /// true. The callback will be fed the function in which the use was
283097a140dSpatrick /// encountered as second argument.
foreachUse__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo284097a140dSpatrick void foreachUse(SmallVectorImpl<Function *> &SCC,
285097a140dSpatrick function_ref<bool(Use &, Function &)> CB) {
286097a140dSpatrick for (Function *F : SCC)
287097a140dSpatrick foreachUse(CB, F);
288097a140dSpatrick }
289097a140dSpatrick
290097a140dSpatrick /// Run the callback \p CB on each use within the function \p F and forget
291097a140dSpatrick /// the use if the result is true.
foreachUse__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo292097a140dSpatrick void foreachUse(function_ref<bool(Use &, Function &)> CB, Function *F) {
293097a140dSpatrick SmallVector<unsigned, 8> ToBeDeleted;
294097a140dSpatrick ToBeDeleted.clear();
295097a140dSpatrick
296097a140dSpatrick unsigned Idx = 0;
297097a140dSpatrick UseVector &UV = getOrCreateUseVector(F);
298097a140dSpatrick
299097a140dSpatrick for (Use *U : UV) {
300097a140dSpatrick if (CB(*U, *F))
301097a140dSpatrick ToBeDeleted.push_back(Idx);
302097a140dSpatrick ++Idx;
303097a140dSpatrick }
304097a140dSpatrick
305097a140dSpatrick // Remove the to-be-deleted indices in reverse order as prior
306097a140dSpatrick // modifications will not modify the smaller indices.
307097a140dSpatrick while (!ToBeDeleted.empty()) {
308097a140dSpatrick unsigned Idx = ToBeDeleted.pop_back_val();
309097a140dSpatrick UV[Idx] = UV.back();
310097a140dSpatrick UV.pop_back();
311097a140dSpatrick }
312097a140dSpatrick }
313097a140dSpatrick
314097a140dSpatrick private:
315097a140dSpatrick /// Map from functions to all uses of this runtime function contained in
316097a140dSpatrick /// them.
317097a140dSpatrick DenseMap<Function *, std::shared_ptr<UseVector>> UsesMap;
31873471bf0Spatrick
31973471bf0Spatrick public:
32073471bf0Spatrick /// Iterators for the uses of this runtime function.
begin__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo32173471bf0Spatrick decltype(UsesMap)::iterator begin() { return UsesMap.begin(); }
end__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo32273471bf0Spatrick decltype(UsesMap)::iterator end() { return UsesMap.end(); }
323097a140dSpatrick };
324097a140dSpatrick
325097a140dSpatrick /// An OpenMP-IR-Builder instance
326097a140dSpatrick OpenMPIRBuilder OMPBuilder;
327097a140dSpatrick
328097a140dSpatrick /// Map from runtime function kind to the runtime function description.
329097a140dSpatrick EnumeratedArray<RuntimeFunctionInfo, RuntimeFunction,
330097a140dSpatrick RuntimeFunction::OMPRTL___last>
331097a140dSpatrick RFIs;
332097a140dSpatrick
33373471bf0Spatrick /// Map from function declarations/definitions to their runtime enum type.
33473471bf0Spatrick DenseMap<Function *, RuntimeFunction> RuntimeFunctionIDMap;
33573471bf0Spatrick
336097a140dSpatrick /// Map from ICV kind to the ICV description.
337097a140dSpatrick EnumeratedArray<InternalControlVarInfo, InternalControlVar,
338097a140dSpatrick InternalControlVar::ICV___last>
339097a140dSpatrick ICVs;
340097a140dSpatrick
341097a140dSpatrick /// Helper to initialize all internal control variable information for those
342097a140dSpatrick /// defined in OMPKinds.def.
initializeInternalControlVars__anon4b47cb5e0111::OMPInformationCache343097a140dSpatrick void initializeInternalControlVars() {
344097a140dSpatrick #define ICV_RT_SET(_Name, RTL) \
345097a140dSpatrick { \
346097a140dSpatrick auto &ICV = ICVs[_Name]; \
347097a140dSpatrick ICV.Setter = RTL; \
348097a140dSpatrick }
349097a140dSpatrick #define ICV_RT_GET(Name, RTL) \
350097a140dSpatrick { \
351097a140dSpatrick auto &ICV = ICVs[Name]; \
352097a140dSpatrick ICV.Getter = RTL; \
353097a140dSpatrick }
354097a140dSpatrick #define ICV_DATA_ENV(Enum, _Name, _EnvVarName, Init) \
355097a140dSpatrick { \
356097a140dSpatrick auto &ICV = ICVs[Enum]; \
357097a140dSpatrick ICV.Name = _Name; \
358097a140dSpatrick ICV.Kind = Enum; \
359097a140dSpatrick ICV.InitKind = Init; \
360097a140dSpatrick ICV.EnvVarName = _EnvVarName; \
361097a140dSpatrick switch (ICV.InitKind) { \
362097a140dSpatrick case ICV_IMPLEMENTATION_DEFINED: \
363097a140dSpatrick ICV.InitValue = nullptr; \
364097a140dSpatrick break; \
365097a140dSpatrick case ICV_ZERO: \
366097a140dSpatrick ICV.InitValue = ConstantInt::get( \
367097a140dSpatrick Type::getInt32Ty(OMPBuilder.Int32->getContext()), 0); \
368097a140dSpatrick break; \
369097a140dSpatrick case ICV_FALSE: \
370097a140dSpatrick ICV.InitValue = ConstantInt::getFalse(OMPBuilder.Int1->getContext()); \
371097a140dSpatrick break; \
372097a140dSpatrick case ICV_LAST: \
373097a140dSpatrick break; \
374097a140dSpatrick } \
375097a140dSpatrick }
376097a140dSpatrick #include "llvm/Frontend/OpenMP/OMPKinds.def"
377097a140dSpatrick }
378097a140dSpatrick
379097a140dSpatrick /// Returns true if the function declaration \p F matches the runtime
380097a140dSpatrick /// function types, that is, return type \p RTFRetType, and argument types
381097a140dSpatrick /// \p RTFArgTypes.
declMatchesRTFTypes__anon4b47cb5e0111::OMPInformationCache382097a140dSpatrick static bool declMatchesRTFTypes(Function *F, Type *RTFRetType,
383097a140dSpatrick SmallVector<Type *, 8> &RTFArgTypes) {
384097a140dSpatrick // TODO: We should output information to the user (under debug output
385097a140dSpatrick // and via remarks).
386097a140dSpatrick
387097a140dSpatrick if (!F)
388097a140dSpatrick return false;
389097a140dSpatrick if (F->getReturnType() != RTFRetType)
390097a140dSpatrick return false;
391097a140dSpatrick if (F->arg_size() != RTFArgTypes.size())
392097a140dSpatrick return false;
393097a140dSpatrick
394*d415bd75Srobert auto *RTFTyIt = RTFArgTypes.begin();
395097a140dSpatrick for (Argument &Arg : F->args()) {
396097a140dSpatrick if (Arg.getType() != *RTFTyIt)
397097a140dSpatrick return false;
398097a140dSpatrick
399097a140dSpatrick ++RTFTyIt;
400097a140dSpatrick }
401097a140dSpatrick
402097a140dSpatrick return true;
403097a140dSpatrick }
404097a140dSpatrick
405097a140dSpatrick // Helper to collect all uses of the declaration in the UsesMap.
collectUses__anon4b47cb5e0111::OMPInformationCache406097a140dSpatrick unsigned collectUses(RuntimeFunctionInfo &RFI, bool CollectStats = true) {
407097a140dSpatrick unsigned NumUses = 0;
408097a140dSpatrick if (!RFI.Declaration)
409097a140dSpatrick return NumUses;
410097a140dSpatrick OMPBuilder.addAttributes(RFI.Kind, *RFI.Declaration);
411097a140dSpatrick
412097a140dSpatrick if (CollectStats) {
413097a140dSpatrick NumOpenMPRuntimeFunctionsIdentified += 1;
414097a140dSpatrick NumOpenMPRuntimeFunctionUsesIdentified += RFI.Declaration->getNumUses();
415097a140dSpatrick }
416097a140dSpatrick
417097a140dSpatrick // TODO: We directly convert uses into proper calls and unknown uses.
418097a140dSpatrick for (Use &U : RFI.Declaration->uses()) {
419097a140dSpatrick if (Instruction *UserI = dyn_cast<Instruction>(U.getUser())) {
420*d415bd75Srobert if (ModuleSlice.empty() || ModuleSlice.count(UserI->getFunction())) {
421097a140dSpatrick RFI.getOrCreateUseVector(UserI->getFunction()).push_back(&U);
422097a140dSpatrick ++NumUses;
423097a140dSpatrick }
424097a140dSpatrick } else {
425097a140dSpatrick RFI.getOrCreateUseVector(nullptr).push_back(&U);
426097a140dSpatrick ++NumUses;
427097a140dSpatrick }
428097a140dSpatrick }
429097a140dSpatrick return NumUses;
430097a140dSpatrick }
431097a140dSpatrick
43273471bf0Spatrick // Helper function to recollect uses of a runtime function.
recollectUsesForFunction__anon4b47cb5e0111::OMPInformationCache43373471bf0Spatrick void recollectUsesForFunction(RuntimeFunction RTF) {
43473471bf0Spatrick auto &RFI = RFIs[RTF];
435097a140dSpatrick RFI.clearUsesMap();
436097a140dSpatrick collectUses(RFI, /*CollectStats*/ false);
437097a140dSpatrick }
43873471bf0Spatrick
43973471bf0Spatrick // Helper function to recollect uses of all runtime functions.
recollectUses__anon4b47cb5e0111::OMPInformationCache44073471bf0Spatrick void recollectUses() {
44173471bf0Spatrick for (int Idx = 0; Idx < RFIs.size(); ++Idx)
44273471bf0Spatrick recollectUsesForFunction(static_cast<RuntimeFunction>(Idx));
443097a140dSpatrick }
444097a140dSpatrick
445*d415bd75Srobert // Helper function to inherit the calling convention of the function callee.
setCallingConvention__anon4b47cb5e0111::OMPInformationCache446*d415bd75Srobert void setCallingConvention(FunctionCallee Callee, CallInst *CI) {
447*d415bd75Srobert if (Function *Fn = dyn_cast<Function>(Callee.getCallee()))
448*d415bd75Srobert CI->setCallingConv(Fn->getCallingConv());
449*d415bd75Srobert }
450*d415bd75Srobert
451*d415bd75Srobert // Helper function to determine if it's legal to create a call to the runtime
452*d415bd75Srobert // functions.
runtimeFnsAvailable__anon4b47cb5e0111::OMPInformationCache453*d415bd75Srobert bool runtimeFnsAvailable(ArrayRef<RuntimeFunction> Fns) {
454*d415bd75Srobert // We can always emit calls if we haven't yet linked in the runtime.
455*d415bd75Srobert if (!OpenMPPostLink)
456*d415bd75Srobert return true;
457*d415bd75Srobert
458*d415bd75Srobert // Once the runtime has been already been linked in we cannot emit calls to
459*d415bd75Srobert // any undefined functions.
460*d415bd75Srobert for (RuntimeFunction Fn : Fns) {
461*d415bd75Srobert RuntimeFunctionInfo &RFI = RFIs[Fn];
462*d415bd75Srobert
463*d415bd75Srobert if (RFI.Declaration && RFI.Declaration->isDeclaration())
464*d415bd75Srobert return false;
465*d415bd75Srobert }
466*d415bd75Srobert return true;
467*d415bd75Srobert }
468*d415bd75Srobert
469097a140dSpatrick /// Helper to initialize all runtime function information for those defined
470097a140dSpatrick /// in OpenMPKinds.def.
initializeRuntimeFunctions__anon4b47cb5e0111::OMPInformationCache471*d415bd75Srobert void initializeRuntimeFunctions(Module &M) {
472097a140dSpatrick
473097a140dSpatrick // Helper macros for handling __VA_ARGS__ in OMP_RTL
474097a140dSpatrick #define OMP_TYPE(VarName, ...) \
475097a140dSpatrick Type *VarName = OMPBuilder.VarName; \
476097a140dSpatrick (void)VarName;
477097a140dSpatrick
478097a140dSpatrick #define OMP_ARRAY_TYPE(VarName, ...) \
479097a140dSpatrick ArrayType *VarName##Ty = OMPBuilder.VarName##Ty; \
480097a140dSpatrick (void)VarName##Ty; \
481097a140dSpatrick PointerType *VarName##PtrTy = OMPBuilder.VarName##PtrTy; \
482097a140dSpatrick (void)VarName##PtrTy;
483097a140dSpatrick
484097a140dSpatrick #define OMP_FUNCTION_TYPE(VarName, ...) \
485097a140dSpatrick FunctionType *VarName = OMPBuilder.VarName; \
486097a140dSpatrick (void)VarName; \
487097a140dSpatrick PointerType *VarName##Ptr = OMPBuilder.VarName##Ptr; \
488097a140dSpatrick (void)VarName##Ptr;
489097a140dSpatrick
490097a140dSpatrick #define OMP_STRUCT_TYPE(VarName, ...) \
491097a140dSpatrick StructType *VarName = OMPBuilder.VarName; \
492097a140dSpatrick (void)VarName; \
493097a140dSpatrick PointerType *VarName##Ptr = OMPBuilder.VarName##Ptr; \
494097a140dSpatrick (void)VarName##Ptr;
495097a140dSpatrick
496097a140dSpatrick #define OMP_RTL(_Enum, _Name, _IsVarArg, _ReturnType, ...) \
497097a140dSpatrick { \
498097a140dSpatrick SmallVector<Type *, 8> ArgsTypes({__VA_ARGS__}); \
499097a140dSpatrick Function *F = M.getFunction(_Name); \
50073471bf0Spatrick RTLFunctions.insert(F); \
501097a140dSpatrick if (declMatchesRTFTypes(F, OMPBuilder._ReturnType, ArgsTypes)) { \
50273471bf0Spatrick RuntimeFunctionIDMap[F] = _Enum; \
503097a140dSpatrick auto &RFI = RFIs[_Enum]; \
504097a140dSpatrick RFI.Kind = _Enum; \
505097a140dSpatrick RFI.Name = _Name; \
506097a140dSpatrick RFI.IsVarArg = _IsVarArg; \
507097a140dSpatrick RFI.ReturnType = OMPBuilder._ReturnType; \
508097a140dSpatrick RFI.ArgumentTypes = std::move(ArgsTypes); \
509097a140dSpatrick RFI.Declaration = F; \
510097a140dSpatrick unsigned NumUses = collectUses(RFI); \
511097a140dSpatrick (void)NumUses; \
512097a140dSpatrick LLVM_DEBUG({ \
513097a140dSpatrick dbgs() << TAG << RFI.Name << (RFI.Declaration ? "" : " not") \
514097a140dSpatrick << " found\n"; \
515097a140dSpatrick if (RFI.Declaration) \
516097a140dSpatrick dbgs() << TAG << "-> got " << NumUses << " uses in " \
517097a140dSpatrick << RFI.getNumFunctionsWithUses() \
518097a140dSpatrick << " different functions.\n"; \
519097a140dSpatrick }); \
520097a140dSpatrick } \
521097a140dSpatrick }
522097a140dSpatrick #include "llvm/Frontend/OpenMP/OMPKinds.def"
523097a140dSpatrick
524*d415bd75Srobert // Remove the `noinline` attribute from `__kmpc`, `ompx::` and `omp_`
525*d415bd75Srobert // functions, except if `optnone` is present.
526*d415bd75Srobert if (isOpenMPDevice(M)) {
527*d415bd75Srobert for (Function &F : M) {
528*d415bd75Srobert for (StringRef Prefix : {"__kmpc", "_ZN4ompx", "omp_"})
529*d415bd75Srobert if (F.hasFnAttribute(Attribute::NoInline) &&
530*d415bd75Srobert F.getName().startswith(Prefix) &&
531*d415bd75Srobert !F.hasFnAttribute(Attribute::OptimizeNone))
532*d415bd75Srobert F.removeFnAttr(Attribute::NoInline);
533*d415bd75Srobert }
534*d415bd75Srobert }
535*d415bd75Srobert
536097a140dSpatrick // TODO: We should attach the attributes defined in OMPKinds.def.
537097a140dSpatrick }
538097a140dSpatrick
539097a140dSpatrick /// Collection of known kernels (\see Kernel) in the module.
540*d415bd75Srobert KernelSet &Kernels;
54173471bf0Spatrick
54273471bf0Spatrick /// Collection of known OpenMP runtime functions..
54373471bf0Spatrick DenseSet<const Function *> RTLFunctions;
544*d415bd75Srobert
545*d415bd75Srobert /// Indicates if we have already linked in the OpenMP device library.
546*d415bd75Srobert bool OpenMPPostLink = false;
54773471bf0Spatrick };
54873471bf0Spatrick
54973471bf0Spatrick template <typename Ty, bool InsertInvalidates = true>
55073471bf0Spatrick struct BooleanStateWithSetVector : public BooleanState {
contains__anon4b47cb5e0111::BooleanStateWithSetVector55173471bf0Spatrick bool contains(const Ty &Elem) const { return Set.contains(Elem); }
insert__anon4b47cb5e0111::BooleanStateWithSetVector55273471bf0Spatrick bool insert(const Ty &Elem) {
55373471bf0Spatrick if (InsertInvalidates)
55473471bf0Spatrick BooleanState::indicatePessimisticFixpoint();
55573471bf0Spatrick return Set.insert(Elem);
55673471bf0Spatrick }
55773471bf0Spatrick
operator []__anon4b47cb5e0111::BooleanStateWithSetVector55873471bf0Spatrick const Ty &operator[](int Idx) const { return Set[Idx]; }
operator ==__anon4b47cb5e0111::BooleanStateWithSetVector55973471bf0Spatrick bool operator==(const BooleanStateWithSetVector &RHS) const {
56073471bf0Spatrick return BooleanState::operator==(RHS) && Set == RHS.Set;
56173471bf0Spatrick }
operator !=__anon4b47cb5e0111::BooleanStateWithSetVector56273471bf0Spatrick bool operator!=(const BooleanStateWithSetVector &RHS) const {
56373471bf0Spatrick return !(*this == RHS);
56473471bf0Spatrick }
56573471bf0Spatrick
empty__anon4b47cb5e0111::BooleanStateWithSetVector56673471bf0Spatrick bool empty() const { return Set.empty(); }
size__anon4b47cb5e0111::BooleanStateWithSetVector56773471bf0Spatrick size_t size() const { return Set.size(); }
56873471bf0Spatrick
56973471bf0Spatrick /// "Clamp" this state with \p RHS.
operator ^=__anon4b47cb5e0111::BooleanStateWithSetVector57073471bf0Spatrick BooleanStateWithSetVector &operator^=(const BooleanStateWithSetVector &RHS) {
57173471bf0Spatrick BooleanState::operator^=(RHS);
57273471bf0Spatrick Set.insert(RHS.Set.begin(), RHS.Set.end());
57373471bf0Spatrick return *this;
57473471bf0Spatrick }
57573471bf0Spatrick
57673471bf0Spatrick private:
57773471bf0Spatrick /// A set to keep track of elements.
57873471bf0Spatrick SetVector<Ty> Set;
57973471bf0Spatrick
58073471bf0Spatrick public:
begin__anon4b47cb5e0111::BooleanStateWithSetVector58173471bf0Spatrick typename decltype(Set)::iterator begin() { return Set.begin(); }
end__anon4b47cb5e0111::BooleanStateWithSetVector58273471bf0Spatrick typename decltype(Set)::iterator end() { return Set.end(); }
begin__anon4b47cb5e0111::BooleanStateWithSetVector58373471bf0Spatrick typename decltype(Set)::const_iterator begin() const { return Set.begin(); }
end__anon4b47cb5e0111::BooleanStateWithSetVector58473471bf0Spatrick typename decltype(Set)::const_iterator end() const { return Set.end(); }
58573471bf0Spatrick };
58673471bf0Spatrick
58773471bf0Spatrick template <typename Ty, bool InsertInvalidates = true>
58873471bf0Spatrick using BooleanStateWithPtrSetVector =
58973471bf0Spatrick BooleanStateWithSetVector<Ty *, InsertInvalidates>;
59073471bf0Spatrick
59173471bf0Spatrick struct KernelInfoState : AbstractState {
59273471bf0Spatrick /// Flag to track if we reached a fixpoint.
59373471bf0Spatrick bool IsAtFixpoint = false;
59473471bf0Spatrick
59573471bf0Spatrick /// The parallel regions (identified by the outlined parallel functions) that
59673471bf0Spatrick /// can be reached from the associated function.
59773471bf0Spatrick BooleanStateWithPtrSetVector<Function, /* InsertInvalidates */ false>
59873471bf0Spatrick ReachedKnownParallelRegions;
59973471bf0Spatrick
60073471bf0Spatrick /// State to track what parallel region we might reach.
60173471bf0Spatrick BooleanStateWithPtrSetVector<CallBase> ReachedUnknownParallelRegions;
60273471bf0Spatrick
60373471bf0Spatrick /// State to track if we are in SPMD-mode, assumed or know, and why we decided
60473471bf0Spatrick /// we cannot be. If it is assumed, then RequiresFullRuntime should also be
60573471bf0Spatrick /// false.
606*d415bd75Srobert BooleanStateWithPtrSetVector<Instruction, false> SPMDCompatibilityTracker;
60773471bf0Spatrick
60873471bf0Spatrick /// The __kmpc_target_init call in this kernel, if any. If we find more than
60973471bf0Spatrick /// one we abort as the kernel is malformed.
61073471bf0Spatrick CallBase *KernelInitCB = nullptr;
61173471bf0Spatrick
61273471bf0Spatrick /// The __kmpc_target_deinit call in this kernel, if any. If we find more than
61373471bf0Spatrick /// one we abort as the kernel is malformed.
61473471bf0Spatrick CallBase *KernelDeinitCB = nullptr;
61573471bf0Spatrick
61673471bf0Spatrick /// Flag to indicate if the associated function is a kernel entry.
61773471bf0Spatrick bool IsKernelEntry = false;
61873471bf0Spatrick
61973471bf0Spatrick /// State to track what kernel entries can reach the associated function.
62073471bf0Spatrick BooleanStateWithPtrSetVector<Function, false> ReachingKernelEntries;
62173471bf0Spatrick
62273471bf0Spatrick /// State to indicate if we can track parallel level of the associated
62373471bf0Spatrick /// function. We will give up tracking if we encounter unknown caller or the
62473471bf0Spatrick /// caller is __kmpc_parallel_51.
62573471bf0Spatrick BooleanStateWithSetVector<uint8_t> ParallelLevels;
62673471bf0Spatrick
627*d415bd75Srobert /// Flag that indicates if the kernel has nested Parallelism
628*d415bd75Srobert bool NestedParallelism = false;
629*d415bd75Srobert
63073471bf0Spatrick /// Abstract State interface
63173471bf0Spatrick ///{
63273471bf0Spatrick
633*d415bd75Srobert KernelInfoState() = default;
KernelInfoState__anon4b47cb5e0111::KernelInfoState63473471bf0Spatrick KernelInfoState(bool BestState) {
63573471bf0Spatrick if (!BestState)
63673471bf0Spatrick indicatePessimisticFixpoint();
63773471bf0Spatrick }
63873471bf0Spatrick
63973471bf0Spatrick /// See AbstractState::isValidState(...)
isValidState__anon4b47cb5e0111::KernelInfoState64073471bf0Spatrick bool isValidState() const override { return true; }
64173471bf0Spatrick
64273471bf0Spatrick /// See AbstractState::isAtFixpoint(...)
isAtFixpoint__anon4b47cb5e0111::KernelInfoState64373471bf0Spatrick bool isAtFixpoint() const override { return IsAtFixpoint; }
64473471bf0Spatrick
64573471bf0Spatrick /// See AbstractState::indicatePessimisticFixpoint(...)
indicatePessimisticFixpoint__anon4b47cb5e0111::KernelInfoState64673471bf0Spatrick ChangeStatus indicatePessimisticFixpoint() override {
64773471bf0Spatrick IsAtFixpoint = true;
648*d415bd75Srobert ParallelLevels.indicatePessimisticFixpoint();
649*d415bd75Srobert ReachingKernelEntries.indicatePessimisticFixpoint();
65073471bf0Spatrick SPMDCompatibilityTracker.indicatePessimisticFixpoint();
651*d415bd75Srobert ReachedKnownParallelRegions.indicatePessimisticFixpoint();
65273471bf0Spatrick ReachedUnknownParallelRegions.indicatePessimisticFixpoint();
65373471bf0Spatrick return ChangeStatus::CHANGED;
65473471bf0Spatrick }
65573471bf0Spatrick
65673471bf0Spatrick /// See AbstractState::indicateOptimisticFixpoint(...)
indicateOptimisticFixpoint__anon4b47cb5e0111::KernelInfoState65773471bf0Spatrick ChangeStatus indicateOptimisticFixpoint() override {
65873471bf0Spatrick IsAtFixpoint = true;
659*d415bd75Srobert ParallelLevels.indicateOptimisticFixpoint();
660*d415bd75Srobert ReachingKernelEntries.indicateOptimisticFixpoint();
661*d415bd75Srobert SPMDCompatibilityTracker.indicateOptimisticFixpoint();
662*d415bd75Srobert ReachedKnownParallelRegions.indicateOptimisticFixpoint();
663*d415bd75Srobert ReachedUnknownParallelRegions.indicateOptimisticFixpoint();
66473471bf0Spatrick return ChangeStatus::UNCHANGED;
66573471bf0Spatrick }
66673471bf0Spatrick
66773471bf0Spatrick /// Return the assumed state
getAssumed__anon4b47cb5e0111::KernelInfoState66873471bf0Spatrick KernelInfoState &getAssumed() { return *this; }
getAssumed__anon4b47cb5e0111::KernelInfoState66973471bf0Spatrick const KernelInfoState &getAssumed() const { return *this; }
67073471bf0Spatrick
operator ==__anon4b47cb5e0111::KernelInfoState67173471bf0Spatrick bool operator==(const KernelInfoState &RHS) const {
67273471bf0Spatrick if (SPMDCompatibilityTracker != RHS.SPMDCompatibilityTracker)
67373471bf0Spatrick return false;
67473471bf0Spatrick if (ReachedKnownParallelRegions != RHS.ReachedKnownParallelRegions)
67573471bf0Spatrick return false;
67673471bf0Spatrick if (ReachedUnknownParallelRegions != RHS.ReachedUnknownParallelRegions)
67773471bf0Spatrick return false;
67873471bf0Spatrick if (ReachingKernelEntries != RHS.ReachingKernelEntries)
67973471bf0Spatrick return false;
680*d415bd75Srobert if (ParallelLevels != RHS.ParallelLevels)
681*d415bd75Srobert return false;
68273471bf0Spatrick return true;
68373471bf0Spatrick }
68473471bf0Spatrick
685*d415bd75Srobert /// Returns true if this kernel contains any OpenMP parallel regions.
mayContainParallelRegion__anon4b47cb5e0111::KernelInfoState686*d415bd75Srobert bool mayContainParallelRegion() {
687*d415bd75Srobert return !ReachedKnownParallelRegions.empty() ||
688*d415bd75Srobert !ReachedUnknownParallelRegions.empty();
689*d415bd75Srobert }
690*d415bd75Srobert
69173471bf0Spatrick /// Return empty set as the best state of potential values.
getBestState__anon4b47cb5e0111::KernelInfoState69273471bf0Spatrick static KernelInfoState getBestState() { return KernelInfoState(true); }
69373471bf0Spatrick
getBestState__anon4b47cb5e0111::KernelInfoState69473471bf0Spatrick static KernelInfoState getBestState(KernelInfoState &KIS) {
69573471bf0Spatrick return getBestState();
69673471bf0Spatrick }
69773471bf0Spatrick
69873471bf0Spatrick /// Return full set as the worst state of potential values.
getWorstState__anon4b47cb5e0111::KernelInfoState69973471bf0Spatrick static KernelInfoState getWorstState() { return KernelInfoState(false); }
70073471bf0Spatrick
70173471bf0Spatrick /// "Clamp" this state with \p KIS.
operator ^=__anon4b47cb5e0111::KernelInfoState70273471bf0Spatrick KernelInfoState operator^=(const KernelInfoState &KIS) {
70373471bf0Spatrick // Do not merge two different _init and _deinit call sites.
70473471bf0Spatrick if (KIS.KernelInitCB) {
70573471bf0Spatrick if (KernelInitCB && KernelInitCB != KIS.KernelInitCB)
706*d415bd75Srobert llvm_unreachable("Kernel that calls another kernel violates OpenMP-Opt "
707*d415bd75Srobert "assumptions.");
70873471bf0Spatrick KernelInitCB = KIS.KernelInitCB;
70973471bf0Spatrick }
71073471bf0Spatrick if (KIS.KernelDeinitCB) {
71173471bf0Spatrick if (KernelDeinitCB && KernelDeinitCB != KIS.KernelDeinitCB)
712*d415bd75Srobert llvm_unreachable("Kernel that calls another kernel violates OpenMP-Opt "
713*d415bd75Srobert "assumptions.");
71473471bf0Spatrick KernelDeinitCB = KIS.KernelDeinitCB;
71573471bf0Spatrick }
71673471bf0Spatrick SPMDCompatibilityTracker ^= KIS.SPMDCompatibilityTracker;
71773471bf0Spatrick ReachedKnownParallelRegions ^= KIS.ReachedKnownParallelRegions;
71873471bf0Spatrick ReachedUnknownParallelRegions ^= KIS.ReachedUnknownParallelRegions;
719*d415bd75Srobert NestedParallelism |= KIS.NestedParallelism;
72073471bf0Spatrick return *this;
72173471bf0Spatrick }
72273471bf0Spatrick
operator &=__anon4b47cb5e0111::KernelInfoState72373471bf0Spatrick KernelInfoState operator&=(const KernelInfoState &KIS) {
72473471bf0Spatrick return (*this ^= KIS);
72573471bf0Spatrick }
72673471bf0Spatrick
72773471bf0Spatrick ///}
72873471bf0Spatrick };
72973471bf0Spatrick
73073471bf0Spatrick /// Used to map the values physically (in the IR) stored in an offload
73173471bf0Spatrick /// array, to a vector in memory.
73273471bf0Spatrick struct OffloadArray {
73373471bf0Spatrick /// Physical array (in the IR).
73473471bf0Spatrick AllocaInst *Array = nullptr;
73573471bf0Spatrick /// Mapped values.
73673471bf0Spatrick SmallVector<Value *, 8> StoredValues;
73773471bf0Spatrick /// Last stores made in the offload array.
73873471bf0Spatrick SmallVector<StoreInst *, 8> LastAccesses;
73973471bf0Spatrick
74073471bf0Spatrick OffloadArray() = default;
74173471bf0Spatrick
74273471bf0Spatrick /// Initializes the OffloadArray with the values stored in \p Array before
74373471bf0Spatrick /// instruction \p Before is reached. Returns false if the initialization
74473471bf0Spatrick /// fails.
74573471bf0Spatrick /// This MUST be used immediately after the construction of the object.
initialize__anon4b47cb5e0111::OffloadArray74673471bf0Spatrick bool initialize(AllocaInst &Array, Instruction &Before) {
74773471bf0Spatrick if (!Array.getAllocatedType()->isArrayTy())
74873471bf0Spatrick return false;
74973471bf0Spatrick
75073471bf0Spatrick if (!getValues(Array, Before))
75173471bf0Spatrick return false;
75273471bf0Spatrick
75373471bf0Spatrick this->Array = &Array;
75473471bf0Spatrick return true;
75573471bf0Spatrick }
75673471bf0Spatrick
75773471bf0Spatrick static const unsigned DeviceIDArgNum = 1;
75873471bf0Spatrick static const unsigned BasePtrsArgNum = 3;
75973471bf0Spatrick static const unsigned PtrsArgNum = 4;
76073471bf0Spatrick static const unsigned SizesArgNum = 5;
76173471bf0Spatrick
76273471bf0Spatrick private:
76373471bf0Spatrick /// Traverses the BasicBlock where \p Array is, collecting the stores made to
76473471bf0Spatrick /// \p Array, leaving StoredValues with the values stored before the
76573471bf0Spatrick /// instruction \p Before is reached.
getValues__anon4b47cb5e0111::OffloadArray76673471bf0Spatrick bool getValues(AllocaInst &Array, Instruction &Before) {
76773471bf0Spatrick // Initialize container.
76873471bf0Spatrick const uint64_t NumValues = Array.getAllocatedType()->getArrayNumElements();
76973471bf0Spatrick StoredValues.assign(NumValues, nullptr);
77073471bf0Spatrick LastAccesses.assign(NumValues, nullptr);
77173471bf0Spatrick
77273471bf0Spatrick // TODO: This assumes the instruction \p Before is in the same
77373471bf0Spatrick // BasicBlock as Array. Make it general, for any control flow graph.
77473471bf0Spatrick BasicBlock *BB = Array.getParent();
77573471bf0Spatrick if (BB != Before.getParent())
77673471bf0Spatrick return false;
77773471bf0Spatrick
77873471bf0Spatrick const DataLayout &DL = Array.getModule()->getDataLayout();
77973471bf0Spatrick const unsigned int PointerSize = DL.getPointerSize();
78073471bf0Spatrick
78173471bf0Spatrick for (Instruction &I : *BB) {
78273471bf0Spatrick if (&I == &Before)
78373471bf0Spatrick break;
78473471bf0Spatrick
78573471bf0Spatrick if (!isa<StoreInst>(&I))
78673471bf0Spatrick continue;
78773471bf0Spatrick
78873471bf0Spatrick auto *S = cast<StoreInst>(&I);
78973471bf0Spatrick int64_t Offset = -1;
79073471bf0Spatrick auto *Dst =
79173471bf0Spatrick GetPointerBaseWithConstantOffset(S->getPointerOperand(), Offset, DL);
79273471bf0Spatrick if (Dst == &Array) {
79373471bf0Spatrick int64_t Idx = Offset / PointerSize;
79473471bf0Spatrick StoredValues[Idx] = getUnderlyingObject(S->getValueOperand());
79573471bf0Spatrick LastAccesses[Idx] = S;
79673471bf0Spatrick }
79773471bf0Spatrick }
79873471bf0Spatrick
79973471bf0Spatrick return isFilled();
80073471bf0Spatrick }
80173471bf0Spatrick
80273471bf0Spatrick /// Returns true if all values in StoredValues and
80373471bf0Spatrick /// LastAccesses are not nullptrs.
isFilled__anon4b47cb5e0111::OffloadArray80473471bf0Spatrick bool isFilled() {
80573471bf0Spatrick const unsigned NumValues = StoredValues.size();
80673471bf0Spatrick for (unsigned I = 0; I < NumValues; ++I) {
80773471bf0Spatrick if (!StoredValues[I] || !LastAccesses[I])
80873471bf0Spatrick return false;
80973471bf0Spatrick }
81073471bf0Spatrick
81173471bf0Spatrick return true;
81273471bf0Spatrick }
813097a140dSpatrick };
814097a140dSpatrick
815097a140dSpatrick struct OpenMPOpt {
816097a140dSpatrick
817097a140dSpatrick using OptimizationRemarkGetter =
818097a140dSpatrick function_ref<OptimizationRemarkEmitter &(Function *)>;
819097a140dSpatrick
OpenMPOpt__anon4b47cb5e0111::OpenMPOpt820097a140dSpatrick OpenMPOpt(SmallVectorImpl<Function *> &SCC, CallGraphUpdater &CGUpdater,
821097a140dSpatrick OptimizationRemarkGetter OREGetter,
822097a140dSpatrick OMPInformationCache &OMPInfoCache, Attributor &A)
823097a140dSpatrick : M(*(*SCC.begin())->getParent()), SCC(SCC), CGUpdater(CGUpdater),
824097a140dSpatrick OREGetter(OREGetter), OMPInfoCache(OMPInfoCache), A(A) {}
825097a140dSpatrick
82673471bf0Spatrick /// Check if any remarks are enabled for openmp-opt
remarksEnabled__anon4b47cb5e0111::OpenMPOpt82773471bf0Spatrick bool remarksEnabled() {
82873471bf0Spatrick auto &Ctx = M.getContext();
82973471bf0Spatrick return Ctx.getDiagHandlerPtr()->isAnyRemarkEnabled(DEBUG_TYPE);
83073471bf0Spatrick }
83173471bf0Spatrick
832097a140dSpatrick /// Run all OpenMP optimizations on the underlying SCC/ModuleSlice.
run__anon4b47cb5e0111::OpenMPOpt83373471bf0Spatrick bool run(bool IsModulePass) {
834097a140dSpatrick if (SCC.empty())
835097a140dSpatrick return false;
836097a140dSpatrick
837097a140dSpatrick bool Changed = false;
838097a140dSpatrick
839097a140dSpatrick LLVM_DEBUG(dbgs() << TAG << "Run on SCC with " << SCC.size()
840097a140dSpatrick << " functions in a slice with "
841097a140dSpatrick << OMPInfoCache.ModuleSlice.size() << " functions\n");
842097a140dSpatrick
84373471bf0Spatrick if (IsModulePass) {
84473471bf0Spatrick Changed |= runAttributor(IsModulePass);
84573471bf0Spatrick
84673471bf0Spatrick // Recollect uses, in case Attributor deleted any.
84773471bf0Spatrick OMPInfoCache.recollectUses();
84873471bf0Spatrick
84973471bf0Spatrick // TODO: This should be folded into buildCustomStateMachine.
85073471bf0Spatrick Changed |= rewriteDeviceCodeStateMachine();
85173471bf0Spatrick
85273471bf0Spatrick if (remarksEnabled())
85373471bf0Spatrick analysisGlobalization();
85473471bf0Spatrick } else {
855097a140dSpatrick if (PrintICVValues)
856097a140dSpatrick printICVs();
857097a140dSpatrick if (PrintOpenMPKernels)
858097a140dSpatrick printKernels();
859097a140dSpatrick
86073471bf0Spatrick Changed |= runAttributor(IsModulePass);
861097a140dSpatrick
862097a140dSpatrick // Recollect uses, in case Attributor deleted any.
863097a140dSpatrick OMPInfoCache.recollectUses();
864097a140dSpatrick
865097a140dSpatrick Changed |= deleteParallelRegions();
866097a140dSpatrick
86773471bf0Spatrick if (HideMemoryTransferLatency)
86873471bf0Spatrick Changed |= hideMemTransfersLatency();
86973471bf0Spatrick Changed |= deduplicateRuntimeCalls();
87073471bf0Spatrick if (EnableParallelRegionMerging) {
87173471bf0Spatrick if (mergeParallelRegions()) {
87273471bf0Spatrick deduplicateRuntimeCalls();
87373471bf0Spatrick Changed = true;
87473471bf0Spatrick }
87573471bf0Spatrick }
87673471bf0Spatrick }
87773471bf0Spatrick
878097a140dSpatrick return Changed;
879097a140dSpatrick }
880097a140dSpatrick
881097a140dSpatrick /// Print initial ICV values for testing.
882097a140dSpatrick /// FIXME: This should be done from the Attributor once it is added.
printICVs__anon4b47cb5e0111::OpenMPOpt883097a140dSpatrick void printICVs() const {
88473471bf0Spatrick InternalControlVar ICVs[] = {ICV_nthreads, ICV_active_levels, ICV_cancel,
88573471bf0Spatrick ICV_proc_bind};
886097a140dSpatrick
887*d415bd75Srobert for (Function *F : SCC) {
888097a140dSpatrick for (auto ICV : ICVs) {
889097a140dSpatrick auto ICVInfo = OMPInfoCache.ICVs[ICV];
89073471bf0Spatrick auto Remark = [&](OptimizationRemarkAnalysis ORA) {
89173471bf0Spatrick return ORA << "OpenMP ICV " << ore::NV("OpenMPICV", ICVInfo.Name)
892097a140dSpatrick << " Value: "
893097a140dSpatrick << (ICVInfo.InitValue
89473471bf0Spatrick ? toString(ICVInfo.InitValue->getValue(), 10, true)
895097a140dSpatrick : "IMPLEMENTATION_DEFINED");
896097a140dSpatrick };
897097a140dSpatrick
89873471bf0Spatrick emitRemark<OptimizationRemarkAnalysis>(F, "OpenMPICVTracker", Remark);
899097a140dSpatrick }
900097a140dSpatrick }
901097a140dSpatrick }
902097a140dSpatrick
903097a140dSpatrick /// Print OpenMP GPU kernels for testing.
printKernels__anon4b47cb5e0111::OpenMPOpt904097a140dSpatrick void printKernels() const {
905097a140dSpatrick for (Function *F : SCC) {
906097a140dSpatrick if (!OMPInfoCache.Kernels.count(F))
907097a140dSpatrick continue;
908097a140dSpatrick
90973471bf0Spatrick auto Remark = [&](OptimizationRemarkAnalysis ORA) {
91073471bf0Spatrick return ORA << "OpenMP GPU kernel "
911097a140dSpatrick << ore::NV("OpenMPGPUKernel", F->getName()) << "\n";
912097a140dSpatrick };
913097a140dSpatrick
91473471bf0Spatrick emitRemark<OptimizationRemarkAnalysis>(F, "OpenMPGPU", Remark);
915097a140dSpatrick }
916097a140dSpatrick }
917097a140dSpatrick
918097a140dSpatrick /// Return the call if \p U is a callee use in a regular call. If \p RFI is
919097a140dSpatrick /// given it has to be the callee or a nullptr is returned.
getCallIfRegularCall__anon4b47cb5e0111::OpenMPOpt920097a140dSpatrick static CallInst *getCallIfRegularCall(
921097a140dSpatrick Use &U, OMPInformationCache::RuntimeFunctionInfo *RFI = nullptr) {
922097a140dSpatrick CallInst *CI = dyn_cast<CallInst>(U.getUser());
923097a140dSpatrick if (CI && CI->isCallee(&U) && !CI->hasOperandBundles() &&
92473471bf0Spatrick (!RFI ||
92573471bf0Spatrick (RFI->Declaration && CI->getCalledFunction() == RFI->Declaration)))
926097a140dSpatrick return CI;
927097a140dSpatrick return nullptr;
928097a140dSpatrick }
929097a140dSpatrick
930097a140dSpatrick /// Return the call if \p V is a regular call. If \p RFI is given it has to be
931097a140dSpatrick /// the callee or a nullptr is returned.
getCallIfRegularCall__anon4b47cb5e0111::OpenMPOpt932097a140dSpatrick static CallInst *getCallIfRegularCall(
933097a140dSpatrick Value &V, OMPInformationCache::RuntimeFunctionInfo *RFI = nullptr) {
934097a140dSpatrick CallInst *CI = dyn_cast<CallInst>(&V);
935097a140dSpatrick if (CI && !CI->hasOperandBundles() &&
93673471bf0Spatrick (!RFI ||
93773471bf0Spatrick (RFI->Declaration && CI->getCalledFunction() == RFI->Declaration)))
938097a140dSpatrick return CI;
939097a140dSpatrick return nullptr;
940097a140dSpatrick }
941097a140dSpatrick
942097a140dSpatrick private:
94373471bf0Spatrick /// Merge parallel regions when it is safe.
mergeParallelRegions__anon4b47cb5e0111::OpenMPOpt94473471bf0Spatrick bool mergeParallelRegions() {
94573471bf0Spatrick const unsigned CallbackCalleeOperand = 2;
94673471bf0Spatrick const unsigned CallbackFirstArgOperand = 3;
94773471bf0Spatrick using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
94873471bf0Spatrick
94973471bf0Spatrick // Check if there are any __kmpc_fork_call calls to merge.
95073471bf0Spatrick OMPInformationCache::RuntimeFunctionInfo &RFI =
95173471bf0Spatrick OMPInfoCache.RFIs[OMPRTL___kmpc_fork_call];
95273471bf0Spatrick
95373471bf0Spatrick if (!RFI.Declaration)
95473471bf0Spatrick return false;
95573471bf0Spatrick
95673471bf0Spatrick // Unmergable calls that prevent merging a parallel region.
95773471bf0Spatrick OMPInformationCache::RuntimeFunctionInfo UnmergableCallsInfo[] = {
95873471bf0Spatrick OMPInfoCache.RFIs[OMPRTL___kmpc_push_proc_bind],
95973471bf0Spatrick OMPInfoCache.RFIs[OMPRTL___kmpc_push_num_threads],
96073471bf0Spatrick };
96173471bf0Spatrick
96273471bf0Spatrick bool Changed = false;
96373471bf0Spatrick LoopInfo *LI = nullptr;
96473471bf0Spatrick DominatorTree *DT = nullptr;
96573471bf0Spatrick
96673471bf0Spatrick SmallDenseMap<BasicBlock *, SmallPtrSet<Instruction *, 4>> BB2PRMap;
96773471bf0Spatrick
96873471bf0Spatrick BasicBlock *StartBB = nullptr, *EndBB = nullptr;
969*d415bd75Srobert auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
97073471bf0Spatrick BasicBlock *CGStartBB = CodeGenIP.getBlock();
97173471bf0Spatrick BasicBlock *CGEndBB =
97273471bf0Spatrick SplitBlock(CGStartBB, &*CodeGenIP.getPoint(), DT, LI);
97373471bf0Spatrick assert(StartBB != nullptr && "StartBB should not be null");
97473471bf0Spatrick CGStartBB->getTerminator()->setSuccessor(0, StartBB);
97573471bf0Spatrick assert(EndBB != nullptr && "EndBB should not be null");
97673471bf0Spatrick EndBB->getTerminator()->setSuccessor(0, CGEndBB);
97773471bf0Spatrick };
97873471bf0Spatrick
97973471bf0Spatrick auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
98073471bf0Spatrick Value &Inner, Value *&ReplacementValue) -> InsertPointTy {
98173471bf0Spatrick ReplacementValue = &Inner;
98273471bf0Spatrick return CodeGenIP;
98373471bf0Spatrick };
98473471bf0Spatrick
98573471bf0Spatrick auto FiniCB = [&](InsertPointTy CodeGenIP) {};
98673471bf0Spatrick
98773471bf0Spatrick /// Create a sequential execution region within a merged parallel region,
98873471bf0Spatrick /// encapsulated in a master construct with a barrier for synchronization.
98973471bf0Spatrick auto CreateSequentialRegion = [&](Function *OuterFn,
99073471bf0Spatrick BasicBlock *OuterPredBB,
99173471bf0Spatrick Instruction *SeqStartI,
99273471bf0Spatrick Instruction *SeqEndI) {
99373471bf0Spatrick // Isolate the instructions of the sequential region to a separate
99473471bf0Spatrick // block.
99573471bf0Spatrick BasicBlock *ParentBB = SeqStartI->getParent();
99673471bf0Spatrick BasicBlock *SeqEndBB =
99773471bf0Spatrick SplitBlock(ParentBB, SeqEndI->getNextNode(), DT, LI);
99873471bf0Spatrick BasicBlock *SeqAfterBB =
99973471bf0Spatrick SplitBlock(SeqEndBB, &*SeqEndBB->getFirstInsertionPt(), DT, LI);
100073471bf0Spatrick BasicBlock *SeqStartBB =
100173471bf0Spatrick SplitBlock(ParentBB, SeqStartI, DT, LI, nullptr, "seq.par.merged");
100273471bf0Spatrick
100373471bf0Spatrick assert(ParentBB->getUniqueSuccessor() == SeqStartBB &&
100473471bf0Spatrick "Expected a different CFG");
100573471bf0Spatrick const DebugLoc DL = ParentBB->getTerminator()->getDebugLoc();
100673471bf0Spatrick ParentBB->getTerminator()->eraseFromParent();
100773471bf0Spatrick
1008*d415bd75Srobert auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
100973471bf0Spatrick BasicBlock *CGStartBB = CodeGenIP.getBlock();
101073471bf0Spatrick BasicBlock *CGEndBB =
101173471bf0Spatrick SplitBlock(CGStartBB, &*CodeGenIP.getPoint(), DT, LI);
101273471bf0Spatrick assert(SeqStartBB != nullptr && "SeqStartBB should not be null");
101373471bf0Spatrick CGStartBB->getTerminator()->setSuccessor(0, SeqStartBB);
101473471bf0Spatrick assert(SeqEndBB != nullptr && "SeqEndBB should not be null");
101573471bf0Spatrick SeqEndBB->getTerminator()->setSuccessor(0, CGEndBB);
101673471bf0Spatrick };
101773471bf0Spatrick auto FiniCB = [&](InsertPointTy CodeGenIP) {};
101873471bf0Spatrick
101973471bf0Spatrick // Find outputs from the sequential region to outside users and
102073471bf0Spatrick // broadcast their values to them.
102173471bf0Spatrick for (Instruction &I : *SeqStartBB) {
102273471bf0Spatrick SmallPtrSet<Instruction *, 4> OutsideUsers;
102373471bf0Spatrick for (User *Usr : I.users()) {
102473471bf0Spatrick Instruction &UsrI = *cast<Instruction>(Usr);
102573471bf0Spatrick // Ignore outputs to LT intrinsics, code extraction for the merged
102673471bf0Spatrick // parallel region will fix them.
102773471bf0Spatrick if (UsrI.isLifetimeStartOrEnd())
102873471bf0Spatrick continue;
102973471bf0Spatrick
103073471bf0Spatrick if (UsrI.getParent() != SeqStartBB)
103173471bf0Spatrick OutsideUsers.insert(&UsrI);
103273471bf0Spatrick }
103373471bf0Spatrick
103473471bf0Spatrick if (OutsideUsers.empty())
103573471bf0Spatrick continue;
103673471bf0Spatrick
103773471bf0Spatrick // Emit an alloca in the outer region to store the broadcasted
103873471bf0Spatrick // value.
103973471bf0Spatrick const DataLayout &DL = M.getDataLayout();
104073471bf0Spatrick AllocaInst *AllocaI = new AllocaInst(
104173471bf0Spatrick I.getType(), DL.getAllocaAddrSpace(), nullptr,
104273471bf0Spatrick I.getName() + ".seq.output.alloc", &OuterFn->front().front());
104373471bf0Spatrick
104473471bf0Spatrick // Emit a store instruction in the sequential BB to update the
104573471bf0Spatrick // value.
104673471bf0Spatrick new StoreInst(&I, AllocaI, SeqStartBB->getTerminator());
104773471bf0Spatrick
104873471bf0Spatrick // Emit a load instruction and replace the use of the output value
104973471bf0Spatrick // with it.
105073471bf0Spatrick for (Instruction *UsrI : OutsideUsers) {
105173471bf0Spatrick LoadInst *LoadI = new LoadInst(
105273471bf0Spatrick I.getType(), AllocaI, I.getName() + ".seq.output.load", UsrI);
105373471bf0Spatrick UsrI->replaceUsesOfWith(&I, LoadI);
105473471bf0Spatrick }
105573471bf0Spatrick }
105673471bf0Spatrick
105773471bf0Spatrick OpenMPIRBuilder::LocationDescription Loc(
105873471bf0Spatrick InsertPointTy(ParentBB, ParentBB->end()), DL);
105973471bf0Spatrick InsertPointTy SeqAfterIP =
106073471bf0Spatrick OMPInfoCache.OMPBuilder.createMaster(Loc, BodyGenCB, FiniCB);
106173471bf0Spatrick
106273471bf0Spatrick OMPInfoCache.OMPBuilder.createBarrier(SeqAfterIP, OMPD_parallel);
106373471bf0Spatrick
106473471bf0Spatrick BranchInst::Create(SeqAfterBB, SeqAfterIP.getBlock());
106573471bf0Spatrick
106673471bf0Spatrick LLVM_DEBUG(dbgs() << TAG << "After sequential inlining " << *OuterFn
106773471bf0Spatrick << "\n");
106873471bf0Spatrick };
106973471bf0Spatrick
107073471bf0Spatrick // Helper to merge the __kmpc_fork_call calls in MergableCIs. They are all
107173471bf0Spatrick // contained in BB and only separated by instructions that can be
107273471bf0Spatrick // redundantly executed in parallel. The block BB is split before the first
107373471bf0Spatrick // call (in MergableCIs) and after the last so the entire region we merge
107473471bf0Spatrick // into a single parallel region is contained in a single basic block
107573471bf0Spatrick // without any other instructions. We use the OpenMPIRBuilder to outline
107673471bf0Spatrick // that block and call the resulting function via __kmpc_fork_call.
1077*d415bd75Srobert auto Merge = [&](const SmallVectorImpl<CallInst *> &MergableCIs,
1078*d415bd75Srobert BasicBlock *BB) {
107973471bf0Spatrick // TODO: Change the interface to allow single CIs expanded, e.g, to
108073471bf0Spatrick // include an outer loop.
108173471bf0Spatrick assert(MergableCIs.size() > 1 && "Assumed multiple mergable CIs");
108273471bf0Spatrick
108373471bf0Spatrick auto Remark = [&](OptimizationRemark OR) {
108473471bf0Spatrick OR << "Parallel region merged with parallel region"
108573471bf0Spatrick << (MergableCIs.size() > 2 ? "s" : "") << " at ";
108673471bf0Spatrick for (auto *CI : llvm::drop_begin(MergableCIs)) {
108773471bf0Spatrick OR << ore::NV("OpenMPParallelMerge", CI->getDebugLoc());
108873471bf0Spatrick if (CI != MergableCIs.back())
108973471bf0Spatrick OR << ", ";
109073471bf0Spatrick }
109173471bf0Spatrick return OR << ".";
109273471bf0Spatrick };
109373471bf0Spatrick
109473471bf0Spatrick emitRemark<OptimizationRemark>(MergableCIs.front(), "OMP150", Remark);
109573471bf0Spatrick
109673471bf0Spatrick Function *OriginalFn = BB->getParent();
109773471bf0Spatrick LLVM_DEBUG(dbgs() << TAG << "Merge " << MergableCIs.size()
109873471bf0Spatrick << " parallel regions in " << OriginalFn->getName()
109973471bf0Spatrick << "\n");
110073471bf0Spatrick
110173471bf0Spatrick // Isolate the calls to merge in a separate block.
110273471bf0Spatrick EndBB = SplitBlock(BB, MergableCIs.back()->getNextNode(), DT, LI);
110373471bf0Spatrick BasicBlock *AfterBB =
110473471bf0Spatrick SplitBlock(EndBB, &*EndBB->getFirstInsertionPt(), DT, LI);
110573471bf0Spatrick StartBB = SplitBlock(BB, MergableCIs.front(), DT, LI, nullptr,
110673471bf0Spatrick "omp.par.merged");
110773471bf0Spatrick
110873471bf0Spatrick assert(BB->getUniqueSuccessor() == StartBB && "Expected a different CFG");
110973471bf0Spatrick const DebugLoc DL = BB->getTerminator()->getDebugLoc();
111073471bf0Spatrick BB->getTerminator()->eraseFromParent();
111173471bf0Spatrick
111273471bf0Spatrick // Create sequential regions for sequential instructions that are
111373471bf0Spatrick // in-between mergable parallel regions.
111473471bf0Spatrick for (auto *It = MergableCIs.begin(), *End = MergableCIs.end() - 1;
111573471bf0Spatrick It != End; ++It) {
111673471bf0Spatrick Instruction *ForkCI = *It;
111773471bf0Spatrick Instruction *NextForkCI = *(It + 1);
111873471bf0Spatrick
111973471bf0Spatrick // Continue if there are not in-between instructions.
112073471bf0Spatrick if (ForkCI->getNextNode() == NextForkCI)
112173471bf0Spatrick continue;
112273471bf0Spatrick
112373471bf0Spatrick CreateSequentialRegion(OriginalFn, BB, ForkCI->getNextNode(),
112473471bf0Spatrick NextForkCI->getPrevNode());
112573471bf0Spatrick }
112673471bf0Spatrick
112773471bf0Spatrick OpenMPIRBuilder::LocationDescription Loc(InsertPointTy(BB, BB->end()),
112873471bf0Spatrick DL);
112973471bf0Spatrick IRBuilder<>::InsertPoint AllocaIP(
113073471bf0Spatrick &OriginalFn->getEntryBlock(),
113173471bf0Spatrick OriginalFn->getEntryBlock().getFirstInsertionPt());
113273471bf0Spatrick // Create the merged parallel region with default proc binding, to
113373471bf0Spatrick // avoid overriding binding settings, and without explicit cancellation.
113473471bf0Spatrick InsertPointTy AfterIP = OMPInfoCache.OMPBuilder.createParallel(
113573471bf0Spatrick Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr,
113673471bf0Spatrick OMP_PROC_BIND_default, /* IsCancellable */ false);
113773471bf0Spatrick BranchInst::Create(AfterBB, AfterIP.getBlock());
113873471bf0Spatrick
113973471bf0Spatrick // Perform the actual outlining.
1140*d415bd75Srobert OMPInfoCache.OMPBuilder.finalize(OriginalFn);
114173471bf0Spatrick
114273471bf0Spatrick Function *OutlinedFn = MergableCIs.front()->getCaller();
114373471bf0Spatrick
114473471bf0Spatrick // Replace the __kmpc_fork_call calls with direct calls to the outlined
114573471bf0Spatrick // callbacks.
114673471bf0Spatrick SmallVector<Value *, 8> Args;
114773471bf0Spatrick for (auto *CI : MergableCIs) {
1148*d415bd75Srobert Value *Callee = CI->getArgOperand(CallbackCalleeOperand);
1149*d415bd75Srobert FunctionType *FT = OMPInfoCache.OMPBuilder.ParallelTask;
115073471bf0Spatrick Args.clear();
115173471bf0Spatrick Args.push_back(OutlinedFn->getArg(0));
115273471bf0Spatrick Args.push_back(OutlinedFn->getArg(1));
1153*d415bd75Srobert for (unsigned U = CallbackFirstArgOperand, E = CI->arg_size(); U < E;
1154*d415bd75Srobert ++U)
115573471bf0Spatrick Args.push_back(CI->getArgOperand(U));
115673471bf0Spatrick
115773471bf0Spatrick CallInst *NewCI = CallInst::Create(FT, Callee, Args, "", CI);
115873471bf0Spatrick if (CI->getDebugLoc())
115973471bf0Spatrick NewCI->setDebugLoc(CI->getDebugLoc());
116073471bf0Spatrick
116173471bf0Spatrick // Forward parameter attributes from the callback to the callee.
1162*d415bd75Srobert for (unsigned U = CallbackFirstArgOperand, E = CI->arg_size(); U < E;
1163*d415bd75Srobert ++U)
1164*d415bd75Srobert for (const Attribute &A : CI->getAttributes().getParamAttrs(U))
116573471bf0Spatrick NewCI->addParamAttr(
116673471bf0Spatrick U - (CallbackFirstArgOperand - CallbackCalleeOperand), A);
116773471bf0Spatrick
116873471bf0Spatrick // Emit an explicit barrier to replace the implicit fork-join barrier.
116973471bf0Spatrick if (CI != MergableCIs.back()) {
117073471bf0Spatrick // TODO: Remove barrier if the merged parallel region includes the
117173471bf0Spatrick // 'nowait' clause.
117273471bf0Spatrick OMPInfoCache.OMPBuilder.createBarrier(
117373471bf0Spatrick InsertPointTy(NewCI->getParent(),
117473471bf0Spatrick NewCI->getNextNode()->getIterator()),
117573471bf0Spatrick OMPD_parallel);
117673471bf0Spatrick }
117773471bf0Spatrick
117873471bf0Spatrick CI->eraseFromParent();
117973471bf0Spatrick }
118073471bf0Spatrick
118173471bf0Spatrick assert(OutlinedFn != OriginalFn && "Outlining failed");
118273471bf0Spatrick CGUpdater.registerOutlinedFunction(*OriginalFn, *OutlinedFn);
118373471bf0Spatrick CGUpdater.reanalyzeFunction(*OriginalFn);
118473471bf0Spatrick
118573471bf0Spatrick NumOpenMPParallelRegionsMerged += MergableCIs.size();
118673471bf0Spatrick
118773471bf0Spatrick return true;
118873471bf0Spatrick };
118973471bf0Spatrick
119073471bf0Spatrick // Helper function that identifes sequences of
119173471bf0Spatrick // __kmpc_fork_call uses in a basic block.
119273471bf0Spatrick auto DetectPRsCB = [&](Use &U, Function &F) {
119373471bf0Spatrick CallInst *CI = getCallIfRegularCall(U, &RFI);
119473471bf0Spatrick BB2PRMap[CI->getParent()].insert(CI);
119573471bf0Spatrick
119673471bf0Spatrick return false;
119773471bf0Spatrick };
119873471bf0Spatrick
119973471bf0Spatrick BB2PRMap.clear();
120073471bf0Spatrick RFI.foreachUse(SCC, DetectPRsCB);
120173471bf0Spatrick SmallVector<SmallVector<CallInst *, 4>, 4> MergableCIsVector;
120273471bf0Spatrick // Find mergable parallel regions within a basic block that are
120373471bf0Spatrick // safe to merge, that is any in-between instructions can safely
120473471bf0Spatrick // execute in parallel after merging.
120573471bf0Spatrick // TODO: support merging across basic-blocks.
120673471bf0Spatrick for (auto &It : BB2PRMap) {
120773471bf0Spatrick auto &CIs = It.getSecond();
120873471bf0Spatrick if (CIs.size() < 2)
120973471bf0Spatrick continue;
121073471bf0Spatrick
121173471bf0Spatrick BasicBlock *BB = It.getFirst();
121273471bf0Spatrick SmallVector<CallInst *, 4> MergableCIs;
121373471bf0Spatrick
121473471bf0Spatrick /// Returns true if the instruction is mergable, false otherwise.
121573471bf0Spatrick /// A terminator instruction is unmergable by definition since merging
121673471bf0Spatrick /// works within a BB. Instructions before the mergable region are
121773471bf0Spatrick /// mergable if they are not calls to OpenMP runtime functions that may
121873471bf0Spatrick /// set different execution parameters for subsequent parallel regions.
121973471bf0Spatrick /// Instructions in-between parallel regions are mergable if they are not
122073471bf0Spatrick /// calls to any non-intrinsic function since that may call a non-mergable
122173471bf0Spatrick /// OpenMP runtime function.
122273471bf0Spatrick auto IsMergable = [&](Instruction &I, bool IsBeforeMergableRegion) {
122373471bf0Spatrick // We do not merge across BBs, hence return false (unmergable) if the
122473471bf0Spatrick // instruction is a terminator.
122573471bf0Spatrick if (I.isTerminator())
122673471bf0Spatrick return false;
122773471bf0Spatrick
122873471bf0Spatrick if (!isa<CallInst>(&I))
122973471bf0Spatrick return true;
123073471bf0Spatrick
123173471bf0Spatrick CallInst *CI = cast<CallInst>(&I);
123273471bf0Spatrick if (IsBeforeMergableRegion) {
123373471bf0Spatrick Function *CalledFunction = CI->getCalledFunction();
123473471bf0Spatrick if (!CalledFunction)
123573471bf0Spatrick return false;
123673471bf0Spatrick // Return false (unmergable) if the call before the parallel
123773471bf0Spatrick // region calls an explicit affinity (proc_bind) or number of
123873471bf0Spatrick // threads (num_threads) compiler-generated function. Those settings
123973471bf0Spatrick // may be incompatible with following parallel regions.
124073471bf0Spatrick // TODO: ICV tracking to detect compatibility.
124173471bf0Spatrick for (const auto &RFI : UnmergableCallsInfo) {
124273471bf0Spatrick if (CalledFunction == RFI.Declaration)
124373471bf0Spatrick return false;
124473471bf0Spatrick }
124573471bf0Spatrick } else {
124673471bf0Spatrick // Return false (unmergable) if there is a call instruction
124773471bf0Spatrick // in-between parallel regions when it is not an intrinsic. It
124873471bf0Spatrick // may call an unmergable OpenMP runtime function in its callpath.
124973471bf0Spatrick // TODO: Keep track of possible OpenMP calls in the callpath.
125073471bf0Spatrick if (!isa<IntrinsicInst>(CI))
125173471bf0Spatrick return false;
125273471bf0Spatrick }
125373471bf0Spatrick
125473471bf0Spatrick return true;
125573471bf0Spatrick };
125673471bf0Spatrick // Find maximal number of parallel region CIs that are safe to merge.
125773471bf0Spatrick for (auto It = BB->begin(), End = BB->end(); It != End;) {
125873471bf0Spatrick Instruction &I = *It;
125973471bf0Spatrick ++It;
126073471bf0Spatrick
126173471bf0Spatrick if (CIs.count(&I)) {
126273471bf0Spatrick MergableCIs.push_back(cast<CallInst>(&I));
126373471bf0Spatrick continue;
126473471bf0Spatrick }
126573471bf0Spatrick
126673471bf0Spatrick // Continue expanding if the instruction is mergable.
126773471bf0Spatrick if (IsMergable(I, MergableCIs.empty()))
126873471bf0Spatrick continue;
126973471bf0Spatrick
127073471bf0Spatrick // Forward the instruction iterator to skip the next parallel region
127173471bf0Spatrick // since there is an unmergable instruction which can affect it.
127273471bf0Spatrick for (; It != End; ++It) {
127373471bf0Spatrick Instruction &SkipI = *It;
127473471bf0Spatrick if (CIs.count(&SkipI)) {
127573471bf0Spatrick LLVM_DEBUG(dbgs() << TAG << "Skip parallel region " << SkipI
127673471bf0Spatrick << " due to " << I << "\n");
127773471bf0Spatrick ++It;
127873471bf0Spatrick break;
127973471bf0Spatrick }
128073471bf0Spatrick }
128173471bf0Spatrick
128273471bf0Spatrick // Store mergable regions found.
128373471bf0Spatrick if (MergableCIs.size() > 1) {
128473471bf0Spatrick MergableCIsVector.push_back(MergableCIs);
128573471bf0Spatrick LLVM_DEBUG(dbgs() << TAG << "Found " << MergableCIs.size()
128673471bf0Spatrick << " parallel regions in block " << BB->getName()
128773471bf0Spatrick << " of function " << BB->getParent()->getName()
128873471bf0Spatrick << "\n";);
128973471bf0Spatrick }
129073471bf0Spatrick
129173471bf0Spatrick MergableCIs.clear();
129273471bf0Spatrick }
129373471bf0Spatrick
129473471bf0Spatrick if (!MergableCIsVector.empty()) {
129573471bf0Spatrick Changed = true;
129673471bf0Spatrick
129773471bf0Spatrick for (auto &MergableCIs : MergableCIsVector)
129873471bf0Spatrick Merge(MergableCIs, BB);
129973471bf0Spatrick MergableCIsVector.clear();
130073471bf0Spatrick }
130173471bf0Spatrick }
130273471bf0Spatrick
130373471bf0Spatrick if (Changed) {
130473471bf0Spatrick /// Re-collect use for fork calls, emitted barrier calls, and
130573471bf0Spatrick /// any emitted master/end_master calls.
130673471bf0Spatrick OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_fork_call);
130773471bf0Spatrick OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_barrier);
130873471bf0Spatrick OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_master);
130973471bf0Spatrick OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_end_master);
131073471bf0Spatrick }
131173471bf0Spatrick
131273471bf0Spatrick return Changed;
131373471bf0Spatrick }
131473471bf0Spatrick
1315097a140dSpatrick /// Try to delete parallel regions if possible.
deleteParallelRegions__anon4b47cb5e0111::OpenMPOpt1316097a140dSpatrick bool deleteParallelRegions() {
1317097a140dSpatrick const unsigned CallbackCalleeOperand = 2;
1318097a140dSpatrick
1319097a140dSpatrick OMPInformationCache::RuntimeFunctionInfo &RFI =
1320097a140dSpatrick OMPInfoCache.RFIs[OMPRTL___kmpc_fork_call];
1321097a140dSpatrick
1322097a140dSpatrick if (!RFI.Declaration)
1323097a140dSpatrick return false;
1324097a140dSpatrick
1325097a140dSpatrick bool Changed = false;
1326097a140dSpatrick auto DeleteCallCB = [&](Use &U, Function &) {
1327097a140dSpatrick CallInst *CI = getCallIfRegularCall(U);
1328097a140dSpatrick if (!CI)
1329097a140dSpatrick return false;
1330097a140dSpatrick auto *Fn = dyn_cast<Function>(
1331097a140dSpatrick CI->getArgOperand(CallbackCalleeOperand)->stripPointerCasts());
1332097a140dSpatrick if (!Fn)
1333097a140dSpatrick return false;
1334097a140dSpatrick if (!Fn->onlyReadsMemory())
1335097a140dSpatrick return false;
1336097a140dSpatrick if (!Fn->hasFnAttribute(Attribute::WillReturn))
1337097a140dSpatrick return false;
1338097a140dSpatrick
1339097a140dSpatrick LLVM_DEBUG(dbgs() << TAG << "Delete read-only parallel region in "
1340097a140dSpatrick << CI->getCaller()->getName() << "\n");
1341097a140dSpatrick
1342097a140dSpatrick auto Remark = [&](OptimizationRemark OR) {
134373471bf0Spatrick return OR << "Removing parallel region with no side-effects.";
1344097a140dSpatrick };
134573471bf0Spatrick emitRemark<OptimizationRemark>(CI, "OMP160", Remark);
1346097a140dSpatrick
1347097a140dSpatrick CGUpdater.removeCallSite(*CI);
1348097a140dSpatrick CI->eraseFromParent();
1349097a140dSpatrick Changed = true;
1350097a140dSpatrick ++NumOpenMPParallelRegionsDeleted;
1351097a140dSpatrick return true;
1352097a140dSpatrick };
1353097a140dSpatrick
1354097a140dSpatrick RFI.foreachUse(SCC, DeleteCallCB);
1355097a140dSpatrick
1356097a140dSpatrick return Changed;
1357097a140dSpatrick }
1358097a140dSpatrick
1359097a140dSpatrick /// Try to eliminate runtime calls by reusing existing ones.
deduplicateRuntimeCalls__anon4b47cb5e0111::OpenMPOpt1360097a140dSpatrick bool deduplicateRuntimeCalls() {
1361097a140dSpatrick bool Changed = false;
1362097a140dSpatrick
1363097a140dSpatrick RuntimeFunction DeduplicableRuntimeCallIDs[] = {
1364097a140dSpatrick OMPRTL_omp_get_num_threads,
1365097a140dSpatrick OMPRTL_omp_in_parallel,
1366097a140dSpatrick OMPRTL_omp_get_cancellation,
1367097a140dSpatrick OMPRTL_omp_get_thread_limit,
1368097a140dSpatrick OMPRTL_omp_get_supported_active_levels,
1369097a140dSpatrick OMPRTL_omp_get_level,
1370097a140dSpatrick OMPRTL_omp_get_ancestor_thread_num,
1371097a140dSpatrick OMPRTL_omp_get_team_size,
1372097a140dSpatrick OMPRTL_omp_get_active_level,
1373097a140dSpatrick OMPRTL_omp_in_final,
1374097a140dSpatrick OMPRTL_omp_get_proc_bind,
1375097a140dSpatrick OMPRTL_omp_get_num_places,
1376097a140dSpatrick OMPRTL_omp_get_num_procs,
1377097a140dSpatrick OMPRTL_omp_get_place_num,
1378097a140dSpatrick OMPRTL_omp_get_partition_num_places,
1379097a140dSpatrick OMPRTL_omp_get_partition_place_nums};
1380097a140dSpatrick
1381097a140dSpatrick // Global-tid is handled separately.
1382097a140dSpatrick SmallSetVector<Value *, 16> GTIdArgs;
1383097a140dSpatrick collectGlobalThreadIdArguments(GTIdArgs);
1384097a140dSpatrick LLVM_DEBUG(dbgs() << TAG << "Found " << GTIdArgs.size()
1385097a140dSpatrick << " global thread ID arguments\n");
1386097a140dSpatrick
1387097a140dSpatrick for (Function *F : SCC) {
1388097a140dSpatrick for (auto DeduplicableRuntimeCallID : DeduplicableRuntimeCallIDs)
138973471bf0Spatrick Changed |= deduplicateRuntimeCalls(
139073471bf0Spatrick *F, OMPInfoCache.RFIs[DeduplicableRuntimeCallID]);
1391097a140dSpatrick
1392097a140dSpatrick // __kmpc_global_thread_num is special as we can replace it with an
1393097a140dSpatrick // argument in enough cases to make it worth trying.
1394097a140dSpatrick Value *GTIdArg = nullptr;
1395097a140dSpatrick for (Argument &Arg : F->args())
1396097a140dSpatrick if (GTIdArgs.count(&Arg)) {
1397097a140dSpatrick GTIdArg = &Arg;
1398097a140dSpatrick break;
1399097a140dSpatrick }
1400097a140dSpatrick Changed |= deduplicateRuntimeCalls(
1401097a140dSpatrick *F, OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num], GTIdArg);
1402097a140dSpatrick }
1403097a140dSpatrick
1404097a140dSpatrick return Changed;
1405097a140dSpatrick }
1406097a140dSpatrick
140773471bf0Spatrick /// Tries to hide the latency of runtime calls that involve host to
140873471bf0Spatrick /// device memory transfers by splitting them into their "issue" and "wait"
140973471bf0Spatrick /// versions. The "issue" is moved upwards as much as possible. The "wait" is
141073471bf0Spatrick /// moved downards as much as possible. The "issue" issues the memory transfer
141173471bf0Spatrick /// asynchronously, returning a handle. The "wait" waits in the returned
141273471bf0Spatrick /// handle for the memory transfer to finish.
hideMemTransfersLatency__anon4b47cb5e0111::OpenMPOpt141373471bf0Spatrick bool hideMemTransfersLatency() {
141473471bf0Spatrick auto &RFI = OMPInfoCache.RFIs[OMPRTL___tgt_target_data_begin_mapper];
141573471bf0Spatrick bool Changed = false;
141673471bf0Spatrick auto SplitMemTransfers = [&](Use &U, Function &Decl) {
141773471bf0Spatrick auto *RTCall = getCallIfRegularCall(U, &RFI);
141873471bf0Spatrick if (!RTCall)
141973471bf0Spatrick return false;
142073471bf0Spatrick
142173471bf0Spatrick OffloadArray OffloadArrays[3];
142273471bf0Spatrick if (!getValuesInOffloadArrays(*RTCall, OffloadArrays))
142373471bf0Spatrick return false;
142473471bf0Spatrick
142573471bf0Spatrick LLVM_DEBUG(dumpValuesInOffloadArrays(OffloadArrays));
142673471bf0Spatrick
142773471bf0Spatrick // TODO: Check if can be moved upwards.
142873471bf0Spatrick bool WasSplit = false;
142973471bf0Spatrick Instruction *WaitMovementPoint = canBeMovedDownwards(*RTCall);
143073471bf0Spatrick if (WaitMovementPoint)
143173471bf0Spatrick WasSplit = splitTargetDataBeginRTC(*RTCall, *WaitMovementPoint);
143273471bf0Spatrick
143373471bf0Spatrick Changed |= WasSplit;
143473471bf0Spatrick return WasSplit;
143573471bf0Spatrick };
1436*d415bd75Srobert if (OMPInfoCache.runtimeFnsAvailable(
1437*d415bd75Srobert {OMPRTL___tgt_target_data_begin_mapper_issue,
1438*d415bd75Srobert OMPRTL___tgt_target_data_begin_mapper_wait}))
143973471bf0Spatrick RFI.foreachUse(SCC, SplitMemTransfers);
144073471bf0Spatrick
144173471bf0Spatrick return Changed;
144273471bf0Spatrick }
144373471bf0Spatrick
analysisGlobalization__anon4b47cb5e0111::OpenMPOpt144473471bf0Spatrick void analysisGlobalization() {
144573471bf0Spatrick auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
144673471bf0Spatrick
144773471bf0Spatrick auto CheckGlobalization = [&](Use &U, Function &Decl) {
144873471bf0Spatrick if (CallInst *CI = getCallIfRegularCall(U, &RFI)) {
144973471bf0Spatrick auto Remark = [&](OptimizationRemarkMissed ORM) {
145073471bf0Spatrick return ORM
145173471bf0Spatrick << "Found thread data sharing on the GPU. "
145273471bf0Spatrick << "Expect degraded performance due to data globalization.";
145373471bf0Spatrick };
145473471bf0Spatrick emitRemark<OptimizationRemarkMissed>(CI, "OMP112", Remark);
145573471bf0Spatrick }
145673471bf0Spatrick
145773471bf0Spatrick return false;
145873471bf0Spatrick };
145973471bf0Spatrick
146073471bf0Spatrick RFI.foreachUse(SCC, CheckGlobalization);
146173471bf0Spatrick }
146273471bf0Spatrick
146373471bf0Spatrick /// Maps the values stored in the offload arrays passed as arguments to
146473471bf0Spatrick /// \p RuntimeCall into the offload arrays in \p OAs.
getValuesInOffloadArrays__anon4b47cb5e0111::OpenMPOpt146573471bf0Spatrick bool getValuesInOffloadArrays(CallInst &RuntimeCall,
146673471bf0Spatrick MutableArrayRef<OffloadArray> OAs) {
146773471bf0Spatrick assert(OAs.size() == 3 && "Need space for three offload arrays!");
146873471bf0Spatrick
146973471bf0Spatrick // A runtime call that involves memory offloading looks something like:
147073471bf0Spatrick // call void @__tgt_target_data_begin_mapper(arg0, arg1,
147173471bf0Spatrick // i8** %offload_baseptrs, i8** %offload_ptrs, i64* %offload_sizes,
147273471bf0Spatrick // ...)
147373471bf0Spatrick // So, the idea is to access the allocas that allocate space for these
147473471bf0Spatrick // offload arrays, offload_baseptrs, offload_ptrs, offload_sizes.
147573471bf0Spatrick // Therefore:
147673471bf0Spatrick // i8** %offload_baseptrs.
147773471bf0Spatrick Value *BasePtrsArg =
147873471bf0Spatrick RuntimeCall.getArgOperand(OffloadArray::BasePtrsArgNum);
147973471bf0Spatrick // i8** %offload_ptrs.
148073471bf0Spatrick Value *PtrsArg = RuntimeCall.getArgOperand(OffloadArray::PtrsArgNum);
148173471bf0Spatrick // i8** %offload_sizes.
148273471bf0Spatrick Value *SizesArg = RuntimeCall.getArgOperand(OffloadArray::SizesArgNum);
148373471bf0Spatrick
148473471bf0Spatrick // Get values stored in **offload_baseptrs.
148573471bf0Spatrick auto *V = getUnderlyingObject(BasePtrsArg);
148673471bf0Spatrick if (!isa<AllocaInst>(V))
148773471bf0Spatrick return false;
148873471bf0Spatrick auto *BasePtrsArray = cast<AllocaInst>(V);
148973471bf0Spatrick if (!OAs[0].initialize(*BasePtrsArray, RuntimeCall))
149073471bf0Spatrick return false;
149173471bf0Spatrick
149273471bf0Spatrick // Get values stored in **offload_baseptrs.
149373471bf0Spatrick V = getUnderlyingObject(PtrsArg);
149473471bf0Spatrick if (!isa<AllocaInst>(V))
149573471bf0Spatrick return false;
149673471bf0Spatrick auto *PtrsArray = cast<AllocaInst>(V);
149773471bf0Spatrick if (!OAs[1].initialize(*PtrsArray, RuntimeCall))
149873471bf0Spatrick return false;
149973471bf0Spatrick
150073471bf0Spatrick // Get values stored in **offload_sizes.
150173471bf0Spatrick V = getUnderlyingObject(SizesArg);
150273471bf0Spatrick // If it's a [constant] global array don't analyze it.
150373471bf0Spatrick if (isa<GlobalValue>(V))
150473471bf0Spatrick return isa<Constant>(V);
150573471bf0Spatrick if (!isa<AllocaInst>(V))
150673471bf0Spatrick return false;
150773471bf0Spatrick
150873471bf0Spatrick auto *SizesArray = cast<AllocaInst>(V);
150973471bf0Spatrick if (!OAs[2].initialize(*SizesArray, RuntimeCall))
151073471bf0Spatrick return false;
151173471bf0Spatrick
151273471bf0Spatrick return true;
151373471bf0Spatrick }
151473471bf0Spatrick
151573471bf0Spatrick /// Prints the values in the OffloadArrays \p OAs using LLVM_DEBUG.
151673471bf0Spatrick /// For now this is a way to test that the function getValuesInOffloadArrays
151773471bf0Spatrick /// is working properly.
151873471bf0Spatrick /// TODO: Move this to a unittest when unittests are available for OpenMPOpt.
dumpValuesInOffloadArrays__anon4b47cb5e0111::OpenMPOpt151973471bf0Spatrick void dumpValuesInOffloadArrays(ArrayRef<OffloadArray> OAs) {
152073471bf0Spatrick assert(OAs.size() == 3 && "There are three offload arrays to debug!");
152173471bf0Spatrick
152273471bf0Spatrick LLVM_DEBUG(dbgs() << TAG << " Successfully got offload values:\n");
152373471bf0Spatrick std::string ValuesStr;
152473471bf0Spatrick raw_string_ostream Printer(ValuesStr);
152573471bf0Spatrick std::string Separator = " --- ";
152673471bf0Spatrick
152773471bf0Spatrick for (auto *BP : OAs[0].StoredValues) {
152873471bf0Spatrick BP->print(Printer);
152973471bf0Spatrick Printer << Separator;
153073471bf0Spatrick }
153173471bf0Spatrick LLVM_DEBUG(dbgs() << "\t\toffload_baseptrs: " << Printer.str() << "\n");
153273471bf0Spatrick ValuesStr.clear();
153373471bf0Spatrick
153473471bf0Spatrick for (auto *P : OAs[1].StoredValues) {
153573471bf0Spatrick P->print(Printer);
153673471bf0Spatrick Printer << Separator;
153773471bf0Spatrick }
153873471bf0Spatrick LLVM_DEBUG(dbgs() << "\t\toffload_ptrs: " << Printer.str() << "\n");
153973471bf0Spatrick ValuesStr.clear();
154073471bf0Spatrick
154173471bf0Spatrick for (auto *S : OAs[2].StoredValues) {
154273471bf0Spatrick S->print(Printer);
154373471bf0Spatrick Printer << Separator;
154473471bf0Spatrick }
154573471bf0Spatrick LLVM_DEBUG(dbgs() << "\t\toffload_sizes: " << Printer.str() << "\n");
154673471bf0Spatrick }
154773471bf0Spatrick
154873471bf0Spatrick /// Returns the instruction where the "wait" counterpart \p RuntimeCall can be
154973471bf0Spatrick /// moved. Returns nullptr if the movement is not possible, or not worth it.
canBeMovedDownwards__anon4b47cb5e0111::OpenMPOpt155073471bf0Spatrick Instruction *canBeMovedDownwards(CallInst &RuntimeCall) {
155173471bf0Spatrick // FIXME: This traverses only the BasicBlock where RuntimeCall is.
155273471bf0Spatrick // Make it traverse the CFG.
155373471bf0Spatrick
155473471bf0Spatrick Instruction *CurrentI = &RuntimeCall;
155573471bf0Spatrick bool IsWorthIt = false;
155673471bf0Spatrick while ((CurrentI = CurrentI->getNextNode())) {
155773471bf0Spatrick
155873471bf0Spatrick // TODO: Once we detect the regions to be offloaded we should use the
155973471bf0Spatrick // alias analysis manager to check if CurrentI may modify one of
156073471bf0Spatrick // the offloaded regions.
156173471bf0Spatrick if (CurrentI->mayHaveSideEffects() || CurrentI->mayReadFromMemory()) {
156273471bf0Spatrick if (IsWorthIt)
156373471bf0Spatrick return CurrentI;
156473471bf0Spatrick
156573471bf0Spatrick return nullptr;
156673471bf0Spatrick }
156773471bf0Spatrick
156873471bf0Spatrick // FIXME: For now if we move it over anything without side effect
156973471bf0Spatrick // is worth it.
157073471bf0Spatrick IsWorthIt = true;
157173471bf0Spatrick }
157273471bf0Spatrick
157373471bf0Spatrick // Return end of BasicBlock.
157473471bf0Spatrick return RuntimeCall.getParent()->getTerminator();
157573471bf0Spatrick }
157673471bf0Spatrick
157773471bf0Spatrick /// Splits \p RuntimeCall into its "issue" and "wait" counterparts.
splitTargetDataBeginRTC__anon4b47cb5e0111::OpenMPOpt157873471bf0Spatrick bool splitTargetDataBeginRTC(CallInst &RuntimeCall,
157973471bf0Spatrick Instruction &WaitMovementPoint) {
158073471bf0Spatrick // Create stack allocated handle (__tgt_async_info) at the beginning of the
158173471bf0Spatrick // function. Used for storing information of the async transfer, allowing to
158273471bf0Spatrick // wait on it later.
158373471bf0Spatrick auto &IRBuilder = OMPInfoCache.OMPBuilder;
1584*d415bd75Srobert Function *F = RuntimeCall.getCaller();
1585*d415bd75Srobert BasicBlock &Entry = F->getEntryBlock();
1586*d415bd75Srobert IRBuilder.Builder.SetInsertPoint(&Entry,
1587*d415bd75Srobert Entry.getFirstNonPHIOrDbgOrAlloca());
1588*d415bd75Srobert Value *Handle = IRBuilder.Builder.CreateAlloca(
1589*d415bd75Srobert IRBuilder.AsyncInfo, /*ArraySize=*/nullptr, "handle");
1590*d415bd75Srobert Handle =
1591*d415bd75Srobert IRBuilder.Builder.CreateAddrSpaceCast(Handle, IRBuilder.AsyncInfoPtr);
159273471bf0Spatrick
159373471bf0Spatrick // Add "issue" runtime call declaration:
159473471bf0Spatrick // declare %struct.tgt_async_info @__tgt_target_data_begin_issue(i64, i32,
159573471bf0Spatrick // i8**, i8**, i64*, i64*)
159673471bf0Spatrick FunctionCallee IssueDecl = IRBuilder.getOrCreateRuntimeFunction(
159773471bf0Spatrick M, OMPRTL___tgt_target_data_begin_mapper_issue);
159873471bf0Spatrick
159973471bf0Spatrick // Change RuntimeCall call site for its asynchronous version.
160073471bf0Spatrick SmallVector<Value *, 16> Args;
160173471bf0Spatrick for (auto &Arg : RuntimeCall.args())
160273471bf0Spatrick Args.push_back(Arg.get());
160373471bf0Spatrick Args.push_back(Handle);
160473471bf0Spatrick
160573471bf0Spatrick CallInst *IssueCallsite =
160673471bf0Spatrick CallInst::Create(IssueDecl, Args, /*NameStr=*/"", &RuntimeCall);
1607*d415bd75Srobert OMPInfoCache.setCallingConvention(IssueDecl, IssueCallsite);
160873471bf0Spatrick RuntimeCall.eraseFromParent();
160973471bf0Spatrick
161073471bf0Spatrick // Add "wait" runtime call declaration:
161173471bf0Spatrick // declare void @__tgt_target_data_begin_wait(i64, %struct.__tgt_async_info)
161273471bf0Spatrick FunctionCallee WaitDecl = IRBuilder.getOrCreateRuntimeFunction(
161373471bf0Spatrick M, OMPRTL___tgt_target_data_begin_mapper_wait);
161473471bf0Spatrick
161573471bf0Spatrick Value *WaitParams[2] = {
161673471bf0Spatrick IssueCallsite->getArgOperand(
161773471bf0Spatrick OffloadArray::DeviceIDArgNum), // device_id.
161873471bf0Spatrick Handle // handle to wait on.
161973471bf0Spatrick };
1620*d415bd75Srobert CallInst *WaitCallsite = CallInst::Create(
1621*d415bd75Srobert WaitDecl, WaitParams, /*NameStr=*/"", &WaitMovementPoint);
1622*d415bd75Srobert OMPInfoCache.setCallingConvention(WaitDecl, WaitCallsite);
162373471bf0Spatrick
162473471bf0Spatrick return true;
162573471bf0Spatrick }
162673471bf0Spatrick
combinedIdentStruct__anon4b47cb5e0111::OpenMPOpt1627097a140dSpatrick static Value *combinedIdentStruct(Value *CurrentIdent, Value *NextIdent,
1628097a140dSpatrick bool GlobalOnly, bool &SingleChoice) {
1629097a140dSpatrick if (CurrentIdent == NextIdent)
1630097a140dSpatrick return CurrentIdent;
1631097a140dSpatrick
1632097a140dSpatrick // TODO: Figure out how to actually combine multiple debug locations. For
1633097a140dSpatrick // now we just keep an existing one if there is a single choice.
1634097a140dSpatrick if (!GlobalOnly || isa<GlobalValue>(NextIdent)) {
1635097a140dSpatrick SingleChoice = !CurrentIdent;
1636097a140dSpatrick return NextIdent;
1637097a140dSpatrick }
1638097a140dSpatrick return nullptr;
1639097a140dSpatrick }
1640097a140dSpatrick
1641097a140dSpatrick /// Return an `struct ident_t*` value that represents the ones used in the
1642097a140dSpatrick /// calls of \p RFI inside of \p F. If \p GlobalOnly is true, we will not
1643097a140dSpatrick /// return a local `struct ident_t*`. For now, if we cannot find a suitable
1644097a140dSpatrick /// return value we create one from scratch. We also do not yet combine
1645097a140dSpatrick /// information, e.g., the source locations, see combinedIdentStruct.
1646097a140dSpatrick Value *
getCombinedIdentFromCallUsesIn__anon4b47cb5e0111::OpenMPOpt1647097a140dSpatrick getCombinedIdentFromCallUsesIn(OMPInformationCache::RuntimeFunctionInfo &RFI,
1648097a140dSpatrick Function &F, bool GlobalOnly) {
1649097a140dSpatrick bool SingleChoice = true;
1650097a140dSpatrick Value *Ident = nullptr;
1651097a140dSpatrick auto CombineIdentStruct = [&](Use &U, Function &Caller) {
1652097a140dSpatrick CallInst *CI = getCallIfRegularCall(U, &RFI);
1653097a140dSpatrick if (!CI || &F != &Caller)
1654097a140dSpatrick return false;
1655097a140dSpatrick Ident = combinedIdentStruct(Ident, CI->getArgOperand(0),
1656097a140dSpatrick /* GlobalOnly */ true, SingleChoice);
1657097a140dSpatrick return false;
1658097a140dSpatrick };
1659097a140dSpatrick RFI.foreachUse(SCC, CombineIdentStruct);
1660097a140dSpatrick
1661097a140dSpatrick if (!Ident || !SingleChoice) {
1662097a140dSpatrick // The IRBuilder uses the insertion block to get to the module, this is
1663097a140dSpatrick // unfortunate but we work around it for now.
1664097a140dSpatrick if (!OMPInfoCache.OMPBuilder.getInsertionPoint().getBlock())
1665097a140dSpatrick OMPInfoCache.OMPBuilder.updateToLocation(OpenMPIRBuilder::InsertPointTy(
1666097a140dSpatrick &F.getEntryBlock(), F.getEntryBlock().begin()));
1667097a140dSpatrick // Create a fallback location if non was found.
1668097a140dSpatrick // TODO: Use the debug locations of the calls instead.
1669*d415bd75Srobert uint32_t SrcLocStrSize;
1670*d415bd75Srobert Constant *Loc =
1671*d415bd75Srobert OMPInfoCache.OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1672*d415bd75Srobert Ident = OMPInfoCache.OMPBuilder.getOrCreateIdent(Loc, SrcLocStrSize);
1673097a140dSpatrick }
1674097a140dSpatrick return Ident;
1675097a140dSpatrick }
1676097a140dSpatrick
1677097a140dSpatrick /// Try to eliminate calls of \p RFI in \p F by reusing an existing one or
1678097a140dSpatrick /// \p ReplVal if given.
deduplicateRuntimeCalls__anon4b47cb5e0111::OpenMPOpt1679097a140dSpatrick bool deduplicateRuntimeCalls(Function &F,
1680097a140dSpatrick OMPInformationCache::RuntimeFunctionInfo &RFI,
1681097a140dSpatrick Value *ReplVal = nullptr) {
1682097a140dSpatrick auto *UV = RFI.getUseVector(F);
1683097a140dSpatrick if (!UV || UV->size() + (ReplVal != nullptr) < 2)
1684097a140dSpatrick return false;
1685097a140dSpatrick
1686097a140dSpatrick LLVM_DEBUG(
1687097a140dSpatrick dbgs() << TAG << "Deduplicate " << UV->size() << " uses of " << RFI.Name
1688097a140dSpatrick << (ReplVal ? " with an existing value\n" : "\n") << "\n");
1689097a140dSpatrick
1690097a140dSpatrick assert((!ReplVal || (isa<Argument>(ReplVal) &&
1691097a140dSpatrick cast<Argument>(ReplVal)->getParent() == &F)) &&
1692097a140dSpatrick "Unexpected replacement value!");
1693097a140dSpatrick
1694097a140dSpatrick // TODO: Use dominance to find a good position instead.
1695097a140dSpatrick auto CanBeMoved = [this](CallBase &CB) {
1696*d415bd75Srobert unsigned NumArgs = CB.arg_size();
1697097a140dSpatrick if (NumArgs == 0)
1698097a140dSpatrick return true;
1699097a140dSpatrick if (CB.getArgOperand(0)->getType() != OMPInfoCache.OMPBuilder.IdentPtr)
1700097a140dSpatrick return false;
1701*d415bd75Srobert for (unsigned U = 1; U < NumArgs; ++U)
1702*d415bd75Srobert if (isa<Instruction>(CB.getArgOperand(U)))
1703097a140dSpatrick return false;
1704097a140dSpatrick return true;
1705097a140dSpatrick };
1706097a140dSpatrick
1707097a140dSpatrick if (!ReplVal) {
1708097a140dSpatrick for (Use *U : *UV)
1709097a140dSpatrick if (CallInst *CI = getCallIfRegularCall(*U, &RFI)) {
1710097a140dSpatrick if (!CanBeMoved(*CI))
1711097a140dSpatrick continue;
1712097a140dSpatrick
171373471bf0Spatrick // If the function is a kernel, dedup will move
171473471bf0Spatrick // the runtime call right after the kernel init callsite. Otherwise,
171573471bf0Spatrick // it will move it to the beginning of the caller function.
171673471bf0Spatrick if (isKernel(F)) {
171773471bf0Spatrick auto &KernelInitRFI = OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
171873471bf0Spatrick auto *KernelInitUV = KernelInitRFI.getUseVector(F);
1719097a140dSpatrick
172073471bf0Spatrick if (KernelInitUV->empty())
172173471bf0Spatrick continue;
172273471bf0Spatrick
172373471bf0Spatrick assert(KernelInitUV->size() == 1 &&
172473471bf0Spatrick "Expected a single __kmpc_target_init in kernel\n");
172573471bf0Spatrick
172673471bf0Spatrick CallInst *KernelInitCI =
172773471bf0Spatrick getCallIfRegularCall(*KernelInitUV->front(), &KernelInitRFI);
172873471bf0Spatrick assert(KernelInitCI &&
172973471bf0Spatrick "Expected a call to __kmpc_target_init in kernel\n");
173073471bf0Spatrick
173173471bf0Spatrick CI->moveAfter(KernelInitCI);
173273471bf0Spatrick } else
1733097a140dSpatrick CI->moveBefore(&*F.getEntryBlock().getFirstInsertionPt());
1734097a140dSpatrick ReplVal = CI;
1735097a140dSpatrick break;
1736097a140dSpatrick }
1737097a140dSpatrick if (!ReplVal)
1738097a140dSpatrick return false;
1739097a140dSpatrick }
1740097a140dSpatrick
1741097a140dSpatrick // If we use a call as a replacement value we need to make sure the ident is
1742097a140dSpatrick // valid at the new location. For now we just pick a global one, either
1743097a140dSpatrick // existing and used by one of the calls, or created from scratch.
1744097a140dSpatrick if (CallBase *CI = dyn_cast<CallBase>(ReplVal)) {
1745*d415bd75Srobert if (!CI->arg_empty() &&
1746097a140dSpatrick CI->getArgOperand(0)->getType() == OMPInfoCache.OMPBuilder.IdentPtr) {
1747097a140dSpatrick Value *Ident = getCombinedIdentFromCallUsesIn(RFI, F,
1748097a140dSpatrick /* GlobalOnly */ true);
1749097a140dSpatrick CI->setArgOperand(0, Ident);
1750097a140dSpatrick }
1751097a140dSpatrick }
1752097a140dSpatrick
1753097a140dSpatrick bool Changed = false;
1754097a140dSpatrick auto ReplaceAndDeleteCB = [&](Use &U, Function &Caller) {
1755097a140dSpatrick CallInst *CI = getCallIfRegularCall(U, &RFI);
1756097a140dSpatrick if (!CI || CI == ReplVal || &F != &Caller)
1757097a140dSpatrick return false;
1758097a140dSpatrick assert(CI->getCaller() == &F && "Unexpected call!");
1759097a140dSpatrick
1760097a140dSpatrick auto Remark = [&](OptimizationRemark OR) {
1761097a140dSpatrick return OR << "OpenMP runtime call "
176273471bf0Spatrick << ore::NV("OpenMPOptRuntime", RFI.Name) << " deduplicated.";
1763097a140dSpatrick };
176473471bf0Spatrick if (CI->getDebugLoc())
176573471bf0Spatrick emitRemark<OptimizationRemark>(CI, "OMP170", Remark);
176673471bf0Spatrick else
176773471bf0Spatrick emitRemark<OptimizationRemark>(&F, "OMP170", Remark);
1768097a140dSpatrick
1769097a140dSpatrick CGUpdater.removeCallSite(*CI);
1770097a140dSpatrick CI->replaceAllUsesWith(ReplVal);
1771097a140dSpatrick CI->eraseFromParent();
1772097a140dSpatrick ++NumOpenMPRuntimeCallsDeduplicated;
1773097a140dSpatrick Changed = true;
1774097a140dSpatrick return true;
1775097a140dSpatrick };
1776097a140dSpatrick RFI.foreachUse(SCC, ReplaceAndDeleteCB);
1777097a140dSpatrick
1778097a140dSpatrick return Changed;
1779097a140dSpatrick }
1780097a140dSpatrick
1781097a140dSpatrick /// Collect arguments that represent the global thread id in \p GTIdArgs.
collectGlobalThreadIdArguments__anon4b47cb5e0111::OpenMPOpt1782097a140dSpatrick void collectGlobalThreadIdArguments(SmallSetVector<Value *, 16> >IdArgs) {
1783097a140dSpatrick // TODO: Below we basically perform a fixpoint iteration with a pessimistic
1784097a140dSpatrick // initialization. We could define an AbstractAttribute instead and
1785097a140dSpatrick // run the Attributor here once it can be run as an SCC pass.
1786097a140dSpatrick
1787097a140dSpatrick // Helper to check the argument \p ArgNo at all call sites of \p F for
1788097a140dSpatrick // a GTId.
1789097a140dSpatrick auto CallArgOpIsGTId = [&](Function &F, unsigned ArgNo, CallInst &RefCI) {
1790097a140dSpatrick if (!F.hasLocalLinkage())
1791097a140dSpatrick return false;
1792097a140dSpatrick for (Use &U : F.uses()) {
1793097a140dSpatrick if (CallInst *CI = getCallIfRegularCall(U)) {
1794097a140dSpatrick Value *ArgOp = CI->getArgOperand(ArgNo);
1795097a140dSpatrick if (CI == &RefCI || GTIdArgs.count(ArgOp) ||
1796097a140dSpatrick getCallIfRegularCall(
1797097a140dSpatrick *ArgOp, &OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num]))
1798097a140dSpatrick continue;
1799097a140dSpatrick }
1800097a140dSpatrick return false;
1801097a140dSpatrick }
1802097a140dSpatrick return true;
1803097a140dSpatrick };
1804097a140dSpatrick
1805097a140dSpatrick // Helper to identify uses of a GTId as GTId arguments.
1806097a140dSpatrick auto AddUserArgs = [&](Value >Id) {
1807097a140dSpatrick for (Use &U : GTId.uses())
1808097a140dSpatrick if (CallInst *CI = dyn_cast<CallInst>(U.getUser()))
1809097a140dSpatrick if (CI->isArgOperand(&U))
1810097a140dSpatrick if (Function *Callee = CI->getCalledFunction())
1811097a140dSpatrick if (CallArgOpIsGTId(*Callee, U.getOperandNo(), *CI))
1812097a140dSpatrick GTIdArgs.insert(Callee->getArg(U.getOperandNo()));
1813097a140dSpatrick };
1814097a140dSpatrick
1815097a140dSpatrick // The argument users of __kmpc_global_thread_num calls are GTIds.
1816097a140dSpatrick OMPInformationCache::RuntimeFunctionInfo &GlobThreadNumRFI =
1817097a140dSpatrick OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num];
1818097a140dSpatrick
1819097a140dSpatrick GlobThreadNumRFI.foreachUse(SCC, [&](Use &U, Function &F) {
1820097a140dSpatrick if (CallInst *CI = getCallIfRegularCall(U, &GlobThreadNumRFI))
1821097a140dSpatrick AddUserArgs(*CI);
1822097a140dSpatrick return false;
1823097a140dSpatrick });
1824097a140dSpatrick
1825097a140dSpatrick // Transitively search for more arguments by looking at the users of the
1826097a140dSpatrick // ones we know already. During the search the GTIdArgs vector is extended
1827097a140dSpatrick // so we cannot cache the size nor can we use a range based for.
1828*d415bd75Srobert for (unsigned U = 0; U < GTIdArgs.size(); ++U)
1829*d415bd75Srobert AddUserArgs(*GTIdArgs[U]);
1830097a140dSpatrick }
1831097a140dSpatrick
1832097a140dSpatrick /// Kernel (=GPU) optimizations and utility functions
1833097a140dSpatrick ///
1834097a140dSpatrick ///{{
1835097a140dSpatrick
1836097a140dSpatrick /// Check if \p F is a kernel, hence entry point for target offloading.
isKernel__anon4b47cb5e0111::OpenMPOpt1837097a140dSpatrick bool isKernel(Function &F) { return OMPInfoCache.Kernels.count(&F); }
1838097a140dSpatrick
1839097a140dSpatrick /// Cache to remember the unique kernel for a function.
1840*d415bd75Srobert DenseMap<Function *, std::optional<Kernel>> UniqueKernelMap;
1841097a140dSpatrick
1842097a140dSpatrick /// Find the unique kernel that will execute \p F, if any.
1843097a140dSpatrick Kernel getUniqueKernelFor(Function &F);
1844097a140dSpatrick
1845097a140dSpatrick /// Find the unique kernel that will execute \p I, if any.
getUniqueKernelFor__anon4b47cb5e0111::OpenMPOpt1846097a140dSpatrick Kernel getUniqueKernelFor(Instruction &I) {
1847097a140dSpatrick return getUniqueKernelFor(*I.getFunction());
1848097a140dSpatrick }
1849097a140dSpatrick
1850097a140dSpatrick /// Rewrite the device (=GPU) code state machine create in non-SPMD mode in
1851097a140dSpatrick /// the cases we can avoid taking the address of a function.
1852097a140dSpatrick bool rewriteDeviceCodeStateMachine();
1853097a140dSpatrick
1854097a140dSpatrick ///
1855097a140dSpatrick ///}}
1856097a140dSpatrick
1857097a140dSpatrick /// Emit a remark generically
1858097a140dSpatrick ///
1859097a140dSpatrick /// This template function can be used to generically emit a remark. The
1860097a140dSpatrick /// RemarkKind should be one of the following:
1861097a140dSpatrick /// - OptimizationRemark to indicate a successful optimization attempt
1862097a140dSpatrick /// - OptimizationRemarkMissed to report a failed optimization attempt
1863097a140dSpatrick /// - OptimizationRemarkAnalysis to provide additional information about an
1864097a140dSpatrick /// optimization attempt
1865097a140dSpatrick ///
1866097a140dSpatrick /// The remark is built using a callback function provided by the caller that
1867097a140dSpatrick /// takes a RemarkKind as input and returns a RemarkKind.
186873471bf0Spatrick template <typename RemarkKind, typename RemarkCallBack>
emitRemark__anon4b47cb5e0111::OpenMPOpt186973471bf0Spatrick void emitRemark(Instruction *I, StringRef RemarkName,
1870097a140dSpatrick RemarkCallBack &&RemarkCB) const {
187173471bf0Spatrick Function *F = I->getParent()->getParent();
1872097a140dSpatrick auto &ORE = OREGetter(F);
1873097a140dSpatrick
187473471bf0Spatrick if (RemarkName.startswith("OMP"))
1875097a140dSpatrick ORE.emit([&]() {
187673471bf0Spatrick return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, I))
187773471bf0Spatrick << " [" << RemarkName << "]";
1878097a140dSpatrick });
187973471bf0Spatrick else
188073471bf0Spatrick ORE.emit(
188173471bf0Spatrick [&]() { return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, I)); });
1882097a140dSpatrick }
1883097a140dSpatrick
188473471bf0Spatrick /// Emit a remark on a function.
188573471bf0Spatrick template <typename RemarkKind, typename RemarkCallBack>
emitRemark__anon4b47cb5e0111::OpenMPOpt188673471bf0Spatrick void emitRemark(Function *F, StringRef RemarkName,
188773471bf0Spatrick RemarkCallBack &&RemarkCB) const {
188873471bf0Spatrick auto &ORE = OREGetter(F);
188973471bf0Spatrick
189073471bf0Spatrick if (RemarkName.startswith("OMP"))
189173471bf0Spatrick ORE.emit([&]() {
189273471bf0Spatrick return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, F))
189373471bf0Spatrick << " [" << RemarkName << "]";
189473471bf0Spatrick });
189573471bf0Spatrick else
189673471bf0Spatrick ORE.emit(
189773471bf0Spatrick [&]() { return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, F)); });
189873471bf0Spatrick }
189973471bf0Spatrick
1900097a140dSpatrick /// The underlying module.
1901097a140dSpatrick Module &M;
1902097a140dSpatrick
1903097a140dSpatrick /// The SCC we are operating on.
1904097a140dSpatrick SmallVectorImpl<Function *> &SCC;
1905097a140dSpatrick
1906097a140dSpatrick /// Callback to update the call graph, the first argument is a removed call,
1907097a140dSpatrick /// the second an optional replacement call.
1908097a140dSpatrick CallGraphUpdater &CGUpdater;
1909097a140dSpatrick
1910097a140dSpatrick /// Callback to get an OptimizationRemarkEmitter from a Function *
1911097a140dSpatrick OptimizationRemarkGetter OREGetter;
1912097a140dSpatrick
1913097a140dSpatrick /// OpenMP-specific information cache. Also Used for Attributor runs.
1914097a140dSpatrick OMPInformationCache &OMPInfoCache;
1915097a140dSpatrick
1916097a140dSpatrick /// Attributor instance.
1917097a140dSpatrick Attributor &A;
1918097a140dSpatrick
1919097a140dSpatrick /// Helper function to run Attributor on SCC.
runAttributor__anon4b47cb5e0111::OpenMPOpt192073471bf0Spatrick bool runAttributor(bool IsModulePass) {
1921097a140dSpatrick if (SCC.empty())
1922097a140dSpatrick return false;
1923097a140dSpatrick
192473471bf0Spatrick registerAAs(IsModulePass);
1925097a140dSpatrick
1926097a140dSpatrick ChangeStatus Changed = A.run();
1927097a140dSpatrick
1928097a140dSpatrick LLVM_DEBUG(dbgs() << "[Attributor] Done with " << SCC.size()
1929097a140dSpatrick << " functions, result: " << Changed << ".\n");
1930097a140dSpatrick
1931097a140dSpatrick return Changed == ChangeStatus::CHANGED;
1932097a140dSpatrick }
1933097a140dSpatrick
193473471bf0Spatrick void registerFoldRuntimeCall(RuntimeFunction RF);
193573471bf0Spatrick
1936097a140dSpatrick /// Populate the Attributor with abstract attribute opportunities in the
1937*d415bd75Srobert /// functions.
193873471bf0Spatrick void registerAAs(bool IsModulePass);
1939*d415bd75Srobert
1940*d415bd75Srobert public:
1941*d415bd75Srobert /// Callback to register AAs for live functions, including internal functions
1942*d415bd75Srobert /// marked live during the traversal.
1943*d415bd75Srobert static void registerAAsForFunction(Attributor &A, const Function &F);
1944097a140dSpatrick };
1945097a140dSpatrick
getUniqueKernelFor(Function & F)1946097a140dSpatrick Kernel OpenMPOpt::getUniqueKernelFor(Function &F) {
1947*d415bd75Srobert if (!OMPInfoCache.ModuleSlice.empty() && !OMPInfoCache.ModuleSlice.count(&F))
1948097a140dSpatrick return nullptr;
1949097a140dSpatrick
1950097a140dSpatrick // Use a scope to keep the lifetime of the CachedKernel short.
1951097a140dSpatrick {
1952*d415bd75Srobert std::optional<Kernel> &CachedKernel = UniqueKernelMap[&F];
1953097a140dSpatrick if (CachedKernel)
1954097a140dSpatrick return *CachedKernel;
1955097a140dSpatrick
1956097a140dSpatrick // TODO: We should use an AA to create an (optimistic and callback
1957097a140dSpatrick // call-aware) call graph. For now we stick to simple patterns that
1958097a140dSpatrick // are less powerful, basically the worst fixpoint.
1959097a140dSpatrick if (isKernel(F)) {
1960097a140dSpatrick CachedKernel = Kernel(&F);
1961097a140dSpatrick return *CachedKernel;
1962097a140dSpatrick }
1963097a140dSpatrick
1964097a140dSpatrick CachedKernel = nullptr;
196573471bf0Spatrick if (!F.hasLocalLinkage()) {
196673471bf0Spatrick
196773471bf0Spatrick // See https://openmp.llvm.org/remarks/OptimizationRemarks.html
196873471bf0Spatrick auto Remark = [&](OptimizationRemarkAnalysis ORA) {
196973471bf0Spatrick return ORA << "Potentially unknown OpenMP target region caller.";
197073471bf0Spatrick };
197173471bf0Spatrick emitRemark<OptimizationRemarkAnalysis>(&F, "OMP100", Remark);
197273471bf0Spatrick
1973097a140dSpatrick return nullptr;
1974097a140dSpatrick }
197573471bf0Spatrick }
1976097a140dSpatrick
1977097a140dSpatrick auto GetUniqueKernelForUse = [&](const Use &U) -> Kernel {
1978097a140dSpatrick if (auto *Cmp = dyn_cast<ICmpInst>(U.getUser())) {
1979097a140dSpatrick // Allow use in equality comparisons.
1980097a140dSpatrick if (Cmp->isEquality())
1981097a140dSpatrick return getUniqueKernelFor(*Cmp);
1982097a140dSpatrick return nullptr;
1983097a140dSpatrick }
1984097a140dSpatrick if (auto *CB = dyn_cast<CallBase>(U.getUser())) {
1985097a140dSpatrick // Allow direct calls.
1986097a140dSpatrick if (CB->isCallee(&U))
1987097a140dSpatrick return getUniqueKernelFor(*CB);
198873471bf0Spatrick
198973471bf0Spatrick OMPInformationCache::RuntimeFunctionInfo &KernelParallelRFI =
199073471bf0Spatrick OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_51];
199173471bf0Spatrick // Allow the use in __kmpc_parallel_51 calls.
199273471bf0Spatrick if (OpenMPOpt::getCallIfRegularCall(*U.getUser(), &KernelParallelRFI))
1993097a140dSpatrick return getUniqueKernelFor(*CB);
1994097a140dSpatrick return nullptr;
1995097a140dSpatrick }
1996097a140dSpatrick // Disallow every other use.
1997097a140dSpatrick return nullptr;
1998097a140dSpatrick };
1999097a140dSpatrick
2000097a140dSpatrick // TODO: In the future we want to track more than just a unique kernel.
2001097a140dSpatrick SmallPtrSet<Kernel, 2> PotentialKernels;
200273471bf0Spatrick OMPInformationCache::foreachUse(F, [&](const Use &U) {
2003097a140dSpatrick PotentialKernels.insert(GetUniqueKernelForUse(U));
2004097a140dSpatrick });
2005097a140dSpatrick
2006097a140dSpatrick Kernel K = nullptr;
2007097a140dSpatrick if (PotentialKernels.size() == 1)
2008097a140dSpatrick K = *PotentialKernels.begin();
2009097a140dSpatrick
2010097a140dSpatrick // Cache the result.
2011097a140dSpatrick UniqueKernelMap[&F] = K;
2012097a140dSpatrick
2013097a140dSpatrick return K;
2014097a140dSpatrick }
2015097a140dSpatrick
rewriteDeviceCodeStateMachine()2016097a140dSpatrick bool OpenMPOpt::rewriteDeviceCodeStateMachine() {
201773471bf0Spatrick OMPInformationCache::RuntimeFunctionInfo &KernelParallelRFI =
201873471bf0Spatrick OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_51];
2019097a140dSpatrick
2020097a140dSpatrick bool Changed = false;
202173471bf0Spatrick if (!KernelParallelRFI)
2022097a140dSpatrick return Changed;
2023097a140dSpatrick
2024*d415bd75Srobert // If we have disabled state machine changes, exit
2025*d415bd75Srobert if (DisableOpenMPOptStateMachineRewrite)
2026*d415bd75Srobert return Changed;
2027*d415bd75Srobert
2028097a140dSpatrick for (Function *F : SCC) {
2029097a140dSpatrick
203073471bf0Spatrick // Check if the function is a use in a __kmpc_parallel_51 call at
2031097a140dSpatrick // all.
2032097a140dSpatrick bool UnknownUse = false;
203373471bf0Spatrick bool KernelParallelUse = false;
2034097a140dSpatrick unsigned NumDirectCalls = 0;
2035097a140dSpatrick
2036097a140dSpatrick SmallVector<Use *, 2> ToBeReplacedStateMachineUses;
203773471bf0Spatrick OMPInformationCache::foreachUse(*F, [&](Use &U) {
2038097a140dSpatrick if (auto *CB = dyn_cast<CallBase>(U.getUser()))
2039097a140dSpatrick if (CB->isCallee(&U)) {
2040097a140dSpatrick ++NumDirectCalls;
2041097a140dSpatrick return;
2042097a140dSpatrick }
2043097a140dSpatrick
2044097a140dSpatrick if (isa<ICmpInst>(U.getUser())) {
2045097a140dSpatrick ToBeReplacedStateMachineUses.push_back(&U);
2046097a140dSpatrick return;
2047097a140dSpatrick }
204873471bf0Spatrick
204973471bf0Spatrick // Find wrapper functions that represent parallel kernels.
205073471bf0Spatrick CallInst *CI =
205173471bf0Spatrick OpenMPOpt::getCallIfRegularCall(*U.getUser(), &KernelParallelRFI);
205273471bf0Spatrick const unsigned int WrapperFunctionArgNo = 6;
205373471bf0Spatrick if (!KernelParallelUse && CI &&
205473471bf0Spatrick CI->getArgOperandNo(&U) == WrapperFunctionArgNo) {
205573471bf0Spatrick KernelParallelUse = true;
2056097a140dSpatrick ToBeReplacedStateMachineUses.push_back(&U);
2057097a140dSpatrick return;
2058097a140dSpatrick }
2059097a140dSpatrick UnknownUse = true;
2060097a140dSpatrick });
2061097a140dSpatrick
206273471bf0Spatrick // Do not emit a remark if we haven't seen a __kmpc_parallel_51
2063097a140dSpatrick // use.
206473471bf0Spatrick if (!KernelParallelUse)
2065097a140dSpatrick continue;
2066097a140dSpatrick
2067097a140dSpatrick // If this ever hits, we should investigate.
2068097a140dSpatrick // TODO: Checking the number of uses is not a necessary restriction and
2069097a140dSpatrick // should be lifted.
2070097a140dSpatrick if (UnknownUse || NumDirectCalls != 1 ||
207173471bf0Spatrick ToBeReplacedStateMachineUses.size() > 2) {
207273471bf0Spatrick auto Remark = [&](OptimizationRemarkAnalysis ORA) {
207373471bf0Spatrick return ORA << "Parallel region is used in "
2074097a140dSpatrick << (UnknownUse ? "unknown" : "unexpected")
207573471bf0Spatrick << " ways. Will not attempt to rewrite the state machine.";
2076097a140dSpatrick };
207773471bf0Spatrick emitRemark<OptimizationRemarkAnalysis>(F, "OMP101", Remark);
2078097a140dSpatrick continue;
2079097a140dSpatrick }
2080097a140dSpatrick
208173471bf0Spatrick // Even if we have __kmpc_parallel_51 calls, we (for now) give
2082097a140dSpatrick // up if the function is not called from a unique kernel.
2083097a140dSpatrick Kernel K = getUniqueKernelFor(*F);
2084097a140dSpatrick if (!K) {
208573471bf0Spatrick auto Remark = [&](OptimizationRemarkAnalysis ORA) {
208673471bf0Spatrick return ORA << "Parallel region is not called from a unique kernel. "
208773471bf0Spatrick "Will not attempt to rewrite the state machine.";
2088097a140dSpatrick };
208973471bf0Spatrick emitRemark<OptimizationRemarkAnalysis>(F, "OMP102", Remark);
2090097a140dSpatrick continue;
2091097a140dSpatrick }
2092097a140dSpatrick
2093097a140dSpatrick // We now know F is a parallel body function called only from the kernel K.
2094097a140dSpatrick // We also identified the state machine uses in which we replace the
2095097a140dSpatrick // function pointer by a new global symbol for identification purposes. This
2096097a140dSpatrick // ensures only direct calls to the function are left.
2097097a140dSpatrick
2098097a140dSpatrick Module &M = *F->getParent();
2099097a140dSpatrick Type *Int8Ty = Type::getInt8Ty(M.getContext());
2100097a140dSpatrick
2101097a140dSpatrick auto *ID = new GlobalVariable(
2102097a140dSpatrick M, Int8Ty, /* isConstant */ true, GlobalValue::PrivateLinkage,
2103097a140dSpatrick UndefValue::get(Int8Ty), F->getName() + ".ID");
2104097a140dSpatrick
2105097a140dSpatrick for (Use *U : ToBeReplacedStateMachineUses)
210673471bf0Spatrick U->set(ConstantExpr::getPointerBitCastOrAddrSpaceCast(
210773471bf0Spatrick ID, U->get()->getType()));
2108097a140dSpatrick
2109097a140dSpatrick ++NumOpenMPParallelRegionsReplacedInGPUStateMachine;
2110097a140dSpatrick
2111097a140dSpatrick Changed = true;
2112097a140dSpatrick }
2113097a140dSpatrick
2114097a140dSpatrick return Changed;
2115097a140dSpatrick }
2116097a140dSpatrick
2117097a140dSpatrick /// Abstract Attribute for tracking ICV values.
2118097a140dSpatrick struct AAICVTracker : public StateWrapper<BooleanState, AbstractAttribute> {
2119097a140dSpatrick using Base = StateWrapper<BooleanState, AbstractAttribute>;
AAICVTracker__anon4b47cb5e0111::AAICVTracker2120097a140dSpatrick AAICVTracker(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
2121097a140dSpatrick
initialize__anon4b47cb5e0111::AAICVTracker212273471bf0Spatrick void initialize(Attributor &A) override {
212373471bf0Spatrick Function *F = getAnchorScope();
212473471bf0Spatrick if (!F || !A.isFunctionIPOAmendable(*F))
212573471bf0Spatrick indicatePessimisticFixpoint();
212673471bf0Spatrick }
212773471bf0Spatrick
2128097a140dSpatrick /// Returns true if value is assumed to be tracked.
isAssumedTracked__anon4b47cb5e0111::AAICVTracker2129097a140dSpatrick bool isAssumedTracked() const { return getAssumed(); }
2130097a140dSpatrick
2131097a140dSpatrick /// Returns true if value is known to be tracked.
isKnownTracked__anon4b47cb5e0111::AAICVTracker2132097a140dSpatrick bool isKnownTracked() const { return getAssumed(); }
2133097a140dSpatrick
2134097a140dSpatrick /// Create an abstract attribute biew for the position \p IRP.
2135097a140dSpatrick static AAICVTracker &createForPosition(const IRPosition &IRP, Attributor &A);
2136097a140dSpatrick
2137097a140dSpatrick /// Return the value with which \p I can be replaced for specific \p ICV.
getReplacementValue__anon4b47cb5e0111::AAICVTracker2138*d415bd75Srobert virtual std::optional<Value *> getReplacementValue(InternalControlVar ICV,
213973471bf0Spatrick const Instruction *I,
214073471bf0Spatrick Attributor &A) const {
2141*d415bd75Srobert return std::nullopt;
214273471bf0Spatrick }
214373471bf0Spatrick
214473471bf0Spatrick /// Return an assumed unique ICV value if a single candidate is found. If
2145*d415bd75Srobert /// there cannot be one, return a nullptr. If it is not clear yet, return
2146*d415bd75Srobert /// std::nullopt.
2147*d415bd75Srobert virtual std::optional<Value *>
214873471bf0Spatrick getUniqueReplacementValue(InternalControlVar ICV) const = 0;
214973471bf0Spatrick
215073471bf0Spatrick // Currently only nthreads is being tracked.
215173471bf0Spatrick // this array will only grow with time.
215273471bf0Spatrick InternalControlVar TrackableICVs[1] = {ICV_nthreads};
2153097a140dSpatrick
2154097a140dSpatrick /// See AbstractAttribute::getName()
getName__anon4b47cb5e0111::AAICVTracker2155097a140dSpatrick const std::string getName() const override { return "AAICVTracker"; }
2156097a140dSpatrick
2157097a140dSpatrick /// See AbstractAttribute::getIdAddr()
getIdAddr__anon4b47cb5e0111::AAICVTracker2158097a140dSpatrick const char *getIdAddr() const override { return &ID; }
2159097a140dSpatrick
2160097a140dSpatrick /// This function should return true if the type of the \p AA is AAICVTracker
classof__anon4b47cb5e0111::AAICVTracker2161097a140dSpatrick static bool classof(const AbstractAttribute *AA) {
2162097a140dSpatrick return (AA->getIdAddr() == &ID);
2163097a140dSpatrick }
2164097a140dSpatrick
2165097a140dSpatrick static const char ID;
2166097a140dSpatrick };
2167097a140dSpatrick
2168097a140dSpatrick struct AAICVTrackerFunction : public AAICVTracker {
AAICVTrackerFunction__anon4b47cb5e0111::AAICVTrackerFunction2169097a140dSpatrick AAICVTrackerFunction(const IRPosition &IRP, Attributor &A)
2170097a140dSpatrick : AAICVTracker(IRP, A) {}
2171097a140dSpatrick
2172097a140dSpatrick // FIXME: come up with better string.
getAsStr__anon4b47cb5e0111::AAICVTrackerFunction217373471bf0Spatrick const std::string getAsStr() const override { return "ICVTrackerFunction"; }
2174097a140dSpatrick
2175097a140dSpatrick // FIXME: come up with some stats.
trackStatistics__anon4b47cb5e0111::AAICVTrackerFunction2176097a140dSpatrick void trackStatistics() const override {}
2177097a140dSpatrick
217873471bf0Spatrick /// We don't manifest anything for this AA.
manifest__anon4b47cb5e0111::AAICVTrackerFunction2179097a140dSpatrick ChangeStatus manifest(Attributor &A) override {
218073471bf0Spatrick return ChangeStatus::UNCHANGED;
2181097a140dSpatrick }
2182097a140dSpatrick
2183097a140dSpatrick // Map of ICV to their values at specific program point.
218473471bf0Spatrick EnumeratedArray<DenseMap<Instruction *, Value *>, InternalControlVar,
2185097a140dSpatrick InternalControlVar::ICV___last>
218673471bf0Spatrick ICVReplacementValuesMap;
2187097a140dSpatrick
updateImpl__anon4b47cb5e0111::AAICVTrackerFunction2188097a140dSpatrick ChangeStatus updateImpl(Attributor &A) override {
2189097a140dSpatrick ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
2190097a140dSpatrick
2191097a140dSpatrick Function *F = getAnchorScope();
2192097a140dSpatrick
2193097a140dSpatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2194097a140dSpatrick
2195097a140dSpatrick for (InternalControlVar ICV : TrackableICVs) {
2196097a140dSpatrick auto &SetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Setter];
2197097a140dSpatrick
219873471bf0Spatrick auto &ValuesMap = ICVReplacementValuesMap[ICV];
2199097a140dSpatrick auto TrackValues = [&](Use &U, Function &) {
2200097a140dSpatrick CallInst *CI = OpenMPOpt::getCallIfRegularCall(U);
2201097a140dSpatrick if (!CI)
2202097a140dSpatrick return false;
2203097a140dSpatrick
2204097a140dSpatrick // FIXME: handle setters with more that 1 arguments.
2205097a140dSpatrick /// Track new value.
220673471bf0Spatrick if (ValuesMap.insert(std::make_pair(CI, CI->getArgOperand(0))).second)
2207097a140dSpatrick HasChanged = ChangeStatus::CHANGED;
2208097a140dSpatrick
2209097a140dSpatrick return false;
2210097a140dSpatrick };
2211097a140dSpatrick
221273471bf0Spatrick auto CallCheck = [&](Instruction &I) {
2213*d415bd75Srobert std::optional<Value *> ReplVal = getValueForCall(A, I, ICV);
2214*d415bd75Srobert if (ReplVal && ValuesMap.insert(std::make_pair(&I, *ReplVal)).second)
221573471bf0Spatrick HasChanged = ChangeStatus::CHANGED;
221673471bf0Spatrick
221773471bf0Spatrick return true;
221873471bf0Spatrick };
221973471bf0Spatrick
222073471bf0Spatrick // Track all changes of an ICV.
2221097a140dSpatrick SetterRFI.foreachUse(TrackValues, F);
222273471bf0Spatrick
222373471bf0Spatrick bool UsedAssumedInformation = false;
222473471bf0Spatrick A.checkForAllInstructions(CallCheck, *this, {Instruction::Call},
222573471bf0Spatrick UsedAssumedInformation,
222673471bf0Spatrick /* CheckBBLivenessOnly */ true);
222773471bf0Spatrick
222873471bf0Spatrick /// TODO: Figure out a way to avoid adding entry in
222973471bf0Spatrick /// ICVReplacementValuesMap
223073471bf0Spatrick Instruction *Entry = &F->getEntryBlock().front();
223173471bf0Spatrick if (HasChanged == ChangeStatus::CHANGED && !ValuesMap.count(Entry))
223273471bf0Spatrick ValuesMap.insert(std::make_pair(Entry, nullptr));
2233097a140dSpatrick }
2234097a140dSpatrick
2235097a140dSpatrick return HasChanged;
2236097a140dSpatrick }
2237097a140dSpatrick
2238*d415bd75Srobert /// Helper to check if \p I is a call and get the value for it if it is
223973471bf0Spatrick /// unique.
getValueForCall__anon4b47cb5e0111::AAICVTrackerFunction2240*d415bd75Srobert std::optional<Value *> getValueForCall(Attributor &A, const Instruction &I,
224173471bf0Spatrick InternalControlVar &ICV) const {
2242097a140dSpatrick
2243*d415bd75Srobert const auto *CB = dyn_cast<CallBase>(&I);
224473471bf0Spatrick if (!CB || CB->hasFnAttr("no_openmp") ||
224573471bf0Spatrick CB->hasFnAttr("no_openmp_routines"))
2246*d415bd75Srobert return std::nullopt;
224773471bf0Spatrick
2248097a140dSpatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2249097a140dSpatrick auto &GetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Getter];
225073471bf0Spatrick auto &SetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Setter];
225173471bf0Spatrick Function *CalledFunction = CB->getCalledFunction();
2252097a140dSpatrick
225373471bf0Spatrick // Indirect call, assume ICV changes.
225473471bf0Spatrick if (CalledFunction == nullptr)
225573471bf0Spatrick return nullptr;
225673471bf0Spatrick if (CalledFunction == GetterRFI.Declaration)
2257*d415bd75Srobert return std::nullopt;
225873471bf0Spatrick if (CalledFunction == SetterRFI.Declaration) {
2259*d415bd75Srobert if (ICVReplacementValuesMap[ICV].count(&I))
2260*d415bd75Srobert return ICVReplacementValuesMap[ICV].lookup(&I);
226173471bf0Spatrick
226273471bf0Spatrick return nullptr;
226373471bf0Spatrick }
226473471bf0Spatrick
226573471bf0Spatrick // Since we don't know, assume it changes the ICV.
226673471bf0Spatrick if (CalledFunction->isDeclaration())
226773471bf0Spatrick return nullptr;
226873471bf0Spatrick
226973471bf0Spatrick const auto &ICVTrackingAA = A.getAAFor<AAICVTracker>(
227073471bf0Spatrick *this, IRPosition::callsite_returned(*CB), DepClassTy::REQUIRED);
227173471bf0Spatrick
2272*d415bd75Srobert if (ICVTrackingAA.isAssumedTracked()) {
2273*d415bd75Srobert std::optional<Value *> URV = ICVTrackingAA.getUniqueReplacementValue(ICV);
2274*d415bd75Srobert if (!URV || (*URV && AA::isValidAtPosition(AA::ValueAndContext(**URV, I),
2275*d415bd75Srobert OMPInfoCache)))
2276*d415bd75Srobert return URV;
2277*d415bd75Srobert }
227873471bf0Spatrick
227973471bf0Spatrick // If we don't know, assume it changes.
228073471bf0Spatrick return nullptr;
228173471bf0Spatrick }
228273471bf0Spatrick
2283*d415bd75Srobert // We don't check unique value for a function, so return std::nullopt.
2284*d415bd75Srobert std::optional<Value *>
getUniqueReplacementValue__anon4b47cb5e0111::AAICVTrackerFunction228573471bf0Spatrick getUniqueReplacementValue(InternalControlVar ICV) const override {
2286*d415bd75Srobert return std::nullopt;
228773471bf0Spatrick }
228873471bf0Spatrick
228973471bf0Spatrick /// Return the value with which \p I can be replaced for specific \p ICV.
getReplacementValue__anon4b47cb5e0111::AAICVTrackerFunction2290*d415bd75Srobert std::optional<Value *> getReplacementValue(InternalControlVar ICV,
229173471bf0Spatrick const Instruction *I,
229273471bf0Spatrick Attributor &A) const override {
229373471bf0Spatrick const auto &ValuesMap = ICVReplacementValuesMap[ICV];
229473471bf0Spatrick if (ValuesMap.count(I))
229573471bf0Spatrick return ValuesMap.lookup(I);
229673471bf0Spatrick
229773471bf0Spatrick SmallVector<const Instruction *, 16> Worklist;
229873471bf0Spatrick SmallPtrSet<const Instruction *, 16> Visited;
229973471bf0Spatrick Worklist.push_back(I);
230073471bf0Spatrick
2301*d415bd75Srobert std::optional<Value *> ReplVal;
230273471bf0Spatrick
230373471bf0Spatrick while (!Worklist.empty()) {
230473471bf0Spatrick const Instruction *CurrInst = Worklist.pop_back_val();
230573471bf0Spatrick if (!Visited.insert(CurrInst).second)
2306097a140dSpatrick continue;
2307097a140dSpatrick
230873471bf0Spatrick const BasicBlock *CurrBB = CurrInst->getParent();
230973471bf0Spatrick
231073471bf0Spatrick // Go up and look for all potential setters/calls that might change the
231173471bf0Spatrick // ICV.
231273471bf0Spatrick while ((CurrInst = CurrInst->getPrevNode())) {
231373471bf0Spatrick if (ValuesMap.count(CurrInst)) {
2314*d415bd75Srobert std::optional<Value *> NewReplVal = ValuesMap.lookup(CurrInst);
231573471bf0Spatrick // Unknown value, track new.
2316*d415bd75Srobert if (!ReplVal) {
231773471bf0Spatrick ReplVal = NewReplVal;
231873471bf0Spatrick break;
231973471bf0Spatrick }
232073471bf0Spatrick
232173471bf0Spatrick // If we found a new value, we can't know the icv value anymore.
2322*d415bd75Srobert if (NewReplVal)
232373471bf0Spatrick if (ReplVal != NewReplVal)
2324097a140dSpatrick return nullptr;
2325097a140dSpatrick
232673471bf0Spatrick break;
2327097a140dSpatrick }
2328097a140dSpatrick
2329*d415bd75Srobert std::optional<Value *> NewReplVal = getValueForCall(A, *CurrInst, ICV);
2330*d415bd75Srobert if (!NewReplVal)
233173471bf0Spatrick continue;
233273471bf0Spatrick
233373471bf0Spatrick // Unknown value, track new.
2334*d415bd75Srobert if (!ReplVal) {
233573471bf0Spatrick ReplVal = NewReplVal;
233673471bf0Spatrick break;
2337097a140dSpatrick }
2338097a140dSpatrick
233973471bf0Spatrick // if (NewReplVal.hasValue())
234073471bf0Spatrick // We found a new value, we can't know the icv value anymore.
234173471bf0Spatrick if (ReplVal != NewReplVal)
2342097a140dSpatrick return nullptr;
2343097a140dSpatrick }
234473471bf0Spatrick
234573471bf0Spatrick // If we are in the same BB and we have a value, we are done.
2346*d415bd75Srobert if (CurrBB == I->getParent() && ReplVal)
234773471bf0Spatrick return ReplVal;
234873471bf0Spatrick
234973471bf0Spatrick // Go through all predecessors and add terminators for analysis.
235073471bf0Spatrick for (const BasicBlock *Pred : predecessors(CurrBB))
235173471bf0Spatrick if (const Instruction *Terminator = Pred->getTerminator())
235273471bf0Spatrick Worklist.push_back(Terminator);
235373471bf0Spatrick }
235473471bf0Spatrick
235573471bf0Spatrick return ReplVal;
235673471bf0Spatrick }
2357097a140dSpatrick };
235873471bf0Spatrick
235973471bf0Spatrick struct AAICVTrackerFunctionReturned : AAICVTracker {
AAICVTrackerFunctionReturned__anon4b47cb5e0111::AAICVTrackerFunctionReturned236073471bf0Spatrick AAICVTrackerFunctionReturned(const IRPosition &IRP, Attributor &A)
236173471bf0Spatrick : AAICVTracker(IRP, A) {}
236273471bf0Spatrick
236373471bf0Spatrick // FIXME: come up with better string.
getAsStr__anon4b47cb5e0111::AAICVTrackerFunctionReturned236473471bf0Spatrick const std::string getAsStr() const override {
236573471bf0Spatrick return "ICVTrackerFunctionReturned";
236673471bf0Spatrick }
236773471bf0Spatrick
236873471bf0Spatrick // FIXME: come up with some stats.
trackStatistics__anon4b47cb5e0111::AAICVTrackerFunctionReturned236973471bf0Spatrick void trackStatistics() const override {}
237073471bf0Spatrick
237173471bf0Spatrick /// We don't manifest anything for this AA.
manifest__anon4b47cb5e0111::AAICVTrackerFunctionReturned237273471bf0Spatrick ChangeStatus manifest(Attributor &A) override {
237373471bf0Spatrick return ChangeStatus::UNCHANGED;
237473471bf0Spatrick }
237573471bf0Spatrick
237673471bf0Spatrick // Map of ICV to their values at specific program point.
2377*d415bd75Srobert EnumeratedArray<std::optional<Value *>, InternalControlVar,
237873471bf0Spatrick InternalControlVar::ICV___last>
237973471bf0Spatrick ICVReplacementValuesMap;
238073471bf0Spatrick
238173471bf0Spatrick /// Return the value with which \p I can be replaced for specific \p ICV.
2382*d415bd75Srobert std::optional<Value *>
getUniqueReplacementValue__anon4b47cb5e0111::AAICVTrackerFunctionReturned238373471bf0Spatrick getUniqueReplacementValue(InternalControlVar ICV) const override {
238473471bf0Spatrick return ICVReplacementValuesMap[ICV];
238573471bf0Spatrick }
238673471bf0Spatrick
updateImpl__anon4b47cb5e0111::AAICVTrackerFunctionReturned238773471bf0Spatrick ChangeStatus updateImpl(Attributor &A) override {
238873471bf0Spatrick ChangeStatus Changed = ChangeStatus::UNCHANGED;
238973471bf0Spatrick const auto &ICVTrackingAA = A.getAAFor<AAICVTracker>(
239073471bf0Spatrick *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED);
239173471bf0Spatrick
239273471bf0Spatrick if (!ICVTrackingAA.isAssumedTracked())
239373471bf0Spatrick return indicatePessimisticFixpoint();
239473471bf0Spatrick
239573471bf0Spatrick for (InternalControlVar ICV : TrackableICVs) {
2396*d415bd75Srobert std::optional<Value *> &ReplVal = ICVReplacementValuesMap[ICV];
2397*d415bd75Srobert std::optional<Value *> UniqueICVValue;
239873471bf0Spatrick
239973471bf0Spatrick auto CheckReturnInst = [&](Instruction &I) {
2400*d415bd75Srobert std::optional<Value *> NewReplVal =
240173471bf0Spatrick ICVTrackingAA.getReplacementValue(ICV, &I, A);
240273471bf0Spatrick
240373471bf0Spatrick // If we found a second ICV value there is no unique returned value.
2404*d415bd75Srobert if (UniqueICVValue && UniqueICVValue != NewReplVal)
240573471bf0Spatrick return false;
240673471bf0Spatrick
240773471bf0Spatrick UniqueICVValue = NewReplVal;
240873471bf0Spatrick
240973471bf0Spatrick return true;
241073471bf0Spatrick };
241173471bf0Spatrick
241273471bf0Spatrick bool UsedAssumedInformation = false;
241373471bf0Spatrick if (!A.checkForAllInstructions(CheckReturnInst, *this, {Instruction::Ret},
241473471bf0Spatrick UsedAssumedInformation,
241573471bf0Spatrick /* CheckBBLivenessOnly */ true))
241673471bf0Spatrick UniqueICVValue = nullptr;
241773471bf0Spatrick
241873471bf0Spatrick if (UniqueICVValue == ReplVal)
241973471bf0Spatrick continue;
242073471bf0Spatrick
242173471bf0Spatrick ReplVal = UniqueICVValue;
242273471bf0Spatrick Changed = ChangeStatus::CHANGED;
242373471bf0Spatrick }
242473471bf0Spatrick
242573471bf0Spatrick return Changed;
242673471bf0Spatrick }
242773471bf0Spatrick };
242873471bf0Spatrick
242973471bf0Spatrick struct AAICVTrackerCallSite : AAICVTracker {
AAICVTrackerCallSite__anon4b47cb5e0111::AAICVTrackerCallSite243073471bf0Spatrick AAICVTrackerCallSite(const IRPosition &IRP, Attributor &A)
243173471bf0Spatrick : AAICVTracker(IRP, A) {}
243273471bf0Spatrick
initialize__anon4b47cb5e0111::AAICVTrackerCallSite243373471bf0Spatrick void initialize(Attributor &A) override {
243473471bf0Spatrick Function *F = getAnchorScope();
243573471bf0Spatrick if (!F || !A.isFunctionIPOAmendable(*F))
243673471bf0Spatrick indicatePessimisticFixpoint();
243773471bf0Spatrick
243873471bf0Spatrick // We only initialize this AA for getters, so we need to know which ICV it
243973471bf0Spatrick // gets.
244073471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
244173471bf0Spatrick for (InternalControlVar ICV : TrackableICVs) {
244273471bf0Spatrick auto ICVInfo = OMPInfoCache.ICVs[ICV];
244373471bf0Spatrick auto &Getter = OMPInfoCache.RFIs[ICVInfo.Getter];
244473471bf0Spatrick if (Getter.Declaration == getAssociatedFunction()) {
244573471bf0Spatrick AssociatedICV = ICVInfo.Kind;
244673471bf0Spatrick return;
244773471bf0Spatrick }
244873471bf0Spatrick }
244973471bf0Spatrick
245073471bf0Spatrick /// Unknown ICV.
245173471bf0Spatrick indicatePessimisticFixpoint();
245273471bf0Spatrick }
245373471bf0Spatrick
manifest__anon4b47cb5e0111::AAICVTrackerCallSite245473471bf0Spatrick ChangeStatus manifest(Attributor &A) override {
2455*d415bd75Srobert if (!ReplVal || !*ReplVal)
245673471bf0Spatrick return ChangeStatus::UNCHANGED;
245773471bf0Spatrick
2458*d415bd75Srobert A.changeAfterManifest(IRPosition::inst(*getCtxI()), **ReplVal);
245973471bf0Spatrick A.deleteAfterManifest(*getCtxI());
246073471bf0Spatrick
246173471bf0Spatrick return ChangeStatus::CHANGED;
246273471bf0Spatrick }
246373471bf0Spatrick
246473471bf0Spatrick // FIXME: come up with better string.
getAsStr__anon4b47cb5e0111::AAICVTrackerCallSite246573471bf0Spatrick const std::string getAsStr() const override { return "ICVTrackerCallSite"; }
246673471bf0Spatrick
246773471bf0Spatrick // FIXME: come up with some stats.
trackStatistics__anon4b47cb5e0111::AAICVTrackerCallSite246873471bf0Spatrick void trackStatistics() const override {}
246973471bf0Spatrick
247073471bf0Spatrick InternalControlVar AssociatedICV;
2471*d415bd75Srobert std::optional<Value *> ReplVal;
247273471bf0Spatrick
updateImpl__anon4b47cb5e0111::AAICVTrackerCallSite247373471bf0Spatrick ChangeStatus updateImpl(Attributor &A) override {
247473471bf0Spatrick const auto &ICVTrackingAA = A.getAAFor<AAICVTracker>(
247573471bf0Spatrick *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED);
247673471bf0Spatrick
247773471bf0Spatrick // We don't have any information, so we assume it changes the ICV.
247873471bf0Spatrick if (!ICVTrackingAA.isAssumedTracked())
247973471bf0Spatrick return indicatePessimisticFixpoint();
248073471bf0Spatrick
2481*d415bd75Srobert std::optional<Value *> NewReplVal =
248273471bf0Spatrick ICVTrackingAA.getReplacementValue(AssociatedICV, getCtxI(), A);
248373471bf0Spatrick
248473471bf0Spatrick if (ReplVal == NewReplVal)
248573471bf0Spatrick return ChangeStatus::UNCHANGED;
248673471bf0Spatrick
248773471bf0Spatrick ReplVal = NewReplVal;
248873471bf0Spatrick return ChangeStatus::CHANGED;
248973471bf0Spatrick }
249073471bf0Spatrick
249173471bf0Spatrick // Return the value with which associated value can be replaced for specific
249273471bf0Spatrick // \p ICV.
2493*d415bd75Srobert std::optional<Value *>
getUniqueReplacementValue__anon4b47cb5e0111::AAICVTrackerCallSite249473471bf0Spatrick getUniqueReplacementValue(InternalControlVar ICV) const override {
249573471bf0Spatrick return ReplVal;
249673471bf0Spatrick }
249773471bf0Spatrick };
249873471bf0Spatrick
249973471bf0Spatrick struct AAICVTrackerCallSiteReturned : AAICVTracker {
AAICVTrackerCallSiteReturned__anon4b47cb5e0111::AAICVTrackerCallSiteReturned250073471bf0Spatrick AAICVTrackerCallSiteReturned(const IRPosition &IRP, Attributor &A)
250173471bf0Spatrick : AAICVTracker(IRP, A) {}
250273471bf0Spatrick
250373471bf0Spatrick // FIXME: come up with better string.
getAsStr__anon4b47cb5e0111::AAICVTrackerCallSiteReturned250473471bf0Spatrick const std::string getAsStr() const override {
250573471bf0Spatrick return "ICVTrackerCallSiteReturned";
250673471bf0Spatrick }
250773471bf0Spatrick
250873471bf0Spatrick // FIXME: come up with some stats.
trackStatistics__anon4b47cb5e0111::AAICVTrackerCallSiteReturned250973471bf0Spatrick void trackStatistics() const override {}
251073471bf0Spatrick
251173471bf0Spatrick /// We don't manifest anything for this AA.
manifest__anon4b47cb5e0111::AAICVTrackerCallSiteReturned251273471bf0Spatrick ChangeStatus manifest(Attributor &A) override {
251373471bf0Spatrick return ChangeStatus::UNCHANGED;
251473471bf0Spatrick }
251573471bf0Spatrick
251673471bf0Spatrick // Map of ICV to their values at specific program point.
2517*d415bd75Srobert EnumeratedArray<std::optional<Value *>, InternalControlVar,
251873471bf0Spatrick InternalControlVar::ICV___last>
251973471bf0Spatrick ICVReplacementValuesMap;
252073471bf0Spatrick
252173471bf0Spatrick /// Return the value with which associated value can be replaced for specific
252273471bf0Spatrick /// \p ICV.
2523*d415bd75Srobert std::optional<Value *>
getUniqueReplacementValue__anon4b47cb5e0111::AAICVTrackerCallSiteReturned252473471bf0Spatrick getUniqueReplacementValue(InternalControlVar ICV) const override {
252573471bf0Spatrick return ICVReplacementValuesMap[ICV];
252673471bf0Spatrick }
252773471bf0Spatrick
updateImpl__anon4b47cb5e0111::AAICVTrackerCallSiteReturned252873471bf0Spatrick ChangeStatus updateImpl(Attributor &A) override {
252973471bf0Spatrick ChangeStatus Changed = ChangeStatus::UNCHANGED;
253073471bf0Spatrick const auto &ICVTrackingAA = A.getAAFor<AAICVTracker>(
253173471bf0Spatrick *this, IRPosition::returned(*getAssociatedFunction()),
253273471bf0Spatrick DepClassTy::REQUIRED);
253373471bf0Spatrick
253473471bf0Spatrick // We don't have any information, so we assume it changes the ICV.
253573471bf0Spatrick if (!ICVTrackingAA.isAssumedTracked())
253673471bf0Spatrick return indicatePessimisticFixpoint();
253773471bf0Spatrick
253873471bf0Spatrick for (InternalControlVar ICV : TrackableICVs) {
2539*d415bd75Srobert std::optional<Value *> &ReplVal = ICVReplacementValuesMap[ICV];
2540*d415bd75Srobert std::optional<Value *> NewReplVal =
254173471bf0Spatrick ICVTrackingAA.getUniqueReplacementValue(ICV);
254273471bf0Spatrick
254373471bf0Spatrick if (ReplVal == NewReplVal)
254473471bf0Spatrick continue;
254573471bf0Spatrick
254673471bf0Spatrick ReplVal = NewReplVal;
254773471bf0Spatrick Changed = ChangeStatus::CHANGED;
254873471bf0Spatrick }
254973471bf0Spatrick return Changed;
255073471bf0Spatrick }
255173471bf0Spatrick };
255273471bf0Spatrick
255373471bf0Spatrick struct AAExecutionDomainFunction : public AAExecutionDomain {
AAExecutionDomainFunction__anon4b47cb5e0111::AAExecutionDomainFunction255473471bf0Spatrick AAExecutionDomainFunction(const IRPosition &IRP, Attributor &A)
255573471bf0Spatrick : AAExecutionDomain(IRP, A) {}
255673471bf0Spatrick
~AAExecutionDomainFunction__anon4b47cb5e0111::AAExecutionDomainFunction2557*d415bd75Srobert ~AAExecutionDomainFunction() {
2558*d415bd75Srobert delete RPOT;
2559*d415bd75Srobert }
2560*d415bd75Srobert
initialize__anon4b47cb5e0111::AAExecutionDomainFunction2561*d415bd75Srobert void initialize(Attributor &A) override {
2562*d415bd75Srobert if (getAnchorScope()->isDeclaration()) {
2563*d415bd75Srobert indicatePessimisticFixpoint();
2564*d415bd75Srobert return;
2565*d415bd75Srobert }
2566*d415bd75Srobert RPOT = new ReversePostOrderTraversal<Function *>(getAnchorScope());
2567*d415bd75Srobert }
2568*d415bd75Srobert
getAsStr__anon4b47cb5e0111::AAExecutionDomainFunction256973471bf0Spatrick const std::string getAsStr() const override {
2570*d415bd75Srobert unsigned TotalBlocks = 0, InitialThreadBlocks = 0;
2571*d415bd75Srobert for (auto &It : BEDMap) {
2572*d415bd75Srobert TotalBlocks++;
2573*d415bd75Srobert InitialThreadBlocks += It.getSecond().IsExecutedByInitialThreadOnly;
2574*d415bd75Srobert }
2575*d415bd75Srobert return "[AAExecutionDomain] " + std::to_string(InitialThreadBlocks) + "/" +
2576*d415bd75Srobert std::to_string(TotalBlocks) + " executed by initial thread only";
257773471bf0Spatrick }
257873471bf0Spatrick
257973471bf0Spatrick /// See AbstractAttribute::trackStatistics().
trackStatistics__anon4b47cb5e0111::AAExecutionDomainFunction258073471bf0Spatrick void trackStatistics() const override {}
258173471bf0Spatrick
manifest__anon4b47cb5e0111::AAExecutionDomainFunction258273471bf0Spatrick ChangeStatus manifest(Attributor &A) override {
258373471bf0Spatrick LLVM_DEBUG({
2584*d415bd75Srobert for (const BasicBlock &BB : *getAnchorScope()) {
2585*d415bd75Srobert if (!isExecutedByInitialThreadOnly(BB))
2586*d415bd75Srobert continue;
258773471bf0Spatrick dbgs() << TAG << " Basic block @" << getAnchorScope()->getName() << " "
2588*d415bd75Srobert << BB.getName() << " is executed by a single thread.\n";
2589*d415bd75Srobert }
259073471bf0Spatrick });
2591*d415bd75Srobert
2592*d415bd75Srobert ChangeStatus Changed = ChangeStatus::UNCHANGED;
2593*d415bd75Srobert
2594*d415bd75Srobert if (DisableOpenMPOptBarrierElimination)
2595*d415bd75Srobert return Changed;
2596*d415bd75Srobert
2597*d415bd75Srobert SmallPtrSet<CallBase *, 16> DeletedBarriers;
2598*d415bd75Srobert auto HandleAlignedBarrier = [&](CallBase *CB) {
2599*d415bd75Srobert const ExecutionDomainTy &ED = CEDMap[CB];
2600*d415bd75Srobert if (!ED.IsReachedFromAlignedBarrierOnly ||
2601*d415bd75Srobert ED.EncounteredNonLocalSideEffect)
2602*d415bd75Srobert return;
2603*d415bd75Srobert
2604*d415bd75Srobert // We can remove this barrier, if it is one, or all aligned barriers
2605*d415bd75Srobert // reaching the kernel end. In the latter case we can transitively work
2606*d415bd75Srobert // our way back until we find a barrier that guards a side-effect if we
2607*d415bd75Srobert // are dealing with the kernel end here.
2608*d415bd75Srobert if (CB) {
2609*d415bd75Srobert DeletedBarriers.insert(CB);
2610*d415bd75Srobert A.deleteAfterManifest(*CB);
2611*d415bd75Srobert ++NumBarriersEliminated;
2612*d415bd75Srobert Changed = ChangeStatus::CHANGED;
2613*d415bd75Srobert } else if (!ED.AlignedBarriers.empty()) {
2614*d415bd75Srobert NumBarriersEliminated += ED.AlignedBarriers.size();
2615*d415bd75Srobert Changed = ChangeStatus::CHANGED;
2616*d415bd75Srobert SmallVector<CallBase *> Worklist(ED.AlignedBarriers.begin(),
2617*d415bd75Srobert ED.AlignedBarriers.end());
2618*d415bd75Srobert SmallSetVector<CallBase *, 16> Visited;
2619*d415bd75Srobert while (!Worklist.empty()) {
2620*d415bd75Srobert CallBase *LastCB = Worklist.pop_back_val();
2621*d415bd75Srobert if (!Visited.insert(LastCB))
2622*d415bd75Srobert continue;
2623*d415bd75Srobert if (!DeletedBarriers.count(LastCB)) {
2624*d415bd75Srobert A.deleteAfterManifest(*LastCB);
2625*d415bd75Srobert continue;
2626*d415bd75Srobert }
2627*d415bd75Srobert // The final aligned barrier (LastCB) reaching the kernel end was
2628*d415bd75Srobert // removed already. This means we can go one step further and remove
2629*d415bd75Srobert // the barriers encoutered last before (LastCB).
2630*d415bd75Srobert const ExecutionDomainTy &LastED = CEDMap[LastCB];
2631*d415bd75Srobert Worklist.append(LastED.AlignedBarriers.begin(),
2632*d415bd75Srobert LastED.AlignedBarriers.end());
2633*d415bd75Srobert }
263473471bf0Spatrick }
263573471bf0Spatrick
2636*d415bd75Srobert // If we actually eliminated a barrier we need to eliminate the associated
2637*d415bd75Srobert // llvm.assumes as well to avoid creating UB.
2638*d415bd75Srobert if (!ED.EncounteredAssumes.empty() && (CB || !ED.AlignedBarriers.empty()))
2639*d415bd75Srobert for (auto *AssumeCB : ED.EncounteredAssumes)
2640*d415bd75Srobert A.deleteAfterManifest(*AssumeCB);
264173471bf0Spatrick };
264273471bf0Spatrick
2643*d415bd75Srobert for (auto *CB : AlignedBarriers)
2644*d415bd75Srobert HandleAlignedBarrier(CB);
264573471bf0Spatrick
264673471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2647*d415bd75Srobert // Handle the "kernel end barrier" for kernels too.
2648*d415bd75Srobert if (OMPInfoCache.Kernels.count(getAnchorScope()))
2649*d415bd75Srobert HandleAlignedBarrier(nullptr);
265073471bf0Spatrick
2651*d415bd75Srobert return Changed;
2652*d415bd75Srobert }
2653*d415bd75Srobert
2654*d415bd75Srobert /// Merge barrier and assumption information from \p PredED into the successor
2655*d415bd75Srobert /// \p ED.
2656*d415bd75Srobert void
2657*d415bd75Srobert mergeInPredecessorBarriersAndAssumptions(Attributor &A, ExecutionDomainTy &ED,
2658*d415bd75Srobert const ExecutionDomainTy &PredED);
2659*d415bd75Srobert
2660*d415bd75Srobert /// Merge all information from \p PredED into the successor \p ED. If
2661*d415bd75Srobert /// \p InitialEdgeOnly is set, only the initial edge will enter the block
2662*d415bd75Srobert /// represented by \p ED from this predecessor.
2663*d415bd75Srobert void mergeInPredecessor(Attributor &A, ExecutionDomainTy &ED,
2664*d415bd75Srobert const ExecutionDomainTy &PredED,
2665*d415bd75Srobert bool InitialEdgeOnly = false);
2666*d415bd75Srobert
2667*d415bd75Srobert /// Accumulate information for the entry block in \p EntryBBED.
2668*d415bd75Srobert void handleEntryBB(Attributor &A, ExecutionDomainTy &EntryBBED);
2669*d415bd75Srobert
2670*d415bd75Srobert /// See AbstractAttribute::updateImpl.
2671*d415bd75Srobert ChangeStatus updateImpl(Attributor &A) override;
2672*d415bd75Srobert
2673*d415bd75Srobert /// Query interface, see AAExecutionDomain
2674*d415bd75Srobert ///{
isExecutedByInitialThreadOnly__anon4b47cb5e0111::AAExecutionDomainFunction2675*d415bd75Srobert bool isExecutedByInitialThreadOnly(const BasicBlock &BB) const override {
2676*d415bd75Srobert if (!isValidState())
2677*d415bd75Srobert return false;
2678*d415bd75Srobert return BEDMap.lookup(&BB).IsExecutedByInitialThreadOnly;
2679*d415bd75Srobert }
2680*d415bd75Srobert
isExecutedInAlignedRegion__anon4b47cb5e0111::AAExecutionDomainFunction2681*d415bd75Srobert bool isExecutedInAlignedRegion(Attributor &A,
2682*d415bd75Srobert const Instruction &I) const override {
2683*d415bd75Srobert assert(I.getFunction() == getAnchorScope() &&
2684*d415bd75Srobert "Instruction is out of scope!");
2685*d415bd75Srobert if (!isValidState())
2686*d415bd75Srobert return false;
2687*d415bd75Srobert
2688*d415bd75Srobert const Instruction *CurI;
2689*d415bd75Srobert
2690*d415bd75Srobert // Check forward until a call or the block end is reached.
2691*d415bd75Srobert CurI = &I;
2692*d415bd75Srobert do {
2693*d415bd75Srobert auto *CB = dyn_cast<CallBase>(CurI);
2694*d415bd75Srobert if (!CB)
2695*d415bd75Srobert continue;
2696*d415bd75Srobert if (CB != &I && AlignedBarriers.contains(const_cast<CallBase *>(CB))) {
2697*d415bd75Srobert break;
2698*d415bd75Srobert }
2699*d415bd75Srobert const auto &It = CEDMap.find(CB);
2700*d415bd75Srobert if (It == CEDMap.end())
2701*d415bd75Srobert continue;
2702*d415bd75Srobert if (!It->getSecond().IsReachingAlignedBarrierOnly)
2703*d415bd75Srobert return false;
2704*d415bd75Srobert break;
2705*d415bd75Srobert } while ((CurI = CurI->getNextNonDebugInstruction()));
2706*d415bd75Srobert
2707*d415bd75Srobert if (!CurI && !BEDMap.lookup(I.getParent()).IsReachingAlignedBarrierOnly)
2708*d415bd75Srobert return false;
2709*d415bd75Srobert
2710*d415bd75Srobert // Check backward until a call or the block beginning is reached.
2711*d415bd75Srobert CurI = &I;
2712*d415bd75Srobert do {
2713*d415bd75Srobert auto *CB = dyn_cast<CallBase>(CurI);
2714*d415bd75Srobert if (!CB)
2715*d415bd75Srobert continue;
2716*d415bd75Srobert if (CB != &I && AlignedBarriers.contains(const_cast<CallBase *>(CB))) {
2717*d415bd75Srobert break;
2718*d415bd75Srobert }
2719*d415bd75Srobert const auto &It = CEDMap.find(CB);
2720*d415bd75Srobert if (It == CEDMap.end())
2721*d415bd75Srobert continue;
2722*d415bd75Srobert if (!AA::isNoSyncInst(A, *CB, *this)) {
2723*d415bd75Srobert if (It->getSecond().IsReachedFromAlignedBarrierOnly) {
2724*d415bd75Srobert break;
2725*d415bd75Srobert }
2726*d415bd75Srobert return false;
2727*d415bd75Srobert }
2728*d415bd75Srobert
2729*d415bd75Srobert Function *Callee = CB->getCalledFunction();
2730*d415bd75Srobert if (!Callee || Callee->isDeclaration())
2731*d415bd75Srobert return false;
2732*d415bd75Srobert const auto &EDAA = A.getAAFor<AAExecutionDomain>(
2733*d415bd75Srobert *this, IRPosition::function(*Callee), DepClassTy::OPTIONAL);
2734*d415bd75Srobert if (!EDAA.getState().isValidState())
2735*d415bd75Srobert return false;
2736*d415bd75Srobert if (!EDAA.getFunctionExecutionDomain().IsReachedFromAlignedBarrierOnly)
2737*d415bd75Srobert return false;
2738*d415bd75Srobert break;
2739*d415bd75Srobert } while ((CurI = CurI->getPrevNonDebugInstruction()));
2740*d415bd75Srobert
2741*d415bd75Srobert if (!CurI &&
2742*d415bd75Srobert !llvm::all_of(
2743*d415bd75Srobert predecessors(I.getParent()), [&](const BasicBlock *PredBB) {
2744*d415bd75Srobert return BEDMap.lookup(PredBB).IsReachedFromAlignedBarrierOnly;
2745*d415bd75Srobert })) {
2746*d415bd75Srobert return false;
2747*d415bd75Srobert }
2748*d415bd75Srobert
2749*d415bd75Srobert // On neither traversal we found a anything but aligned barriers.
2750*d415bd75Srobert return true;
2751*d415bd75Srobert }
2752*d415bd75Srobert
getExecutionDomain__anon4b47cb5e0111::AAExecutionDomainFunction2753*d415bd75Srobert ExecutionDomainTy getExecutionDomain(const BasicBlock &BB) const override {
2754*d415bd75Srobert assert(isValidState() &&
2755*d415bd75Srobert "No request should be made against an invalid state!");
2756*d415bd75Srobert return BEDMap.lookup(&BB);
2757*d415bd75Srobert }
getExecutionDomain__anon4b47cb5e0111::AAExecutionDomainFunction2758*d415bd75Srobert ExecutionDomainTy getExecutionDomain(const CallBase &CB) const override {
2759*d415bd75Srobert assert(isValidState() &&
2760*d415bd75Srobert "No request should be made against an invalid state!");
2761*d415bd75Srobert return CEDMap.lookup(&CB);
2762*d415bd75Srobert }
getFunctionExecutionDomain__anon4b47cb5e0111::AAExecutionDomainFunction2763*d415bd75Srobert ExecutionDomainTy getFunctionExecutionDomain() const override {
2764*d415bd75Srobert assert(isValidState() &&
2765*d415bd75Srobert "No request should be made against an invalid state!");
2766*d415bd75Srobert return BEDMap.lookup(nullptr);
2767*d415bd75Srobert }
2768*d415bd75Srobert ///}
2769*d415bd75Srobert
2770*d415bd75Srobert // Check if the edge into the successor block contains a condition that only
2771*d415bd75Srobert // lets the main thread execute it.
isInitialThreadOnlyEdge__anon4b47cb5e0111::AAExecutionDomainFunction2772*d415bd75Srobert static bool isInitialThreadOnlyEdge(Attributor &A, BranchInst *Edge,
2773*d415bd75Srobert BasicBlock &SuccessorBB) {
277473471bf0Spatrick if (!Edge || !Edge->isConditional())
277573471bf0Spatrick return false;
2776*d415bd75Srobert if (Edge->getSuccessor(0) != &SuccessorBB)
277773471bf0Spatrick return false;
277873471bf0Spatrick
277973471bf0Spatrick auto *Cmp = dyn_cast<CmpInst>(Edge->getCondition());
278073471bf0Spatrick if (!Cmp || !Cmp->isTrueWhenEqual() || !Cmp->isEquality())
278173471bf0Spatrick return false;
278273471bf0Spatrick
278373471bf0Spatrick ConstantInt *C = dyn_cast<ConstantInt>(Cmp->getOperand(1));
278473471bf0Spatrick if (!C)
278573471bf0Spatrick return false;
278673471bf0Spatrick
278773471bf0Spatrick // Match: -1 == __kmpc_target_init (for non-SPMD kernels only!)
278873471bf0Spatrick if (C->isAllOnesValue()) {
278973471bf0Spatrick auto *CB = dyn_cast<CallBase>(Cmp->getOperand(0));
2790*d415bd75Srobert auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2791*d415bd75Srobert auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
279273471bf0Spatrick CB = CB ? OpenMPOpt::getCallIfRegularCall(*CB, &RFI) : nullptr;
279373471bf0Spatrick if (!CB)
279473471bf0Spatrick return false;
2795*d415bd75Srobert const int InitModeArgNo = 1;
2796*d415bd75Srobert auto *ModeCI = dyn_cast<ConstantInt>(CB->getOperand(InitModeArgNo));
2797*d415bd75Srobert return ModeCI && (ModeCI->getSExtValue() & OMP_TGT_EXEC_MODE_GENERIC);
2798*d415bd75Srobert }
2799*d415bd75Srobert
2800*d415bd75Srobert if (C->isZero()) {
2801*d415bd75Srobert // Match: 0 == llvm.nvvm.read.ptx.sreg.tid.x()
2802*d415bd75Srobert if (auto *II = dyn_cast<IntrinsicInst>(Cmp->getOperand(0)))
2803*d415bd75Srobert if (II->getIntrinsicID() == Intrinsic::nvvm_read_ptx_sreg_tid_x)
2804*d415bd75Srobert return true;
2805*d415bd75Srobert
2806*d415bd75Srobert // Match: 0 == llvm.amdgcn.workitem.id.x()
2807*d415bd75Srobert if (auto *II = dyn_cast<IntrinsicInst>(Cmp->getOperand(0)))
2808*d415bd75Srobert if (II->getIntrinsicID() == Intrinsic::amdgcn_workitem_id_x)
2809*d415bd75Srobert return true;
281073471bf0Spatrick }
281173471bf0Spatrick
281273471bf0Spatrick return false;
281373471bf0Spatrick };
281473471bf0Spatrick
2815*d415bd75Srobert /// Mapping containing information per block.
2816*d415bd75Srobert DenseMap<const BasicBlock *, ExecutionDomainTy> BEDMap;
2817*d415bd75Srobert DenseMap<const CallBase *, ExecutionDomainTy> CEDMap;
2818*d415bd75Srobert SmallSetVector<CallBase *, 16> AlignedBarriers;
281973471bf0Spatrick
2820*d415bd75Srobert ReversePostOrderTraversal<Function *> *RPOT = nullptr;
282173471bf0Spatrick };
282273471bf0Spatrick
mergeInPredecessorBarriersAndAssumptions(Attributor & A,ExecutionDomainTy & ED,const ExecutionDomainTy & PredED)2823*d415bd75Srobert void AAExecutionDomainFunction::mergeInPredecessorBarriersAndAssumptions(
2824*d415bd75Srobert Attributor &A, ExecutionDomainTy &ED, const ExecutionDomainTy &PredED) {
2825*d415bd75Srobert for (auto *EA : PredED.EncounteredAssumes)
2826*d415bd75Srobert ED.addAssumeInst(A, *EA);
2827*d415bd75Srobert
2828*d415bd75Srobert for (auto *AB : PredED.AlignedBarriers)
2829*d415bd75Srobert ED.addAlignedBarrier(A, *AB);
283073471bf0Spatrick }
283173471bf0Spatrick
mergeInPredecessor(Attributor & A,ExecutionDomainTy & ED,const ExecutionDomainTy & PredED,bool InitialEdgeOnly)2832*d415bd75Srobert void AAExecutionDomainFunction::mergeInPredecessor(
2833*d415bd75Srobert Attributor &A, ExecutionDomainTy &ED, const ExecutionDomainTy &PredED,
2834*d415bd75Srobert bool InitialEdgeOnly) {
2835*d415bd75Srobert ED.IsExecutedByInitialThreadOnly =
2836*d415bd75Srobert InitialEdgeOnly || (PredED.IsExecutedByInitialThreadOnly &&
2837*d415bd75Srobert ED.IsExecutedByInitialThreadOnly);
2838*d415bd75Srobert
2839*d415bd75Srobert ED.IsReachedFromAlignedBarrierOnly = ED.IsReachedFromAlignedBarrierOnly &&
2840*d415bd75Srobert PredED.IsReachedFromAlignedBarrierOnly;
2841*d415bd75Srobert ED.EncounteredNonLocalSideEffect =
2842*d415bd75Srobert ED.EncounteredNonLocalSideEffect | PredED.EncounteredNonLocalSideEffect;
2843*d415bd75Srobert if (ED.IsReachedFromAlignedBarrierOnly)
2844*d415bd75Srobert mergeInPredecessorBarriersAndAssumptions(A, ED, PredED);
2845*d415bd75Srobert else
2846*d415bd75Srobert ED.clearAssumeInstAndAlignedBarriers();
2847*d415bd75Srobert }
2848*d415bd75Srobert
handleEntryBB(Attributor & A,ExecutionDomainTy & EntryBBED)2849*d415bd75Srobert void AAExecutionDomainFunction::handleEntryBB(Attributor &A,
2850*d415bd75Srobert ExecutionDomainTy &EntryBBED) {
2851*d415bd75Srobert SmallVector<ExecutionDomainTy> PredExecDomains;
2852*d415bd75Srobert auto PredForCallSite = [&](AbstractCallSite ACS) {
2853*d415bd75Srobert const auto &EDAA = A.getAAFor<AAExecutionDomain>(
2854*d415bd75Srobert *this, IRPosition::function(*ACS.getInstruction()->getFunction()),
2855*d415bd75Srobert DepClassTy::OPTIONAL);
2856*d415bd75Srobert if (!EDAA.getState().isValidState())
2857*d415bd75Srobert return false;
2858*d415bd75Srobert PredExecDomains.emplace_back(
2859*d415bd75Srobert EDAA.getExecutionDomain(*cast<CallBase>(ACS.getInstruction())));
2860*d415bd75Srobert return true;
2861*d415bd75Srobert };
2862*d415bd75Srobert
2863*d415bd75Srobert bool AllCallSitesKnown;
2864*d415bd75Srobert if (A.checkForAllCallSites(PredForCallSite, *this,
2865*d415bd75Srobert /* RequiresAllCallSites */ true,
2866*d415bd75Srobert AllCallSitesKnown)) {
2867*d415bd75Srobert for (const auto &PredED : PredExecDomains)
2868*d415bd75Srobert mergeInPredecessor(A, EntryBBED, PredED);
2869*d415bd75Srobert
2870*d415bd75Srobert } else {
2871*d415bd75Srobert // We could not find all predecessors, so this is either a kernel or a
2872*d415bd75Srobert // function with external linkage (or with some other weird uses).
2873*d415bd75Srobert auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2874*d415bd75Srobert if (OMPInfoCache.Kernels.count(getAnchorScope())) {
2875*d415bd75Srobert EntryBBED.IsExecutedByInitialThreadOnly = false;
2876*d415bd75Srobert EntryBBED.IsReachedFromAlignedBarrierOnly = true;
2877*d415bd75Srobert EntryBBED.EncounteredNonLocalSideEffect = false;
2878*d415bd75Srobert } else {
2879*d415bd75Srobert EntryBBED.IsExecutedByInitialThreadOnly = false;
2880*d415bd75Srobert EntryBBED.IsReachedFromAlignedBarrierOnly = false;
2881*d415bd75Srobert EntryBBED.EncounteredNonLocalSideEffect = true;
2882*d415bd75Srobert }
2883*d415bd75Srobert }
2884*d415bd75Srobert
2885*d415bd75Srobert auto &FnED = BEDMap[nullptr];
2886*d415bd75Srobert FnED.IsReachingAlignedBarrierOnly &=
2887*d415bd75Srobert EntryBBED.IsReachedFromAlignedBarrierOnly;
2888*d415bd75Srobert }
2889*d415bd75Srobert
updateImpl(Attributor & A)2890*d415bd75Srobert ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) {
2891*d415bd75Srobert
2892*d415bd75Srobert bool Changed = false;
2893*d415bd75Srobert
2894*d415bd75Srobert // Helper to deal with an aligned barrier encountered during the forward
2895*d415bd75Srobert // traversal. \p CB is the aligned barrier, \p ED is the execution domain when
2896*d415bd75Srobert // it was encountered.
2897*d415bd75Srobert auto HandleAlignedBarrier = [&](CallBase *CB, ExecutionDomainTy &ED) {
2898*d415bd75Srobert if (CB)
2899*d415bd75Srobert Changed |= AlignedBarriers.insert(CB);
2900*d415bd75Srobert // First, update the barrier ED kept in the separate CEDMap.
2901*d415bd75Srobert auto &CallED = CEDMap[CB];
2902*d415bd75Srobert mergeInPredecessor(A, CallED, ED);
2903*d415bd75Srobert // Next adjust the ED we use for the traversal.
2904*d415bd75Srobert ED.EncounteredNonLocalSideEffect = false;
2905*d415bd75Srobert ED.IsReachedFromAlignedBarrierOnly = true;
2906*d415bd75Srobert // Aligned barrier collection has to come last.
2907*d415bd75Srobert ED.clearAssumeInstAndAlignedBarriers();
2908*d415bd75Srobert if (CB)
2909*d415bd75Srobert ED.addAlignedBarrier(A, *CB);
2910*d415bd75Srobert };
2911*d415bd75Srobert
2912*d415bd75Srobert auto &LivenessAA =
2913*d415bd75Srobert A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
2914*d415bd75Srobert
2915*d415bd75Srobert // Set \p R to \V and report true if that changed \p R.
2916*d415bd75Srobert auto SetAndRecord = [&](bool &R, bool V) {
2917*d415bd75Srobert bool Eq = (R == V);
2918*d415bd75Srobert R = V;
2919*d415bd75Srobert return !Eq;
2920*d415bd75Srobert };
2921*d415bd75Srobert
2922*d415bd75Srobert auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2923*d415bd75Srobert
2924*d415bd75Srobert Function *F = getAnchorScope();
2925*d415bd75Srobert BasicBlock &EntryBB = F->getEntryBlock();
2926*d415bd75Srobert bool IsKernel = OMPInfoCache.Kernels.count(F);
2927*d415bd75Srobert
2928*d415bd75Srobert SmallVector<Instruction *> SyncInstWorklist;
2929*d415bd75Srobert for (auto &RIt : *RPOT) {
2930*d415bd75Srobert BasicBlock &BB = *RIt;
2931*d415bd75Srobert
2932*d415bd75Srobert bool IsEntryBB = &BB == &EntryBB;
2933*d415bd75Srobert // TODO: We use local reasoning since we don't have a divergence analysis
2934*d415bd75Srobert // running as well. We could basically allow uniform branches here.
2935*d415bd75Srobert bool AlignedBarrierLastInBlock = IsEntryBB && IsKernel;
2936*d415bd75Srobert ExecutionDomainTy ED;
2937*d415bd75Srobert // Propagate "incoming edges" into information about this block.
2938*d415bd75Srobert if (IsEntryBB) {
2939*d415bd75Srobert handleEntryBB(A, ED);
2940*d415bd75Srobert } else {
2941*d415bd75Srobert // For live non-entry blocks we only propagate
2942*d415bd75Srobert // information via live edges.
2943*d415bd75Srobert if (LivenessAA.isAssumedDead(&BB))
2944*d415bd75Srobert continue;
2945*d415bd75Srobert
2946*d415bd75Srobert for (auto *PredBB : predecessors(&BB)) {
2947*d415bd75Srobert if (LivenessAA.isEdgeDead(PredBB, &BB))
2948*d415bd75Srobert continue;
2949*d415bd75Srobert bool InitialEdgeOnly = isInitialThreadOnlyEdge(
2950*d415bd75Srobert A, dyn_cast<BranchInst>(PredBB->getTerminator()), BB);
2951*d415bd75Srobert mergeInPredecessor(A, ED, BEDMap[PredBB], InitialEdgeOnly);
2952*d415bd75Srobert }
2953*d415bd75Srobert }
2954*d415bd75Srobert
2955*d415bd75Srobert // Now we traverse the block, accumulate effects in ED and attach
2956*d415bd75Srobert // information to calls.
2957*d415bd75Srobert for (Instruction &I : BB) {
2958*d415bd75Srobert bool UsedAssumedInformation;
2959*d415bd75Srobert if (A.isAssumedDead(I, *this, &LivenessAA, UsedAssumedInformation,
2960*d415bd75Srobert /* CheckBBLivenessOnly */ false, DepClassTy::OPTIONAL,
2961*d415bd75Srobert /* CheckForDeadStore */ true))
2962*d415bd75Srobert continue;
2963*d415bd75Srobert
2964*d415bd75Srobert // Asummes and "assume-like" (dbg, lifetime, ...) are handled first, the
2965*d415bd75Srobert // former is collected the latter is ignored.
2966*d415bd75Srobert if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
2967*d415bd75Srobert if (auto *AI = dyn_cast_or_null<AssumeInst>(II)) {
2968*d415bd75Srobert ED.addAssumeInst(A, *AI);
2969*d415bd75Srobert continue;
2970*d415bd75Srobert }
2971*d415bd75Srobert // TODO: Should we also collect and delete lifetime markers?
2972*d415bd75Srobert if (II->isAssumeLikeIntrinsic())
2973*d415bd75Srobert continue;
2974*d415bd75Srobert }
2975*d415bd75Srobert
2976*d415bd75Srobert auto *CB = dyn_cast<CallBase>(&I);
2977*d415bd75Srobert bool IsNoSync = AA::isNoSyncInst(A, I, *this);
2978*d415bd75Srobert bool IsAlignedBarrier =
2979*d415bd75Srobert !IsNoSync && CB &&
2980*d415bd75Srobert AANoSync::isAlignedBarrier(*CB, AlignedBarrierLastInBlock);
2981*d415bd75Srobert
2982*d415bd75Srobert AlignedBarrierLastInBlock &= IsNoSync;
2983*d415bd75Srobert
2984*d415bd75Srobert // Next we check for calls. Aligned barriers are handled
2985*d415bd75Srobert // explicitly, everything else is kept for the backward traversal and will
2986*d415bd75Srobert // also affect our state.
2987*d415bd75Srobert if (CB) {
2988*d415bd75Srobert if (IsAlignedBarrier) {
2989*d415bd75Srobert HandleAlignedBarrier(CB, ED);
2990*d415bd75Srobert AlignedBarrierLastInBlock = true;
2991*d415bd75Srobert continue;
2992*d415bd75Srobert }
2993*d415bd75Srobert
2994*d415bd75Srobert // Check the pointer(s) of a memory intrinsic explicitly.
2995*d415bd75Srobert if (isa<MemIntrinsic>(&I)) {
2996*d415bd75Srobert if (!ED.EncounteredNonLocalSideEffect &&
2997*d415bd75Srobert AA::isPotentiallyAffectedByBarrier(A, I, *this))
2998*d415bd75Srobert ED.EncounteredNonLocalSideEffect = true;
2999*d415bd75Srobert if (!IsNoSync) {
3000*d415bd75Srobert ED.IsReachedFromAlignedBarrierOnly = false;
3001*d415bd75Srobert SyncInstWorklist.push_back(&I);
3002*d415bd75Srobert }
3003*d415bd75Srobert continue;
3004*d415bd75Srobert }
3005*d415bd75Srobert
3006*d415bd75Srobert // Record how we entered the call, then accumulate the effect of the
3007*d415bd75Srobert // call in ED for potential use by the callee.
3008*d415bd75Srobert auto &CallED = CEDMap[CB];
3009*d415bd75Srobert mergeInPredecessor(A, CallED, ED);
3010*d415bd75Srobert
3011*d415bd75Srobert // If we have a sync-definition we can check if it starts/ends in an
3012*d415bd75Srobert // aligned barrier. If we are unsure we assume any sync breaks
3013*d415bd75Srobert // alignment.
3014*d415bd75Srobert Function *Callee = CB->getCalledFunction();
3015*d415bd75Srobert if (!IsNoSync && Callee && !Callee->isDeclaration()) {
3016*d415bd75Srobert const auto &EDAA = A.getAAFor<AAExecutionDomain>(
3017*d415bd75Srobert *this, IRPosition::function(*Callee), DepClassTy::OPTIONAL);
3018*d415bd75Srobert if (EDAA.getState().isValidState()) {
3019*d415bd75Srobert const auto &CalleeED = EDAA.getFunctionExecutionDomain();
3020*d415bd75Srobert ED.IsReachedFromAlignedBarrierOnly =
3021*d415bd75Srobert CallED.IsReachedFromAlignedBarrierOnly =
3022*d415bd75Srobert CalleeED.IsReachedFromAlignedBarrierOnly;
3023*d415bd75Srobert AlignedBarrierLastInBlock = ED.IsReachedFromAlignedBarrierOnly;
3024*d415bd75Srobert if (IsNoSync || !CalleeED.IsReachedFromAlignedBarrierOnly)
3025*d415bd75Srobert ED.EncounteredNonLocalSideEffect |=
3026*d415bd75Srobert CalleeED.EncounteredNonLocalSideEffect;
3027*d415bd75Srobert else
3028*d415bd75Srobert ED.EncounteredNonLocalSideEffect =
3029*d415bd75Srobert CalleeED.EncounteredNonLocalSideEffect;
3030*d415bd75Srobert if (!CalleeED.IsReachingAlignedBarrierOnly)
3031*d415bd75Srobert SyncInstWorklist.push_back(&I);
3032*d415bd75Srobert if (CalleeED.IsReachedFromAlignedBarrierOnly)
3033*d415bd75Srobert mergeInPredecessorBarriersAndAssumptions(A, ED, CalleeED);
3034*d415bd75Srobert continue;
3035*d415bd75Srobert }
3036*d415bd75Srobert }
3037*d415bd75Srobert if (!IsNoSync)
3038*d415bd75Srobert ED.IsReachedFromAlignedBarrierOnly =
3039*d415bd75Srobert CallED.IsReachedFromAlignedBarrierOnly = false;
3040*d415bd75Srobert AlignedBarrierLastInBlock &= ED.IsReachedFromAlignedBarrierOnly;
3041*d415bd75Srobert ED.EncounteredNonLocalSideEffect |= !CB->doesNotAccessMemory();
3042*d415bd75Srobert if (!IsNoSync)
3043*d415bd75Srobert SyncInstWorklist.push_back(&I);
3044*d415bd75Srobert }
3045*d415bd75Srobert
3046*d415bd75Srobert if (!I.mayHaveSideEffects() && !I.mayReadFromMemory())
3047*d415bd75Srobert continue;
3048*d415bd75Srobert
3049*d415bd75Srobert // If we have a callee we try to use fine-grained information to
3050*d415bd75Srobert // determine local side-effects.
3051*d415bd75Srobert if (CB) {
3052*d415bd75Srobert const auto &MemAA = A.getAAFor<AAMemoryLocation>(
3053*d415bd75Srobert *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);
3054*d415bd75Srobert
3055*d415bd75Srobert auto AccessPred = [&](const Instruction *I, const Value *Ptr,
3056*d415bd75Srobert AAMemoryLocation::AccessKind,
3057*d415bd75Srobert AAMemoryLocation::MemoryLocationsKind) {
3058*d415bd75Srobert return !AA::isPotentiallyAffectedByBarrier(A, {Ptr}, *this, I);
3059*d415bd75Srobert };
3060*d415bd75Srobert if (MemAA.getState().isValidState() &&
3061*d415bd75Srobert MemAA.checkForAllAccessesToMemoryKind(
3062*d415bd75Srobert AccessPred, AAMemoryLocation::ALL_LOCATIONS))
3063*d415bd75Srobert continue;
3064*d415bd75Srobert }
3065*d415bd75Srobert
3066*d415bd75Srobert if (!I.mayHaveSideEffects() && OMPInfoCache.isOnlyUsedByAssume(I))
3067*d415bd75Srobert continue;
3068*d415bd75Srobert
3069*d415bd75Srobert if (auto *LI = dyn_cast<LoadInst>(&I))
3070*d415bd75Srobert if (LI->hasMetadata(LLVMContext::MD_invariant_load))
3071*d415bd75Srobert continue;
3072*d415bd75Srobert
3073*d415bd75Srobert if (!ED.EncounteredNonLocalSideEffect &&
3074*d415bd75Srobert AA::isPotentiallyAffectedByBarrier(A, I, *this))
3075*d415bd75Srobert ED.EncounteredNonLocalSideEffect = true;
3076*d415bd75Srobert }
3077*d415bd75Srobert
3078*d415bd75Srobert if (!isa<UnreachableInst>(BB.getTerminator()) &&
3079*d415bd75Srobert !BB.getTerminator()->getNumSuccessors()) {
3080*d415bd75Srobert
3081*d415bd75Srobert auto &FnED = BEDMap[nullptr];
3082*d415bd75Srobert mergeInPredecessor(A, FnED, ED);
3083*d415bd75Srobert
3084*d415bd75Srobert if (IsKernel)
3085*d415bd75Srobert HandleAlignedBarrier(nullptr, ED);
3086*d415bd75Srobert }
3087*d415bd75Srobert
3088*d415bd75Srobert ExecutionDomainTy &StoredED = BEDMap[&BB];
3089*d415bd75Srobert ED.IsReachingAlignedBarrierOnly = StoredED.IsReachingAlignedBarrierOnly;
3090*d415bd75Srobert
3091*d415bd75Srobert // Check if we computed anything different as part of the forward
3092*d415bd75Srobert // traversal. We do not take assumptions and aligned barriers into account
3093*d415bd75Srobert // as they do not influence the state we iterate. Backward traversal values
3094*d415bd75Srobert // are handled later on.
3095*d415bd75Srobert if (ED.IsExecutedByInitialThreadOnly !=
3096*d415bd75Srobert StoredED.IsExecutedByInitialThreadOnly ||
3097*d415bd75Srobert ED.IsReachedFromAlignedBarrierOnly !=
3098*d415bd75Srobert StoredED.IsReachedFromAlignedBarrierOnly ||
3099*d415bd75Srobert ED.EncounteredNonLocalSideEffect !=
3100*d415bd75Srobert StoredED.EncounteredNonLocalSideEffect)
3101*d415bd75Srobert Changed = true;
3102*d415bd75Srobert
3103*d415bd75Srobert // Update the state with the new value.
3104*d415bd75Srobert StoredED = std::move(ED);
3105*d415bd75Srobert }
3106*d415bd75Srobert
3107*d415bd75Srobert // Propagate (non-aligned) sync instruction effects backwards until the
3108*d415bd75Srobert // entry is hit or an aligned barrier.
3109*d415bd75Srobert SmallSetVector<BasicBlock *, 16> Visited;
3110*d415bd75Srobert while (!SyncInstWorklist.empty()) {
3111*d415bd75Srobert Instruction *SyncInst = SyncInstWorklist.pop_back_val();
3112*d415bd75Srobert Instruction *CurInst = SyncInst;
3113*d415bd75Srobert bool HitAlignedBarrier = false;
3114*d415bd75Srobert while ((CurInst = CurInst->getPrevNode())) {
3115*d415bd75Srobert auto *CB = dyn_cast<CallBase>(CurInst);
3116*d415bd75Srobert if (!CB)
3117*d415bd75Srobert continue;
3118*d415bd75Srobert auto &CallED = CEDMap[CB];
3119*d415bd75Srobert if (SetAndRecord(CallED.IsReachingAlignedBarrierOnly, false))
3120*d415bd75Srobert Changed = true;
3121*d415bd75Srobert HitAlignedBarrier = AlignedBarriers.count(CB);
3122*d415bd75Srobert if (HitAlignedBarrier)
3123*d415bd75Srobert break;
3124*d415bd75Srobert }
3125*d415bd75Srobert if (HitAlignedBarrier)
3126*d415bd75Srobert continue;
3127*d415bd75Srobert BasicBlock *SyncBB = SyncInst->getParent();
3128*d415bd75Srobert for (auto *PredBB : predecessors(SyncBB)) {
3129*d415bd75Srobert if (LivenessAA.isEdgeDead(PredBB, SyncBB))
3130*d415bd75Srobert continue;
3131*d415bd75Srobert if (!Visited.insert(PredBB))
3132*d415bd75Srobert continue;
3133*d415bd75Srobert SyncInstWorklist.push_back(PredBB->getTerminator());
3134*d415bd75Srobert auto &PredED = BEDMap[PredBB];
3135*d415bd75Srobert if (SetAndRecord(PredED.IsReachingAlignedBarrierOnly, false))
3136*d415bd75Srobert Changed = true;
3137*d415bd75Srobert }
3138*d415bd75Srobert if (SyncBB != &EntryBB)
3139*d415bd75Srobert continue;
3140*d415bd75Srobert auto &FnED = BEDMap[nullptr];
3141*d415bd75Srobert if (SetAndRecord(FnED.IsReachingAlignedBarrierOnly, false))
3142*d415bd75Srobert Changed = true;
3143*d415bd75Srobert }
3144*d415bd75Srobert
3145*d415bd75Srobert return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
314673471bf0Spatrick }
314773471bf0Spatrick
314873471bf0Spatrick /// Try to replace memory allocation calls called by a single thread with a
314973471bf0Spatrick /// static buffer of shared memory.
315073471bf0Spatrick struct AAHeapToShared : public StateWrapper<BooleanState, AbstractAttribute> {
315173471bf0Spatrick using Base = StateWrapper<BooleanState, AbstractAttribute>;
AAHeapToShared__anon4b47cb5e0111::AAHeapToShared315273471bf0Spatrick AAHeapToShared(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
315373471bf0Spatrick
315473471bf0Spatrick /// Create an abstract attribute view for the position \p IRP.
315573471bf0Spatrick static AAHeapToShared &createForPosition(const IRPosition &IRP,
315673471bf0Spatrick Attributor &A);
315773471bf0Spatrick
315873471bf0Spatrick /// Returns true if HeapToShared conversion is assumed to be possible.
315973471bf0Spatrick virtual bool isAssumedHeapToShared(CallBase &CB) const = 0;
316073471bf0Spatrick
316173471bf0Spatrick /// Returns true if HeapToShared conversion is assumed and the CB is a
316273471bf0Spatrick /// callsite to a free operation to be removed.
316373471bf0Spatrick virtual bool isAssumedHeapToSharedRemovedFree(CallBase &CB) const = 0;
316473471bf0Spatrick
316573471bf0Spatrick /// See AbstractAttribute::getName().
getName__anon4b47cb5e0111::AAHeapToShared316673471bf0Spatrick const std::string getName() const override { return "AAHeapToShared"; }
316773471bf0Spatrick
316873471bf0Spatrick /// See AbstractAttribute::getIdAddr().
getIdAddr__anon4b47cb5e0111::AAHeapToShared316973471bf0Spatrick const char *getIdAddr() const override { return &ID; }
317073471bf0Spatrick
317173471bf0Spatrick /// This function should return true if the type of the \p AA is
317273471bf0Spatrick /// AAHeapToShared.
classof__anon4b47cb5e0111::AAHeapToShared317373471bf0Spatrick static bool classof(const AbstractAttribute *AA) {
317473471bf0Spatrick return (AA->getIdAddr() == &ID);
317573471bf0Spatrick }
317673471bf0Spatrick
317773471bf0Spatrick /// Unique ID (due to the unique address)
317873471bf0Spatrick static const char ID;
317973471bf0Spatrick };
318073471bf0Spatrick
318173471bf0Spatrick struct AAHeapToSharedFunction : public AAHeapToShared {
AAHeapToSharedFunction__anon4b47cb5e0111::AAHeapToSharedFunction318273471bf0Spatrick AAHeapToSharedFunction(const IRPosition &IRP, Attributor &A)
318373471bf0Spatrick : AAHeapToShared(IRP, A) {}
318473471bf0Spatrick
getAsStr__anon4b47cb5e0111::AAHeapToSharedFunction318573471bf0Spatrick const std::string getAsStr() const override {
318673471bf0Spatrick return "[AAHeapToShared] " + std::to_string(MallocCalls.size()) +
318773471bf0Spatrick " malloc calls eligible.";
318873471bf0Spatrick }
318973471bf0Spatrick
319073471bf0Spatrick /// See AbstractAttribute::trackStatistics().
trackStatistics__anon4b47cb5e0111::AAHeapToSharedFunction319173471bf0Spatrick void trackStatistics() const override {}
319273471bf0Spatrick
319373471bf0Spatrick /// This functions finds free calls that will be removed by the
319473471bf0Spatrick /// HeapToShared transformation.
findPotentialRemovedFreeCalls__anon4b47cb5e0111::AAHeapToSharedFunction319573471bf0Spatrick void findPotentialRemovedFreeCalls(Attributor &A) {
319673471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
319773471bf0Spatrick auto &FreeRFI = OMPInfoCache.RFIs[OMPRTL___kmpc_free_shared];
319873471bf0Spatrick
319973471bf0Spatrick PotentialRemovedFreeCalls.clear();
320073471bf0Spatrick // Update free call users of found malloc calls.
320173471bf0Spatrick for (CallBase *CB : MallocCalls) {
320273471bf0Spatrick SmallVector<CallBase *, 4> FreeCalls;
320373471bf0Spatrick for (auto *U : CB->users()) {
320473471bf0Spatrick CallBase *C = dyn_cast<CallBase>(U);
320573471bf0Spatrick if (C && C->getCalledFunction() == FreeRFI.Declaration)
320673471bf0Spatrick FreeCalls.push_back(C);
320773471bf0Spatrick }
320873471bf0Spatrick
320973471bf0Spatrick if (FreeCalls.size() != 1)
321073471bf0Spatrick continue;
321173471bf0Spatrick
321273471bf0Spatrick PotentialRemovedFreeCalls.insert(FreeCalls.front());
321373471bf0Spatrick }
321473471bf0Spatrick }
321573471bf0Spatrick
initialize__anon4b47cb5e0111::AAHeapToSharedFunction321673471bf0Spatrick void initialize(Attributor &A) override {
3217*d415bd75Srobert if (DisableOpenMPOptDeglobalization) {
3218*d415bd75Srobert indicatePessimisticFixpoint();
3219*d415bd75Srobert return;
3220*d415bd75Srobert }
3221*d415bd75Srobert
322273471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
322373471bf0Spatrick auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
3224*d415bd75Srobert if (!RFI.Declaration)
3225*d415bd75Srobert return;
322673471bf0Spatrick
3227*d415bd75Srobert Attributor::SimplifictionCallbackTy SCB =
3228*d415bd75Srobert [](const IRPosition &, const AbstractAttribute *,
3229*d415bd75Srobert bool &) -> std::optional<Value *> { return nullptr; };
3230*d415bd75Srobert
3231*d415bd75Srobert Function *F = getAnchorScope();
323273471bf0Spatrick for (User *U : RFI.Declaration->users())
3233*d415bd75Srobert if (CallBase *CB = dyn_cast<CallBase>(U)) {
3234*d415bd75Srobert if (CB->getFunction() != F)
3235*d415bd75Srobert continue;
323673471bf0Spatrick MallocCalls.insert(CB);
3237*d415bd75Srobert A.registerSimplificationCallback(IRPosition::callsite_returned(*CB),
3238*d415bd75Srobert SCB);
3239*d415bd75Srobert }
324073471bf0Spatrick
324173471bf0Spatrick findPotentialRemovedFreeCalls(A);
324273471bf0Spatrick }
324373471bf0Spatrick
isAssumedHeapToShared__anon4b47cb5e0111::AAHeapToSharedFunction324473471bf0Spatrick bool isAssumedHeapToShared(CallBase &CB) const override {
324573471bf0Spatrick return isValidState() && MallocCalls.count(&CB);
324673471bf0Spatrick }
324773471bf0Spatrick
isAssumedHeapToSharedRemovedFree__anon4b47cb5e0111::AAHeapToSharedFunction324873471bf0Spatrick bool isAssumedHeapToSharedRemovedFree(CallBase &CB) const override {
324973471bf0Spatrick return isValidState() && PotentialRemovedFreeCalls.count(&CB);
325073471bf0Spatrick }
325173471bf0Spatrick
manifest__anon4b47cb5e0111::AAHeapToSharedFunction325273471bf0Spatrick ChangeStatus manifest(Attributor &A) override {
325373471bf0Spatrick if (MallocCalls.empty())
325473471bf0Spatrick return ChangeStatus::UNCHANGED;
325573471bf0Spatrick
325673471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
325773471bf0Spatrick auto &FreeCall = OMPInfoCache.RFIs[OMPRTL___kmpc_free_shared];
325873471bf0Spatrick
325973471bf0Spatrick Function *F = getAnchorScope();
326073471bf0Spatrick auto *HS = A.lookupAAFor<AAHeapToStack>(IRPosition::function(*F), this,
326173471bf0Spatrick DepClassTy::OPTIONAL);
326273471bf0Spatrick
326373471bf0Spatrick ChangeStatus Changed = ChangeStatus::UNCHANGED;
326473471bf0Spatrick for (CallBase *CB : MallocCalls) {
326573471bf0Spatrick // Skip replacing this if HeapToStack has already claimed it.
326673471bf0Spatrick if (HS && HS->isAssumedHeapToStack(*CB))
326773471bf0Spatrick continue;
326873471bf0Spatrick
326973471bf0Spatrick // Find the unique free call to remove it.
327073471bf0Spatrick SmallVector<CallBase *, 4> FreeCalls;
327173471bf0Spatrick for (auto *U : CB->users()) {
327273471bf0Spatrick CallBase *C = dyn_cast<CallBase>(U);
327373471bf0Spatrick if (C && C->getCalledFunction() == FreeCall.Declaration)
327473471bf0Spatrick FreeCalls.push_back(C);
327573471bf0Spatrick }
327673471bf0Spatrick if (FreeCalls.size() != 1)
327773471bf0Spatrick continue;
327873471bf0Spatrick
3279*d415bd75Srobert auto *AllocSize = cast<ConstantInt>(CB->getArgOperand(0));
328073471bf0Spatrick
3281*d415bd75Srobert if (AllocSize->getZExtValue() + SharedMemoryUsed > SharedMemoryLimit) {
3282*d415bd75Srobert LLVM_DEBUG(dbgs() << TAG << "Cannot replace call " << *CB
3283*d415bd75Srobert << " with shared memory."
3284*d415bd75Srobert << " Shared memory usage is limited to "
3285*d415bd75Srobert << SharedMemoryLimit << " bytes\n");
3286*d415bd75Srobert continue;
3287*d415bd75Srobert }
3288*d415bd75Srobert
3289*d415bd75Srobert LLVM_DEBUG(dbgs() << TAG << "Replace globalization call " << *CB
3290*d415bd75Srobert << " with " << AllocSize->getZExtValue()
329173471bf0Spatrick << " bytes of shared memory\n");
329273471bf0Spatrick
329373471bf0Spatrick // Create a new shared memory buffer of the same size as the allocation
329473471bf0Spatrick // and replace all the uses of the original allocation with it.
329573471bf0Spatrick Module *M = CB->getModule();
329673471bf0Spatrick Type *Int8Ty = Type::getInt8Ty(M->getContext());
329773471bf0Spatrick Type *Int8ArrTy = ArrayType::get(Int8Ty, AllocSize->getZExtValue());
329873471bf0Spatrick auto *SharedMem = new GlobalVariable(
329973471bf0Spatrick *M, Int8ArrTy, /* IsConstant */ false, GlobalValue::InternalLinkage,
3300*d415bd75Srobert UndefValue::get(Int8ArrTy), CB->getName() + "_shared", nullptr,
330173471bf0Spatrick GlobalValue::NotThreadLocal,
330273471bf0Spatrick static_cast<unsigned>(AddressSpace::Shared));
330373471bf0Spatrick auto *NewBuffer =
330473471bf0Spatrick ConstantExpr::getPointerCast(SharedMem, Int8Ty->getPointerTo());
330573471bf0Spatrick
330673471bf0Spatrick auto Remark = [&](OptimizationRemark OR) {
330773471bf0Spatrick return OR << "Replaced globalized variable with "
330873471bf0Spatrick << ore::NV("SharedMemory", AllocSize->getZExtValue())
330973471bf0Spatrick << ((AllocSize->getZExtValue() != 1) ? " bytes " : " byte ")
331073471bf0Spatrick << "of shared memory.";
331173471bf0Spatrick };
331273471bf0Spatrick A.emitRemark<OptimizationRemark>(CB, "OMP111", Remark);
331373471bf0Spatrick
3314*d415bd75Srobert MaybeAlign Alignment = CB->getRetAlign();
3315*d415bd75Srobert assert(Alignment &&
3316*d415bd75Srobert "HeapToShared on allocation without alignment attribute");
3317*d415bd75Srobert SharedMem->setAlignment(MaybeAlign(Alignment));
331873471bf0Spatrick
3319*d415bd75Srobert A.changeAfterManifest(IRPosition::callsite_returned(*CB), *NewBuffer);
332073471bf0Spatrick A.deleteAfterManifest(*CB);
332173471bf0Spatrick A.deleteAfterManifest(*FreeCalls.front());
332273471bf0Spatrick
3323*d415bd75Srobert SharedMemoryUsed += AllocSize->getZExtValue();
3324*d415bd75Srobert NumBytesMovedToSharedMemory = SharedMemoryUsed;
332573471bf0Spatrick Changed = ChangeStatus::CHANGED;
332673471bf0Spatrick }
332773471bf0Spatrick
332873471bf0Spatrick return Changed;
332973471bf0Spatrick }
333073471bf0Spatrick
updateImpl__anon4b47cb5e0111::AAHeapToSharedFunction333173471bf0Spatrick ChangeStatus updateImpl(Attributor &A) override {
3332*d415bd75Srobert if (MallocCalls.empty())
3333*d415bd75Srobert return indicatePessimisticFixpoint();
333473471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
333573471bf0Spatrick auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
3336*d415bd75Srobert if (!RFI.Declaration)
3337*d415bd75Srobert return ChangeStatus::UNCHANGED;
3338*d415bd75Srobert
333973471bf0Spatrick Function *F = getAnchorScope();
334073471bf0Spatrick
334173471bf0Spatrick auto NumMallocCalls = MallocCalls.size();
334273471bf0Spatrick
334373471bf0Spatrick // Only consider malloc calls executed by a single thread with a constant.
334473471bf0Spatrick for (User *U : RFI.Declaration->users()) {
3345*d415bd75Srobert if (CallBase *CB = dyn_cast<CallBase>(U)) {
3346*d415bd75Srobert if (CB->getCaller() != F)
3347*d415bd75Srobert continue;
3348*d415bd75Srobert if (!MallocCalls.count(CB))
3349*d415bd75Srobert continue;
3350*d415bd75Srobert if (!isa<ConstantInt>(CB->getArgOperand(0))) {
3351*d415bd75Srobert MallocCalls.remove(CB);
3352*d415bd75Srobert continue;
3353*d415bd75Srobert }
335473471bf0Spatrick const auto &ED = A.getAAFor<AAExecutionDomain>(
335573471bf0Spatrick *this, IRPosition::function(*F), DepClassTy::REQUIRED);
3356*d415bd75Srobert if (!ED.isExecutedByInitialThreadOnly(*CB))
3357*d415bd75Srobert MallocCalls.remove(CB);
3358*d415bd75Srobert }
335973471bf0Spatrick }
336073471bf0Spatrick
336173471bf0Spatrick findPotentialRemovedFreeCalls(A);
336273471bf0Spatrick
336373471bf0Spatrick if (NumMallocCalls != MallocCalls.size())
336473471bf0Spatrick return ChangeStatus::CHANGED;
336573471bf0Spatrick
336673471bf0Spatrick return ChangeStatus::UNCHANGED;
336773471bf0Spatrick }
336873471bf0Spatrick
336973471bf0Spatrick /// Collection of all malloc calls in a function.
3370*d415bd75Srobert SmallSetVector<CallBase *, 4> MallocCalls;
337173471bf0Spatrick /// Collection of potentially removed free calls in a function.
337273471bf0Spatrick SmallPtrSet<CallBase *, 4> PotentialRemovedFreeCalls;
3373*d415bd75Srobert /// The total amount of shared memory that has been used for HeapToShared.
3374*d415bd75Srobert unsigned SharedMemoryUsed = 0;
337573471bf0Spatrick };
337673471bf0Spatrick
337773471bf0Spatrick struct AAKernelInfo : public StateWrapper<KernelInfoState, AbstractAttribute> {
337873471bf0Spatrick using Base = StateWrapper<KernelInfoState, AbstractAttribute>;
AAKernelInfo__anon4b47cb5e0111::AAKernelInfo337973471bf0Spatrick AAKernelInfo(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
338073471bf0Spatrick
338173471bf0Spatrick /// Statistics are tracked as part of manifest for now.
trackStatistics__anon4b47cb5e0111::AAKernelInfo338273471bf0Spatrick void trackStatistics() const override {}
338373471bf0Spatrick
338473471bf0Spatrick /// See AbstractAttribute::getAsStr()
getAsStr__anon4b47cb5e0111::AAKernelInfo338573471bf0Spatrick const std::string getAsStr() const override {
338673471bf0Spatrick if (!isValidState())
338773471bf0Spatrick return "<invalid>";
338873471bf0Spatrick return std::string(SPMDCompatibilityTracker.isAssumed() ? "SPMD"
338973471bf0Spatrick : "generic") +
339073471bf0Spatrick std::string(SPMDCompatibilityTracker.isAtFixpoint() ? " [FIX]"
339173471bf0Spatrick : "") +
339273471bf0Spatrick std::string(" #PRs: ") +
3393*d415bd75Srobert (ReachedKnownParallelRegions.isValidState()
3394*d415bd75Srobert ? std::to_string(ReachedKnownParallelRegions.size())
3395*d415bd75Srobert : "<invalid>") +
339673471bf0Spatrick ", #Unknown PRs: " +
3397*d415bd75Srobert (ReachedUnknownParallelRegions.isValidState()
3398*d415bd75Srobert ? std::to_string(ReachedUnknownParallelRegions.size())
3399*d415bd75Srobert : "<invalid>") +
3400*d415bd75Srobert ", #Reaching Kernels: " +
3401*d415bd75Srobert (ReachingKernelEntries.isValidState()
3402*d415bd75Srobert ? std::to_string(ReachingKernelEntries.size())
3403*d415bd75Srobert : "<invalid>") +
3404*d415bd75Srobert ", #ParLevels: " +
3405*d415bd75Srobert (ParallelLevels.isValidState()
3406*d415bd75Srobert ? std::to_string(ParallelLevels.size())
3407*d415bd75Srobert : "<invalid>");
340873471bf0Spatrick }
340973471bf0Spatrick
341073471bf0Spatrick /// Create an abstract attribute biew for the position \p IRP.
341173471bf0Spatrick static AAKernelInfo &createForPosition(const IRPosition &IRP, Attributor &A);
341273471bf0Spatrick
341373471bf0Spatrick /// See AbstractAttribute::getName()
getName__anon4b47cb5e0111::AAKernelInfo341473471bf0Spatrick const std::string getName() const override { return "AAKernelInfo"; }
341573471bf0Spatrick
341673471bf0Spatrick /// See AbstractAttribute::getIdAddr()
getIdAddr__anon4b47cb5e0111::AAKernelInfo341773471bf0Spatrick const char *getIdAddr() const override { return &ID; }
341873471bf0Spatrick
341973471bf0Spatrick /// This function should return true if the type of the \p AA is AAKernelInfo
classof__anon4b47cb5e0111::AAKernelInfo342073471bf0Spatrick static bool classof(const AbstractAttribute *AA) {
342173471bf0Spatrick return (AA->getIdAddr() == &ID);
342273471bf0Spatrick }
342373471bf0Spatrick
342473471bf0Spatrick static const char ID;
342573471bf0Spatrick };
342673471bf0Spatrick
342773471bf0Spatrick /// The function kernel info abstract attribute, basically, what can we say
342873471bf0Spatrick /// about a function with regards to the KernelInfoState.
342973471bf0Spatrick struct AAKernelInfoFunction : AAKernelInfo {
AAKernelInfoFunction__anon4b47cb5e0111::AAKernelInfoFunction343073471bf0Spatrick AAKernelInfoFunction(const IRPosition &IRP, Attributor &A)
343173471bf0Spatrick : AAKernelInfo(IRP, A) {}
343273471bf0Spatrick
3433*d415bd75Srobert SmallPtrSet<Instruction *, 4> GuardedInstructions;
3434*d415bd75Srobert
getGuardedInstructions__anon4b47cb5e0111::AAKernelInfoFunction3435*d415bd75Srobert SmallPtrSetImpl<Instruction *> &getGuardedInstructions() {
3436*d415bd75Srobert return GuardedInstructions;
3437*d415bd75Srobert }
3438*d415bd75Srobert
343973471bf0Spatrick /// See AbstractAttribute::initialize(...).
initialize__anon4b47cb5e0111::AAKernelInfoFunction344073471bf0Spatrick void initialize(Attributor &A) override {
344173471bf0Spatrick // This is a high-level transform that might change the constant arguments
344273471bf0Spatrick // of the init and dinit calls. We need to tell the Attributor about this
344373471bf0Spatrick // to avoid other parts using the current constant value for simpliication.
344473471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
344573471bf0Spatrick
344673471bf0Spatrick Function *Fn = getAnchorScope();
344773471bf0Spatrick
344873471bf0Spatrick OMPInformationCache::RuntimeFunctionInfo &InitRFI =
344973471bf0Spatrick OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
345073471bf0Spatrick OMPInformationCache::RuntimeFunctionInfo &DeinitRFI =
345173471bf0Spatrick OMPInfoCache.RFIs[OMPRTL___kmpc_target_deinit];
345273471bf0Spatrick
345373471bf0Spatrick // For kernels we perform more initialization work, first we find the init
345473471bf0Spatrick // and deinit calls.
345573471bf0Spatrick auto StoreCallBase = [](Use &U,
345673471bf0Spatrick OMPInformationCache::RuntimeFunctionInfo &RFI,
345773471bf0Spatrick CallBase *&Storage) {
345873471bf0Spatrick CallBase *CB = OpenMPOpt::getCallIfRegularCall(U, &RFI);
345973471bf0Spatrick assert(CB &&
346073471bf0Spatrick "Unexpected use of __kmpc_target_init or __kmpc_target_deinit!");
346173471bf0Spatrick assert(!Storage &&
346273471bf0Spatrick "Multiple uses of __kmpc_target_init or __kmpc_target_deinit!");
346373471bf0Spatrick Storage = CB;
346473471bf0Spatrick return false;
346573471bf0Spatrick };
346673471bf0Spatrick InitRFI.foreachUse(
346773471bf0Spatrick [&](Use &U, Function &) {
346873471bf0Spatrick StoreCallBase(U, InitRFI, KernelInitCB);
346973471bf0Spatrick return false;
347073471bf0Spatrick },
347173471bf0Spatrick Fn);
347273471bf0Spatrick DeinitRFI.foreachUse(
347373471bf0Spatrick [&](Use &U, Function &) {
347473471bf0Spatrick StoreCallBase(U, DeinitRFI, KernelDeinitCB);
347573471bf0Spatrick return false;
347673471bf0Spatrick },
347773471bf0Spatrick Fn);
347873471bf0Spatrick
3479*d415bd75Srobert // Ignore kernels without initializers such as global constructors.
3480*d415bd75Srobert if (!KernelInitCB || !KernelDeinitCB)
3481*d415bd75Srobert return;
3482*d415bd75Srobert
3483*d415bd75Srobert // Add itself to the reaching kernel and set IsKernelEntry.
3484*d415bd75Srobert ReachingKernelEntries.insert(Fn);
3485*d415bd75Srobert IsKernelEntry = true;
348673471bf0Spatrick
348773471bf0Spatrick // For kernels we might need to initialize/finalize the IsSPMD state and
348873471bf0Spatrick // we need to register a simplification callback so that the Attributor
348973471bf0Spatrick // knows the constant arguments to __kmpc_target_init and
349073471bf0Spatrick // __kmpc_target_deinit might actually change.
349173471bf0Spatrick
349273471bf0Spatrick Attributor::SimplifictionCallbackTy StateMachineSimplifyCB =
349373471bf0Spatrick [&](const IRPosition &IRP, const AbstractAttribute *AA,
3494*d415bd75Srobert bool &UsedAssumedInformation) -> std::optional<Value *> {
349573471bf0Spatrick // IRP represents the "use generic state machine" argument of an
349673471bf0Spatrick // __kmpc_target_init call. We will answer this one with the internal
349773471bf0Spatrick // state. As long as we are not in an invalid state, we will create a
349873471bf0Spatrick // custom state machine so the value should be a `i1 false`. If we are
349973471bf0Spatrick // in an invalid state, we won't change the value that is in the IR.
3500*d415bd75Srobert if (!ReachedKnownParallelRegions.isValidState())
3501*d415bd75Srobert return nullptr;
3502*d415bd75Srobert // If we have disabled state machine rewrites, don't make a custom one.
3503*d415bd75Srobert if (DisableOpenMPOptStateMachineRewrite)
350473471bf0Spatrick return nullptr;
350573471bf0Spatrick if (AA)
350673471bf0Spatrick A.recordDependence(*this, *AA, DepClassTy::OPTIONAL);
350773471bf0Spatrick UsedAssumedInformation = !isAtFixpoint();
350873471bf0Spatrick auto *FalseVal =
3509*d415bd75Srobert ConstantInt::getBool(IRP.getAnchorValue().getContext(), false);
351073471bf0Spatrick return FalseVal;
351173471bf0Spatrick };
351273471bf0Spatrick
3513*d415bd75Srobert Attributor::SimplifictionCallbackTy ModeSimplifyCB =
351473471bf0Spatrick [&](const IRPosition &IRP, const AbstractAttribute *AA,
3515*d415bd75Srobert bool &UsedAssumedInformation) -> std::optional<Value *> {
351673471bf0Spatrick // IRP represents the "SPMDCompatibilityTracker" argument of an
351773471bf0Spatrick // __kmpc_target_init or
351873471bf0Spatrick // __kmpc_target_deinit call. We will answer this one with the internal
351973471bf0Spatrick // state.
352073471bf0Spatrick if (!SPMDCompatibilityTracker.isValidState())
352173471bf0Spatrick return nullptr;
352273471bf0Spatrick if (!SPMDCompatibilityTracker.isAtFixpoint()) {
352373471bf0Spatrick if (AA)
352473471bf0Spatrick A.recordDependence(*this, *AA, DepClassTy::OPTIONAL);
352573471bf0Spatrick UsedAssumedInformation = true;
352673471bf0Spatrick } else {
352773471bf0Spatrick UsedAssumedInformation = false;
352873471bf0Spatrick }
3529*d415bd75Srobert auto *Val = ConstantInt::getSigned(
3530*d415bd75Srobert IntegerType::getInt8Ty(IRP.getAnchorValue().getContext()),
3531*d415bd75Srobert SPMDCompatibilityTracker.isAssumed() ? OMP_TGT_EXEC_MODE_SPMD
3532*d415bd75Srobert : OMP_TGT_EXEC_MODE_GENERIC);
353373471bf0Spatrick return Val;
353473471bf0Spatrick };
353573471bf0Spatrick
3536*d415bd75Srobert constexpr const int InitModeArgNo = 1;
3537*d415bd75Srobert constexpr const int DeinitModeArgNo = 1;
353873471bf0Spatrick constexpr const int InitUseStateMachineArgNo = 2;
353973471bf0Spatrick A.registerSimplificationCallback(
354073471bf0Spatrick IRPosition::callsite_argument(*KernelInitCB, InitUseStateMachineArgNo),
354173471bf0Spatrick StateMachineSimplifyCB);
354273471bf0Spatrick A.registerSimplificationCallback(
3543*d415bd75Srobert IRPosition::callsite_argument(*KernelInitCB, InitModeArgNo),
3544*d415bd75Srobert ModeSimplifyCB);
354573471bf0Spatrick A.registerSimplificationCallback(
3546*d415bd75Srobert IRPosition::callsite_argument(*KernelDeinitCB, DeinitModeArgNo),
3547*d415bd75Srobert ModeSimplifyCB);
354873471bf0Spatrick
354973471bf0Spatrick // Check if we know we are in SPMD-mode already.
3550*d415bd75Srobert ConstantInt *ModeArg =
3551*d415bd75Srobert dyn_cast<ConstantInt>(KernelInitCB->getArgOperand(InitModeArgNo));
3552*d415bd75Srobert if (ModeArg && (ModeArg->getSExtValue() & OMP_TGT_EXEC_MODE_SPMD))
355373471bf0Spatrick SPMDCompatibilityTracker.indicateOptimisticFixpoint();
3554*d415bd75Srobert // This is a generic region but SPMDization is disabled so stop tracking.
3555*d415bd75Srobert else if (DisableOpenMPOptSPMDization)
3556*d415bd75Srobert SPMDCompatibilityTracker.indicatePessimisticFixpoint();
3557*d415bd75Srobert
3558*d415bd75Srobert // Register virtual uses of functions we might need to preserve.
3559*d415bd75Srobert auto RegisterVirtualUse = [&](RuntimeFunction RFKind,
3560*d415bd75Srobert Attributor::VirtualUseCallbackTy &CB) {
3561*d415bd75Srobert if (!OMPInfoCache.RFIs[RFKind].Declaration)
3562*d415bd75Srobert return;
3563*d415bd75Srobert A.registerVirtualUseCallback(*OMPInfoCache.RFIs[RFKind].Declaration, CB);
3564*d415bd75Srobert };
3565*d415bd75Srobert
3566*d415bd75Srobert // Add a dependence to ensure updates if the state changes.
3567*d415bd75Srobert auto AddDependence = [](Attributor &A, const AAKernelInfo *KI,
3568*d415bd75Srobert const AbstractAttribute *QueryingAA) {
3569*d415bd75Srobert if (QueryingAA) {
3570*d415bd75Srobert A.recordDependence(*KI, *QueryingAA, DepClassTy::OPTIONAL);
3571*d415bd75Srobert }
3572*d415bd75Srobert return true;
3573*d415bd75Srobert };
3574*d415bd75Srobert
3575*d415bd75Srobert Attributor::VirtualUseCallbackTy CustomStateMachineUseCB =
3576*d415bd75Srobert [&](Attributor &A, const AbstractAttribute *QueryingAA) {
3577*d415bd75Srobert // Whenever we create a custom state machine we will insert calls to
3578*d415bd75Srobert // __kmpc_get_hardware_num_threads_in_block,
3579*d415bd75Srobert // __kmpc_get_warp_size,
3580*d415bd75Srobert // __kmpc_barrier_simple_generic,
3581*d415bd75Srobert // __kmpc_kernel_parallel, and
3582*d415bd75Srobert // __kmpc_kernel_end_parallel.
3583*d415bd75Srobert // Not needed if we are on track for SPMDzation.
3584*d415bd75Srobert if (SPMDCompatibilityTracker.isValidState())
3585*d415bd75Srobert return AddDependence(A, this, QueryingAA);
3586*d415bd75Srobert // Not needed if we can't rewrite due to an invalid state.
3587*d415bd75Srobert if (!ReachedKnownParallelRegions.isValidState())
3588*d415bd75Srobert return AddDependence(A, this, QueryingAA);
3589*d415bd75Srobert return false;
3590*d415bd75Srobert };
3591*d415bd75Srobert
3592*d415bd75Srobert // Not needed if we are pre-runtime merge.
3593*d415bd75Srobert if (!KernelInitCB->getCalledFunction()->isDeclaration()) {
3594*d415bd75Srobert RegisterVirtualUse(OMPRTL___kmpc_get_hardware_num_threads_in_block,
3595*d415bd75Srobert CustomStateMachineUseCB);
3596*d415bd75Srobert RegisterVirtualUse(OMPRTL___kmpc_get_warp_size, CustomStateMachineUseCB);
3597*d415bd75Srobert RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_generic,
3598*d415bd75Srobert CustomStateMachineUseCB);
3599*d415bd75Srobert RegisterVirtualUse(OMPRTL___kmpc_kernel_parallel,
3600*d415bd75Srobert CustomStateMachineUseCB);
3601*d415bd75Srobert RegisterVirtualUse(OMPRTL___kmpc_kernel_end_parallel,
3602*d415bd75Srobert CustomStateMachineUseCB);
3603*d415bd75Srobert }
3604*d415bd75Srobert
3605*d415bd75Srobert // If we do not perform SPMDzation we do not need the virtual uses below.
3606*d415bd75Srobert if (SPMDCompatibilityTracker.isAtFixpoint())
3607*d415bd75Srobert return;
3608*d415bd75Srobert
3609*d415bd75Srobert Attributor::VirtualUseCallbackTy HWThreadIdUseCB =
3610*d415bd75Srobert [&](Attributor &A, const AbstractAttribute *QueryingAA) {
3611*d415bd75Srobert // Whenever we perform SPMDzation we will insert
3612*d415bd75Srobert // __kmpc_get_hardware_thread_id_in_block calls.
3613*d415bd75Srobert if (!SPMDCompatibilityTracker.isValidState())
3614*d415bd75Srobert return AddDependence(A, this, QueryingAA);
3615*d415bd75Srobert return false;
3616*d415bd75Srobert };
3617*d415bd75Srobert RegisterVirtualUse(OMPRTL___kmpc_get_hardware_thread_id_in_block,
3618*d415bd75Srobert HWThreadIdUseCB);
3619*d415bd75Srobert
3620*d415bd75Srobert Attributor::VirtualUseCallbackTy SPMDBarrierUseCB =
3621*d415bd75Srobert [&](Attributor &A, const AbstractAttribute *QueryingAA) {
3622*d415bd75Srobert // Whenever we perform SPMDzation with guarding we will insert
3623*d415bd75Srobert // __kmpc_simple_barrier_spmd calls. If SPMDzation failed, there is
3624*d415bd75Srobert // nothing to guard, or there are no parallel regions, we don't need
3625*d415bd75Srobert // the calls.
3626*d415bd75Srobert if (!SPMDCompatibilityTracker.isValidState())
3627*d415bd75Srobert return AddDependence(A, this, QueryingAA);
3628*d415bd75Srobert if (SPMDCompatibilityTracker.empty())
3629*d415bd75Srobert return AddDependence(A, this, QueryingAA);
3630*d415bd75Srobert if (!mayContainParallelRegion())
3631*d415bd75Srobert return AddDependence(A, this, QueryingAA);
3632*d415bd75Srobert return false;
3633*d415bd75Srobert };
3634*d415bd75Srobert RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_spmd, SPMDBarrierUseCB);
3635*d415bd75Srobert }
3636*d415bd75Srobert
3637*d415bd75Srobert /// Sanitize the string \p S such that it is a suitable global symbol name.
sanitizeForGlobalName__anon4b47cb5e0111::AAKernelInfoFunction3638*d415bd75Srobert static std::string sanitizeForGlobalName(std::string S) {
3639*d415bd75Srobert std::replace_if(
3640*d415bd75Srobert S.begin(), S.end(),
3641*d415bd75Srobert [](const char C) {
3642*d415bd75Srobert return !((C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') ||
3643*d415bd75Srobert (C >= '0' && C <= '9') || C == '_');
3644*d415bd75Srobert },
3645*d415bd75Srobert '.');
3646*d415bd75Srobert return S;
364773471bf0Spatrick }
364873471bf0Spatrick
364973471bf0Spatrick /// Modify the IR based on the KernelInfoState as the fixpoint iteration is
365073471bf0Spatrick /// finished now.
manifest__anon4b47cb5e0111::AAKernelInfoFunction365173471bf0Spatrick ChangeStatus manifest(Attributor &A) override {
365273471bf0Spatrick // If we are not looking at a kernel with __kmpc_target_init and
365373471bf0Spatrick // __kmpc_target_deinit call we cannot actually manifest the information.
365473471bf0Spatrick if (!KernelInitCB || !KernelDeinitCB)
365573471bf0Spatrick return ChangeStatus::UNCHANGED;
365673471bf0Spatrick
3657*d415bd75Srobert /// Insert nested Parallelism global variable
3658*d415bd75Srobert Function *Kernel = getAnchorScope();
3659*d415bd75Srobert Module &M = *Kernel->getParent();
3660*d415bd75Srobert Type *Int8Ty = Type::getInt8Ty(M.getContext());
3661*d415bd75Srobert new GlobalVariable(M, Int8Ty, /* isConstant */ true,
3662*d415bd75Srobert GlobalValue::WeakAnyLinkage,
3663*d415bd75Srobert ConstantInt::get(Int8Ty, NestedParallelism ? 1 : 0),
3664*d415bd75Srobert Kernel->getName() + "_nested_parallelism");
366573471bf0Spatrick
366673471bf0Spatrick // If we can we change the execution mode to SPMD-mode otherwise we build a
366773471bf0Spatrick // custom state machine.
3668*d415bd75Srobert ChangeStatus Changed = ChangeStatus::UNCHANGED;
3669*d415bd75Srobert if (!changeToSPMDMode(A, Changed)) {
3670*d415bd75Srobert if (!KernelInitCB->getCalledFunction()->isDeclaration())
3671*d415bd75Srobert return buildCustomStateMachine(A);
367273471bf0Spatrick }
367373471bf0Spatrick
3674*d415bd75Srobert return Changed;
3675*d415bd75Srobert }
3676*d415bd75Srobert
insertInstructionGuardsHelper__anon4b47cb5e0111::AAKernelInfoFunction3677*d415bd75Srobert void insertInstructionGuardsHelper(Attributor &A) {
367873471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
367973471bf0Spatrick
3680*d415bd75Srobert auto CreateGuardedRegion = [&](Instruction *RegionStartI,
3681*d415bd75Srobert Instruction *RegionEndI) {
3682*d415bd75Srobert LoopInfo *LI = nullptr;
3683*d415bd75Srobert DominatorTree *DT = nullptr;
3684*d415bd75Srobert MemorySSAUpdater *MSU = nullptr;
3685*d415bd75Srobert using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3686*d415bd75Srobert
3687*d415bd75Srobert BasicBlock *ParentBB = RegionStartI->getParent();
3688*d415bd75Srobert Function *Fn = ParentBB->getParent();
3689*d415bd75Srobert Module &M = *Fn->getParent();
3690*d415bd75Srobert
3691*d415bd75Srobert // Create all the blocks and logic.
3692*d415bd75Srobert // ParentBB:
3693*d415bd75Srobert // goto RegionCheckTidBB
3694*d415bd75Srobert // RegionCheckTidBB:
3695*d415bd75Srobert // Tid = __kmpc_hardware_thread_id()
3696*d415bd75Srobert // if (Tid != 0)
3697*d415bd75Srobert // goto RegionBarrierBB
3698*d415bd75Srobert // RegionStartBB:
3699*d415bd75Srobert // <execute instructions guarded>
3700*d415bd75Srobert // goto RegionEndBB
3701*d415bd75Srobert // RegionEndBB:
3702*d415bd75Srobert // <store escaping values to shared mem>
3703*d415bd75Srobert // goto RegionBarrierBB
3704*d415bd75Srobert // RegionBarrierBB:
3705*d415bd75Srobert // __kmpc_simple_barrier_spmd()
3706*d415bd75Srobert // // second barrier is omitted if lacking escaping values.
3707*d415bd75Srobert // <load escaping values from shared mem>
3708*d415bd75Srobert // __kmpc_simple_barrier_spmd()
3709*d415bd75Srobert // goto RegionExitBB
3710*d415bd75Srobert // RegionExitBB:
3711*d415bd75Srobert // <execute rest of instructions>
3712*d415bd75Srobert
3713*d415bd75Srobert BasicBlock *RegionEndBB = SplitBlock(ParentBB, RegionEndI->getNextNode(),
3714*d415bd75Srobert DT, LI, MSU, "region.guarded.end");
3715*d415bd75Srobert BasicBlock *RegionBarrierBB =
3716*d415bd75Srobert SplitBlock(RegionEndBB, &*RegionEndBB->getFirstInsertionPt(), DT, LI,
3717*d415bd75Srobert MSU, "region.barrier");
3718*d415bd75Srobert BasicBlock *RegionExitBB =
3719*d415bd75Srobert SplitBlock(RegionBarrierBB, &*RegionBarrierBB->getFirstInsertionPt(),
3720*d415bd75Srobert DT, LI, MSU, "region.exit");
3721*d415bd75Srobert BasicBlock *RegionStartBB =
3722*d415bd75Srobert SplitBlock(ParentBB, RegionStartI, DT, LI, MSU, "region.guarded");
3723*d415bd75Srobert
3724*d415bd75Srobert assert(ParentBB->getUniqueSuccessor() == RegionStartBB &&
3725*d415bd75Srobert "Expected a different CFG");
3726*d415bd75Srobert
3727*d415bd75Srobert BasicBlock *RegionCheckTidBB = SplitBlock(
3728*d415bd75Srobert ParentBB, ParentBB->getTerminator(), DT, LI, MSU, "region.check.tid");
3729*d415bd75Srobert
3730*d415bd75Srobert // Register basic blocks with the Attributor.
3731*d415bd75Srobert A.registerManifestAddedBasicBlock(*RegionEndBB);
3732*d415bd75Srobert A.registerManifestAddedBasicBlock(*RegionBarrierBB);
3733*d415bd75Srobert A.registerManifestAddedBasicBlock(*RegionExitBB);
3734*d415bd75Srobert A.registerManifestAddedBasicBlock(*RegionStartBB);
3735*d415bd75Srobert A.registerManifestAddedBasicBlock(*RegionCheckTidBB);
3736*d415bd75Srobert
3737*d415bd75Srobert bool HasBroadcastValues = false;
3738*d415bd75Srobert // Find escaping outputs from the guarded region to outside users and
3739*d415bd75Srobert // broadcast their values to them.
3740*d415bd75Srobert for (Instruction &I : *RegionStartBB) {
3741*d415bd75Srobert SmallPtrSet<Instruction *, 4> OutsideUsers;
3742*d415bd75Srobert for (User *Usr : I.users()) {
3743*d415bd75Srobert Instruction &UsrI = *cast<Instruction>(Usr);
3744*d415bd75Srobert if (UsrI.getParent() != RegionStartBB)
3745*d415bd75Srobert OutsideUsers.insert(&UsrI);
3746*d415bd75Srobert }
3747*d415bd75Srobert
3748*d415bd75Srobert if (OutsideUsers.empty())
3749*d415bd75Srobert continue;
3750*d415bd75Srobert
3751*d415bd75Srobert HasBroadcastValues = true;
3752*d415bd75Srobert
3753*d415bd75Srobert // Emit a global variable in shared memory to store the broadcasted
3754*d415bd75Srobert // value.
3755*d415bd75Srobert auto *SharedMem = new GlobalVariable(
3756*d415bd75Srobert M, I.getType(), /* IsConstant */ false,
3757*d415bd75Srobert GlobalValue::InternalLinkage, UndefValue::get(I.getType()),
3758*d415bd75Srobert sanitizeForGlobalName(
3759*d415bd75Srobert (I.getName() + ".guarded.output.alloc").str()),
3760*d415bd75Srobert nullptr, GlobalValue::NotThreadLocal,
3761*d415bd75Srobert static_cast<unsigned>(AddressSpace::Shared));
3762*d415bd75Srobert
3763*d415bd75Srobert // Emit a store instruction to update the value.
3764*d415bd75Srobert new StoreInst(&I, SharedMem, RegionEndBB->getTerminator());
3765*d415bd75Srobert
3766*d415bd75Srobert LoadInst *LoadI = new LoadInst(I.getType(), SharedMem,
3767*d415bd75Srobert I.getName() + ".guarded.output.load",
3768*d415bd75Srobert RegionBarrierBB->getTerminator());
3769*d415bd75Srobert
3770*d415bd75Srobert // Emit a load instruction and replace uses of the output value.
3771*d415bd75Srobert for (Instruction *UsrI : OutsideUsers)
3772*d415bd75Srobert UsrI->replaceUsesOfWith(&I, LoadI);
3773*d415bd75Srobert }
3774*d415bd75Srobert
3775*d415bd75Srobert auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3776*d415bd75Srobert
3777*d415bd75Srobert // Go to tid check BB in ParentBB.
3778*d415bd75Srobert const DebugLoc DL = ParentBB->getTerminator()->getDebugLoc();
3779*d415bd75Srobert ParentBB->getTerminator()->eraseFromParent();
3780*d415bd75Srobert OpenMPIRBuilder::LocationDescription Loc(
3781*d415bd75Srobert InsertPointTy(ParentBB, ParentBB->end()), DL);
3782*d415bd75Srobert OMPInfoCache.OMPBuilder.updateToLocation(Loc);
3783*d415bd75Srobert uint32_t SrcLocStrSize;
3784*d415bd75Srobert auto *SrcLocStr =
3785*d415bd75Srobert OMPInfoCache.OMPBuilder.getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3786*d415bd75Srobert Value *Ident =
3787*d415bd75Srobert OMPInfoCache.OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3788*d415bd75Srobert BranchInst::Create(RegionCheckTidBB, ParentBB)->setDebugLoc(DL);
3789*d415bd75Srobert
3790*d415bd75Srobert // Add check for Tid in RegionCheckTidBB
3791*d415bd75Srobert RegionCheckTidBB->getTerminator()->eraseFromParent();
3792*d415bd75Srobert OpenMPIRBuilder::LocationDescription LocRegionCheckTid(
3793*d415bd75Srobert InsertPointTy(RegionCheckTidBB, RegionCheckTidBB->end()), DL);
3794*d415bd75Srobert OMPInfoCache.OMPBuilder.updateToLocation(LocRegionCheckTid);
3795*d415bd75Srobert FunctionCallee HardwareTidFn =
3796*d415bd75Srobert OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
3797*d415bd75Srobert M, OMPRTL___kmpc_get_hardware_thread_id_in_block);
3798*d415bd75Srobert CallInst *Tid =
3799*d415bd75Srobert OMPInfoCache.OMPBuilder.Builder.CreateCall(HardwareTidFn, {});
3800*d415bd75Srobert Tid->setDebugLoc(DL);
3801*d415bd75Srobert OMPInfoCache.setCallingConvention(HardwareTidFn, Tid);
3802*d415bd75Srobert Value *TidCheck = OMPInfoCache.OMPBuilder.Builder.CreateIsNull(Tid);
3803*d415bd75Srobert OMPInfoCache.OMPBuilder.Builder
3804*d415bd75Srobert .CreateCondBr(TidCheck, RegionStartBB, RegionBarrierBB)
3805*d415bd75Srobert ->setDebugLoc(DL);
3806*d415bd75Srobert
3807*d415bd75Srobert // First barrier for synchronization, ensures main thread has updated
3808*d415bd75Srobert // values.
3809*d415bd75Srobert FunctionCallee BarrierFn =
3810*d415bd75Srobert OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
3811*d415bd75Srobert M, OMPRTL___kmpc_barrier_simple_spmd);
3812*d415bd75Srobert OMPInfoCache.OMPBuilder.updateToLocation(InsertPointTy(
3813*d415bd75Srobert RegionBarrierBB, RegionBarrierBB->getFirstInsertionPt()));
3814*d415bd75Srobert CallInst *Barrier =
3815*d415bd75Srobert OMPInfoCache.OMPBuilder.Builder.CreateCall(BarrierFn, {Ident, Tid});
3816*d415bd75Srobert Barrier->setDebugLoc(DL);
3817*d415bd75Srobert OMPInfoCache.setCallingConvention(BarrierFn, Barrier);
3818*d415bd75Srobert
3819*d415bd75Srobert // Second barrier ensures workers have read broadcast values.
3820*d415bd75Srobert if (HasBroadcastValues) {
3821*d415bd75Srobert CallInst *Barrier = CallInst::Create(BarrierFn, {Ident, Tid}, "",
3822*d415bd75Srobert RegionBarrierBB->getTerminator());
3823*d415bd75Srobert Barrier->setDebugLoc(DL);
3824*d415bd75Srobert OMPInfoCache.setCallingConvention(BarrierFn, Barrier);
3825*d415bd75Srobert }
3826*d415bd75Srobert };
3827*d415bd75Srobert
3828*d415bd75Srobert auto &AllocSharedRFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
3829*d415bd75Srobert SmallPtrSet<BasicBlock *, 8> Visited;
3830*d415bd75Srobert for (Instruction *GuardedI : SPMDCompatibilityTracker) {
3831*d415bd75Srobert BasicBlock *BB = GuardedI->getParent();
3832*d415bd75Srobert if (!Visited.insert(BB).second)
3833*d415bd75Srobert continue;
3834*d415bd75Srobert
3835*d415bd75Srobert SmallVector<std::pair<Instruction *, Instruction *>> Reorders;
3836*d415bd75Srobert Instruction *LastEffect = nullptr;
3837*d415bd75Srobert BasicBlock::reverse_iterator IP = BB->rbegin(), IPEnd = BB->rend();
3838*d415bd75Srobert while (++IP != IPEnd) {
3839*d415bd75Srobert if (!IP->mayHaveSideEffects() && !IP->mayReadFromMemory())
3840*d415bd75Srobert continue;
3841*d415bd75Srobert Instruction *I = &*IP;
3842*d415bd75Srobert if (OpenMPOpt::getCallIfRegularCall(*I, &AllocSharedRFI))
3843*d415bd75Srobert continue;
3844*d415bd75Srobert if (!I->user_empty() || !SPMDCompatibilityTracker.contains(I)) {
3845*d415bd75Srobert LastEffect = nullptr;
3846*d415bd75Srobert continue;
3847*d415bd75Srobert }
3848*d415bd75Srobert if (LastEffect)
3849*d415bd75Srobert Reorders.push_back({I, LastEffect});
3850*d415bd75Srobert LastEffect = &*IP;
3851*d415bd75Srobert }
3852*d415bd75Srobert for (auto &Reorder : Reorders)
3853*d415bd75Srobert Reorder.first->moveBefore(Reorder.second);
3854*d415bd75Srobert }
3855*d415bd75Srobert
3856*d415bd75Srobert SmallVector<std::pair<Instruction *, Instruction *>, 4> GuardedRegions;
3857*d415bd75Srobert
3858*d415bd75Srobert for (Instruction *GuardedI : SPMDCompatibilityTracker) {
3859*d415bd75Srobert BasicBlock *BB = GuardedI->getParent();
3860*d415bd75Srobert auto *CalleeAA = A.lookupAAFor<AAKernelInfo>(
3861*d415bd75Srobert IRPosition::function(*GuardedI->getFunction()), nullptr,
3862*d415bd75Srobert DepClassTy::NONE);
3863*d415bd75Srobert assert(CalleeAA != nullptr && "Expected Callee AAKernelInfo");
3864*d415bd75Srobert auto &CalleeAAFunction = *cast<AAKernelInfoFunction>(CalleeAA);
3865*d415bd75Srobert // Continue if instruction is already guarded.
3866*d415bd75Srobert if (CalleeAAFunction.getGuardedInstructions().contains(GuardedI))
3867*d415bd75Srobert continue;
3868*d415bd75Srobert
3869*d415bd75Srobert Instruction *GuardedRegionStart = nullptr, *GuardedRegionEnd = nullptr;
3870*d415bd75Srobert for (Instruction &I : *BB) {
3871*d415bd75Srobert // If instruction I needs to be guarded update the guarded region
3872*d415bd75Srobert // bounds.
3873*d415bd75Srobert if (SPMDCompatibilityTracker.contains(&I)) {
3874*d415bd75Srobert CalleeAAFunction.getGuardedInstructions().insert(&I);
3875*d415bd75Srobert if (GuardedRegionStart)
3876*d415bd75Srobert GuardedRegionEnd = &I;
3877*d415bd75Srobert else
3878*d415bd75Srobert GuardedRegionStart = GuardedRegionEnd = &I;
3879*d415bd75Srobert
3880*d415bd75Srobert continue;
3881*d415bd75Srobert }
3882*d415bd75Srobert
3883*d415bd75Srobert // Instruction I does not need guarding, store
3884*d415bd75Srobert // any region found and reset bounds.
3885*d415bd75Srobert if (GuardedRegionStart) {
3886*d415bd75Srobert GuardedRegions.push_back(
3887*d415bd75Srobert std::make_pair(GuardedRegionStart, GuardedRegionEnd));
3888*d415bd75Srobert GuardedRegionStart = nullptr;
3889*d415bd75Srobert GuardedRegionEnd = nullptr;
3890*d415bd75Srobert }
3891*d415bd75Srobert }
3892*d415bd75Srobert }
3893*d415bd75Srobert
3894*d415bd75Srobert for (auto &GR : GuardedRegions)
3895*d415bd75Srobert CreateGuardedRegion(GR.first, GR.second);
3896*d415bd75Srobert }
3897*d415bd75Srobert
forceSingleThreadPerWorkgroupHelper__anon4b47cb5e0111::AAKernelInfoFunction3898*d415bd75Srobert void forceSingleThreadPerWorkgroupHelper(Attributor &A) {
3899*d415bd75Srobert // Only allow 1 thread per workgroup to continue executing the user code.
3900*d415bd75Srobert //
3901*d415bd75Srobert // InitCB = __kmpc_target_init(...)
3902*d415bd75Srobert // ThreadIdInBlock = __kmpc_get_hardware_thread_id_in_block();
3903*d415bd75Srobert // if (ThreadIdInBlock != 0) return;
3904*d415bd75Srobert // UserCode:
3905*d415bd75Srobert // // user code
3906*d415bd75Srobert //
3907*d415bd75Srobert auto &Ctx = getAnchorValue().getContext();
3908*d415bd75Srobert Function *Kernel = getAssociatedFunction();
3909*d415bd75Srobert assert(Kernel && "Expected an associated function!");
3910*d415bd75Srobert
3911*d415bd75Srobert // Create block for user code to branch to from initial block.
3912*d415bd75Srobert BasicBlock *InitBB = KernelInitCB->getParent();
3913*d415bd75Srobert BasicBlock *UserCodeBB = InitBB->splitBasicBlock(
3914*d415bd75Srobert KernelInitCB->getNextNode(), "main.thread.user_code");
3915*d415bd75Srobert BasicBlock *ReturnBB =
3916*d415bd75Srobert BasicBlock::Create(Ctx, "exit.threads", Kernel, UserCodeBB);
3917*d415bd75Srobert
3918*d415bd75Srobert // Register blocks with attributor:
3919*d415bd75Srobert A.registerManifestAddedBasicBlock(*InitBB);
3920*d415bd75Srobert A.registerManifestAddedBasicBlock(*UserCodeBB);
3921*d415bd75Srobert A.registerManifestAddedBasicBlock(*ReturnBB);
3922*d415bd75Srobert
3923*d415bd75Srobert // Debug location:
3924*d415bd75Srobert const DebugLoc &DLoc = KernelInitCB->getDebugLoc();
3925*d415bd75Srobert ReturnInst::Create(Ctx, ReturnBB)->setDebugLoc(DLoc);
3926*d415bd75Srobert InitBB->getTerminator()->eraseFromParent();
3927*d415bd75Srobert
3928*d415bd75Srobert // Prepare call to OMPRTL___kmpc_get_hardware_thread_id_in_block.
3929*d415bd75Srobert Module &M = *Kernel->getParent();
3930*d415bd75Srobert auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3931*d415bd75Srobert FunctionCallee ThreadIdInBlockFn =
3932*d415bd75Srobert OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
3933*d415bd75Srobert M, OMPRTL___kmpc_get_hardware_thread_id_in_block);
3934*d415bd75Srobert
3935*d415bd75Srobert // Get thread ID in block.
3936*d415bd75Srobert CallInst *ThreadIdInBlock =
3937*d415bd75Srobert CallInst::Create(ThreadIdInBlockFn, "thread_id.in.block", InitBB);
3938*d415bd75Srobert OMPInfoCache.setCallingConvention(ThreadIdInBlockFn, ThreadIdInBlock);
3939*d415bd75Srobert ThreadIdInBlock->setDebugLoc(DLoc);
3940*d415bd75Srobert
3941*d415bd75Srobert // Eliminate all threads in the block with ID not equal to 0:
3942*d415bd75Srobert Instruction *IsMainThread =
3943*d415bd75Srobert ICmpInst::Create(ICmpInst::ICmp, CmpInst::ICMP_NE, ThreadIdInBlock,
3944*d415bd75Srobert ConstantInt::get(ThreadIdInBlock->getType(), 0),
3945*d415bd75Srobert "thread.is_main", InitBB);
3946*d415bd75Srobert IsMainThread->setDebugLoc(DLoc);
3947*d415bd75Srobert BranchInst::Create(ReturnBB, UserCodeBB, IsMainThread, InitBB);
3948*d415bd75Srobert }
3949*d415bd75Srobert
changeToSPMDMode__anon4b47cb5e0111::AAKernelInfoFunction3950*d415bd75Srobert bool changeToSPMDMode(Attributor &A, ChangeStatus &Changed) {
3951*d415bd75Srobert auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3952*d415bd75Srobert
3953*d415bd75Srobert // We cannot change to SPMD mode if the runtime functions aren't availible.
3954*d415bd75Srobert if (!OMPInfoCache.runtimeFnsAvailable(
3955*d415bd75Srobert {OMPRTL___kmpc_get_hardware_thread_id_in_block,
3956*d415bd75Srobert OMPRTL___kmpc_barrier_simple_spmd}))
3957*d415bd75Srobert return false;
3958*d415bd75Srobert
395973471bf0Spatrick if (!SPMDCompatibilityTracker.isAssumed()) {
396073471bf0Spatrick for (Instruction *NonCompatibleI : SPMDCompatibilityTracker) {
396173471bf0Spatrick if (!NonCompatibleI)
396273471bf0Spatrick continue;
396373471bf0Spatrick
396473471bf0Spatrick // Skip diagnostics on calls to known OpenMP runtime functions for now.
396573471bf0Spatrick if (auto *CB = dyn_cast<CallBase>(NonCompatibleI))
396673471bf0Spatrick if (OMPInfoCache.RTLFunctions.contains(CB->getCalledFunction()))
396773471bf0Spatrick continue;
396873471bf0Spatrick
396973471bf0Spatrick auto Remark = [&](OptimizationRemarkAnalysis ORA) {
397073471bf0Spatrick ORA << "Value has potential side effects preventing SPMD-mode "
397173471bf0Spatrick "execution";
397273471bf0Spatrick if (isa<CallBase>(NonCompatibleI)) {
397373471bf0Spatrick ORA << ". Add `__attribute__((assume(\"ompx_spmd_amenable\")))` to "
397473471bf0Spatrick "the called function to override";
397573471bf0Spatrick }
397673471bf0Spatrick return ORA << ".";
397773471bf0Spatrick };
397873471bf0Spatrick A.emitRemark<OptimizationRemarkAnalysis>(NonCompatibleI, "OMP121",
397973471bf0Spatrick Remark);
398073471bf0Spatrick
398173471bf0Spatrick LLVM_DEBUG(dbgs() << TAG << "SPMD-incompatible side-effect: "
398273471bf0Spatrick << *NonCompatibleI << "\n");
398373471bf0Spatrick }
398473471bf0Spatrick
398573471bf0Spatrick return false;
398673471bf0Spatrick }
398773471bf0Spatrick
3988*d415bd75Srobert // Get the actual kernel, could be the caller of the anchor scope if we have
3989*d415bd75Srobert // a debug wrapper.
399073471bf0Spatrick Function *Kernel = getAnchorScope();
3991*d415bd75Srobert if (Kernel->hasLocalLinkage()) {
3992*d415bd75Srobert assert(Kernel->hasOneUse() && "Unexpected use of debug kernel wrapper.");
3993*d415bd75Srobert auto *CB = cast<CallBase>(Kernel->user_back());
3994*d415bd75Srobert Kernel = CB->getCaller();
3995*d415bd75Srobert }
3996*d415bd75Srobert assert(OMPInfoCache.Kernels.count(Kernel) && "Expected kernel function!");
3997*d415bd75Srobert
3998*d415bd75Srobert // Check if the kernel is already in SPMD mode, if so, return success.
399973471bf0Spatrick GlobalVariable *ExecMode = Kernel->getParent()->getGlobalVariable(
400073471bf0Spatrick (Kernel->getName() + "_exec_mode").str());
400173471bf0Spatrick assert(ExecMode && "Kernel without exec mode?");
4002*d415bd75Srobert assert(ExecMode->getInitializer() && "ExecMode doesn't have initializer!");
400373471bf0Spatrick
400473471bf0Spatrick // Set the global exec mode flag to indicate SPMD-Generic mode.
4005*d415bd75Srobert assert(isa<ConstantInt>(ExecMode->getInitializer()) &&
4006*d415bd75Srobert "ExecMode is not an integer!");
4007*d415bd75Srobert const int8_t ExecModeVal =
4008*d415bd75Srobert cast<ConstantInt>(ExecMode->getInitializer())->getSExtValue();
4009*d415bd75Srobert if (ExecModeVal != OMP_TGT_EXEC_MODE_GENERIC)
4010*d415bd75Srobert return true;
4011*d415bd75Srobert
4012*d415bd75Srobert // We will now unconditionally modify the IR, indicate a change.
4013*d415bd75Srobert Changed = ChangeStatus::CHANGED;
4014*d415bd75Srobert
4015*d415bd75Srobert // Do not use instruction guards when no parallel is present inside
4016*d415bd75Srobert // the target region.
4017*d415bd75Srobert if (mayContainParallelRegion())
4018*d415bd75Srobert insertInstructionGuardsHelper(A);
4019*d415bd75Srobert else
4020*d415bd75Srobert forceSingleThreadPerWorkgroupHelper(A);
4021*d415bd75Srobert
4022*d415bd75Srobert // Adjust the global exec mode flag that tells the runtime what mode this
4023*d415bd75Srobert // kernel is executed in.
4024*d415bd75Srobert assert(ExecModeVal == OMP_TGT_EXEC_MODE_GENERIC &&
4025*d415bd75Srobert "Initially non-SPMD kernel has SPMD exec mode!");
402673471bf0Spatrick ExecMode->setInitializer(
4027*d415bd75Srobert ConstantInt::get(ExecMode->getInitializer()->getType(),
4028*d415bd75Srobert ExecModeVal | OMP_TGT_EXEC_MODE_GENERIC_SPMD));
402973471bf0Spatrick
403073471bf0Spatrick // Next rewrite the init and deinit calls to indicate we use SPMD-mode now.
4031*d415bd75Srobert const int InitModeArgNo = 1;
4032*d415bd75Srobert const int DeinitModeArgNo = 1;
403373471bf0Spatrick const int InitUseStateMachineArgNo = 2;
403473471bf0Spatrick
403573471bf0Spatrick auto &Ctx = getAnchorValue().getContext();
4036*d415bd75Srobert A.changeUseAfterManifest(
4037*d415bd75Srobert KernelInitCB->getArgOperandUse(InitModeArgNo),
4038*d415bd75Srobert *ConstantInt::getSigned(IntegerType::getInt8Ty(Ctx),
4039*d415bd75Srobert OMP_TGT_EXEC_MODE_SPMD));
404073471bf0Spatrick A.changeUseAfterManifest(
404173471bf0Spatrick KernelInitCB->getArgOperandUse(InitUseStateMachineArgNo),
4042*d415bd75Srobert *ConstantInt::getBool(Ctx, false));
404373471bf0Spatrick A.changeUseAfterManifest(
4044*d415bd75Srobert KernelDeinitCB->getArgOperandUse(DeinitModeArgNo),
4045*d415bd75Srobert *ConstantInt::getSigned(IntegerType::getInt8Ty(Ctx),
4046*d415bd75Srobert OMP_TGT_EXEC_MODE_SPMD));
404773471bf0Spatrick
404873471bf0Spatrick ++NumOpenMPTargetRegionKernelsSPMD;
404973471bf0Spatrick
405073471bf0Spatrick auto Remark = [&](OptimizationRemark OR) {
405173471bf0Spatrick return OR << "Transformed generic-mode kernel to SPMD-mode.";
405273471bf0Spatrick };
405373471bf0Spatrick A.emitRemark<OptimizationRemark>(KernelInitCB, "OMP120", Remark);
405473471bf0Spatrick return true;
405573471bf0Spatrick };
405673471bf0Spatrick
buildCustomStateMachine__anon4b47cb5e0111::AAKernelInfoFunction405773471bf0Spatrick ChangeStatus buildCustomStateMachine(Attributor &A) {
4058*d415bd75Srobert // If we have disabled state machine rewrites, don't make a custom one
4059*d415bd75Srobert if (DisableOpenMPOptStateMachineRewrite)
4060*d415bd75Srobert return ChangeStatus::UNCHANGED;
406173471bf0Spatrick
4062*d415bd75Srobert // Don't rewrite the state machine if we are not in a valid state.
4063*d415bd75Srobert if (!ReachedKnownParallelRegions.isValidState())
4064*d415bd75Srobert return ChangeStatus::UNCHANGED;
4065*d415bd75Srobert
4066*d415bd75Srobert auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
4067*d415bd75Srobert if (!OMPInfoCache.runtimeFnsAvailable(
4068*d415bd75Srobert {OMPRTL___kmpc_get_hardware_num_threads_in_block,
4069*d415bd75Srobert OMPRTL___kmpc_get_warp_size, OMPRTL___kmpc_barrier_simple_generic,
4070*d415bd75Srobert OMPRTL___kmpc_kernel_parallel, OMPRTL___kmpc_kernel_end_parallel}))
4071*d415bd75Srobert return ChangeStatus::UNCHANGED;
4072*d415bd75Srobert
4073*d415bd75Srobert const int InitModeArgNo = 1;
407473471bf0Spatrick const int InitUseStateMachineArgNo = 2;
407573471bf0Spatrick
407673471bf0Spatrick // Check if the current configuration is non-SPMD and generic state machine.
407773471bf0Spatrick // If we already have SPMD mode or a custom state machine we do not need to
407873471bf0Spatrick // go any further. If it is anything but a constant something is weird and
407973471bf0Spatrick // we give up.
408073471bf0Spatrick ConstantInt *UseStateMachine = dyn_cast<ConstantInt>(
408173471bf0Spatrick KernelInitCB->getArgOperand(InitUseStateMachineArgNo));
4082*d415bd75Srobert ConstantInt *Mode =
4083*d415bd75Srobert dyn_cast<ConstantInt>(KernelInitCB->getArgOperand(InitModeArgNo));
408473471bf0Spatrick
408573471bf0Spatrick // If we are stuck with generic mode, try to create a custom device (=GPU)
408673471bf0Spatrick // state machine which is specialized for the parallel regions that are
408773471bf0Spatrick // reachable by the kernel.
4088*d415bd75Srobert if (!UseStateMachine || UseStateMachine->isZero() || !Mode ||
4089*d415bd75Srobert (Mode->getSExtValue() & OMP_TGT_EXEC_MODE_SPMD))
409073471bf0Spatrick return ChangeStatus::UNCHANGED;
409173471bf0Spatrick
409273471bf0Spatrick // If not SPMD mode, indicate we use a custom state machine now.
409373471bf0Spatrick auto &Ctx = getAnchorValue().getContext();
4094*d415bd75Srobert auto *FalseVal = ConstantInt::getBool(Ctx, false);
409573471bf0Spatrick A.changeUseAfterManifest(
409673471bf0Spatrick KernelInitCB->getArgOperandUse(InitUseStateMachineArgNo), *FalseVal);
409773471bf0Spatrick
409873471bf0Spatrick // If we don't actually need a state machine we are done here. This can
409973471bf0Spatrick // happen if there simply are no parallel regions. In the resulting kernel
410073471bf0Spatrick // all worker threads will simply exit right away, leaving the main thread
410173471bf0Spatrick // to do the work alone.
4102*d415bd75Srobert if (!mayContainParallelRegion()) {
410373471bf0Spatrick ++NumOpenMPTargetRegionKernelsWithoutStateMachine;
410473471bf0Spatrick
410573471bf0Spatrick auto Remark = [&](OptimizationRemark OR) {
410673471bf0Spatrick return OR << "Removing unused state machine from generic-mode kernel.";
410773471bf0Spatrick };
410873471bf0Spatrick A.emitRemark<OptimizationRemark>(KernelInitCB, "OMP130", Remark);
410973471bf0Spatrick
411073471bf0Spatrick return ChangeStatus::CHANGED;
411173471bf0Spatrick }
411273471bf0Spatrick
411373471bf0Spatrick // Keep track in the statistics of our new shiny custom state machine.
411473471bf0Spatrick if (ReachedUnknownParallelRegions.empty()) {
411573471bf0Spatrick ++NumOpenMPTargetRegionKernelsCustomStateMachineWithoutFallback;
411673471bf0Spatrick
411773471bf0Spatrick auto Remark = [&](OptimizationRemark OR) {
411873471bf0Spatrick return OR << "Rewriting generic-mode kernel with a customized state "
411973471bf0Spatrick "machine.";
412073471bf0Spatrick };
412173471bf0Spatrick A.emitRemark<OptimizationRemark>(KernelInitCB, "OMP131", Remark);
412273471bf0Spatrick } else {
412373471bf0Spatrick ++NumOpenMPTargetRegionKernelsCustomStateMachineWithFallback;
412473471bf0Spatrick
412573471bf0Spatrick auto Remark = [&](OptimizationRemarkAnalysis OR) {
412673471bf0Spatrick return OR << "Generic-mode kernel is executed with a customized state "
412773471bf0Spatrick "machine that requires a fallback.";
412873471bf0Spatrick };
412973471bf0Spatrick A.emitRemark<OptimizationRemarkAnalysis>(KernelInitCB, "OMP132", Remark);
413073471bf0Spatrick
413173471bf0Spatrick // Tell the user why we ended up with a fallback.
413273471bf0Spatrick for (CallBase *UnknownParallelRegionCB : ReachedUnknownParallelRegions) {
413373471bf0Spatrick if (!UnknownParallelRegionCB)
413473471bf0Spatrick continue;
413573471bf0Spatrick auto Remark = [&](OptimizationRemarkAnalysis ORA) {
413673471bf0Spatrick return ORA << "Call may contain unknown parallel regions. Use "
413773471bf0Spatrick << "`__attribute__((assume(\"omp_no_parallelism\")))` to "
413873471bf0Spatrick "override.";
413973471bf0Spatrick };
414073471bf0Spatrick A.emitRemark<OptimizationRemarkAnalysis>(UnknownParallelRegionCB,
414173471bf0Spatrick "OMP133", Remark);
414273471bf0Spatrick }
414373471bf0Spatrick }
414473471bf0Spatrick
414573471bf0Spatrick // Create all the blocks:
414673471bf0Spatrick //
414773471bf0Spatrick // InitCB = __kmpc_target_init(...)
4148*d415bd75Srobert // BlockHwSize =
4149*d415bd75Srobert // __kmpc_get_hardware_num_threads_in_block();
4150*d415bd75Srobert // WarpSize = __kmpc_get_warp_size();
4151*d415bd75Srobert // BlockSize = BlockHwSize - WarpSize;
4152*d415bd75Srobert // IsWorkerCheckBB: bool IsWorker = InitCB != -1;
415373471bf0Spatrick // if (IsWorker) {
4154*d415bd75Srobert // if (InitCB >= BlockSize) return;
4155*d415bd75Srobert // SMBeginBB: __kmpc_barrier_simple_generic(...);
415673471bf0Spatrick // void *WorkFn;
415773471bf0Spatrick // bool Active = __kmpc_kernel_parallel(&WorkFn);
415873471bf0Spatrick // if (!WorkFn) return;
415973471bf0Spatrick // SMIsActiveCheckBB: if (Active) {
416073471bf0Spatrick // SMIfCascadeCurrentBB: if (WorkFn == <ParFn0>)
416173471bf0Spatrick // ParFn0(...);
416273471bf0Spatrick // SMIfCascadeCurrentBB: else if (WorkFn == <ParFn1>)
416373471bf0Spatrick // ParFn1(...);
416473471bf0Spatrick // ...
416573471bf0Spatrick // SMIfCascadeCurrentBB: else
416673471bf0Spatrick // ((WorkFnTy*)WorkFn)(...);
416773471bf0Spatrick // SMEndParallelBB: __kmpc_kernel_end_parallel(...);
416873471bf0Spatrick // }
4169*d415bd75Srobert // SMDoneBB: __kmpc_barrier_simple_generic(...);
417073471bf0Spatrick // goto SMBeginBB;
417173471bf0Spatrick // }
417273471bf0Spatrick // UserCodeEntryBB: // user code
417373471bf0Spatrick // __kmpc_target_deinit(...)
417473471bf0Spatrick //
417573471bf0Spatrick Function *Kernel = getAssociatedFunction();
417673471bf0Spatrick assert(Kernel && "Expected an associated function!");
417773471bf0Spatrick
417873471bf0Spatrick BasicBlock *InitBB = KernelInitCB->getParent();
417973471bf0Spatrick BasicBlock *UserCodeEntryBB = InitBB->splitBasicBlock(
418073471bf0Spatrick KernelInitCB->getNextNode(), "thread.user_code.check");
4181*d415bd75Srobert BasicBlock *IsWorkerCheckBB =
4182*d415bd75Srobert BasicBlock::Create(Ctx, "is_worker_check", Kernel, UserCodeEntryBB);
418373471bf0Spatrick BasicBlock *StateMachineBeginBB = BasicBlock::Create(
418473471bf0Spatrick Ctx, "worker_state_machine.begin", Kernel, UserCodeEntryBB);
418573471bf0Spatrick BasicBlock *StateMachineFinishedBB = BasicBlock::Create(
418673471bf0Spatrick Ctx, "worker_state_machine.finished", Kernel, UserCodeEntryBB);
418773471bf0Spatrick BasicBlock *StateMachineIsActiveCheckBB = BasicBlock::Create(
418873471bf0Spatrick Ctx, "worker_state_machine.is_active.check", Kernel, UserCodeEntryBB);
418973471bf0Spatrick BasicBlock *StateMachineIfCascadeCurrentBB =
419073471bf0Spatrick BasicBlock::Create(Ctx, "worker_state_machine.parallel_region.check",
419173471bf0Spatrick Kernel, UserCodeEntryBB);
419273471bf0Spatrick BasicBlock *StateMachineEndParallelBB =
419373471bf0Spatrick BasicBlock::Create(Ctx, "worker_state_machine.parallel_region.end",
419473471bf0Spatrick Kernel, UserCodeEntryBB);
419573471bf0Spatrick BasicBlock *StateMachineDoneBarrierBB = BasicBlock::Create(
419673471bf0Spatrick Ctx, "worker_state_machine.done.barrier", Kernel, UserCodeEntryBB);
419773471bf0Spatrick A.registerManifestAddedBasicBlock(*InitBB);
419873471bf0Spatrick A.registerManifestAddedBasicBlock(*UserCodeEntryBB);
4199*d415bd75Srobert A.registerManifestAddedBasicBlock(*IsWorkerCheckBB);
420073471bf0Spatrick A.registerManifestAddedBasicBlock(*StateMachineBeginBB);
420173471bf0Spatrick A.registerManifestAddedBasicBlock(*StateMachineFinishedBB);
420273471bf0Spatrick A.registerManifestAddedBasicBlock(*StateMachineIsActiveCheckBB);
420373471bf0Spatrick A.registerManifestAddedBasicBlock(*StateMachineIfCascadeCurrentBB);
420473471bf0Spatrick A.registerManifestAddedBasicBlock(*StateMachineEndParallelBB);
420573471bf0Spatrick A.registerManifestAddedBasicBlock(*StateMachineDoneBarrierBB);
420673471bf0Spatrick
420773471bf0Spatrick const DebugLoc &DLoc = KernelInitCB->getDebugLoc();
420873471bf0Spatrick ReturnInst::Create(Ctx, StateMachineFinishedBB)->setDebugLoc(DLoc);
420973471bf0Spatrick InitBB->getTerminator()->eraseFromParent();
4210*d415bd75Srobert
421173471bf0Spatrick Instruction *IsWorker =
421273471bf0Spatrick ICmpInst::Create(ICmpInst::ICmp, llvm::CmpInst::ICMP_NE, KernelInitCB,
421373471bf0Spatrick ConstantInt::get(KernelInitCB->getType(), -1),
421473471bf0Spatrick "thread.is_worker", InitBB);
421573471bf0Spatrick IsWorker->setDebugLoc(DLoc);
4216*d415bd75Srobert BranchInst::Create(IsWorkerCheckBB, UserCodeEntryBB, IsWorker, InitBB);
421773471bf0Spatrick
421873471bf0Spatrick Module &M = *Kernel->getParent();
4219*d415bd75Srobert FunctionCallee BlockHwSizeFn =
4220*d415bd75Srobert OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4221*d415bd75Srobert M, OMPRTL___kmpc_get_hardware_num_threads_in_block);
4222*d415bd75Srobert FunctionCallee WarpSizeFn =
4223*d415bd75Srobert OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4224*d415bd75Srobert M, OMPRTL___kmpc_get_warp_size);
4225*d415bd75Srobert CallInst *BlockHwSize =
4226*d415bd75Srobert CallInst::Create(BlockHwSizeFn, "block.hw_size", IsWorkerCheckBB);
4227*d415bd75Srobert OMPInfoCache.setCallingConvention(BlockHwSizeFn, BlockHwSize);
4228*d415bd75Srobert BlockHwSize->setDebugLoc(DLoc);
4229*d415bd75Srobert CallInst *WarpSize =
4230*d415bd75Srobert CallInst::Create(WarpSizeFn, "warp.size", IsWorkerCheckBB);
4231*d415bd75Srobert OMPInfoCache.setCallingConvention(WarpSizeFn, WarpSize);
4232*d415bd75Srobert WarpSize->setDebugLoc(DLoc);
4233*d415bd75Srobert Instruction *BlockSize = BinaryOperator::CreateSub(
4234*d415bd75Srobert BlockHwSize, WarpSize, "block.size", IsWorkerCheckBB);
4235*d415bd75Srobert BlockSize->setDebugLoc(DLoc);
4236*d415bd75Srobert Instruction *IsMainOrWorker = ICmpInst::Create(
4237*d415bd75Srobert ICmpInst::ICmp, llvm::CmpInst::ICMP_SLT, KernelInitCB, BlockSize,
4238*d415bd75Srobert "thread.is_main_or_worker", IsWorkerCheckBB);
4239*d415bd75Srobert IsMainOrWorker->setDebugLoc(DLoc);
4240*d415bd75Srobert BranchInst::Create(StateMachineBeginBB, StateMachineFinishedBB,
4241*d415bd75Srobert IsMainOrWorker, IsWorkerCheckBB);
424273471bf0Spatrick
424373471bf0Spatrick // Create local storage for the work function pointer.
424473471bf0Spatrick const DataLayout &DL = M.getDataLayout();
424573471bf0Spatrick Type *VoidPtrTy = Type::getInt8PtrTy(Ctx);
424673471bf0Spatrick Instruction *WorkFnAI =
424773471bf0Spatrick new AllocaInst(VoidPtrTy, DL.getAllocaAddrSpace(), nullptr,
424873471bf0Spatrick "worker.work_fn.addr", &Kernel->getEntryBlock().front());
424973471bf0Spatrick WorkFnAI->setDebugLoc(DLoc);
425073471bf0Spatrick
425173471bf0Spatrick OMPInfoCache.OMPBuilder.updateToLocation(
425273471bf0Spatrick OpenMPIRBuilder::LocationDescription(
425373471bf0Spatrick IRBuilder<>::InsertPoint(StateMachineBeginBB,
425473471bf0Spatrick StateMachineBeginBB->end()),
425573471bf0Spatrick DLoc));
425673471bf0Spatrick
425773471bf0Spatrick Value *Ident = KernelInitCB->getArgOperand(0);
425873471bf0Spatrick Value *GTid = KernelInitCB;
425973471bf0Spatrick
426073471bf0Spatrick FunctionCallee BarrierFn =
426173471bf0Spatrick OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4262*d415bd75Srobert M, OMPRTL___kmpc_barrier_simple_generic);
4263*d415bd75Srobert CallInst *Barrier =
4264*d415bd75Srobert CallInst::Create(BarrierFn, {Ident, GTid}, "", StateMachineBeginBB);
4265*d415bd75Srobert OMPInfoCache.setCallingConvention(BarrierFn, Barrier);
4266*d415bd75Srobert Barrier->setDebugLoc(DLoc);
426773471bf0Spatrick
426873471bf0Spatrick if (WorkFnAI->getType()->getPointerAddressSpace() !=
426973471bf0Spatrick (unsigned int)AddressSpace::Generic) {
427073471bf0Spatrick WorkFnAI = new AddrSpaceCastInst(
427173471bf0Spatrick WorkFnAI,
427273471bf0Spatrick PointerType::getWithSamePointeeType(
427373471bf0Spatrick cast<PointerType>(WorkFnAI->getType()),
427473471bf0Spatrick (unsigned int)AddressSpace::Generic),
427573471bf0Spatrick WorkFnAI->getName() + ".generic", StateMachineBeginBB);
427673471bf0Spatrick WorkFnAI->setDebugLoc(DLoc);
427773471bf0Spatrick }
427873471bf0Spatrick
427973471bf0Spatrick FunctionCallee KernelParallelFn =
428073471bf0Spatrick OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
428173471bf0Spatrick M, OMPRTL___kmpc_kernel_parallel);
4282*d415bd75Srobert CallInst *IsActiveWorker = CallInst::Create(
428373471bf0Spatrick KernelParallelFn, {WorkFnAI}, "worker.is_active", StateMachineBeginBB);
4284*d415bd75Srobert OMPInfoCache.setCallingConvention(KernelParallelFn, IsActiveWorker);
428573471bf0Spatrick IsActiveWorker->setDebugLoc(DLoc);
428673471bf0Spatrick Instruction *WorkFn = new LoadInst(VoidPtrTy, WorkFnAI, "worker.work_fn",
428773471bf0Spatrick StateMachineBeginBB);
428873471bf0Spatrick WorkFn->setDebugLoc(DLoc);
428973471bf0Spatrick
429073471bf0Spatrick FunctionType *ParallelRegionFnTy = FunctionType::get(
429173471bf0Spatrick Type::getVoidTy(Ctx), {Type::getInt16Ty(Ctx), Type::getInt32Ty(Ctx)},
429273471bf0Spatrick false);
429373471bf0Spatrick Value *WorkFnCast = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
429473471bf0Spatrick WorkFn, ParallelRegionFnTy->getPointerTo(), "worker.work_fn.addr_cast",
429573471bf0Spatrick StateMachineBeginBB);
429673471bf0Spatrick
429773471bf0Spatrick Instruction *IsDone =
429873471bf0Spatrick ICmpInst::Create(ICmpInst::ICmp, llvm::CmpInst::ICMP_EQ, WorkFn,
429973471bf0Spatrick Constant::getNullValue(VoidPtrTy), "worker.is_done",
430073471bf0Spatrick StateMachineBeginBB);
430173471bf0Spatrick IsDone->setDebugLoc(DLoc);
430273471bf0Spatrick BranchInst::Create(StateMachineFinishedBB, StateMachineIsActiveCheckBB,
430373471bf0Spatrick IsDone, StateMachineBeginBB)
430473471bf0Spatrick ->setDebugLoc(DLoc);
430573471bf0Spatrick
430673471bf0Spatrick BranchInst::Create(StateMachineIfCascadeCurrentBB,
430773471bf0Spatrick StateMachineDoneBarrierBB, IsActiveWorker,
430873471bf0Spatrick StateMachineIsActiveCheckBB)
430973471bf0Spatrick ->setDebugLoc(DLoc);
431073471bf0Spatrick
431173471bf0Spatrick Value *ZeroArg =
431273471bf0Spatrick Constant::getNullValue(ParallelRegionFnTy->getParamType(0));
431373471bf0Spatrick
431473471bf0Spatrick // Now that we have most of the CFG skeleton it is time for the if-cascade
431573471bf0Spatrick // that checks the function pointer we got from the runtime against the
431673471bf0Spatrick // parallel regions we expect, if there are any.
4317*d415bd75Srobert for (int I = 0, E = ReachedKnownParallelRegions.size(); I < E; ++I) {
4318*d415bd75Srobert auto *ParallelRegion = ReachedKnownParallelRegions[I];
431973471bf0Spatrick BasicBlock *PRExecuteBB = BasicBlock::Create(
432073471bf0Spatrick Ctx, "worker_state_machine.parallel_region.execute", Kernel,
432173471bf0Spatrick StateMachineEndParallelBB);
432273471bf0Spatrick CallInst::Create(ParallelRegion, {ZeroArg, GTid}, "", PRExecuteBB)
432373471bf0Spatrick ->setDebugLoc(DLoc);
432473471bf0Spatrick BranchInst::Create(StateMachineEndParallelBB, PRExecuteBB)
432573471bf0Spatrick ->setDebugLoc(DLoc);
432673471bf0Spatrick
432773471bf0Spatrick BasicBlock *PRNextBB =
432873471bf0Spatrick BasicBlock::Create(Ctx, "worker_state_machine.parallel_region.check",
432973471bf0Spatrick Kernel, StateMachineEndParallelBB);
433073471bf0Spatrick
433173471bf0Spatrick // Check if we need to compare the pointer at all or if we can just
433273471bf0Spatrick // call the parallel region function.
433373471bf0Spatrick Value *IsPR;
4334*d415bd75Srobert if (I + 1 < E || !ReachedUnknownParallelRegions.empty()) {
433573471bf0Spatrick Instruction *CmpI = ICmpInst::Create(
433673471bf0Spatrick ICmpInst::ICmp, llvm::CmpInst::ICMP_EQ, WorkFnCast, ParallelRegion,
433773471bf0Spatrick "worker.check_parallel_region", StateMachineIfCascadeCurrentBB);
433873471bf0Spatrick CmpI->setDebugLoc(DLoc);
433973471bf0Spatrick IsPR = CmpI;
434073471bf0Spatrick } else {
434173471bf0Spatrick IsPR = ConstantInt::getTrue(Ctx);
434273471bf0Spatrick }
434373471bf0Spatrick
434473471bf0Spatrick BranchInst::Create(PRExecuteBB, PRNextBB, IsPR,
434573471bf0Spatrick StateMachineIfCascadeCurrentBB)
434673471bf0Spatrick ->setDebugLoc(DLoc);
434773471bf0Spatrick StateMachineIfCascadeCurrentBB = PRNextBB;
434873471bf0Spatrick }
434973471bf0Spatrick
435073471bf0Spatrick // At the end of the if-cascade we place the indirect function pointer call
435173471bf0Spatrick // in case we might need it, that is if there can be parallel regions we
435273471bf0Spatrick // have not handled in the if-cascade above.
435373471bf0Spatrick if (!ReachedUnknownParallelRegions.empty()) {
435473471bf0Spatrick StateMachineIfCascadeCurrentBB->setName(
435573471bf0Spatrick "worker_state_machine.parallel_region.fallback.execute");
435673471bf0Spatrick CallInst::Create(ParallelRegionFnTy, WorkFnCast, {ZeroArg, GTid}, "",
435773471bf0Spatrick StateMachineIfCascadeCurrentBB)
435873471bf0Spatrick ->setDebugLoc(DLoc);
435973471bf0Spatrick }
436073471bf0Spatrick BranchInst::Create(StateMachineEndParallelBB,
436173471bf0Spatrick StateMachineIfCascadeCurrentBB)
436273471bf0Spatrick ->setDebugLoc(DLoc);
436373471bf0Spatrick
4364*d415bd75Srobert FunctionCallee EndParallelFn =
4365*d415bd75Srobert OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4366*d415bd75Srobert M, OMPRTL___kmpc_kernel_end_parallel);
4367*d415bd75Srobert CallInst *EndParallel =
4368*d415bd75Srobert CallInst::Create(EndParallelFn, {}, "", StateMachineEndParallelBB);
4369*d415bd75Srobert OMPInfoCache.setCallingConvention(EndParallelFn, EndParallel);
4370*d415bd75Srobert EndParallel->setDebugLoc(DLoc);
437173471bf0Spatrick BranchInst::Create(StateMachineDoneBarrierBB, StateMachineEndParallelBB)
437273471bf0Spatrick ->setDebugLoc(DLoc);
437373471bf0Spatrick
437473471bf0Spatrick CallInst::Create(BarrierFn, {Ident, GTid}, "", StateMachineDoneBarrierBB)
437573471bf0Spatrick ->setDebugLoc(DLoc);
437673471bf0Spatrick BranchInst::Create(StateMachineBeginBB, StateMachineDoneBarrierBB)
437773471bf0Spatrick ->setDebugLoc(DLoc);
437873471bf0Spatrick
437973471bf0Spatrick return ChangeStatus::CHANGED;
438073471bf0Spatrick }
438173471bf0Spatrick
438273471bf0Spatrick /// Fixpoint iteration update function. Will be called every time a dependence
438373471bf0Spatrick /// changed its state (and in the beginning).
updateImpl__anon4b47cb5e0111::AAKernelInfoFunction438473471bf0Spatrick ChangeStatus updateImpl(Attributor &A) override {
438573471bf0Spatrick KernelInfoState StateBefore = getState();
438673471bf0Spatrick
438773471bf0Spatrick // Callback to check a read/write instruction.
438873471bf0Spatrick auto CheckRWInst = [&](Instruction &I) {
438973471bf0Spatrick // We handle calls later.
439073471bf0Spatrick if (isa<CallBase>(I))
439173471bf0Spatrick return true;
439273471bf0Spatrick // We only care about write effects.
439373471bf0Spatrick if (!I.mayWriteToMemory())
439473471bf0Spatrick return true;
439573471bf0Spatrick if (auto *SI = dyn_cast<StoreInst>(&I)) {
4396*d415bd75Srobert const auto &UnderlyingObjsAA = A.getAAFor<AAUnderlyingObjects>(
4397*d415bd75Srobert *this, IRPosition::value(*SI->getPointerOperand()),
4398*d415bd75Srobert DepClassTy::OPTIONAL);
4399*d415bd75Srobert auto &HS = A.getAAFor<AAHeapToStack>(
4400*d415bd75Srobert *this, IRPosition::function(*I.getFunction()),
4401*d415bd75Srobert DepClassTy::OPTIONAL);
4402*d415bd75Srobert if (UnderlyingObjsAA.forallUnderlyingObjects([&](Value &Obj) {
4403*d415bd75Srobert if (AA::isAssumedThreadLocalObject(A, Obj, *this))
4404*d415bd75Srobert return true;
4405*d415bd75Srobert // Check for AAHeapToStack moved objects which must not be
4406*d415bd75Srobert // guarded.
4407*d415bd75Srobert auto *CB = dyn_cast<CallBase>(&Obj);
4408*d415bd75Srobert return CB && HS.isAssumedHeapToStack(*CB);
4409*d415bd75Srobert }))
441073471bf0Spatrick return true;
441173471bf0Spatrick }
4412*d415bd75Srobert
4413*d415bd75Srobert // Insert instruction that needs guarding.
441473471bf0Spatrick SPMDCompatibilityTracker.insert(&I);
441573471bf0Spatrick return true;
441673471bf0Spatrick };
441773471bf0Spatrick
441873471bf0Spatrick bool UsedAssumedInformationInCheckRWInst = false;
441973471bf0Spatrick if (!SPMDCompatibilityTracker.isAtFixpoint())
442073471bf0Spatrick if (!A.checkForAllReadWriteInstructions(
442173471bf0Spatrick CheckRWInst, *this, UsedAssumedInformationInCheckRWInst))
442273471bf0Spatrick SPMDCompatibilityTracker.indicatePessimisticFixpoint();
442373471bf0Spatrick
4424*d415bd75Srobert bool UsedAssumedInformationFromReachingKernels = false;
442573471bf0Spatrick if (!IsKernelEntry) {
442673471bf0Spatrick updateParallelLevels(A);
4427*d415bd75Srobert
4428*d415bd75Srobert bool AllReachingKernelsKnown = true;
4429*d415bd75Srobert updateReachingKernelEntries(A, AllReachingKernelsKnown);
4430*d415bd75Srobert UsedAssumedInformationFromReachingKernels = !AllReachingKernelsKnown;
4431*d415bd75Srobert
4432*d415bd75Srobert if (!SPMDCompatibilityTracker.empty()) {
4433*d415bd75Srobert if (!ParallelLevels.isValidState())
4434*d415bd75Srobert SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4435*d415bd75Srobert else if (!ReachingKernelEntries.isValidState())
4436*d415bd75Srobert SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4437*d415bd75Srobert else {
4438*d415bd75Srobert // Check if all reaching kernels agree on the mode as we can otherwise
4439*d415bd75Srobert // not guard instructions. We might not be sure about the mode so we
4440*d415bd75Srobert // we cannot fix the internal spmd-zation state either.
4441*d415bd75Srobert int SPMD = 0, Generic = 0;
4442*d415bd75Srobert for (auto *Kernel : ReachingKernelEntries) {
4443*d415bd75Srobert auto &CBAA = A.getAAFor<AAKernelInfo>(
4444*d415bd75Srobert *this, IRPosition::function(*Kernel), DepClassTy::OPTIONAL);
4445*d415bd75Srobert if (CBAA.SPMDCompatibilityTracker.isValidState() &&
4446*d415bd75Srobert CBAA.SPMDCompatibilityTracker.isAssumed())
4447*d415bd75Srobert ++SPMD;
4448*d415bd75Srobert else
4449*d415bd75Srobert ++Generic;
4450*d415bd75Srobert if (!CBAA.SPMDCompatibilityTracker.isAtFixpoint())
4451*d415bd75Srobert UsedAssumedInformationFromReachingKernels = true;
4452*d415bd75Srobert }
4453*d415bd75Srobert if (SPMD != 0 && Generic != 0)
4454*d415bd75Srobert SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4455*d415bd75Srobert }
4456*d415bd75Srobert }
445773471bf0Spatrick }
445873471bf0Spatrick
445973471bf0Spatrick // Callback to check a call instruction.
4460*d415bd75Srobert bool AllParallelRegionStatesWereFixed = true;
446173471bf0Spatrick bool AllSPMDStatesWereFixed = true;
446273471bf0Spatrick auto CheckCallInst = [&](Instruction &I) {
446373471bf0Spatrick auto &CB = cast<CallBase>(I);
446473471bf0Spatrick auto &CBAA = A.getAAFor<AAKernelInfo>(
446573471bf0Spatrick *this, IRPosition::callsite_function(CB), DepClassTy::OPTIONAL);
446673471bf0Spatrick getState() ^= CBAA.getState();
446773471bf0Spatrick AllSPMDStatesWereFixed &= CBAA.SPMDCompatibilityTracker.isAtFixpoint();
4468*d415bd75Srobert AllParallelRegionStatesWereFixed &=
4469*d415bd75Srobert CBAA.ReachedKnownParallelRegions.isAtFixpoint();
4470*d415bd75Srobert AllParallelRegionStatesWereFixed &=
4471*d415bd75Srobert CBAA.ReachedUnknownParallelRegions.isAtFixpoint();
447273471bf0Spatrick return true;
447373471bf0Spatrick };
447473471bf0Spatrick
447573471bf0Spatrick bool UsedAssumedInformationInCheckCallInst = false;
447673471bf0Spatrick if (!A.checkForAllCallLikeInstructions(
4477*d415bd75Srobert CheckCallInst, *this, UsedAssumedInformationInCheckCallInst)) {
4478*d415bd75Srobert LLVM_DEBUG(dbgs() << TAG
4479*d415bd75Srobert << "Failed to visit all call-like instructions!\n";);
448073471bf0Spatrick return indicatePessimisticFixpoint();
4481*d415bd75Srobert }
4482*d415bd75Srobert
4483*d415bd75Srobert // If we haven't used any assumed information for the reached parallel
4484*d415bd75Srobert // region states we can fix it.
4485*d415bd75Srobert if (!UsedAssumedInformationInCheckCallInst &&
4486*d415bd75Srobert AllParallelRegionStatesWereFixed) {
4487*d415bd75Srobert ReachedKnownParallelRegions.indicateOptimisticFixpoint();
4488*d415bd75Srobert ReachedUnknownParallelRegions.indicateOptimisticFixpoint();
4489*d415bd75Srobert }
449073471bf0Spatrick
449173471bf0Spatrick // If we haven't used any assumed information for the SPMD state we can fix
449273471bf0Spatrick // it.
449373471bf0Spatrick if (!UsedAssumedInformationInCheckRWInst &&
4494*d415bd75Srobert !UsedAssumedInformationInCheckCallInst &&
4495*d415bd75Srobert !UsedAssumedInformationFromReachingKernels && AllSPMDStatesWereFixed)
449673471bf0Spatrick SPMDCompatibilityTracker.indicateOptimisticFixpoint();
449773471bf0Spatrick
449873471bf0Spatrick return StateBefore == getState() ? ChangeStatus::UNCHANGED
449973471bf0Spatrick : ChangeStatus::CHANGED;
450073471bf0Spatrick }
450173471bf0Spatrick
450273471bf0Spatrick private:
450373471bf0Spatrick /// Update info regarding reaching kernels.
updateReachingKernelEntries__anon4b47cb5e0111::AAKernelInfoFunction4504*d415bd75Srobert void updateReachingKernelEntries(Attributor &A,
4505*d415bd75Srobert bool &AllReachingKernelsKnown) {
450673471bf0Spatrick auto PredCallSite = [&](AbstractCallSite ACS) {
450773471bf0Spatrick Function *Caller = ACS.getInstruction()->getFunction();
450873471bf0Spatrick
450973471bf0Spatrick assert(Caller && "Caller is nullptr");
451073471bf0Spatrick
451173471bf0Spatrick auto &CAA = A.getOrCreateAAFor<AAKernelInfo>(
451273471bf0Spatrick IRPosition::function(*Caller), this, DepClassTy::REQUIRED);
451373471bf0Spatrick if (CAA.ReachingKernelEntries.isValidState()) {
451473471bf0Spatrick ReachingKernelEntries ^= CAA.ReachingKernelEntries;
451573471bf0Spatrick return true;
451673471bf0Spatrick }
451773471bf0Spatrick
451873471bf0Spatrick // We lost track of the caller of the associated function, any kernel
451973471bf0Spatrick // could reach now.
452073471bf0Spatrick ReachingKernelEntries.indicatePessimisticFixpoint();
452173471bf0Spatrick
452273471bf0Spatrick return true;
452373471bf0Spatrick };
452473471bf0Spatrick
452573471bf0Spatrick if (!A.checkForAllCallSites(PredCallSite, *this,
452673471bf0Spatrick true /* RequireAllCallSites */,
4527*d415bd75Srobert AllReachingKernelsKnown))
452873471bf0Spatrick ReachingKernelEntries.indicatePessimisticFixpoint();
452973471bf0Spatrick }
453073471bf0Spatrick
453173471bf0Spatrick /// Update info regarding parallel levels.
updateParallelLevels__anon4b47cb5e0111::AAKernelInfoFunction453273471bf0Spatrick void updateParallelLevels(Attributor &A) {
453373471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
453473471bf0Spatrick OMPInformationCache::RuntimeFunctionInfo &Parallel51RFI =
453573471bf0Spatrick OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_51];
453673471bf0Spatrick
453773471bf0Spatrick auto PredCallSite = [&](AbstractCallSite ACS) {
453873471bf0Spatrick Function *Caller = ACS.getInstruction()->getFunction();
453973471bf0Spatrick
454073471bf0Spatrick assert(Caller && "Caller is nullptr");
454173471bf0Spatrick
454273471bf0Spatrick auto &CAA =
454373471bf0Spatrick A.getOrCreateAAFor<AAKernelInfo>(IRPosition::function(*Caller));
454473471bf0Spatrick if (CAA.ParallelLevels.isValidState()) {
454573471bf0Spatrick // Any function that is called by `__kmpc_parallel_51` will not be
454673471bf0Spatrick // folded as the parallel level in the function is updated. In order to
454773471bf0Spatrick // get it right, all the analysis would depend on the implentation. That
454873471bf0Spatrick // said, if in the future any change to the implementation, the analysis
454973471bf0Spatrick // could be wrong. As a consequence, we are just conservative here.
455073471bf0Spatrick if (Caller == Parallel51RFI.Declaration) {
455173471bf0Spatrick ParallelLevels.indicatePessimisticFixpoint();
455273471bf0Spatrick return true;
455373471bf0Spatrick }
455473471bf0Spatrick
455573471bf0Spatrick ParallelLevels ^= CAA.ParallelLevels;
455673471bf0Spatrick
455773471bf0Spatrick return true;
455873471bf0Spatrick }
455973471bf0Spatrick
456073471bf0Spatrick // We lost track of the caller of the associated function, any kernel
456173471bf0Spatrick // could reach now.
456273471bf0Spatrick ParallelLevels.indicatePessimisticFixpoint();
456373471bf0Spatrick
456473471bf0Spatrick return true;
456573471bf0Spatrick };
456673471bf0Spatrick
456773471bf0Spatrick bool AllCallSitesKnown = true;
456873471bf0Spatrick if (!A.checkForAllCallSites(PredCallSite, *this,
456973471bf0Spatrick true /* RequireAllCallSites */,
457073471bf0Spatrick AllCallSitesKnown))
457173471bf0Spatrick ParallelLevels.indicatePessimisticFixpoint();
457273471bf0Spatrick }
457373471bf0Spatrick };
457473471bf0Spatrick
457573471bf0Spatrick /// The call site kernel info abstract attribute, basically, what can we say
457673471bf0Spatrick /// about a call site with regards to the KernelInfoState. For now this simply
457773471bf0Spatrick /// forwards the information from the callee.
457873471bf0Spatrick struct AAKernelInfoCallSite : AAKernelInfo {
AAKernelInfoCallSite__anon4b47cb5e0111::AAKernelInfoCallSite457973471bf0Spatrick AAKernelInfoCallSite(const IRPosition &IRP, Attributor &A)
458073471bf0Spatrick : AAKernelInfo(IRP, A) {}
458173471bf0Spatrick
458273471bf0Spatrick /// See AbstractAttribute::initialize(...).
initialize__anon4b47cb5e0111::AAKernelInfoCallSite458373471bf0Spatrick void initialize(Attributor &A) override {
458473471bf0Spatrick AAKernelInfo::initialize(A);
458573471bf0Spatrick
458673471bf0Spatrick CallBase &CB = cast<CallBase>(getAssociatedValue());
458773471bf0Spatrick Function *Callee = getAssociatedFunction();
458873471bf0Spatrick
4589*d415bd75Srobert auto &AssumptionAA = A.getAAFor<AAAssumptionInfo>(
4590*d415bd75Srobert *this, IRPosition::callsite_function(CB), DepClassTy::OPTIONAL);
459173471bf0Spatrick
459273471bf0Spatrick // Check for SPMD-mode assumptions.
4593*d415bd75Srobert if (AssumptionAA.hasAssumption("ompx_spmd_amenable")) {
459473471bf0Spatrick SPMDCompatibilityTracker.indicateOptimisticFixpoint();
4595*d415bd75Srobert indicateOptimisticFixpoint();
4596*d415bd75Srobert }
459773471bf0Spatrick
459873471bf0Spatrick // First weed out calls we do not care about, that is readonly/readnone
459973471bf0Spatrick // calls, intrinsics, and "no_openmp" calls. Neither of these can reach a
460073471bf0Spatrick // parallel region or anything else we are looking for.
460173471bf0Spatrick if (!CB.mayWriteToMemory() || isa<IntrinsicInst>(CB)) {
460273471bf0Spatrick indicateOptimisticFixpoint();
460373471bf0Spatrick return;
460473471bf0Spatrick }
460573471bf0Spatrick
460673471bf0Spatrick // Next we check if we know the callee. If it is a known OpenMP function
460773471bf0Spatrick // we will handle them explicitly in the switch below. If it is not, we
460873471bf0Spatrick // will use an AAKernelInfo object on the callee to gather information and
460973471bf0Spatrick // merge that into the current state. The latter happens in the updateImpl.
461073471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
461173471bf0Spatrick const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(Callee);
461273471bf0Spatrick if (It == OMPInfoCache.RuntimeFunctionIDMap.end()) {
461373471bf0Spatrick // Unknown caller or declarations are not analyzable, we give up.
461473471bf0Spatrick if (!Callee || !A.isFunctionIPOAmendable(*Callee)) {
461573471bf0Spatrick
461673471bf0Spatrick // Unknown callees might contain parallel regions, except if they have
461773471bf0Spatrick // an appropriate assumption attached.
4618*d415bd75Srobert if (!(AssumptionAA.hasAssumption("omp_no_openmp") ||
4619*d415bd75Srobert AssumptionAA.hasAssumption("omp_no_parallelism")))
462073471bf0Spatrick ReachedUnknownParallelRegions.insert(&CB);
462173471bf0Spatrick
462273471bf0Spatrick // If SPMDCompatibilityTracker is not fixed, we need to give up on the
462373471bf0Spatrick // idea we can run something unknown in SPMD-mode.
4624*d415bd75Srobert if (!SPMDCompatibilityTracker.isAtFixpoint()) {
4625*d415bd75Srobert SPMDCompatibilityTracker.indicatePessimisticFixpoint();
462673471bf0Spatrick SPMDCompatibilityTracker.insert(&CB);
4627*d415bd75Srobert }
462873471bf0Spatrick
462973471bf0Spatrick // We have updated the state for this unknown call properly, there won't
463073471bf0Spatrick // be any change so we indicate a fixpoint.
463173471bf0Spatrick indicateOptimisticFixpoint();
463273471bf0Spatrick }
463373471bf0Spatrick // If the callee is known and can be used in IPO, we will update the state
463473471bf0Spatrick // based on the callee state in updateImpl.
463573471bf0Spatrick return;
463673471bf0Spatrick }
463773471bf0Spatrick
463873471bf0Spatrick const unsigned int WrapperFunctionArgNo = 6;
463973471bf0Spatrick RuntimeFunction RF = It->getSecond();
464073471bf0Spatrick switch (RF) {
464173471bf0Spatrick // All the functions we know are compatible with SPMD mode.
464273471bf0Spatrick case OMPRTL___kmpc_is_spmd_exec_mode:
4643*d415bd75Srobert case OMPRTL___kmpc_distribute_static_fini:
464473471bf0Spatrick case OMPRTL___kmpc_for_static_fini:
464573471bf0Spatrick case OMPRTL___kmpc_global_thread_num:
464673471bf0Spatrick case OMPRTL___kmpc_get_hardware_num_threads_in_block:
464773471bf0Spatrick case OMPRTL___kmpc_get_hardware_num_blocks:
464873471bf0Spatrick case OMPRTL___kmpc_single:
464973471bf0Spatrick case OMPRTL___kmpc_end_single:
465073471bf0Spatrick case OMPRTL___kmpc_master:
465173471bf0Spatrick case OMPRTL___kmpc_end_master:
465273471bf0Spatrick case OMPRTL___kmpc_barrier:
4653*d415bd75Srobert case OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2:
4654*d415bd75Srobert case OMPRTL___kmpc_nvptx_teams_reduce_nowait_v2:
4655*d415bd75Srobert case OMPRTL___kmpc_nvptx_end_reduce_nowait:
465673471bf0Spatrick break;
4657*d415bd75Srobert case OMPRTL___kmpc_distribute_static_init_4:
4658*d415bd75Srobert case OMPRTL___kmpc_distribute_static_init_4u:
4659*d415bd75Srobert case OMPRTL___kmpc_distribute_static_init_8:
4660*d415bd75Srobert case OMPRTL___kmpc_distribute_static_init_8u:
466173471bf0Spatrick case OMPRTL___kmpc_for_static_init_4:
466273471bf0Spatrick case OMPRTL___kmpc_for_static_init_4u:
466373471bf0Spatrick case OMPRTL___kmpc_for_static_init_8:
466473471bf0Spatrick case OMPRTL___kmpc_for_static_init_8u: {
466573471bf0Spatrick // Check the schedule and allow static schedule in SPMD mode.
466673471bf0Spatrick unsigned ScheduleArgOpNo = 2;
466773471bf0Spatrick auto *ScheduleTypeCI =
466873471bf0Spatrick dyn_cast<ConstantInt>(CB.getArgOperand(ScheduleArgOpNo));
466973471bf0Spatrick unsigned ScheduleTypeVal =
467073471bf0Spatrick ScheduleTypeCI ? ScheduleTypeCI->getZExtValue() : 0;
467173471bf0Spatrick switch (OMPScheduleType(ScheduleTypeVal)) {
4672*d415bd75Srobert case OMPScheduleType::UnorderedStatic:
4673*d415bd75Srobert case OMPScheduleType::UnorderedStaticChunked:
4674*d415bd75Srobert case OMPScheduleType::OrderedDistribute:
4675*d415bd75Srobert case OMPScheduleType::OrderedDistributeChunked:
467673471bf0Spatrick break;
467773471bf0Spatrick default:
4678*d415bd75Srobert SPMDCompatibilityTracker.indicatePessimisticFixpoint();
467973471bf0Spatrick SPMDCompatibilityTracker.insert(&CB);
468073471bf0Spatrick break;
468173471bf0Spatrick };
468273471bf0Spatrick } break;
468373471bf0Spatrick case OMPRTL___kmpc_target_init:
468473471bf0Spatrick KernelInitCB = &CB;
468573471bf0Spatrick break;
468673471bf0Spatrick case OMPRTL___kmpc_target_deinit:
468773471bf0Spatrick KernelDeinitCB = &CB;
468873471bf0Spatrick break;
468973471bf0Spatrick case OMPRTL___kmpc_parallel_51:
469073471bf0Spatrick if (auto *ParallelRegion = dyn_cast<Function>(
469173471bf0Spatrick CB.getArgOperand(WrapperFunctionArgNo)->stripPointerCasts())) {
469273471bf0Spatrick ReachedKnownParallelRegions.insert(ParallelRegion);
4693*d415bd75Srobert /// Check nested parallelism
4694*d415bd75Srobert auto &FnAA = A.getAAFor<AAKernelInfo>(
4695*d415bd75Srobert *this, IRPosition::function(*ParallelRegion), DepClassTy::OPTIONAL);
4696*d415bd75Srobert NestedParallelism |= !FnAA.getState().isValidState() ||
4697*d415bd75Srobert !FnAA.ReachedKnownParallelRegions.empty() ||
4698*d415bd75Srobert !FnAA.ReachedUnknownParallelRegions.empty();
469973471bf0Spatrick break;
470073471bf0Spatrick }
470173471bf0Spatrick // The condition above should usually get the parallel region function
470273471bf0Spatrick // pointer and record it. In the off chance it doesn't we assume the
470373471bf0Spatrick // worst.
470473471bf0Spatrick ReachedUnknownParallelRegions.insert(&CB);
470573471bf0Spatrick break;
470673471bf0Spatrick case OMPRTL___kmpc_omp_task:
470773471bf0Spatrick // We do not look into tasks right now, just give up.
4708*d415bd75Srobert SPMDCompatibilityTracker.indicatePessimisticFixpoint();
470973471bf0Spatrick SPMDCompatibilityTracker.insert(&CB);
471073471bf0Spatrick ReachedUnknownParallelRegions.insert(&CB);
471173471bf0Spatrick break;
471273471bf0Spatrick case OMPRTL___kmpc_alloc_shared:
471373471bf0Spatrick case OMPRTL___kmpc_free_shared:
471473471bf0Spatrick // Return without setting a fixpoint, to be resolved in updateImpl.
471573471bf0Spatrick return;
471673471bf0Spatrick default:
471773471bf0Spatrick // Unknown OpenMP runtime calls cannot be executed in SPMD-mode,
4718*d415bd75Srobert // generally. However, they do not hide parallel regions.
4719*d415bd75Srobert SPMDCompatibilityTracker.indicatePessimisticFixpoint();
472073471bf0Spatrick SPMDCompatibilityTracker.insert(&CB);
472173471bf0Spatrick break;
472273471bf0Spatrick }
472373471bf0Spatrick // All other OpenMP runtime calls will not reach parallel regions so they
472473471bf0Spatrick // can be safely ignored for now. Since it is a known OpenMP runtime call we
472573471bf0Spatrick // have now modeled all effects and there is no need for any update.
472673471bf0Spatrick indicateOptimisticFixpoint();
472773471bf0Spatrick }
472873471bf0Spatrick
updateImpl__anon4b47cb5e0111::AAKernelInfoCallSite472973471bf0Spatrick ChangeStatus updateImpl(Attributor &A) override {
473073471bf0Spatrick // TODO: Once we have call site specific value information we can provide
473173471bf0Spatrick // call site specific liveness information and then it makes
473273471bf0Spatrick // sense to specialize attributes for call sites arguments instead of
473373471bf0Spatrick // redirecting requests to the callee argument.
473473471bf0Spatrick Function *F = getAssociatedFunction();
473573471bf0Spatrick
473673471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
473773471bf0Spatrick const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(F);
473873471bf0Spatrick
473973471bf0Spatrick // If F is not a runtime function, propagate the AAKernelInfo of the callee.
474073471bf0Spatrick if (It == OMPInfoCache.RuntimeFunctionIDMap.end()) {
474173471bf0Spatrick const IRPosition &FnPos = IRPosition::function(*F);
474273471bf0Spatrick auto &FnAA = A.getAAFor<AAKernelInfo>(*this, FnPos, DepClassTy::REQUIRED);
474373471bf0Spatrick if (getState() == FnAA.getState())
474473471bf0Spatrick return ChangeStatus::UNCHANGED;
474573471bf0Spatrick getState() = FnAA.getState();
474673471bf0Spatrick return ChangeStatus::CHANGED;
474773471bf0Spatrick }
474873471bf0Spatrick
474973471bf0Spatrick // F is a runtime function that allocates or frees memory, check
475073471bf0Spatrick // AAHeapToStack and AAHeapToShared.
475173471bf0Spatrick KernelInfoState StateBefore = getState();
475273471bf0Spatrick assert((It->getSecond() == OMPRTL___kmpc_alloc_shared ||
475373471bf0Spatrick It->getSecond() == OMPRTL___kmpc_free_shared) &&
475473471bf0Spatrick "Expected a __kmpc_alloc_shared or __kmpc_free_shared runtime call");
475573471bf0Spatrick
475673471bf0Spatrick CallBase &CB = cast<CallBase>(getAssociatedValue());
475773471bf0Spatrick
475873471bf0Spatrick auto &HeapToStackAA = A.getAAFor<AAHeapToStack>(
475973471bf0Spatrick *this, IRPosition::function(*CB.getCaller()), DepClassTy::OPTIONAL);
476073471bf0Spatrick auto &HeapToSharedAA = A.getAAFor<AAHeapToShared>(
476173471bf0Spatrick *this, IRPosition::function(*CB.getCaller()), DepClassTy::OPTIONAL);
476273471bf0Spatrick
476373471bf0Spatrick RuntimeFunction RF = It->getSecond();
476473471bf0Spatrick
476573471bf0Spatrick switch (RF) {
476673471bf0Spatrick // If neither HeapToStack nor HeapToShared assume the call is removed,
476773471bf0Spatrick // assume SPMD incompatibility.
476873471bf0Spatrick case OMPRTL___kmpc_alloc_shared:
476973471bf0Spatrick if (!HeapToStackAA.isAssumedHeapToStack(CB) &&
477073471bf0Spatrick !HeapToSharedAA.isAssumedHeapToShared(CB))
477173471bf0Spatrick SPMDCompatibilityTracker.insert(&CB);
477273471bf0Spatrick break;
477373471bf0Spatrick case OMPRTL___kmpc_free_shared:
477473471bf0Spatrick if (!HeapToStackAA.isAssumedHeapToStackRemovedFree(CB) &&
477573471bf0Spatrick !HeapToSharedAA.isAssumedHeapToSharedRemovedFree(CB))
477673471bf0Spatrick SPMDCompatibilityTracker.insert(&CB);
477773471bf0Spatrick break;
477873471bf0Spatrick default:
4779*d415bd75Srobert SPMDCompatibilityTracker.indicatePessimisticFixpoint();
478073471bf0Spatrick SPMDCompatibilityTracker.insert(&CB);
478173471bf0Spatrick }
478273471bf0Spatrick
478373471bf0Spatrick return StateBefore == getState() ? ChangeStatus::UNCHANGED
478473471bf0Spatrick : ChangeStatus::CHANGED;
478573471bf0Spatrick }
478673471bf0Spatrick };
478773471bf0Spatrick
478873471bf0Spatrick struct AAFoldRuntimeCall
478973471bf0Spatrick : public StateWrapper<BooleanState, AbstractAttribute> {
479073471bf0Spatrick using Base = StateWrapper<BooleanState, AbstractAttribute>;
479173471bf0Spatrick
AAFoldRuntimeCall__anon4b47cb5e0111::AAFoldRuntimeCall479273471bf0Spatrick AAFoldRuntimeCall(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
479373471bf0Spatrick
479473471bf0Spatrick /// Statistics are tracked as part of manifest for now.
trackStatistics__anon4b47cb5e0111::AAFoldRuntimeCall479573471bf0Spatrick void trackStatistics() const override {}
479673471bf0Spatrick
479773471bf0Spatrick /// Create an abstract attribute biew for the position \p IRP.
479873471bf0Spatrick static AAFoldRuntimeCall &createForPosition(const IRPosition &IRP,
479973471bf0Spatrick Attributor &A);
480073471bf0Spatrick
480173471bf0Spatrick /// See AbstractAttribute::getName()
getName__anon4b47cb5e0111::AAFoldRuntimeCall480273471bf0Spatrick const std::string getName() const override { return "AAFoldRuntimeCall"; }
480373471bf0Spatrick
480473471bf0Spatrick /// See AbstractAttribute::getIdAddr()
getIdAddr__anon4b47cb5e0111::AAFoldRuntimeCall480573471bf0Spatrick const char *getIdAddr() const override { return &ID; }
480673471bf0Spatrick
480773471bf0Spatrick /// This function should return true if the type of the \p AA is
480873471bf0Spatrick /// AAFoldRuntimeCall
classof__anon4b47cb5e0111::AAFoldRuntimeCall480973471bf0Spatrick static bool classof(const AbstractAttribute *AA) {
481073471bf0Spatrick return (AA->getIdAddr() == &ID);
481173471bf0Spatrick }
481273471bf0Spatrick
481373471bf0Spatrick static const char ID;
481473471bf0Spatrick };
481573471bf0Spatrick
481673471bf0Spatrick struct AAFoldRuntimeCallCallSiteReturned : AAFoldRuntimeCall {
AAFoldRuntimeCallCallSiteReturned__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned481773471bf0Spatrick AAFoldRuntimeCallCallSiteReturned(const IRPosition &IRP, Attributor &A)
481873471bf0Spatrick : AAFoldRuntimeCall(IRP, A) {}
481973471bf0Spatrick
482073471bf0Spatrick /// See AbstractAttribute::getAsStr()
getAsStr__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned482173471bf0Spatrick const std::string getAsStr() const override {
482273471bf0Spatrick if (!isValidState())
482373471bf0Spatrick return "<invalid>";
482473471bf0Spatrick
482573471bf0Spatrick std::string Str("simplified value: ");
482673471bf0Spatrick
4827*d415bd75Srobert if (!SimplifiedValue)
482873471bf0Spatrick return Str + std::string("none");
482973471bf0Spatrick
4830*d415bd75Srobert if (!*SimplifiedValue)
483173471bf0Spatrick return Str + std::string("nullptr");
483273471bf0Spatrick
4833*d415bd75Srobert if (ConstantInt *CI = dyn_cast<ConstantInt>(*SimplifiedValue))
483473471bf0Spatrick return Str + std::to_string(CI->getSExtValue());
483573471bf0Spatrick
483673471bf0Spatrick return Str + std::string("unknown");
483773471bf0Spatrick }
483873471bf0Spatrick
initialize__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned483973471bf0Spatrick void initialize(Attributor &A) override {
4840*d415bd75Srobert if (DisableOpenMPOptFolding)
4841*d415bd75Srobert indicatePessimisticFixpoint();
4842*d415bd75Srobert
484373471bf0Spatrick Function *Callee = getAssociatedFunction();
484473471bf0Spatrick
484573471bf0Spatrick auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
484673471bf0Spatrick const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(Callee);
484773471bf0Spatrick assert(It != OMPInfoCache.RuntimeFunctionIDMap.end() &&
484873471bf0Spatrick "Expected a known OpenMP runtime function");
484973471bf0Spatrick
485073471bf0Spatrick RFKind = It->getSecond();
485173471bf0Spatrick
485273471bf0Spatrick CallBase &CB = cast<CallBase>(getAssociatedValue());
485373471bf0Spatrick A.registerSimplificationCallback(
485473471bf0Spatrick IRPosition::callsite_returned(CB),
485573471bf0Spatrick [&](const IRPosition &IRP, const AbstractAttribute *AA,
4856*d415bd75Srobert bool &UsedAssumedInformation) -> std::optional<Value *> {
4857*d415bd75Srobert assert((isValidState() ||
4858*d415bd75Srobert (SimplifiedValue && *SimplifiedValue == nullptr)) &&
485973471bf0Spatrick "Unexpected invalid state!");
486073471bf0Spatrick
486173471bf0Spatrick if (!isAtFixpoint()) {
486273471bf0Spatrick UsedAssumedInformation = true;
486373471bf0Spatrick if (AA)
486473471bf0Spatrick A.recordDependence(*this, *AA, DepClassTy::OPTIONAL);
486573471bf0Spatrick }
486673471bf0Spatrick return SimplifiedValue;
486773471bf0Spatrick });
486873471bf0Spatrick }
486973471bf0Spatrick
updateImpl__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned487073471bf0Spatrick ChangeStatus updateImpl(Attributor &A) override {
487173471bf0Spatrick ChangeStatus Changed = ChangeStatus::UNCHANGED;
487273471bf0Spatrick switch (RFKind) {
487373471bf0Spatrick case OMPRTL___kmpc_is_spmd_exec_mode:
487473471bf0Spatrick Changed |= foldIsSPMDExecMode(A);
487573471bf0Spatrick break;
487673471bf0Spatrick case OMPRTL___kmpc_parallel_level:
487773471bf0Spatrick Changed |= foldParallelLevel(A);
487873471bf0Spatrick break;
487973471bf0Spatrick case OMPRTL___kmpc_get_hardware_num_threads_in_block:
488073471bf0Spatrick Changed = Changed | foldKernelFnAttribute(A, "omp_target_thread_limit");
488173471bf0Spatrick break;
488273471bf0Spatrick case OMPRTL___kmpc_get_hardware_num_blocks:
488373471bf0Spatrick Changed = Changed | foldKernelFnAttribute(A, "omp_target_num_teams");
488473471bf0Spatrick break;
488573471bf0Spatrick default:
488673471bf0Spatrick llvm_unreachable("Unhandled OpenMP runtime function!");
488773471bf0Spatrick }
488873471bf0Spatrick
488973471bf0Spatrick return Changed;
489073471bf0Spatrick }
489173471bf0Spatrick
manifest__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned489273471bf0Spatrick ChangeStatus manifest(Attributor &A) override {
489373471bf0Spatrick ChangeStatus Changed = ChangeStatus::UNCHANGED;
489473471bf0Spatrick
4895*d415bd75Srobert if (SimplifiedValue && *SimplifiedValue) {
4896*d415bd75Srobert Instruction &I = *getCtxI();
4897*d415bd75Srobert A.changeAfterManifest(IRPosition::inst(I), **SimplifiedValue);
4898*d415bd75Srobert A.deleteAfterManifest(I);
489973471bf0Spatrick
4900*d415bd75Srobert CallBase *CB = dyn_cast<CallBase>(&I);
4901*d415bd75Srobert auto Remark = [&](OptimizationRemark OR) {
4902*d415bd75Srobert if (auto *C = dyn_cast<ConstantInt>(*SimplifiedValue))
4903*d415bd75Srobert return OR << "Replacing OpenMP runtime call "
4904*d415bd75Srobert << CB->getCalledFunction()->getName() << " with "
4905*d415bd75Srobert << ore::NV("FoldedValue", C->getZExtValue()) << ".";
4906*d415bd75Srobert return OR << "Replacing OpenMP runtime call "
4907*d415bd75Srobert << CB->getCalledFunction()->getName() << ".";
4908*d415bd75Srobert };
4909*d415bd75Srobert
4910*d415bd75Srobert if (CB && EnableVerboseRemarks)
4911*d415bd75Srobert A.emitRemark<OptimizationRemark>(CB, "OMP180", Remark);
4912*d415bd75Srobert
4913*d415bd75Srobert LLVM_DEBUG(dbgs() << TAG << "Replacing runtime call: " << I << " with "
491473471bf0Spatrick << **SimplifiedValue << "\n");
491573471bf0Spatrick
491673471bf0Spatrick Changed = ChangeStatus::CHANGED;
491773471bf0Spatrick }
491873471bf0Spatrick
491973471bf0Spatrick return Changed;
492073471bf0Spatrick }
492173471bf0Spatrick
indicatePessimisticFixpoint__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned492273471bf0Spatrick ChangeStatus indicatePessimisticFixpoint() override {
492373471bf0Spatrick SimplifiedValue = nullptr;
492473471bf0Spatrick return AAFoldRuntimeCall::indicatePessimisticFixpoint();
492573471bf0Spatrick }
492673471bf0Spatrick
492773471bf0Spatrick private:
492873471bf0Spatrick /// Fold __kmpc_is_spmd_exec_mode into a constant if possible.
foldIsSPMDExecMode__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned492973471bf0Spatrick ChangeStatus foldIsSPMDExecMode(Attributor &A) {
4930*d415bd75Srobert std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
493173471bf0Spatrick
493273471bf0Spatrick unsigned AssumedSPMDCount = 0, KnownSPMDCount = 0;
493373471bf0Spatrick unsigned AssumedNonSPMDCount = 0, KnownNonSPMDCount = 0;
493473471bf0Spatrick auto &CallerKernelInfoAA = A.getAAFor<AAKernelInfo>(
493573471bf0Spatrick *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED);
493673471bf0Spatrick
493773471bf0Spatrick if (!CallerKernelInfoAA.ReachingKernelEntries.isValidState())
493873471bf0Spatrick return indicatePessimisticFixpoint();
493973471bf0Spatrick
494073471bf0Spatrick for (Kernel K : CallerKernelInfoAA.ReachingKernelEntries) {
494173471bf0Spatrick auto &AA = A.getAAFor<AAKernelInfo>(*this, IRPosition::function(*K),
494273471bf0Spatrick DepClassTy::REQUIRED);
494373471bf0Spatrick
494473471bf0Spatrick if (!AA.isValidState()) {
494573471bf0Spatrick SimplifiedValue = nullptr;
494673471bf0Spatrick return indicatePessimisticFixpoint();
494773471bf0Spatrick }
494873471bf0Spatrick
494973471bf0Spatrick if (AA.SPMDCompatibilityTracker.isAssumed()) {
495073471bf0Spatrick if (AA.SPMDCompatibilityTracker.isAtFixpoint())
495173471bf0Spatrick ++KnownSPMDCount;
495273471bf0Spatrick else
495373471bf0Spatrick ++AssumedSPMDCount;
495473471bf0Spatrick } else {
495573471bf0Spatrick if (AA.SPMDCompatibilityTracker.isAtFixpoint())
495673471bf0Spatrick ++KnownNonSPMDCount;
495773471bf0Spatrick else
495873471bf0Spatrick ++AssumedNonSPMDCount;
495973471bf0Spatrick }
496073471bf0Spatrick }
496173471bf0Spatrick
496273471bf0Spatrick if ((AssumedSPMDCount + KnownSPMDCount) &&
496373471bf0Spatrick (AssumedNonSPMDCount + KnownNonSPMDCount))
496473471bf0Spatrick return indicatePessimisticFixpoint();
496573471bf0Spatrick
496673471bf0Spatrick auto &Ctx = getAnchorValue().getContext();
496773471bf0Spatrick if (KnownSPMDCount || AssumedSPMDCount) {
496873471bf0Spatrick assert(KnownNonSPMDCount == 0 && AssumedNonSPMDCount == 0 &&
496973471bf0Spatrick "Expected only SPMD kernels!");
497073471bf0Spatrick // All reaching kernels are in SPMD mode. Update all function calls to
497173471bf0Spatrick // __kmpc_is_spmd_exec_mode to 1.
497273471bf0Spatrick SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx), true);
497373471bf0Spatrick } else if (KnownNonSPMDCount || AssumedNonSPMDCount) {
497473471bf0Spatrick assert(KnownSPMDCount == 0 && AssumedSPMDCount == 0 &&
497573471bf0Spatrick "Expected only non-SPMD kernels!");
497673471bf0Spatrick // All reaching kernels are in non-SPMD mode. Update all function
497773471bf0Spatrick // calls to __kmpc_is_spmd_exec_mode to 0.
497873471bf0Spatrick SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx), false);
497973471bf0Spatrick } else {
498073471bf0Spatrick // We have empty reaching kernels, therefore we cannot tell if the
498173471bf0Spatrick // associated call site can be folded. At this moment, SimplifiedValue
498273471bf0Spatrick // must be none.
4983*d415bd75Srobert assert(!SimplifiedValue && "SimplifiedValue should be none");
498473471bf0Spatrick }
498573471bf0Spatrick
498673471bf0Spatrick return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
498773471bf0Spatrick : ChangeStatus::CHANGED;
498873471bf0Spatrick }
498973471bf0Spatrick
499073471bf0Spatrick /// Fold __kmpc_parallel_level into a constant if possible.
foldParallelLevel__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned499173471bf0Spatrick ChangeStatus foldParallelLevel(Attributor &A) {
4992*d415bd75Srobert std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
499373471bf0Spatrick
499473471bf0Spatrick auto &CallerKernelInfoAA = A.getAAFor<AAKernelInfo>(
499573471bf0Spatrick *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED);
499673471bf0Spatrick
499773471bf0Spatrick if (!CallerKernelInfoAA.ParallelLevels.isValidState())
499873471bf0Spatrick return indicatePessimisticFixpoint();
499973471bf0Spatrick
500073471bf0Spatrick if (!CallerKernelInfoAA.ReachingKernelEntries.isValidState())
500173471bf0Spatrick return indicatePessimisticFixpoint();
500273471bf0Spatrick
500373471bf0Spatrick if (CallerKernelInfoAA.ReachingKernelEntries.empty()) {
5004*d415bd75Srobert assert(!SimplifiedValue &&
500573471bf0Spatrick "SimplifiedValue should keep none at this point");
500673471bf0Spatrick return ChangeStatus::UNCHANGED;
500773471bf0Spatrick }
500873471bf0Spatrick
500973471bf0Spatrick unsigned AssumedSPMDCount = 0, KnownSPMDCount = 0;
501073471bf0Spatrick unsigned AssumedNonSPMDCount = 0, KnownNonSPMDCount = 0;
501173471bf0Spatrick for (Kernel K : CallerKernelInfoAA.ReachingKernelEntries) {
501273471bf0Spatrick auto &AA = A.getAAFor<AAKernelInfo>(*this, IRPosition::function(*K),
501373471bf0Spatrick DepClassTy::REQUIRED);
501473471bf0Spatrick if (!AA.SPMDCompatibilityTracker.isValidState())
501573471bf0Spatrick return indicatePessimisticFixpoint();
501673471bf0Spatrick
501773471bf0Spatrick if (AA.SPMDCompatibilityTracker.isAssumed()) {
501873471bf0Spatrick if (AA.SPMDCompatibilityTracker.isAtFixpoint())
501973471bf0Spatrick ++KnownSPMDCount;
502073471bf0Spatrick else
502173471bf0Spatrick ++AssumedSPMDCount;
502273471bf0Spatrick } else {
502373471bf0Spatrick if (AA.SPMDCompatibilityTracker.isAtFixpoint())
502473471bf0Spatrick ++KnownNonSPMDCount;
502573471bf0Spatrick else
502673471bf0Spatrick ++AssumedNonSPMDCount;
502773471bf0Spatrick }
502873471bf0Spatrick }
502973471bf0Spatrick
503073471bf0Spatrick if ((AssumedSPMDCount + KnownSPMDCount) &&
503173471bf0Spatrick (AssumedNonSPMDCount + KnownNonSPMDCount))
503273471bf0Spatrick return indicatePessimisticFixpoint();
503373471bf0Spatrick
503473471bf0Spatrick auto &Ctx = getAnchorValue().getContext();
503573471bf0Spatrick // If the caller can only be reached by SPMD kernel entries, the parallel
503673471bf0Spatrick // level is 1. Similarly, if the caller can only be reached by non-SPMD
503773471bf0Spatrick // kernel entries, it is 0.
503873471bf0Spatrick if (AssumedSPMDCount || KnownSPMDCount) {
503973471bf0Spatrick assert(KnownNonSPMDCount == 0 && AssumedNonSPMDCount == 0 &&
504073471bf0Spatrick "Expected only SPMD kernels!");
504173471bf0Spatrick SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx), 1);
504273471bf0Spatrick } else {
504373471bf0Spatrick assert(KnownSPMDCount == 0 && AssumedSPMDCount == 0 &&
504473471bf0Spatrick "Expected only non-SPMD kernels!");
504573471bf0Spatrick SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx), 0);
504673471bf0Spatrick }
504773471bf0Spatrick return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
504873471bf0Spatrick : ChangeStatus::CHANGED;
504973471bf0Spatrick }
505073471bf0Spatrick
foldKernelFnAttribute__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned505173471bf0Spatrick ChangeStatus foldKernelFnAttribute(Attributor &A, llvm::StringRef Attr) {
505273471bf0Spatrick // Specialize only if all the calls agree with the attribute constant value
505373471bf0Spatrick int32_t CurrentAttrValue = -1;
5054*d415bd75Srobert std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
505573471bf0Spatrick
505673471bf0Spatrick auto &CallerKernelInfoAA = A.getAAFor<AAKernelInfo>(
505773471bf0Spatrick *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED);
505873471bf0Spatrick
505973471bf0Spatrick if (!CallerKernelInfoAA.ReachingKernelEntries.isValidState())
506073471bf0Spatrick return indicatePessimisticFixpoint();
506173471bf0Spatrick
506273471bf0Spatrick // Iterate over the kernels that reach this function
506373471bf0Spatrick for (Kernel K : CallerKernelInfoAA.ReachingKernelEntries) {
5064*d415bd75Srobert int32_t NextAttrVal = K->getFnAttributeAsParsedInteger(Attr, -1);
506573471bf0Spatrick
506673471bf0Spatrick if (NextAttrVal == -1 ||
506773471bf0Spatrick (CurrentAttrValue != -1 && CurrentAttrValue != NextAttrVal))
506873471bf0Spatrick return indicatePessimisticFixpoint();
506973471bf0Spatrick CurrentAttrValue = NextAttrVal;
507073471bf0Spatrick }
507173471bf0Spatrick
507273471bf0Spatrick if (CurrentAttrValue != -1) {
507373471bf0Spatrick auto &Ctx = getAnchorValue().getContext();
507473471bf0Spatrick SimplifiedValue =
507573471bf0Spatrick ConstantInt::get(Type::getInt32Ty(Ctx), CurrentAttrValue);
507673471bf0Spatrick }
507773471bf0Spatrick return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
507873471bf0Spatrick : ChangeStatus::CHANGED;
507973471bf0Spatrick }
508073471bf0Spatrick
508173471bf0Spatrick /// An optional value the associated value is assumed to fold to. That is, we
508273471bf0Spatrick /// assume the associated value (which is a call) can be replaced by this
508373471bf0Spatrick /// simplified value.
5084*d415bd75Srobert std::optional<Value *> SimplifiedValue;
508573471bf0Spatrick
508673471bf0Spatrick /// The runtime function kind of the callee of the associated call site.
508773471bf0Spatrick RuntimeFunction RFKind;
508873471bf0Spatrick };
508973471bf0Spatrick
5090097a140dSpatrick } // namespace
5091097a140dSpatrick
509273471bf0Spatrick /// Register folding callsite
registerFoldRuntimeCall(RuntimeFunction RF)509373471bf0Spatrick void OpenMPOpt::registerFoldRuntimeCall(RuntimeFunction RF) {
509473471bf0Spatrick auto &RFI = OMPInfoCache.RFIs[RF];
509573471bf0Spatrick RFI.foreachUse(SCC, [&](Use &U, Function &F) {
509673471bf0Spatrick CallInst *CI = OpenMPOpt::getCallIfRegularCall(U, &RFI);
509773471bf0Spatrick if (!CI)
509873471bf0Spatrick return false;
509973471bf0Spatrick A.getOrCreateAAFor<AAFoldRuntimeCall>(
510073471bf0Spatrick IRPosition::callsite_returned(*CI), /* QueryingAA */ nullptr,
510173471bf0Spatrick DepClassTy::NONE, /* ForceUpdate */ false,
510273471bf0Spatrick /* UpdateAfterInit */ false);
510373471bf0Spatrick return false;
510473471bf0Spatrick });
510573471bf0Spatrick }
510673471bf0Spatrick
registerAAs(bool IsModulePass)510773471bf0Spatrick void OpenMPOpt::registerAAs(bool IsModulePass) {
510873471bf0Spatrick if (SCC.empty())
510973471bf0Spatrick return;
5110*d415bd75Srobert
511173471bf0Spatrick if (IsModulePass) {
511273471bf0Spatrick // Ensure we create the AAKernelInfo AAs first and without triggering an
511373471bf0Spatrick // update. This will make sure we register all value simplification
511473471bf0Spatrick // callbacks before any other AA has the chance to create an AAValueSimplify
511573471bf0Spatrick // or similar.
5116*d415bd75Srobert auto CreateKernelInfoCB = [&](Use &, Function &Kernel) {
511773471bf0Spatrick A.getOrCreateAAFor<AAKernelInfo>(
5118*d415bd75Srobert IRPosition::function(Kernel), /* QueryingAA */ nullptr,
511973471bf0Spatrick DepClassTy::NONE, /* ForceUpdate */ false,
512073471bf0Spatrick /* UpdateAfterInit */ false);
5121*d415bd75Srobert return false;
5122*d415bd75Srobert };
5123*d415bd75Srobert OMPInformationCache::RuntimeFunctionInfo &InitRFI =
5124*d415bd75Srobert OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
5125*d415bd75Srobert InitRFI.foreachUse(SCC, CreateKernelInfoCB);
512673471bf0Spatrick
512773471bf0Spatrick registerFoldRuntimeCall(OMPRTL___kmpc_is_spmd_exec_mode);
512873471bf0Spatrick registerFoldRuntimeCall(OMPRTL___kmpc_parallel_level);
512973471bf0Spatrick registerFoldRuntimeCall(OMPRTL___kmpc_get_hardware_num_threads_in_block);
513073471bf0Spatrick registerFoldRuntimeCall(OMPRTL___kmpc_get_hardware_num_blocks);
513173471bf0Spatrick }
513273471bf0Spatrick
513373471bf0Spatrick // Create CallSite AA for all Getters.
5134*d415bd75Srobert if (DeduceICVValues) {
513573471bf0Spatrick for (int Idx = 0; Idx < OMPInfoCache.ICVs.size() - 1; ++Idx) {
513673471bf0Spatrick auto ICVInfo = OMPInfoCache.ICVs[static_cast<InternalControlVar>(Idx)];
513773471bf0Spatrick
513873471bf0Spatrick auto &GetterRFI = OMPInfoCache.RFIs[ICVInfo.Getter];
513973471bf0Spatrick
514073471bf0Spatrick auto CreateAA = [&](Use &U, Function &Caller) {
514173471bf0Spatrick CallInst *CI = OpenMPOpt::getCallIfRegularCall(U, &GetterRFI);
514273471bf0Spatrick if (!CI)
514373471bf0Spatrick return false;
514473471bf0Spatrick
514573471bf0Spatrick auto &CB = cast<CallBase>(*CI);
514673471bf0Spatrick
514773471bf0Spatrick IRPosition CBPos = IRPosition::callsite_function(CB);
514873471bf0Spatrick A.getOrCreateAAFor<AAICVTracker>(CBPos);
514973471bf0Spatrick return false;
515073471bf0Spatrick };
515173471bf0Spatrick
515273471bf0Spatrick GetterRFI.foreachUse(SCC, CreateAA);
515373471bf0Spatrick }
5154*d415bd75Srobert }
515573471bf0Spatrick
515673471bf0Spatrick // Create an ExecutionDomain AA for every function and a HeapToStack AA for
515773471bf0Spatrick // every function if there is a device kernel.
515873471bf0Spatrick if (!isOpenMPDevice(M))
515973471bf0Spatrick return;
516073471bf0Spatrick
516173471bf0Spatrick for (auto *F : SCC) {
516273471bf0Spatrick if (F->isDeclaration())
516373471bf0Spatrick continue;
516473471bf0Spatrick
5165*d415bd75Srobert // We look at internal functions only on-demand but if any use is not a
5166*d415bd75Srobert // direct call or outside the current set of analyzed functions, we have
5167*d415bd75Srobert // to do it eagerly.
5168*d415bd75Srobert if (F->hasLocalLinkage()) {
5169*d415bd75Srobert if (llvm::all_of(F->uses(), [this](const Use &U) {
5170*d415bd75Srobert const auto *CB = dyn_cast<CallBase>(U.getUser());
5171*d415bd75Srobert return CB && CB->isCallee(&U) &&
5172*d415bd75Srobert A.isRunOn(const_cast<Function *>(CB->getCaller()));
5173*d415bd75Srobert }))
5174*d415bd75Srobert continue;
5175*d415bd75Srobert }
5176*d415bd75Srobert registerAAsForFunction(A, *F);
5177*d415bd75Srobert }
5178*d415bd75Srobert }
517973471bf0Spatrick
registerAAsForFunction(Attributor & A,const Function & F)5180*d415bd75Srobert void OpenMPOpt::registerAAsForFunction(Attributor &A, const Function &F) {
5181*d415bd75Srobert if (!DisableOpenMPOptDeglobalization)
5182*d415bd75Srobert A.getOrCreateAAFor<AAHeapToShared>(IRPosition::function(F));
5183*d415bd75Srobert A.getOrCreateAAFor<AAExecutionDomain>(IRPosition::function(F));
5184*d415bd75Srobert if (!DisableOpenMPOptDeglobalization)
5185*d415bd75Srobert A.getOrCreateAAFor<AAHeapToStack>(IRPosition::function(F));
5186*d415bd75Srobert
5187*d415bd75Srobert for (auto &I : instructions(F)) {
518873471bf0Spatrick if (auto *LI = dyn_cast<LoadInst>(&I)) {
518973471bf0Spatrick bool UsedAssumedInformation = false;
519073471bf0Spatrick A.getAssumedSimplified(IRPosition::value(*LI), /* AA */ nullptr,
5191*d415bd75Srobert UsedAssumedInformation, AA::Interprocedural);
5192*d415bd75Srobert continue;
5193*d415bd75Srobert }
5194*d415bd75Srobert if (auto *SI = dyn_cast<StoreInst>(&I)) {
5195*d415bd75Srobert A.getOrCreateAAFor<AAIsDead>(IRPosition::value(*SI));
5196*d415bd75Srobert continue;
5197*d415bd75Srobert }
5198*d415bd75Srobert if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
5199*d415bd75Srobert if (II->getIntrinsicID() == Intrinsic::assume) {
5200*d415bd75Srobert A.getOrCreateAAFor<AAPotentialValues>(
5201*d415bd75Srobert IRPosition::value(*II->getArgOperand(0)));
5202*d415bd75Srobert continue;
520373471bf0Spatrick }
520473471bf0Spatrick }
520573471bf0Spatrick }
520673471bf0Spatrick }
520773471bf0Spatrick
5208097a140dSpatrick const char AAICVTracker::ID = 0;
520973471bf0Spatrick const char AAKernelInfo::ID = 0;
521073471bf0Spatrick const char AAExecutionDomain::ID = 0;
521173471bf0Spatrick const char AAHeapToShared::ID = 0;
521273471bf0Spatrick const char AAFoldRuntimeCall::ID = 0;
5213097a140dSpatrick
createForPosition(const IRPosition & IRP,Attributor & A)5214097a140dSpatrick AAICVTracker &AAICVTracker::createForPosition(const IRPosition &IRP,
5215097a140dSpatrick Attributor &A) {
5216097a140dSpatrick AAICVTracker *AA = nullptr;
5217097a140dSpatrick switch (IRP.getPositionKind()) {
5218097a140dSpatrick case IRPosition::IRP_INVALID:
5219097a140dSpatrick case IRPosition::IRP_FLOAT:
5220097a140dSpatrick case IRPosition::IRP_ARGUMENT:
5221097a140dSpatrick case IRPosition::IRP_CALL_SITE_ARGUMENT:
5222097a140dSpatrick llvm_unreachable("ICVTracker can only be created for function position!");
522373471bf0Spatrick case IRPosition::IRP_RETURNED:
522473471bf0Spatrick AA = new (A.Allocator) AAICVTrackerFunctionReturned(IRP, A);
522573471bf0Spatrick break;
522673471bf0Spatrick case IRPosition::IRP_CALL_SITE_RETURNED:
522773471bf0Spatrick AA = new (A.Allocator) AAICVTrackerCallSiteReturned(IRP, A);
522873471bf0Spatrick break;
522973471bf0Spatrick case IRPosition::IRP_CALL_SITE:
523073471bf0Spatrick AA = new (A.Allocator) AAICVTrackerCallSite(IRP, A);
523173471bf0Spatrick break;
5232097a140dSpatrick case IRPosition::IRP_FUNCTION:
5233097a140dSpatrick AA = new (A.Allocator) AAICVTrackerFunction(IRP, A);
5234097a140dSpatrick break;
5235097a140dSpatrick }
5236097a140dSpatrick
5237097a140dSpatrick return *AA;
5238097a140dSpatrick }
5239097a140dSpatrick
createForPosition(const IRPosition & IRP,Attributor & A)524073471bf0Spatrick AAExecutionDomain &AAExecutionDomain::createForPosition(const IRPosition &IRP,
524173471bf0Spatrick Attributor &A) {
524273471bf0Spatrick AAExecutionDomainFunction *AA = nullptr;
524373471bf0Spatrick switch (IRP.getPositionKind()) {
524473471bf0Spatrick case IRPosition::IRP_INVALID:
524573471bf0Spatrick case IRPosition::IRP_FLOAT:
524673471bf0Spatrick case IRPosition::IRP_ARGUMENT:
524773471bf0Spatrick case IRPosition::IRP_CALL_SITE_ARGUMENT:
524873471bf0Spatrick case IRPosition::IRP_RETURNED:
524973471bf0Spatrick case IRPosition::IRP_CALL_SITE_RETURNED:
525073471bf0Spatrick case IRPosition::IRP_CALL_SITE:
525173471bf0Spatrick llvm_unreachable(
525273471bf0Spatrick "AAExecutionDomain can only be created for function position!");
525373471bf0Spatrick case IRPosition::IRP_FUNCTION:
525473471bf0Spatrick AA = new (A.Allocator) AAExecutionDomainFunction(IRP, A);
525573471bf0Spatrick break;
525673471bf0Spatrick }
525773471bf0Spatrick
525873471bf0Spatrick return *AA;
525973471bf0Spatrick }
526073471bf0Spatrick
createForPosition(const IRPosition & IRP,Attributor & A)526173471bf0Spatrick AAHeapToShared &AAHeapToShared::createForPosition(const IRPosition &IRP,
526273471bf0Spatrick Attributor &A) {
526373471bf0Spatrick AAHeapToSharedFunction *AA = nullptr;
526473471bf0Spatrick switch (IRP.getPositionKind()) {
526573471bf0Spatrick case IRPosition::IRP_INVALID:
526673471bf0Spatrick case IRPosition::IRP_FLOAT:
526773471bf0Spatrick case IRPosition::IRP_ARGUMENT:
526873471bf0Spatrick case IRPosition::IRP_CALL_SITE_ARGUMENT:
526973471bf0Spatrick case IRPosition::IRP_RETURNED:
527073471bf0Spatrick case IRPosition::IRP_CALL_SITE_RETURNED:
527173471bf0Spatrick case IRPosition::IRP_CALL_SITE:
527273471bf0Spatrick llvm_unreachable(
527373471bf0Spatrick "AAHeapToShared can only be created for function position!");
527473471bf0Spatrick case IRPosition::IRP_FUNCTION:
527573471bf0Spatrick AA = new (A.Allocator) AAHeapToSharedFunction(IRP, A);
527673471bf0Spatrick break;
527773471bf0Spatrick }
527873471bf0Spatrick
527973471bf0Spatrick return *AA;
528073471bf0Spatrick }
528173471bf0Spatrick
createForPosition(const IRPosition & IRP,Attributor & A)528273471bf0Spatrick AAKernelInfo &AAKernelInfo::createForPosition(const IRPosition &IRP,
528373471bf0Spatrick Attributor &A) {
528473471bf0Spatrick AAKernelInfo *AA = nullptr;
528573471bf0Spatrick switch (IRP.getPositionKind()) {
528673471bf0Spatrick case IRPosition::IRP_INVALID:
528773471bf0Spatrick case IRPosition::IRP_FLOAT:
528873471bf0Spatrick case IRPosition::IRP_ARGUMENT:
528973471bf0Spatrick case IRPosition::IRP_RETURNED:
529073471bf0Spatrick case IRPosition::IRP_CALL_SITE_RETURNED:
529173471bf0Spatrick case IRPosition::IRP_CALL_SITE_ARGUMENT:
529273471bf0Spatrick llvm_unreachable("KernelInfo can only be created for function position!");
529373471bf0Spatrick case IRPosition::IRP_CALL_SITE:
529473471bf0Spatrick AA = new (A.Allocator) AAKernelInfoCallSite(IRP, A);
529573471bf0Spatrick break;
529673471bf0Spatrick case IRPosition::IRP_FUNCTION:
529773471bf0Spatrick AA = new (A.Allocator) AAKernelInfoFunction(IRP, A);
529873471bf0Spatrick break;
529973471bf0Spatrick }
530073471bf0Spatrick
530173471bf0Spatrick return *AA;
530273471bf0Spatrick }
530373471bf0Spatrick
createForPosition(const IRPosition & IRP,Attributor & A)530473471bf0Spatrick AAFoldRuntimeCall &AAFoldRuntimeCall::createForPosition(const IRPosition &IRP,
530573471bf0Spatrick Attributor &A) {
530673471bf0Spatrick AAFoldRuntimeCall *AA = nullptr;
530773471bf0Spatrick switch (IRP.getPositionKind()) {
530873471bf0Spatrick case IRPosition::IRP_INVALID:
530973471bf0Spatrick case IRPosition::IRP_FLOAT:
531073471bf0Spatrick case IRPosition::IRP_ARGUMENT:
531173471bf0Spatrick case IRPosition::IRP_RETURNED:
531273471bf0Spatrick case IRPosition::IRP_FUNCTION:
531373471bf0Spatrick case IRPosition::IRP_CALL_SITE:
531473471bf0Spatrick case IRPosition::IRP_CALL_SITE_ARGUMENT:
531573471bf0Spatrick llvm_unreachable("KernelInfo can only be created for call site position!");
531673471bf0Spatrick case IRPosition::IRP_CALL_SITE_RETURNED:
531773471bf0Spatrick AA = new (A.Allocator) AAFoldRuntimeCallCallSiteReturned(IRP, A);
531873471bf0Spatrick break;
531973471bf0Spatrick }
532073471bf0Spatrick
532173471bf0Spatrick return *AA;
532273471bf0Spatrick }
532373471bf0Spatrick
run(Module & M,ModuleAnalysisManager & AM)532473471bf0Spatrick PreservedAnalyses OpenMPOptPass::run(Module &M, ModuleAnalysisManager &AM) {
532573471bf0Spatrick if (!containsOpenMP(M))
532673471bf0Spatrick return PreservedAnalyses::all();
532773471bf0Spatrick if (DisableOpenMPOptimizations)
5328097a140dSpatrick return PreservedAnalyses::all();
5329097a140dSpatrick
533073471bf0Spatrick FunctionAnalysisManager &FAM =
533173471bf0Spatrick AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
533273471bf0Spatrick KernelSet Kernels = getDeviceKernels(M);
533373471bf0Spatrick
5334*d415bd75Srobert if (PrintModuleBeforeOptimizations)
5335*d415bd75Srobert LLVM_DEBUG(dbgs() << TAG << "Module before OpenMPOpt Module Pass:\n" << M);
5336*d415bd75Srobert
533773471bf0Spatrick auto IsCalled = [&](Function &F) {
533873471bf0Spatrick if (Kernels.contains(&F))
533973471bf0Spatrick return true;
534073471bf0Spatrick for (const User *U : F.users())
534173471bf0Spatrick if (!isa<BlockAddress>(U))
534273471bf0Spatrick return true;
534373471bf0Spatrick return false;
534473471bf0Spatrick };
534573471bf0Spatrick
534673471bf0Spatrick auto EmitRemark = [&](Function &F) {
534773471bf0Spatrick auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
534873471bf0Spatrick ORE.emit([&]() {
534973471bf0Spatrick OptimizationRemarkAnalysis ORA(DEBUG_TYPE, "OMP140", &F);
535073471bf0Spatrick return ORA << "Could not internalize function. "
535173471bf0Spatrick << "Some optimizations may not be possible. [OMP140]";
535273471bf0Spatrick });
535373471bf0Spatrick };
535473471bf0Spatrick
535573471bf0Spatrick // Create internal copies of each function if this is a kernel Module. This
535673471bf0Spatrick // allows iterprocedural passes to see every call edge.
535773471bf0Spatrick DenseMap<Function *, Function *> InternalizedMap;
535873471bf0Spatrick if (isOpenMPDevice(M)) {
535973471bf0Spatrick SmallPtrSet<Function *, 16> InternalizeFns;
536073471bf0Spatrick for (Function &F : M)
536173471bf0Spatrick if (!F.isDeclaration() && !Kernels.contains(&F) && IsCalled(F) &&
536273471bf0Spatrick !DisableInternalization) {
536373471bf0Spatrick if (Attributor::isInternalizable(F)) {
536473471bf0Spatrick InternalizeFns.insert(&F);
536573471bf0Spatrick } else if (!F.hasLocalLinkage() && !F.hasFnAttribute(Attribute::Cold)) {
536673471bf0Spatrick EmitRemark(F);
536773471bf0Spatrick }
536873471bf0Spatrick }
536973471bf0Spatrick
537073471bf0Spatrick Attributor::internalizeFunctions(InternalizeFns, InternalizedMap);
537173471bf0Spatrick }
537273471bf0Spatrick
537373471bf0Spatrick // Look at every function in the Module unless it was internalized.
5374*d415bd75Srobert SetVector<Function *> Functions;
537573471bf0Spatrick SmallVector<Function *, 16> SCC;
537673471bf0Spatrick for (Function &F : M)
5377*d415bd75Srobert if (!F.isDeclaration() && !InternalizedMap.lookup(&F)) {
537873471bf0Spatrick SCC.push_back(&F);
5379*d415bd75Srobert Functions.insert(&F);
5380*d415bd75Srobert }
538173471bf0Spatrick
538273471bf0Spatrick if (SCC.empty())
538373471bf0Spatrick return PreservedAnalyses::all();
538473471bf0Spatrick
538573471bf0Spatrick AnalysisGetter AG(FAM);
538673471bf0Spatrick
538773471bf0Spatrick auto OREGetter = [&FAM](Function *F) -> OptimizationRemarkEmitter & {
538873471bf0Spatrick return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*F);
538973471bf0Spatrick };
539073471bf0Spatrick
539173471bf0Spatrick BumpPtrAllocator Allocator;
539273471bf0Spatrick CallGraphUpdater CGUpdater;
539373471bf0Spatrick
5394*d415bd75Srobert bool PostLink = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||
5395*d415bd75Srobert LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink;
5396*d415bd75Srobert OMPInformationCache InfoCache(M, AG, Allocator, /*CGSCC*/ nullptr, Kernels,
5397*d415bd75Srobert PostLink);
539873471bf0Spatrick
5399*d415bd75Srobert unsigned MaxFixpointIterations =
5400*d415bd75Srobert (isOpenMPDevice(M)) ? SetFixpointIterations : 32;
5401*d415bd75Srobert
5402*d415bd75Srobert AttributorConfig AC(CGUpdater);
5403*d415bd75Srobert AC.DefaultInitializeLiveInternals = false;
5404*d415bd75Srobert AC.IsModulePass = true;
5405*d415bd75Srobert AC.RewriteSignatures = false;
5406*d415bd75Srobert AC.MaxFixpointIterations = MaxFixpointIterations;
5407*d415bd75Srobert AC.OREGetter = OREGetter;
5408*d415bd75Srobert AC.PassName = DEBUG_TYPE;
5409*d415bd75Srobert AC.InitializationCallback = OpenMPOpt::registerAAsForFunction;
5410*d415bd75Srobert
5411*d415bd75Srobert Attributor A(Functions, InfoCache, AC);
541273471bf0Spatrick
541373471bf0Spatrick OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
541473471bf0Spatrick bool Changed = OMPOpt.run(true);
5415*d415bd75Srobert
5416*d415bd75Srobert // Optionally inline device functions for potentially better performance.
5417*d415bd75Srobert if (AlwaysInlineDeviceFunctions && isOpenMPDevice(M))
5418*d415bd75Srobert for (Function &F : M)
5419*d415bd75Srobert if (!F.isDeclaration() && !Kernels.contains(&F) &&
5420*d415bd75Srobert !F.hasFnAttribute(Attribute::NoInline))
5421*d415bd75Srobert F.addFnAttr(Attribute::AlwaysInline);
5422*d415bd75Srobert
5423*d415bd75Srobert if (PrintModuleAfterOptimizations)
5424*d415bd75Srobert LLVM_DEBUG(dbgs() << TAG << "Module after OpenMPOpt Module Pass:\n" << M);
5425*d415bd75Srobert
542673471bf0Spatrick if (Changed)
542773471bf0Spatrick return PreservedAnalyses::none();
542873471bf0Spatrick
542973471bf0Spatrick return PreservedAnalyses::all();
543073471bf0Spatrick }
543173471bf0Spatrick
run(LazyCallGraph::SCC & C,CGSCCAnalysisManager & AM,LazyCallGraph & CG,CGSCCUpdateResult & UR)543273471bf0Spatrick PreservedAnalyses OpenMPOptCGSCCPass::run(LazyCallGraph::SCC &C,
543373471bf0Spatrick CGSCCAnalysisManager &AM,
543473471bf0Spatrick LazyCallGraph &CG,
543573471bf0Spatrick CGSCCUpdateResult &UR) {
543673471bf0Spatrick if (!containsOpenMP(*C.begin()->getFunction().getParent()))
543773471bf0Spatrick return PreservedAnalyses::all();
5438097a140dSpatrick if (DisableOpenMPOptimizations)
5439097a140dSpatrick return PreservedAnalyses::all();
5440097a140dSpatrick
5441097a140dSpatrick SmallVector<Function *, 16> SCC;
544273471bf0Spatrick // If there are kernels in the module, we have to run on all SCC's.
544373471bf0Spatrick for (LazyCallGraph::Node &N : C) {
544473471bf0Spatrick Function *Fn = &N.getFunction();
544573471bf0Spatrick SCC.push_back(Fn);
544673471bf0Spatrick }
5447097a140dSpatrick
5448097a140dSpatrick if (SCC.empty())
5449097a140dSpatrick return PreservedAnalyses::all();
5450097a140dSpatrick
545173471bf0Spatrick Module &M = *C.begin()->getFunction().getParent();
545273471bf0Spatrick
5453*d415bd75Srobert if (PrintModuleBeforeOptimizations)
5454*d415bd75Srobert LLVM_DEBUG(dbgs() << TAG << "Module before OpenMPOpt CGSCC Pass:\n" << M);
5455*d415bd75Srobert
545673471bf0Spatrick KernelSet Kernels = getDeviceKernels(M);
545773471bf0Spatrick
5458097a140dSpatrick FunctionAnalysisManager &FAM =
5459097a140dSpatrick AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
5460097a140dSpatrick
5461097a140dSpatrick AnalysisGetter AG(FAM);
5462097a140dSpatrick
5463097a140dSpatrick auto OREGetter = [&FAM](Function *F) -> OptimizationRemarkEmitter & {
5464097a140dSpatrick return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*F);
5465097a140dSpatrick };
5466097a140dSpatrick
546773471bf0Spatrick BumpPtrAllocator Allocator;
5468097a140dSpatrick CallGraphUpdater CGUpdater;
5469097a140dSpatrick CGUpdater.initialize(CG, C, AM, UR);
5470097a140dSpatrick
5471*d415bd75Srobert bool PostLink = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||
5472*d415bd75Srobert LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink;
5473097a140dSpatrick SetVector<Function *> Functions(SCC.begin(), SCC.end());
5474097a140dSpatrick OMPInformationCache InfoCache(*(Functions.back()->getParent()), AG, Allocator,
5475*d415bd75Srobert /*CGSCC*/ &Functions, Kernels, PostLink);
5476097a140dSpatrick
5477*d415bd75Srobert unsigned MaxFixpointIterations =
5478*d415bd75Srobert (isOpenMPDevice(M)) ? SetFixpointIterations : 32;
5479*d415bd75Srobert
5480*d415bd75Srobert AttributorConfig AC(CGUpdater);
5481*d415bd75Srobert AC.DefaultInitializeLiveInternals = false;
5482*d415bd75Srobert AC.IsModulePass = false;
5483*d415bd75Srobert AC.RewriteSignatures = false;
5484*d415bd75Srobert AC.MaxFixpointIterations = MaxFixpointIterations;
5485*d415bd75Srobert AC.OREGetter = OREGetter;
5486*d415bd75Srobert AC.PassName = DEBUG_TYPE;
5487*d415bd75Srobert AC.InitializationCallback = OpenMPOpt::registerAAsForFunction;
5488*d415bd75Srobert
5489*d415bd75Srobert Attributor A(Functions, InfoCache, AC);
5490097a140dSpatrick
5491097a140dSpatrick OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
549273471bf0Spatrick bool Changed = OMPOpt.run(false);
5493*d415bd75Srobert
5494*d415bd75Srobert if (PrintModuleAfterOptimizations)
5495*d415bd75Srobert LLVM_DEBUG(dbgs() << TAG << "Module after OpenMPOpt CGSCC Pass:\n" << M);
5496*d415bd75Srobert
5497097a140dSpatrick if (Changed)
5498097a140dSpatrick return PreservedAnalyses::none();
5499097a140dSpatrick
5500097a140dSpatrick return PreservedAnalyses::all();
5501097a140dSpatrick }
5502097a140dSpatrick
getDeviceKernels(Module & M)550373471bf0Spatrick KernelSet llvm::omp::getDeviceKernels(Module &M) {
550473471bf0Spatrick // TODO: Create a more cross-platform way of determining device kernels.
5505*d415bd75Srobert NamedMDNode *MD = M.getNamedMetadata("nvvm.annotations");
550673471bf0Spatrick KernelSet Kernels;
550773471bf0Spatrick
5508097a140dSpatrick if (!MD)
550973471bf0Spatrick return Kernels;
5510097a140dSpatrick
5511097a140dSpatrick for (auto *Op : MD->operands()) {
5512097a140dSpatrick if (Op->getNumOperands() < 2)
5513097a140dSpatrick continue;
5514097a140dSpatrick MDString *KindID = dyn_cast<MDString>(Op->getOperand(1));
5515097a140dSpatrick if (!KindID || KindID->getString() != "kernel")
5516097a140dSpatrick continue;
5517097a140dSpatrick
5518097a140dSpatrick Function *KernelFn =
5519097a140dSpatrick mdconst::dyn_extract_or_null<Function>(Op->getOperand(0));
5520097a140dSpatrick if (!KernelFn)
5521097a140dSpatrick continue;
5522097a140dSpatrick
5523097a140dSpatrick ++NumOpenMPTargetRegionKernels;
5524097a140dSpatrick
5525097a140dSpatrick Kernels.insert(KernelFn);
5526097a140dSpatrick }
552773471bf0Spatrick
552873471bf0Spatrick return Kernels;
5529097a140dSpatrick }
5530097a140dSpatrick
containsOpenMP(Module & M)553173471bf0Spatrick bool llvm::omp::containsOpenMP(Module &M) {
553273471bf0Spatrick Metadata *MD = M.getModuleFlag("openmp");
553373471bf0Spatrick if (!MD)
553473471bf0Spatrick return false;
5535097a140dSpatrick
5536097a140dSpatrick return true;
5537097a140dSpatrick }
5538097a140dSpatrick
isOpenMPDevice(Module & M)553973471bf0Spatrick bool llvm::omp::isOpenMPDevice(Module &M) {
554073471bf0Spatrick Metadata *MD = M.getModuleFlag("openmp-device");
554173471bf0Spatrick if (!MD)
554273471bf0Spatrick return false;
554373471bf0Spatrick
554473471bf0Spatrick return true;
5545097a140dSpatrick }
5546