xref: /openbsd-src/gnu/llvm/llvm/lib/Transforms/IPO/OpenMPOpt.cpp (revision d415bd752c734aee168c4ee86ff32e8cc249eb16)
1097a140dSpatrick //===-- IPO/OpenMPOpt.cpp - Collection of OpenMP specific optimizations ---===//
2097a140dSpatrick //
3097a140dSpatrick // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4097a140dSpatrick // See https://llvm.org/LICENSE.txt for license information.
5097a140dSpatrick // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6097a140dSpatrick //
7097a140dSpatrick //===----------------------------------------------------------------------===//
8097a140dSpatrick //
9097a140dSpatrick // OpenMP specific optimizations:
10097a140dSpatrick //
11097a140dSpatrick // - Deduplication of runtime calls, e.g., omp_get_thread_num.
1273471bf0Spatrick // - Replacing globalized device memory with stack memory.
1373471bf0Spatrick // - Replacing globalized device memory with shared memory.
1473471bf0Spatrick // - Parallel region merging.
1573471bf0Spatrick // - Transforming generic-mode device kernels to SPMD mode.
1673471bf0Spatrick // - Specializing the state machine for generic-mode device kernels.
17097a140dSpatrick //
18097a140dSpatrick //===----------------------------------------------------------------------===//
19097a140dSpatrick 
20097a140dSpatrick #include "llvm/Transforms/IPO/OpenMPOpt.h"
21097a140dSpatrick 
22097a140dSpatrick #include "llvm/ADT/EnumeratedArray.h"
2373471bf0Spatrick #include "llvm/ADT/PostOrderIterator.h"
24*d415bd75Srobert #include "llvm/ADT/SetVector.h"
25*d415bd75Srobert #include "llvm/ADT/SmallVector.h"
26097a140dSpatrick #include "llvm/ADT/Statistic.h"
27*d415bd75Srobert #include "llvm/ADT/StringRef.h"
28097a140dSpatrick #include "llvm/Analysis/CallGraph.h"
29097a140dSpatrick #include "llvm/Analysis/CallGraphSCCPass.h"
30*d415bd75Srobert #include "llvm/Analysis/MemoryLocation.h"
31097a140dSpatrick #include "llvm/Analysis/OptimizationRemarkEmitter.h"
3273471bf0Spatrick #include "llvm/Analysis/ValueTracking.h"
33097a140dSpatrick #include "llvm/Frontend/OpenMP/OMPConstants.h"
34097a140dSpatrick #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
3573471bf0Spatrick #include "llvm/IR/Assumptions.h"
36*d415bd75Srobert #include "llvm/IR/BasicBlock.h"
37*d415bd75Srobert #include "llvm/IR/Constants.h"
3873471bf0Spatrick #include "llvm/IR/DiagnosticInfo.h"
3973471bf0Spatrick #include "llvm/IR/GlobalValue.h"
40*d415bd75Srobert #include "llvm/IR/GlobalVariable.h"
4173471bf0Spatrick #include "llvm/IR/Instruction.h"
42*d415bd75Srobert #include "llvm/IR/Instructions.h"
4373471bf0Spatrick #include "llvm/IR/IntrinsicInst.h"
44*d415bd75Srobert #include "llvm/IR/IntrinsicsAMDGPU.h"
45*d415bd75Srobert #include "llvm/IR/IntrinsicsNVPTX.h"
46*d415bd75Srobert #include "llvm/IR/LLVMContext.h"
47097a140dSpatrick #include "llvm/InitializePasses.h"
48097a140dSpatrick #include "llvm/Support/CommandLine.h"
49*d415bd75Srobert #include "llvm/Support/Debug.h"
50097a140dSpatrick #include "llvm/Transforms/IPO/Attributor.h"
5173471bf0Spatrick #include "llvm/Transforms/Utils/BasicBlockUtils.h"
52097a140dSpatrick #include "llvm/Transforms/Utils/CallGraphUpdater.h"
53*d415bd75Srobert 
54*d415bd75Srobert #include <algorithm>
55*d415bd75Srobert #include <optional>
56*d415bd75Srobert #include <string>
57097a140dSpatrick 
58097a140dSpatrick using namespace llvm;
59097a140dSpatrick using namespace omp;
60097a140dSpatrick 
61097a140dSpatrick #define DEBUG_TYPE "openmp-opt"
62097a140dSpatrick 
63097a140dSpatrick static cl::opt<bool> DisableOpenMPOptimizations(
64*d415bd75Srobert     "openmp-opt-disable", cl::desc("Disable OpenMP specific optimizations."),
65*d415bd75Srobert     cl::Hidden, cl::init(false));
66097a140dSpatrick 
6773471bf0Spatrick static cl::opt<bool> EnableParallelRegionMerging(
68*d415bd75Srobert     "openmp-opt-enable-merging",
6973471bf0Spatrick     cl::desc("Enable the OpenMP region merging optimization."), cl::Hidden,
7073471bf0Spatrick     cl::init(false));
7173471bf0Spatrick 
7273471bf0Spatrick static cl::opt<bool>
73*d415bd75Srobert     DisableInternalization("openmp-opt-disable-internalization",
7473471bf0Spatrick                            cl::desc("Disable function internalization."),
7573471bf0Spatrick                            cl::Hidden, cl::init(false));
7673471bf0Spatrick 
77*d415bd75Srobert static cl::opt<bool> DeduceICVValues("openmp-deduce-icv-values",
78*d415bd75Srobert                                      cl::init(false), cl::Hidden);
79097a140dSpatrick static cl::opt<bool> PrintICVValues("openmp-print-icv-values", cl::init(false),
80097a140dSpatrick                                     cl::Hidden);
81097a140dSpatrick static cl::opt<bool> PrintOpenMPKernels("openmp-print-gpu-kernels",
82097a140dSpatrick                                         cl::init(false), cl::Hidden);
83097a140dSpatrick 
8473471bf0Spatrick static cl::opt<bool> HideMemoryTransferLatency(
8573471bf0Spatrick     "openmp-hide-memory-transfer-latency",
8673471bf0Spatrick     cl::desc("[WIP] Tries to hide the latency of host to device memory"
8773471bf0Spatrick              " transfers"),
8873471bf0Spatrick     cl::Hidden, cl::init(false));
8973471bf0Spatrick 
90*d415bd75Srobert static cl::opt<bool> DisableOpenMPOptDeglobalization(
91*d415bd75Srobert     "openmp-opt-disable-deglobalization",
92*d415bd75Srobert     cl::desc("Disable OpenMP optimizations involving deglobalization."),
93*d415bd75Srobert     cl::Hidden, cl::init(false));
94*d415bd75Srobert 
95*d415bd75Srobert static cl::opt<bool> DisableOpenMPOptSPMDization(
96*d415bd75Srobert     "openmp-opt-disable-spmdization",
97*d415bd75Srobert     cl::desc("Disable OpenMP optimizations involving SPMD-ization."),
98*d415bd75Srobert     cl::Hidden, cl::init(false));
99*d415bd75Srobert 
100*d415bd75Srobert static cl::opt<bool> DisableOpenMPOptFolding(
101*d415bd75Srobert     "openmp-opt-disable-folding",
102*d415bd75Srobert     cl::desc("Disable OpenMP optimizations involving folding."), cl::Hidden,
103*d415bd75Srobert     cl::init(false));
104*d415bd75Srobert 
105*d415bd75Srobert static cl::opt<bool> DisableOpenMPOptStateMachineRewrite(
106*d415bd75Srobert     "openmp-opt-disable-state-machine-rewrite",
107*d415bd75Srobert     cl::desc("Disable OpenMP optimizations that replace the state machine."),
108*d415bd75Srobert     cl::Hidden, cl::init(false));
109*d415bd75Srobert 
110*d415bd75Srobert static cl::opt<bool> DisableOpenMPOptBarrierElimination(
111*d415bd75Srobert     "openmp-opt-disable-barrier-elimination",
112*d415bd75Srobert     cl::desc("Disable OpenMP optimizations that eliminate barriers."),
113*d415bd75Srobert     cl::Hidden, cl::init(false));
114*d415bd75Srobert 
115*d415bd75Srobert static cl::opt<bool> PrintModuleAfterOptimizations(
116*d415bd75Srobert     "openmp-opt-print-module-after",
117*d415bd75Srobert     cl::desc("Print the current module after OpenMP optimizations."),
118*d415bd75Srobert     cl::Hidden, cl::init(false));
119*d415bd75Srobert 
120*d415bd75Srobert static cl::opt<bool> PrintModuleBeforeOptimizations(
121*d415bd75Srobert     "openmp-opt-print-module-before",
122*d415bd75Srobert     cl::desc("Print the current module before OpenMP optimizations."),
123*d415bd75Srobert     cl::Hidden, cl::init(false));
124*d415bd75Srobert 
125*d415bd75Srobert static cl::opt<bool> AlwaysInlineDeviceFunctions(
126*d415bd75Srobert     "openmp-opt-inline-device",
127*d415bd75Srobert     cl::desc("Inline all applicible functions on the device."), cl::Hidden,
128*d415bd75Srobert     cl::init(false));
129*d415bd75Srobert 
130*d415bd75Srobert static cl::opt<bool>
131*d415bd75Srobert     EnableVerboseRemarks("openmp-opt-verbose-remarks",
132*d415bd75Srobert                          cl::desc("Enables more verbose remarks."), cl::Hidden,
133*d415bd75Srobert                          cl::init(false));
134*d415bd75Srobert 
135*d415bd75Srobert static cl::opt<unsigned>
136*d415bd75Srobert     SetFixpointIterations("openmp-opt-max-iterations", cl::Hidden,
137*d415bd75Srobert                           cl::desc("Maximal number of attributor iterations."),
138*d415bd75Srobert                           cl::init(256));
139*d415bd75Srobert 
140*d415bd75Srobert static cl::opt<unsigned>
141*d415bd75Srobert     SharedMemoryLimit("openmp-opt-shared-limit", cl::Hidden,
142*d415bd75Srobert                       cl::desc("Maximum amount of shared memory to use."),
143*d415bd75Srobert                       cl::init(std::numeric_limits<unsigned>::max()));
144*d415bd75Srobert 
145097a140dSpatrick STATISTIC(NumOpenMPRuntimeCallsDeduplicated,
146097a140dSpatrick           "Number of OpenMP runtime calls deduplicated");
147097a140dSpatrick STATISTIC(NumOpenMPParallelRegionsDeleted,
148097a140dSpatrick           "Number of OpenMP parallel regions deleted");
149097a140dSpatrick STATISTIC(NumOpenMPRuntimeFunctionsIdentified,
150097a140dSpatrick           "Number of OpenMP runtime functions identified");
151097a140dSpatrick STATISTIC(NumOpenMPRuntimeFunctionUsesIdentified,
152097a140dSpatrick           "Number of OpenMP runtime function uses identified");
153097a140dSpatrick STATISTIC(NumOpenMPTargetRegionKernels,
154097a140dSpatrick           "Number of OpenMP target region entry points (=kernels) identified");
15573471bf0Spatrick STATISTIC(NumOpenMPTargetRegionKernelsSPMD,
15673471bf0Spatrick           "Number of OpenMP target region entry points (=kernels) executed in "
15773471bf0Spatrick           "SPMD-mode instead of generic-mode");
15873471bf0Spatrick STATISTIC(NumOpenMPTargetRegionKernelsWithoutStateMachine,
15973471bf0Spatrick           "Number of OpenMP target region entry points (=kernels) executed in "
16073471bf0Spatrick           "generic-mode without a state machines");
16173471bf0Spatrick STATISTIC(NumOpenMPTargetRegionKernelsCustomStateMachineWithFallback,
16273471bf0Spatrick           "Number of OpenMP target region entry points (=kernels) executed in "
16373471bf0Spatrick           "generic-mode with customized state machines with fallback");
16473471bf0Spatrick STATISTIC(NumOpenMPTargetRegionKernelsCustomStateMachineWithoutFallback,
16573471bf0Spatrick           "Number of OpenMP target region entry points (=kernels) executed in "
16673471bf0Spatrick           "generic-mode with customized state machines without fallback");
167097a140dSpatrick STATISTIC(
168097a140dSpatrick     NumOpenMPParallelRegionsReplacedInGPUStateMachine,
169097a140dSpatrick     "Number of OpenMP parallel regions replaced with ID in GPU state machines");
17073471bf0Spatrick STATISTIC(NumOpenMPParallelRegionsMerged,
17173471bf0Spatrick           "Number of OpenMP parallel regions merged");
17273471bf0Spatrick STATISTIC(NumBytesMovedToSharedMemory,
17373471bf0Spatrick           "Amount of memory pushed to shared memory");
174*d415bd75Srobert STATISTIC(NumBarriersEliminated, "Number of redundant barriers eliminated");
175097a140dSpatrick 
176097a140dSpatrick #if !defined(NDEBUG)
177097a140dSpatrick static constexpr auto TAG = "[" DEBUG_TYPE "]";
178097a140dSpatrick #endif
179097a140dSpatrick 
180097a140dSpatrick namespace {
181097a140dSpatrick 
18273471bf0Spatrick struct AAHeapToShared;
18373471bf0Spatrick 
184097a140dSpatrick struct AAICVTracker;
185097a140dSpatrick 
186097a140dSpatrick /// OpenMP specific information. For now, stores RFIs and ICVs also needed for
187097a140dSpatrick /// Attributor runs.
188097a140dSpatrick struct OMPInformationCache : public InformationCache {
OMPInformationCache__anon4b47cb5e0111::OMPInformationCache189097a140dSpatrick   OMPInformationCache(Module &M, AnalysisGetter &AG,
190*d415bd75Srobert                       BumpPtrAllocator &Allocator, SetVector<Function *> *CGSCC,
191*d415bd75Srobert                       KernelSet &Kernels, bool OpenMPPostLink)
192*d415bd75Srobert       : InformationCache(M, AG, Allocator, CGSCC), OMPBuilder(M),
193*d415bd75Srobert         Kernels(Kernels), OpenMPPostLink(OpenMPPostLink) {
194097a140dSpatrick 
195097a140dSpatrick     OMPBuilder.initialize();
196*d415bd75Srobert     initializeRuntimeFunctions(M);
197097a140dSpatrick     initializeInternalControlVars();
198097a140dSpatrick   }
199097a140dSpatrick 
200097a140dSpatrick   /// Generic information that describes an internal control variable.
201097a140dSpatrick   struct InternalControlVarInfo {
202097a140dSpatrick     /// The kind, as described by InternalControlVar enum.
203097a140dSpatrick     InternalControlVar Kind;
204097a140dSpatrick 
205097a140dSpatrick     /// The name of the ICV.
206097a140dSpatrick     StringRef Name;
207097a140dSpatrick 
208097a140dSpatrick     /// Environment variable associated with this ICV.
209097a140dSpatrick     StringRef EnvVarName;
210097a140dSpatrick 
211097a140dSpatrick     /// Initial value kind.
212097a140dSpatrick     ICVInitValue InitKind;
213097a140dSpatrick 
214097a140dSpatrick     /// Initial value.
215097a140dSpatrick     ConstantInt *InitValue;
216097a140dSpatrick 
217097a140dSpatrick     /// Setter RTL function associated with this ICV.
218097a140dSpatrick     RuntimeFunction Setter;
219097a140dSpatrick 
220097a140dSpatrick     /// Getter RTL function associated with this ICV.
221097a140dSpatrick     RuntimeFunction Getter;
222097a140dSpatrick 
223097a140dSpatrick     /// RTL Function corresponding to the override clause of this ICV
224097a140dSpatrick     RuntimeFunction Clause;
225097a140dSpatrick   };
226097a140dSpatrick 
227097a140dSpatrick   /// Generic information that describes a runtime function
228097a140dSpatrick   struct RuntimeFunctionInfo {
229097a140dSpatrick 
230097a140dSpatrick     /// The kind, as described by the RuntimeFunction enum.
231097a140dSpatrick     RuntimeFunction Kind;
232097a140dSpatrick 
233097a140dSpatrick     /// The name of the function.
234097a140dSpatrick     StringRef Name;
235097a140dSpatrick 
236097a140dSpatrick     /// Flag to indicate a variadic function.
237097a140dSpatrick     bool IsVarArg;
238097a140dSpatrick 
239097a140dSpatrick     /// The return type of the function.
240097a140dSpatrick     Type *ReturnType;
241097a140dSpatrick 
242097a140dSpatrick     /// The argument types of the function.
243097a140dSpatrick     SmallVector<Type *, 8> ArgumentTypes;
244097a140dSpatrick 
245097a140dSpatrick     /// The declaration if available.
246097a140dSpatrick     Function *Declaration = nullptr;
247097a140dSpatrick 
248097a140dSpatrick     /// Uses of this runtime function per function containing the use.
249097a140dSpatrick     using UseVector = SmallVector<Use *, 16>;
250097a140dSpatrick 
251097a140dSpatrick     /// Clear UsesMap for runtime function.
clearUsesMap__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo252097a140dSpatrick     void clearUsesMap() { UsesMap.clear(); }
253097a140dSpatrick 
254097a140dSpatrick     /// Boolean conversion that is true if the runtime function was found.
operator bool__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo255097a140dSpatrick     operator bool() const { return Declaration; }
256097a140dSpatrick 
257097a140dSpatrick     /// Return the vector of uses in function \p F.
getOrCreateUseVector__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo258097a140dSpatrick     UseVector &getOrCreateUseVector(Function *F) {
259097a140dSpatrick       std::shared_ptr<UseVector> &UV = UsesMap[F];
260097a140dSpatrick       if (!UV)
261097a140dSpatrick         UV = std::make_shared<UseVector>();
262097a140dSpatrick       return *UV;
263097a140dSpatrick     }
264097a140dSpatrick 
265097a140dSpatrick     /// Return the vector of uses in function \p F or `nullptr` if there are
266097a140dSpatrick     /// none.
getUseVector__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo267097a140dSpatrick     const UseVector *getUseVector(Function &F) const {
268097a140dSpatrick       auto I = UsesMap.find(&F);
269097a140dSpatrick       if (I != UsesMap.end())
270097a140dSpatrick         return I->second.get();
271097a140dSpatrick       return nullptr;
272097a140dSpatrick     }
273097a140dSpatrick 
274097a140dSpatrick     /// Return how many functions contain uses of this runtime function.
getNumFunctionsWithUses__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo275097a140dSpatrick     size_t getNumFunctionsWithUses() const { return UsesMap.size(); }
276097a140dSpatrick 
277097a140dSpatrick     /// Return the number of arguments (or the minimal number for variadic
278097a140dSpatrick     /// functions).
getNumArgs__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo279097a140dSpatrick     size_t getNumArgs() const { return ArgumentTypes.size(); }
280097a140dSpatrick 
281097a140dSpatrick     /// Run the callback \p CB on each use and forget the use if the result is
282097a140dSpatrick     /// true. The callback will be fed the function in which the use was
283097a140dSpatrick     /// encountered as second argument.
foreachUse__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo284097a140dSpatrick     void foreachUse(SmallVectorImpl<Function *> &SCC,
285097a140dSpatrick                     function_ref<bool(Use &, Function &)> CB) {
286097a140dSpatrick       for (Function *F : SCC)
287097a140dSpatrick         foreachUse(CB, F);
288097a140dSpatrick     }
289097a140dSpatrick 
290097a140dSpatrick     /// Run the callback \p CB on each use within the function \p F and forget
291097a140dSpatrick     /// the use if the result is true.
foreachUse__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo292097a140dSpatrick     void foreachUse(function_ref<bool(Use &, Function &)> CB, Function *F) {
293097a140dSpatrick       SmallVector<unsigned, 8> ToBeDeleted;
294097a140dSpatrick       ToBeDeleted.clear();
295097a140dSpatrick 
296097a140dSpatrick       unsigned Idx = 0;
297097a140dSpatrick       UseVector &UV = getOrCreateUseVector(F);
298097a140dSpatrick 
299097a140dSpatrick       for (Use *U : UV) {
300097a140dSpatrick         if (CB(*U, *F))
301097a140dSpatrick           ToBeDeleted.push_back(Idx);
302097a140dSpatrick         ++Idx;
303097a140dSpatrick       }
304097a140dSpatrick 
305097a140dSpatrick       // Remove the to-be-deleted indices in reverse order as prior
306097a140dSpatrick       // modifications will not modify the smaller indices.
307097a140dSpatrick       while (!ToBeDeleted.empty()) {
308097a140dSpatrick         unsigned Idx = ToBeDeleted.pop_back_val();
309097a140dSpatrick         UV[Idx] = UV.back();
310097a140dSpatrick         UV.pop_back();
311097a140dSpatrick       }
312097a140dSpatrick     }
313097a140dSpatrick 
314097a140dSpatrick   private:
315097a140dSpatrick     /// Map from functions to all uses of this runtime function contained in
316097a140dSpatrick     /// them.
317097a140dSpatrick     DenseMap<Function *, std::shared_ptr<UseVector>> UsesMap;
31873471bf0Spatrick 
31973471bf0Spatrick   public:
32073471bf0Spatrick     /// Iterators for the uses of this runtime function.
begin__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo32173471bf0Spatrick     decltype(UsesMap)::iterator begin() { return UsesMap.begin(); }
end__anon4b47cb5e0111::OMPInformationCache::RuntimeFunctionInfo32273471bf0Spatrick     decltype(UsesMap)::iterator end() { return UsesMap.end(); }
323097a140dSpatrick   };
324097a140dSpatrick 
325097a140dSpatrick   /// An OpenMP-IR-Builder instance
326097a140dSpatrick   OpenMPIRBuilder OMPBuilder;
327097a140dSpatrick 
328097a140dSpatrick   /// Map from runtime function kind to the runtime function description.
329097a140dSpatrick   EnumeratedArray<RuntimeFunctionInfo, RuntimeFunction,
330097a140dSpatrick                   RuntimeFunction::OMPRTL___last>
331097a140dSpatrick       RFIs;
332097a140dSpatrick 
33373471bf0Spatrick   /// Map from function declarations/definitions to their runtime enum type.
33473471bf0Spatrick   DenseMap<Function *, RuntimeFunction> RuntimeFunctionIDMap;
33573471bf0Spatrick 
336097a140dSpatrick   /// Map from ICV kind to the ICV description.
337097a140dSpatrick   EnumeratedArray<InternalControlVarInfo, InternalControlVar,
338097a140dSpatrick                   InternalControlVar::ICV___last>
339097a140dSpatrick       ICVs;
340097a140dSpatrick 
341097a140dSpatrick   /// Helper to initialize all internal control variable information for those
342097a140dSpatrick   /// defined in OMPKinds.def.
initializeInternalControlVars__anon4b47cb5e0111::OMPInformationCache343097a140dSpatrick   void initializeInternalControlVars() {
344097a140dSpatrick #define ICV_RT_SET(_Name, RTL)                                                 \
345097a140dSpatrick   {                                                                            \
346097a140dSpatrick     auto &ICV = ICVs[_Name];                                                   \
347097a140dSpatrick     ICV.Setter = RTL;                                                          \
348097a140dSpatrick   }
349097a140dSpatrick #define ICV_RT_GET(Name, RTL)                                                  \
350097a140dSpatrick   {                                                                            \
351097a140dSpatrick     auto &ICV = ICVs[Name];                                                    \
352097a140dSpatrick     ICV.Getter = RTL;                                                          \
353097a140dSpatrick   }
354097a140dSpatrick #define ICV_DATA_ENV(Enum, _Name, _EnvVarName, Init)                           \
355097a140dSpatrick   {                                                                            \
356097a140dSpatrick     auto &ICV = ICVs[Enum];                                                    \
357097a140dSpatrick     ICV.Name = _Name;                                                          \
358097a140dSpatrick     ICV.Kind = Enum;                                                           \
359097a140dSpatrick     ICV.InitKind = Init;                                                       \
360097a140dSpatrick     ICV.EnvVarName = _EnvVarName;                                              \
361097a140dSpatrick     switch (ICV.InitKind) {                                                    \
362097a140dSpatrick     case ICV_IMPLEMENTATION_DEFINED:                                           \
363097a140dSpatrick       ICV.InitValue = nullptr;                                                 \
364097a140dSpatrick       break;                                                                   \
365097a140dSpatrick     case ICV_ZERO:                                                             \
366097a140dSpatrick       ICV.InitValue = ConstantInt::get(                                        \
367097a140dSpatrick           Type::getInt32Ty(OMPBuilder.Int32->getContext()), 0);                \
368097a140dSpatrick       break;                                                                   \
369097a140dSpatrick     case ICV_FALSE:                                                            \
370097a140dSpatrick       ICV.InitValue = ConstantInt::getFalse(OMPBuilder.Int1->getContext());    \
371097a140dSpatrick       break;                                                                   \
372097a140dSpatrick     case ICV_LAST:                                                             \
373097a140dSpatrick       break;                                                                   \
374097a140dSpatrick     }                                                                          \
375097a140dSpatrick   }
376097a140dSpatrick #include "llvm/Frontend/OpenMP/OMPKinds.def"
377097a140dSpatrick   }
378097a140dSpatrick 
379097a140dSpatrick   /// Returns true if the function declaration \p F matches the runtime
380097a140dSpatrick   /// function types, that is, return type \p RTFRetType, and argument types
381097a140dSpatrick   /// \p RTFArgTypes.
declMatchesRTFTypes__anon4b47cb5e0111::OMPInformationCache382097a140dSpatrick   static bool declMatchesRTFTypes(Function *F, Type *RTFRetType,
383097a140dSpatrick                                   SmallVector<Type *, 8> &RTFArgTypes) {
384097a140dSpatrick     // TODO: We should output information to the user (under debug output
385097a140dSpatrick     //       and via remarks).
386097a140dSpatrick 
387097a140dSpatrick     if (!F)
388097a140dSpatrick       return false;
389097a140dSpatrick     if (F->getReturnType() != RTFRetType)
390097a140dSpatrick       return false;
391097a140dSpatrick     if (F->arg_size() != RTFArgTypes.size())
392097a140dSpatrick       return false;
393097a140dSpatrick 
394*d415bd75Srobert     auto *RTFTyIt = RTFArgTypes.begin();
395097a140dSpatrick     for (Argument &Arg : F->args()) {
396097a140dSpatrick       if (Arg.getType() != *RTFTyIt)
397097a140dSpatrick         return false;
398097a140dSpatrick 
399097a140dSpatrick       ++RTFTyIt;
400097a140dSpatrick     }
401097a140dSpatrick 
402097a140dSpatrick     return true;
403097a140dSpatrick   }
404097a140dSpatrick 
405097a140dSpatrick   // Helper to collect all uses of the declaration in the UsesMap.
collectUses__anon4b47cb5e0111::OMPInformationCache406097a140dSpatrick   unsigned collectUses(RuntimeFunctionInfo &RFI, bool CollectStats = true) {
407097a140dSpatrick     unsigned NumUses = 0;
408097a140dSpatrick     if (!RFI.Declaration)
409097a140dSpatrick       return NumUses;
410097a140dSpatrick     OMPBuilder.addAttributes(RFI.Kind, *RFI.Declaration);
411097a140dSpatrick 
412097a140dSpatrick     if (CollectStats) {
413097a140dSpatrick       NumOpenMPRuntimeFunctionsIdentified += 1;
414097a140dSpatrick       NumOpenMPRuntimeFunctionUsesIdentified += RFI.Declaration->getNumUses();
415097a140dSpatrick     }
416097a140dSpatrick 
417097a140dSpatrick     // TODO: We directly convert uses into proper calls and unknown uses.
418097a140dSpatrick     for (Use &U : RFI.Declaration->uses()) {
419097a140dSpatrick       if (Instruction *UserI = dyn_cast<Instruction>(U.getUser())) {
420*d415bd75Srobert         if (ModuleSlice.empty() || ModuleSlice.count(UserI->getFunction())) {
421097a140dSpatrick           RFI.getOrCreateUseVector(UserI->getFunction()).push_back(&U);
422097a140dSpatrick           ++NumUses;
423097a140dSpatrick         }
424097a140dSpatrick       } else {
425097a140dSpatrick         RFI.getOrCreateUseVector(nullptr).push_back(&U);
426097a140dSpatrick         ++NumUses;
427097a140dSpatrick       }
428097a140dSpatrick     }
429097a140dSpatrick     return NumUses;
430097a140dSpatrick   }
431097a140dSpatrick 
43273471bf0Spatrick   // Helper function to recollect uses of a runtime function.
recollectUsesForFunction__anon4b47cb5e0111::OMPInformationCache43373471bf0Spatrick   void recollectUsesForFunction(RuntimeFunction RTF) {
43473471bf0Spatrick     auto &RFI = RFIs[RTF];
435097a140dSpatrick     RFI.clearUsesMap();
436097a140dSpatrick     collectUses(RFI, /*CollectStats*/ false);
437097a140dSpatrick   }
43873471bf0Spatrick 
43973471bf0Spatrick   // Helper function to recollect uses of all runtime functions.
recollectUses__anon4b47cb5e0111::OMPInformationCache44073471bf0Spatrick   void recollectUses() {
44173471bf0Spatrick     for (int Idx = 0; Idx < RFIs.size(); ++Idx)
44273471bf0Spatrick       recollectUsesForFunction(static_cast<RuntimeFunction>(Idx));
443097a140dSpatrick   }
444097a140dSpatrick 
445*d415bd75Srobert   // Helper function to inherit the calling convention of the function callee.
setCallingConvention__anon4b47cb5e0111::OMPInformationCache446*d415bd75Srobert   void setCallingConvention(FunctionCallee Callee, CallInst *CI) {
447*d415bd75Srobert     if (Function *Fn = dyn_cast<Function>(Callee.getCallee()))
448*d415bd75Srobert       CI->setCallingConv(Fn->getCallingConv());
449*d415bd75Srobert   }
450*d415bd75Srobert 
451*d415bd75Srobert   // Helper function to determine if it's legal to create a call to the runtime
452*d415bd75Srobert   // functions.
runtimeFnsAvailable__anon4b47cb5e0111::OMPInformationCache453*d415bd75Srobert   bool runtimeFnsAvailable(ArrayRef<RuntimeFunction> Fns) {
454*d415bd75Srobert     // We can always emit calls if we haven't yet linked in the runtime.
455*d415bd75Srobert     if (!OpenMPPostLink)
456*d415bd75Srobert       return true;
457*d415bd75Srobert 
458*d415bd75Srobert     // Once the runtime has been already been linked in we cannot emit calls to
459*d415bd75Srobert     // any undefined functions.
460*d415bd75Srobert     for (RuntimeFunction Fn : Fns) {
461*d415bd75Srobert       RuntimeFunctionInfo &RFI = RFIs[Fn];
462*d415bd75Srobert 
463*d415bd75Srobert       if (RFI.Declaration && RFI.Declaration->isDeclaration())
464*d415bd75Srobert         return false;
465*d415bd75Srobert     }
466*d415bd75Srobert     return true;
467*d415bd75Srobert   }
468*d415bd75Srobert 
469097a140dSpatrick   /// Helper to initialize all runtime function information for those defined
470097a140dSpatrick   /// in OpenMPKinds.def.
initializeRuntimeFunctions__anon4b47cb5e0111::OMPInformationCache471*d415bd75Srobert   void initializeRuntimeFunctions(Module &M) {
472097a140dSpatrick 
473097a140dSpatrick     // Helper macros for handling __VA_ARGS__ in OMP_RTL
474097a140dSpatrick #define OMP_TYPE(VarName, ...)                                                 \
475097a140dSpatrick   Type *VarName = OMPBuilder.VarName;                                          \
476097a140dSpatrick   (void)VarName;
477097a140dSpatrick 
478097a140dSpatrick #define OMP_ARRAY_TYPE(VarName, ...)                                           \
479097a140dSpatrick   ArrayType *VarName##Ty = OMPBuilder.VarName##Ty;                             \
480097a140dSpatrick   (void)VarName##Ty;                                                           \
481097a140dSpatrick   PointerType *VarName##PtrTy = OMPBuilder.VarName##PtrTy;                     \
482097a140dSpatrick   (void)VarName##PtrTy;
483097a140dSpatrick 
484097a140dSpatrick #define OMP_FUNCTION_TYPE(VarName, ...)                                        \
485097a140dSpatrick   FunctionType *VarName = OMPBuilder.VarName;                                  \
486097a140dSpatrick   (void)VarName;                                                               \
487097a140dSpatrick   PointerType *VarName##Ptr = OMPBuilder.VarName##Ptr;                         \
488097a140dSpatrick   (void)VarName##Ptr;
489097a140dSpatrick 
490097a140dSpatrick #define OMP_STRUCT_TYPE(VarName, ...)                                          \
491097a140dSpatrick   StructType *VarName = OMPBuilder.VarName;                                    \
492097a140dSpatrick   (void)VarName;                                                               \
493097a140dSpatrick   PointerType *VarName##Ptr = OMPBuilder.VarName##Ptr;                         \
494097a140dSpatrick   (void)VarName##Ptr;
495097a140dSpatrick 
496097a140dSpatrick #define OMP_RTL(_Enum, _Name, _IsVarArg, _ReturnType, ...)                     \
497097a140dSpatrick   {                                                                            \
498097a140dSpatrick     SmallVector<Type *, 8> ArgsTypes({__VA_ARGS__});                           \
499097a140dSpatrick     Function *F = M.getFunction(_Name);                                        \
50073471bf0Spatrick     RTLFunctions.insert(F);                                                    \
501097a140dSpatrick     if (declMatchesRTFTypes(F, OMPBuilder._ReturnType, ArgsTypes)) {           \
50273471bf0Spatrick       RuntimeFunctionIDMap[F] = _Enum;                                         \
503097a140dSpatrick       auto &RFI = RFIs[_Enum];                                                 \
504097a140dSpatrick       RFI.Kind = _Enum;                                                        \
505097a140dSpatrick       RFI.Name = _Name;                                                        \
506097a140dSpatrick       RFI.IsVarArg = _IsVarArg;                                                \
507097a140dSpatrick       RFI.ReturnType = OMPBuilder._ReturnType;                                 \
508097a140dSpatrick       RFI.ArgumentTypes = std::move(ArgsTypes);                                \
509097a140dSpatrick       RFI.Declaration = F;                                                     \
510097a140dSpatrick       unsigned NumUses = collectUses(RFI);                                     \
511097a140dSpatrick       (void)NumUses;                                                           \
512097a140dSpatrick       LLVM_DEBUG({                                                             \
513097a140dSpatrick         dbgs() << TAG << RFI.Name << (RFI.Declaration ? "" : " not")           \
514097a140dSpatrick                << " found\n";                                                  \
515097a140dSpatrick         if (RFI.Declaration)                                                   \
516097a140dSpatrick           dbgs() << TAG << "-> got " << NumUses << " uses in "                 \
517097a140dSpatrick                  << RFI.getNumFunctionsWithUses()                              \
518097a140dSpatrick                  << " different functions.\n";                                 \
519097a140dSpatrick       });                                                                      \
520097a140dSpatrick     }                                                                          \
521097a140dSpatrick   }
522097a140dSpatrick #include "llvm/Frontend/OpenMP/OMPKinds.def"
523097a140dSpatrick 
524*d415bd75Srobert     // Remove the `noinline` attribute from `__kmpc`, `ompx::` and `omp_`
525*d415bd75Srobert     // functions, except if `optnone` is present.
526*d415bd75Srobert     if (isOpenMPDevice(M)) {
527*d415bd75Srobert       for (Function &F : M) {
528*d415bd75Srobert         for (StringRef Prefix : {"__kmpc", "_ZN4ompx", "omp_"})
529*d415bd75Srobert           if (F.hasFnAttribute(Attribute::NoInline) &&
530*d415bd75Srobert               F.getName().startswith(Prefix) &&
531*d415bd75Srobert               !F.hasFnAttribute(Attribute::OptimizeNone))
532*d415bd75Srobert             F.removeFnAttr(Attribute::NoInline);
533*d415bd75Srobert       }
534*d415bd75Srobert     }
535*d415bd75Srobert 
536097a140dSpatrick     // TODO: We should attach the attributes defined in OMPKinds.def.
537097a140dSpatrick   }
538097a140dSpatrick 
539097a140dSpatrick   /// Collection of known kernels (\see Kernel) in the module.
540*d415bd75Srobert   KernelSet &Kernels;
54173471bf0Spatrick 
54273471bf0Spatrick   /// Collection of known OpenMP runtime functions..
54373471bf0Spatrick   DenseSet<const Function *> RTLFunctions;
544*d415bd75Srobert 
545*d415bd75Srobert   /// Indicates if we have already linked in the OpenMP device library.
546*d415bd75Srobert   bool OpenMPPostLink = false;
54773471bf0Spatrick };
54873471bf0Spatrick 
54973471bf0Spatrick template <typename Ty, bool InsertInvalidates = true>
55073471bf0Spatrick struct BooleanStateWithSetVector : public BooleanState {
contains__anon4b47cb5e0111::BooleanStateWithSetVector55173471bf0Spatrick   bool contains(const Ty &Elem) const { return Set.contains(Elem); }
insert__anon4b47cb5e0111::BooleanStateWithSetVector55273471bf0Spatrick   bool insert(const Ty &Elem) {
55373471bf0Spatrick     if (InsertInvalidates)
55473471bf0Spatrick       BooleanState::indicatePessimisticFixpoint();
55573471bf0Spatrick     return Set.insert(Elem);
55673471bf0Spatrick   }
55773471bf0Spatrick 
operator []__anon4b47cb5e0111::BooleanStateWithSetVector55873471bf0Spatrick   const Ty &operator[](int Idx) const { return Set[Idx]; }
operator ==__anon4b47cb5e0111::BooleanStateWithSetVector55973471bf0Spatrick   bool operator==(const BooleanStateWithSetVector &RHS) const {
56073471bf0Spatrick     return BooleanState::operator==(RHS) && Set == RHS.Set;
56173471bf0Spatrick   }
operator !=__anon4b47cb5e0111::BooleanStateWithSetVector56273471bf0Spatrick   bool operator!=(const BooleanStateWithSetVector &RHS) const {
56373471bf0Spatrick     return !(*this == RHS);
56473471bf0Spatrick   }
56573471bf0Spatrick 
empty__anon4b47cb5e0111::BooleanStateWithSetVector56673471bf0Spatrick   bool empty() const { return Set.empty(); }
size__anon4b47cb5e0111::BooleanStateWithSetVector56773471bf0Spatrick   size_t size() const { return Set.size(); }
56873471bf0Spatrick 
56973471bf0Spatrick   /// "Clamp" this state with \p RHS.
operator ^=__anon4b47cb5e0111::BooleanStateWithSetVector57073471bf0Spatrick   BooleanStateWithSetVector &operator^=(const BooleanStateWithSetVector &RHS) {
57173471bf0Spatrick     BooleanState::operator^=(RHS);
57273471bf0Spatrick     Set.insert(RHS.Set.begin(), RHS.Set.end());
57373471bf0Spatrick     return *this;
57473471bf0Spatrick   }
57573471bf0Spatrick 
57673471bf0Spatrick private:
57773471bf0Spatrick   /// A set to keep track of elements.
57873471bf0Spatrick   SetVector<Ty> Set;
57973471bf0Spatrick 
58073471bf0Spatrick public:
begin__anon4b47cb5e0111::BooleanStateWithSetVector58173471bf0Spatrick   typename decltype(Set)::iterator begin() { return Set.begin(); }
end__anon4b47cb5e0111::BooleanStateWithSetVector58273471bf0Spatrick   typename decltype(Set)::iterator end() { return Set.end(); }
begin__anon4b47cb5e0111::BooleanStateWithSetVector58373471bf0Spatrick   typename decltype(Set)::const_iterator begin() const { return Set.begin(); }
end__anon4b47cb5e0111::BooleanStateWithSetVector58473471bf0Spatrick   typename decltype(Set)::const_iterator end() const { return Set.end(); }
58573471bf0Spatrick };
58673471bf0Spatrick 
58773471bf0Spatrick template <typename Ty, bool InsertInvalidates = true>
58873471bf0Spatrick using BooleanStateWithPtrSetVector =
58973471bf0Spatrick     BooleanStateWithSetVector<Ty *, InsertInvalidates>;
59073471bf0Spatrick 
59173471bf0Spatrick struct KernelInfoState : AbstractState {
59273471bf0Spatrick   /// Flag to track if we reached a fixpoint.
59373471bf0Spatrick   bool IsAtFixpoint = false;
59473471bf0Spatrick 
59573471bf0Spatrick   /// The parallel regions (identified by the outlined parallel functions) that
59673471bf0Spatrick   /// can be reached from the associated function.
59773471bf0Spatrick   BooleanStateWithPtrSetVector<Function, /* InsertInvalidates */ false>
59873471bf0Spatrick       ReachedKnownParallelRegions;
59973471bf0Spatrick 
60073471bf0Spatrick   /// State to track what parallel region we might reach.
60173471bf0Spatrick   BooleanStateWithPtrSetVector<CallBase> ReachedUnknownParallelRegions;
60273471bf0Spatrick 
60373471bf0Spatrick   /// State to track if we are in SPMD-mode, assumed or know, and why we decided
60473471bf0Spatrick   /// we cannot be. If it is assumed, then RequiresFullRuntime should also be
60573471bf0Spatrick   /// false.
606*d415bd75Srobert   BooleanStateWithPtrSetVector<Instruction, false> SPMDCompatibilityTracker;
60773471bf0Spatrick 
60873471bf0Spatrick   /// The __kmpc_target_init call in this kernel, if any. If we find more than
60973471bf0Spatrick   /// one we abort as the kernel is malformed.
61073471bf0Spatrick   CallBase *KernelInitCB = nullptr;
61173471bf0Spatrick 
61273471bf0Spatrick   /// The __kmpc_target_deinit call in this kernel, if any. If we find more than
61373471bf0Spatrick   /// one we abort as the kernel is malformed.
61473471bf0Spatrick   CallBase *KernelDeinitCB = nullptr;
61573471bf0Spatrick 
61673471bf0Spatrick   /// Flag to indicate if the associated function is a kernel entry.
61773471bf0Spatrick   bool IsKernelEntry = false;
61873471bf0Spatrick 
61973471bf0Spatrick   /// State to track what kernel entries can reach the associated function.
62073471bf0Spatrick   BooleanStateWithPtrSetVector<Function, false> ReachingKernelEntries;
62173471bf0Spatrick 
62273471bf0Spatrick   /// State to indicate if we can track parallel level of the associated
62373471bf0Spatrick   /// function. We will give up tracking if we encounter unknown caller or the
62473471bf0Spatrick   /// caller is __kmpc_parallel_51.
62573471bf0Spatrick   BooleanStateWithSetVector<uint8_t> ParallelLevels;
62673471bf0Spatrick 
627*d415bd75Srobert   /// Flag that indicates if the kernel has nested Parallelism
628*d415bd75Srobert   bool NestedParallelism = false;
629*d415bd75Srobert 
63073471bf0Spatrick   /// Abstract State interface
63173471bf0Spatrick   ///{
63273471bf0Spatrick 
633*d415bd75Srobert   KernelInfoState() = default;
KernelInfoState__anon4b47cb5e0111::KernelInfoState63473471bf0Spatrick   KernelInfoState(bool BestState) {
63573471bf0Spatrick     if (!BestState)
63673471bf0Spatrick       indicatePessimisticFixpoint();
63773471bf0Spatrick   }
63873471bf0Spatrick 
63973471bf0Spatrick   /// See AbstractState::isValidState(...)
isValidState__anon4b47cb5e0111::KernelInfoState64073471bf0Spatrick   bool isValidState() const override { return true; }
64173471bf0Spatrick 
64273471bf0Spatrick   /// See AbstractState::isAtFixpoint(...)
isAtFixpoint__anon4b47cb5e0111::KernelInfoState64373471bf0Spatrick   bool isAtFixpoint() const override { return IsAtFixpoint; }
64473471bf0Spatrick 
64573471bf0Spatrick   /// See AbstractState::indicatePessimisticFixpoint(...)
indicatePessimisticFixpoint__anon4b47cb5e0111::KernelInfoState64673471bf0Spatrick   ChangeStatus indicatePessimisticFixpoint() override {
64773471bf0Spatrick     IsAtFixpoint = true;
648*d415bd75Srobert     ParallelLevels.indicatePessimisticFixpoint();
649*d415bd75Srobert     ReachingKernelEntries.indicatePessimisticFixpoint();
65073471bf0Spatrick     SPMDCompatibilityTracker.indicatePessimisticFixpoint();
651*d415bd75Srobert     ReachedKnownParallelRegions.indicatePessimisticFixpoint();
65273471bf0Spatrick     ReachedUnknownParallelRegions.indicatePessimisticFixpoint();
65373471bf0Spatrick     return ChangeStatus::CHANGED;
65473471bf0Spatrick   }
65573471bf0Spatrick 
65673471bf0Spatrick   /// See AbstractState::indicateOptimisticFixpoint(...)
indicateOptimisticFixpoint__anon4b47cb5e0111::KernelInfoState65773471bf0Spatrick   ChangeStatus indicateOptimisticFixpoint() override {
65873471bf0Spatrick     IsAtFixpoint = true;
659*d415bd75Srobert     ParallelLevels.indicateOptimisticFixpoint();
660*d415bd75Srobert     ReachingKernelEntries.indicateOptimisticFixpoint();
661*d415bd75Srobert     SPMDCompatibilityTracker.indicateOptimisticFixpoint();
662*d415bd75Srobert     ReachedKnownParallelRegions.indicateOptimisticFixpoint();
663*d415bd75Srobert     ReachedUnknownParallelRegions.indicateOptimisticFixpoint();
66473471bf0Spatrick     return ChangeStatus::UNCHANGED;
66573471bf0Spatrick   }
66673471bf0Spatrick 
66773471bf0Spatrick   /// Return the assumed state
getAssumed__anon4b47cb5e0111::KernelInfoState66873471bf0Spatrick   KernelInfoState &getAssumed() { return *this; }
getAssumed__anon4b47cb5e0111::KernelInfoState66973471bf0Spatrick   const KernelInfoState &getAssumed() const { return *this; }
67073471bf0Spatrick 
operator ==__anon4b47cb5e0111::KernelInfoState67173471bf0Spatrick   bool operator==(const KernelInfoState &RHS) const {
67273471bf0Spatrick     if (SPMDCompatibilityTracker != RHS.SPMDCompatibilityTracker)
67373471bf0Spatrick       return false;
67473471bf0Spatrick     if (ReachedKnownParallelRegions != RHS.ReachedKnownParallelRegions)
67573471bf0Spatrick       return false;
67673471bf0Spatrick     if (ReachedUnknownParallelRegions != RHS.ReachedUnknownParallelRegions)
67773471bf0Spatrick       return false;
67873471bf0Spatrick     if (ReachingKernelEntries != RHS.ReachingKernelEntries)
67973471bf0Spatrick       return false;
680*d415bd75Srobert     if (ParallelLevels != RHS.ParallelLevels)
681*d415bd75Srobert       return false;
68273471bf0Spatrick     return true;
68373471bf0Spatrick   }
68473471bf0Spatrick 
685*d415bd75Srobert   /// Returns true if this kernel contains any OpenMP parallel regions.
mayContainParallelRegion__anon4b47cb5e0111::KernelInfoState686*d415bd75Srobert   bool mayContainParallelRegion() {
687*d415bd75Srobert     return !ReachedKnownParallelRegions.empty() ||
688*d415bd75Srobert            !ReachedUnknownParallelRegions.empty();
689*d415bd75Srobert   }
690*d415bd75Srobert 
69173471bf0Spatrick   /// Return empty set as the best state of potential values.
getBestState__anon4b47cb5e0111::KernelInfoState69273471bf0Spatrick   static KernelInfoState getBestState() { return KernelInfoState(true); }
69373471bf0Spatrick 
getBestState__anon4b47cb5e0111::KernelInfoState69473471bf0Spatrick   static KernelInfoState getBestState(KernelInfoState &KIS) {
69573471bf0Spatrick     return getBestState();
69673471bf0Spatrick   }
69773471bf0Spatrick 
69873471bf0Spatrick   /// Return full set as the worst state of potential values.
getWorstState__anon4b47cb5e0111::KernelInfoState69973471bf0Spatrick   static KernelInfoState getWorstState() { return KernelInfoState(false); }
70073471bf0Spatrick 
70173471bf0Spatrick   /// "Clamp" this state with \p KIS.
operator ^=__anon4b47cb5e0111::KernelInfoState70273471bf0Spatrick   KernelInfoState operator^=(const KernelInfoState &KIS) {
70373471bf0Spatrick     // Do not merge two different _init and _deinit call sites.
70473471bf0Spatrick     if (KIS.KernelInitCB) {
70573471bf0Spatrick       if (KernelInitCB && KernelInitCB != KIS.KernelInitCB)
706*d415bd75Srobert         llvm_unreachable("Kernel that calls another kernel violates OpenMP-Opt "
707*d415bd75Srobert                          "assumptions.");
70873471bf0Spatrick       KernelInitCB = KIS.KernelInitCB;
70973471bf0Spatrick     }
71073471bf0Spatrick     if (KIS.KernelDeinitCB) {
71173471bf0Spatrick       if (KernelDeinitCB && KernelDeinitCB != KIS.KernelDeinitCB)
712*d415bd75Srobert         llvm_unreachable("Kernel that calls another kernel violates OpenMP-Opt "
713*d415bd75Srobert                          "assumptions.");
71473471bf0Spatrick       KernelDeinitCB = KIS.KernelDeinitCB;
71573471bf0Spatrick     }
71673471bf0Spatrick     SPMDCompatibilityTracker ^= KIS.SPMDCompatibilityTracker;
71773471bf0Spatrick     ReachedKnownParallelRegions ^= KIS.ReachedKnownParallelRegions;
71873471bf0Spatrick     ReachedUnknownParallelRegions ^= KIS.ReachedUnknownParallelRegions;
719*d415bd75Srobert     NestedParallelism |= KIS.NestedParallelism;
72073471bf0Spatrick     return *this;
72173471bf0Spatrick   }
72273471bf0Spatrick 
operator &=__anon4b47cb5e0111::KernelInfoState72373471bf0Spatrick   KernelInfoState operator&=(const KernelInfoState &KIS) {
72473471bf0Spatrick     return (*this ^= KIS);
72573471bf0Spatrick   }
72673471bf0Spatrick 
72773471bf0Spatrick   ///}
72873471bf0Spatrick };
72973471bf0Spatrick 
73073471bf0Spatrick /// Used to map the values physically (in the IR) stored in an offload
73173471bf0Spatrick /// array, to a vector in memory.
73273471bf0Spatrick struct OffloadArray {
73373471bf0Spatrick   /// Physical array (in the IR).
73473471bf0Spatrick   AllocaInst *Array = nullptr;
73573471bf0Spatrick   /// Mapped values.
73673471bf0Spatrick   SmallVector<Value *, 8> StoredValues;
73773471bf0Spatrick   /// Last stores made in the offload array.
73873471bf0Spatrick   SmallVector<StoreInst *, 8> LastAccesses;
73973471bf0Spatrick 
74073471bf0Spatrick   OffloadArray() = default;
74173471bf0Spatrick 
74273471bf0Spatrick   /// Initializes the OffloadArray with the values stored in \p Array before
74373471bf0Spatrick   /// instruction \p Before is reached. Returns false if the initialization
74473471bf0Spatrick   /// fails.
74573471bf0Spatrick   /// This MUST be used immediately after the construction of the object.
initialize__anon4b47cb5e0111::OffloadArray74673471bf0Spatrick   bool initialize(AllocaInst &Array, Instruction &Before) {
74773471bf0Spatrick     if (!Array.getAllocatedType()->isArrayTy())
74873471bf0Spatrick       return false;
74973471bf0Spatrick 
75073471bf0Spatrick     if (!getValues(Array, Before))
75173471bf0Spatrick       return false;
75273471bf0Spatrick 
75373471bf0Spatrick     this->Array = &Array;
75473471bf0Spatrick     return true;
75573471bf0Spatrick   }
75673471bf0Spatrick 
75773471bf0Spatrick   static const unsigned DeviceIDArgNum = 1;
75873471bf0Spatrick   static const unsigned BasePtrsArgNum = 3;
75973471bf0Spatrick   static const unsigned PtrsArgNum = 4;
76073471bf0Spatrick   static const unsigned SizesArgNum = 5;
76173471bf0Spatrick 
76273471bf0Spatrick private:
76373471bf0Spatrick   /// Traverses the BasicBlock where \p Array is, collecting the stores made to
76473471bf0Spatrick   /// \p Array, leaving StoredValues with the values stored before the
76573471bf0Spatrick   /// instruction \p Before is reached.
getValues__anon4b47cb5e0111::OffloadArray76673471bf0Spatrick   bool getValues(AllocaInst &Array, Instruction &Before) {
76773471bf0Spatrick     // Initialize container.
76873471bf0Spatrick     const uint64_t NumValues = Array.getAllocatedType()->getArrayNumElements();
76973471bf0Spatrick     StoredValues.assign(NumValues, nullptr);
77073471bf0Spatrick     LastAccesses.assign(NumValues, nullptr);
77173471bf0Spatrick 
77273471bf0Spatrick     // TODO: This assumes the instruction \p Before is in the same
77373471bf0Spatrick     //  BasicBlock as Array. Make it general, for any control flow graph.
77473471bf0Spatrick     BasicBlock *BB = Array.getParent();
77573471bf0Spatrick     if (BB != Before.getParent())
77673471bf0Spatrick       return false;
77773471bf0Spatrick 
77873471bf0Spatrick     const DataLayout &DL = Array.getModule()->getDataLayout();
77973471bf0Spatrick     const unsigned int PointerSize = DL.getPointerSize();
78073471bf0Spatrick 
78173471bf0Spatrick     for (Instruction &I : *BB) {
78273471bf0Spatrick       if (&I == &Before)
78373471bf0Spatrick         break;
78473471bf0Spatrick 
78573471bf0Spatrick       if (!isa<StoreInst>(&I))
78673471bf0Spatrick         continue;
78773471bf0Spatrick 
78873471bf0Spatrick       auto *S = cast<StoreInst>(&I);
78973471bf0Spatrick       int64_t Offset = -1;
79073471bf0Spatrick       auto *Dst =
79173471bf0Spatrick           GetPointerBaseWithConstantOffset(S->getPointerOperand(), Offset, DL);
79273471bf0Spatrick       if (Dst == &Array) {
79373471bf0Spatrick         int64_t Idx = Offset / PointerSize;
79473471bf0Spatrick         StoredValues[Idx] = getUnderlyingObject(S->getValueOperand());
79573471bf0Spatrick         LastAccesses[Idx] = S;
79673471bf0Spatrick       }
79773471bf0Spatrick     }
79873471bf0Spatrick 
79973471bf0Spatrick     return isFilled();
80073471bf0Spatrick   }
80173471bf0Spatrick 
80273471bf0Spatrick   /// Returns true if all values in StoredValues and
80373471bf0Spatrick   /// LastAccesses are not nullptrs.
isFilled__anon4b47cb5e0111::OffloadArray80473471bf0Spatrick   bool isFilled() {
80573471bf0Spatrick     const unsigned NumValues = StoredValues.size();
80673471bf0Spatrick     for (unsigned I = 0; I < NumValues; ++I) {
80773471bf0Spatrick       if (!StoredValues[I] || !LastAccesses[I])
80873471bf0Spatrick         return false;
80973471bf0Spatrick     }
81073471bf0Spatrick 
81173471bf0Spatrick     return true;
81273471bf0Spatrick   }
813097a140dSpatrick };
814097a140dSpatrick 
815097a140dSpatrick struct OpenMPOpt {
816097a140dSpatrick 
817097a140dSpatrick   using OptimizationRemarkGetter =
818097a140dSpatrick       function_ref<OptimizationRemarkEmitter &(Function *)>;
819097a140dSpatrick 
OpenMPOpt__anon4b47cb5e0111::OpenMPOpt820097a140dSpatrick   OpenMPOpt(SmallVectorImpl<Function *> &SCC, CallGraphUpdater &CGUpdater,
821097a140dSpatrick             OptimizationRemarkGetter OREGetter,
822097a140dSpatrick             OMPInformationCache &OMPInfoCache, Attributor &A)
823097a140dSpatrick       : M(*(*SCC.begin())->getParent()), SCC(SCC), CGUpdater(CGUpdater),
824097a140dSpatrick         OREGetter(OREGetter), OMPInfoCache(OMPInfoCache), A(A) {}
825097a140dSpatrick 
82673471bf0Spatrick   /// Check if any remarks are enabled for openmp-opt
remarksEnabled__anon4b47cb5e0111::OpenMPOpt82773471bf0Spatrick   bool remarksEnabled() {
82873471bf0Spatrick     auto &Ctx = M.getContext();
82973471bf0Spatrick     return Ctx.getDiagHandlerPtr()->isAnyRemarkEnabled(DEBUG_TYPE);
83073471bf0Spatrick   }
83173471bf0Spatrick 
832097a140dSpatrick   /// Run all OpenMP optimizations on the underlying SCC/ModuleSlice.
run__anon4b47cb5e0111::OpenMPOpt83373471bf0Spatrick   bool run(bool IsModulePass) {
834097a140dSpatrick     if (SCC.empty())
835097a140dSpatrick       return false;
836097a140dSpatrick 
837097a140dSpatrick     bool Changed = false;
838097a140dSpatrick 
839097a140dSpatrick     LLVM_DEBUG(dbgs() << TAG << "Run on SCC with " << SCC.size()
840097a140dSpatrick                       << " functions in a slice with "
841097a140dSpatrick                       << OMPInfoCache.ModuleSlice.size() << " functions\n");
842097a140dSpatrick 
84373471bf0Spatrick     if (IsModulePass) {
84473471bf0Spatrick       Changed |= runAttributor(IsModulePass);
84573471bf0Spatrick 
84673471bf0Spatrick       // Recollect uses, in case Attributor deleted any.
84773471bf0Spatrick       OMPInfoCache.recollectUses();
84873471bf0Spatrick 
84973471bf0Spatrick       // TODO: This should be folded into buildCustomStateMachine.
85073471bf0Spatrick       Changed |= rewriteDeviceCodeStateMachine();
85173471bf0Spatrick 
85273471bf0Spatrick       if (remarksEnabled())
85373471bf0Spatrick         analysisGlobalization();
85473471bf0Spatrick     } else {
855097a140dSpatrick       if (PrintICVValues)
856097a140dSpatrick         printICVs();
857097a140dSpatrick       if (PrintOpenMPKernels)
858097a140dSpatrick         printKernels();
859097a140dSpatrick 
86073471bf0Spatrick       Changed |= runAttributor(IsModulePass);
861097a140dSpatrick 
862097a140dSpatrick       // Recollect uses, in case Attributor deleted any.
863097a140dSpatrick       OMPInfoCache.recollectUses();
864097a140dSpatrick 
865097a140dSpatrick       Changed |= deleteParallelRegions();
866097a140dSpatrick 
86773471bf0Spatrick       if (HideMemoryTransferLatency)
86873471bf0Spatrick         Changed |= hideMemTransfersLatency();
86973471bf0Spatrick       Changed |= deduplicateRuntimeCalls();
87073471bf0Spatrick       if (EnableParallelRegionMerging) {
87173471bf0Spatrick         if (mergeParallelRegions()) {
87273471bf0Spatrick           deduplicateRuntimeCalls();
87373471bf0Spatrick           Changed = true;
87473471bf0Spatrick         }
87573471bf0Spatrick       }
87673471bf0Spatrick     }
87773471bf0Spatrick 
878097a140dSpatrick     return Changed;
879097a140dSpatrick   }
880097a140dSpatrick 
881097a140dSpatrick   /// Print initial ICV values for testing.
882097a140dSpatrick   /// FIXME: This should be done from the Attributor once it is added.
printICVs__anon4b47cb5e0111::OpenMPOpt883097a140dSpatrick   void printICVs() const {
88473471bf0Spatrick     InternalControlVar ICVs[] = {ICV_nthreads, ICV_active_levels, ICV_cancel,
88573471bf0Spatrick                                  ICV_proc_bind};
886097a140dSpatrick 
887*d415bd75Srobert     for (Function *F : SCC) {
888097a140dSpatrick       for (auto ICV : ICVs) {
889097a140dSpatrick         auto ICVInfo = OMPInfoCache.ICVs[ICV];
89073471bf0Spatrick         auto Remark = [&](OptimizationRemarkAnalysis ORA) {
89173471bf0Spatrick           return ORA << "OpenMP ICV " << ore::NV("OpenMPICV", ICVInfo.Name)
892097a140dSpatrick                      << " Value: "
893097a140dSpatrick                      << (ICVInfo.InitValue
89473471bf0Spatrick                              ? toString(ICVInfo.InitValue->getValue(), 10, true)
895097a140dSpatrick                              : "IMPLEMENTATION_DEFINED");
896097a140dSpatrick         };
897097a140dSpatrick 
89873471bf0Spatrick         emitRemark<OptimizationRemarkAnalysis>(F, "OpenMPICVTracker", Remark);
899097a140dSpatrick       }
900097a140dSpatrick     }
901097a140dSpatrick   }
902097a140dSpatrick 
903097a140dSpatrick   /// Print OpenMP GPU kernels for testing.
printKernels__anon4b47cb5e0111::OpenMPOpt904097a140dSpatrick   void printKernels() const {
905097a140dSpatrick     for (Function *F : SCC) {
906097a140dSpatrick       if (!OMPInfoCache.Kernels.count(F))
907097a140dSpatrick         continue;
908097a140dSpatrick 
90973471bf0Spatrick       auto Remark = [&](OptimizationRemarkAnalysis ORA) {
91073471bf0Spatrick         return ORA << "OpenMP GPU kernel "
911097a140dSpatrick                    << ore::NV("OpenMPGPUKernel", F->getName()) << "\n";
912097a140dSpatrick       };
913097a140dSpatrick 
91473471bf0Spatrick       emitRemark<OptimizationRemarkAnalysis>(F, "OpenMPGPU", Remark);
915097a140dSpatrick     }
916097a140dSpatrick   }
917097a140dSpatrick 
918097a140dSpatrick   /// Return the call if \p U is a callee use in a regular call. If \p RFI is
919097a140dSpatrick   /// given it has to be the callee or a nullptr is returned.
getCallIfRegularCall__anon4b47cb5e0111::OpenMPOpt920097a140dSpatrick   static CallInst *getCallIfRegularCall(
921097a140dSpatrick       Use &U, OMPInformationCache::RuntimeFunctionInfo *RFI = nullptr) {
922097a140dSpatrick     CallInst *CI = dyn_cast<CallInst>(U.getUser());
923097a140dSpatrick     if (CI && CI->isCallee(&U) && !CI->hasOperandBundles() &&
92473471bf0Spatrick         (!RFI ||
92573471bf0Spatrick          (RFI->Declaration && CI->getCalledFunction() == RFI->Declaration)))
926097a140dSpatrick       return CI;
927097a140dSpatrick     return nullptr;
928097a140dSpatrick   }
929097a140dSpatrick 
930097a140dSpatrick   /// Return the call if \p V is a regular call. If \p RFI is given it has to be
931097a140dSpatrick   /// the callee or a nullptr is returned.
getCallIfRegularCall__anon4b47cb5e0111::OpenMPOpt932097a140dSpatrick   static CallInst *getCallIfRegularCall(
933097a140dSpatrick       Value &V, OMPInformationCache::RuntimeFunctionInfo *RFI = nullptr) {
934097a140dSpatrick     CallInst *CI = dyn_cast<CallInst>(&V);
935097a140dSpatrick     if (CI && !CI->hasOperandBundles() &&
93673471bf0Spatrick         (!RFI ||
93773471bf0Spatrick          (RFI->Declaration && CI->getCalledFunction() == RFI->Declaration)))
938097a140dSpatrick       return CI;
939097a140dSpatrick     return nullptr;
940097a140dSpatrick   }
941097a140dSpatrick 
942097a140dSpatrick private:
94373471bf0Spatrick   /// Merge parallel regions when it is safe.
mergeParallelRegions__anon4b47cb5e0111::OpenMPOpt94473471bf0Spatrick   bool mergeParallelRegions() {
94573471bf0Spatrick     const unsigned CallbackCalleeOperand = 2;
94673471bf0Spatrick     const unsigned CallbackFirstArgOperand = 3;
94773471bf0Spatrick     using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
94873471bf0Spatrick 
94973471bf0Spatrick     // Check if there are any __kmpc_fork_call calls to merge.
95073471bf0Spatrick     OMPInformationCache::RuntimeFunctionInfo &RFI =
95173471bf0Spatrick         OMPInfoCache.RFIs[OMPRTL___kmpc_fork_call];
95273471bf0Spatrick 
95373471bf0Spatrick     if (!RFI.Declaration)
95473471bf0Spatrick       return false;
95573471bf0Spatrick 
95673471bf0Spatrick     // Unmergable calls that prevent merging a parallel region.
95773471bf0Spatrick     OMPInformationCache::RuntimeFunctionInfo UnmergableCallsInfo[] = {
95873471bf0Spatrick         OMPInfoCache.RFIs[OMPRTL___kmpc_push_proc_bind],
95973471bf0Spatrick         OMPInfoCache.RFIs[OMPRTL___kmpc_push_num_threads],
96073471bf0Spatrick     };
96173471bf0Spatrick 
96273471bf0Spatrick     bool Changed = false;
96373471bf0Spatrick     LoopInfo *LI = nullptr;
96473471bf0Spatrick     DominatorTree *DT = nullptr;
96573471bf0Spatrick 
96673471bf0Spatrick     SmallDenseMap<BasicBlock *, SmallPtrSet<Instruction *, 4>> BB2PRMap;
96773471bf0Spatrick 
96873471bf0Spatrick     BasicBlock *StartBB = nullptr, *EndBB = nullptr;
969*d415bd75Srobert     auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
97073471bf0Spatrick       BasicBlock *CGStartBB = CodeGenIP.getBlock();
97173471bf0Spatrick       BasicBlock *CGEndBB =
97273471bf0Spatrick           SplitBlock(CGStartBB, &*CodeGenIP.getPoint(), DT, LI);
97373471bf0Spatrick       assert(StartBB != nullptr && "StartBB should not be null");
97473471bf0Spatrick       CGStartBB->getTerminator()->setSuccessor(0, StartBB);
97573471bf0Spatrick       assert(EndBB != nullptr && "EndBB should not be null");
97673471bf0Spatrick       EndBB->getTerminator()->setSuccessor(0, CGEndBB);
97773471bf0Spatrick     };
97873471bf0Spatrick 
97973471bf0Spatrick     auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
98073471bf0Spatrick                       Value &Inner, Value *&ReplacementValue) -> InsertPointTy {
98173471bf0Spatrick       ReplacementValue = &Inner;
98273471bf0Spatrick       return CodeGenIP;
98373471bf0Spatrick     };
98473471bf0Spatrick 
98573471bf0Spatrick     auto FiniCB = [&](InsertPointTy CodeGenIP) {};
98673471bf0Spatrick 
98773471bf0Spatrick     /// Create a sequential execution region within a merged parallel region,
98873471bf0Spatrick     /// encapsulated in a master construct with a barrier for synchronization.
98973471bf0Spatrick     auto CreateSequentialRegion = [&](Function *OuterFn,
99073471bf0Spatrick                                       BasicBlock *OuterPredBB,
99173471bf0Spatrick                                       Instruction *SeqStartI,
99273471bf0Spatrick                                       Instruction *SeqEndI) {
99373471bf0Spatrick       // Isolate the instructions of the sequential region to a separate
99473471bf0Spatrick       // block.
99573471bf0Spatrick       BasicBlock *ParentBB = SeqStartI->getParent();
99673471bf0Spatrick       BasicBlock *SeqEndBB =
99773471bf0Spatrick           SplitBlock(ParentBB, SeqEndI->getNextNode(), DT, LI);
99873471bf0Spatrick       BasicBlock *SeqAfterBB =
99973471bf0Spatrick           SplitBlock(SeqEndBB, &*SeqEndBB->getFirstInsertionPt(), DT, LI);
100073471bf0Spatrick       BasicBlock *SeqStartBB =
100173471bf0Spatrick           SplitBlock(ParentBB, SeqStartI, DT, LI, nullptr, "seq.par.merged");
100273471bf0Spatrick 
100373471bf0Spatrick       assert(ParentBB->getUniqueSuccessor() == SeqStartBB &&
100473471bf0Spatrick              "Expected a different CFG");
100573471bf0Spatrick       const DebugLoc DL = ParentBB->getTerminator()->getDebugLoc();
100673471bf0Spatrick       ParentBB->getTerminator()->eraseFromParent();
100773471bf0Spatrick 
1008*d415bd75Srobert       auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
100973471bf0Spatrick         BasicBlock *CGStartBB = CodeGenIP.getBlock();
101073471bf0Spatrick         BasicBlock *CGEndBB =
101173471bf0Spatrick             SplitBlock(CGStartBB, &*CodeGenIP.getPoint(), DT, LI);
101273471bf0Spatrick         assert(SeqStartBB != nullptr && "SeqStartBB should not be null");
101373471bf0Spatrick         CGStartBB->getTerminator()->setSuccessor(0, SeqStartBB);
101473471bf0Spatrick         assert(SeqEndBB != nullptr && "SeqEndBB should not be null");
101573471bf0Spatrick         SeqEndBB->getTerminator()->setSuccessor(0, CGEndBB);
101673471bf0Spatrick       };
101773471bf0Spatrick       auto FiniCB = [&](InsertPointTy CodeGenIP) {};
101873471bf0Spatrick 
101973471bf0Spatrick       // Find outputs from the sequential region to outside users and
102073471bf0Spatrick       // broadcast their values to them.
102173471bf0Spatrick       for (Instruction &I : *SeqStartBB) {
102273471bf0Spatrick         SmallPtrSet<Instruction *, 4> OutsideUsers;
102373471bf0Spatrick         for (User *Usr : I.users()) {
102473471bf0Spatrick           Instruction &UsrI = *cast<Instruction>(Usr);
102573471bf0Spatrick           // Ignore outputs to LT intrinsics, code extraction for the merged
102673471bf0Spatrick           // parallel region will fix them.
102773471bf0Spatrick           if (UsrI.isLifetimeStartOrEnd())
102873471bf0Spatrick             continue;
102973471bf0Spatrick 
103073471bf0Spatrick           if (UsrI.getParent() != SeqStartBB)
103173471bf0Spatrick             OutsideUsers.insert(&UsrI);
103273471bf0Spatrick         }
103373471bf0Spatrick 
103473471bf0Spatrick         if (OutsideUsers.empty())
103573471bf0Spatrick           continue;
103673471bf0Spatrick 
103773471bf0Spatrick         // Emit an alloca in the outer region to store the broadcasted
103873471bf0Spatrick         // value.
103973471bf0Spatrick         const DataLayout &DL = M.getDataLayout();
104073471bf0Spatrick         AllocaInst *AllocaI = new AllocaInst(
104173471bf0Spatrick             I.getType(), DL.getAllocaAddrSpace(), nullptr,
104273471bf0Spatrick             I.getName() + ".seq.output.alloc", &OuterFn->front().front());
104373471bf0Spatrick 
104473471bf0Spatrick         // Emit a store instruction in the sequential BB to update the
104573471bf0Spatrick         // value.
104673471bf0Spatrick         new StoreInst(&I, AllocaI, SeqStartBB->getTerminator());
104773471bf0Spatrick 
104873471bf0Spatrick         // Emit a load instruction and replace the use of the output value
104973471bf0Spatrick         // with it.
105073471bf0Spatrick         for (Instruction *UsrI : OutsideUsers) {
105173471bf0Spatrick           LoadInst *LoadI = new LoadInst(
105273471bf0Spatrick               I.getType(), AllocaI, I.getName() + ".seq.output.load", UsrI);
105373471bf0Spatrick           UsrI->replaceUsesOfWith(&I, LoadI);
105473471bf0Spatrick         }
105573471bf0Spatrick       }
105673471bf0Spatrick 
105773471bf0Spatrick       OpenMPIRBuilder::LocationDescription Loc(
105873471bf0Spatrick           InsertPointTy(ParentBB, ParentBB->end()), DL);
105973471bf0Spatrick       InsertPointTy SeqAfterIP =
106073471bf0Spatrick           OMPInfoCache.OMPBuilder.createMaster(Loc, BodyGenCB, FiniCB);
106173471bf0Spatrick 
106273471bf0Spatrick       OMPInfoCache.OMPBuilder.createBarrier(SeqAfterIP, OMPD_parallel);
106373471bf0Spatrick 
106473471bf0Spatrick       BranchInst::Create(SeqAfterBB, SeqAfterIP.getBlock());
106573471bf0Spatrick 
106673471bf0Spatrick       LLVM_DEBUG(dbgs() << TAG << "After sequential inlining " << *OuterFn
106773471bf0Spatrick                         << "\n");
106873471bf0Spatrick     };
106973471bf0Spatrick 
107073471bf0Spatrick     // Helper to merge the __kmpc_fork_call calls in MergableCIs. They are all
107173471bf0Spatrick     // contained in BB and only separated by instructions that can be
107273471bf0Spatrick     // redundantly executed in parallel. The block BB is split before the first
107373471bf0Spatrick     // call (in MergableCIs) and after the last so the entire region we merge
107473471bf0Spatrick     // into a single parallel region is contained in a single basic block
107573471bf0Spatrick     // without any other instructions. We use the OpenMPIRBuilder to outline
107673471bf0Spatrick     // that block and call the resulting function via __kmpc_fork_call.
1077*d415bd75Srobert     auto Merge = [&](const SmallVectorImpl<CallInst *> &MergableCIs,
1078*d415bd75Srobert                      BasicBlock *BB) {
107973471bf0Spatrick       // TODO: Change the interface to allow single CIs expanded, e.g, to
108073471bf0Spatrick       // include an outer loop.
108173471bf0Spatrick       assert(MergableCIs.size() > 1 && "Assumed multiple mergable CIs");
108273471bf0Spatrick 
108373471bf0Spatrick       auto Remark = [&](OptimizationRemark OR) {
108473471bf0Spatrick         OR << "Parallel region merged with parallel region"
108573471bf0Spatrick            << (MergableCIs.size() > 2 ? "s" : "") << " at ";
108673471bf0Spatrick         for (auto *CI : llvm::drop_begin(MergableCIs)) {
108773471bf0Spatrick           OR << ore::NV("OpenMPParallelMerge", CI->getDebugLoc());
108873471bf0Spatrick           if (CI != MergableCIs.back())
108973471bf0Spatrick             OR << ", ";
109073471bf0Spatrick         }
109173471bf0Spatrick         return OR << ".";
109273471bf0Spatrick       };
109373471bf0Spatrick 
109473471bf0Spatrick       emitRemark<OptimizationRemark>(MergableCIs.front(), "OMP150", Remark);
109573471bf0Spatrick 
109673471bf0Spatrick       Function *OriginalFn = BB->getParent();
109773471bf0Spatrick       LLVM_DEBUG(dbgs() << TAG << "Merge " << MergableCIs.size()
109873471bf0Spatrick                         << " parallel regions in " << OriginalFn->getName()
109973471bf0Spatrick                         << "\n");
110073471bf0Spatrick 
110173471bf0Spatrick       // Isolate the calls to merge in a separate block.
110273471bf0Spatrick       EndBB = SplitBlock(BB, MergableCIs.back()->getNextNode(), DT, LI);
110373471bf0Spatrick       BasicBlock *AfterBB =
110473471bf0Spatrick           SplitBlock(EndBB, &*EndBB->getFirstInsertionPt(), DT, LI);
110573471bf0Spatrick       StartBB = SplitBlock(BB, MergableCIs.front(), DT, LI, nullptr,
110673471bf0Spatrick                            "omp.par.merged");
110773471bf0Spatrick 
110873471bf0Spatrick       assert(BB->getUniqueSuccessor() == StartBB && "Expected a different CFG");
110973471bf0Spatrick       const DebugLoc DL = BB->getTerminator()->getDebugLoc();
111073471bf0Spatrick       BB->getTerminator()->eraseFromParent();
111173471bf0Spatrick 
111273471bf0Spatrick       // Create sequential regions for sequential instructions that are
111373471bf0Spatrick       // in-between mergable parallel regions.
111473471bf0Spatrick       for (auto *It = MergableCIs.begin(), *End = MergableCIs.end() - 1;
111573471bf0Spatrick            It != End; ++It) {
111673471bf0Spatrick         Instruction *ForkCI = *It;
111773471bf0Spatrick         Instruction *NextForkCI = *(It + 1);
111873471bf0Spatrick 
111973471bf0Spatrick         // Continue if there are not in-between instructions.
112073471bf0Spatrick         if (ForkCI->getNextNode() == NextForkCI)
112173471bf0Spatrick           continue;
112273471bf0Spatrick 
112373471bf0Spatrick         CreateSequentialRegion(OriginalFn, BB, ForkCI->getNextNode(),
112473471bf0Spatrick                                NextForkCI->getPrevNode());
112573471bf0Spatrick       }
112673471bf0Spatrick 
112773471bf0Spatrick       OpenMPIRBuilder::LocationDescription Loc(InsertPointTy(BB, BB->end()),
112873471bf0Spatrick                                                DL);
112973471bf0Spatrick       IRBuilder<>::InsertPoint AllocaIP(
113073471bf0Spatrick           &OriginalFn->getEntryBlock(),
113173471bf0Spatrick           OriginalFn->getEntryBlock().getFirstInsertionPt());
113273471bf0Spatrick       // Create the merged parallel region with default proc binding, to
113373471bf0Spatrick       // avoid overriding binding settings, and without explicit cancellation.
113473471bf0Spatrick       InsertPointTy AfterIP = OMPInfoCache.OMPBuilder.createParallel(
113573471bf0Spatrick           Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr,
113673471bf0Spatrick           OMP_PROC_BIND_default, /* IsCancellable */ false);
113773471bf0Spatrick       BranchInst::Create(AfterBB, AfterIP.getBlock());
113873471bf0Spatrick 
113973471bf0Spatrick       // Perform the actual outlining.
1140*d415bd75Srobert       OMPInfoCache.OMPBuilder.finalize(OriginalFn);
114173471bf0Spatrick 
114273471bf0Spatrick       Function *OutlinedFn = MergableCIs.front()->getCaller();
114373471bf0Spatrick 
114473471bf0Spatrick       // Replace the __kmpc_fork_call calls with direct calls to the outlined
114573471bf0Spatrick       // callbacks.
114673471bf0Spatrick       SmallVector<Value *, 8> Args;
114773471bf0Spatrick       for (auto *CI : MergableCIs) {
1148*d415bd75Srobert         Value *Callee = CI->getArgOperand(CallbackCalleeOperand);
1149*d415bd75Srobert         FunctionType *FT = OMPInfoCache.OMPBuilder.ParallelTask;
115073471bf0Spatrick         Args.clear();
115173471bf0Spatrick         Args.push_back(OutlinedFn->getArg(0));
115273471bf0Spatrick         Args.push_back(OutlinedFn->getArg(1));
1153*d415bd75Srobert         for (unsigned U = CallbackFirstArgOperand, E = CI->arg_size(); U < E;
1154*d415bd75Srobert              ++U)
115573471bf0Spatrick           Args.push_back(CI->getArgOperand(U));
115673471bf0Spatrick 
115773471bf0Spatrick         CallInst *NewCI = CallInst::Create(FT, Callee, Args, "", CI);
115873471bf0Spatrick         if (CI->getDebugLoc())
115973471bf0Spatrick           NewCI->setDebugLoc(CI->getDebugLoc());
116073471bf0Spatrick 
116173471bf0Spatrick         // Forward parameter attributes from the callback to the callee.
1162*d415bd75Srobert         for (unsigned U = CallbackFirstArgOperand, E = CI->arg_size(); U < E;
1163*d415bd75Srobert              ++U)
1164*d415bd75Srobert           for (const Attribute &A : CI->getAttributes().getParamAttrs(U))
116573471bf0Spatrick             NewCI->addParamAttr(
116673471bf0Spatrick                 U - (CallbackFirstArgOperand - CallbackCalleeOperand), A);
116773471bf0Spatrick 
116873471bf0Spatrick         // Emit an explicit barrier to replace the implicit fork-join barrier.
116973471bf0Spatrick         if (CI != MergableCIs.back()) {
117073471bf0Spatrick           // TODO: Remove barrier if the merged parallel region includes the
117173471bf0Spatrick           // 'nowait' clause.
117273471bf0Spatrick           OMPInfoCache.OMPBuilder.createBarrier(
117373471bf0Spatrick               InsertPointTy(NewCI->getParent(),
117473471bf0Spatrick                             NewCI->getNextNode()->getIterator()),
117573471bf0Spatrick               OMPD_parallel);
117673471bf0Spatrick         }
117773471bf0Spatrick 
117873471bf0Spatrick         CI->eraseFromParent();
117973471bf0Spatrick       }
118073471bf0Spatrick 
118173471bf0Spatrick       assert(OutlinedFn != OriginalFn && "Outlining failed");
118273471bf0Spatrick       CGUpdater.registerOutlinedFunction(*OriginalFn, *OutlinedFn);
118373471bf0Spatrick       CGUpdater.reanalyzeFunction(*OriginalFn);
118473471bf0Spatrick 
118573471bf0Spatrick       NumOpenMPParallelRegionsMerged += MergableCIs.size();
118673471bf0Spatrick 
118773471bf0Spatrick       return true;
118873471bf0Spatrick     };
118973471bf0Spatrick 
119073471bf0Spatrick     // Helper function that identifes sequences of
119173471bf0Spatrick     // __kmpc_fork_call uses in a basic block.
119273471bf0Spatrick     auto DetectPRsCB = [&](Use &U, Function &F) {
119373471bf0Spatrick       CallInst *CI = getCallIfRegularCall(U, &RFI);
119473471bf0Spatrick       BB2PRMap[CI->getParent()].insert(CI);
119573471bf0Spatrick 
119673471bf0Spatrick       return false;
119773471bf0Spatrick     };
119873471bf0Spatrick 
119973471bf0Spatrick     BB2PRMap.clear();
120073471bf0Spatrick     RFI.foreachUse(SCC, DetectPRsCB);
120173471bf0Spatrick     SmallVector<SmallVector<CallInst *, 4>, 4> MergableCIsVector;
120273471bf0Spatrick     // Find mergable parallel regions within a basic block that are
120373471bf0Spatrick     // safe to merge, that is any in-between instructions can safely
120473471bf0Spatrick     // execute in parallel after merging.
120573471bf0Spatrick     // TODO: support merging across basic-blocks.
120673471bf0Spatrick     for (auto &It : BB2PRMap) {
120773471bf0Spatrick       auto &CIs = It.getSecond();
120873471bf0Spatrick       if (CIs.size() < 2)
120973471bf0Spatrick         continue;
121073471bf0Spatrick 
121173471bf0Spatrick       BasicBlock *BB = It.getFirst();
121273471bf0Spatrick       SmallVector<CallInst *, 4> MergableCIs;
121373471bf0Spatrick 
121473471bf0Spatrick       /// Returns true if the instruction is mergable, false otherwise.
121573471bf0Spatrick       /// A terminator instruction is unmergable by definition since merging
121673471bf0Spatrick       /// works within a BB. Instructions before the mergable region are
121773471bf0Spatrick       /// mergable if they are not calls to OpenMP runtime functions that may
121873471bf0Spatrick       /// set different execution parameters for subsequent parallel regions.
121973471bf0Spatrick       /// Instructions in-between parallel regions are mergable if they are not
122073471bf0Spatrick       /// calls to any non-intrinsic function since that may call a non-mergable
122173471bf0Spatrick       /// OpenMP runtime function.
122273471bf0Spatrick       auto IsMergable = [&](Instruction &I, bool IsBeforeMergableRegion) {
122373471bf0Spatrick         // We do not merge across BBs, hence return false (unmergable) if the
122473471bf0Spatrick         // instruction is a terminator.
122573471bf0Spatrick         if (I.isTerminator())
122673471bf0Spatrick           return false;
122773471bf0Spatrick 
122873471bf0Spatrick         if (!isa<CallInst>(&I))
122973471bf0Spatrick           return true;
123073471bf0Spatrick 
123173471bf0Spatrick         CallInst *CI = cast<CallInst>(&I);
123273471bf0Spatrick         if (IsBeforeMergableRegion) {
123373471bf0Spatrick           Function *CalledFunction = CI->getCalledFunction();
123473471bf0Spatrick           if (!CalledFunction)
123573471bf0Spatrick             return false;
123673471bf0Spatrick           // Return false (unmergable) if the call before the parallel
123773471bf0Spatrick           // region calls an explicit affinity (proc_bind) or number of
123873471bf0Spatrick           // threads (num_threads) compiler-generated function. Those settings
123973471bf0Spatrick           // may be incompatible with following parallel regions.
124073471bf0Spatrick           // TODO: ICV tracking to detect compatibility.
124173471bf0Spatrick           for (const auto &RFI : UnmergableCallsInfo) {
124273471bf0Spatrick             if (CalledFunction == RFI.Declaration)
124373471bf0Spatrick               return false;
124473471bf0Spatrick           }
124573471bf0Spatrick         } else {
124673471bf0Spatrick           // Return false (unmergable) if there is a call instruction
124773471bf0Spatrick           // in-between parallel regions when it is not an intrinsic. It
124873471bf0Spatrick           // may call an unmergable OpenMP runtime function in its callpath.
124973471bf0Spatrick           // TODO: Keep track of possible OpenMP calls in the callpath.
125073471bf0Spatrick           if (!isa<IntrinsicInst>(CI))
125173471bf0Spatrick             return false;
125273471bf0Spatrick         }
125373471bf0Spatrick 
125473471bf0Spatrick         return true;
125573471bf0Spatrick       };
125673471bf0Spatrick       // Find maximal number of parallel region CIs that are safe to merge.
125773471bf0Spatrick       for (auto It = BB->begin(), End = BB->end(); It != End;) {
125873471bf0Spatrick         Instruction &I = *It;
125973471bf0Spatrick         ++It;
126073471bf0Spatrick 
126173471bf0Spatrick         if (CIs.count(&I)) {
126273471bf0Spatrick           MergableCIs.push_back(cast<CallInst>(&I));
126373471bf0Spatrick           continue;
126473471bf0Spatrick         }
126573471bf0Spatrick 
126673471bf0Spatrick         // Continue expanding if the instruction is mergable.
126773471bf0Spatrick         if (IsMergable(I, MergableCIs.empty()))
126873471bf0Spatrick           continue;
126973471bf0Spatrick 
127073471bf0Spatrick         // Forward the instruction iterator to skip the next parallel region
127173471bf0Spatrick         // since there is an unmergable instruction which can affect it.
127273471bf0Spatrick         for (; It != End; ++It) {
127373471bf0Spatrick           Instruction &SkipI = *It;
127473471bf0Spatrick           if (CIs.count(&SkipI)) {
127573471bf0Spatrick             LLVM_DEBUG(dbgs() << TAG << "Skip parallel region " << SkipI
127673471bf0Spatrick                               << " due to " << I << "\n");
127773471bf0Spatrick             ++It;
127873471bf0Spatrick             break;
127973471bf0Spatrick           }
128073471bf0Spatrick         }
128173471bf0Spatrick 
128273471bf0Spatrick         // Store mergable regions found.
128373471bf0Spatrick         if (MergableCIs.size() > 1) {
128473471bf0Spatrick           MergableCIsVector.push_back(MergableCIs);
128573471bf0Spatrick           LLVM_DEBUG(dbgs() << TAG << "Found " << MergableCIs.size()
128673471bf0Spatrick                             << " parallel regions in block " << BB->getName()
128773471bf0Spatrick                             << " of function " << BB->getParent()->getName()
128873471bf0Spatrick                             << "\n";);
128973471bf0Spatrick         }
129073471bf0Spatrick 
129173471bf0Spatrick         MergableCIs.clear();
129273471bf0Spatrick       }
129373471bf0Spatrick 
129473471bf0Spatrick       if (!MergableCIsVector.empty()) {
129573471bf0Spatrick         Changed = true;
129673471bf0Spatrick 
129773471bf0Spatrick         for (auto &MergableCIs : MergableCIsVector)
129873471bf0Spatrick           Merge(MergableCIs, BB);
129973471bf0Spatrick         MergableCIsVector.clear();
130073471bf0Spatrick       }
130173471bf0Spatrick     }
130273471bf0Spatrick 
130373471bf0Spatrick     if (Changed) {
130473471bf0Spatrick       /// Re-collect use for fork calls, emitted barrier calls, and
130573471bf0Spatrick       /// any emitted master/end_master calls.
130673471bf0Spatrick       OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_fork_call);
130773471bf0Spatrick       OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_barrier);
130873471bf0Spatrick       OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_master);
130973471bf0Spatrick       OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_end_master);
131073471bf0Spatrick     }
131173471bf0Spatrick 
131273471bf0Spatrick     return Changed;
131373471bf0Spatrick   }
131473471bf0Spatrick 
1315097a140dSpatrick   /// Try to delete parallel regions if possible.
deleteParallelRegions__anon4b47cb5e0111::OpenMPOpt1316097a140dSpatrick   bool deleteParallelRegions() {
1317097a140dSpatrick     const unsigned CallbackCalleeOperand = 2;
1318097a140dSpatrick 
1319097a140dSpatrick     OMPInformationCache::RuntimeFunctionInfo &RFI =
1320097a140dSpatrick         OMPInfoCache.RFIs[OMPRTL___kmpc_fork_call];
1321097a140dSpatrick 
1322097a140dSpatrick     if (!RFI.Declaration)
1323097a140dSpatrick       return false;
1324097a140dSpatrick 
1325097a140dSpatrick     bool Changed = false;
1326097a140dSpatrick     auto DeleteCallCB = [&](Use &U, Function &) {
1327097a140dSpatrick       CallInst *CI = getCallIfRegularCall(U);
1328097a140dSpatrick       if (!CI)
1329097a140dSpatrick         return false;
1330097a140dSpatrick       auto *Fn = dyn_cast<Function>(
1331097a140dSpatrick           CI->getArgOperand(CallbackCalleeOperand)->stripPointerCasts());
1332097a140dSpatrick       if (!Fn)
1333097a140dSpatrick         return false;
1334097a140dSpatrick       if (!Fn->onlyReadsMemory())
1335097a140dSpatrick         return false;
1336097a140dSpatrick       if (!Fn->hasFnAttribute(Attribute::WillReturn))
1337097a140dSpatrick         return false;
1338097a140dSpatrick 
1339097a140dSpatrick       LLVM_DEBUG(dbgs() << TAG << "Delete read-only parallel region in "
1340097a140dSpatrick                         << CI->getCaller()->getName() << "\n");
1341097a140dSpatrick 
1342097a140dSpatrick       auto Remark = [&](OptimizationRemark OR) {
134373471bf0Spatrick         return OR << "Removing parallel region with no side-effects.";
1344097a140dSpatrick       };
134573471bf0Spatrick       emitRemark<OptimizationRemark>(CI, "OMP160", Remark);
1346097a140dSpatrick 
1347097a140dSpatrick       CGUpdater.removeCallSite(*CI);
1348097a140dSpatrick       CI->eraseFromParent();
1349097a140dSpatrick       Changed = true;
1350097a140dSpatrick       ++NumOpenMPParallelRegionsDeleted;
1351097a140dSpatrick       return true;
1352097a140dSpatrick     };
1353097a140dSpatrick 
1354097a140dSpatrick     RFI.foreachUse(SCC, DeleteCallCB);
1355097a140dSpatrick 
1356097a140dSpatrick     return Changed;
1357097a140dSpatrick   }
1358097a140dSpatrick 
1359097a140dSpatrick   /// Try to eliminate runtime calls by reusing existing ones.
deduplicateRuntimeCalls__anon4b47cb5e0111::OpenMPOpt1360097a140dSpatrick   bool deduplicateRuntimeCalls() {
1361097a140dSpatrick     bool Changed = false;
1362097a140dSpatrick 
1363097a140dSpatrick     RuntimeFunction DeduplicableRuntimeCallIDs[] = {
1364097a140dSpatrick         OMPRTL_omp_get_num_threads,
1365097a140dSpatrick         OMPRTL_omp_in_parallel,
1366097a140dSpatrick         OMPRTL_omp_get_cancellation,
1367097a140dSpatrick         OMPRTL_omp_get_thread_limit,
1368097a140dSpatrick         OMPRTL_omp_get_supported_active_levels,
1369097a140dSpatrick         OMPRTL_omp_get_level,
1370097a140dSpatrick         OMPRTL_omp_get_ancestor_thread_num,
1371097a140dSpatrick         OMPRTL_omp_get_team_size,
1372097a140dSpatrick         OMPRTL_omp_get_active_level,
1373097a140dSpatrick         OMPRTL_omp_in_final,
1374097a140dSpatrick         OMPRTL_omp_get_proc_bind,
1375097a140dSpatrick         OMPRTL_omp_get_num_places,
1376097a140dSpatrick         OMPRTL_omp_get_num_procs,
1377097a140dSpatrick         OMPRTL_omp_get_place_num,
1378097a140dSpatrick         OMPRTL_omp_get_partition_num_places,
1379097a140dSpatrick         OMPRTL_omp_get_partition_place_nums};
1380097a140dSpatrick 
1381097a140dSpatrick     // Global-tid is handled separately.
1382097a140dSpatrick     SmallSetVector<Value *, 16> GTIdArgs;
1383097a140dSpatrick     collectGlobalThreadIdArguments(GTIdArgs);
1384097a140dSpatrick     LLVM_DEBUG(dbgs() << TAG << "Found " << GTIdArgs.size()
1385097a140dSpatrick                       << " global thread ID arguments\n");
1386097a140dSpatrick 
1387097a140dSpatrick     for (Function *F : SCC) {
1388097a140dSpatrick       for (auto DeduplicableRuntimeCallID : DeduplicableRuntimeCallIDs)
138973471bf0Spatrick         Changed |= deduplicateRuntimeCalls(
139073471bf0Spatrick             *F, OMPInfoCache.RFIs[DeduplicableRuntimeCallID]);
1391097a140dSpatrick 
1392097a140dSpatrick       // __kmpc_global_thread_num is special as we can replace it with an
1393097a140dSpatrick       // argument in enough cases to make it worth trying.
1394097a140dSpatrick       Value *GTIdArg = nullptr;
1395097a140dSpatrick       for (Argument &Arg : F->args())
1396097a140dSpatrick         if (GTIdArgs.count(&Arg)) {
1397097a140dSpatrick           GTIdArg = &Arg;
1398097a140dSpatrick           break;
1399097a140dSpatrick         }
1400097a140dSpatrick       Changed |= deduplicateRuntimeCalls(
1401097a140dSpatrick           *F, OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num], GTIdArg);
1402097a140dSpatrick     }
1403097a140dSpatrick 
1404097a140dSpatrick     return Changed;
1405097a140dSpatrick   }
1406097a140dSpatrick 
140773471bf0Spatrick   /// Tries to hide the latency of runtime calls that involve host to
140873471bf0Spatrick   /// device memory transfers by splitting them into their "issue" and "wait"
140973471bf0Spatrick   /// versions. The "issue" is moved upwards as much as possible. The "wait" is
141073471bf0Spatrick   /// moved downards as much as possible. The "issue" issues the memory transfer
141173471bf0Spatrick   /// asynchronously, returning a handle. The "wait" waits in the returned
141273471bf0Spatrick   /// handle for the memory transfer to finish.
hideMemTransfersLatency__anon4b47cb5e0111::OpenMPOpt141373471bf0Spatrick   bool hideMemTransfersLatency() {
141473471bf0Spatrick     auto &RFI = OMPInfoCache.RFIs[OMPRTL___tgt_target_data_begin_mapper];
141573471bf0Spatrick     bool Changed = false;
141673471bf0Spatrick     auto SplitMemTransfers = [&](Use &U, Function &Decl) {
141773471bf0Spatrick       auto *RTCall = getCallIfRegularCall(U, &RFI);
141873471bf0Spatrick       if (!RTCall)
141973471bf0Spatrick         return false;
142073471bf0Spatrick 
142173471bf0Spatrick       OffloadArray OffloadArrays[3];
142273471bf0Spatrick       if (!getValuesInOffloadArrays(*RTCall, OffloadArrays))
142373471bf0Spatrick         return false;
142473471bf0Spatrick 
142573471bf0Spatrick       LLVM_DEBUG(dumpValuesInOffloadArrays(OffloadArrays));
142673471bf0Spatrick 
142773471bf0Spatrick       // TODO: Check if can be moved upwards.
142873471bf0Spatrick       bool WasSplit = false;
142973471bf0Spatrick       Instruction *WaitMovementPoint = canBeMovedDownwards(*RTCall);
143073471bf0Spatrick       if (WaitMovementPoint)
143173471bf0Spatrick         WasSplit = splitTargetDataBeginRTC(*RTCall, *WaitMovementPoint);
143273471bf0Spatrick 
143373471bf0Spatrick       Changed |= WasSplit;
143473471bf0Spatrick       return WasSplit;
143573471bf0Spatrick     };
1436*d415bd75Srobert     if (OMPInfoCache.runtimeFnsAvailable(
1437*d415bd75Srobert             {OMPRTL___tgt_target_data_begin_mapper_issue,
1438*d415bd75Srobert              OMPRTL___tgt_target_data_begin_mapper_wait}))
143973471bf0Spatrick       RFI.foreachUse(SCC, SplitMemTransfers);
144073471bf0Spatrick 
144173471bf0Spatrick     return Changed;
144273471bf0Spatrick   }
144373471bf0Spatrick 
analysisGlobalization__anon4b47cb5e0111::OpenMPOpt144473471bf0Spatrick   void analysisGlobalization() {
144573471bf0Spatrick     auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
144673471bf0Spatrick 
144773471bf0Spatrick     auto CheckGlobalization = [&](Use &U, Function &Decl) {
144873471bf0Spatrick       if (CallInst *CI = getCallIfRegularCall(U, &RFI)) {
144973471bf0Spatrick         auto Remark = [&](OptimizationRemarkMissed ORM) {
145073471bf0Spatrick           return ORM
145173471bf0Spatrick                  << "Found thread data sharing on the GPU. "
145273471bf0Spatrick                  << "Expect degraded performance due to data globalization.";
145373471bf0Spatrick         };
145473471bf0Spatrick         emitRemark<OptimizationRemarkMissed>(CI, "OMP112", Remark);
145573471bf0Spatrick       }
145673471bf0Spatrick 
145773471bf0Spatrick       return false;
145873471bf0Spatrick     };
145973471bf0Spatrick 
146073471bf0Spatrick     RFI.foreachUse(SCC, CheckGlobalization);
146173471bf0Spatrick   }
146273471bf0Spatrick 
146373471bf0Spatrick   /// Maps the values stored in the offload arrays passed as arguments to
146473471bf0Spatrick   /// \p RuntimeCall into the offload arrays in \p OAs.
getValuesInOffloadArrays__anon4b47cb5e0111::OpenMPOpt146573471bf0Spatrick   bool getValuesInOffloadArrays(CallInst &RuntimeCall,
146673471bf0Spatrick                                 MutableArrayRef<OffloadArray> OAs) {
146773471bf0Spatrick     assert(OAs.size() == 3 && "Need space for three offload arrays!");
146873471bf0Spatrick 
146973471bf0Spatrick     // A runtime call that involves memory offloading looks something like:
147073471bf0Spatrick     // call void @__tgt_target_data_begin_mapper(arg0, arg1,
147173471bf0Spatrick     //   i8** %offload_baseptrs, i8** %offload_ptrs, i64* %offload_sizes,
147273471bf0Spatrick     // ...)
147373471bf0Spatrick     // So, the idea is to access the allocas that allocate space for these
147473471bf0Spatrick     // offload arrays, offload_baseptrs, offload_ptrs, offload_sizes.
147573471bf0Spatrick     // Therefore:
147673471bf0Spatrick     // i8** %offload_baseptrs.
147773471bf0Spatrick     Value *BasePtrsArg =
147873471bf0Spatrick         RuntimeCall.getArgOperand(OffloadArray::BasePtrsArgNum);
147973471bf0Spatrick     // i8** %offload_ptrs.
148073471bf0Spatrick     Value *PtrsArg = RuntimeCall.getArgOperand(OffloadArray::PtrsArgNum);
148173471bf0Spatrick     // i8** %offload_sizes.
148273471bf0Spatrick     Value *SizesArg = RuntimeCall.getArgOperand(OffloadArray::SizesArgNum);
148373471bf0Spatrick 
148473471bf0Spatrick     // Get values stored in **offload_baseptrs.
148573471bf0Spatrick     auto *V = getUnderlyingObject(BasePtrsArg);
148673471bf0Spatrick     if (!isa<AllocaInst>(V))
148773471bf0Spatrick       return false;
148873471bf0Spatrick     auto *BasePtrsArray = cast<AllocaInst>(V);
148973471bf0Spatrick     if (!OAs[0].initialize(*BasePtrsArray, RuntimeCall))
149073471bf0Spatrick       return false;
149173471bf0Spatrick 
149273471bf0Spatrick     // Get values stored in **offload_baseptrs.
149373471bf0Spatrick     V = getUnderlyingObject(PtrsArg);
149473471bf0Spatrick     if (!isa<AllocaInst>(V))
149573471bf0Spatrick       return false;
149673471bf0Spatrick     auto *PtrsArray = cast<AllocaInst>(V);
149773471bf0Spatrick     if (!OAs[1].initialize(*PtrsArray, RuntimeCall))
149873471bf0Spatrick       return false;
149973471bf0Spatrick 
150073471bf0Spatrick     // Get values stored in **offload_sizes.
150173471bf0Spatrick     V = getUnderlyingObject(SizesArg);
150273471bf0Spatrick     // If it's a [constant] global array don't analyze it.
150373471bf0Spatrick     if (isa<GlobalValue>(V))
150473471bf0Spatrick       return isa<Constant>(V);
150573471bf0Spatrick     if (!isa<AllocaInst>(V))
150673471bf0Spatrick       return false;
150773471bf0Spatrick 
150873471bf0Spatrick     auto *SizesArray = cast<AllocaInst>(V);
150973471bf0Spatrick     if (!OAs[2].initialize(*SizesArray, RuntimeCall))
151073471bf0Spatrick       return false;
151173471bf0Spatrick 
151273471bf0Spatrick     return true;
151373471bf0Spatrick   }
151473471bf0Spatrick 
151573471bf0Spatrick   /// Prints the values in the OffloadArrays \p OAs using LLVM_DEBUG.
151673471bf0Spatrick   /// For now this is a way to test that the function getValuesInOffloadArrays
151773471bf0Spatrick   /// is working properly.
151873471bf0Spatrick   /// TODO: Move this to a unittest when unittests are available for OpenMPOpt.
dumpValuesInOffloadArrays__anon4b47cb5e0111::OpenMPOpt151973471bf0Spatrick   void dumpValuesInOffloadArrays(ArrayRef<OffloadArray> OAs) {
152073471bf0Spatrick     assert(OAs.size() == 3 && "There are three offload arrays to debug!");
152173471bf0Spatrick 
152273471bf0Spatrick     LLVM_DEBUG(dbgs() << TAG << " Successfully got offload values:\n");
152373471bf0Spatrick     std::string ValuesStr;
152473471bf0Spatrick     raw_string_ostream Printer(ValuesStr);
152573471bf0Spatrick     std::string Separator = " --- ";
152673471bf0Spatrick 
152773471bf0Spatrick     for (auto *BP : OAs[0].StoredValues) {
152873471bf0Spatrick       BP->print(Printer);
152973471bf0Spatrick       Printer << Separator;
153073471bf0Spatrick     }
153173471bf0Spatrick     LLVM_DEBUG(dbgs() << "\t\toffload_baseptrs: " << Printer.str() << "\n");
153273471bf0Spatrick     ValuesStr.clear();
153373471bf0Spatrick 
153473471bf0Spatrick     for (auto *P : OAs[1].StoredValues) {
153573471bf0Spatrick       P->print(Printer);
153673471bf0Spatrick       Printer << Separator;
153773471bf0Spatrick     }
153873471bf0Spatrick     LLVM_DEBUG(dbgs() << "\t\toffload_ptrs: " << Printer.str() << "\n");
153973471bf0Spatrick     ValuesStr.clear();
154073471bf0Spatrick 
154173471bf0Spatrick     for (auto *S : OAs[2].StoredValues) {
154273471bf0Spatrick       S->print(Printer);
154373471bf0Spatrick       Printer << Separator;
154473471bf0Spatrick     }
154573471bf0Spatrick     LLVM_DEBUG(dbgs() << "\t\toffload_sizes: " << Printer.str() << "\n");
154673471bf0Spatrick   }
154773471bf0Spatrick 
154873471bf0Spatrick   /// Returns the instruction where the "wait" counterpart \p RuntimeCall can be
154973471bf0Spatrick   /// moved. Returns nullptr if the movement is not possible, or not worth it.
canBeMovedDownwards__anon4b47cb5e0111::OpenMPOpt155073471bf0Spatrick   Instruction *canBeMovedDownwards(CallInst &RuntimeCall) {
155173471bf0Spatrick     // FIXME: This traverses only the BasicBlock where RuntimeCall is.
155273471bf0Spatrick     //  Make it traverse the CFG.
155373471bf0Spatrick 
155473471bf0Spatrick     Instruction *CurrentI = &RuntimeCall;
155573471bf0Spatrick     bool IsWorthIt = false;
155673471bf0Spatrick     while ((CurrentI = CurrentI->getNextNode())) {
155773471bf0Spatrick 
155873471bf0Spatrick       // TODO: Once we detect the regions to be offloaded we should use the
155973471bf0Spatrick       //  alias analysis manager to check if CurrentI may modify one of
156073471bf0Spatrick       //  the offloaded regions.
156173471bf0Spatrick       if (CurrentI->mayHaveSideEffects() || CurrentI->mayReadFromMemory()) {
156273471bf0Spatrick         if (IsWorthIt)
156373471bf0Spatrick           return CurrentI;
156473471bf0Spatrick 
156573471bf0Spatrick         return nullptr;
156673471bf0Spatrick       }
156773471bf0Spatrick 
156873471bf0Spatrick       // FIXME: For now if we move it over anything without side effect
156973471bf0Spatrick       //  is worth it.
157073471bf0Spatrick       IsWorthIt = true;
157173471bf0Spatrick     }
157273471bf0Spatrick 
157373471bf0Spatrick     // Return end of BasicBlock.
157473471bf0Spatrick     return RuntimeCall.getParent()->getTerminator();
157573471bf0Spatrick   }
157673471bf0Spatrick 
157773471bf0Spatrick   /// Splits \p RuntimeCall into its "issue" and "wait" counterparts.
splitTargetDataBeginRTC__anon4b47cb5e0111::OpenMPOpt157873471bf0Spatrick   bool splitTargetDataBeginRTC(CallInst &RuntimeCall,
157973471bf0Spatrick                                Instruction &WaitMovementPoint) {
158073471bf0Spatrick     // Create stack allocated handle (__tgt_async_info) at the beginning of the
158173471bf0Spatrick     // function. Used for storing information of the async transfer, allowing to
158273471bf0Spatrick     // wait on it later.
158373471bf0Spatrick     auto &IRBuilder = OMPInfoCache.OMPBuilder;
1584*d415bd75Srobert     Function *F = RuntimeCall.getCaller();
1585*d415bd75Srobert     BasicBlock &Entry = F->getEntryBlock();
1586*d415bd75Srobert     IRBuilder.Builder.SetInsertPoint(&Entry,
1587*d415bd75Srobert                                      Entry.getFirstNonPHIOrDbgOrAlloca());
1588*d415bd75Srobert     Value *Handle = IRBuilder.Builder.CreateAlloca(
1589*d415bd75Srobert         IRBuilder.AsyncInfo, /*ArraySize=*/nullptr, "handle");
1590*d415bd75Srobert     Handle =
1591*d415bd75Srobert         IRBuilder.Builder.CreateAddrSpaceCast(Handle, IRBuilder.AsyncInfoPtr);
159273471bf0Spatrick 
159373471bf0Spatrick     // Add "issue" runtime call declaration:
159473471bf0Spatrick     // declare %struct.tgt_async_info @__tgt_target_data_begin_issue(i64, i32,
159573471bf0Spatrick     //   i8**, i8**, i64*, i64*)
159673471bf0Spatrick     FunctionCallee IssueDecl = IRBuilder.getOrCreateRuntimeFunction(
159773471bf0Spatrick         M, OMPRTL___tgt_target_data_begin_mapper_issue);
159873471bf0Spatrick 
159973471bf0Spatrick     // Change RuntimeCall call site for its asynchronous version.
160073471bf0Spatrick     SmallVector<Value *, 16> Args;
160173471bf0Spatrick     for (auto &Arg : RuntimeCall.args())
160273471bf0Spatrick       Args.push_back(Arg.get());
160373471bf0Spatrick     Args.push_back(Handle);
160473471bf0Spatrick 
160573471bf0Spatrick     CallInst *IssueCallsite =
160673471bf0Spatrick         CallInst::Create(IssueDecl, Args, /*NameStr=*/"", &RuntimeCall);
1607*d415bd75Srobert     OMPInfoCache.setCallingConvention(IssueDecl, IssueCallsite);
160873471bf0Spatrick     RuntimeCall.eraseFromParent();
160973471bf0Spatrick 
161073471bf0Spatrick     // Add "wait" runtime call declaration:
161173471bf0Spatrick     // declare void @__tgt_target_data_begin_wait(i64, %struct.__tgt_async_info)
161273471bf0Spatrick     FunctionCallee WaitDecl = IRBuilder.getOrCreateRuntimeFunction(
161373471bf0Spatrick         M, OMPRTL___tgt_target_data_begin_mapper_wait);
161473471bf0Spatrick 
161573471bf0Spatrick     Value *WaitParams[2] = {
161673471bf0Spatrick         IssueCallsite->getArgOperand(
161773471bf0Spatrick             OffloadArray::DeviceIDArgNum), // device_id.
161873471bf0Spatrick         Handle                             // handle to wait on.
161973471bf0Spatrick     };
1620*d415bd75Srobert     CallInst *WaitCallsite = CallInst::Create(
1621*d415bd75Srobert         WaitDecl, WaitParams, /*NameStr=*/"", &WaitMovementPoint);
1622*d415bd75Srobert     OMPInfoCache.setCallingConvention(WaitDecl, WaitCallsite);
162373471bf0Spatrick 
162473471bf0Spatrick     return true;
162573471bf0Spatrick   }
162673471bf0Spatrick 
combinedIdentStruct__anon4b47cb5e0111::OpenMPOpt1627097a140dSpatrick   static Value *combinedIdentStruct(Value *CurrentIdent, Value *NextIdent,
1628097a140dSpatrick                                     bool GlobalOnly, bool &SingleChoice) {
1629097a140dSpatrick     if (CurrentIdent == NextIdent)
1630097a140dSpatrick       return CurrentIdent;
1631097a140dSpatrick 
1632097a140dSpatrick     // TODO: Figure out how to actually combine multiple debug locations. For
1633097a140dSpatrick     //       now we just keep an existing one if there is a single choice.
1634097a140dSpatrick     if (!GlobalOnly || isa<GlobalValue>(NextIdent)) {
1635097a140dSpatrick       SingleChoice = !CurrentIdent;
1636097a140dSpatrick       return NextIdent;
1637097a140dSpatrick     }
1638097a140dSpatrick     return nullptr;
1639097a140dSpatrick   }
1640097a140dSpatrick 
1641097a140dSpatrick   /// Return an `struct ident_t*` value that represents the ones used in the
1642097a140dSpatrick   /// calls of \p RFI inside of \p F. If \p GlobalOnly is true, we will not
1643097a140dSpatrick   /// return a local `struct ident_t*`. For now, if we cannot find a suitable
1644097a140dSpatrick   /// return value we create one from scratch. We also do not yet combine
1645097a140dSpatrick   /// information, e.g., the source locations, see combinedIdentStruct.
1646097a140dSpatrick   Value *
getCombinedIdentFromCallUsesIn__anon4b47cb5e0111::OpenMPOpt1647097a140dSpatrick   getCombinedIdentFromCallUsesIn(OMPInformationCache::RuntimeFunctionInfo &RFI,
1648097a140dSpatrick                                  Function &F, bool GlobalOnly) {
1649097a140dSpatrick     bool SingleChoice = true;
1650097a140dSpatrick     Value *Ident = nullptr;
1651097a140dSpatrick     auto CombineIdentStruct = [&](Use &U, Function &Caller) {
1652097a140dSpatrick       CallInst *CI = getCallIfRegularCall(U, &RFI);
1653097a140dSpatrick       if (!CI || &F != &Caller)
1654097a140dSpatrick         return false;
1655097a140dSpatrick       Ident = combinedIdentStruct(Ident, CI->getArgOperand(0),
1656097a140dSpatrick                                   /* GlobalOnly */ true, SingleChoice);
1657097a140dSpatrick       return false;
1658097a140dSpatrick     };
1659097a140dSpatrick     RFI.foreachUse(SCC, CombineIdentStruct);
1660097a140dSpatrick 
1661097a140dSpatrick     if (!Ident || !SingleChoice) {
1662097a140dSpatrick       // The IRBuilder uses the insertion block to get to the module, this is
1663097a140dSpatrick       // unfortunate but we work around it for now.
1664097a140dSpatrick       if (!OMPInfoCache.OMPBuilder.getInsertionPoint().getBlock())
1665097a140dSpatrick         OMPInfoCache.OMPBuilder.updateToLocation(OpenMPIRBuilder::InsertPointTy(
1666097a140dSpatrick             &F.getEntryBlock(), F.getEntryBlock().begin()));
1667097a140dSpatrick       // Create a fallback location if non was found.
1668097a140dSpatrick       // TODO: Use the debug locations of the calls instead.
1669*d415bd75Srobert       uint32_t SrcLocStrSize;
1670*d415bd75Srobert       Constant *Loc =
1671*d415bd75Srobert           OMPInfoCache.OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1672*d415bd75Srobert       Ident = OMPInfoCache.OMPBuilder.getOrCreateIdent(Loc, SrcLocStrSize);
1673097a140dSpatrick     }
1674097a140dSpatrick     return Ident;
1675097a140dSpatrick   }
1676097a140dSpatrick 
1677097a140dSpatrick   /// Try to eliminate calls of \p RFI in \p F by reusing an existing one or
1678097a140dSpatrick   /// \p ReplVal if given.
deduplicateRuntimeCalls__anon4b47cb5e0111::OpenMPOpt1679097a140dSpatrick   bool deduplicateRuntimeCalls(Function &F,
1680097a140dSpatrick                                OMPInformationCache::RuntimeFunctionInfo &RFI,
1681097a140dSpatrick                                Value *ReplVal = nullptr) {
1682097a140dSpatrick     auto *UV = RFI.getUseVector(F);
1683097a140dSpatrick     if (!UV || UV->size() + (ReplVal != nullptr) < 2)
1684097a140dSpatrick       return false;
1685097a140dSpatrick 
1686097a140dSpatrick     LLVM_DEBUG(
1687097a140dSpatrick         dbgs() << TAG << "Deduplicate " << UV->size() << " uses of " << RFI.Name
1688097a140dSpatrick                << (ReplVal ? " with an existing value\n" : "\n") << "\n");
1689097a140dSpatrick 
1690097a140dSpatrick     assert((!ReplVal || (isa<Argument>(ReplVal) &&
1691097a140dSpatrick                          cast<Argument>(ReplVal)->getParent() == &F)) &&
1692097a140dSpatrick            "Unexpected replacement value!");
1693097a140dSpatrick 
1694097a140dSpatrick     // TODO: Use dominance to find a good position instead.
1695097a140dSpatrick     auto CanBeMoved = [this](CallBase &CB) {
1696*d415bd75Srobert       unsigned NumArgs = CB.arg_size();
1697097a140dSpatrick       if (NumArgs == 0)
1698097a140dSpatrick         return true;
1699097a140dSpatrick       if (CB.getArgOperand(0)->getType() != OMPInfoCache.OMPBuilder.IdentPtr)
1700097a140dSpatrick         return false;
1701*d415bd75Srobert       for (unsigned U = 1; U < NumArgs; ++U)
1702*d415bd75Srobert         if (isa<Instruction>(CB.getArgOperand(U)))
1703097a140dSpatrick           return false;
1704097a140dSpatrick       return true;
1705097a140dSpatrick     };
1706097a140dSpatrick 
1707097a140dSpatrick     if (!ReplVal) {
1708097a140dSpatrick       for (Use *U : *UV)
1709097a140dSpatrick         if (CallInst *CI = getCallIfRegularCall(*U, &RFI)) {
1710097a140dSpatrick           if (!CanBeMoved(*CI))
1711097a140dSpatrick             continue;
1712097a140dSpatrick 
171373471bf0Spatrick           // If the function is a kernel, dedup will move
171473471bf0Spatrick           // the runtime call right after the kernel init callsite. Otherwise,
171573471bf0Spatrick           // it will move it to the beginning of the caller function.
171673471bf0Spatrick           if (isKernel(F)) {
171773471bf0Spatrick             auto &KernelInitRFI = OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
171873471bf0Spatrick             auto *KernelInitUV = KernelInitRFI.getUseVector(F);
1719097a140dSpatrick 
172073471bf0Spatrick             if (KernelInitUV->empty())
172173471bf0Spatrick               continue;
172273471bf0Spatrick 
172373471bf0Spatrick             assert(KernelInitUV->size() == 1 &&
172473471bf0Spatrick                    "Expected a single __kmpc_target_init in kernel\n");
172573471bf0Spatrick 
172673471bf0Spatrick             CallInst *KernelInitCI =
172773471bf0Spatrick                 getCallIfRegularCall(*KernelInitUV->front(), &KernelInitRFI);
172873471bf0Spatrick             assert(KernelInitCI &&
172973471bf0Spatrick                    "Expected a call to __kmpc_target_init in kernel\n");
173073471bf0Spatrick 
173173471bf0Spatrick             CI->moveAfter(KernelInitCI);
173273471bf0Spatrick           } else
1733097a140dSpatrick             CI->moveBefore(&*F.getEntryBlock().getFirstInsertionPt());
1734097a140dSpatrick           ReplVal = CI;
1735097a140dSpatrick           break;
1736097a140dSpatrick         }
1737097a140dSpatrick       if (!ReplVal)
1738097a140dSpatrick         return false;
1739097a140dSpatrick     }
1740097a140dSpatrick 
1741097a140dSpatrick     // If we use a call as a replacement value we need to make sure the ident is
1742097a140dSpatrick     // valid at the new location. For now we just pick a global one, either
1743097a140dSpatrick     // existing and used by one of the calls, or created from scratch.
1744097a140dSpatrick     if (CallBase *CI = dyn_cast<CallBase>(ReplVal)) {
1745*d415bd75Srobert       if (!CI->arg_empty() &&
1746097a140dSpatrick           CI->getArgOperand(0)->getType() == OMPInfoCache.OMPBuilder.IdentPtr) {
1747097a140dSpatrick         Value *Ident = getCombinedIdentFromCallUsesIn(RFI, F,
1748097a140dSpatrick                                                       /* GlobalOnly */ true);
1749097a140dSpatrick         CI->setArgOperand(0, Ident);
1750097a140dSpatrick       }
1751097a140dSpatrick     }
1752097a140dSpatrick 
1753097a140dSpatrick     bool Changed = false;
1754097a140dSpatrick     auto ReplaceAndDeleteCB = [&](Use &U, Function &Caller) {
1755097a140dSpatrick       CallInst *CI = getCallIfRegularCall(U, &RFI);
1756097a140dSpatrick       if (!CI || CI == ReplVal || &F != &Caller)
1757097a140dSpatrick         return false;
1758097a140dSpatrick       assert(CI->getCaller() == &F && "Unexpected call!");
1759097a140dSpatrick 
1760097a140dSpatrick       auto Remark = [&](OptimizationRemark OR) {
1761097a140dSpatrick         return OR << "OpenMP runtime call "
176273471bf0Spatrick                   << ore::NV("OpenMPOptRuntime", RFI.Name) << " deduplicated.";
1763097a140dSpatrick       };
176473471bf0Spatrick       if (CI->getDebugLoc())
176573471bf0Spatrick         emitRemark<OptimizationRemark>(CI, "OMP170", Remark);
176673471bf0Spatrick       else
176773471bf0Spatrick         emitRemark<OptimizationRemark>(&F, "OMP170", Remark);
1768097a140dSpatrick 
1769097a140dSpatrick       CGUpdater.removeCallSite(*CI);
1770097a140dSpatrick       CI->replaceAllUsesWith(ReplVal);
1771097a140dSpatrick       CI->eraseFromParent();
1772097a140dSpatrick       ++NumOpenMPRuntimeCallsDeduplicated;
1773097a140dSpatrick       Changed = true;
1774097a140dSpatrick       return true;
1775097a140dSpatrick     };
1776097a140dSpatrick     RFI.foreachUse(SCC, ReplaceAndDeleteCB);
1777097a140dSpatrick 
1778097a140dSpatrick     return Changed;
1779097a140dSpatrick   }
1780097a140dSpatrick 
1781097a140dSpatrick   /// Collect arguments that represent the global thread id in \p GTIdArgs.
collectGlobalThreadIdArguments__anon4b47cb5e0111::OpenMPOpt1782097a140dSpatrick   void collectGlobalThreadIdArguments(SmallSetVector<Value *, 16> &GTIdArgs) {
1783097a140dSpatrick     // TODO: Below we basically perform a fixpoint iteration with a pessimistic
1784097a140dSpatrick     //       initialization. We could define an AbstractAttribute instead and
1785097a140dSpatrick     //       run the Attributor here once it can be run as an SCC pass.
1786097a140dSpatrick 
1787097a140dSpatrick     // Helper to check the argument \p ArgNo at all call sites of \p F for
1788097a140dSpatrick     // a GTId.
1789097a140dSpatrick     auto CallArgOpIsGTId = [&](Function &F, unsigned ArgNo, CallInst &RefCI) {
1790097a140dSpatrick       if (!F.hasLocalLinkage())
1791097a140dSpatrick         return false;
1792097a140dSpatrick       for (Use &U : F.uses()) {
1793097a140dSpatrick         if (CallInst *CI = getCallIfRegularCall(U)) {
1794097a140dSpatrick           Value *ArgOp = CI->getArgOperand(ArgNo);
1795097a140dSpatrick           if (CI == &RefCI || GTIdArgs.count(ArgOp) ||
1796097a140dSpatrick               getCallIfRegularCall(
1797097a140dSpatrick                   *ArgOp, &OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num]))
1798097a140dSpatrick             continue;
1799097a140dSpatrick         }
1800097a140dSpatrick         return false;
1801097a140dSpatrick       }
1802097a140dSpatrick       return true;
1803097a140dSpatrick     };
1804097a140dSpatrick 
1805097a140dSpatrick     // Helper to identify uses of a GTId as GTId arguments.
1806097a140dSpatrick     auto AddUserArgs = [&](Value &GTId) {
1807097a140dSpatrick       for (Use &U : GTId.uses())
1808097a140dSpatrick         if (CallInst *CI = dyn_cast<CallInst>(U.getUser()))
1809097a140dSpatrick           if (CI->isArgOperand(&U))
1810097a140dSpatrick             if (Function *Callee = CI->getCalledFunction())
1811097a140dSpatrick               if (CallArgOpIsGTId(*Callee, U.getOperandNo(), *CI))
1812097a140dSpatrick                 GTIdArgs.insert(Callee->getArg(U.getOperandNo()));
1813097a140dSpatrick     };
1814097a140dSpatrick 
1815097a140dSpatrick     // The argument users of __kmpc_global_thread_num calls are GTIds.
1816097a140dSpatrick     OMPInformationCache::RuntimeFunctionInfo &GlobThreadNumRFI =
1817097a140dSpatrick         OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num];
1818097a140dSpatrick 
1819097a140dSpatrick     GlobThreadNumRFI.foreachUse(SCC, [&](Use &U, Function &F) {
1820097a140dSpatrick       if (CallInst *CI = getCallIfRegularCall(U, &GlobThreadNumRFI))
1821097a140dSpatrick         AddUserArgs(*CI);
1822097a140dSpatrick       return false;
1823097a140dSpatrick     });
1824097a140dSpatrick 
1825097a140dSpatrick     // Transitively search for more arguments by looking at the users of the
1826097a140dSpatrick     // ones we know already. During the search the GTIdArgs vector is extended
1827097a140dSpatrick     // so we cannot cache the size nor can we use a range based for.
1828*d415bd75Srobert     for (unsigned U = 0; U < GTIdArgs.size(); ++U)
1829*d415bd75Srobert       AddUserArgs(*GTIdArgs[U]);
1830097a140dSpatrick   }
1831097a140dSpatrick 
1832097a140dSpatrick   /// Kernel (=GPU) optimizations and utility functions
1833097a140dSpatrick   ///
1834097a140dSpatrick   ///{{
1835097a140dSpatrick 
1836097a140dSpatrick   /// Check if \p F is a kernel, hence entry point for target offloading.
isKernel__anon4b47cb5e0111::OpenMPOpt1837097a140dSpatrick   bool isKernel(Function &F) { return OMPInfoCache.Kernels.count(&F); }
1838097a140dSpatrick 
1839097a140dSpatrick   /// Cache to remember the unique kernel for a function.
1840*d415bd75Srobert   DenseMap<Function *, std::optional<Kernel>> UniqueKernelMap;
1841097a140dSpatrick 
1842097a140dSpatrick   /// Find the unique kernel that will execute \p F, if any.
1843097a140dSpatrick   Kernel getUniqueKernelFor(Function &F);
1844097a140dSpatrick 
1845097a140dSpatrick   /// Find the unique kernel that will execute \p I, if any.
getUniqueKernelFor__anon4b47cb5e0111::OpenMPOpt1846097a140dSpatrick   Kernel getUniqueKernelFor(Instruction &I) {
1847097a140dSpatrick     return getUniqueKernelFor(*I.getFunction());
1848097a140dSpatrick   }
1849097a140dSpatrick 
1850097a140dSpatrick   /// Rewrite the device (=GPU) code state machine create in non-SPMD mode in
1851097a140dSpatrick   /// the cases we can avoid taking the address of a function.
1852097a140dSpatrick   bool rewriteDeviceCodeStateMachine();
1853097a140dSpatrick 
1854097a140dSpatrick   ///
1855097a140dSpatrick   ///}}
1856097a140dSpatrick 
1857097a140dSpatrick   /// Emit a remark generically
1858097a140dSpatrick   ///
1859097a140dSpatrick   /// This template function can be used to generically emit a remark. The
1860097a140dSpatrick   /// RemarkKind should be one of the following:
1861097a140dSpatrick   ///   - OptimizationRemark to indicate a successful optimization attempt
1862097a140dSpatrick   ///   - OptimizationRemarkMissed to report a failed optimization attempt
1863097a140dSpatrick   ///   - OptimizationRemarkAnalysis to provide additional information about an
1864097a140dSpatrick   ///     optimization attempt
1865097a140dSpatrick   ///
1866097a140dSpatrick   /// The remark is built using a callback function provided by the caller that
1867097a140dSpatrick   /// takes a RemarkKind as input and returns a RemarkKind.
186873471bf0Spatrick   template <typename RemarkKind, typename RemarkCallBack>
emitRemark__anon4b47cb5e0111::OpenMPOpt186973471bf0Spatrick   void emitRemark(Instruction *I, StringRef RemarkName,
1870097a140dSpatrick                   RemarkCallBack &&RemarkCB) const {
187173471bf0Spatrick     Function *F = I->getParent()->getParent();
1872097a140dSpatrick     auto &ORE = OREGetter(F);
1873097a140dSpatrick 
187473471bf0Spatrick     if (RemarkName.startswith("OMP"))
1875097a140dSpatrick       ORE.emit([&]() {
187673471bf0Spatrick         return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, I))
187773471bf0Spatrick                << " [" << RemarkName << "]";
1878097a140dSpatrick       });
187973471bf0Spatrick     else
188073471bf0Spatrick       ORE.emit(
188173471bf0Spatrick           [&]() { return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, I)); });
1882097a140dSpatrick   }
1883097a140dSpatrick 
188473471bf0Spatrick   /// Emit a remark on a function.
188573471bf0Spatrick   template <typename RemarkKind, typename RemarkCallBack>
emitRemark__anon4b47cb5e0111::OpenMPOpt188673471bf0Spatrick   void emitRemark(Function *F, StringRef RemarkName,
188773471bf0Spatrick                   RemarkCallBack &&RemarkCB) const {
188873471bf0Spatrick     auto &ORE = OREGetter(F);
188973471bf0Spatrick 
189073471bf0Spatrick     if (RemarkName.startswith("OMP"))
189173471bf0Spatrick       ORE.emit([&]() {
189273471bf0Spatrick         return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, F))
189373471bf0Spatrick                << " [" << RemarkName << "]";
189473471bf0Spatrick       });
189573471bf0Spatrick     else
189673471bf0Spatrick       ORE.emit(
189773471bf0Spatrick           [&]() { return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, F)); });
189873471bf0Spatrick   }
189973471bf0Spatrick 
1900097a140dSpatrick   /// The underlying module.
1901097a140dSpatrick   Module &M;
1902097a140dSpatrick 
1903097a140dSpatrick   /// The SCC we are operating on.
1904097a140dSpatrick   SmallVectorImpl<Function *> &SCC;
1905097a140dSpatrick 
1906097a140dSpatrick   /// Callback to update the call graph, the first argument is a removed call,
1907097a140dSpatrick   /// the second an optional replacement call.
1908097a140dSpatrick   CallGraphUpdater &CGUpdater;
1909097a140dSpatrick 
1910097a140dSpatrick   /// Callback to get an OptimizationRemarkEmitter from a Function *
1911097a140dSpatrick   OptimizationRemarkGetter OREGetter;
1912097a140dSpatrick 
1913097a140dSpatrick   /// OpenMP-specific information cache. Also Used for Attributor runs.
1914097a140dSpatrick   OMPInformationCache &OMPInfoCache;
1915097a140dSpatrick 
1916097a140dSpatrick   /// Attributor instance.
1917097a140dSpatrick   Attributor &A;
1918097a140dSpatrick 
1919097a140dSpatrick   /// Helper function to run Attributor on SCC.
runAttributor__anon4b47cb5e0111::OpenMPOpt192073471bf0Spatrick   bool runAttributor(bool IsModulePass) {
1921097a140dSpatrick     if (SCC.empty())
1922097a140dSpatrick       return false;
1923097a140dSpatrick 
192473471bf0Spatrick     registerAAs(IsModulePass);
1925097a140dSpatrick 
1926097a140dSpatrick     ChangeStatus Changed = A.run();
1927097a140dSpatrick 
1928097a140dSpatrick     LLVM_DEBUG(dbgs() << "[Attributor] Done with " << SCC.size()
1929097a140dSpatrick                       << " functions, result: " << Changed << ".\n");
1930097a140dSpatrick 
1931097a140dSpatrick     return Changed == ChangeStatus::CHANGED;
1932097a140dSpatrick   }
1933097a140dSpatrick 
193473471bf0Spatrick   void registerFoldRuntimeCall(RuntimeFunction RF);
193573471bf0Spatrick 
1936097a140dSpatrick   /// Populate the Attributor with abstract attribute opportunities in the
1937*d415bd75Srobert   /// functions.
193873471bf0Spatrick   void registerAAs(bool IsModulePass);
1939*d415bd75Srobert 
1940*d415bd75Srobert public:
1941*d415bd75Srobert   /// Callback to register AAs for live functions, including internal functions
1942*d415bd75Srobert   /// marked live during the traversal.
1943*d415bd75Srobert   static void registerAAsForFunction(Attributor &A, const Function &F);
1944097a140dSpatrick };
1945097a140dSpatrick 
getUniqueKernelFor(Function & F)1946097a140dSpatrick Kernel OpenMPOpt::getUniqueKernelFor(Function &F) {
1947*d415bd75Srobert   if (!OMPInfoCache.ModuleSlice.empty() && !OMPInfoCache.ModuleSlice.count(&F))
1948097a140dSpatrick     return nullptr;
1949097a140dSpatrick 
1950097a140dSpatrick   // Use a scope to keep the lifetime of the CachedKernel short.
1951097a140dSpatrick   {
1952*d415bd75Srobert     std::optional<Kernel> &CachedKernel = UniqueKernelMap[&F];
1953097a140dSpatrick     if (CachedKernel)
1954097a140dSpatrick       return *CachedKernel;
1955097a140dSpatrick 
1956097a140dSpatrick     // TODO: We should use an AA to create an (optimistic and callback
1957097a140dSpatrick     //       call-aware) call graph. For now we stick to simple patterns that
1958097a140dSpatrick     //       are less powerful, basically the worst fixpoint.
1959097a140dSpatrick     if (isKernel(F)) {
1960097a140dSpatrick       CachedKernel = Kernel(&F);
1961097a140dSpatrick       return *CachedKernel;
1962097a140dSpatrick     }
1963097a140dSpatrick 
1964097a140dSpatrick     CachedKernel = nullptr;
196573471bf0Spatrick     if (!F.hasLocalLinkage()) {
196673471bf0Spatrick 
196773471bf0Spatrick       // See https://openmp.llvm.org/remarks/OptimizationRemarks.html
196873471bf0Spatrick       auto Remark = [&](OptimizationRemarkAnalysis ORA) {
196973471bf0Spatrick         return ORA << "Potentially unknown OpenMP target region caller.";
197073471bf0Spatrick       };
197173471bf0Spatrick       emitRemark<OptimizationRemarkAnalysis>(&F, "OMP100", Remark);
197273471bf0Spatrick 
1973097a140dSpatrick       return nullptr;
1974097a140dSpatrick     }
197573471bf0Spatrick   }
1976097a140dSpatrick 
1977097a140dSpatrick   auto GetUniqueKernelForUse = [&](const Use &U) -> Kernel {
1978097a140dSpatrick     if (auto *Cmp = dyn_cast<ICmpInst>(U.getUser())) {
1979097a140dSpatrick       // Allow use in equality comparisons.
1980097a140dSpatrick       if (Cmp->isEquality())
1981097a140dSpatrick         return getUniqueKernelFor(*Cmp);
1982097a140dSpatrick       return nullptr;
1983097a140dSpatrick     }
1984097a140dSpatrick     if (auto *CB = dyn_cast<CallBase>(U.getUser())) {
1985097a140dSpatrick       // Allow direct calls.
1986097a140dSpatrick       if (CB->isCallee(&U))
1987097a140dSpatrick         return getUniqueKernelFor(*CB);
198873471bf0Spatrick 
198973471bf0Spatrick       OMPInformationCache::RuntimeFunctionInfo &KernelParallelRFI =
199073471bf0Spatrick           OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_51];
199173471bf0Spatrick       // Allow the use in __kmpc_parallel_51 calls.
199273471bf0Spatrick       if (OpenMPOpt::getCallIfRegularCall(*U.getUser(), &KernelParallelRFI))
1993097a140dSpatrick         return getUniqueKernelFor(*CB);
1994097a140dSpatrick       return nullptr;
1995097a140dSpatrick     }
1996097a140dSpatrick     // Disallow every other use.
1997097a140dSpatrick     return nullptr;
1998097a140dSpatrick   };
1999097a140dSpatrick 
2000097a140dSpatrick   // TODO: In the future we want to track more than just a unique kernel.
2001097a140dSpatrick   SmallPtrSet<Kernel, 2> PotentialKernels;
200273471bf0Spatrick   OMPInformationCache::foreachUse(F, [&](const Use &U) {
2003097a140dSpatrick     PotentialKernels.insert(GetUniqueKernelForUse(U));
2004097a140dSpatrick   });
2005097a140dSpatrick 
2006097a140dSpatrick   Kernel K = nullptr;
2007097a140dSpatrick   if (PotentialKernels.size() == 1)
2008097a140dSpatrick     K = *PotentialKernels.begin();
2009097a140dSpatrick 
2010097a140dSpatrick   // Cache the result.
2011097a140dSpatrick   UniqueKernelMap[&F] = K;
2012097a140dSpatrick 
2013097a140dSpatrick   return K;
2014097a140dSpatrick }
2015097a140dSpatrick 
rewriteDeviceCodeStateMachine()2016097a140dSpatrick bool OpenMPOpt::rewriteDeviceCodeStateMachine() {
201773471bf0Spatrick   OMPInformationCache::RuntimeFunctionInfo &KernelParallelRFI =
201873471bf0Spatrick       OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_51];
2019097a140dSpatrick 
2020097a140dSpatrick   bool Changed = false;
202173471bf0Spatrick   if (!KernelParallelRFI)
2022097a140dSpatrick     return Changed;
2023097a140dSpatrick 
2024*d415bd75Srobert   // If we have disabled state machine changes, exit
2025*d415bd75Srobert   if (DisableOpenMPOptStateMachineRewrite)
2026*d415bd75Srobert     return Changed;
2027*d415bd75Srobert 
2028097a140dSpatrick   for (Function *F : SCC) {
2029097a140dSpatrick 
203073471bf0Spatrick     // Check if the function is a use in a __kmpc_parallel_51 call at
2031097a140dSpatrick     // all.
2032097a140dSpatrick     bool UnknownUse = false;
203373471bf0Spatrick     bool KernelParallelUse = false;
2034097a140dSpatrick     unsigned NumDirectCalls = 0;
2035097a140dSpatrick 
2036097a140dSpatrick     SmallVector<Use *, 2> ToBeReplacedStateMachineUses;
203773471bf0Spatrick     OMPInformationCache::foreachUse(*F, [&](Use &U) {
2038097a140dSpatrick       if (auto *CB = dyn_cast<CallBase>(U.getUser()))
2039097a140dSpatrick         if (CB->isCallee(&U)) {
2040097a140dSpatrick           ++NumDirectCalls;
2041097a140dSpatrick           return;
2042097a140dSpatrick         }
2043097a140dSpatrick 
2044097a140dSpatrick       if (isa<ICmpInst>(U.getUser())) {
2045097a140dSpatrick         ToBeReplacedStateMachineUses.push_back(&U);
2046097a140dSpatrick         return;
2047097a140dSpatrick       }
204873471bf0Spatrick 
204973471bf0Spatrick       // Find wrapper functions that represent parallel kernels.
205073471bf0Spatrick       CallInst *CI =
205173471bf0Spatrick           OpenMPOpt::getCallIfRegularCall(*U.getUser(), &KernelParallelRFI);
205273471bf0Spatrick       const unsigned int WrapperFunctionArgNo = 6;
205373471bf0Spatrick       if (!KernelParallelUse && CI &&
205473471bf0Spatrick           CI->getArgOperandNo(&U) == WrapperFunctionArgNo) {
205573471bf0Spatrick         KernelParallelUse = true;
2056097a140dSpatrick         ToBeReplacedStateMachineUses.push_back(&U);
2057097a140dSpatrick         return;
2058097a140dSpatrick       }
2059097a140dSpatrick       UnknownUse = true;
2060097a140dSpatrick     });
2061097a140dSpatrick 
206273471bf0Spatrick     // Do not emit a remark if we haven't seen a __kmpc_parallel_51
2063097a140dSpatrick     // use.
206473471bf0Spatrick     if (!KernelParallelUse)
2065097a140dSpatrick       continue;
2066097a140dSpatrick 
2067097a140dSpatrick     // If this ever hits, we should investigate.
2068097a140dSpatrick     // TODO: Checking the number of uses is not a necessary restriction and
2069097a140dSpatrick     // should be lifted.
2070097a140dSpatrick     if (UnknownUse || NumDirectCalls != 1 ||
207173471bf0Spatrick         ToBeReplacedStateMachineUses.size() > 2) {
207273471bf0Spatrick       auto Remark = [&](OptimizationRemarkAnalysis ORA) {
207373471bf0Spatrick         return ORA << "Parallel region is used in "
2074097a140dSpatrick                    << (UnknownUse ? "unknown" : "unexpected")
207573471bf0Spatrick                    << " ways. Will not attempt to rewrite the state machine.";
2076097a140dSpatrick       };
207773471bf0Spatrick       emitRemark<OptimizationRemarkAnalysis>(F, "OMP101", Remark);
2078097a140dSpatrick       continue;
2079097a140dSpatrick     }
2080097a140dSpatrick 
208173471bf0Spatrick     // Even if we have __kmpc_parallel_51 calls, we (for now) give
2082097a140dSpatrick     // up if the function is not called from a unique kernel.
2083097a140dSpatrick     Kernel K = getUniqueKernelFor(*F);
2084097a140dSpatrick     if (!K) {
208573471bf0Spatrick       auto Remark = [&](OptimizationRemarkAnalysis ORA) {
208673471bf0Spatrick         return ORA << "Parallel region is not called from a unique kernel. "
208773471bf0Spatrick                       "Will not attempt to rewrite the state machine.";
2088097a140dSpatrick       };
208973471bf0Spatrick       emitRemark<OptimizationRemarkAnalysis>(F, "OMP102", Remark);
2090097a140dSpatrick       continue;
2091097a140dSpatrick     }
2092097a140dSpatrick 
2093097a140dSpatrick     // We now know F is a parallel body function called only from the kernel K.
2094097a140dSpatrick     // We also identified the state machine uses in which we replace the
2095097a140dSpatrick     // function pointer by a new global symbol for identification purposes. This
2096097a140dSpatrick     // ensures only direct calls to the function are left.
2097097a140dSpatrick 
2098097a140dSpatrick     Module &M = *F->getParent();
2099097a140dSpatrick     Type *Int8Ty = Type::getInt8Ty(M.getContext());
2100097a140dSpatrick 
2101097a140dSpatrick     auto *ID = new GlobalVariable(
2102097a140dSpatrick         M, Int8Ty, /* isConstant */ true, GlobalValue::PrivateLinkage,
2103097a140dSpatrick         UndefValue::get(Int8Ty), F->getName() + ".ID");
2104097a140dSpatrick 
2105097a140dSpatrick     for (Use *U : ToBeReplacedStateMachineUses)
210673471bf0Spatrick       U->set(ConstantExpr::getPointerBitCastOrAddrSpaceCast(
210773471bf0Spatrick           ID, U->get()->getType()));
2108097a140dSpatrick 
2109097a140dSpatrick     ++NumOpenMPParallelRegionsReplacedInGPUStateMachine;
2110097a140dSpatrick 
2111097a140dSpatrick     Changed = true;
2112097a140dSpatrick   }
2113097a140dSpatrick 
2114097a140dSpatrick   return Changed;
2115097a140dSpatrick }
2116097a140dSpatrick 
2117097a140dSpatrick /// Abstract Attribute for tracking ICV values.
2118097a140dSpatrick struct AAICVTracker : public StateWrapper<BooleanState, AbstractAttribute> {
2119097a140dSpatrick   using Base = StateWrapper<BooleanState, AbstractAttribute>;
AAICVTracker__anon4b47cb5e0111::AAICVTracker2120097a140dSpatrick   AAICVTracker(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
2121097a140dSpatrick 
initialize__anon4b47cb5e0111::AAICVTracker212273471bf0Spatrick   void initialize(Attributor &A) override {
212373471bf0Spatrick     Function *F = getAnchorScope();
212473471bf0Spatrick     if (!F || !A.isFunctionIPOAmendable(*F))
212573471bf0Spatrick       indicatePessimisticFixpoint();
212673471bf0Spatrick   }
212773471bf0Spatrick 
2128097a140dSpatrick   /// Returns true if value is assumed to be tracked.
isAssumedTracked__anon4b47cb5e0111::AAICVTracker2129097a140dSpatrick   bool isAssumedTracked() const { return getAssumed(); }
2130097a140dSpatrick 
2131097a140dSpatrick   /// Returns true if value is known to be tracked.
isKnownTracked__anon4b47cb5e0111::AAICVTracker2132097a140dSpatrick   bool isKnownTracked() const { return getAssumed(); }
2133097a140dSpatrick 
2134097a140dSpatrick   /// Create an abstract attribute biew for the position \p IRP.
2135097a140dSpatrick   static AAICVTracker &createForPosition(const IRPosition &IRP, Attributor &A);
2136097a140dSpatrick 
2137097a140dSpatrick   /// Return the value with which \p I can be replaced for specific \p ICV.
getReplacementValue__anon4b47cb5e0111::AAICVTracker2138*d415bd75Srobert   virtual std::optional<Value *> getReplacementValue(InternalControlVar ICV,
213973471bf0Spatrick                                                      const Instruction *I,
214073471bf0Spatrick                                                      Attributor &A) const {
2141*d415bd75Srobert     return std::nullopt;
214273471bf0Spatrick   }
214373471bf0Spatrick 
214473471bf0Spatrick   /// Return an assumed unique ICV value if a single candidate is found. If
2145*d415bd75Srobert   /// there cannot be one, return a nullptr. If it is not clear yet, return
2146*d415bd75Srobert   /// std::nullopt.
2147*d415bd75Srobert   virtual std::optional<Value *>
214873471bf0Spatrick   getUniqueReplacementValue(InternalControlVar ICV) const = 0;
214973471bf0Spatrick 
215073471bf0Spatrick   // Currently only nthreads is being tracked.
215173471bf0Spatrick   // this array will only grow with time.
215273471bf0Spatrick   InternalControlVar TrackableICVs[1] = {ICV_nthreads};
2153097a140dSpatrick 
2154097a140dSpatrick   /// See AbstractAttribute::getName()
getName__anon4b47cb5e0111::AAICVTracker2155097a140dSpatrick   const std::string getName() const override { return "AAICVTracker"; }
2156097a140dSpatrick 
2157097a140dSpatrick   /// See AbstractAttribute::getIdAddr()
getIdAddr__anon4b47cb5e0111::AAICVTracker2158097a140dSpatrick   const char *getIdAddr() const override { return &ID; }
2159097a140dSpatrick 
2160097a140dSpatrick   /// This function should return true if the type of the \p AA is AAICVTracker
classof__anon4b47cb5e0111::AAICVTracker2161097a140dSpatrick   static bool classof(const AbstractAttribute *AA) {
2162097a140dSpatrick     return (AA->getIdAddr() == &ID);
2163097a140dSpatrick   }
2164097a140dSpatrick 
2165097a140dSpatrick   static const char ID;
2166097a140dSpatrick };
2167097a140dSpatrick 
2168097a140dSpatrick struct AAICVTrackerFunction : public AAICVTracker {
AAICVTrackerFunction__anon4b47cb5e0111::AAICVTrackerFunction2169097a140dSpatrick   AAICVTrackerFunction(const IRPosition &IRP, Attributor &A)
2170097a140dSpatrick       : AAICVTracker(IRP, A) {}
2171097a140dSpatrick 
2172097a140dSpatrick   // FIXME: come up with better string.
getAsStr__anon4b47cb5e0111::AAICVTrackerFunction217373471bf0Spatrick   const std::string getAsStr() const override { return "ICVTrackerFunction"; }
2174097a140dSpatrick 
2175097a140dSpatrick   // FIXME: come up with some stats.
trackStatistics__anon4b47cb5e0111::AAICVTrackerFunction2176097a140dSpatrick   void trackStatistics() const override {}
2177097a140dSpatrick 
217873471bf0Spatrick   /// We don't manifest anything for this AA.
manifest__anon4b47cb5e0111::AAICVTrackerFunction2179097a140dSpatrick   ChangeStatus manifest(Attributor &A) override {
218073471bf0Spatrick     return ChangeStatus::UNCHANGED;
2181097a140dSpatrick   }
2182097a140dSpatrick 
2183097a140dSpatrick   // Map of ICV to their values at specific program point.
218473471bf0Spatrick   EnumeratedArray<DenseMap<Instruction *, Value *>, InternalControlVar,
2185097a140dSpatrick                   InternalControlVar::ICV___last>
218673471bf0Spatrick       ICVReplacementValuesMap;
2187097a140dSpatrick 
updateImpl__anon4b47cb5e0111::AAICVTrackerFunction2188097a140dSpatrick   ChangeStatus updateImpl(Attributor &A) override {
2189097a140dSpatrick     ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
2190097a140dSpatrick 
2191097a140dSpatrick     Function *F = getAnchorScope();
2192097a140dSpatrick 
2193097a140dSpatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2194097a140dSpatrick 
2195097a140dSpatrick     for (InternalControlVar ICV : TrackableICVs) {
2196097a140dSpatrick       auto &SetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Setter];
2197097a140dSpatrick 
219873471bf0Spatrick       auto &ValuesMap = ICVReplacementValuesMap[ICV];
2199097a140dSpatrick       auto TrackValues = [&](Use &U, Function &) {
2200097a140dSpatrick         CallInst *CI = OpenMPOpt::getCallIfRegularCall(U);
2201097a140dSpatrick         if (!CI)
2202097a140dSpatrick           return false;
2203097a140dSpatrick 
2204097a140dSpatrick         // FIXME: handle setters with more that 1 arguments.
2205097a140dSpatrick         /// Track new value.
220673471bf0Spatrick         if (ValuesMap.insert(std::make_pair(CI, CI->getArgOperand(0))).second)
2207097a140dSpatrick           HasChanged = ChangeStatus::CHANGED;
2208097a140dSpatrick 
2209097a140dSpatrick         return false;
2210097a140dSpatrick       };
2211097a140dSpatrick 
221273471bf0Spatrick       auto CallCheck = [&](Instruction &I) {
2213*d415bd75Srobert         std::optional<Value *> ReplVal = getValueForCall(A, I, ICV);
2214*d415bd75Srobert         if (ReplVal && ValuesMap.insert(std::make_pair(&I, *ReplVal)).second)
221573471bf0Spatrick           HasChanged = ChangeStatus::CHANGED;
221673471bf0Spatrick 
221773471bf0Spatrick         return true;
221873471bf0Spatrick       };
221973471bf0Spatrick 
222073471bf0Spatrick       // Track all changes of an ICV.
2221097a140dSpatrick       SetterRFI.foreachUse(TrackValues, F);
222273471bf0Spatrick 
222373471bf0Spatrick       bool UsedAssumedInformation = false;
222473471bf0Spatrick       A.checkForAllInstructions(CallCheck, *this, {Instruction::Call},
222573471bf0Spatrick                                 UsedAssumedInformation,
222673471bf0Spatrick                                 /* CheckBBLivenessOnly */ true);
222773471bf0Spatrick 
222873471bf0Spatrick       /// TODO: Figure out a way to avoid adding entry in
222973471bf0Spatrick       /// ICVReplacementValuesMap
223073471bf0Spatrick       Instruction *Entry = &F->getEntryBlock().front();
223173471bf0Spatrick       if (HasChanged == ChangeStatus::CHANGED && !ValuesMap.count(Entry))
223273471bf0Spatrick         ValuesMap.insert(std::make_pair(Entry, nullptr));
2233097a140dSpatrick     }
2234097a140dSpatrick 
2235097a140dSpatrick     return HasChanged;
2236097a140dSpatrick   }
2237097a140dSpatrick 
2238*d415bd75Srobert   /// Helper to check if \p I is a call and get the value for it if it is
223973471bf0Spatrick   /// unique.
getValueForCall__anon4b47cb5e0111::AAICVTrackerFunction2240*d415bd75Srobert   std::optional<Value *> getValueForCall(Attributor &A, const Instruction &I,
224173471bf0Spatrick                                          InternalControlVar &ICV) const {
2242097a140dSpatrick 
2243*d415bd75Srobert     const auto *CB = dyn_cast<CallBase>(&I);
224473471bf0Spatrick     if (!CB || CB->hasFnAttr("no_openmp") ||
224573471bf0Spatrick         CB->hasFnAttr("no_openmp_routines"))
2246*d415bd75Srobert       return std::nullopt;
224773471bf0Spatrick 
2248097a140dSpatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2249097a140dSpatrick     auto &GetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Getter];
225073471bf0Spatrick     auto &SetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Setter];
225173471bf0Spatrick     Function *CalledFunction = CB->getCalledFunction();
2252097a140dSpatrick 
225373471bf0Spatrick     // Indirect call, assume ICV changes.
225473471bf0Spatrick     if (CalledFunction == nullptr)
225573471bf0Spatrick       return nullptr;
225673471bf0Spatrick     if (CalledFunction == GetterRFI.Declaration)
2257*d415bd75Srobert       return std::nullopt;
225873471bf0Spatrick     if (CalledFunction == SetterRFI.Declaration) {
2259*d415bd75Srobert       if (ICVReplacementValuesMap[ICV].count(&I))
2260*d415bd75Srobert         return ICVReplacementValuesMap[ICV].lookup(&I);
226173471bf0Spatrick 
226273471bf0Spatrick       return nullptr;
226373471bf0Spatrick     }
226473471bf0Spatrick 
226573471bf0Spatrick     // Since we don't know, assume it changes the ICV.
226673471bf0Spatrick     if (CalledFunction->isDeclaration())
226773471bf0Spatrick       return nullptr;
226873471bf0Spatrick 
226973471bf0Spatrick     const auto &ICVTrackingAA = A.getAAFor<AAICVTracker>(
227073471bf0Spatrick         *this, IRPosition::callsite_returned(*CB), DepClassTy::REQUIRED);
227173471bf0Spatrick 
2272*d415bd75Srobert     if (ICVTrackingAA.isAssumedTracked()) {
2273*d415bd75Srobert       std::optional<Value *> URV = ICVTrackingAA.getUniqueReplacementValue(ICV);
2274*d415bd75Srobert       if (!URV || (*URV && AA::isValidAtPosition(AA::ValueAndContext(**URV, I),
2275*d415bd75Srobert                                                  OMPInfoCache)))
2276*d415bd75Srobert         return URV;
2277*d415bd75Srobert     }
227873471bf0Spatrick 
227973471bf0Spatrick     // If we don't know, assume it changes.
228073471bf0Spatrick     return nullptr;
228173471bf0Spatrick   }
228273471bf0Spatrick 
2283*d415bd75Srobert   // We don't check unique value for a function, so return std::nullopt.
2284*d415bd75Srobert   std::optional<Value *>
getUniqueReplacementValue__anon4b47cb5e0111::AAICVTrackerFunction228573471bf0Spatrick   getUniqueReplacementValue(InternalControlVar ICV) const override {
2286*d415bd75Srobert     return std::nullopt;
228773471bf0Spatrick   }
228873471bf0Spatrick 
228973471bf0Spatrick   /// Return the value with which \p I can be replaced for specific \p ICV.
getReplacementValue__anon4b47cb5e0111::AAICVTrackerFunction2290*d415bd75Srobert   std::optional<Value *> getReplacementValue(InternalControlVar ICV,
229173471bf0Spatrick                                              const Instruction *I,
229273471bf0Spatrick                                              Attributor &A) const override {
229373471bf0Spatrick     const auto &ValuesMap = ICVReplacementValuesMap[ICV];
229473471bf0Spatrick     if (ValuesMap.count(I))
229573471bf0Spatrick       return ValuesMap.lookup(I);
229673471bf0Spatrick 
229773471bf0Spatrick     SmallVector<const Instruction *, 16> Worklist;
229873471bf0Spatrick     SmallPtrSet<const Instruction *, 16> Visited;
229973471bf0Spatrick     Worklist.push_back(I);
230073471bf0Spatrick 
2301*d415bd75Srobert     std::optional<Value *> ReplVal;
230273471bf0Spatrick 
230373471bf0Spatrick     while (!Worklist.empty()) {
230473471bf0Spatrick       const Instruction *CurrInst = Worklist.pop_back_val();
230573471bf0Spatrick       if (!Visited.insert(CurrInst).second)
2306097a140dSpatrick         continue;
2307097a140dSpatrick 
230873471bf0Spatrick       const BasicBlock *CurrBB = CurrInst->getParent();
230973471bf0Spatrick 
231073471bf0Spatrick       // Go up and look for all potential setters/calls that might change the
231173471bf0Spatrick       // ICV.
231273471bf0Spatrick       while ((CurrInst = CurrInst->getPrevNode())) {
231373471bf0Spatrick         if (ValuesMap.count(CurrInst)) {
2314*d415bd75Srobert           std::optional<Value *> NewReplVal = ValuesMap.lookup(CurrInst);
231573471bf0Spatrick           // Unknown value, track new.
2316*d415bd75Srobert           if (!ReplVal) {
231773471bf0Spatrick             ReplVal = NewReplVal;
231873471bf0Spatrick             break;
231973471bf0Spatrick           }
232073471bf0Spatrick 
232173471bf0Spatrick           // If we found a new value, we can't know the icv value anymore.
2322*d415bd75Srobert           if (NewReplVal)
232373471bf0Spatrick             if (ReplVal != NewReplVal)
2324097a140dSpatrick               return nullptr;
2325097a140dSpatrick 
232673471bf0Spatrick           break;
2327097a140dSpatrick         }
2328097a140dSpatrick 
2329*d415bd75Srobert         std::optional<Value *> NewReplVal = getValueForCall(A, *CurrInst, ICV);
2330*d415bd75Srobert         if (!NewReplVal)
233173471bf0Spatrick           continue;
233273471bf0Spatrick 
233373471bf0Spatrick         // Unknown value, track new.
2334*d415bd75Srobert         if (!ReplVal) {
233573471bf0Spatrick           ReplVal = NewReplVal;
233673471bf0Spatrick           break;
2337097a140dSpatrick         }
2338097a140dSpatrick 
233973471bf0Spatrick         // if (NewReplVal.hasValue())
234073471bf0Spatrick         // We found a new value, we can't know the icv value anymore.
234173471bf0Spatrick         if (ReplVal != NewReplVal)
2342097a140dSpatrick           return nullptr;
2343097a140dSpatrick       }
234473471bf0Spatrick 
234573471bf0Spatrick       // If we are in the same BB and we have a value, we are done.
2346*d415bd75Srobert       if (CurrBB == I->getParent() && ReplVal)
234773471bf0Spatrick         return ReplVal;
234873471bf0Spatrick 
234973471bf0Spatrick       // Go through all predecessors and add terminators for analysis.
235073471bf0Spatrick       for (const BasicBlock *Pred : predecessors(CurrBB))
235173471bf0Spatrick         if (const Instruction *Terminator = Pred->getTerminator())
235273471bf0Spatrick           Worklist.push_back(Terminator);
235373471bf0Spatrick     }
235473471bf0Spatrick 
235573471bf0Spatrick     return ReplVal;
235673471bf0Spatrick   }
2357097a140dSpatrick };
235873471bf0Spatrick 
235973471bf0Spatrick struct AAICVTrackerFunctionReturned : AAICVTracker {
AAICVTrackerFunctionReturned__anon4b47cb5e0111::AAICVTrackerFunctionReturned236073471bf0Spatrick   AAICVTrackerFunctionReturned(const IRPosition &IRP, Attributor &A)
236173471bf0Spatrick       : AAICVTracker(IRP, A) {}
236273471bf0Spatrick 
236373471bf0Spatrick   // FIXME: come up with better string.
getAsStr__anon4b47cb5e0111::AAICVTrackerFunctionReturned236473471bf0Spatrick   const std::string getAsStr() const override {
236573471bf0Spatrick     return "ICVTrackerFunctionReturned";
236673471bf0Spatrick   }
236773471bf0Spatrick 
236873471bf0Spatrick   // FIXME: come up with some stats.
trackStatistics__anon4b47cb5e0111::AAICVTrackerFunctionReturned236973471bf0Spatrick   void trackStatistics() const override {}
237073471bf0Spatrick 
237173471bf0Spatrick   /// We don't manifest anything for this AA.
manifest__anon4b47cb5e0111::AAICVTrackerFunctionReturned237273471bf0Spatrick   ChangeStatus manifest(Attributor &A) override {
237373471bf0Spatrick     return ChangeStatus::UNCHANGED;
237473471bf0Spatrick   }
237573471bf0Spatrick 
237673471bf0Spatrick   // Map of ICV to their values at specific program point.
2377*d415bd75Srobert   EnumeratedArray<std::optional<Value *>, InternalControlVar,
237873471bf0Spatrick                   InternalControlVar::ICV___last>
237973471bf0Spatrick       ICVReplacementValuesMap;
238073471bf0Spatrick 
238173471bf0Spatrick   /// Return the value with which \p I can be replaced for specific \p ICV.
2382*d415bd75Srobert   std::optional<Value *>
getUniqueReplacementValue__anon4b47cb5e0111::AAICVTrackerFunctionReturned238373471bf0Spatrick   getUniqueReplacementValue(InternalControlVar ICV) const override {
238473471bf0Spatrick     return ICVReplacementValuesMap[ICV];
238573471bf0Spatrick   }
238673471bf0Spatrick 
updateImpl__anon4b47cb5e0111::AAICVTrackerFunctionReturned238773471bf0Spatrick   ChangeStatus updateImpl(Attributor &A) override {
238873471bf0Spatrick     ChangeStatus Changed = ChangeStatus::UNCHANGED;
238973471bf0Spatrick     const auto &ICVTrackingAA = A.getAAFor<AAICVTracker>(
239073471bf0Spatrick         *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED);
239173471bf0Spatrick 
239273471bf0Spatrick     if (!ICVTrackingAA.isAssumedTracked())
239373471bf0Spatrick       return indicatePessimisticFixpoint();
239473471bf0Spatrick 
239573471bf0Spatrick     for (InternalControlVar ICV : TrackableICVs) {
2396*d415bd75Srobert       std::optional<Value *> &ReplVal = ICVReplacementValuesMap[ICV];
2397*d415bd75Srobert       std::optional<Value *> UniqueICVValue;
239873471bf0Spatrick 
239973471bf0Spatrick       auto CheckReturnInst = [&](Instruction &I) {
2400*d415bd75Srobert         std::optional<Value *> NewReplVal =
240173471bf0Spatrick             ICVTrackingAA.getReplacementValue(ICV, &I, A);
240273471bf0Spatrick 
240373471bf0Spatrick         // If we found a second ICV value there is no unique returned value.
2404*d415bd75Srobert         if (UniqueICVValue && UniqueICVValue != NewReplVal)
240573471bf0Spatrick           return false;
240673471bf0Spatrick 
240773471bf0Spatrick         UniqueICVValue = NewReplVal;
240873471bf0Spatrick 
240973471bf0Spatrick         return true;
241073471bf0Spatrick       };
241173471bf0Spatrick 
241273471bf0Spatrick       bool UsedAssumedInformation = false;
241373471bf0Spatrick       if (!A.checkForAllInstructions(CheckReturnInst, *this, {Instruction::Ret},
241473471bf0Spatrick                                      UsedAssumedInformation,
241573471bf0Spatrick                                      /* CheckBBLivenessOnly */ true))
241673471bf0Spatrick         UniqueICVValue = nullptr;
241773471bf0Spatrick 
241873471bf0Spatrick       if (UniqueICVValue == ReplVal)
241973471bf0Spatrick         continue;
242073471bf0Spatrick 
242173471bf0Spatrick       ReplVal = UniqueICVValue;
242273471bf0Spatrick       Changed = ChangeStatus::CHANGED;
242373471bf0Spatrick     }
242473471bf0Spatrick 
242573471bf0Spatrick     return Changed;
242673471bf0Spatrick   }
242773471bf0Spatrick };
242873471bf0Spatrick 
242973471bf0Spatrick struct AAICVTrackerCallSite : AAICVTracker {
AAICVTrackerCallSite__anon4b47cb5e0111::AAICVTrackerCallSite243073471bf0Spatrick   AAICVTrackerCallSite(const IRPosition &IRP, Attributor &A)
243173471bf0Spatrick       : AAICVTracker(IRP, A) {}
243273471bf0Spatrick 
initialize__anon4b47cb5e0111::AAICVTrackerCallSite243373471bf0Spatrick   void initialize(Attributor &A) override {
243473471bf0Spatrick     Function *F = getAnchorScope();
243573471bf0Spatrick     if (!F || !A.isFunctionIPOAmendable(*F))
243673471bf0Spatrick       indicatePessimisticFixpoint();
243773471bf0Spatrick 
243873471bf0Spatrick     // We only initialize this AA for getters, so we need to know which ICV it
243973471bf0Spatrick     // gets.
244073471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
244173471bf0Spatrick     for (InternalControlVar ICV : TrackableICVs) {
244273471bf0Spatrick       auto ICVInfo = OMPInfoCache.ICVs[ICV];
244373471bf0Spatrick       auto &Getter = OMPInfoCache.RFIs[ICVInfo.Getter];
244473471bf0Spatrick       if (Getter.Declaration == getAssociatedFunction()) {
244573471bf0Spatrick         AssociatedICV = ICVInfo.Kind;
244673471bf0Spatrick         return;
244773471bf0Spatrick       }
244873471bf0Spatrick     }
244973471bf0Spatrick 
245073471bf0Spatrick     /// Unknown ICV.
245173471bf0Spatrick     indicatePessimisticFixpoint();
245273471bf0Spatrick   }
245373471bf0Spatrick 
manifest__anon4b47cb5e0111::AAICVTrackerCallSite245473471bf0Spatrick   ChangeStatus manifest(Attributor &A) override {
2455*d415bd75Srobert     if (!ReplVal || !*ReplVal)
245673471bf0Spatrick       return ChangeStatus::UNCHANGED;
245773471bf0Spatrick 
2458*d415bd75Srobert     A.changeAfterManifest(IRPosition::inst(*getCtxI()), **ReplVal);
245973471bf0Spatrick     A.deleteAfterManifest(*getCtxI());
246073471bf0Spatrick 
246173471bf0Spatrick     return ChangeStatus::CHANGED;
246273471bf0Spatrick   }
246373471bf0Spatrick 
246473471bf0Spatrick   // FIXME: come up with better string.
getAsStr__anon4b47cb5e0111::AAICVTrackerCallSite246573471bf0Spatrick   const std::string getAsStr() const override { return "ICVTrackerCallSite"; }
246673471bf0Spatrick 
246773471bf0Spatrick   // FIXME: come up with some stats.
trackStatistics__anon4b47cb5e0111::AAICVTrackerCallSite246873471bf0Spatrick   void trackStatistics() const override {}
246973471bf0Spatrick 
247073471bf0Spatrick   InternalControlVar AssociatedICV;
2471*d415bd75Srobert   std::optional<Value *> ReplVal;
247273471bf0Spatrick 
updateImpl__anon4b47cb5e0111::AAICVTrackerCallSite247373471bf0Spatrick   ChangeStatus updateImpl(Attributor &A) override {
247473471bf0Spatrick     const auto &ICVTrackingAA = A.getAAFor<AAICVTracker>(
247573471bf0Spatrick         *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED);
247673471bf0Spatrick 
247773471bf0Spatrick     // We don't have any information, so we assume it changes the ICV.
247873471bf0Spatrick     if (!ICVTrackingAA.isAssumedTracked())
247973471bf0Spatrick       return indicatePessimisticFixpoint();
248073471bf0Spatrick 
2481*d415bd75Srobert     std::optional<Value *> NewReplVal =
248273471bf0Spatrick         ICVTrackingAA.getReplacementValue(AssociatedICV, getCtxI(), A);
248373471bf0Spatrick 
248473471bf0Spatrick     if (ReplVal == NewReplVal)
248573471bf0Spatrick       return ChangeStatus::UNCHANGED;
248673471bf0Spatrick 
248773471bf0Spatrick     ReplVal = NewReplVal;
248873471bf0Spatrick     return ChangeStatus::CHANGED;
248973471bf0Spatrick   }
249073471bf0Spatrick 
249173471bf0Spatrick   // Return the value with which associated value can be replaced for specific
249273471bf0Spatrick   // \p ICV.
2493*d415bd75Srobert   std::optional<Value *>
getUniqueReplacementValue__anon4b47cb5e0111::AAICVTrackerCallSite249473471bf0Spatrick   getUniqueReplacementValue(InternalControlVar ICV) const override {
249573471bf0Spatrick     return ReplVal;
249673471bf0Spatrick   }
249773471bf0Spatrick };
249873471bf0Spatrick 
249973471bf0Spatrick struct AAICVTrackerCallSiteReturned : AAICVTracker {
AAICVTrackerCallSiteReturned__anon4b47cb5e0111::AAICVTrackerCallSiteReturned250073471bf0Spatrick   AAICVTrackerCallSiteReturned(const IRPosition &IRP, Attributor &A)
250173471bf0Spatrick       : AAICVTracker(IRP, A) {}
250273471bf0Spatrick 
250373471bf0Spatrick   // FIXME: come up with better string.
getAsStr__anon4b47cb5e0111::AAICVTrackerCallSiteReturned250473471bf0Spatrick   const std::string getAsStr() const override {
250573471bf0Spatrick     return "ICVTrackerCallSiteReturned";
250673471bf0Spatrick   }
250773471bf0Spatrick 
250873471bf0Spatrick   // FIXME: come up with some stats.
trackStatistics__anon4b47cb5e0111::AAICVTrackerCallSiteReturned250973471bf0Spatrick   void trackStatistics() const override {}
251073471bf0Spatrick 
251173471bf0Spatrick   /// We don't manifest anything for this AA.
manifest__anon4b47cb5e0111::AAICVTrackerCallSiteReturned251273471bf0Spatrick   ChangeStatus manifest(Attributor &A) override {
251373471bf0Spatrick     return ChangeStatus::UNCHANGED;
251473471bf0Spatrick   }
251573471bf0Spatrick 
251673471bf0Spatrick   // Map of ICV to their values at specific program point.
2517*d415bd75Srobert   EnumeratedArray<std::optional<Value *>, InternalControlVar,
251873471bf0Spatrick                   InternalControlVar::ICV___last>
251973471bf0Spatrick       ICVReplacementValuesMap;
252073471bf0Spatrick 
252173471bf0Spatrick   /// Return the value with which associated value can be replaced for specific
252273471bf0Spatrick   /// \p ICV.
2523*d415bd75Srobert   std::optional<Value *>
getUniqueReplacementValue__anon4b47cb5e0111::AAICVTrackerCallSiteReturned252473471bf0Spatrick   getUniqueReplacementValue(InternalControlVar ICV) const override {
252573471bf0Spatrick     return ICVReplacementValuesMap[ICV];
252673471bf0Spatrick   }
252773471bf0Spatrick 
updateImpl__anon4b47cb5e0111::AAICVTrackerCallSiteReturned252873471bf0Spatrick   ChangeStatus updateImpl(Attributor &A) override {
252973471bf0Spatrick     ChangeStatus Changed = ChangeStatus::UNCHANGED;
253073471bf0Spatrick     const auto &ICVTrackingAA = A.getAAFor<AAICVTracker>(
253173471bf0Spatrick         *this, IRPosition::returned(*getAssociatedFunction()),
253273471bf0Spatrick         DepClassTy::REQUIRED);
253373471bf0Spatrick 
253473471bf0Spatrick     // We don't have any information, so we assume it changes the ICV.
253573471bf0Spatrick     if (!ICVTrackingAA.isAssumedTracked())
253673471bf0Spatrick       return indicatePessimisticFixpoint();
253773471bf0Spatrick 
253873471bf0Spatrick     for (InternalControlVar ICV : TrackableICVs) {
2539*d415bd75Srobert       std::optional<Value *> &ReplVal = ICVReplacementValuesMap[ICV];
2540*d415bd75Srobert       std::optional<Value *> NewReplVal =
254173471bf0Spatrick           ICVTrackingAA.getUniqueReplacementValue(ICV);
254273471bf0Spatrick 
254373471bf0Spatrick       if (ReplVal == NewReplVal)
254473471bf0Spatrick         continue;
254573471bf0Spatrick 
254673471bf0Spatrick       ReplVal = NewReplVal;
254773471bf0Spatrick       Changed = ChangeStatus::CHANGED;
254873471bf0Spatrick     }
254973471bf0Spatrick     return Changed;
255073471bf0Spatrick   }
255173471bf0Spatrick };
255273471bf0Spatrick 
255373471bf0Spatrick struct AAExecutionDomainFunction : public AAExecutionDomain {
AAExecutionDomainFunction__anon4b47cb5e0111::AAExecutionDomainFunction255473471bf0Spatrick   AAExecutionDomainFunction(const IRPosition &IRP, Attributor &A)
255573471bf0Spatrick       : AAExecutionDomain(IRP, A) {}
255673471bf0Spatrick 
~AAExecutionDomainFunction__anon4b47cb5e0111::AAExecutionDomainFunction2557*d415bd75Srobert   ~AAExecutionDomainFunction() {
2558*d415bd75Srobert     delete RPOT;
2559*d415bd75Srobert   }
2560*d415bd75Srobert 
initialize__anon4b47cb5e0111::AAExecutionDomainFunction2561*d415bd75Srobert   void initialize(Attributor &A) override {
2562*d415bd75Srobert     if (getAnchorScope()->isDeclaration()) {
2563*d415bd75Srobert       indicatePessimisticFixpoint();
2564*d415bd75Srobert       return;
2565*d415bd75Srobert     }
2566*d415bd75Srobert     RPOT = new ReversePostOrderTraversal<Function *>(getAnchorScope());
2567*d415bd75Srobert   }
2568*d415bd75Srobert 
getAsStr__anon4b47cb5e0111::AAExecutionDomainFunction256973471bf0Spatrick   const std::string getAsStr() const override {
2570*d415bd75Srobert     unsigned TotalBlocks = 0, InitialThreadBlocks = 0;
2571*d415bd75Srobert     for (auto &It : BEDMap) {
2572*d415bd75Srobert       TotalBlocks++;
2573*d415bd75Srobert       InitialThreadBlocks += It.getSecond().IsExecutedByInitialThreadOnly;
2574*d415bd75Srobert     }
2575*d415bd75Srobert     return "[AAExecutionDomain] " + std::to_string(InitialThreadBlocks) + "/" +
2576*d415bd75Srobert            std::to_string(TotalBlocks) + " executed by initial thread only";
257773471bf0Spatrick   }
257873471bf0Spatrick 
257973471bf0Spatrick   /// See AbstractAttribute::trackStatistics().
trackStatistics__anon4b47cb5e0111::AAExecutionDomainFunction258073471bf0Spatrick   void trackStatistics() const override {}
258173471bf0Spatrick 
manifest__anon4b47cb5e0111::AAExecutionDomainFunction258273471bf0Spatrick   ChangeStatus manifest(Attributor &A) override {
258373471bf0Spatrick     LLVM_DEBUG({
2584*d415bd75Srobert       for (const BasicBlock &BB : *getAnchorScope()) {
2585*d415bd75Srobert         if (!isExecutedByInitialThreadOnly(BB))
2586*d415bd75Srobert           continue;
258773471bf0Spatrick         dbgs() << TAG << " Basic block @" << getAnchorScope()->getName() << " "
2588*d415bd75Srobert                << BB.getName() << " is executed by a single thread.\n";
2589*d415bd75Srobert       }
259073471bf0Spatrick     });
2591*d415bd75Srobert 
2592*d415bd75Srobert     ChangeStatus Changed = ChangeStatus::UNCHANGED;
2593*d415bd75Srobert 
2594*d415bd75Srobert     if (DisableOpenMPOptBarrierElimination)
2595*d415bd75Srobert       return Changed;
2596*d415bd75Srobert 
2597*d415bd75Srobert     SmallPtrSet<CallBase *, 16> DeletedBarriers;
2598*d415bd75Srobert     auto HandleAlignedBarrier = [&](CallBase *CB) {
2599*d415bd75Srobert       const ExecutionDomainTy &ED = CEDMap[CB];
2600*d415bd75Srobert       if (!ED.IsReachedFromAlignedBarrierOnly ||
2601*d415bd75Srobert           ED.EncounteredNonLocalSideEffect)
2602*d415bd75Srobert         return;
2603*d415bd75Srobert 
2604*d415bd75Srobert       // We can remove this barrier, if it is one, or all aligned barriers
2605*d415bd75Srobert       // reaching the kernel end. In the latter case we can transitively work
2606*d415bd75Srobert       // our way back until we find a barrier that guards a side-effect if we
2607*d415bd75Srobert       // are dealing with the kernel end here.
2608*d415bd75Srobert       if (CB) {
2609*d415bd75Srobert         DeletedBarriers.insert(CB);
2610*d415bd75Srobert         A.deleteAfterManifest(*CB);
2611*d415bd75Srobert         ++NumBarriersEliminated;
2612*d415bd75Srobert         Changed = ChangeStatus::CHANGED;
2613*d415bd75Srobert       } else if (!ED.AlignedBarriers.empty()) {
2614*d415bd75Srobert         NumBarriersEliminated += ED.AlignedBarriers.size();
2615*d415bd75Srobert         Changed = ChangeStatus::CHANGED;
2616*d415bd75Srobert         SmallVector<CallBase *> Worklist(ED.AlignedBarriers.begin(),
2617*d415bd75Srobert                                          ED.AlignedBarriers.end());
2618*d415bd75Srobert         SmallSetVector<CallBase *, 16> Visited;
2619*d415bd75Srobert         while (!Worklist.empty()) {
2620*d415bd75Srobert           CallBase *LastCB = Worklist.pop_back_val();
2621*d415bd75Srobert           if (!Visited.insert(LastCB))
2622*d415bd75Srobert             continue;
2623*d415bd75Srobert           if (!DeletedBarriers.count(LastCB)) {
2624*d415bd75Srobert             A.deleteAfterManifest(*LastCB);
2625*d415bd75Srobert             continue;
2626*d415bd75Srobert           }
2627*d415bd75Srobert           // The final aligned barrier (LastCB) reaching the kernel end was
2628*d415bd75Srobert           // removed already. This means we can go one step further and remove
2629*d415bd75Srobert           // the barriers encoutered last before (LastCB).
2630*d415bd75Srobert           const ExecutionDomainTy &LastED = CEDMap[LastCB];
2631*d415bd75Srobert           Worklist.append(LastED.AlignedBarriers.begin(),
2632*d415bd75Srobert                           LastED.AlignedBarriers.end());
2633*d415bd75Srobert         }
263473471bf0Spatrick       }
263573471bf0Spatrick 
2636*d415bd75Srobert       // If we actually eliminated a barrier we need to eliminate the associated
2637*d415bd75Srobert       // llvm.assumes as well to avoid creating UB.
2638*d415bd75Srobert       if (!ED.EncounteredAssumes.empty() && (CB || !ED.AlignedBarriers.empty()))
2639*d415bd75Srobert         for (auto *AssumeCB : ED.EncounteredAssumes)
2640*d415bd75Srobert           A.deleteAfterManifest(*AssumeCB);
264173471bf0Spatrick     };
264273471bf0Spatrick 
2643*d415bd75Srobert     for (auto *CB : AlignedBarriers)
2644*d415bd75Srobert       HandleAlignedBarrier(CB);
264573471bf0Spatrick 
264673471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2647*d415bd75Srobert     // Handle the "kernel end barrier" for kernels too.
2648*d415bd75Srobert     if (OMPInfoCache.Kernels.count(getAnchorScope()))
2649*d415bd75Srobert       HandleAlignedBarrier(nullptr);
265073471bf0Spatrick 
2651*d415bd75Srobert     return Changed;
2652*d415bd75Srobert   }
2653*d415bd75Srobert 
2654*d415bd75Srobert   /// Merge barrier and assumption information from \p PredED into the successor
2655*d415bd75Srobert   /// \p ED.
2656*d415bd75Srobert   void
2657*d415bd75Srobert   mergeInPredecessorBarriersAndAssumptions(Attributor &A, ExecutionDomainTy &ED,
2658*d415bd75Srobert                                            const ExecutionDomainTy &PredED);
2659*d415bd75Srobert 
2660*d415bd75Srobert   /// Merge all information from \p PredED into the successor \p ED. If
2661*d415bd75Srobert   /// \p InitialEdgeOnly is set, only the initial edge will enter the block
2662*d415bd75Srobert   /// represented by \p ED from this predecessor.
2663*d415bd75Srobert   void mergeInPredecessor(Attributor &A, ExecutionDomainTy &ED,
2664*d415bd75Srobert                           const ExecutionDomainTy &PredED,
2665*d415bd75Srobert                           bool InitialEdgeOnly = false);
2666*d415bd75Srobert 
2667*d415bd75Srobert   /// Accumulate information for the entry block in \p EntryBBED.
2668*d415bd75Srobert   void handleEntryBB(Attributor &A, ExecutionDomainTy &EntryBBED);
2669*d415bd75Srobert 
2670*d415bd75Srobert   /// See AbstractAttribute::updateImpl.
2671*d415bd75Srobert   ChangeStatus updateImpl(Attributor &A) override;
2672*d415bd75Srobert 
2673*d415bd75Srobert   /// Query interface, see AAExecutionDomain
2674*d415bd75Srobert   ///{
isExecutedByInitialThreadOnly__anon4b47cb5e0111::AAExecutionDomainFunction2675*d415bd75Srobert   bool isExecutedByInitialThreadOnly(const BasicBlock &BB) const override {
2676*d415bd75Srobert     if (!isValidState())
2677*d415bd75Srobert       return false;
2678*d415bd75Srobert     return BEDMap.lookup(&BB).IsExecutedByInitialThreadOnly;
2679*d415bd75Srobert   }
2680*d415bd75Srobert 
isExecutedInAlignedRegion__anon4b47cb5e0111::AAExecutionDomainFunction2681*d415bd75Srobert   bool isExecutedInAlignedRegion(Attributor &A,
2682*d415bd75Srobert                                  const Instruction &I) const override {
2683*d415bd75Srobert     assert(I.getFunction() == getAnchorScope() &&
2684*d415bd75Srobert            "Instruction is out of scope!");
2685*d415bd75Srobert     if (!isValidState())
2686*d415bd75Srobert       return false;
2687*d415bd75Srobert 
2688*d415bd75Srobert     const Instruction *CurI;
2689*d415bd75Srobert 
2690*d415bd75Srobert     // Check forward until a call or the block end is reached.
2691*d415bd75Srobert     CurI = &I;
2692*d415bd75Srobert     do {
2693*d415bd75Srobert       auto *CB = dyn_cast<CallBase>(CurI);
2694*d415bd75Srobert       if (!CB)
2695*d415bd75Srobert         continue;
2696*d415bd75Srobert       if (CB != &I && AlignedBarriers.contains(const_cast<CallBase *>(CB))) {
2697*d415bd75Srobert         break;
2698*d415bd75Srobert       }
2699*d415bd75Srobert       const auto &It = CEDMap.find(CB);
2700*d415bd75Srobert       if (It == CEDMap.end())
2701*d415bd75Srobert         continue;
2702*d415bd75Srobert       if (!It->getSecond().IsReachingAlignedBarrierOnly)
2703*d415bd75Srobert         return false;
2704*d415bd75Srobert       break;
2705*d415bd75Srobert     } while ((CurI = CurI->getNextNonDebugInstruction()));
2706*d415bd75Srobert 
2707*d415bd75Srobert     if (!CurI && !BEDMap.lookup(I.getParent()).IsReachingAlignedBarrierOnly)
2708*d415bd75Srobert       return false;
2709*d415bd75Srobert 
2710*d415bd75Srobert     // Check backward until a call or the block beginning is reached.
2711*d415bd75Srobert     CurI = &I;
2712*d415bd75Srobert     do {
2713*d415bd75Srobert       auto *CB = dyn_cast<CallBase>(CurI);
2714*d415bd75Srobert       if (!CB)
2715*d415bd75Srobert         continue;
2716*d415bd75Srobert       if (CB != &I && AlignedBarriers.contains(const_cast<CallBase *>(CB))) {
2717*d415bd75Srobert         break;
2718*d415bd75Srobert       }
2719*d415bd75Srobert       const auto &It = CEDMap.find(CB);
2720*d415bd75Srobert       if (It == CEDMap.end())
2721*d415bd75Srobert         continue;
2722*d415bd75Srobert       if (!AA::isNoSyncInst(A, *CB, *this)) {
2723*d415bd75Srobert         if (It->getSecond().IsReachedFromAlignedBarrierOnly) {
2724*d415bd75Srobert           break;
2725*d415bd75Srobert         }
2726*d415bd75Srobert         return false;
2727*d415bd75Srobert       }
2728*d415bd75Srobert 
2729*d415bd75Srobert       Function *Callee = CB->getCalledFunction();
2730*d415bd75Srobert       if (!Callee || Callee->isDeclaration())
2731*d415bd75Srobert         return false;
2732*d415bd75Srobert       const auto &EDAA = A.getAAFor<AAExecutionDomain>(
2733*d415bd75Srobert           *this, IRPosition::function(*Callee), DepClassTy::OPTIONAL);
2734*d415bd75Srobert       if (!EDAA.getState().isValidState())
2735*d415bd75Srobert         return false;
2736*d415bd75Srobert       if (!EDAA.getFunctionExecutionDomain().IsReachedFromAlignedBarrierOnly)
2737*d415bd75Srobert         return false;
2738*d415bd75Srobert       break;
2739*d415bd75Srobert     } while ((CurI = CurI->getPrevNonDebugInstruction()));
2740*d415bd75Srobert 
2741*d415bd75Srobert     if (!CurI &&
2742*d415bd75Srobert         !llvm::all_of(
2743*d415bd75Srobert             predecessors(I.getParent()), [&](const BasicBlock *PredBB) {
2744*d415bd75Srobert               return BEDMap.lookup(PredBB).IsReachedFromAlignedBarrierOnly;
2745*d415bd75Srobert             })) {
2746*d415bd75Srobert       return false;
2747*d415bd75Srobert     }
2748*d415bd75Srobert 
2749*d415bd75Srobert     // On neither traversal we found a anything but aligned barriers.
2750*d415bd75Srobert     return true;
2751*d415bd75Srobert   }
2752*d415bd75Srobert 
getExecutionDomain__anon4b47cb5e0111::AAExecutionDomainFunction2753*d415bd75Srobert   ExecutionDomainTy getExecutionDomain(const BasicBlock &BB) const override {
2754*d415bd75Srobert     assert(isValidState() &&
2755*d415bd75Srobert            "No request should be made against an invalid state!");
2756*d415bd75Srobert     return BEDMap.lookup(&BB);
2757*d415bd75Srobert   }
getExecutionDomain__anon4b47cb5e0111::AAExecutionDomainFunction2758*d415bd75Srobert   ExecutionDomainTy getExecutionDomain(const CallBase &CB) const override {
2759*d415bd75Srobert     assert(isValidState() &&
2760*d415bd75Srobert            "No request should be made against an invalid state!");
2761*d415bd75Srobert     return CEDMap.lookup(&CB);
2762*d415bd75Srobert   }
getFunctionExecutionDomain__anon4b47cb5e0111::AAExecutionDomainFunction2763*d415bd75Srobert   ExecutionDomainTy getFunctionExecutionDomain() const override {
2764*d415bd75Srobert     assert(isValidState() &&
2765*d415bd75Srobert            "No request should be made against an invalid state!");
2766*d415bd75Srobert     return BEDMap.lookup(nullptr);
2767*d415bd75Srobert   }
2768*d415bd75Srobert   ///}
2769*d415bd75Srobert 
2770*d415bd75Srobert   // Check if the edge into the successor block contains a condition that only
2771*d415bd75Srobert   // lets the main thread execute it.
isInitialThreadOnlyEdge__anon4b47cb5e0111::AAExecutionDomainFunction2772*d415bd75Srobert   static bool isInitialThreadOnlyEdge(Attributor &A, BranchInst *Edge,
2773*d415bd75Srobert                                       BasicBlock &SuccessorBB) {
277473471bf0Spatrick     if (!Edge || !Edge->isConditional())
277573471bf0Spatrick       return false;
2776*d415bd75Srobert     if (Edge->getSuccessor(0) != &SuccessorBB)
277773471bf0Spatrick       return false;
277873471bf0Spatrick 
277973471bf0Spatrick     auto *Cmp = dyn_cast<CmpInst>(Edge->getCondition());
278073471bf0Spatrick     if (!Cmp || !Cmp->isTrueWhenEqual() || !Cmp->isEquality())
278173471bf0Spatrick       return false;
278273471bf0Spatrick 
278373471bf0Spatrick     ConstantInt *C = dyn_cast<ConstantInt>(Cmp->getOperand(1));
278473471bf0Spatrick     if (!C)
278573471bf0Spatrick       return false;
278673471bf0Spatrick 
278773471bf0Spatrick     // Match: -1 == __kmpc_target_init (for non-SPMD kernels only!)
278873471bf0Spatrick     if (C->isAllOnesValue()) {
278973471bf0Spatrick       auto *CB = dyn_cast<CallBase>(Cmp->getOperand(0));
2790*d415bd75Srobert       auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2791*d415bd75Srobert       auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
279273471bf0Spatrick       CB = CB ? OpenMPOpt::getCallIfRegularCall(*CB, &RFI) : nullptr;
279373471bf0Spatrick       if (!CB)
279473471bf0Spatrick         return false;
2795*d415bd75Srobert       const int InitModeArgNo = 1;
2796*d415bd75Srobert       auto *ModeCI = dyn_cast<ConstantInt>(CB->getOperand(InitModeArgNo));
2797*d415bd75Srobert       return ModeCI && (ModeCI->getSExtValue() & OMP_TGT_EXEC_MODE_GENERIC);
2798*d415bd75Srobert     }
2799*d415bd75Srobert 
2800*d415bd75Srobert     if (C->isZero()) {
2801*d415bd75Srobert       // Match: 0 == llvm.nvvm.read.ptx.sreg.tid.x()
2802*d415bd75Srobert       if (auto *II = dyn_cast<IntrinsicInst>(Cmp->getOperand(0)))
2803*d415bd75Srobert         if (II->getIntrinsicID() == Intrinsic::nvvm_read_ptx_sreg_tid_x)
2804*d415bd75Srobert           return true;
2805*d415bd75Srobert 
2806*d415bd75Srobert       // Match: 0 == llvm.amdgcn.workitem.id.x()
2807*d415bd75Srobert       if (auto *II = dyn_cast<IntrinsicInst>(Cmp->getOperand(0)))
2808*d415bd75Srobert         if (II->getIntrinsicID() == Intrinsic::amdgcn_workitem_id_x)
2809*d415bd75Srobert           return true;
281073471bf0Spatrick     }
281173471bf0Spatrick 
281273471bf0Spatrick     return false;
281373471bf0Spatrick   };
281473471bf0Spatrick 
2815*d415bd75Srobert   /// Mapping containing information per block.
2816*d415bd75Srobert   DenseMap<const BasicBlock *, ExecutionDomainTy> BEDMap;
2817*d415bd75Srobert   DenseMap<const CallBase *, ExecutionDomainTy> CEDMap;
2818*d415bd75Srobert   SmallSetVector<CallBase *, 16> AlignedBarriers;
281973471bf0Spatrick 
2820*d415bd75Srobert   ReversePostOrderTraversal<Function *> *RPOT = nullptr;
282173471bf0Spatrick };
282273471bf0Spatrick 
mergeInPredecessorBarriersAndAssumptions(Attributor & A,ExecutionDomainTy & ED,const ExecutionDomainTy & PredED)2823*d415bd75Srobert void AAExecutionDomainFunction::mergeInPredecessorBarriersAndAssumptions(
2824*d415bd75Srobert     Attributor &A, ExecutionDomainTy &ED, const ExecutionDomainTy &PredED) {
2825*d415bd75Srobert   for (auto *EA : PredED.EncounteredAssumes)
2826*d415bd75Srobert     ED.addAssumeInst(A, *EA);
2827*d415bd75Srobert 
2828*d415bd75Srobert   for (auto *AB : PredED.AlignedBarriers)
2829*d415bd75Srobert     ED.addAlignedBarrier(A, *AB);
283073471bf0Spatrick }
283173471bf0Spatrick 
mergeInPredecessor(Attributor & A,ExecutionDomainTy & ED,const ExecutionDomainTy & PredED,bool InitialEdgeOnly)2832*d415bd75Srobert void AAExecutionDomainFunction::mergeInPredecessor(
2833*d415bd75Srobert     Attributor &A, ExecutionDomainTy &ED, const ExecutionDomainTy &PredED,
2834*d415bd75Srobert     bool InitialEdgeOnly) {
2835*d415bd75Srobert   ED.IsExecutedByInitialThreadOnly =
2836*d415bd75Srobert       InitialEdgeOnly || (PredED.IsExecutedByInitialThreadOnly &&
2837*d415bd75Srobert                           ED.IsExecutedByInitialThreadOnly);
2838*d415bd75Srobert 
2839*d415bd75Srobert   ED.IsReachedFromAlignedBarrierOnly = ED.IsReachedFromAlignedBarrierOnly &&
2840*d415bd75Srobert                                        PredED.IsReachedFromAlignedBarrierOnly;
2841*d415bd75Srobert   ED.EncounteredNonLocalSideEffect =
2842*d415bd75Srobert       ED.EncounteredNonLocalSideEffect | PredED.EncounteredNonLocalSideEffect;
2843*d415bd75Srobert   if (ED.IsReachedFromAlignedBarrierOnly)
2844*d415bd75Srobert     mergeInPredecessorBarriersAndAssumptions(A, ED, PredED);
2845*d415bd75Srobert   else
2846*d415bd75Srobert     ED.clearAssumeInstAndAlignedBarriers();
2847*d415bd75Srobert }
2848*d415bd75Srobert 
handleEntryBB(Attributor & A,ExecutionDomainTy & EntryBBED)2849*d415bd75Srobert void AAExecutionDomainFunction::handleEntryBB(Attributor &A,
2850*d415bd75Srobert                                               ExecutionDomainTy &EntryBBED) {
2851*d415bd75Srobert   SmallVector<ExecutionDomainTy> PredExecDomains;
2852*d415bd75Srobert   auto PredForCallSite = [&](AbstractCallSite ACS) {
2853*d415bd75Srobert     const auto &EDAA = A.getAAFor<AAExecutionDomain>(
2854*d415bd75Srobert         *this, IRPosition::function(*ACS.getInstruction()->getFunction()),
2855*d415bd75Srobert         DepClassTy::OPTIONAL);
2856*d415bd75Srobert     if (!EDAA.getState().isValidState())
2857*d415bd75Srobert       return false;
2858*d415bd75Srobert     PredExecDomains.emplace_back(
2859*d415bd75Srobert         EDAA.getExecutionDomain(*cast<CallBase>(ACS.getInstruction())));
2860*d415bd75Srobert     return true;
2861*d415bd75Srobert   };
2862*d415bd75Srobert 
2863*d415bd75Srobert   bool AllCallSitesKnown;
2864*d415bd75Srobert   if (A.checkForAllCallSites(PredForCallSite, *this,
2865*d415bd75Srobert                              /* RequiresAllCallSites */ true,
2866*d415bd75Srobert                              AllCallSitesKnown)) {
2867*d415bd75Srobert     for (const auto &PredED : PredExecDomains)
2868*d415bd75Srobert       mergeInPredecessor(A, EntryBBED, PredED);
2869*d415bd75Srobert 
2870*d415bd75Srobert   } else {
2871*d415bd75Srobert     // We could not find all predecessors, so this is either a kernel or a
2872*d415bd75Srobert     // function with external linkage (or with some other weird uses).
2873*d415bd75Srobert     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2874*d415bd75Srobert     if (OMPInfoCache.Kernels.count(getAnchorScope())) {
2875*d415bd75Srobert       EntryBBED.IsExecutedByInitialThreadOnly = false;
2876*d415bd75Srobert       EntryBBED.IsReachedFromAlignedBarrierOnly = true;
2877*d415bd75Srobert       EntryBBED.EncounteredNonLocalSideEffect = false;
2878*d415bd75Srobert     } else {
2879*d415bd75Srobert       EntryBBED.IsExecutedByInitialThreadOnly = false;
2880*d415bd75Srobert       EntryBBED.IsReachedFromAlignedBarrierOnly = false;
2881*d415bd75Srobert       EntryBBED.EncounteredNonLocalSideEffect = true;
2882*d415bd75Srobert     }
2883*d415bd75Srobert   }
2884*d415bd75Srobert 
2885*d415bd75Srobert   auto &FnED = BEDMap[nullptr];
2886*d415bd75Srobert   FnED.IsReachingAlignedBarrierOnly &=
2887*d415bd75Srobert       EntryBBED.IsReachedFromAlignedBarrierOnly;
2888*d415bd75Srobert }
2889*d415bd75Srobert 
updateImpl(Attributor & A)2890*d415bd75Srobert ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) {
2891*d415bd75Srobert 
2892*d415bd75Srobert   bool Changed = false;
2893*d415bd75Srobert 
2894*d415bd75Srobert   // Helper to deal with an aligned barrier encountered during the forward
2895*d415bd75Srobert   // traversal. \p CB is the aligned barrier, \p ED is the execution domain when
2896*d415bd75Srobert   // it was encountered.
2897*d415bd75Srobert   auto HandleAlignedBarrier = [&](CallBase *CB, ExecutionDomainTy &ED) {
2898*d415bd75Srobert     if (CB)
2899*d415bd75Srobert       Changed |= AlignedBarriers.insert(CB);
2900*d415bd75Srobert     // First, update the barrier ED kept in the separate CEDMap.
2901*d415bd75Srobert     auto &CallED = CEDMap[CB];
2902*d415bd75Srobert     mergeInPredecessor(A, CallED, ED);
2903*d415bd75Srobert     // Next adjust the ED we use for the traversal.
2904*d415bd75Srobert     ED.EncounteredNonLocalSideEffect = false;
2905*d415bd75Srobert     ED.IsReachedFromAlignedBarrierOnly = true;
2906*d415bd75Srobert     // Aligned barrier collection has to come last.
2907*d415bd75Srobert     ED.clearAssumeInstAndAlignedBarriers();
2908*d415bd75Srobert     if (CB)
2909*d415bd75Srobert       ED.addAlignedBarrier(A, *CB);
2910*d415bd75Srobert   };
2911*d415bd75Srobert 
2912*d415bd75Srobert   auto &LivenessAA =
2913*d415bd75Srobert       A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
2914*d415bd75Srobert 
2915*d415bd75Srobert   // Set \p R to \V and report true if that changed \p R.
2916*d415bd75Srobert   auto SetAndRecord = [&](bool &R, bool V) {
2917*d415bd75Srobert     bool Eq = (R == V);
2918*d415bd75Srobert     R = V;
2919*d415bd75Srobert     return !Eq;
2920*d415bd75Srobert   };
2921*d415bd75Srobert 
2922*d415bd75Srobert   auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2923*d415bd75Srobert 
2924*d415bd75Srobert   Function *F = getAnchorScope();
2925*d415bd75Srobert   BasicBlock &EntryBB = F->getEntryBlock();
2926*d415bd75Srobert   bool IsKernel = OMPInfoCache.Kernels.count(F);
2927*d415bd75Srobert 
2928*d415bd75Srobert   SmallVector<Instruction *> SyncInstWorklist;
2929*d415bd75Srobert   for (auto &RIt : *RPOT) {
2930*d415bd75Srobert     BasicBlock &BB = *RIt;
2931*d415bd75Srobert 
2932*d415bd75Srobert     bool IsEntryBB = &BB == &EntryBB;
2933*d415bd75Srobert     // TODO: We use local reasoning since we don't have a divergence analysis
2934*d415bd75Srobert     // 	     running as well. We could basically allow uniform branches here.
2935*d415bd75Srobert     bool AlignedBarrierLastInBlock = IsEntryBB && IsKernel;
2936*d415bd75Srobert     ExecutionDomainTy ED;
2937*d415bd75Srobert     // Propagate "incoming edges" into information about this block.
2938*d415bd75Srobert     if (IsEntryBB) {
2939*d415bd75Srobert       handleEntryBB(A, ED);
2940*d415bd75Srobert     } else {
2941*d415bd75Srobert       // For live non-entry blocks we only propagate
2942*d415bd75Srobert       // information via live edges.
2943*d415bd75Srobert       if (LivenessAA.isAssumedDead(&BB))
2944*d415bd75Srobert         continue;
2945*d415bd75Srobert 
2946*d415bd75Srobert       for (auto *PredBB : predecessors(&BB)) {
2947*d415bd75Srobert         if (LivenessAA.isEdgeDead(PredBB, &BB))
2948*d415bd75Srobert           continue;
2949*d415bd75Srobert         bool InitialEdgeOnly = isInitialThreadOnlyEdge(
2950*d415bd75Srobert             A, dyn_cast<BranchInst>(PredBB->getTerminator()), BB);
2951*d415bd75Srobert         mergeInPredecessor(A, ED, BEDMap[PredBB], InitialEdgeOnly);
2952*d415bd75Srobert       }
2953*d415bd75Srobert     }
2954*d415bd75Srobert 
2955*d415bd75Srobert     // Now we traverse the block, accumulate effects in ED and attach
2956*d415bd75Srobert     // information to calls.
2957*d415bd75Srobert     for (Instruction &I : BB) {
2958*d415bd75Srobert       bool UsedAssumedInformation;
2959*d415bd75Srobert       if (A.isAssumedDead(I, *this, &LivenessAA, UsedAssumedInformation,
2960*d415bd75Srobert                           /* CheckBBLivenessOnly */ false, DepClassTy::OPTIONAL,
2961*d415bd75Srobert                           /* CheckForDeadStore */ true))
2962*d415bd75Srobert         continue;
2963*d415bd75Srobert 
2964*d415bd75Srobert       // Asummes and "assume-like" (dbg, lifetime, ...) are handled first, the
2965*d415bd75Srobert       // former is collected the latter is ignored.
2966*d415bd75Srobert       if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
2967*d415bd75Srobert         if (auto *AI = dyn_cast_or_null<AssumeInst>(II)) {
2968*d415bd75Srobert           ED.addAssumeInst(A, *AI);
2969*d415bd75Srobert           continue;
2970*d415bd75Srobert         }
2971*d415bd75Srobert         // TODO: Should we also collect and delete lifetime markers?
2972*d415bd75Srobert         if (II->isAssumeLikeIntrinsic())
2973*d415bd75Srobert           continue;
2974*d415bd75Srobert       }
2975*d415bd75Srobert 
2976*d415bd75Srobert       auto *CB = dyn_cast<CallBase>(&I);
2977*d415bd75Srobert       bool IsNoSync = AA::isNoSyncInst(A, I, *this);
2978*d415bd75Srobert       bool IsAlignedBarrier =
2979*d415bd75Srobert           !IsNoSync && CB &&
2980*d415bd75Srobert           AANoSync::isAlignedBarrier(*CB, AlignedBarrierLastInBlock);
2981*d415bd75Srobert 
2982*d415bd75Srobert       AlignedBarrierLastInBlock &= IsNoSync;
2983*d415bd75Srobert 
2984*d415bd75Srobert       // Next we check for calls. Aligned barriers are handled
2985*d415bd75Srobert       // explicitly, everything else is kept for the backward traversal and will
2986*d415bd75Srobert       // also affect our state.
2987*d415bd75Srobert       if (CB) {
2988*d415bd75Srobert         if (IsAlignedBarrier) {
2989*d415bd75Srobert           HandleAlignedBarrier(CB, ED);
2990*d415bd75Srobert           AlignedBarrierLastInBlock = true;
2991*d415bd75Srobert           continue;
2992*d415bd75Srobert         }
2993*d415bd75Srobert 
2994*d415bd75Srobert         // Check the pointer(s) of a memory intrinsic explicitly.
2995*d415bd75Srobert         if (isa<MemIntrinsic>(&I)) {
2996*d415bd75Srobert           if (!ED.EncounteredNonLocalSideEffect &&
2997*d415bd75Srobert               AA::isPotentiallyAffectedByBarrier(A, I, *this))
2998*d415bd75Srobert             ED.EncounteredNonLocalSideEffect = true;
2999*d415bd75Srobert           if (!IsNoSync) {
3000*d415bd75Srobert             ED.IsReachedFromAlignedBarrierOnly = false;
3001*d415bd75Srobert             SyncInstWorklist.push_back(&I);
3002*d415bd75Srobert           }
3003*d415bd75Srobert           continue;
3004*d415bd75Srobert         }
3005*d415bd75Srobert 
3006*d415bd75Srobert         // Record how we entered the call, then accumulate the effect of the
3007*d415bd75Srobert         // call in ED for potential use by the callee.
3008*d415bd75Srobert         auto &CallED = CEDMap[CB];
3009*d415bd75Srobert         mergeInPredecessor(A, CallED, ED);
3010*d415bd75Srobert 
3011*d415bd75Srobert         // If we have a sync-definition we can check if it starts/ends in an
3012*d415bd75Srobert         // aligned barrier. If we are unsure we assume any sync breaks
3013*d415bd75Srobert         // alignment.
3014*d415bd75Srobert         Function *Callee = CB->getCalledFunction();
3015*d415bd75Srobert         if (!IsNoSync && Callee && !Callee->isDeclaration()) {
3016*d415bd75Srobert           const auto &EDAA = A.getAAFor<AAExecutionDomain>(
3017*d415bd75Srobert               *this, IRPosition::function(*Callee), DepClassTy::OPTIONAL);
3018*d415bd75Srobert           if (EDAA.getState().isValidState()) {
3019*d415bd75Srobert             const auto &CalleeED = EDAA.getFunctionExecutionDomain();
3020*d415bd75Srobert             ED.IsReachedFromAlignedBarrierOnly =
3021*d415bd75Srobert                 CallED.IsReachedFromAlignedBarrierOnly =
3022*d415bd75Srobert                     CalleeED.IsReachedFromAlignedBarrierOnly;
3023*d415bd75Srobert             AlignedBarrierLastInBlock = ED.IsReachedFromAlignedBarrierOnly;
3024*d415bd75Srobert             if (IsNoSync || !CalleeED.IsReachedFromAlignedBarrierOnly)
3025*d415bd75Srobert               ED.EncounteredNonLocalSideEffect |=
3026*d415bd75Srobert                   CalleeED.EncounteredNonLocalSideEffect;
3027*d415bd75Srobert             else
3028*d415bd75Srobert               ED.EncounteredNonLocalSideEffect =
3029*d415bd75Srobert                   CalleeED.EncounteredNonLocalSideEffect;
3030*d415bd75Srobert             if (!CalleeED.IsReachingAlignedBarrierOnly)
3031*d415bd75Srobert               SyncInstWorklist.push_back(&I);
3032*d415bd75Srobert             if (CalleeED.IsReachedFromAlignedBarrierOnly)
3033*d415bd75Srobert               mergeInPredecessorBarriersAndAssumptions(A, ED, CalleeED);
3034*d415bd75Srobert             continue;
3035*d415bd75Srobert           }
3036*d415bd75Srobert         }
3037*d415bd75Srobert         if (!IsNoSync)
3038*d415bd75Srobert           ED.IsReachedFromAlignedBarrierOnly =
3039*d415bd75Srobert               CallED.IsReachedFromAlignedBarrierOnly = false;
3040*d415bd75Srobert         AlignedBarrierLastInBlock &= ED.IsReachedFromAlignedBarrierOnly;
3041*d415bd75Srobert         ED.EncounteredNonLocalSideEffect |= !CB->doesNotAccessMemory();
3042*d415bd75Srobert         if (!IsNoSync)
3043*d415bd75Srobert           SyncInstWorklist.push_back(&I);
3044*d415bd75Srobert       }
3045*d415bd75Srobert 
3046*d415bd75Srobert       if (!I.mayHaveSideEffects() && !I.mayReadFromMemory())
3047*d415bd75Srobert         continue;
3048*d415bd75Srobert 
3049*d415bd75Srobert       // If we have a callee we try to use fine-grained information to
3050*d415bd75Srobert       // determine local side-effects.
3051*d415bd75Srobert       if (CB) {
3052*d415bd75Srobert         const auto &MemAA = A.getAAFor<AAMemoryLocation>(
3053*d415bd75Srobert             *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);
3054*d415bd75Srobert 
3055*d415bd75Srobert         auto AccessPred = [&](const Instruction *I, const Value *Ptr,
3056*d415bd75Srobert                               AAMemoryLocation::AccessKind,
3057*d415bd75Srobert                               AAMemoryLocation::MemoryLocationsKind) {
3058*d415bd75Srobert           return !AA::isPotentiallyAffectedByBarrier(A, {Ptr}, *this, I);
3059*d415bd75Srobert         };
3060*d415bd75Srobert         if (MemAA.getState().isValidState() &&
3061*d415bd75Srobert             MemAA.checkForAllAccessesToMemoryKind(
3062*d415bd75Srobert                 AccessPred, AAMemoryLocation::ALL_LOCATIONS))
3063*d415bd75Srobert           continue;
3064*d415bd75Srobert       }
3065*d415bd75Srobert 
3066*d415bd75Srobert       if (!I.mayHaveSideEffects() && OMPInfoCache.isOnlyUsedByAssume(I))
3067*d415bd75Srobert         continue;
3068*d415bd75Srobert 
3069*d415bd75Srobert       if (auto *LI = dyn_cast<LoadInst>(&I))
3070*d415bd75Srobert         if (LI->hasMetadata(LLVMContext::MD_invariant_load))
3071*d415bd75Srobert           continue;
3072*d415bd75Srobert 
3073*d415bd75Srobert       if (!ED.EncounteredNonLocalSideEffect &&
3074*d415bd75Srobert           AA::isPotentiallyAffectedByBarrier(A, I, *this))
3075*d415bd75Srobert         ED.EncounteredNonLocalSideEffect = true;
3076*d415bd75Srobert     }
3077*d415bd75Srobert 
3078*d415bd75Srobert     if (!isa<UnreachableInst>(BB.getTerminator()) &&
3079*d415bd75Srobert         !BB.getTerminator()->getNumSuccessors()) {
3080*d415bd75Srobert 
3081*d415bd75Srobert       auto &FnED = BEDMap[nullptr];
3082*d415bd75Srobert       mergeInPredecessor(A, FnED, ED);
3083*d415bd75Srobert 
3084*d415bd75Srobert       if (IsKernel)
3085*d415bd75Srobert         HandleAlignedBarrier(nullptr, ED);
3086*d415bd75Srobert     }
3087*d415bd75Srobert 
3088*d415bd75Srobert     ExecutionDomainTy &StoredED = BEDMap[&BB];
3089*d415bd75Srobert     ED.IsReachingAlignedBarrierOnly = StoredED.IsReachingAlignedBarrierOnly;
3090*d415bd75Srobert 
3091*d415bd75Srobert     // Check if we computed anything different as part of the forward
3092*d415bd75Srobert     // traversal. We do not take assumptions and aligned barriers into account
3093*d415bd75Srobert     // as they do not influence the state we iterate. Backward traversal values
3094*d415bd75Srobert     // are handled later on.
3095*d415bd75Srobert     if (ED.IsExecutedByInitialThreadOnly !=
3096*d415bd75Srobert             StoredED.IsExecutedByInitialThreadOnly ||
3097*d415bd75Srobert         ED.IsReachedFromAlignedBarrierOnly !=
3098*d415bd75Srobert             StoredED.IsReachedFromAlignedBarrierOnly ||
3099*d415bd75Srobert         ED.EncounteredNonLocalSideEffect !=
3100*d415bd75Srobert             StoredED.EncounteredNonLocalSideEffect)
3101*d415bd75Srobert       Changed = true;
3102*d415bd75Srobert 
3103*d415bd75Srobert     // Update the state with the new value.
3104*d415bd75Srobert     StoredED = std::move(ED);
3105*d415bd75Srobert   }
3106*d415bd75Srobert 
3107*d415bd75Srobert   // Propagate (non-aligned) sync instruction effects backwards until the
3108*d415bd75Srobert   // entry is hit or an aligned barrier.
3109*d415bd75Srobert   SmallSetVector<BasicBlock *, 16> Visited;
3110*d415bd75Srobert   while (!SyncInstWorklist.empty()) {
3111*d415bd75Srobert     Instruction *SyncInst = SyncInstWorklist.pop_back_val();
3112*d415bd75Srobert     Instruction *CurInst = SyncInst;
3113*d415bd75Srobert     bool HitAlignedBarrier = false;
3114*d415bd75Srobert     while ((CurInst = CurInst->getPrevNode())) {
3115*d415bd75Srobert       auto *CB = dyn_cast<CallBase>(CurInst);
3116*d415bd75Srobert       if (!CB)
3117*d415bd75Srobert         continue;
3118*d415bd75Srobert       auto &CallED = CEDMap[CB];
3119*d415bd75Srobert       if (SetAndRecord(CallED.IsReachingAlignedBarrierOnly, false))
3120*d415bd75Srobert         Changed = true;
3121*d415bd75Srobert       HitAlignedBarrier = AlignedBarriers.count(CB);
3122*d415bd75Srobert       if (HitAlignedBarrier)
3123*d415bd75Srobert         break;
3124*d415bd75Srobert     }
3125*d415bd75Srobert     if (HitAlignedBarrier)
3126*d415bd75Srobert       continue;
3127*d415bd75Srobert     BasicBlock *SyncBB = SyncInst->getParent();
3128*d415bd75Srobert     for (auto *PredBB : predecessors(SyncBB)) {
3129*d415bd75Srobert       if (LivenessAA.isEdgeDead(PredBB, SyncBB))
3130*d415bd75Srobert         continue;
3131*d415bd75Srobert       if (!Visited.insert(PredBB))
3132*d415bd75Srobert         continue;
3133*d415bd75Srobert       SyncInstWorklist.push_back(PredBB->getTerminator());
3134*d415bd75Srobert       auto &PredED = BEDMap[PredBB];
3135*d415bd75Srobert       if (SetAndRecord(PredED.IsReachingAlignedBarrierOnly, false))
3136*d415bd75Srobert         Changed = true;
3137*d415bd75Srobert     }
3138*d415bd75Srobert     if (SyncBB != &EntryBB)
3139*d415bd75Srobert       continue;
3140*d415bd75Srobert     auto &FnED = BEDMap[nullptr];
3141*d415bd75Srobert     if (SetAndRecord(FnED.IsReachingAlignedBarrierOnly, false))
3142*d415bd75Srobert       Changed = true;
3143*d415bd75Srobert   }
3144*d415bd75Srobert 
3145*d415bd75Srobert   return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
314673471bf0Spatrick }
314773471bf0Spatrick 
314873471bf0Spatrick /// Try to replace memory allocation calls called by a single thread with a
314973471bf0Spatrick /// static buffer of shared memory.
315073471bf0Spatrick struct AAHeapToShared : public StateWrapper<BooleanState, AbstractAttribute> {
315173471bf0Spatrick   using Base = StateWrapper<BooleanState, AbstractAttribute>;
AAHeapToShared__anon4b47cb5e0111::AAHeapToShared315273471bf0Spatrick   AAHeapToShared(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
315373471bf0Spatrick 
315473471bf0Spatrick   /// Create an abstract attribute view for the position \p IRP.
315573471bf0Spatrick   static AAHeapToShared &createForPosition(const IRPosition &IRP,
315673471bf0Spatrick                                            Attributor &A);
315773471bf0Spatrick 
315873471bf0Spatrick   /// Returns true if HeapToShared conversion is assumed to be possible.
315973471bf0Spatrick   virtual bool isAssumedHeapToShared(CallBase &CB) const = 0;
316073471bf0Spatrick 
316173471bf0Spatrick   /// Returns true if HeapToShared conversion is assumed and the CB is a
316273471bf0Spatrick   /// callsite to a free operation to be removed.
316373471bf0Spatrick   virtual bool isAssumedHeapToSharedRemovedFree(CallBase &CB) const = 0;
316473471bf0Spatrick 
316573471bf0Spatrick   /// See AbstractAttribute::getName().
getName__anon4b47cb5e0111::AAHeapToShared316673471bf0Spatrick   const std::string getName() const override { return "AAHeapToShared"; }
316773471bf0Spatrick 
316873471bf0Spatrick   /// See AbstractAttribute::getIdAddr().
getIdAddr__anon4b47cb5e0111::AAHeapToShared316973471bf0Spatrick   const char *getIdAddr() const override { return &ID; }
317073471bf0Spatrick 
317173471bf0Spatrick   /// This function should return true if the type of the \p AA is
317273471bf0Spatrick   /// AAHeapToShared.
classof__anon4b47cb5e0111::AAHeapToShared317373471bf0Spatrick   static bool classof(const AbstractAttribute *AA) {
317473471bf0Spatrick     return (AA->getIdAddr() == &ID);
317573471bf0Spatrick   }
317673471bf0Spatrick 
317773471bf0Spatrick   /// Unique ID (due to the unique address)
317873471bf0Spatrick   static const char ID;
317973471bf0Spatrick };
318073471bf0Spatrick 
318173471bf0Spatrick struct AAHeapToSharedFunction : public AAHeapToShared {
AAHeapToSharedFunction__anon4b47cb5e0111::AAHeapToSharedFunction318273471bf0Spatrick   AAHeapToSharedFunction(const IRPosition &IRP, Attributor &A)
318373471bf0Spatrick       : AAHeapToShared(IRP, A) {}
318473471bf0Spatrick 
getAsStr__anon4b47cb5e0111::AAHeapToSharedFunction318573471bf0Spatrick   const std::string getAsStr() const override {
318673471bf0Spatrick     return "[AAHeapToShared] " + std::to_string(MallocCalls.size()) +
318773471bf0Spatrick            " malloc calls eligible.";
318873471bf0Spatrick   }
318973471bf0Spatrick 
319073471bf0Spatrick   /// See AbstractAttribute::trackStatistics().
trackStatistics__anon4b47cb5e0111::AAHeapToSharedFunction319173471bf0Spatrick   void trackStatistics() const override {}
319273471bf0Spatrick 
319373471bf0Spatrick   /// This functions finds free calls that will be removed by the
319473471bf0Spatrick   /// HeapToShared transformation.
findPotentialRemovedFreeCalls__anon4b47cb5e0111::AAHeapToSharedFunction319573471bf0Spatrick   void findPotentialRemovedFreeCalls(Attributor &A) {
319673471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
319773471bf0Spatrick     auto &FreeRFI = OMPInfoCache.RFIs[OMPRTL___kmpc_free_shared];
319873471bf0Spatrick 
319973471bf0Spatrick     PotentialRemovedFreeCalls.clear();
320073471bf0Spatrick     // Update free call users of found malloc calls.
320173471bf0Spatrick     for (CallBase *CB : MallocCalls) {
320273471bf0Spatrick       SmallVector<CallBase *, 4> FreeCalls;
320373471bf0Spatrick       for (auto *U : CB->users()) {
320473471bf0Spatrick         CallBase *C = dyn_cast<CallBase>(U);
320573471bf0Spatrick         if (C && C->getCalledFunction() == FreeRFI.Declaration)
320673471bf0Spatrick           FreeCalls.push_back(C);
320773471bf0Spatrick       }
320873471bf0Spatrick 
320973471bf0Spatrick       if (FreeCalls.size() != 1)
321073471bf0Spatrick         continue;
321173471bf0Spatrick 
321273471bf0Spatrick       PotentialRemovedFreeCalls.insert(FreeCalls.front());
321373471bf0Spatrick     }
321473471bf0Spatrick   }
321573471bf0Spatrick 
initialize__anon4b47cb5e0111::AAHeapToSharedFunction321673471bf0Spatrick   void initialize(Attributor &A) override {
3217*d415bd75Srobert     if (DisableOpenMPOptDeglobalization) {
3218*d415bd75Srobert       indicatePessimisticFixpoint();
3219*d415bd75Srobert       return;
3220*d415bd75Srobert     }
3221*d415bd75Srobert 
322273471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
322373471bf0Spatrick     auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
3224*d415bd75Srobert     if (!RFI.Declaration)
3225*d415bd75Srobert       return;
322673471bf0Spatrick 
3227*d415bd75Srobert     Attributor::SimplifictionCallbackTy SCB =
3228*d415bd75Srobert         [](const IRPosition &, const AbstractAttribute *,
3229*d415bd75Srobert            bool &) -> std::optional<Value *> { return nullptr; };
3230*d415bd75Srobert 
3231*d415bd75Srobert     Function *F = getAnchorScope();
323273471bf0Spatrick     for (User *U : RFI.Declaration->users())
3233*d415bd75Srobert       if (CallBase *CB = dyn_cast<CallBase>(U)) {
3234*d415bd75Srobert         if (CB->getFunction() != F)
3235*d415bd75Srobert           continue;
323673471bf0Spatrick         MallocCalls.insert(CB);
3237*d415bd75Srobert         A.registerSimplificationCallback(IRPosition::callsite_returned(*CB),
3238*d415bd75Srobert                                          SCB);
3239*d415bd75Srobert       }
324073471bf0Spatrick 
324173471bf0Spatrick     findPotentialRemovedFreeCalls(A);
324273471bf0Spatrick   }
324373471bf0Spatrick 
isAssumedHeapToShared__anon4b47cb5e0111::AAHeapToSharedFunction324473471bf0Spatrick   bool isAssumedHeapToShared(CallBase &CB) const override {
324573471bf0Spatrick     return isValidState() && MallocCalls.count(&CB);
324673471bf0Spatrick   }
324773471bf0Spatrick 
isAssumedHeapToSharedRemovedFree__anon4b47cb5e0111::AAHeapToSharedFunction324873471bf0Spatrick   bool isAssumedHeapToSharedRemovedFree(CallBase &CB) const override {
324973471bf0Spatrick     return isValidState() && PotentialRemovedFreeCalls.count(&CB);
325073471bf0Spatrick   }
325173471bf0Spatrick 
manifest__anon4b47cb5e0111::AAHeapToSharedFunction325273471bf0Spatrick   ChangeStatus manifest(Attributor &A) override {
325373471bf0Spatrick     if (MallocCalls.empty())
325473471bf0Spatrick       return ChangeStatus::UNCHANGED;
325573471bf0Spatrick 
325673471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
325773471bf0Spatrick     auto &FreeCall = OMPInfoCache.RFIs[OMPRTL___kmpc_free_shared];
325873471bf0Spatrick 
325973471bf0Spatrick     Function *F = getAnchorScope();
326073471bf0Spatrick     auto *HS = A.lookupAAFor<AAHeapToStack>(IRPosition::function(*F), this,
326173471bf0Spatrick                                             DepClassTy::OPTIONAL);
326273471bf0Spatrick 
326373471bf0Spatrick     ChangeStatus Changed = ChangeStatus::UNCHANGED;
326473471bf0Spatrick     for (CallBase *CB : MallocCalls) {
326573471bf0Spatrick       // Skip replacing this if HeapToStack has already claimed it.
326673471bf0Spatrick       if (HS && HS->isAssumedHeapToStack(*CB))
326773471bf0Spatrick         continue;
326873471bf0Spatrick 
326973471bf0Spatrick       // Find the unique free call to remove it.
327073471bf0Spatrick       SmallVector<CallBase *, 4> FreeCalls;
327173471bf0Spatrick       for (auto *U : CB->users()) {
327273471bf0Spatrick         CallBase *C = dyn_cast<CallBase>(U);
327373471bf0Spatrick         if (C && C->getCalledFunction() == FreeCall.Declaration)
327473471bf0Spatrick           FreeCalls.push_back(C);
327573471bf0Spatrick       }
327673471bf0Spatrick       if (FreeCalls.size() != 1)
327773471bf0Spatrick         continue;
327873471bf0Spatrick 
3279*d415bd75Srobert       auto *AllocSize = cast<ConstantInt>(CB->getArgOperand(0));
328073471bf0Spatrick 
3281*d415bd75Srobert       if (AllocSize->getZExtValue() + SharedMemoryUsed > SharedMemoryLimit) {
3282*d415bd75Srobert         LLVM_DEBUG(dbgs() << TAG << "Cannot replace call " << *CB
3283*d415bd75Srobert                           << " with shared memory."
3284*d415bd75Srobert                           << " Shared memory usage is limited to "
3285*d415bd75Srobert                           << SharedMemoryLimit << " bytes\n");
3286*d415bd75Srobert         continue;
3287*d415bd75Srobert       }
3288*d415bd75Srobert 
3289*d415bd75Srobert       LLVM_DEBUG(dbgs() << TAG << "Replace globalization call " << *CB
3290*d415bd75Srobert                         << " with " << AllocSize->getZExtValue()
329173471bf0Spatrick                         << " bytes of shared memory\n");
329273471bf0Spatrick 
329373471bf0Spatrick       // Create a new shared memory buffer of the same size as the allocation
329473471bf0Spatrick       // and replace all the uses of the original allocation with it.
329573471bf0Spatrick       Module *M = CB->getModule();
329673471bf0Spatrick       Type *Int8Ty = Type::getInt8Ty(M->getContext());
329773471bf0Spatrick       Type *Int8ArrTy = ArrayType::get(Int8Ty, AllocSize->getZExtValue());
329873471bf0Spatrick       auto *SharedMem = new GlobalVariable(
329973471bf0Spatrick           *M, Int8ArrTy, /* IsConstant */ false, GlobalValue::InternalLinkage,
3300*d415bd75Srobert           UndefValue::get(Int8ArrTy), CB->getName() + "_shared", nullptr,
330173471bf0Spatrick           GlobalValue::NotThreadLocal,
330273471bf0Spatrick           static_cast<unsigned>(AddressSpace::Shared));
330373471bf0Spatrick       auto *NewBuffer =
330473471bf0Spatrick           ConstantExpr::getPointerCast(SharedMem, Int8Ty->getPointerTo());
330573471bf0Spatrick 
330673471bf0Spatrick       auto Remark = [&](OptimizationRemark OR) {
330773471bf0Spatrick         return OR << "Replaced globalized variable with "
330873471bf0Spatrick                   << ore::NV("SharedMemory", AllocSize->getZExtValue())
330973471bf0Spatrick                   << ((AllocSize->getZExtValue() != 1) ? " bytes " : " byte ")
331073471bf0Spatrick                   << "of shared memory.";
331173471bf0Spatrick       };
331273471bf0Spatrick       A.emitRemark<OptimizationRemark>(CB, "OMP111", Remark);
331373471bf0Spatrick 
3314*d415bd75Srobert       MaybeAlign Alignment = CB->getRetAlign();
3315*d415bd75Srobert       assert(Alignment &&
3316*d415bd75Srobert              "HeapToShared on allocation without alignment attribute");
3317*d415bd75Srobert       SharedMem->setAlignment(MaybeAlign(Alignment));
331873471bf0Spatrick 
3319*d415bd75Srobert       A.changeAfterManifest(IRPosition::callsite_returned(*CB), *NewBuffer);
332073471bf0Spatrick       A.deleteAfterManifest(*CB);
332173471bf0Spatrick       A.deleteAfterManifest(*FreeCalls.front());
332273471bf0Spatrick 
3323*d415bd75Srobert       SharedMemoryUsed += AllocSize->getZExtValue();
3324*d415bd75Srobert       NumBytesMovedToSharedMemory = SharedMemoryUsed;
332573471bf0Spatrick       Changed = ChangeStatus::CHANGED;
332673471bf0Spatrick     }
332773471bf0Spatrick 
332873471bf0Spatrick     return Changed;
332973471bf0Spatrick   }
333073471bf0Spatrick 
updateImpl__anon4b47cb5e0111::AAHeapToSharedFunction333173471bf0Spatrick   ChangeStatus updateImpl(Attributor &A) override {
3332*d415bd75Srobert     if (MallocCalls.empty())
3333*d415bd75Srobert       return indicatePessimisticFixpoint();
333473471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
333573471bf0Spatrick     auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
3336*d415bd75Srobert     if (!RFI.Declaration)
3337*d415bd75Srobert       return ChangeStatus::UNCHANGED;
3338*d415bd75Srobert 
333973471bf0Spatrick     Function *F = getAnchorScope();
334073471bf0Spatrick 
334173471bf0Spatrick     auto NumMallocCalls = MallocCalls.size();
334273471bf0Spatrick 
334373471bf0Spatrick     // Only consider malloc calls executed by a single thread with a constant.
334473471bf0Spatrick     for (User *U : RFI.Declaration->users()) {
3345*d415bd75Srobert       if (CallBase *CB = dyn_cast<CallBase>(U)) {
3346*d415bd75Srobert         if (CB->getCaller() != F)
3347*d415bd75Srobert           continue;
3348*d415bd75Srobert         if (!MallocCalls.count(CB))
3349*d415bd75Srobert           continue;
3350*d415bd75Srobert         if (!isa<ConstantInt>(CB->getArgOperand(0))) {
3351*d415bd75Srobert           MallocCalls.remove(CB);
3352*d415bd75Srobert           continue;
3353*d415bd75Srobert         }
335473471bf0Spatrick         const auto &ED = A.getAAFor<AAExecutionDomain>(
335573471bf0Spatrick             *this, IRPosition::function(*F), DepClassTy::REQUIRED);
3356*d415bd75Srobert         if (!ED.isExecutedByInitialThreadOnly(*CB))
3357*d415bd75Srobert           MallocCalls.remove(CB);
3358*d415bd75Srobert       }
335973471bf0Spatrick     }
336073471bf0Spatrick 
336173471bf0Spatrick     findPotentialRemovedFreeCalls(A);
336273471bf0Spatrick 
336373471bf0Spatrick     if (NumMallocCalls != MallocCalls.size())
336473471bf0Spatrick       return ChangeStatus::CHANGED;
336573471bf0Spatrick 
336673471bf0Spatrick     return ChangeStatus::UNCHANGED;
336773471bf0Spatrick   }
336873471bf0Spatrick 
336973471bf0Spatrick   /// Collection of all malloc calls in a function.
3370*d415bd75Srobert   SmallSetVector<CallBase *, 4> MallocCalls;
337173471bf0Spatrick   /// Collection of potentially removed free calls in a function.
337273471bf0Spatrick   SmallPtrSet<CallBase *, 4> PotentialRemovedFreeCalls;
3373*d415bd75Srobert   /// The total amount of shared memory that has been used for HeapToShared.
3374*d415bd75Srobert   unsigned SharedMemoryUsed = 0;
337573471bf0Spatrick };
337673471bf0Spatrick 
337773471bf0Spatrick struct AAKernelInfo : public StateWrapper<KernelInfoState, AbstractAttribute> {
337873471bf0Spatrick   using Base = StateWrapper<KernelInfoState, AbstractAttribute>;
AAKernelInfo__anon4b47cb5e0111::AAKernelInfo337973471bf0Spatrick   AAKernelInfo(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
338073471bf0Spatrick 
338173471bf0Spatrick   /// Statistics are tracked as part of manifest for now.
trackStatistics__anon4b47cb5e0111::AAKernelInfo338273471bf0Spatrick   void trackStatistics() const override {}
338373471bf0Spatrick 
338473471bf0Spatrick   /// See AbstractAttribute::getAsStr()
getAsStr__anon4b47cb5e0111::AAKernelInfo338573471bf0Spatrick   const std::string getAsStr() const override {
338673471bf0Spatrick     if (!isValidState())
338773471bf0Spatrick       return "<invalid>";
338873471bf0Spatrick     return std::string(SPMDCompatibilityTracker.isAssumed() ? "SPMD"
338973471bf0Spatrick                                                             : "generic") +
339073471bf0Spatrick            std::string(SPMDCompatibilityTracker.isAtFixpoint() ? " [FIX]"
339173471bf0Spatrick                                                                : "") +
339273471bf0Spatrick            std::string(" #PRs: ") +
3393*d415bd75Srobert            (ReachedKnownParallelRegions.isValidState()
3394*d415bd75Srobert                 ? std::to_string(ReachedKnownParallelRegions.size())
3395*d415bd75Srobert                 : "<invalid>") +
339673471bf0Spatrick            ", #Unknown PRs: " +
3397*d415bd75Srobert            (ReachedUnknownParallelRegions.isValidState()
3398*d415bd75Srobert                 ? std::to_string(ReachedUnknownParallelRegions.size())
3399*d415bd75Srobert                 : "<invalid>") +
3400*d415bd75Srobert            ", #Reaching Kernels: " +
3401*d415bd75Srobert            (ReachingKernelEntries.isValidState()
3402*d415bd75Srobert                 ? std::to_string(ReachingKernelEntries.size())
3403*d415bd75Srobert                 : "<invalid>") +
3404*d415bd75Srobert            ", #ParLevels: " +
3405*d415bd75Srobert            (ParallelLevels.isValidState()
3406*d415bd75Srobert                 ? std::to_string(ParallelLevels.size())
3407*d415bd75Srobert                 : "<invalid>");
340873471bf0Spatrick   }
340973471bf0Spatrick 
341073471bf0Spatrick   /// Create an abstract attribute biew for the position \p IRP.
341173471bf0Spatrick   static AAKernelInfo &createForPosition(const IRPosition &IRP, Attributor &A);
341273471bf0Spatrick 
341373471bf0Spatrick   /// See AbstractAttribute::getName()
getName__anon4b47cb5e0111::AAKernelInfo341473471bf0Spatrick   const std::string getName() const override { return "AAKernelInfo"; }
341573471bf0Spatrick 
341673471bf0Spatrick   /// See AbstractAttribute::getIdAddr()
getIdAddr__anon4b47cb5e0111::AAKernelInfo341773471bf0Spatrick   const char *getIdAddr() const override { return &ID; }
341873471bf0Spatrick 
341973471bf0Spatrick   /// This function should return true if the type of the \p AA is AAKernelInfo
classof__anon4b47cb5e0111::AAKernelInfo342073471bf0Spatrick   static bool classof(const AbstractAttribute *AA) {
342173471bf0Spatrick     return (AA->getIdAddr() == &ID);
342273471bf0Spatrick   }
342373471bf0Spatrick 
342473471bf0Spatrick   static const char ID;
342573471bf0Spatrick };
342673471bf0Spatrick 
342773471bf0Spatrick /// The function kernel info abstract attribute, basically, what can we say
342873471bf0Spatrick /// about a function with regards to the KernelInfoState.
342973471bf0Spatrick struct AAKernelInfoFunction : AAKernelInfo {
AAKernelInfoFunction__anon4b47cb5e0111::AAKernelInfoFunction343073471bf0Spatrick   AAKernelInfoFunction(const IRPosition &IRP, Attributor &A)
343173471bf0Spatrick       : AAKernelInfo(IRP, A) {}
343273471bf0Spatrick 
3433*d415bd75Srobert   SmallPtrSet<Instruction *, 4> GuardedInstructions;
3434*d415bd75Srobert 
getGuardedInstructions__anon4b47cb5e0111::AAKernelInfoFunction3435*d415bd75Srobert   SmallPtrSetImpl<Instruction *> &getGuardedInstructions() {
3436*d415bd75Srobert     return GuardedInstructions;
3437*d415bd75Srobert   }
3438*d415bd75Srobert 
343973471bf0Spatrick   /// See AbstractAttribute::initialize(...).
initialize__anon4b47cb5e0111::AAKernelInfoFunction344073471bf0Spatrick   void initialize(Attributor &A) override {
344173471bf0Spatrick     // This is a high-level transform that might change the constant arguments
344273471bf0Spatrick     // of the init and dinit calls. We need to tell the Attributor about this
344373471bf0Spatrick     // to avoid other parts using the current constant value for simpliication.
344473471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
344573471bf0Spatrick 
344673471bf0Spatrick     Function *Fn = getAnchorScope();
344773471bf0Spatrick 
344873471bf0Spatrick     OMPInformationCache::RuntimeFunctionInfo &InitRFI =
344973471bf0Spatrick         OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
345073471bf0Spatrick     OMPInformationCache::RuntimeFunctionInfo &DeinitRFI =
345173471bf0Spatrick         OMPInfoCache.RFIs[OMPRTL___kmpc_target_deinit];
345273471bf0Spatrick 
345373471bf0Spatrick     // For kernels we perform more initialization work, first we find the init
345473471bf0Spatrick     // and deinit calls.
345573471bf0Spatrick     auto StoreCallBase = [](Use &U,
345673471bf0Spatrick                             OMPInformationCache::RuntimeFunctionInfo &RFI,
345773471bf0Spatrick                             CallBase *&Storage) {
345873471bf0Spatrick       CallBase *CB = OpenMPOpt::getCallIfRegularCall(U, &RFI);
345973471bf0Spatrick       assert(CB &&
346073471bf0Spatrick              "Unexpected use of __kmpc_target_init or __kmpc_target_deinit!");
346173471bf0Spatrick       assert(!Storage &&
346273471bf0Spatrick              "Multiple uses of __kmpc_target_init or __kmpc_target_deinit!");
346373471bf0Spatrick       Storage = CB;
346473471bf0Spatrick       return false;
346573471bf0Spatrick     };
346673471bf0Spatrick     InitRFI.foreachUse(
346773471bf0Spatrick         [&](Use &U, Function &) {
346873471bf0Spatrick           StoreCallBase(U, InitRFI, KernelInitCB);
346973471bf0Spatrick           return false;
347073471bf0Spatrick         },
347173471bf0Spatrick         Fn);
347273471bf0Spatrick     DeinitRFI.foreachUse(
347373471bf0Spatrick         [&](Use &U, Function &) {
347473471bf0Spatrick           StoreCallBase(U, DeinitRFI, KernelDeinitCB);
347573471bf0Spatrick           return false;
347673471bf0Spatrick         },
347773471bf0Spatrick         Fn);
347873471bf0Spatrick 
3479*d415bd75Srobert     // Ignore kernels without initializers such as global constructors.
3480*d415bd75Srobert     if (!KernelInitCB || !KernelDeinitCB)
3481*d415bd75Srobert       return;
3482*d415bd75Srobert 
3483*d415bd75Srobert     // Add itself to the reaching kernel and set IsKernelEntry.
3484*d415bd75Srobert     ReachingKernelEntries.insert(Fn);
3485*d415bd75Srobert     IsKernelEntry = true;
348673471bf0Spatrick 
348773471bf0Spatrick     // For kernels we might need to initialize/finalize the IsSPMD state and
348873471bf0Spatrick     // we need to register a simplification callback so that the Attributor
348973471bf0Spatrick     // knows the constant arguments to __kmpc_target_init and
349073471bf0Spatrick     // __kmpc_target_deinit might actually change.
349173471bf0Spatrick 
349273471bf0Spatrick     Attributor::SimplifictionCallbackTy StateMachineSimplifyCB =
349373471bf0Spatrick         [&](const IRPosition &IRP, const AbstractAttribute *AA,
3494*d415bd75Srobert             bool &UsedAssumedInformation) -> std::optional<Value *> {
349573471bf0Spatrick       // IRP represents the "use generic state machine" argument of an
349673471bf0Spatrick       // __kmpc_target_init call. We will answer this one with the internal
349773471bf0Spatrick       // state. As long as we are not in an invalid state, we will create a
349873471bf0Spatrick       // custom state machine so the value should be a `i1 false`. If we are
349973471bf0Spatrick       // in an invalid state, we won't change the value that is in the IR.
3500*d415bd75Srobert       if (!ReachedKnownParallelRegions.isValidState())
3501*d415bd75Srobert         return nullptr;
3502*d415bd75Srobert       // If we have disabled state machine rewrites, don't make a custom one.
3503*d415bd75Srobert       if (DisableOpenMPOptStateMachineRewrite)
350473471bf0Spatrick         return nullptr;
350573471bf0Spatrick       if (AA)
350673471bf0Spatrick         A.recordDependence(*this, *AA, DepClassTy::OPTIONAL);
350773471bf0Spatrick       UsedAssumedInformation = !isAtFixpoint();
350873471bf0Spatrick       auto *FalseVal =
3509*d415bd75Srobert           ConstantInt::getBool(IRP.getAnchorValue().getContext(), false);
351073471bf0Spatrick       return FalseVal;
351173471bf0Spatrick     };
351273471bf0Spatrick 
3513*d415bd75Srobert     Attributor::SimplifictionCallbackTy ModeSimplifyCB =
351473471bf0Spatrick         [&](const IRPosition &IRP, const AbstractAttribute *AA,
3515*d415bd75Srobert             bool &UsedAssumedInformation) -> std::optional<Value *> {
351673471bf0Spatrick       // IRP represents the "SPMDCompatibilityTracker" argument of an
351773471bf0Spatrick       // __kmpc_target_init or
351873471bf0Spatrick       // __kmpc_target_deinit call. We will answer this one with the internal
351973471bf0Spatrick       // state.
352073471bf0Spatrick       if (!SPMDCompatibilityTracker.isValidState())
352173471bf0Spatrick         return nullptr;
352273471bf0Spatrick       if (!SPMDCompatibilityTracker.isAtFixpoint()) {
352373471bf0Spatrick         if (AA)
352473471bf0Spatrick           A.recordDependence(*this, *AA, DepClassTy::OPTIONAL);
352573471bf0Spatrick         UsedAssumedInformation = true;
352673471bf0Spatrick       } else {
352773471bf0Spatrick         UsedAssumedInformation = false;
352873471bf0Spatrick       }
3529*d415bd75Srobert       auto *Val = ConstantInt::getSigned(
3530*d415bd75Srobert           IntegerType::getInt8Ty(IRP.getAnchorValue().getContext()),
3531*d415bd75Srobert           SPMDCompatibilityTracker.isAssumed() ? OMP_TGT_EXEC_MODE_SPMD
3532*d415bd75Srobert                                                : OMP_TGT_EXEC_MODE_GENERIC);
353373471bf0Spatrick       return Val;
353473471bf0Spatrick     };
353573471bf0Spatrick 
3536*d415bd75Srobert     constexpr const int InitModeArgNo = 1;
3537*d415bd75Srobert     constexpr const int DeinitModeArgNo = 1;
353873471bf0Spatrick     constexpr const int InitUseStateMachineArgNo = 2;
353973471bf0Spatrick     A.registerSimplificationCallback(
354073471bf0Spatrick         IRPosition::callsite_argument(*KernelInitCB, InitUseStateMachineArgNo),
354173471bf0Spatrick         StateMachineSimplifyCB);
354273471bf0Spatrick     A.registerSimplificationCallback(
3543*d415bd75Srobert         IRPosition::callsite_argument(*KernelInitCB, InitModeArgNo),
3544*d415bd75Srobert         ModeSimplifyCB);
354573471bf0Spatrick     A.registerSimplificationCallback(
3546*d415bd75Srobert         IRPosition::callsite_argument(*KernelDeinitCB, DeinitModeArgNo),
3547*d415bd75Srobert         ModeSimplifyCB);
354873471bf0Spatrick 
354973471bf0Spatrick     // Check if we know we are in SPMD-mode already.
3550*d415bd75Srobert     ConstantInt *ModeArg =
3551*d415bd75Srobert         dyn_cast<ConstantInt>(KernelInitCB->getArgOperand(InitModeArgNo));
3552*d415bd75Srobert     if (ModeArg && (ModeArg->getSExtValue() & OMP_TGT_EXEC_MODE_SPMD))
355373471bf0Spatrick       SPMDCompatibilityTracker.indicateOptimisticFixpoint();
3554*d415bd75Srobert     // This is a generic region but SPMDization is disabled so stop tracking.
3555*d415bd75Srobert     else if (DisableOpenMPOptSPMDization)
3556*d415bd75Srobert       SPMDCompatibilityTracker.indicatePessimisticFixpoint();
3557*d415bd75Srobert 
3558*d415bd75Srobert     // Register virtual uses of functions we might need to preserve.
3559*d415bd75Srobert     auto RegisterVirtualUse = [&](RuntimeFunction RFKind,
3560*d415bd75Srobert                                   Attributor::VirtualUseCallbackTy &CB) {
3561*d415bd75Srobert       if (!OMPInfoCache.RFIs[RFKind].Declaration)
3562*d415bd75Srobert         return;
3563*d415bd75Srobert       A.registerVirtualUseCallback(*OMPInfoCache.RFIs[RFKind].Declaration, CB);
3564*d415bd75Srobert     };
3565*d415bd75Srobert 
3566*d415bd75Srobert     // Add a dependence to ensure updates if the state changes.
3567*d415bd75Srobert     auto AddDependence = [](Attributor &A, const AAKernelInfo *KI,
3568*d415bd75Srobert                             const AbstractAttribute *QueryingAA) {
3569*d415bd75Srobert       if (QueryingAA) {
3570*d415bd75Srobert         A.recordDependence(*KI, *QueryingAA, DepClassTy::OPTIONAL);
3571*d415bd75Srobert       }
3572*d415bd75Srobert       return true;
3573*d415bd75Srobert     };
3574*d415bd75Srobert 
3575*d415bd75Srobert     Attributor::VirtualUseCallbackTy CustomStateMachineUseCB =
3576*d415bd75Srobert         [&](Attributor &A, const AbstractAttribute *QueryingAA) {
3577*d415bd75Srobert           // Whenever we create a custom state machine we will insert calls to
3578*d415bd75Srobert           // __kmpc_get_hardware_num_threads_in_block,
3579*d415bd75Srobert           // __kmpc_get_warp_size,
3580*d415bd75Srobert           // __kmpc_barrier_simple_generic,
3581*d415bd75Srobert           // __kmpc_kernel_parallel, and
3582*d415bd75Srobert           // __kmpc_kernel_end_parallel.
3583*d415bd75Srobert           // Not needed if we are on track for SPMDzation.
3584*d415bd75Srobert           if (SPMDCompatibilityTracker.isValidState())
3585*d415bd75Srobert             return AddDependence(A, this, QueryingAA);
3586*d415bd75Srobert           // Not needed if we can't rewrite due to an invalid state.
3587*d415bd75Srobert           if (!ReachedKnownParallelRegions.isValidState())
3588*d415bd75Srobert             return AddDependence(A, this, QueryingAA);
3589*d415bd75Srobert           return false;
3590*d415bd75Srobert         };
3591*d415bd75Srobert 
3592*d415bd75Srobert     // Not needed if we are pre-runtime merge.
3593*d415bd75Srobert     if (!KernelInitCB->getCalledFunction()->isDeclaration()) {
3594*d415bd75Srobert       RegisterVirtualUse(OMPRTL___kmpc_get_hardware_num_threads_in_block,
3595*d415bd75Srobert                          CustomStateMachineUseCB);
3596*d415bd75Srobert       RegisterVirtualUse(OMPRTL___kmpc_get_warp_size, CustomStateMachineUseCB);
3597*d415bd75Srobert       RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_generic,
3598*d415bd75Srobert                          CustomStateMachineUseCB);
3599*d415bd75Srobert       RegisterVirtualUse(OMPRTL___kmpc_kernel_parallel,
3600*d415bd75Srobert                          CustomStateMachineUseCB);
3601*d415bd75Srobert       RegisterVirtualUse(OMPRTL___kmpc_kernel_end_parallel,
3602*d415bd75Srobert                          CustomStateMachineUseCB);
3603*d415bd75Srobert     }
3604*d415bd75Srobert 
3605*d415bd75Srobert     // If we do not perform SPMDzation we do not need the virtual uses below.
3606*d415bd75Srobert     if (SPMDCompatibilityTracker.isAtFixpoint())
3607*d415bd75Srobert       return;
3608*d415bd75Srobert 
3609*d415bd75Srobert     Attributor::VirtualUseCallbackTy HWThreadIdUseCB =
3610*d415bd75Srobert         [&](Attributor &A, const AbstractAttribute *QueryingAA) {
3611*d415bd75Srobert           // Whenever we perform SPMDzation we will insert
3612*d415bd75Srobert           // __kmpc_get_hardware_thread_id_in_block calls.
3613*d415bd75Srobert           if (!SPMDCompatibilityTracker.isValidState())
3614*d415bd75Srobert             return AddDependence(A, this, QueryingAA);
3615*d415bd75Srobert           return false;
3616*d415bd75Srobert         };
3617*d415bd75Srobert     RegisterVirtualUse(OMPRTL___kmpc_get_hardware_thread_id_in_block,
3618*d415bd75Srobert                        HWThreadIdUseCB);
3619*d415bd75Srobert 
3620*d415bd75Srobert     Attributor::VirtualUseCallbackTy SPMDBarrierUseCB =
3621*d415bd75Srobert         [&](Attributor &A, const AbstractAttribute *QueryingAA) {
3622*d415bd75Srobert           // Whenever we perform SPMDzation with guarding we will insert
3623*d415bd75Srobert           // __kmpc_simple_barrier_spmd calls. If SPMDzation failed, there is
3624*d415bd75Srobert           // nothing to guard, or there are no parallel regions, we don't need
3625*d415bd75Srobert           // the calls.
3626*d415bd75Srobert           if (!SPMDCompatibilityTracker.isValidState())
3627*d415bd75Srobert             return AddDependence(A, this, QueryingAA);
3628*d415bd75Srobert           if (SPMDCompatibilityTracker.empty())
3629*d415bd75Srobert             return AddDependence(A, this, QueryingAA);
3630*d415bd75Srobert           if (!mayContainParallelRegion())
3631*d415bd75Srobert             return AddDependence(A, this, QueryingAA);
3632*d415bd75Srobert           return false;
3633*d415bd75Srobert         };
3634*d415bd75Srobert     RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_spmd, SPMDBarrierUseCB);
3635*d415bd75Srobert   }
3636*d415bd75Srobert 
3637*d415bd75Srobert   /// Sanitize the string \p S such that it is a suitable global symbol name.
sanitizeForGlobalName__anon4b47cb5e0111::AAKernelInfoFunction3638*d415bd75Srobert   static std::string sanitizeForGlobalName(std::string S) {
3639*d415bd75Srobert     std::replace_if(
3640*d415bd75Srobert         S.begin(), S.end(),
3641*d415bd75Srobert         [](const char C) {
3642*d415bd75Srobert           return !((C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') ||
3643*d415bd75Srobert                    (C >= '0' && C <= '9') || C == '_');
3644*d415bd75Srobert         },
3645*d415bd75Srobert         '.');
3646*d415bd75Srobert     return S;
364773471bf0Spatrick   }
364873471bf0Spatrick 
364973471bf0Spatrick   /// Modify the IR based on the KernelInfoState as the fixpoint iteration is
365073471bf0Spatrick   /// finished now.
manifest__anon4b47cb5e0111::AAKernelInfoFunction365173471bf0Spatrick   ChangeStatus manifest(Attributor &A) override {
365273471bf0Spatrick     // If we are not looking at a kernel with __kmpc_target_init and
365373471bf0Spatrick     // __kmpc_target_deinit call we cannot actually manifest the information.
365473471bf0Spatrick     if (!KernelInitCB || !KernelDeinitCB)
365573471bf0Spatrick       return ChangeStatus::UNCHANGED;
365673471bf0Spatrick 
3657*d415bd75Srobert     /// Insert nested Parallelism global variable
3658*d415bd75Srobert     Function *Kernel = getAnchorScope();
3659*d415bd75Srobert     Module &M = *Kernel->getParent();
3660*d415bd75Srobert     Type *Int8Ty = Type::getInt8Ty(M.getContext());
3661*d415bd75Srobert     new GlobalVariable(M, Int8Ty, /* isConstant */ true,
3662*d415bd75Srobert                        GlobalValue::WeakAnyLinkage,
3663*d415bd75Srobert                        ConstantInt::get(Int8Ty, NestedParallelism ? 1 : 0),
3664*d415bd75Srobert                        Kernel->getName() + "_nested_parallelism");
366573471bf0Spatrick 
366673471bf0Spatrick     // If we can we change the execution mode to SPMD-mode otherwise we build a
366773471bf0Spatrick     // custom state machine.
3668*d415bd75Srobert     ChangeStatus Changed = ChangeStatus::UNCHANGED;
3669*d415bd75Srobert     if (!changeToSPMDMode(A, Changed)) {
3670*d415bd75Srobert       if (!KernelInitCB->getCalledFunction()->isDeclaration())
3671*d415bd75Srobert         return buildCustomStateMachine(A);
367273471bf0Spatrick     }
367373471bf0Spatrick 
3674*d415bd75Srobert     return Changed;
3675*d415bd75Srobert   }
3676*d415bd75Srobert 
insertInstructionGuardsHelper__anon4b47cb5e0111::AAKernelInfoFunction3677*d415bd75Srobert   void insertInstructionGuardsHelper(Attributor &A) {
367873471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
367973471bf0Spatrick 
3680*d415bd75Srobert     auto CreateGuardedRegion = [&](Instruction *RegionStartI,
3681*d415bd75Srobert                                    Instruction *RegionEndI) {
3682*d415bd75Srobert       LoopInfo *LI = nullptr;
3683*d415bd75Srobert       DominatorTree *DT = nullptr;
3684*d415bd75Srobert       MemorySSAUpdater *MSU = nullptr;
3685*d415bd75Srobert       using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3686*d415bd75Srobert 
3687*d415bd75Srobert       BasicBlock *ParentBB = RegionStartI->getParent();
3688*d415bd75Srobert       Function *Fn = ParentBB->getParent();
3689*d415bd75Srobert       Module &M = *Fn->getParent();
3690*d415bd75Srobert 
3691*d415bd75Srobert       // Create all the blocks and logic.
3692*d415bd75Srobert       // ParentBB:
3693*d415bd75Srobert       //    goto RegionCheckTidBB
3694*d415bd75Srobert       // RegionCheckTidBB:
3695*d415bd75Srobert       //    Tid = __kmpc_hardware_thread_id()
3696*d415bd75Srobert       //    if (Tid != 0)
3697*d415bd75Srobert       //        goto RegionBarrierBB
3698*d415bd75Srobert       // RegionStartBB:
3699*d415bd75Srobert       //    <execute instructions guarded>
3700*d415bd75Srobert       //    goto RegionEndBB
3701*d415bd75Srobert       // RegionEndBB:
3702*d415bd75Srobert       //    <store escaping values to shared mem>
3703*d415bd75Srobert       //    goto RegionBarrierBB
3704*d415bd75Srobert       //  RegionBarrierBB:
3705*d415bd75Srobert       //    __kmpc_simple_barrier_spmd()
3706*d415bd75Srobert       //    // second barrier is omitted if lacking escaping values.
3707*d415bd75Srobert       //    <load escaping values from shared mem>
3708*d415bd75Srobert       //    __kmpc_simple_barrier_spmd()
3709*d415bd75Srobert       //    goto RegionExitBB
3710*d415bd75Srobert       // RegionExitBB:
3711*d415bd75Srobert       //    <execute rest of instructions>
3712*d415bd75Srobert 
3713*d415bd75Srobert       BasicBlock *RegionEndBB = SplitBlock(ParentBB, RegionEndI->getNextNode(),
3714*d415bd75Srobert                                            DT, LI, MSU, "region.guarded.end");
3715*d415bd75Srobert       BasicBlock *RegionBarrierBB =
3716*d415bd75Srobert           SplitBlock(RegionEndBB, &*RegionEndBB->getFirstInsertionPt(), DT, LI,
3717*d415bd75Srobert                      MSU, "region.barrier");
3718*d415bd75Srobert       BasicBlock *RegionExitBB =
3719*d415bd75Srobert           SplitBlock(RegionBarrierBB, &*RegionBarrierBB->getFirstInsertionPt(),
3720*d415bd75Srobert                      DT, LI, MSU, "region.exit");
3721*d415bd75Srobert       BasicBlock *RegionStartBB =
3722*d415bd75Srobert           SplitBlock(ParentBB, RegionStartI, DT, LI, MSU, "region.guarded");
3723*d415bd75Srobert 
3724*d415bd75Srobert       assert(ParentBB->getUniqueSuccessor() == RegionStartBB &&
3725*d415bd75Srobert              "Expected a different CFG");
3726*d415bd75Srobert 
3727*d415bd75Srobert       BasicBlock *RegionCheckTidBB = SplitBlock(
3728*d415bd75Srobert           ParentBB, ParentBB->getTerminator(), DT, LI, MSU, "region.check.tid");
3729*d415bd75Srobert 
3730*d415bd75Srobert       // Register basic blocks with the Attributor.
3731*d415bd75Srobert       A.registerManifestAddedBasicBlock(*RegionEndBB);
3732*d415bd75Srobert       A.registerManifestAddedBasicBlock(*RegionBarrierBB);
3733*d415bd75Srobert       A.registerManifestAddedBasicBlock(*RegionExitBB);
3734*d415bd75Srobert       A.registerManifestAddedBasicBlock(*RegionStartBB);
3735*d415bd75Srobert       A.registerManifestAddedBasicBlock(*RegionCheckTidBB);
3736*d415bd75Srobert 
3737*d415bd75Srobert       bool HasBroadcastValues = false;
3738*d415bd75Srobert       // Find escaping outputs from the guarded region to outside users and
3739*d415bd75Srobert       // broadcast their values to them.
3740*d415bd75Srobert       for (Instruction &I : *RegionStartBB) {
3741*d415bd75Srobert         SmallPtrSet<Instruction *, 4> OutsideUsers;
3742*d415bd75Srobert         for (User *Usr : I.users()) {
3743*d415bd75Srobert           Instruction &UsrI = *cast<Instruction>(Usr);
3744*d415bd75Srobert           if (UsrI.getParent() != RegionStartBB)
3745*d415bd75Srobert             OutsideUsers.insert(&UsrI);
3746*d415bd75Srobert         }
3747*d415bd75Srobert 
3748*d415bd75Srobert         if (OutsideUsers.empty())
3749*d415bd75Srobert           continue;
3750*d415bd75Srobert 
3751*d415bd75Srobert         HasBroadcastValues = true;
3752*d415bd75Srobert 
3753*d415bd75Srobert         // Emit a global variable in shared memory to store the broadcasted
3754*d415bd75Srobert         // value.
3755*d415bd75Srobert         auto *SharedMem = new GlobalVariable(
3756*d415bd75Srobert             M, I.getType(), /* IsConstant */ false,
3757*d415bd75Srobert             GlobalValue::InternalLinkage, UndefValue::get(I.getType()),
3758*d415bd75Srobert             sanitizeForGlobalName(
3759*d415bd75Srobert                 (I.getName() + ".guarded.output.alloc").str()),
3760*d415bd75Srobert             nullptr, GlobalValue::NotThreadLocal,
3761*d415bd75Srobert             static_cast<unsigned>(AddressSpace::Shared));
3762*d415bd75Srobert 
3763*d415bd75Srobert         // Emit a store instruction to update the value.
3764*d415bd75Srobert         new StoreInst(&I, SharedMem, RegionEndBB->getTerminator());
3765*d415bd75Srobert 
3766*d415bd75Srobert         LoadInst *LoadI = new LoadInst(I.getType(), SharedMem,
3767*d415bd75Srobert                                        I.getName() + ".guarded.output.load",
3768*d415bd75Srobert                                        RegionBarrierBB->getTerminator());
3769*d415bd75Srobert 
3770*d415bd75Srobert         // Emit a load instruction and replace uses of the output value.
3771*d415bd75Srobert         for (Instruction *UsrI : OutsideUsers)
3772*d415bd75Srobert           UsrI->replaceUsesOfWith(&I, LoadI);
3773*d415bd75Srobert       }
3774*d415bd75Srobert 
3775*d415bd75Srobert       auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3776*d415bd75Srobert 
3777*d415bd75Srobert       // Go to tid check BB in ParentBB.
3778*d415bd75Srobert       const DebugLoc DL = ParentBB->getTerminator()->getDebugLoc();
3779*d415bd75Srobert       ParentBB->getTerminator()->eraseFromParent();
3780*d415bd75Srobert       OpenMPIRBuilder::LocationDescription Loc(
3781*d415bd75Srobert           InsertPointTy(ParentBB, ParentBB->end()), DL);
3782*d415bd75Srobert       OMPInfoCache.OMPBuilder.updateToLocation(Loc);
3783*d415bd75Srobert       uint32_t SrcLocStrSize;
3784*d415bd75Srobert       auto *SrcLocStr =
3785*d415bd75Srobert           OMPInfoCache.OMPBuilder.getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3786*d415bd75Srobert       Value *Ident =
3787*d415bd75Srobert           OMPInfoCache.OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3788*d415bd75Srobert       BranchInst::Create(RegionCheckTidBB, ParentBB)->setDebugLoc(DL);
3789*d415bd75Srobert 
3790*d415bd75Srobert       // Add check for Tid in RegionCheckTidBB
3791*d415bd75Srobert       RegionCheckTidBB->getTerminator()->eraseFromParent();
3792*d415bd75Srobert       OpenMPIRBuilder::LocationDescription LocRegionCheckTid(
3793*d415bd75Srobert           InsertPointTy(RegionCheckTidBB, RegionCheckTidBB->end()), DL);
3794*d415bd75Srobert       OMPInfoCache.OMPBuilder.updateToLocation(LocRegionCheckTid);
3795*d415bd75Srobert       FunctionCallee HardwareTidFn =
3796*d415bd75Srobert           OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
3797*d415bd75Srobert               M, OMPRTL___kmpc_get_hardware_thread_id_in_block);
3798*d415bd75Srobert       CallInst *Tid =
3799*d415bd75Srobert           OMPInfoCache.OMPBuilder.Builder.CreateCall(HardwareTidFn, {});
3800*d415bd75Srobert       Tid->setDebugLoc(DL);
3801*d415bd75Srobert       OMPInfoCache.setCallingConvention(HardwareTidFn, Tid);
3802*d415bd75Srobert       Value *TidCheck = OMPInfoCache.OMPBuilder.Builder.CreateIsNull(Tid);
3803*d415bd75Srobert       OMPInfoCache.OMPBuilder.Builder
3804*d415bd75Srobert           .CreateCondBr(TidCheck, RegionStartBB, RegionBarrierBB)
3805*d415bd75Srobert           ->setDebugLoc(DL);
3806*d415bd75Srobert 
3807*d415bd75Srobert       // First barrier for synchronization, ensures main thread has updated
3808*d415bd75Srobert       // values.
3809*d415bd75Srobert       FunctionCallee BarrierFn =
3810*d415bd75Srobert           OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
3811*d415bd75Srobert               M, OMPRTL___kmpc_barrier_simple_spmd);
3812*d415bd75Srobert       OMPInfoCache.OMPBuilder.updateToLocation(InsertPointTy(
3813*d415bd75Srobert           RegionBarrierBB, RegionBarrierBB->getFirstInsertionPt()));
3814*d415bd75Srobert       CallInst *Barrier =
3815*d415bd75Srobert           OMPInfoCache.OMPBuilder.Builder.CreateCall(BarrierFn, {Ident, Tid});
3816*d415bd75Srobert       Barrier->setDebugLoc(DL);
3817*d415bd75Srobert       OMPInfoCache.setCallingConvention(BarrierFn, Barrier);
3818*d415bd75Srobert 
3819*d415bd75Srobert       // Second barrier ensures workers have read broadcast values.
3820*d415bd75Srobert       if (HasBroadcastValues) {
3821*d415bd75Srobert         CallInst *Barrier = CallInst::Create(BarrierFn, {Ident, Tid}, "",
3822*d415bd75Srobert                                              RegionBarrierBB->getTerminator());
3823*d415bd75Srobert         Barrier->setDebugLoc(DL);
3824*d415bd75Srobert         OMPInfoCache.setCallingConvention(BarrierFn, Barrier);
3825*d415bd75Srobert       }
3826*d415bd75Srobert     };
3827*d415bd75Srobert 
3828*d415bd75Srobert     auto &AllocSharedRFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
3829*d415bd75Srobert     SmallPtrSet<BasicBlock *, 8> Visited;
3830*d415bd75Srobert     for (Instruction *GuardedI : SPMDCompatibilityTracker) {
3831*d415bd75Srobert       BasicBlock *BB = GuardedI->getParent();
3832*d415bd75Srobert       if (!Visited.insert(BB).second)
3833*d415bd75Srobert         continue;
3834*d415bd75Srobert 
3835*d415bd75Srobert       SmallVector<std::pair<Instruction *, Instruction *>> Reorders;
3836*d415bd75Srobert       Instruction *LastEffect = nullptr;
3837*d415bd75Srobert       BasicBlock::reverse_iterator IP = BB->rbegin(), IPEnd = BB->rend();
3838*d415bd75Srobert       while (++IP != IPEnd) {
3839*d415bd75Srobert         if (!IP->mayHaveSideEffects() && !IP->mayReadFromMemory())
3840*d415bd75Srobert           continue;
3841*d415bd75Srobert         Instruction *I = &*IP;
3842*d415bd75Srobert         if (OpenMPOpt::getCallIfRegularCall(*I, &AllocSharedRFI))
3843*d415bd75Srobert           continue;
3844*d415bd75Srobert         if (!I->user_empty() || !SPMDCompatibilityTracker.contains(I)) {
3845*d415bd75Srobert           LastEffect = nullptr;
3846*d415bd75Srobert           continue;
3847*d415bd75Srobert         }
3848*d415bd75Srobert         if (LastEffect)
3849*d415bd75Srobert           Reorders.push_back({I, LastEffect});
3850*d415bd75Srobert         LastEffect = &*IP;
3851*d415bd75Srobert       }
3852*d415bd75Srobert       for (auto &Reorder : Reorders)
3853*d415bd75Srobert         Reorder.first->moveBefore(Reorder.second);
3854*d415bd75Srobert     }
3855*d415bd75Srobert 
3856*d415bd75Srobert     SmallVector<std::pair<Instruction *, Instruction *>, 4> GuardedRegions;
3857*d415bd75Srobert 
3858*d415bd75Srobert     for (Instruction *GuardedI : SPMDCompatibilityTracker) {
3859*d415bd75Srobert       BasicBlock *BB = GuardedI->getParent();
3860*d415bd75Srobert       auto *CalleeAA = A.lookupAAFor<AAKernelInfo>(
3861*d415bd75Srobert           IRPosition::function(*GuardedI->getFunction()), nullptr,
3862*d415bd75Srobert           DepClassTy::NONE);
3863*d415bd75Srobert       assert(CalleeAA != nullptr && "Expected Callee AAKernelInfo");
3864*d415bd75Srobert       auto &CalleeAAFunction = *cast<AAKernelInfoFunction>(CalleeAA);
3865*d415bd75Srobert       // Continue if instruction is already guarded.
3866*d415bd75Srobert       if (CalleeAAFunction.getGuardedInstructions().contains(GuardedI))
3867*d415bd75Srobert         continue;
3868*d415bd75Srobert 
3869*d415bd75Srobert       Instruction *GuardedRegionStart = nullptr, *GuardedRegionEnd = nullptr;
3870*d415bd75Srobert       for (Instruction &I : *BB) {
3871*d415bd75Srobert         // If instruction I needs to be guarded update the guarded region
3872*d415bd75Srobert         // bounds.
3873*d415bd75Srobert         if (SPMDCompatibilityTracker.contains(&I)) {
3874*d415bd75Srobert           CalleeAAFunction.getGuardedInstructions().insert(&I);
3875*d415bd75Srobert           if (GuardedRegionStart)
3876*d415bd75Srobert             GuardedRegionEnd = &I;
3877*d415bd75Srobert           else
3878*d415bd75Srobert             GuardedRegionStart = GuardedRegionEnd = &I;
3879*d415bd75Srobert 
3880*d415bd75Srobert           continue;
3881*d415bd75Srobert         }
3882*d415bd75Srobert 
3883*d415bd75Srobert         // Instruction I does not need guarding, store
3884*d415bd75Srobert         // any region found and reset bounds.
3885*d415bd75Srobert         if (GuardedRegionStart) {
3886*d415bd75Srobert           GuardedRegions.push_back(
3887*d415bd75Srobert               std::make_pair(GuardedRegionStart, GuardedRegionEnd));
3888*d415bd75Srobert           GuardedRegionStart = nullptr;
3889*d415bd75Srobert           GuardedRegionEnd = nullptr;
3890*d415bd75Srobert         }
3891*d415bd75Srobert       }
3892*d415bd75Srobert     }
3893*d415bd75Srobert 
3894*d415bd75Srobert     for (auto &GR : GuardedRegions)
3895*d415bd75Srobert       CreateGuardedRegion(GR.first, GR.second);
3896*d415bd75Srobert   }
3897*d415bd75Srobert 
forceSingleThreadPerWorkgroupHelper__anon4b47cb5e0111::AAKernelInfoFunction3898*d415bd75Srobert   void forceSingleThreadPerWorkgroupHelper(Attributor &A) {
3899*d415bd75Srobert     // Only allow 1 thread per workgroup to continue executing the user code.
3900*d415bd75Srobert     //
3901*d415bd75Srobert     //     InitCB = __kmpc_target_init(...)
3902*d415bd75Srobert     //     ThreadIdInBlock = __kmpc_get_hardware_thread_id_in_block();
3903*d415bd75Srobert     //     if (ThreadIdInBlock != 0) return;
3904*d415bd75Srobert     // UserCode:
3905*d415bd75Srobert     //     // user code
3906*d415bd75Srobert     //
3907*d415bd75Srobert     auto &Ctx = getAnchorValue().getContext();
3908*d415bd75Srobert     Function *Kernel = getAssociatedFunction();
3909*d415bd75Srobert     assert(Kernel && "Expected an associated function!");
3910*d415bd75Srobert 
3911*d415bd75Srobert     // Create block for user code to branch to from initial block.
3912*d415bd75Srobert     BasicBlock *InitBB = KernelInitCB->getParent();
3913*d415bd75Srobert     BasicBlock *UserCodeBB = InitBB->splitBasicBlock(
3914*d415bd75Srobert         KernelInitCB->getNextNode(), "main.thread.user_code");
3915*d415bd75Srobert     BasicBlock *ReturnBB =
3916*d415bd75Srobert         BasicBlock::Create(Ctx, "exit.threads", Kernel, UserCodeBB);
3917*d415bd75Srobert 
3918*d415bd75Srobert     // Register blocks with attributor:
3919*d415bd75Srobert     A.registerManifestAddedBasicBlock(*InitBB);
3920*d415bd75Srobert     A.registerManifestAddedBasicBlock(*UserCodeBB);
3921*d415bd75Srobert     A.registerManifestAddedBasicBlock(*ReturnBB);
3922*d415bd75Srobert 
3923*d415bd75Srobert     // Debug location:
3924*d415bd75Srobert     const DebugLoc &DLoc = KernelInitCB->getDebugLoc();
3925*d415bd75Srobert     ReturnInst::Create(Ctx, ReturnBB)->setDebugLoc(DLoc);
3926*d415bd75Srobert     InitBB->getTerminator()->eraseFromParent();
3927*d415bd75Srobert 
3928*d415bd75Srobert     // Prepare call to OMPRTL___kmpc_get_hardware_thread_id_in_block.
3929*d415bd75Srobert     Module &M = *Kernel->getParent();
3930*d415bd75Srobert     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3931*d415bd75Srobert     FunctionCallee ThreadIdInBlockFn =
3932*d415bd75Srobert         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
3933*d415bd75Srobert             M, OMPRTL___kmpc_get_hardware_thread_id_in_block);
3934*d415bd75Srobert 
3935*d415bd75Srobert     // Get thread ID in block.
3936*d415bd75Srobert     CallInst *ThreadIdInBlock =
3937*d415bd75Srobert         CallInst::Create(ThreadIdInBlockFn, "thread_id.in.block", InitBB);
3938*d415bd75Srobert     OMPInfoCache.setCallingConvention(ThreadIdInBlockFn, ThreadIdInBlock);
3939*d415bd75Srobert     ThreadIdInBlock->setDebugLoc(DLoc);
3940*d415bd75Srobert 
3941*d415bd75Srobert     // Eliminate all threads in the block with ID not equal to 0:
3942*d415bd75Srobert     Instruction *IsMainThread =
3943*d415bd75Srobert         ICmpInst::Create(ICmpInst::ICmp, CmpInst::ICMP_NE, ThreadIdInBlock,
3944*d415bd75Srobert                          ConstantInt::get(ThreadIdInBlock->getType(), 0),
3945*d415bd75Srobert                          "thread.is_main", InitBB);
3946*d415bd75Srobert     IsMainThread->setDebugLoc(DLoc);
3947*d415bd75Srobert     BranchInst::Create(ReturnBB, UserCodeBB, IsMainThread, InitBB);
3948*d415bd75Srobert   }
3949*d415bd75Srobert 
changeToSPMDMode__anon4b47cb5e0111::AAKernelInfoFunction3950*d415bd75Srobert   bool changeToSPMDMode(Attributor &A, ChangeStatus &Changed) {
3951*d415bd75Srobert     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3952*d415bd75Srobert 
3953*d415bd75Srobert     // We cannot change to SPMD mode if the runtime functions aren't availible.
3954*d415bd75Srobert     if (!OMPInfoCache.runtimeFnsAvailable(
3955*d415bd75Srobert             {OMPRTL___kmpc_get_hardware_thread_id_in_block,
3956*d415bd75Srobert              OMPRTL___kmpc_barrier_simple_spmd}))
3957*d415bd75Srobert       return false;
3958*d415bd75Srobert 
395973471bf0Spatrick     if (!SPMDCompatibilityTracker.isAssumed()) {
396073471bf0Spatrick       for (Instruction *NonCompatibleI : SPMDCompatibilityTracker) {
396173471bf0Spatrick         if (!NonCompatibleI)
396273471bf0Spatrick           continue;
396373471bf0Spatrick 
396473471bf0Spatrick         // Skip diagnostics on calls to known OpenMP runtime functions for now.
396573471bf0Spatrick         if (auto *CB = dyn_cast<CallBase>(NonCompatibleI))
396673471bf0Spatrick           if (OMPInfoCache.RTLFunctions.contains(CB->getCalledFunction()))
396773471bf0Spatrick             continue;
396873471bf0Spatrick 
396973471bf0Spatrick         auto Remark = [&](OptimizationRemarkAnalysis ORA) {
397073471bf0Spatrick           ORA << "Value has potential side effects preventing SPMD-mode "
397173471bf0Spatrick                  "execution";
397273471bf0Spatrick           if (isa<CallBase>(NonCompatibleI)) {
397373471bf0Spatrick             ORA << ". Add `__attribute__((assume(\"ompx_spmd_amenable\")))` to "
397473471bf0Spatrick                    "the called function to override";
397573471bf0Spatrick           }
397673471bf0Spatrick           return ORA << ".";
397773471bf0Spatrick         };
397873471bf0Spatrick         A.emitRemark<OptimizationRemarkAnalysis>(NonCompatibleI, "OMP121",
397973471bf0Spatrick                                                  Remark);
398073471bf0Spatrick 
398173471bf0Spatrick         LLVM_DEBUG(dbgs() << TAG << "SPMD-incompatible side-effect: "
398273471bf0Spatrick                           << *NonCompatibleI << "\n");
398373471bf0Spatrick       }
398473471bf0Spatrick 
398573471bf0Spatrick       return false;
398673471bf0Spatrick     }
398773471bf0Spatrick 
3988*d415bd75Srobert     // Get the actual kernel, could be the caller of the anchor scope if we have
3989*d415bd75Srobert     // a debug wrapper.
399073471bf0Spatrick     Function *Kernel = getAnchorScope();
3991*d415bd75Srobert     if (Kernel->hasLocalLinkage()) {
3992*d415bd75Srobert       assert(Kernel->hasOneUse() && "Unexpected use of debug kernel wrapper.");
3993*d415bd75Srobert       auto *CB = cast<CallBase>(Kernel->user_back());
3994*d415bd75Srobert       Kernel = CB->getCaller();
3995*d415bd75Srobert     }
3996*d415bd75Srobert     assert(OMPInfoCache.Kernels.count(Kernel) && "Expected kernel function!");
3997*d415bd75Srobert 
3998*d415bd75Srobert     // Check if the kernel is already in SPMD mode, if so, return success.
399973471bf0Spatrick     GlobalVariable *ExecMode = Kernel->getParent()->getGlobalVariable(
400073471bf0Spatrick         (Kernel->getName() + "_exec_mode").str());
400173471bf0Spatrick     assert(ExecMode && "Kernel without exec mode?");
4002*d415bd75Srobert     assert(ExecMode->getInitializer() && "ExecMode doesn't have initializer!");
400373471bf0Spatrick 
400473471bf0Spatrick     // Set the global exec mode flag to indicate SPMD-Generic mode.
4005*d415bd75Srobert     assert(isa<ConstantInt>(ExecMode->getInitializer()) &&
4006*d415bd75Srobert            "ExecMode is not an integer!");
4007*d415bd75Srobert     const int8_t ExecModeVal =
4008*d415bd75Srobert         cast<ConstantInt>(ExecMode->getInitializer())->getSExtValue();
4009*d415bd75Srobert     if (ExecModeVal != OMP_TGT_EXEC_MODE_GENERIC)
4010*d415bd75Srobert       return true;
4011*d415bd75Srobert 
4012*d415bd75Srobert     // We will now unconditionally modify the IR, indicate a change.
4013*d415bd75Srobert     Changed = ChangeStatus::CHANGED;
4014*d415bd75Srobert 
4015*d415bd75Srobert     // Do not use instruction guards when no parallel is present inside
4016*d415bd75Srobert     // the target region.
4017*d415bd75Srobert     if (mayContainParallelRegion())
4018*d415bd75Srobert       insertInstructionGuardsHelper(A);
4019*d415bd75Srobert     else
4020*d415bd75Srobert       forceSingleThreadPerWorkgroupHelper(A);
4021*d415bd75Srobert 
4022*d415bd75Srobert     // Adjust the global exec mode flag that tells the runtime what mode this
4023*d415bd75Srobert     // kernel is executed in.
4024*d415bd75Srobert     assert(ExecModeVal == OMP_TGT_EXEC_MODE_GENERIC &&
4025*d415bd75Srobert            "Initially non-SPMD kernel has SPMD exec mode!");
402673471bf0Spatrick     ExecMode->setInitializer(
4027*d415bd75Srobert         ConstantInt::get(ExecMode->getInitializer()->getType(),
4028*d415bd75Srobert                          ExecModeVal | OMP_TGT_EXEC_MODE_GENERIC_SPMD));
402973471bf0Spatrick 
403073471bf0Spatrick     // Next rewrite the init and deinit calls to indicate we use SPMD-mode now.
4031*d415bd75Srobert     const int InitModeArgNo = 1;
4032*d415bd75Srobert     const int DeinitModeArgNo = 1;
403373471bf0Spatrick     const int InitUseStateMachineArgNo = 2;
403473471bf0Spatrick 
403573471bf0Spatrick     auto &Ctx = getAnchorValue().getContext();
4036*d415bd75Srobert     A.changeUseAfterManifest(
4037*d415bd75Srobert         KernelInitCB->getArgOperandUse(InitModeArgNo),
4038*d415bd75Srobert         *ConstantInt::getSigned(IntegerType::getInt8Ty(Ctx),
4039*d415bd75Srobert                                 OMP_TGT_EXEC_MODE_SPMD));
404073471bf0Spatrick     A.changeUseAfterManifest(
404173471bf0Spatrick         KernelInitCB->getArgOperandUse(InitUseStateMachineArgNo),
4042*d415bd75Srobert         *ConstantInt::getBool(Ctx, false));
404373471bf0Spatrick     A.changeUseAfterManifest(
4044*d415bd75Srobert         KernelDeinitCB->getArgOperandUse(DeinitModeArgNo),
4045*d415bd75Srobert         *ConstantInt::getSigned(IntegerType::getInt8Ty(Ctx),
4046*d415bd75Srobert                                 OMP_TGT_EXEC_MODE_SPMD));
404773471bf0Spatrick 
404873471bf0Spatrick     ++NumOpenMPTargetRegionKernelsSPMD;
404973471bf0Spatrick 
405073471bf0Spatrick     auto Remark = [&](OptimizationRemark OR) {
405173471bf0Spatrick       return OR << "Transformed generic-mode kernel to SPMD-mode.";
405273471bf0Spatrick     };
405373471bf0Spatrick     A.emitRemark<OptimizationRemark>(KernelInitCB, "OMP120", Remark);
405473471bf0Spatrick     return true;
405573471bf0Spatrick   };
405673471bf0Spatrick 
buildCustomStateMachine__anon4b47cb5e0111::AAKernelInfoFunction405773471bf0Spatrick   ChangeStatus buildCustomStateMachine(Attributor &A) {
4058*d415bd75Srobert     // If we have disabled state machine rewrites, don't make a custom one
4059*d415bd75Srobert     if (DisableOpenMPOptStateMachineRewrite)
4060*d415bd75Srobert       return ChangeStatus::UNCHANGED;
406173471bf0Spatrick 
4062*d415bd75Srobert     // Don't rewrite the state machine if we are not in a valid state.
4063*d415bd75Srobert     if (!ReachedKnownParallelRegions.isValidState())
4064*d415bd75Srobert       return ChangeStatus::UNCHANGED;
4065*d415bd75Srobert 
4066*d415bd75Srobert     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
4067*d415bd75Srobert     if (!OMPInfoCache.runtimeFnsAvailable(
4068*d415bd75Srobert             {OMPRTL___kmpc_get_hardware_num_threads_in_block,
4069*d415bd75Srobert              OMPRTL___kmpc_get_warp_size, OMPRTL___kmpc_barrier_simple_generic,
4070*d415bd75Srobert              OMPRTL___kmpc_kernel_parallel, OMPRTL___kmpc_kernel_end_parallel}))
4071*d415bd75Srobert       return ChangeStatus::UNCHANGED;
4072*d415bd75Srobert 
4073*d415bd75Srobert     const int InitModeArgNo = 1;
407473471bf0Spatrick     const int InitUseStateMachineArgNo = 2;
407573471bf0Spatrick 
407673471bf0Spatrick     // Check if the current configuration is non-SPMD and generic state machine.
407773471bf0Spatrick     // If we already have SPMD mode or a custom state machine we do not need to
407873471bf0Spatrick     // go any further. If it is anything but a constant something is weird and
407973471bf0Spatrick     // we give up.
408073471bf0Spatrick     ConstantInt *UseStateMachine = dyn_cast<ConstantInt>(
408173471bf0Spatrick         KernelInitCB->getArgOperand(InitUseStateMachineArgNo));
4082*d415bd75Srobert     ConstantInt *Mode =
4083*d415bd75Srobert         dyn_cast<ConstantInt>(KernelInitCB->getArgOperand(InitModeArgNo));
408473471bf0Spatrick 
408573471bf0Spatrick     // If we are stuck with generic mode, try to create a custom device (=GPU)
408673471bf0Spatrick     // state machine which is specialized for the parallel regions that are
408773471bf0Spatrick     // reachable by the kernel.
4088*d415bd75Srobert     if (!UseStateMachine || UseStateMachine->isZero() || !Mode ||
4089*d415bd75Srobert         (Mode->getSExtValue() & OMP_TGT_EXEC_MODE_SPMD))
409073471bf0Spatrick       return ChangeStatus::UNCHANGED;
409173471bf0Spatrick 
409273471bf0Spatrick     // If not SPMD mode, indicate we use a custom state machine now.
409373471bf0Spatrick     auto &Ctx = getAnchorValue().getContext();
4094*d415bd75Srobert     auto *FalseVal = ConstantInt::getBool(Ctx, false);
409573471bf0Spatrick     A.changeUseAfterManifest(
409673471bf0Spatrick         KernelInitCB->getArgOperandUse(InitUseStateMachineArgNo), *FalseVal);
409773471bf0Spatrick 
409873471bf0Spatrick     // If we don't actually need a state machine we are done here. This can
409973471bf0Spatrick     // happen if there simply are no parallel regions. In the resulting kernel
410073471bf0Spatrick     // all worker threads will simply exit right away, leaving the main thread
410173471bf0Spatrick     // to do the work alone.
4102*d415bd75Srobert     if (!mayContainParallelRegion()) {
410373471bf0Spatrick       ++NumOpenMPTargetRegionKernelsWithoutStateMachine;
410473471bf0Spatrick 
410573471bf0Spatrick       auto Remark = [&](OptimizationRemark OR) {
410673471bf0Spatrick         return OR << "Removing unused state machine from generic-mode kernel.";
410773471bf0Spatrick       };
410873471bf0Spatrick       A.emitRemark<OptimizationRemark>(KernelInitCB, "OMP130", Remark);
410973471bf0Spatrick 
411073471bf0Spatrick       return ChangeStatus::CHANGED;
411173471bf0Spatrick     }
411273471bf0Spatrick 
411373471bf0Spatrick     // Keep track in the statistics of our new shiny custom state machine.
411473471bf0Spatrick     if (ReachedUnknownParallelRegions.empty()) {
411573471bf0Spatrick       ++NumOpenMPTargetRegionKernelsCustomStateMachineWithoutFallback;
411673471bf0Spatrick 
411773471bf0Spatrick       auto Remark = [&](OptimizationRemark OR) {
411873471bf0Spatrick         return OR << "Rewriting generic-mode kernel with a customized state "
411973471bf0Spatrick                      "machine.";
412073471bf0Spatrick       };
412173471bf0Spatrick       A.emitRemark<OptimizationRemark>(KernelInitCB, "OMP131", Remark);
412273471bf0Spatrick     } else {
412373471bf0Spatrick       ++NumOpenMPTargetRegionKernelsCustomStateMachineWithFallback;
412473471bf0Spatrick 
412573471bf0Spatrick       auto Remark = [&](OptimizationRemarkAnalysis OR) {
412673471bf0Spatrick         return OR << "Generic-mode kernel is executed with a customized state "
412773471bf0Spatrick                      "machine that requires a fallback.";
412873471bf0Spatrick       };
412973471bf0Spatrick       A.emitRemark<OptimizationRemarkAnalysis>(KernelInitCB, "OMP132", Remark);
413073471bf0Spatrick 
413173471bf0Spatrick       // Tell the user why we ended up with a fallback.
413273471bf0Spatrick       for (CallBase *UnknownParallelRegionCB : ReachedUnknownParallelRegions) {
413373471bf0Spatrick         if (!UnknownParallelRegionCB)
413473471bf0Spatrick           continue;
413573471bf0Spatrick         auto Remark = [&](OptimizationRemarkAnalysis ORA) {
413673471bf0Spatrick           return ORA << "Call may contain unknown parallel regions. Use "
413773471bf0Spatrick                      << "`__attribute__((assume(\"omp_no_parallelism\")))` to "
413873471bf0Spatrick                         "override.";
413973471bf0Spatrick         };
414073471bf0Spatrick         A.emitRemark<OptimizationRemarkAnalysis>(UnknownParallelRegionCB,
414173471bf0Spatrick                                                  "OMP133", Remark);
414273471bf0Spatrick       }
414373471bf0Spatrick     }
414473471bf0Spatrick 
414573471bf0Spatrick     // Create all the blocks:
414673471bf0Spatrick     //
414773471bf0Spatrick     //                       InitCB = __kmpc_target_init(...)
4148*d415bd75Srobert     //                       BlockHwSize =
4149*d415bd75Srobert     //                         __kmpc_get_hardware_num_threads_in_block();
4150*d415bd75Srobert     //                       WarpSize = __kmpc_get_warp_size();
4151*d415bd75Srobert     //                       BlockSize = BlockHwSize - WarpSize;
4152*d415bd75Srobert     // IsWorkerCheckBB:      bool IsWorker = InitCB != -1;
415373471bf0Spatrick     //                       if (IsWorker) {
4154*d415bd75Srobert     //                         if (InitCB >= BlockSize) return;
4155*d415bd75Srobert     // SMBeginBB:               __kmpc_barrier_simple_generic(...);
415673471bf0Spatrick     //                         void *WorkFn;
415773471bf0Spatrick     //                         bool Active = __kmpc_kernel_parallel(&WorkFn);
415873471bf0Spatrick     //                         if (!WorkFn) return;
415973471bf0Spatrick     // SMIsActiveCheckBB:       if (Active) {
416073471bf0Spatrick     // SMIfCascadeCurrentBB:      if      (WorkFn == <ParFn0>)
416173471bf0Spatrick     //                              ParFn0(...);
416273471bf0Spatrick     // SMIfCascadeCurrentBB:      else if (WorkFn == <ParFn1>)
416373471bf0Spatrick     //                              ParFn1(...);
416473471bf0Spatrick     //                            ...
416573471bf0Spatrick     // SMIfCascadeCurrentBB:      else
416673471bf0Spatrick     //                              ((WorkFnTy*)WorkFn)(...);
416773471bf0Spatrick     // SMEndParallelBB:           __kmpc_kernel_end_parallel(...);
416873471bf0Spatrick     //                          }
4169*d415bd75Srobert     // SMDoneBB:                __kmpc_barrier_simple_generic(...);
417073471bf0Spatrick     //                          goto SMBeginBB;
417173471bf0Spatrick     //                       }
417273471bf0Spatrick     // UserCodeEntryBB:      // user code
417373471bf0Spatrick     //                       __kmpc_target_deinit(...)
417473471bf0Spatrick     //
417573471bf0Spatrick     Function *Kernel = getAssociatedFunction();
417673471bf0Spatrick     assert(Kernel && "Expected an associated function!");
417773471bf0Spatrick 
417873471bf0Spatrick     BasicBlock *InitBB = KernelInitCB->getParent();
417973471bf0Spatrick     BasicBlock *UserCodeEntryBB = InitBB->splitBasicBlock(
418073471bf0Spatrick         KernelInitCB->getNextNode(), "thread.user_code.check");
4181*d415bd75Srobert     BasicBlock *IsWorkerCheckBB =
4182*d415bd75Srobert         BasicBlock::Create(Ctx, "is_worker_check", Kernel, UserCodeEntryBB);
418373471bf0Spatrick     BasicBlock *StateMachineBeginBB = BasicBlock::Create(
418473471bf0Spatrick         Ctx, "worker_state_machine.begin", Kernel, UserCodeEntryBB);
418573471bf0Spatrick     BasicBlock *StateMachineFinishedBB = BasicBlock::Create(
418673471bf0Spatrick         Ctx, "worker_state_machine.finished", Kernel, UserCodeEntryBB);
418773471bf0Spatrick     BasicBlock *StateMachineIsActiveCheckBB = BasicBlock::Create(
418873471bf0Spatrick         Ctx, "worker_state_machine.is_active.check", Kernel, UserCodeEntryBB);
418973471bf0Spatrick     BasicBlock *StateMachineIfCascadeCurrentBB =
419073471bf0Spatrick         BasicBlock::Create(Ctx, "worker_state_machine.parallel_region.check",
419173471bf0Spatrick                            Kernel, UserCodeEntryBB);
419273471bf0Spatrick     BasicBlock *StateMachineEndParallelBB =
419373471bf0Spatrick         BasicBlock::Create(Ctx, "worker_state_machine.parallel_region.end",
419473471bf0Spatrick                            Kernel, UserCodeEntryBB);
419573471bf0Spatrick     BasicBlock *StateMachineDoneBarrierBB = BasicBlock::Create(
419673471bf0Spatrick         Ctx, "worker_state_machine.done.barrier", Kernel, UserCodeEntryBB);
419773471bf0Spatrick     A.registerManifestAddedBasicBlock(*InitBB);
419873471bf0Spatrick     A.registerManifestAddedBasicBlock(*UserCodeEntryBB);
4199*d415bd75Srobert     A.registerManifestAddedBasicBlock(*IsWorkerCheckBB);
420073471bf0Spatrick     A.registerManifestAddedBasicBlock(*StateMachineBeginBB);
420173471bf0Spatrick     A.registerManifestAddedBasicBlock(*StateMachineFinishedBB);
420273471bf0Spatrick     A.registerManifestAddedBasicBlock(*StateMachineIsActiveCheckBB);
420373471bf0Spatrick     A.registerManifestAddedBasicBlock(*StateMachineIfCascadeCurrentBB);
420473471bf0Spatrick     A.registerManifestAddedBasicBlock(*StateMachineEndParallelBB);
420573471bf0Spatrick     A.registerManifestAddedBasicBlock(*StateMachineDoneBarrierBB);
420673471bf0Spatrick 
420773471bf0Spatrick     const DebugLoc &DLoc = KernelInitCB->getDebugLoc();
420873471bf0Spatrick     ReturnInst::Create(Ctx, StateMachineFinishedBB)->setDebugLoc(DLoc);
420973471bf0Spatrick     InitBB->getTerminator()->eraseFromParent();
4210*d415bd75Srobert 
421173471bf0Spatrick     Instruction *IsWorker =
421273471bf0Spatrick         ICmpInst::Create(ICmpInst::ICmp, llvm::CmpInst::ICMP_NE, KernelInitCB,
421373471bf0Spatrick                          ConstantInt::get(KernelInitCB->getType(), -1),
421473471bf0Spatrick                          "thread.is_worker", InitBB);
421573471bf0Spatrick     IsWorker->setDebugLoc(DLoc);
4216*d415bd75Srobert     BranchInst::Create(IsWorkerCheckBB, UserCodeEntryBB, IsWorker, InitBB);
421773471bf0Spatrick 
421873471bf0Spatrick     Module &M = *Kernel->getParent();
4219*d415bd75Srobert     FunctionCallee BlockHwSizeFn =
4220*d415bd75Srobert         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4221*d415bd75Srobert             M, OMPRTL___kmpc_get_hardware_num_threads_in_block);
4222*d415bd75Srobert     FunctionCallee WarpSizeFn =
4223*d415bd75Srobert         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4224*d415bd75Srobert             M, OMPRTL___kmpc_get_warp_size);
4225*d415bd75Srobert     CallInst *BlockHwSize =
4226*d415bd75Srobert         CallInst::Create(BlockHwSizeFn, "block.hw_size", IsWorkerCheckBB);
4227*d415bd75Srobert     OMPInfoCache.setCallingConvention(BlockHwSizeFn, BlockHwSize);
4228*d415bd75Srobert     BlockHwSize->setDebugLoc(DLoc);
4229*d415bd75Srobert     CallInst *WarpSize =
4230*d415bd75Srobert         CallInst::Create(WarpSizeFn, "warp.size", IsWorkerCheckBB);
4231*d415bd75Srobert     OMPInfoCache.setCallingConvention(WarpSizeFn, WarpSize);
4232*d415bd75Srobert     WarpSize->setDebugLoc(DLoc);
4233*d415bd75Srobert     Instruction *BlockSize = BinaryOperator::CreateSub(
4234*d415bd75Srobert         BlockHwSize, WarpSize, "block.size", IsWorkerCheckBB);
4235*d415bd75Srobert     BlockSize->setDebugLoc(DLoc);
4236*d415bd75Srobert     Instruction *IsMainOrWorker = ICmpInst::Create(
4237*d415bd75Srobert         ICmpInst::ICmp, llvm::CmpInst::ICMP_SLT, KernelInitCB, BlockSize,
4238*d415bd75Srobert         "thread.is_main_or_worker", IsWorkerCheckBB);
4239*d415bd75Srobert     IsMainOrWorker->setDebugLoc(DLoc);
4240*d415bd75Srobert     BranchInst::Create(StateMachineBeginBB, StateMachineFinishedBB,
4241*d415bd75Srobert                        IsMainOrWorker, IsWorkerCheckBB);
424273471bf0Spatrick 
424373471bf0Spatrick     // Create local storage for the work function pointer.
424473471bf0Spatrick     const DataLayout &DL = M.getDataLayout();
424573471bf0Spatrick     Type *VoidPtrTy = Type::getInt8PtrTy(Ctx);
424673471bf0Spatrick     Instruction *WorkFnAI =
424773471bf0Spatrick         new AllocaInst(VoidPtrTy, DL.getAllocaAddrSpace(), nullptr,
424873471bf0Spatrick                        "worker.work_fn.addr", &Kernel->getEntryBlock().front());
424973471bf0Spatrick     WorkFnAI->setDebugLoc(DLoc);
425073471bf0Spatrick 
425173471bf0Spatrick     OMPInfoCache.OMPBuilder.updateToLocation(
425273471bf0Spatrick         OpenMPIRBuilder::LocationDescription(
425373471bf0Spatrick             IRBuilder<>::InsertPoint(StateMachineBeginBB,
425473471bf0Spatrick                                      StateMachineBeginBB->end()),
425573471bf0Spatrick             DLoc));
425673471bf0Spatrick 
425773471bf0Spatrick     Value *Ident = KernelInitCB->getArgOperand(0);
425873471bf0Spatrick     Value *GTid = KernelInitCB;
425973471bf0Spatrick 
426073471bf0Spatrick     FunctionCallee BarrierFn =
426173471bf0Spatrick         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4262*d415bd75Srobert             M, OMPRTL___kmpc_barrier_simple_generic);
4263*d415bd75Srobert     CallInst *Barrier =
4264*d415bd75Srobert         CallInst::Create(BarrierFn, {Ident, GTid}, "", StateMachineBeginBB);
4265*d415bd75Srobert     OMPInfoCache.setCallingConvention(BarrierFn, Barrier);
4266*d415bd75Srobert     Barrier->setDebugLoc(DLoc);
426773471bf0Spatrick 
426873471bf0Spatrick     if (WorkFnAI->getType()->getPointerAddressSpace() !=
426973471bf0Spatrick         (unsigned int)AddressSpace::Generic) {
427073471bf0Spatrick       WorkFnAI = new AddrSpaceCastInst(
427173471bf0Spatrick           WorkFnAI,
427273471bf0Spatrick           PointerType::getWithSamePointeeType(
427373471bf0Spatrick               cast<PointerType>(WorkFnAI->getType()),
427473471bf0Spatrick               (unsigned int)AddressSpace::Generic),
427573471bf0Spatrick           WorkFnAI->getName() + ".generic", StateMachineBeginBB);
427673471bf0Spatrick       WorkFnAI->setDebugLoc(DLoc);
427773471bf0Spatrick     }
427873471bf0Spatrick 
427973471bf0Spatrick     FunctionCallee KernelParallelFn =
428073471bf0Spatrick         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
428173471bf0Spatrick             M, OMPRTL___kmpc_kernel_parallel);
4282*d415bd75Srobert     CallInst *IsActiveWorker = CallInst::Create(
428373471bf0Spatrick         KernelParallelFn, {WorkFnAI}, "worker.is_active", StateMachineBeginBB);
4284*d415bd75Srobert     OMPInfoCache.setCallingConvention(KernelParallelFn, IsActiveWorker);
428573471bf0Spatrick     IsActiveWorker->setDebugLoc(DLoc);
428673471bf0Spatrick     Instruction *WorkFn = new LoadInst(VoidPtrTy, WorkFnAI, "worker.work_fn",
428773471bf0Spatrick                                        StateMachineBeginBB);
428873471bf0Spatrick     WorkFn->setDebugLoc(DLoc);
428973471bf0Spatrick 
429073471bf0Spatrick     FunctionType *ParallelRegionFnTy = FunctionType::get(
429173471bf0Spatrick         Type::getVoidTy(Ctx), {Type::getInt16Ty(Ctx), Type::getInt32Ty(Ctx)},
429273471bf0Spatrick         false);
429373471bf0Spatrick     Value *WorkFnCast = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
429473471bf0Spatrick         WorkFn, ParallelRegionFnTy->getPointerTo(), "worker.work_fn.addr_cast",
429573471bf0Spatrick         StateMachineBeginBB);
429673471bf0Spatrick 
429773471bf0Spatrick     Instruction *IsDone =
429873471bf0Spatrick         ICmpInst::Create(ICmpInst::ICmp, llvm::CmpInst::ICMP_EQ, WorkFn,
429973471bf0Spatrick                          Constant::getNullValue(VoidPtrTy), "worker.is_done",
430073471bf0Spatrick                          StateMachineBeginBB);
430173471bf0Spatrick     IsDone->setDebugLoc(DLoc);
430273471bf0Spatrick     BranchInst::Create(StateMachineFinishedBB, StateMachineIsActiveCheckBB,
430373471bf0Spatrick                        IsDone, StateMachineBeginBB)
430473471bf0Spatrick         ->setDebugLoc(DLoc);
430573471bf0Spatrick 
430673471bf0Spatrick     BranchInst::Create(StateMachineIfCascadeCurrentBB,
430773471bf0Spatrick                        StateMachineDoneBarrierBB, IsActiveWorker,
430873471bf0Spatrick                        StateMachineIsActiveCheckBB)
430973471bf0Spatrick         ->setDebugLoc(DLoc);
431073471bf0Spatrick 
431173471bf0Spatrick     Value *ZeroArg =
431273471bf0Spatrick         Constant::getNullValue(ParallelRegionFnTy->getParamType(0));
431373471bf0Spatrick 
431473471bf0Spatrick     // Now that we have most of the CFG skeleton it is time for the if-cascade
431573471bf0Spatrick     // that checks the function pointer we got from the runtime against the
431673471bf0Spatrick     // parallel regions we expect, if there are any.
4317*d415bd75Srobert     for (int I = 0, E = ReachedKnownParallelRegions.size(); I < E; ++I) {
4318*d415bd75Srobert       auto *ParallelRegion = ReachedKnownParallelRegions[I];
431973471bf0Spatrick       BasicBlock *PRExecuteBB = BasicBlock::Create(
432073471bf0Spatrick           Ctx, "worker_state_machine.parallel_region.execute", Kernel,
432173471bf0Spatrick           StateMachineEndParallelBB);
432273471bf0Spatrick       CallInst::Create(ParallelRegion, {ZeroArg, GTid}, "", PRExecuteBB)
432373471bf0Spatrick           ->setDebugLoc(DLoc);
432473471bf0Spatrick       BranchInst::Create(StateMachineEndParallelBB, PRExecuteBB)
432573471bf0Spatrick           ->setDebugLoc(DLoc);
432673471bf0Spatrick 
432773471bf0Spatrick       BasicBlock *PRNextBB =
432873471bf0Spatrick           BasicBlock::Create(Ctx, "worker_state_machine.parallel_region.check",
432973471bf0Spatrick                              Kernel, StateMachineEndParallelBB);
433073471bf0Spatrick 
433173471bf0Spatrick       // Check if we need to compare the pointer at all or if we can just
433273471bf0Spatrick       // call the parallel region function.
433373471bf0Spatrick       Value *IsPR;
4334*d415bd75Srobert       if (I + 1 < E || !ReachedUnknownParallelRegions.empty()) {
433573471bf0Spatrick         Instruction *CmpI = ICmpInst::Create(
433673471bf0Spatrick             ICmpInst::ICmp, llvm::CmpInst::ICMP_EQ, WorkFnCast, ParallelRegion,
433773471bf0Spatrick             "worker.check_parallel_region", StateMachineIfCascadeCurrentBB);
433873471bf0Spatrick         CmpI->setDebugLoc(DLoc);
433973471bf0Spatrick         IsPR = CmpI;
434073471bf0Spatrick       } else {
434173471bf0Spatrick         IsPR = ConstantInt::getTrue(Ctx);
434273471bf0Spatrick       }
434373471bf0Spatrick 
434473471bf0Spatrick       BranchInst::Create(PRExecuteBB, PRNextBB, IsPR,
434573471bf0Spatrick                          StateMachineIfCascadeCurrentBB)
434673471bf0Spatrick           ->setDebugLoc(DLoc);
434773471bf0Spatrick       StateMachineIfCascadeCurrentBB = PRNextBB;
434873471bf0Spatrick     }
434973471bf0Spatrick 
435073471bf0Spatrick     // At the end of the if-cascade we place the indirect function pointer call
435173471bf0Spatrick     // in case we might need it, that is if there can be parallel regions we
435273471bf0Spatrick     // have not handled in the if-cascade above.
435373471bf0Spatrick     if (!ReachedUnknownParallelRegions.empty()) {
435473471bf0Spatrick       StateMachineIfCascadeCurrentBB->setName(
435573471bf0Spatrick           "worker_state_machine.parallel_region.fallback.execute");
435673471bf0Spatrick       CallInst::Create(ParallelRegionFnTy, WorkFnCast, {ZeroArg, GTid}, "",
435773471bf0Spatrick                        StateMachineIfCascadeCurrentBB)
435873471bf0Spatrick           ->setDebugLoc(DLoc);
435973471bf0Spatrick     }
436073471bf0Spatrick     BranchInst::Create(StateMachineEndParallelBB,
436173471bf0Spatrick                        StateMachineIfCascadeCurrentBB)
436273471bf0Spatrick         ->setDebugLoc(DLoc);
436373471bf0Spatrick 
4364*d415bd75Srobert     FunctionCallee EndParallelFn =
4365*d415bd75Srobert         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4366*d415bd75Srobert             M, OMPRTL___kmpc_kernel_end_parallel);
4367*d415bd75Srobert     CallInst *EndParallel =
4368*d415bd75Srobert         CallInst::Create(EndParallelFn, {}, "", StateMachineEndParallelBB);
4369*d415bd75Srobert     OMPInfoCache.setCallingConvention(EndParallelFn, EndParallel);
4370*d415bd75Srobert     EndParallel->setDebugLoc(DLoc);
437173471bf0Spatrick     BranchInst::Create(StateMachineDoneBarrierBB, StateMachineEndParallelBB)
437273471bf0Spatrick         ->setDebugLoc(DLoc);
437373471bf0Spatrick 
437473471bf0Spatrick     CallInst::Create(BarrierFn, {Ident, GTid}, "", StateMachineDoneBarrierBB)
437573471bf0Spatrick         ->setDebugLoc(DLoc);
437673471bf0Spatrick     BranchInst::Create(StateMachineBeginBB, StateMachineDoneBarrierBB)
437773471bf0Spatrick         ->setDebugLoc(DLoc);
437873471bf0Spatrick 
437973471bf0Spatrick     return ChangeStatus::CHANGED;
438073471bf0Spatrick   }
438173471bf0Spatrick 
438273471bf0Spatrick   /// Fixpoint iteration update function. Will be called every time a dependence
438373471bf0Spatrick   /// changed its state (and in the beginning).
updateImpl__anon4b47cb5e0111::AAKernelInfoFunction438473471bf0Spatrick   ChangeStatus updateImpl(Attributor &A) override {
438573471bf0Spatrick     KernelInfoState StateBefore = getState();
438673471bf0Spatrick 
438773471bf0Spatrick     // Callback to check a read/write instruction.
438873471bf0Spatrick     auto CheckRWInst = [&](Instruction &I) {
438973471bf0Spatrick       // We handle calls later.
439073471bf0Spatrick       if (isa<CallBase>(I))
439173471bf0Spatrick         return true;
439273471bf0Spatrick       // We only care about write effects.
439373471bf0Spatrick       if (!I.mayWriteToMemory())
439473471bf0Spatrick         return true;
439573471bf0Spatrick       if (auto *SI = dyn_cast<StoreInst>(&I)) {
4396*d415bd75Srobert         const auto &UnderlyingObjsAA = A.getAAFor<AAUnderlyingObjects>(
4397*d415bd75Srobert             *this, IRPosition::value(*SI->getPointerOperand()),
4398*d415bd75Srobert             DepClassTy::OPTIONAL);
4399*d415bd75Srobert         auto &HS = A.getAAFor<AAHeapToStack>(
4400*d415bd75Srobert             *this, IRPosition::function(*I.getFunction()),
4401*d415bd75Srobert             DepClassTy::OPTIONAL);
4402*d415bd75Srobert         if (UnderlyingObjsAA.forallUnderlyingObjects([&](Value &Obj) {
4403*d415bd75Srobert               if (AA::isAssumedThreadLocalObject(A, Obj, *this))
4404*d415bd75Srobert                 return true;
4405*d415bd75Srobert               // Check for AAHeapToStack moved objects which must not be
4406*d415bd75Srobert               // guarded.
4407*d415bd75Srobert               auto *CB = dyn_cast<CallBase>(&Obj);
4408*d415bd75Srobert               return CB && HS.isAssumedHeapToStack(*CB);
4409*d415bd75Srobert             }))
441073471bf0Spatrick           return true;
441173471bf0Spatrick       }
4412*d415bd75Srobert 
4413*d415bd75Srobert       // Insert instruction that needs guarding.
441473471bf0Spatrick       SPMDCompatibilityTracker.insert(&I);
441573471bf0Spatrick       return true;
441673471bf0Spatrick     };
441773471bf0Spatrick 
441873471bf0Spatrick     bool UsedAssumedInformationInCheckRWInst = false;
441973471bf0Spatrick     if (!SPMDCompatibilityTracker.isAtFixpoint())
442073471bf0Spatrick       if (!A.checkForAllReadWriteInstructions(
442173471bf0Spatrick               CheckRWInst, *this, UsedAssumedInformationInCheckRWInst))
442273471bf0Spatrick         SPMDCompatibilityTracker.indicatePessimisticFixpoint();
442373471bf0Spatrick 
4424*d415bd75Srobert     bool UsedAssumedInformationFromReachingKernels = false;
442573471bf0Spatrick     if (!IsKernelEntry) {
442673471bf0Spatrick       updateParallelLevels(A);
4427*d415bd75Srobert 
4428*d415bd75Srobert       bool AllReachingKernelsKnown = true;
4429*d415bd75Srobert       updateReachingKernelEntries(A, AllReachingKernelsKnown);
4430*d415bd75Srobert       UsedAssumedInformationFromReachingKernels = !AllReachingKernelsKnown;
4431*d415bd75Srobert 
4432*d415bd75Srobert       if (!SPMDCompatibilityTracker.empty()) {
4433*d415bd75Srobert         if (!ParallelLevels.isValidState())
4434*d415bd75Srobert           SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4435*d415bd75Srobert         else if (!ReachingKernelEntries.isValidState())
4436*d415bd75Srobert           SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4437*d415bd75Srobert         else {
4438*d415bd75Srobert           // Check if all reaching kernels agree on the mode as we can otherwise
4439*d415bd75Srobert           // not guard instructions. We might not be sure about the mode so we
4440*d415bd75Srobert           // we cannot fix the internal spmd-zation state either.
4441*d415bd75Srobert           int SPMD = 0, Generic = 0;
4442*d415bd75Srobert           for (auto *Kernel : ReachingKernelEntries) {
4443*d415bd75Srobert             auto &CBAA = A.getAAFor<AAKernelInfo>(
4444*d415bd75Srobert                 *this, IRPosition::function(*Kernel), DepClassTy::OPTIONAL);
4445*d415bd75Srobert             if (CBAA.SPMDCompatibilityTracker.isValidState() &&
4446*d415bd75Srobert                 CBAA.SPMDCompatibilityTracker.isAssumed())
4447*d415bd75Srobert               ++SPMD;
4448*d415bd75Srobert             else
4449*d415bd75Srobert               ++Generic;
4450*d415bd75Srobert             if (!CBAA.SPMDCompatibilityTracker.isAtFixpoint())
4451*d415bd75Srobert               UsedAssumedInformationFromReachingKernels = true;
4452*d415bd75Srobert           }
4453*d415bd75Srobert           if (SPMD != 0 && Generic != 0)
4454*d415bd75Srobert             SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4455*d415bd75Srobert         }
4456*d415bd75Srobert       }
445773471bf0Spatrick     }
445873471bf0Spatrick 
445973471bf0Spatrick     // Callback to check a call instruction.
4460*d415bd75Srobert     bool AllParallelRegionStatesWereFixed = true;
446173471bf0Spatrick     bool AllSPMDStatesWereFixed = true;
446273471bf0Spatrick     auto CheckCallInst = [&](Instruction &I) {
446373471bf0Spatrick       auto &CB = cast<CallBase>(I);
446473471bf0Spatrick       auto &CBAA = A.getAAFor<AAKernelInfo>(
446573471bf0Spatrick           *this, IRPosition::callsite_function(CB), DepClassTy::OPTIONAL);
446673471bf0Spatrick       getState() ^= CBAA.getState();
446773471bf0Spatrick       AllSPMDStatesWereFixed &= CBAA.SPMDCompatibilityTracker.isAtFixpoint();
4468*d415bd75Srobert       AllParallelRegionStatesWereFixed &=
4469*d415bd75Srobert           CBAA.ReachedKnownParallelRegions.isAtFixpoint();
4470*d415bd75Srobert       AllParallelRegionStatesWereFixed &=
4471*d415bd75Srobert           CBAA.ReachedUnknownParallelRegions.isAtFixpoint();
447273471bf0Spatrick       return true;
447373471bf0Spatrick     };
447473471bf0Spatrick 
447573471bf0Spatrick     bool UsedAssumedInformationInCheckCallInst = false;
447673471bf0Spatrick     if (!A.checkForAllCallLikeInstructions(
4477*d415bd75Srobert             CheckCallInst, *this, UsedAssumedInformationInCheckCallInst)) {
4478*d415bd75Srobert       LLVM_DEBUG(dbgs() << TAG
4479*d415bd75Srobert                         << "Failed to visit all call-like instructions!\n";);
448073471bf0Spatrick       return indicatePessimisticFixpoint();
4481*d415bd75Srobert     }
4482*d415bd75Srobert 
4483*d415bd75Srobert     // If we haven't used any assumed information for the reached parallel
4484*d415bd75Srobert     // region states we can fix it.
4485*d415bd75Srobert     if (!UsedAssumedInformationInCheckCallInst &&
4486*d415bd75Srobert         AllParallelRegionStatesWereFixed) {
4487*d415bd75Srobert       ReachedKnownParallelRegions.indicateOptimisticFixpoint();
4488*d415bd75Srobert       ReachedUnknownParallelRegions.indicateOptimisticFixpoint();
4489*d415bd75Srobert     }
449073471bf0Spatrick 
449173471bf0Spatrick     // If we haven't used any assumed information for the SPMD state we can fix
449273471bf0Spatrick     // it.
449373471bf0Spatrick     if (!UsedAssumedInformationInCheckRWInst &&
4494*d415bd75Srobert         !UsedAssumedInformationInCheckCallInst &&
4495*d415bd75Srobert         !UsedAssumedInformationFromReachingKernels && AllSPMDStatesWereFixed)
449673471bf0Spatrick       SPMDCompatibilityTracker.indicateOptimisticFixpoint();
449773471bf0Spatrick 
449873471bf0Spatrick     return StateBefore == getState() ? ChangeStatus::UNCHANGED
449973471bf0Spatrick                                      : ChangeStatus::CHANGED;
450073471bf0Spatrick   }
450173471bf0Spatrick 
450273471bf0Spatrick private:
450373471bf0Spatrick   /// Update info regarding reaching kernels.
updateReachingKernelEntries__anon4b47cb5e0111::AAKernelInfoFunction4504*d415bd75Srobert   void updateReachingKernelEntries(Attributor &A,
4505*d415bd75Srobert                                    bool &AllReachingKernelsKnown) {
450673471bf0Spatrick     auto PredCallSite = [&](AbstractCallSite ACS) {
450773471bf0Spatrick       Function *Caller = ACS.getInstruction()->getFunction();
450873471bf0Spatrick 
450973471bf0Spatrick       assert(Caller && "Caller is nullptr");
451073471bf0Spatrick 
451173471bf0Spatrick       auto &CAA = A.getOrCreateAAFor<AAKernelInfo>(
451273471bf0Spatrick           IRPosition::function(*Caller), this, DepClassTy::REQUIRED);
451373471bf0Spatrick       if (CAA.ReachingKernelEntries.isValidState()) {
451473471bf0Spatrick         ReachingKernelEntries ^= CAA.ReachingKernelEntries;
451573471bf0Spatrick         return true;
451673471bf0Spatrick       }
451773471bf0Spatrick 
451873471bf0Spatrick       // We lost track of the caller of the associated function, any kernel
451973471bf0Spatrick       // could reach now.
452073471bf0Spatrick       ReachingKernelEntries.indicatePessimisticFixpoint();
452173471bf0Spatrick 
452273471bf0Spatrick       return true;
452373471bf0Spatrick     };
452473471bf0Spatrick 
452573471bf0Spatrick     if (!A.checkForAllCallSites(PredCallSite, *this,
452673471bf0Spatrick                                 true /* RequireAllCallSites */,
4527*d415bd75Srobert                                 AllReachingKernelsKnown))
452873471bf0Spatrick       ReachingKernelEntries.indicatePessimisticFixpoint();
452973471bf0Spatrick   }
453073471bf0Spatrick 
453173471bf0Spatrick   /// Update info regarding parallel levels.
updateParallelLevels__anon4b47cb5e0111::AAKernelInfoFunction453273471bf0Spatrick   void updateParallelLevels(Attributor &A) {
453373471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
453473471bf0Spatrick     OMPInformationCache::RuntimeFunctionInfo &Parallel51RFI =
453573471bf0Spatrick         OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_51];
453673471bf0Spatrick 
453773471bf0Spatrick     auto PredCallSite = [&](AbstractCallSite ACS) {
453873471bf0Spatrick       Function *Caller = ACS.getInstruction()->getFunction();
453973471bf0Spatrick 
454073471bf0Spatrick       assert(Caller && "Caller is nullptr");
454173471bf0Spatrick 
454273471bf0Spatrick       auto &CAA =
454373471bf0Spatrick           A.getOrCreateAAFor<AAKernelInfo>(IRPosition::function(*Caller));
454473471bf0Spatrick       if (CAA.ParallelLevels.isValidState()) {
454573471bf0Spatrick         // Any function that is called by `__kmpc_parallel_51` will not be
454673471bf0Spatrick         // folded as the parallel level in the function is updated. In order to
454773471bf0Spatrick         // get it right, all the analysis would depend on the implentation. That
454873471bf0Spatrick         // said, if in the future any change to the implementation, the analysis
454973471bf0Spatrick         // could be wrong. As a consequence, we are just conservative here.
455073471bf0Spatrick         if (Caller == Parallel51RFI.Declaration) {
455173471bf0Spatrick           ParallelLevels.indicatePessimisticFixpoint();
455273471bf0Spatrick           return true;
455373471bf0Spatrick         }
455473471bf0Spatrick 
455573471bf0Spatrick         ParallelLevels ^= CAA.ParallelLevels;
455673471bf0Spatrick 
455773471bf0Spatrick         return true;
455873471bf0Spatrick       }
455973471bf0Spatrick 
456073471bf0Spatrick       // We lost track of the caller of the associated function, any kernel
456173471bf0Spatrick       // could reach now.
456273471bf0Spatrick       ParallelLevels.indicatePessimisticFixpoint();
456373471bf0Spatrick 
456473471bf0Spatrick       return true;
456573471bf0Spatrick     };
456673471bf0Spatrick 
456773471bf0Spatrick     bool AllCallSitesKnown = true;
456873471bf0Spatrick     if (!A.checkForAllCallSites(PredCallSite, *this,
456973471bf0Spatrick                                 true /* RequireAllCallSites */,
457073471bf0Spatrick                                 AllCallSitesKnown))
457173471bf0Spatrick       ParallelLevels.indicatePessimisticFixpoint();
457273471bf0Spatrick   }
457373471bf0Spatrick };
457473471bf0Spatrick 
457573471bf0Spatrick /// The call site kernel info abstract attribute, basically, what can we say
457673471bf0Spatrick /// about a call site with regards to the KernelInfoState. For now this simply
457773471bf0Spatrick /// forwards the information from the callee.
457873471bf0Spatrick struct AAKernelInfoCallSite : AAKernelInfo {
AAKernelInfoCallSite__anon4b47cb5e0111::AAKernelInfoCallSite457973471bf0Spatrick   AAKernelInfoCallSite(const IRPosition &IRP, Attributor &A)
458073471bf0Spatrick       : AAKernelInfo(IRP, A) {}
458173471bf0Spatrick 
458273471bf0Spatrick   /// See AbstractAttribute::initialize(...).
initialize__anon4b47cb5e0111::AAKernelInfoCallSite458373471bf0Spatrick   void initialize(Attributor &A) override {
458473471bf0Spatrick     AAKernelInfo::initialize(A);
458573471bf0Spatrick 
458673471bf0Spatrick     CallBase &CB = cast<CallBase>(getAssociatedValue());
458773471bf0Spatrick     Function *Callee = getAssociatedFunction();
458873471bf0Spatrick 
4589*d415bd75Srobert     auto &AssumptionAA = A.getAAFor<AAAssumptionInfo>(
4590*d415bd75Srobert         *this, IRPosition::callsite_function(CB), DepClassTy::OPTIONAL);
459173471bf0Spatrick 
459273471bf0Spatrick     // Check for SPMD-mode assumptions.
4593*d415bd75Srobert     if (AssumptionAA.hasAssumption("ompx_spmd_amenable")) {
459473471bf0Spatrick       SPMDCompatibilityTracker.indicateOptimisticFixpoint();
4595*d415bd75Srobert       indicateOptimisticFixpoint();
4596*d415bd75Srobert     }
459773471bf0Spatrick 
459873471bf0Spatrick     // First weed out calls we do not care about, that is readonly/readnone
459973471bf0Spatrick     // calls, intrinsics, and "no_openmp" calls. Neither of these can reach a
460073471bf0Spatrick     // parallel region or anything else we are looking for.
460173471bf0Spatrick     if (!CB.mayWriteToMemory() || isa<IntrinsicInst>(CB)) {
460273471bf0Spatrick       indicateOptimisticFixpoint();
460373471bf0Spatrick       return;
460473471bf0Spatrick     }
460573471bf0Spatrick 
460673471bf0Spatrick     // Next we check if we know the callee. If it is a known OpenMP function
460773471bf0Spatrick     // we will handle them explicitly in the switch below. If it is not, we
460873471bf0Spatrick     // will use an AAKernelInfo object on the callee to gather information and
460973471bf0Spatrick     // merge that into the current state. The latter happens in the updateImpl.
461073471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
461173471bf0Spatrick     const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(Callee);
461273471bf0Spatrick     if (It == OMPInfoCache.RuntimeFunctionIDMap.end()) {
461373471bf0Spatrick       // Unknown caller or declarations are not analyzable, we give up.
461473471bf0Spatrick       if (!Callee || !A.isFunctionIPOAmendable(*Callee)) {
461573471bf0Spatrick 
461673471bf0Spatrick         // Unknown callees might contain parallel regions, except if they have
461773471bf0Spatrick         // an appropriate assumption attached.
4618*d415bd75Srobert         if (!(AssumptionAA.hasAssumption("omp_no_openmp") ||
4619*d415bd75Srobert               AssumptionAA.hasAssumption("omp_no_parallelism")))
462073471bf0Spatrick           ReachedUnknownParallelRegions.insert(&CB);
462173471bf0Spatrick 
462273471bf0Spatrick         // If SPMDCompatibilityTracker is not fixed, we need to give up on the
462373471bf0Spatrick         // idea we can run something unknown in SPMD-mode.
4624*d415bd75Srobert         if (!SPMDCompatibilityTracker.isAtFixpoint()) {
4625*d415bd75Srobert           SPMDCompatibilityTracker.indicatePessimisticFixpoint();
462673471bf0Spatrick           SPMDCompatibilityTracker.insert(&CB);
4627*d415bd75Srobert         }
462873471bf0Spatrick 
462973471bf0Spatrick         // We have updated the state for this unknown call properly, there won't
463073471bf0Spatrick         // be any change so we indicate a fixpoint.
463173471bf0Spatrick         indicateOptimisticFixpoint();
463273471bf0Spatrick       }
463373471bf0Spatrick       // If the callee is known and can be used in IPO, we will update the state
463473471bf0Spatrick       // based on the callee state in updateImpl.
463573471bf0Spatrick       return;
463673471bf0Spatrick     }
463773471bf0Spatrick 
463873471bf0Spatrick     const unsigned int WrapperFunctionArgNo = 6;
463973471bf0Spatrick     RuntimeFunction RF = It->getSecond();
464073471bf0Spatrick     switch (RF) {
464173471bf0Spatrick     // All the functions we know are compatible with SPMD mode.
464273471bf0Spatrick     case OMPRTL___kmpc_is_spmd_exec_mode:
4643*d415bd75Srobert     case OMPRTL___kmpc_distribute_static_fini:
464473471bf0Spatrick     case OMPRTL___kmpc_for_static_fini:
464573471bf0Spatrick     case OMPRTL___kmpc_global_thread_num:
464673471bf0Spatrick     case OMPRTL___kmpc_get_hardware_num_threads_in_block:
464773471bf0Spatrick     case OMPRTL___kmpc_get_hardware_num_blocks:
464873471bf0Spatrick     case OMPRTL___kmpc_single:
464973471bf0Spatrick     case OMPRTL___kmpc_end_single:
465073471bf0Spatrick     case OMPRTL___kmpc_master:
465173471bf0Spatrick     case OMPRTL___kmpc_end_master:
465273471bf0Spatrick     case OMPRTL___kmpc_barrier:
4653*d415bd75Srobert     case OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2:
4654*d415bd75Srobert     case OMPRTL___kmpc_nvptx_teams_reduce_nowait_v2:
4655*d415bd75Srobert     case OMPRTL___kmpc_nvptx_end_reduce_nowait:
465673471bf0Spatrick       break;
4657*d415bd75Srobert     case OMPRTL___kmpc_distribute_static_init_4:
4658*d415bd75Srobert     case OMPRTL___kmpc_distribute_static_init_4u:
4659*d415bd75Srobert     case OMPRTL___kmpc_distribute_static_init_8:
4660*d415bd75Srobert     case OMPRTL___kmpc_distribute_static_init_8u:
466173471bf0Spatrick     case OMPRTL___kmpc_for_static_init_4:
466273471bf0Spatrick     case OMPRTL___kmpc_for_static_init_4u:
466373471bf0Spatrick     case OMPRTL___kmpc_for_static_init_8:
466473471bf0Spatrick     case OMPRTL___kmpc_for_static_init_8u: {
466573471bf0Spatrick       // Check the schedule and allow static schedule in SPMD mode.
466673471bf0Spatrick       unsigned ScheduleArgOpNo = 2;
466773471bf0Spatrick       auto *ScheduleTypeCI =
466873471bf0Spatrick           dyn_cast<ConstantInt>(CB.getArgOperand(ScheduleArgOpNo));
466973471bf0Spatrick       unsigned ScheduleTypeVal =
467073471bf0Spatrick           ScheduleTypeCI ? ScheduleTypeCI->getZExtValue() : 0;
467173471bf0Spatrick       switch (OMPScheduleType(ScheduleTypeVal)) {
4672*d415bd75Srobert       case OMPScheduleType::UnorderedStatic:
4673*d415bd75Srobert       case OMPScheduleType::UnorderedStaticChunked:
4674*d415bd75Srobert       case OMPScheduleType::OrderedDistribute:
4675*d415bd75Srobert       case OMPScheduleType::OrderedDistributeChunked:
467673471bf0Spatrick         break;
467773471bf0Spatrick       default:
4678*d415bd75Srobert         SPMDCompatibilityTracker.indicatePessimisticFixpoint();
467973471bf0Spatrick         SPMDCompatibilityTracker.insert(&CB);
468073471bf0Spatrick         break;
468173471bf0Spatrick       };
468273471bf0Spatrick     } break;
468373471bf0Spatrick     case OMPRTL___kmpc_target_init:
468473471bf0Spatrick       KernelInitCB = &CB;
468573471bf0Spatrick       break;
468673471bf0Spatrick     case OMPRTL___kmpc_target_deinit:
468773471bf0Spatrick       KernelDeinitCB = &CB;
468873471bf0Spatrick       break;
468973471bf0Spatrick     case OMPRTL___kmpc_parallel_51:
469073471bf0Spatrick       if (auto *ParallelRegion = dyn_cast<Function>(
469173471bf0Spatrick               CB.getArgOperand(WrapperFunctionArgNo)->stripPointerCasts())) {
469273471bf0Spatrick         ReachedKnownParallelRegions.insert(ParallelRegion);
4693*d415bd75Srobert         /// Check nested parallelism
4694*d415bd75Srobert         auto &FnAA = A.getAAFor<AAKernelInfo>(
4695*d415bd75Srobert             *this, IRPosition::function(*ParallelRegion), DepClassTy::OPTIONAL);
4696*d415bd75Srobert         NestedParallelism |= !FnAA.getState().isValidState() ||
4697*d415bd75Srobert                              !FnAA.ReachedKnownParallelRegions.empty() ||
4698*d415bd75Srobert                              !FnAA.ReachedUnknownParallelRegions.empty();
469973471bf0Spatrick         break;
470073471bf0Spatrick       }
470173471bf0Spatrick       // The condition above should usually get the parallel region function
470273471bf0Spatrick       // pointer and record it. In the off chance it doesn't we assume the
470373471bf0Spatrick       // worst.
470473471bf0Spatrick       ReachedUnknownParallelRegions.insert(&CB);
470573471bf0Spatrick       break;
470673471bf0Spatrick     case OMPRTL___kmpc_omp_task:
470773471bf0Spatrick       // We do not look into tasks right now, just give up.
4708*d415bd75Srobert       SPMDCompatibilityTracker.indicatePessimisticFixpoint();
470973471bf0Spatrick       SPMDCompatibilityTracker.insert(&CB);
471073471bf0Spatrick       ReachedUnknownParallelRegions.insert(&CB);
471173471bf0Spatrick       break;
471273471bf0Spatrick     case OMPRTL___kmpc_alloc_shared:
471373471bf0Spatrick     case OMPRTL___kmpc_free_shared:
471473471bf0Spatrick       // Return without setting a fixpoint, to be resolved in updateImpl.
471573471bf0Spatrick       return;
471673471bf0Spatrick     default:
471773471bf0Spatrick       // Unknown OpenMP runtime calls cannot be executed in SPMD-mode,
4718*d415bd75Srobert       // generally. However, they do not hide parallel regions.
4719*d415bd75Srobert       SPMDCompatibilityTracker.indicatePessimisticFixpoint();
472073471bf0Spatrick       SPMDCompatibilityTracker.insert(&CB);
472173471bf0Spatrick       break;
472273471bf0Spatrick     }
472373471bf0Spatrick     // All other OpenMP runtime calls will not reach parallel regions so they
472473471bf0Spatrick     // can be safely ignored for now. Since it is a known OpenMP runtime call we
472573471bf0Spatrick     // have now modeled all effects and there is no need for any update.
472673471bf0Spatrick     indicateOptimisticFixpoint();
472773471bf0Spatrick   }
472873471bf0Spatrick 
updateImpl__anon4b47cb5e0111::AAKernelInfoCallSite472973471bf0Spatrick   ChangeStatus updateImpl(Attributor &A) override {
473073471bf0Spatrick     // TODO: Once we have call site specific value information we can provide
473173471bf0Spatrick     //       call site specific liveness information and then it makes
473273471bf0Spatrick     //       sense to specialize attributes for call sites arguments instead of
473373471bf0Spatrick     //       redirecting requests to the callee argument.
473473471bf0Spatrick     Function *F = getAssociatedFunction();
473573471bf0Spatrick 
473673471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
473773471bf0Spatrick     const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(F);
473873471bf0Spatrick 
473973471bf0Spatrick     // If F is not a runtime function, propagate the AAKernelInfo of the callee.
474073471bf0Spatrick     if (It == OMPInfoCache.RuntimeFunctionIDMap.end()) {
474173471bf0Spatrick       const IRPosition &FnPos = IRPosition::function(*F);
474273471bf0Spatrick       auto &FnAA = A.getAAFor<AAKernelInfo>(*this, FnPos, DepClassTy::REQUIRED);
474373471bf0Spatrick       if (getState() == FnAA.getState())
474473471bf0Spatrick         return ChangeStatus::UNCHANGED;
474573471bf0Spatrick       getState() = FnAA.getState();
474673471bf0Spatrick       return ChangeStatus::CHANGED;
474773471bf0Spatrick     }
474873471bf0Spatrick 
474973471bf0Spatrick     // F is a runtime function that allocates or frees memory, check
475073471bf0Spatrick     // AAHeapToStack and AAHeapToShared.
475173471bf0Spatrick     KernelInfoState StateBefore = getState();
475273471bf0Spatrick     assert((It->getSecond() == OMPRTL___kmpc_alloc_shared ||
475373471bf0Spatrick             It->getSecond() == OMPRTL___kmpc_free_shared) &&
475473471bf0Spatrick            "Expected a __kmpc_alloc_shared or __kmpc_free_shared runtime call");
475573471bf0Spatrick 
475673471bf0Spatrick     CallBase &CB = cast<CallBase>(getAssociatedValue());
475773471bf0Spatrick 
475873471bf0Spatrick     auto &HeapToStackAA = A.getAAFor<AAHeapToStack>(
475973471bf0Spatrick         *this, IRPosition::function(*CB.getCaller()), DepClassTy::OPTIONAL);
476073471bf0Spatrick     auto &HeapToSharedAA = A.getAAFor<AAHeapToShared>(
476173471bf0Spatrick         *this, IRPosition::function(*CB.getCaller()), DepClassTy::OPTIONAL);
476273471bf0Spatrick 
476373471bf0Spatrick     RuntimeFunction RF = It->getSecond();
476473471bf0Spatrick 
476573471bf0Spatrick     switch (RF) {
476673471bf0Spatrick     // If neither HeapToStack nor HeapToShared assume the call is removed,
476773471bf0Spatrick     // assume SPMD incompatibility.
476873471bf0Spatrick     case OMPRTL___kmpc_alloc_shared:
476973471bf0Spatrick       if (!HeapToStackAA.isAssumedHeapToStack(CB) &&
477073471bf0Spatrick           !HeapToSharedAA.isAssumedHeapToShared(CB))
477173471bf0Spatrick         SPMDCompatibilityTracker.insert(&CB);
477273471bf0Spatrick       break;
477373471bf0Spatrick     case OMPRTL___kmpc_free_shared:
477473471bf0Spatrick       if (!HeapToStackAA.isAssumedHeapToStackRemovedFree(CB) &&
477573471bf0Spatrick           !HeapToSharedAA.isAssumedHeapToSharedRemovedFree(CB))
477673471bf0Spatrick         SPMDCompatibilityTracker.insert(&CB);
477773471bf0Spatrick       break;
477873471bf0Spatrick     default:
4779*d415bd75Srobert       SPMDCompatibilityTracker.indicatePessimisticFixpoint();
478073471bf0Spatrick       SPMDCompatibilityTracker.insert(&CB);
478173471bf0Spatrick     }
478273471bf0Spatrick 
478373471bf0Spatrick     return StateBefore == getState() ? ChangeStatus::UNCHANGED
478473471bf0Spatrick                                      : ChangeStatus::CHANGED;
478573471bf0Spatrick   }
478673471bf0Spatrick };
478773471bf0Spatrick 
478873471bf0Spatrick struct AAFoldRuntimeCall
478973471bf0Spatrick     : public StateWrapper<BooleanState, AbstractAttribute> {
479073471bf0Spatrick   using Base = StateWrapper<BooleanState, AbstractAttribute>;
479173471bf0Spatrick 
AAFoldRuntimeCall__anon4b47cb5e0111::AAFoldRuntimeCall479273471bf0Spatrick   AAFoldRuntimeCall(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
479373471bf0Spatrick 
479473471bf0Spatrick   /// Statistics are tracked as part of manifest for now.
trackStatistics__anon4b47cb5e0111::AAFoldRuntimeCall479573471bf0Spatrick   void trackStatistics() const override {}
479673471bf0Spatrick 
479773471bf0Spatrick   /// Create an abstract attribute biew for the position \p IRP.
479873471bf0Spatrick   static AAFoldRuntimeCall &createForPosition(const IRPosition &IRP,
479973471bf0Spatrick                                               Attributor &A);
480073471bf0Spatrick 
480173471bf0Spatrick   /// See AbstractAttribute::getName()
getName__anon4b47cb5e0111::AAFoldRuntimeCall480273471bf0Spatrick   const std::string getName() const override { return "AAFoldRuntimeCall"; }
480373471bf0Spatrick 
480473471bf0Spatrick   /// See AbstractAttribute::getIdAddr()
getIdAddr__anon4b47cb5e0111::AAFoldRuntimeCall480573471bf0Spatrick   const char *getIdAddr() const override { return &ID; }
480673471bf0Spatrick 
480773471bf0Spatrick   /// This function should return true if the type of the \p AA is
480873471bf0Spatrick   /// AAFoldRuntimeCall
classof__anon4b47cb5e0111::AAFoldRuntimeCall480973471bf0Spatrick   static bool classof(const AbstractAttribute *AA) {
481073471bf0Spatrick     return (AA->getIdAddr() == &ID);
481173471bf0Spatrick   }
481273471bf0Spatrick 
481373471bf0Spatrick   static const char ID;
481473471bf0Spatrick };
481573471bf0Spatrick 
481673471bf0Spatrick struct AAFoldRuntimeCallCallSiteReturned : AAFoldRuntimeCall {
AAFoldRuntimeCallCallSiteReturned__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned481773471bf0Spatrick   AAFoldRuntimeCallCallSiteReturned(const IRPosition &IRP, Attributor &A)
481873471bf0Spatrick       : AAFoldRuntimeCall(IRP, A) {}
481973471bf0Spatrick 
482073471bf0Spatrick   /// See AbstractAttribute::getAsStr()
getAsStr__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned482173471bf0Spatrick   const std::string getAsStr() const override {
482273471bf0Spatrick     if (!isValidState())
482373471bf0Spatrick       return "<invalid>";
482473471bf0Spatrick 
482573471bf0Spatrick     std::string Str("simplified value: ");
482673471bf0Spatrick 
4827*d415bd75Srobert     if (!SimplifiedValue)
482873471bf0Spatrick       return Str + std::string("none");
482973471bf0Spatrick 
4830*d415bd75Srobert     if (!*SimplifiedValue)
483173471bf0Spatrick       return Str + std::string("nullptr");
483273471bf0Spatrick 
4833*d415bd75Srobert     if (ConstantInt *CI = dyn_cast<ConstantInt>(*SimplifiedValue))
483473471bf0Spatrick       return Str + std::to_string(CI->getSExtValue());
483573471bf0Spatrick 
483673471bf0Spatrick     return Str + std::string("unknown");
483773471bf0Spatrick   }
483873471bf0Spatrick 
initialize__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned483973471bf0Spatrick   void initialize(Attributor &A) override {
4840*d415bd75Srobert     if (DisableOpenMPOptFolding)
4841*d415bd75Srobert       indicatePessimisticFixpoint();
4842*d415bd75Srobert 
484373471bf0Spatrick     Function *Callee = getAssociatedFunction();
484473471bf0Spatrick 
484573471bf0Spatrick     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
484673471bf0Spatrick     const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(Callee);
484773471bf0Spatrick     assert(It != OMPInfoCache.RuntimeFunctionIDMap.end() &&
484873471bf0Spatrick            "Expected a known OpenMP runtime function");
484973471bf0Spatrick 
485073471bf0Spatrick     RFKind = It->getSecond();
485173471bf0Spatrick 
485273471bf0Spatrick     CallBase &CB = cast<CallBase>(getAssociatedValue());
485373471bf0Spatrick     A.registerSimplificationCallback(
485473471bf0Spatrick         IRPosition::callsite_returned(CB),
485573471bf0Spatrick         [&](const IRPosition &IRP, const AbstractAttribute *AA,
4856*d415bd75Srobert             bool &UsedAssumedInformation) -> std::optional<Value *> {
4857*d415bd75Srobert           assert((isValidState() ||
4858*d415bd75Srobert                   (SimplifiedValue && *SimplifiedValue == nullptr)) &&
485973471bf0Spatrick                  "Unexpected invalid state!");
486073471bf0Spatrick 
486173471bf0Spatrick           if (!isAtFixpoint()) {
486273471bf0Spatrick             UsedAssumedInformation = true;
486373471bf0Spatrick             if (AA)
486473471bf0Spatrick               A.recordDependence(*this, *AA, DepClassTy::OPTIONAL);
486573471bf0Spatrick           }
486673471bf0Spatrick           return SimplifiedValue;
486773471bf0Spatrick         });
486873471bf0Spatrick   }
486973471bf0Spatrick 
updateImpl__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned487073471bf0Spatrick   ChangeStatus updateImpl(Attributor &A) override {
487173471bf0Spatrick     ChangeStatus Changed = ChangeStatus::UNCHANGED;
487273471bf0Spatrick     switch (RFKind) {
487373471bf0Spatrick     case OMPRTL___kmpc_is_spmd_exec_mode:
487473471bf0Spatrick       Changed |= foldIsSPMDExecMode(A);
487573471bf0Spatrick       break;
487673471bf0Spatrick     case OMPRTL___kmpc_parallel_level:
487773471bf0Spatrick       Changed |= foldParallelLevel(A);
487873471bf0Spatrick       break;
487973471bf0Spatrick     case OMPRTL___kmpc_get_hardware_num_threads_in_block:
488073471bf0Spatrick       Changed = Changed | foldKernelFnAttribute(A, "omp_target_thread_limit");
488173471bf0Spatrick       break;
488273471bf0Spatrick     case OMPRTL___kmpc_get_hardware_num_blocks:
488373471bf0Spatrick       Changed = Changed | foldKernelFnAttribute(A, "omp_target_num_teams");
488473471bf0Spatrick       break;
488573471bf0Spatrick     default:
488673471bf0Spatrick       llvm_unreachable("Unhandled OpenMP runtime function!");
488773471bf0Spatrick     }
488873471bf0Spatrick 
488973471bf0Spatrick     return Changed;
489073471bf0Spatrick   }
489173471bf0Spatrick 
manifest__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned489273471bf0Spatrick   ChangeStatus manifest(Attributor &A) override {
489373471bf0Spatrick     ChangeStatus Changed = ChangeStatus::UNCHANGED;
489473471bf0Spatrick 
4895*d415bd75Srobert     if (SimplifiedValue && *SimplifiedValue) {
4896*d415bd75Srobert       Instruction &I = *getCtxI();
4897*d415bd75Srobert       A.changeAfterManifest(IRPosition::inst(I), **SimplifiedValue);
4898*d415bd75Srobert       A.deleteAfterManifest(I);
489973471bf0Spatrick 
4900*d415bd75Srobert       CallBase *CB = dyn_cast<CallBase>(&I);
4901*d415bd75Srobert       auto Remark = [&](OptimizationRemark OR) {
4902*d415bd75Srobert         if (auto *C = dyn_cast<ConstantInt>(*SimplifiedValue))
4903*d415bd75Srobert           return OR << "Replacing OpenMP runtime call "
4904*d415bd75Srobert                     << CB->getCalledFunction()->getName() << " with "
4905*d415bd75Srobert                     << ore::NV("FoldedValue", C->getZExtValue()) << ".";
4906*d415bd75Srobert         return OR << "Replacing OpenMP runtime call "
4907*d415bd75Srobert                   << CB->getCalledFunction()->getName() << ".";
4908*d415bd75Srobert       };
4909*d415bd75Srobert 
4910*d415bd75Srobert       if (CB && EnableVerboseRemarks)
4911*d415bd75Srobert         A.emitRemark<OptimizationRemark>(CB, "OMP180", Remark);
4912*d415bd75Srobert 
4913*d415bd75Srobert       LLVM_DEBUG(dbgs() << TAG << "Replacing runtime call: " << I << " with "
491473471bf0Spatrick                         << **SimplifiedValue << "\n");
491573471bf0Spatrick 
491673471bf0Spatrick       Changed = ChangeStatus::CHANGED;
491773471bf0Spatrick     }
491873471bf0Spatrick 
491973471bf0Spatrick     return Changed;
492073471bf0Spatrick   }
492173471bf0Spatrick 
indicatePessimisticFixpoint__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned492273471bf0Spatrick   ChangeStatus indicatePessimisticFixpoint() override {
492373471bf0Spatrick     SimplifiedValue = nullptr;
492473471bf0Spatrick     return AAFoldRuntimeCall::indicatePessimisticFixpoint();
492573471bf0Spatrick   }
492673471bf0Spatrick 
492773471bf0Spatrick private:
492873471bf0Spatrick   /// Fold __kmpc_is_spmd_exec_mode into a constant if possible.
foldIsSPMDExecMode__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned492973471bf0Spatrick   ChangeStatus foldIsSPMDExecMode(Attributor &A) {
4930*d415bd75Srobert     std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
493173471bf0Spatrick 
493273471bf0Spatrick     unsigned AssumedSPMDCount = 0, KnownSPMDCount = 0;
493373471bf0Spatrick     unsigned AssumedNonSPMDCount = 0, KnownNonSPMDCount = 0;
493473471bf0Spatrick     auto &CallerKernelInfoAA = A.getAAFor<AAKernelInfo>(
493573471bf0Spatrick         *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED);
493673471bf0Spatrick 
493773471bf0Spatrick     if (!CallerKernelInfoAA.ReachingKernelEntries.isValidState())
493873471bf0Spatrick       return indicatePessimisticFixpoint();
493973471bf0Spatrick 
494073471bf0Spatrick     for (Kernel K : CallerKernelInfoAA.ReachingKernelEntries) {
494173471bf0Spatrick       auto &AA = A.getAAFor<AAKernelInfo>(*this, IRPosition::function(*K),
494273471bf0Spatrick                                           DepClassTy::REQUIRED);
494373471bf0Spatrick 
494473471bf0Spatrick       if (!AA.isValidState()) {
494573471bf0Spatrick         SimplifiedValue = nullptr;
494673471bf0Spatrick         return indicatePessimisticFixpoint();
494773471bf0Spatrick       }
494873471bf0Spatrick 
494973471bf0Spatrick       if (AA.SPMDCompatibilityTracker.isAssumed()) {
495073471bf0Spatrick         if (AA.SPMDCompatibilityTracker.isAtFixpoint())
495173471bf0Spatrick           ++KnownSPMDCount;
495273471bf0Spatrick         else
495373471bf0Spatrick           ++AssumedSPMDCount;
495473471bf0Spatrick       } else {
495573471bf0Spatrick         if (AA.SPMDCompatibilityTracker.isAtFixpoint())
495673471bf0Spatrick           ++KnownNonSPMDCount;
495773471bf0Spatrick         else
495873471bf0Spatrick           ++AssumedNonSPMDCount;
495973471bf0Spatrick       }
496073471bf0Spatrick     }
496173471bf0Spatrick 
496273471bf0Spatrick     if ((AssumedSPMDCount + KnownSPMDCount) &&
496373471bf0Spatrick         (AssumedNonSPMDCount + KnownNonSPMDCount))
496473471bf0Spatrick       return indicatePessimisticFixpoint();
496573471bf0Spatrick 
496673471bf0Spatrick     auto &Ctx = getAnchorValue().getContext();
496773471bf0Spatrick     if (KnownSPMDCount || AssumedSPMDCount) {
496873471bf0Spatrick       assert(KnownNonSPMDCount == 0 && AssumedNonSPMDCount == 0 &&
496973471bf0Spatrick              "Expected only SPMD kernels!");
497073471bf0Spatrick       // All reaching kernels are in SPMD mode. Update all function calls to
497173471bf0Spatrick       // __kmpc_is_spmd_exec_mode to 1.
497273471bf0Spatrick       SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx), true);
497373471bf0Spatrick     } else if (KnownNonSPMDCount || AssumedNonSPMDCount) {
497473471bf0Spatrick       assert(KnownSPMDCount == 0 && AssumedSPMDCount == 0 &&
497573471bf0Spatrick              "Expected only non-SPMD kernels!");
497673471bf0Spatrick       // All reaching kernels are in non-SPMD mode. Update all function
497773471bf0Spatrick       // calls to __kmpc_is_spmd_exec_mode to 0.
497873471bf0Spatrick       SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx), false);
497973471bf0Spatrick     } else {
498073471bf0Spatrick       // We have empty reaching kernels, therefore we cannot tell if the
498173471bf0Spatrick       // associated call site can be folded. At this moment, SimplifiedValue
498273471bf0Spatrick       // must be none.
4983*d415bd75Srobert       assert(!SimplifiedValue && "SimplifiedValue should be none");
498473471bf0Spatrick     }
498573471bf0Spatrick 
498673471bf0Spatrick     return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
498773471bf0Spatrick                                                     : ChangeStatus::CHANGED;
498873471bf0Spatrick   }
498973471bf0Spatrick 
499073471bf0Spatrick   /// Fold __kmpc_parallel_level into a constant if possible.
foldParallelLevel__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned499173471bf0Spatrick   ChangeStatus foldParallelLevel(Attributor &A) {
4992*d415bd75Srobert     std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
499373471bf0Spatrick 
499473471bf0Spatrick     auto &CallerKernelInfoAA = A.getAAFor<AAKernelInfo>(
499573471bf0Spatrick         *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED);
499673471bf0Spatrick 
499773471bf0Spatrick     if (!CallerKernelInfoAA.ParallelLevels.isValidState())
499873471bf0Spatrick       return indicatePessimisticFixpoint();
499973471bf0Spatrick 
500073471bf0Spatrick     if (!CallerKernelInfoAA.ReachingKernelEntries.isValidState())
500173471bf0Spatrick       return indicatePessimisticFixpoint();
500273471bf0Spatrick 
500373471bf0Spatrick     if (CallerKernelInfoAA.ReachingKernelEntries.empty()) {
5004*d415bd75Srobert       assert(!SimplifiedValue &&
500573471bf0Spatrick              "SimplifiedValue should keep none at this point");
500673471bf0Spatrick       return ChangeStatus::UNCHANGED;
500773471bf0Spatrick     }
500873471bf0Spatrick 
500973471bf0Spatrick     unsigned AssumedSPMDCount = 0, KnownSPMDCount = 0;
501073471bf0Spatrick     unsigned AssumedNonSPMDCount = 0, KnownNonSPMDCount = 0;
501173471bf0Spatrick     for (Kernel K : CallerKernelInfoAA.ReachingKernelEntries) {
501273471bf0Spatrick       auto &AA = A.getAAFor<AAKernelInfo>(*this, IRPosition::function(*K),
501373471bf0Spatrick                                           DepClassTy::REQUIRED);
501473471bf0Spatrick       if (!AA.SPMDCompatibilityTracker.isValidState())
501573471bf0Spatrick         return indicatePessimisticFixpoint();
501673471bf0Spatrick 
501773471bf0Spatrick       if (AA.SPMDCompatibilityTracker.isAssumed()) {
501873471bf0Spatrick         if (AA.SPMDCompatibilityTracker.isAtFixpoint())
501973471bf0Spatrick           ++KnownSPMDCount;
502073471bf0Spatrick         else
502173471bf0Spatrick           ++AssumedSPMDCount;
502273471bf0Spatrick       } else {
502373471bf0Spatrick         if (AA.SPMDCompatibilityTracker.isAtFixpoint())
502473471bf0Spatrick           ++KnownNonSPMDCount;
502573471bf0Spatrick         else
502673471bf0Spatrick           ++AssumedNonSPMDCount;
502773471bf0Spatrick       }
502873471bf0Spatrick     }
502973471bf0Spatrick 
503073471bf0Spatrick     if ((AssumedSPMDCount + KnownSPMDCount) &&
503173471bf0Spatrick         (AssumedNonSPMDCount + KnownNonSPMDCount))
503273471bf0Spatrick       return indicatePessimisticFixpoint();
503373471bf0Spatrick 
503473471bf0Spatrick     auto &Ctx = getAnchorValue().getContext();
503573471bf0Spatrick     // If the caller can only be reached by SPMD kernel entries, the parallel
503673471bf0Spatrick     // level is 1. Similarly, if the caller can only be reached by non-SPMD
503773471bf0Spatrick     // kernel entries, it is 0.
503873471bf0Spatrick     if (AssumedSPMDCount || KnownSPMDCount) {
503973471bf0Spatrick       assert(KnownNonSPMDCount == 0 && AssumedNonSPMDCount == 0 &&
504073471bf0Spatrick              "Expected only SPMD kernels!");
504173471bf0Spatrick       SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx), 1);
504273471bf0Spatrick     } else {
504373471bf0Spatrick       assert(KnownSPMDCount == 0 && AssumedSPMDCount == 0 &&
504473471bf0Spatrick              "Expected only non-SPMD kernels!");
504573471bf0Spatrick       SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx), 0);
504673471bf0Spatrick     }
504773471bf0Spatrick     return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
504873471bf0Spatrick                                                     : ChangeStatus::CHANGED;
504973471bf0Spatrick   }
505073471bf0Spatrick 
foldKernelFnAttribute__anon4b47cb5e0111::AAFoldRuntimeCallCallSiteReturned505173471bf0Spatrick   ChangeStatus foldKernelFnAttribute(Attributor &A, llvm::StringRef Attr) {
505273471bf0Spatrick     // Specialize only if all the calls agree with the attribute constant value
505373471bf0Spatrick     int32_t CurrentAttrValue = -1;
5054*d415bd75Srobert     std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
505573471bf0Spatrick 
505673471bf0Spatrick     auto &CallerKernelInfoAA = A.getAAFor<AAKernelInfo>(
505773471bf0Spatrick         *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED);
505873471bf0Spatrick 
505973471bf0Spatrick     if (!CallerKernelInfoAA.ReachingKernelEntries.isValidState())
506073471bf0Spatrick       return indicatePessimisticFixpoint();
506173471bf0Spatrick 
506273471bf0Spatrick     // Iterate over the kernels that reach this function
506373471bf0Spatrick     for (Kernel K : CallerKernelInfoAA.ReachingKernelEntries) {
5064*d415bd75Srobert       int32_t NextAttrVal = K->getFnAttributeAsParsedInteger(Attr, -1);
506573471bf0Spatrick 
506673471bf0Spatrick       if (NextAttrVal == -1 ||
506773471bf0Spatrick           (CurrentAttrValue != -1 && CurrentAttrValue != NextAttrVal))
506873471bf0Spatrick         return indicatePessimisticFixpoint();
506973471bf0Spatrick       CurrentAttrValue = NextAttrVal;
507073471bf0Spatrick     }
507173471bf0Spatrick 
507273471bf0Spatrick     if (CurrentAttrValue != -1) {
507373471bf0Spatrick       auto &Ctx = getAnchorValue().getContext();
507473471bf0Spatrick       SimplifiedValue =
507573471bf0Spatrick           ConstantInt::get(Type::getInt32Ty(Ctx), CurrentAttrValue);
507673471bf0Spatrick     }
507773471bf0Spatrick     return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
507873471bf0Spatrick                                                     : ChangeStatus::CHANGED;
507973471bf0Spatrick   }
508073471bf0Spatrick 
508173471bf0Spatrick   /// An optional value the associated value is assumed to fold to. That is, we
508273471bf0Spatrick   /// assume the associated value (which is a call) can be replaced by this
508373471bf0Spatrick   /// simplified value.
5084*d415bd75Srobert   std::optional<Value *> SimplifiedValue;
508573471bf0Spatrick 
508673471bf0Spatrick   /// The runtime function kind of the callee of the associated call site.
508773471bf0Spatrick   RuntimeFunction RFKind;
508873471bf0Spatrick };
508973471bf0Spatrick 
5090097a140dSpatrick } // namespace
5091097a140dSpatrick 
509273471bf0Spatrick /// Register folding callsite
registerFoldRuntimeCall(RuntimeFunction RF)509373471bf0Spatrick void OpenMPOpt::registerFoldRuntimeCall(RuntimeFunction RF) {
509473471bf0Spatrick   auto &RFI = OMPInfoCache.RFIs[RF];
509573471bf0Spatrick   RFI.foreachUse(SCC, [&](Use &U, Function &F) {
509673471bf0Spatrick     CallInst *CI = OpenMPOpt::getCallIfRegularCall(U, &RFI);
509773471bf0Spatrick     if (!CI)
509873471bf0Spatrick       return false;
509973471bf0Spatrick     A.getOrCreateAAFor<AAFoldRuntimeCall>(
510073471bf0Spatrick         IRPosition::callsite_returned(*CI), /* QueryingAA */ nullptr,
510173471bf0Spatrick         DepClassTy::NONE, /* ForceUpdate */ false,
510273471bf0Spatrick         /* UpdateAfterInit */ false);
510373471bf0Spatrick     return false;
510473471bf0Spatrick   });
510573471bf0Spatrick }
510673471bf0Spatrick 
registerAAs(bool IsModulePass)510773471bf0Spatrick void OpenMPOpt::registerAAs(bool IsModulePass) {
510873471bf0Spatrick   if (SCC.empty())
510973471bf0Spatrick     return;
5110*d415bd75Srobert 
511173471bf0Spatrick   if (IsModulePass) {
511273471bf0Spatrick     // Ensure we create the AAKernelInfo AAs first and without triggering an
511373471bf0Spatrick     // update. This will make sure we register all value simplification
511473471bf0Spatrick     // callbacks before any other AA has the chance to create an AAValueSimplify
511573471bf0Spatrick     // or similar.
5116*d415bd75Srobert     auto CreateKernelInfoCB = [&](Use &, Function &Kernel) {
511773471bf0Spatrick       A.getOrCreateAAFor<AAKernelInfo>(
5118*d415bd75Srobert           IRPosition::function(Kernel), /* QueryingAA */ nullptr,
511973471bf0Spatrick           DepClassTy::NONE, /* ForceUpdate */ false,
512073471bf0Spatrick           /* UpdateAfterInit */ false);
5121*d415bd75Srobert       return false;
5122*d415bd75Srobert     };
5123*d415bd75Srobert     OMPInformationCache::RuntimeFunctionInfo &InitRFI =
5124*d415bd75Srobert         OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
5125*d415bd75Srobert     InitRFI.foreachUse(SCC, CreateKernelInfoCB);
512673471bf0Spatrick 
512773471bf0Spatrick     registerFoldRuntimeCall(OMPRTL___kmpc_is_spmd_exec_mode);
512873471bf0Spatrick     registerFoldRuntimeCall(OMPRTL___kmpc_parallel_level);
512973471bf0Spatrick     registerFoldRuntimeCall(OMPRTL___kmpc_get_hardware_num_threads_in_block);
513073471bf0Spatrick     registerFoldRuntimeCall(OMPRTL___kmpc_get_hardware_num_blocks);
513173471bf0Spatrick   }
513273471bf0Spatrick 
513373471bf0Spatrick   // Create CallSite AA for all Getters.
5134*d415bd75Srobert   if (DeduceICVValues) {
513573471bf0Spatrick     for (int Idx = 0; Idx < OMPInfoCache.ICVs.size() - 1; ++Idx) {
513673471bf0Spatrick       auto ICVInfo = OMPInfoCache.ICVs[static_cast<InternalControlVar>(Idx)];
513773471bf0Spatrick 
513873471bf0Spatrick       auto &GetterRFI = OMPInfoCache.RFIs[ICVInfo.Getter];
513973471bf0Spatrick 
514073471bf0Spatrick       auto CreateAA = [&](Use &U, Function &Caller) {
514173471bf0Spatrick         CallInst *CI = OpenMPOpt::getCallIfRegularCall(U, &GetterRFI);
514273471bf0Spatrick         if (!CI)
514373471bf0Spatrick           return false;
514473471bf0Spatrick 
514573471bf0Spatrick         auto &CB = cast<CallBase>(*CI);
514673471bf0Spatrick 
514773471bf0Spatrick         IRPosition CBPos = IRPosition::callsite_function(CB);
514873471bf0Spatrick         A.getOrCreateAAFor<AAICVTracker>(CBPos);
514973471bf0Spatrick         return false;
515073471bf0Spatrick       };
515173471bf0Spatrick 
515273471bf0Spatrick       GetterRFI.foreachUse(SCC, CreateAA);
515373471bf0Spatrick     }
5154*d415bd75Srobert   }
515573471bf0Spatrick 
515673471bf0Spatrick   // Create an ExecutionDomain AA for every function and a HeapToStack AA for
515773471bf0Spatrick   // every function if there is a device kernel.
515873471bf0Spatrick   if (!isOpenMPDevice(M))
515973471bf0Spatrick     return;
516073471bf0Spatrick 
516173471bf0Spatrick   for (auto *F : SCC) {
516273471bf0Spatrick     if (F->isDeclaration())
516373471bf0Spatrick       continue;
516473471bf0Spatrick 
5165*d415bd75Srobert     // We look at internal functions only on-demand but if any use is not a
5166*d415bd75Srobert     // direct call or outside the current set of analyzed functions, we have
5167*d415bd75Srobert     // to do it eagerly.
5168*d415bd75Srobert     if (F->hasLocalLinkage()) {
5169*d415bd75Srobert       if (llvm::all_of(F->uses(), [this](const Use &U) {
5170*d415bd75Srobert             const auto *CB = dyn_cast<CallBase>(U.getUser());
5171*d415bd75Srobert             return CB && CB->isCallee(&U) &&
5172*d415bd75Srobert                    A.isRunOn(const_cast<Function *>(CB->getCaller()));
5173*d415bd75Srobert           }))
5174*d415bd75Srobert         continue;
5175*d415bd75Srobert     }
5176*d415bd75Srobert     registerAAsForFunction(A, *F);
5177*d415bd75Srobert   }
5178*d415bd75Srobert }
517973471bf0Spatrick 
registerAAsForFunction(Attributor & A,const Function & F)5180*d415bd75Srobert void OpenMPOpt::registerAAsForFunction(Attributor &A, const Function &F) {
5181*d415bd75Srobert   if (!DisableOpenMPOptDeglobalization)
5182*d415bd75Srobert     A.getOrCreateAAFor<AAHeapToShared>(IRPosition::function(F));
5183*d415bd75Srobert   A.getOrCreateAAFor<AAExecutionDomain>(IRPosition::function(F));
5184*d415bd75Srobert   if (!DisableOpenMPOptDeglobalization)
5185*d415bd75Srobert     A.getOrCreateAAFor<AAHeapToStack>(IRPosition::function(F));
5186*d415bd75Srobert 
5187*d415bd75Srobert   for (auto &I : instructions(F)) {
518873471bf0Spatrick     if (auto *LI = dyn_cast<LoadInst>(&I)) {
518973471bf0Spatrick       bool UsedAssumedInformation = false;
519073471bf0Spatrick       A.getAssumedSimplified(IRPosition::value(*LI), /* AA */ nullptr,
5191*d415bd75Srobert                              UsedAssumedInformation, AA::Interprocedural);
5192*d415bd75Srobert       continue;
5193*d415bd75Srobert     }
5194*d415bd75Srobert     if (auto *SI = dyn_cast<StoreInst>(&I)) {
5195*d415bd75Srobert       A.getOrCreateAAFor<AAIsDead>(IRPosition::value(*SI));
5196*d415bd75Srobert       continue;
5197*d415bd75Srobert     }
5198*d415bd75Srobert     if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
5199*d415bd75Srobert       if (II->getIntrinsicID() == Intrinsic::assume) {
5200*d415bd75Srobert         A.getOrCreateAAFor<AAPotentialValues>(
5201*d415bd75Srobert             IRPosition::value(*II->getArgOperand(0)));
5202*d415bd75Srobert         continue;
520373471bf0Spatrick       }
520473471bf0Spatrick     }
520573471bf0Spatrick   }
520673471bf0Spatrick }
520773471bf0Spatrick 
5208097a140dSpatrick const char AAICVTracker::ID = 0;
520973471bf0Spatrick const char AAKernelInfo::ID = 0;
521073471bf0Spatrick const char AAExecutionDomain::ID = 0;
521173471bf0Spatrick const char AAHeapToShared::ID = 0;
521273471bf0Spatrick const char AAFoldRuntimeCall::ID = 0;
5213097a140dSpatrick 
createForPosition(const IRPosition & IRP,Attributor & A)5214097a140dSpatrick AAICVTracker &AAICVTracker::createForPosition(const IRPosition &IRP,
5215097a140dSpatrick                                               Attributor &A) {
5216097a140dSpatrick   AAICVTracker *AA = nullptr;
5217097a140dSpatrick   switch (IRP.getPositionKind()) {
5218097a140dSpatrick   case IRPosition::IRP_INVALID:
5219097a140dSpatrick   case IRPosition::IRP_FLOAT:
5220097a140dSpatrick   case IRPosition::IRP_ARGUMENT:
5221097a140dSpatrick   case IRPosition::IRP_CALL_SITE_ARGUMENT:
5222097a140dSpatrick     llvm_unreachable("ICVTracker can only be created for function position!");
522373471bf0Spatrick   case IRPosition::IRP_RETURNED:
522473471bf0Spatrick     AA = new (A.Allocator) AAICVTrackerFunctionReturned(IRP, A);
522573471bf0Spatrick     break;
522673471bf0Spatrick   case IRPosition::IRP_CALL_SITE_RETURNED:
522773471bf0Spatrick     AA = new (A.Allocator) AAICVTrackerCallSiteReturned(IRP, A);
522873471bf0Spatrick     break;
522973471bf0Spatrick   case IRPosition::IRP_CALL_SITE:
523073471bf0Spatrick     AA = new (A.Allocator) AAICVTrackerCallSite(IRP, A);
523173471bf0Spatrick     break;
5232097a140dSpatrick   case IRPosition::IRP_FUNCTION:
5233097a140dSpatrick     AA = new (A.Allocator) AAICVTrackerFunction(IRP, A);
5234097a140dSpatrick     break;
5235097a140dSpatrick   }
5236097a140dSpatrick 
5237097a140dSpatrick   return *AA;
5238097a140dSpatrick }
5239097a140dSpatrick 
createForPosition(const IRPosition & IRP,Attributor & A)524073471bf0Spatrick AAExecutionDomain &AAExecutionDomain::createForPosition(const IRPosition &IRP,
524173471bf0Spatrick                                                         Attributor &A) {
524273471bf0Spatrick   AAExecutionDomainFunction *AA = nullptr;
524373471bf0Spatrick   switch (IRP.getPositionKind()) {
524473471bf0Spatrick   case IRPosition::IRP_INVALID:
524573471bf0Spatrick   case IRPosition::IRP_FLOAT:
524673471bf0Spatrick   case IRPosition::IRP_ARGUMENT:
524773471bf0Spatrick   case IRPosition::IRP_CALL_SITE_ARGUMENT:
524873471bf0Spatrick   case IRPosition::IRP_RETURNED:
524973471bf0Spatrick   case IRPosition::IRP_CALL_SITE_RETURNED:
525073471bf0Spatrick   case IRPosition::IRP_CALL_SITE:
525173471bf0Spatrick     llvm_unreachable(
525273471bf0Spatrick         "AAExecutionDomain can only be created for function position!");
525373471bf0Spatrick   case IRPosition::IRP_FUNCTION:
525473471bf0Spatrick     AA = new (A.Allocator) AAExecutionDomainFunction(IRP, A);
525573471bf0Spatrick     break;
525673471bf0Spatrick   }
525773471bf0Spatrick 
525873471bf0Spatrick   return *AA;
525973471bf0Spatrick }
526073471bf0Spatrick 
createForPosition(const IRPosition & IRP,Attributor & A)526173471bf0Spatrick AAHeapToShared &AAHeapToShared::createForPosition(const IRPosition &IRP,
526273471bf0Spatrick                                                   Attributor &A) {
526373471bf0Spatrick   AAHeapToSharedFunction *AA = nullptr;
526473471bf0Spatrick   switch (IRP.getPositionKind()) {
526573471bf0Spatrick   case IRPosition::IRP_INVALID:
526673471bf0Spatrick   case IRPosition::IRP_FLOAT:
526773471bf0Spatrick   case IRPosition::IRP_ARGUMENT:
526873471bf0Spatrick   case IRPosition::IRP_CALL_SITE_ARGUMENT:
526973471bf0Spatrick   case IRPosition::IRP_RETURNED:
527073471bf0Spatrick   case IRPosition::IRP_CALL_SITE_RETURNED:
527173471bf0Spatrick   case IRPosition::IRP_CALL_SITE:
527273471bf0Spatrick     llvm_unreachable(
527373471bf0Spatrick         "AAHeapToShared can only be created for function position!");
527473471bf0Spatrick   case IRPosition::IRP_FUNCTION:
527573471bf0Spatrick     AA = new (A.Allocator) AAHeapToSharedFunction(IRP, A);
527673471bf0Spatrick     break;
527773471bf0Spatrick   }
527873471bf0Spatrick 
527973471bf0Spatrick   return *AA;
528073471bf0Spatrick }
528173471bf0Spatrick 
createForPosition(const IRPosition & IRP,Attributor & A)528273471bf0Spatrick AAKernelInfo &AAKernelInfo::createForPosition(const IRPosition &IRP,
528373471bf0Spatrick                                               Attributor &A) {
528473471bf0Spatrick   AAKernelInfo *AA = nullptr;
528573471bf0Spatrick   switch (IRP.getPositionKind()) {
528673471bf0Spatrick   case IRPosition::IRP_INVALID:
528773471bf0Spatrick   case IRPosition::IRP_FLOAT:
528873471bf0Spatrick   case IRPosition::IRP_ARGUMENT:
528973471bf0Spatrick   case IRPosition::IRP_RETURNED:
529073471bf0Spatrick   case IRPosition::IRP_CALL_SITE_RETURNED:
529173471bf0Spatrick   case IRPosition::IRP_CALL_SITE_ARGUMENT:
529273471bf0Spatrick     llvm_unreachable("KernelInfo can only be created for function position!");
529373471bf0Spatrick   case IRPosition::IRP_CALL_SITE:
529473471bf0Spatrick     AA = new (A.Allocator) AAKernelInfoCallSite(IRP, A);
529573471bf0Spatrick     break;
529673471bf0Spatrick   case IRPosition::IRP_FUNCTION:
529773471bf0Spatrick     AA = new (A.Allocator) AAKernelInfoFunction(IRP, A);
529873471bf0Spatrick     break;
529973471bf0Spatrick   }
530073471bf0Spatrick 
530173471bf0Spatrick   return *AA;
530273471bf0Spatrick }
530373471bf0Spatrick 
createForPosition(const IRPosition & IRP,Attributor & A)530473471bf0Spatrick AAFoldRuntimeCall &AAFoldRuntimeCall::createForPosition(const IRPosition &IRP,
530573471bf0Spatrick                                                         Attributor &A) {
530673471bf0Spatrick   AAFoldRuntimeCall *AA = nullptr;
530773471bf0Spatrick   switch (IRP.getPositionKind()) {
530873471bf0Spatrick   case IRPosition::IRP_INVALID:
530973471bf0Spatrick   case IRPosition::IRP_FLOAT:
531073471bf0Spatrick   case IRPosition::IRP_ARGUMENT:
531173471bf0Spatrick   case IRPosition::IRP_RETURNED:
531273471bf0Spatrick   case IRPosition::IRP_FUNCTION:
531373471bf0Spatrick   case IRPosition::IRP_CALL_SITE:
531473471bf0Spatrick   case IRPosition::IRP_CALL_SITE_ARGUMENT:
531573471bf0Spatrick     llvm_unreachable("KernelInfo can only be created for call site position!");
531673471bf0Spatrick   case IRPosition::IRP_CALL_SITE_RETURNED:
531773471bf0Spatrick     AA = new (A.Allocator) AAFoldRuntimeCallCallSiteReturned(IRP, A);
531873471bf0Spatrick     break;
531973471bf0Spatrick   }
532073471bf0Spatrick 
532173471bf0Spatrick   return *AA;
532273471bf0Spatrick }
532373471bf0Spatrick 
run(Module & M,ModuleAnalysisManager & AM)532473471bf0Spatrick PreservedAnalyses OpenMPOptPass::run(Module &M, ModuleAnalysisManager &AM) {
532573471bf0Spatrick   if (!containsOpenMP(M))
532673471bf0Spatrick     return PreservedAnalyses::all();
532773471bf0Spatrick   if (DisableOpenMPOptimizations)
5328097a140dSpatrick     return PreservedAnalyses::all();
5329097a140dSpatrick 
533073471bf0Spatrick   FunctionAnalysisManager &FAM =
533173471bf0Spatrick       AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
533273471bf0Spatrick   KernelSet Kernels = getDeviceKernels(M);
533373471bf0Spatrick 
5334*d415bd75Srobert   if (PrintModuleBeforeOptimizations)
5335*d415bd75Srobert     LLVM_DEBUG(dbgs() << TAG << "Module before OpenMPOpt Module Pass:\n" << M);
5336*d415bd75Srobert 
533773471bf0Spatrick   auto IsCalled = [&](Function &F) {
533873471bf0Spatrick     if (Kernels.contains(&F))
533973471bf0Spatrick       return true;
534073471bf0Spatrick     for (const User *U : F.users())
534173471bf0Spatrick       if (!isa<BlockAddress>(U))
534273471bf0Spatrick         return true;
534373471bf0Spatrick     return false;
534473471bf0Spatrick   };
534573471bf0Spatrick 
534673471bf0Spatrick   auto EmitRemark = [&](Function &F) {
534773471bf0Spatrick     auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
534873471bf0Spatrick     ORE.emit([&]() {
534973471bf0Spatrick       OptimizationRemarkAnalysis ORA(DEBUG_TYPE, "OMP140", &F);
535073471bf0Spatrick       return ORA << "Could not internalize function. "
535173471bf0Spatrick                  << "Some optimizations may not be possible. [OMP140]";
535273471bf0Spatrick     });
535373471bf0Spatrick   };
535473471bf0Spatrick 
535573471bf0Spatrick   // Create internal copies of each function if this is a kernel Module. This
535673471bf0Spatrick   // allows iterprocedural passes to see every call edge.
535773471bf0Spatrick   DenseMap<Function *, Function *> InternalizedMap;
535873471bf0Spatrick   if (isOpenMPDevice(M)) {
535973471bf0Spatrick     SmallPtrSet<Function *, 16> InternalizeFns;
536073471bf0Spatrick     for (Function &F : M)
536173471bf0Spatrick       if (!F.isDeclaration() && !Kernels.contains(&F) && IsCalled(F) &&
536273471bf0Spatrick           !DisableInternalization) {
536373471bf0Spatrick         if (Attributor::isInternalizable(F)) {
536473471bf0Spatrick           InternalizeFns.insert(&F);
536573471bf0Spatrick         } else if (!F.hasLocalLinkage() && !F.hasFnAttribute(Attribute::Cold)) {
536673471bf0Spatrick           EmitRemark(F);
536773471bf0Spatrick         }
536873471bf0Spatrick       }
536973471bf0Spatrick 
537073471bf0Spatrick     Attributor::internalizeFunctions(InternalizeFns, InternalizedMap);
537173471bf0Spatrick   }
537273471bf0Spatrick 
537373471bf0Spatrick   // Look at every function in the Module unless it was internalized.
5374*d415bd75Srobert   SetVector<Function *> Functions;
537573471bf0Spatrick   SmallVector<Function *, 16> SCC;
537673471bf0Spatrick   for (Function &F : M)
5377*d415bd75Srobert     if (!F.isDeclaration() && !InternalizedMap.lookup(&F)) {
537873471bf0Spatrick       SCC.push_back(&F);
5379*d415bd75Srobert       Functions.insert(&F);
5380*d415bd75Srobert     }
538173471bf0Spatrick 
538273471bf0Spatrick   if (SCC.empty())
538373471bf0Spatrick     return PreservedAnalyses::all();
538473471bf0Spatrick 
538573471bf0Spatrick   AnalysisGetter AG(FAM);
538673471bf0Spatrick 
538773471bf0Spatrick   auto OREGetter = [&FAM](Function *F) -> OptimizationRemarkEmitter & {
538873471bf0Spatrick     return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*F);
538973471bf0Spatrick   };
539073471bf0Spatrick 
539173471bf0Spatrick   BumpPtrAllocator Allocator;
539273471bf0Spatrick   CallGraphUpdater CGUpdater;
539373471bf0Spatrick 
5394*d415bd75Srobert   bool PostLink = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||
5395*d415bd75Srobert                   LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink;
5396*d415bd75Srobert   OMPInformationCache InfoCache(M, AG, Allocator, /*CGSCC*/ nullptr, Kernels,
5397*d415bd75Srobert                                 PostLink);
539873471bf0Spatrick 
5399*d415bd75Srobert   unsigned MaxFixpointIterations =
5400*d415bd75Srobert       (isOpenMPDevice(M)) ? SetFixpointIterations : 32;
5401*d415bd75Srobert 
5402*d415bd75Srobert   AttributorConfig AC(CGUpdater);
5403*d415bd75Srobert   AC.DefaultInitializeLiveInternals = false;
5404*d415bd75Srobert   AC.IsModulePass = true;
5405*d415bd75Srobert   AC.RewriteSignatures = false;
5406*d415bd75Srobert   AC.MaxFixpointIterations = MaxFixpointIterations;
5407*d415bd75Srobert   AC.OREGetter = OREGetter;
5408*d415bd75Srobert   AC.PassName = DEBUG_TYPE;
5409*d415bd75Srobert   AC.InitializationCallback = OpenMPOpt::registerAAsForFunction;
5410*d415bd75Srobert 
5411*d415bd75Srobert   Attributor A(Functions, InfoCache, AC);
541273471bf0Spatrick 
541373471bf0Spatrick   OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
541473471bf0Spatrick   bool Changed = OMPOpt.run(true);
5415*d415bd75Srobert 
5416*d415bd75Srobert   // Optionally inline device functions for potentially better performance.
5417*d415bd75Srobert   if (AlwaysInlineDeviceFunctions && isOpenMPDevice(M))
5418*d415bd75Srobert     for (Function &F : M)
5419*d415bd75Srobert       if (!F.isDeclaration() && !Kernels.contains(&F) &&
5420*d415bd75Srobert           !F.hasFnAttribute(Attribute::NoInline))
5421*d415bd75Srobert         F.addFnAttr(Attribute::AlwaysInline);
5422*d415bd75Srobert 
5423*d415bd75Srobert   if (PrintModuleAfterOptimizations)
5424*d415bd75Srobert     LLVM_DEBUG(dbgs() << TAG << "Module after OpenMPOpt Module Pass:\n" << M);
5425*d415bd75Srobert 
542673471bf0Spatrick   if (Changed)
542773471bf0Spatrick     return PreservedAnalyses::none();
542873471bf0Spatrick 
542973471bf0Spatrick   return PreservedAnalyses::all();
543073471bf0Spatrick }
543173471bf0Spatrick 
run(LazyCallGraph::SCC & C,CGSCCAnalysisManager & AM,LazyCallGraph & CG,CGSCCUpdateResult & UR)543273471bf0Spatrick PreservedAnalyses OpenMPOptCGSCCPass::run(LazyCallGraph::SCC &C,
543373471bf0Spatrick                                           CGSCCAnalysisManager &AM,
543473471bf0Spatrick                                           LazyCallGraph &CG,
543573471bf0Spatrick                                           CGSCCUpdateResult &UR) {
543673471bf0Spatrick   if (!containsOpenMP(*C.begin()->getFunction().getParent()))
543773471bf0Spatrick     return PreservedAnalyses::all();
5438097a140dSpatrick   if (DisableOpenMPOptimizations)
5439097a140dSpatrick     return PreservedAnalyses::all();
5440097a140dSpatrick 
5441097a140dSpatrick   SmallVector<Function *, 16> SCC;
544273471bf0Spatrick   // If there are kernels in the module, we have to run on all SCC's.
544373471bf0Spatrick   for (LazyCallGraph::Node &N : C) {
544473471bf0Spatrick     Function *Fn = &N.getFunction();
544573471bf0Spatrick     SCC.push_back(Fn);
544673471bf0Spatrick   }
5447097a140dSpatrick 
5448097a140dSpatrick   if (SCC.empty())
5449097a140dSpatrick     return PreservedAnalyses::all();
5450097a140dSpatrick 
545173471bf0Spatrick   Module &M = *C.begin()->getFunction().getParent();
545273471bf0Spatrick 
5453*d415bd75Srobert   if (PrintModuleBeforeOptimizations)
5454*d415bd75Srobert     LLVM_DEBUG(dbgs() << TAG << "Module before OpenMPOpt CGSCC Pass:\n" << M);
5455*d415bd75Srobert 
545673471bf0Spatrick   KernelSet Kernels = getDeviceKernels(M);
545773471bf0Spatrick 
5458097a140dSpatrick   FunctionAnalysisManager &FAM =
5459097a140dSpatrick       AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
5460097a140dSpatrick 
5461097a140dSpatrick   AnalysisGetter AG(FAM);
5462097a140dSpatrick 
5463097a140dSpatrick   auto OREGetter = [&FAM](Function *F) -> OptimizationRemarkEmitter & {
5464097a140dSpatrick     return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*F);
5465097a140dSpatrick   };
5466097a140dSpatrick 
546773471bf0Spatrick   BumpPtrAllocator Allocator;
5468097a140dSpatrick   CallGraphUpdater CGUpdater;
5469097a140dSpatrick   CGUpdater.initialize(CG, C, AM, UR);
5470097a140dSpatrick 
5471*d415bd75Srobert   bool PostLink = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||
5472*d415bd75Srobert                   LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink;
5473097a140dSpatrick   SetVector<Function *> Functions(SCC.begin(), SCC.end());
5474097a140dSpatrick   OMPInformationCache InfoCache(*(Functions.back()->getParent()), AG, Allocator,
5475*d415bd75Srobert                                 /*CGSCC*/ &Functions, Kernels, PostLink);
5476097a140dSpatrick 
5477*d415bd75Srobert   unsigned MaxFixpointIterations =
5478*d415bd75Srobert       (isOpenMPDevice(M)) ? SetFixpointIterations : 32;
5479*d415bd75Srobert 
5480*d415bd75Srobert   AttributorConfig AC(CGUpdater);
5481*d415bd75Srobert   AC.DefaultInitializeLiveInternals = false;
5482*d415bd75Srobert   AC.IsModulePass = false;
5483*d415bd75Srobert   AC.RewriteSignatures = false;
5484*d415bd75Srobert   AC.MaxFixpointIterations = MaxFixpointIterations;
5485*d415bd75Srobert   AC.OREGetter = OREGetter;
5486*d415bd75Srobert   AC.PassName = DEBUG_TYPE;
5487*d415bd75Srobert   AC.InitializationCallback = OpenMPOpt::registerAAsForFunction;
5488*d415bd75Srobert 
5489*d415bd75Srobert   Attributor A(Functions, InfoCache, AC);
5490097a140dSpatrick 
5491097a140dSpatrick   OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
549273471bf0Spatrick   bool Changed = OMPOpt.run(false);
5493*d415bd75Srobert 
5494*d415bd75Srobert   if (PrintModuleAfterOptimizations)
5495*d415bd75Srobert     LLVM_DEBUG(dbgs() << TAG << "Module after OpenMPOpt CGSCC Pass:\n" << M);
5496*d415bd75Srobert 
5497097a140dSpatrick   if (Changed)
5498097a140dSpatrick     return PreservedAnalyses::none();
5499097a140dSpatrick 
5500097a140dSpatrick   return PreservedAnalyses::all();
5501097a140dSpatrick }
5502097a140dSpatrick 
getDeviceKernels(Module & M)550373471bf0Spatrick KernelSet llvm::omp::getDeviceKernels(Module &M) {
550473471bf0Spatrick   // TODO: Create a more cross-platform way of determining device kernels.
5505*d415bd75Srobert   NamedMDNode *MD = M.getNamedMetadata("nvvm.annotations");
550673471bf0Spatrick   KernelSet Kernels;
550773471bf0Spatrick 
5508097a140dSpatrick   if (!MD)
550973471bf0Spatrick     return Kernels;
5510097a140dSpatrick 
5511097a140dSpatrick   for (auto *Op : MD->operands()) {
5512097a140dSpatrick     if (Op->getNumOperands() < 2)
5513097a140dSpatrick       continue;
5514097a140dSpatrick     MDString *KindID = dyn_cast<MDString>(Op->getOperand(1));
5515097a140dSpatrick     if (!KindID || KindID->getString() != "kernel")
5516097a140dSpatrick       continue;
5517097a140dSpatrick 
5518097a140dSpatrick     Function *KernelFn =
5519097a140dSpatrick         mdconst::dyn_extract_or_null<Function>(Op->getOperand(0));
5520097a140dSpatrick     if (!KernelFn)
5521097a140dSpatrick       continue;
5522097a140dSpatrick 
5523097a140dSpatrick     ++NumOpenMPTargetRegionKernels;
5524097a140dSpatrick 
5525097a140dSpatrick     Kernels.insert(KernelFn);
5526097a140dSpatrick   }
552773471bf0Spatrick 
552873471bf0Spatrick   return Kernels;
5529097a140dSpatrick }
5530097a140dSpatrick 
containsOpenMP(Module & M)553173471bf0Spatrick bool llvm::omp::containsOpenMP(Module &M) {
553273471bf0Spatrick   Metadata *MD = M.getModuleFlag("openmp");
553373471bf0Spatrick   if (!MD)
553473471bf0Spatrick     return false;
5535097a140dSpatrick 
5536097a140dSpatrick   return true;
5537097a140dSpatrick }
5538097a140dSpatrick 
isOpenMPDevice(Module & M)553973471bf0Spatrick bool llvm::omp::isOpenMPDevice(Module &M) {
554073471bf0Spatrick   Metadata *MD = M.getModuleFlag("openmp-device");
554173471bf0Spatrick   if (!MD)
554273471bf0Spatrick     return false;
554373471bf0Spatrick 
554473471bf0Spatrick   return true;
5545097a140dSpatrick }
5546