xref: /llvm-project/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp (revision 8818d13ed2fa5df316e4871c6cf1c6e534850b9f) 
1 //===- SIMemoryLegalizer.cpp ----------------------------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief Memory legalizer - implements memory model. More information can be
12 /// found here:
13 ///   http://llvm.org/docs/AMDGPUUsage.html#memory-model
14 //
15 //===----------------------------------------------------------------------===//
16 
17 #include "AMDGPU.h"
18 #include "AMDGPUMachineModuleInfo.h"
19 #include "AMDGPUSubtarget.h"
20 #include "SIDefines.h"
21 #include "SIInstrInfo.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/None.h"
24 #include "llvm/ADT/Optional.h"
25 #include "llvm/CodeGen/MachineBasicBlock.h"
26 #include "llvm/CodeGen/MachineFunction.h"
27 #include "llvm/CodeGen/MachineFunctionPass.h"
28 #include "llvm/CodeGen/MachineInstrBuilder.h"
29 #include "llvm/CodeGen/MachineMemOperand.h"
30 #include "llvm/CodeGen/MachineModuleInfo.h"
31 #include "llvm/CodeGen/MachineOperand.h"
32 #include "llvm/IR/DebugLoc.h"
33 #include "llvm/IR/DiagnosticInfo.h"
34 #include "llvm/IR/Function.h"
35 #include "llvm/IR/LLVMContext.h"
36 #include "llvm/MC/MCInstrDesc.h"
37 #include "llvm/Pass.h"
38 #include "llvm/Support/AtomicOrdering.h"
39 #include <cassert>
40 #include <list>
41 
42 using namespace llvm;
43 using namespace llvm::AMDGPU;
44 
45 #define DEBUG_TYPE "si-memory-legalizer"
46 #define PASS_NAME "SI Memory Legalizer"
47 
48 namespace {
49 
50 class SIMemOpInfo final {
51 private:
52   SyncScope::ID SSID = SyncScope::System;
53   AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
54   AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic;
55   bool IsNonTemporal = false;
56 
57   SIMemOpInfo(SyncScope::ID SSID, AtomicOrdering Ordering)
58       : SSID(SSID), Ordering(Ordering) {}
59 
60   SIMemOpInfo(SyncScope::ID SSID, AtomicOrdering Ordering,
61               AtomicOrdering FailureOrdering, bool IsNonTemporal = false)
62       : SSID(SSID), Ordering(Ordering), FailureOrdering(FailureOrdering),
63         IsNonTemporal(IsNonTemporal) {}
64 
65   /// \returns Info constructed from \p MI, which has at least machine memory
66   /// operand.
67   static Optional<SIMemOpInfo> constructFromMIWithMMO(
68       const MachineBasicBlock::iterator &MI);
69 
70 public:
71   /// \returns Synchronization scope ID of the machine instruction used to
72   /// create this SIMemOpInfo.
73   SyncScope::ID getSSID() const {
74     return SSID;
75   }
76   /// \returns Ordering constraint of the machine instruction used to
77   /// create this SIMemOpInfo.
78   AtomicOrdering getOrdering() const {
79     return Ordering;
80   }
81   /// \returns Failure ordering constraint of the machine instruction used to
82   /// create this SIMemOpInfo.
83   AtomicOrdering getFailureOrdering() const {
84     return FailureOrdering;
85   }
86   /// \returns True if memory access of the machine instruction used to
87   /// create this SIMemOpInfo is non-temporal, false otherwise.
88   bool isNonTemporal() const {
89     return IsNonTemporal;
90   }
91 
92   /// \returns True if ordering constraint of the machine instruction used to
93   /// create this SIMemOpInfo is unordered or higher, false otherwise.
94   bool isAtomic() const {
95     return Ordering != AtomicOrdering::NotAtomic;
96   }
97 
98   /// \returns Load info if \p MI is a load operation, "None" otherwise.
99   static Optional<SIMemOpInfo> getLoadInfo(
100       const MachineBasicBlock::iterator &MI);
101   /// \returns Store info if \p MI is a store operation, "None" otherwise.
102   static Optional<SIMemOpInfo> getStoreInfo(
103       const MachineBasicBlock::iterator &MI);
104   /// \returns Atomic fence info if \p MI is an atomic fence operation,
105   /// "None" otherwise.
106   static Optional<SIMemOpInfo> getAtomicFenceInfo(
107       const MachineBasicBlock::iterator &MI);
108   /// \returns Atomic cmpxchg info if \p MI is an atomic cmpxchg operation,
109   /// "None" otherwise.
110   static Optional<SIMemOpInfo> getAtomicCmpxchgInfo(
111       const MachineBasicBlock::iterator &MI);
112   /// \returns Atomic rmw info if \p MI is an atomic rmw operation,
113   /// "None" otherwise.
114   static Optional<SIMemOpInfo> getAtomicRmwInfo(
115       const MachineBasicBlock::iterator &MI);
116 
117   /// \brief Reports unknown synchronization scope used in \p MI to LLVM
118   /// context.
119   static void reportUnknownSyncScope(
120       const MachineBasicBlock::iterator &MI);
121 };
122 
123 class SIMemoryLegalizer final : public MachineFunctionPass {
124 private:
125   /// \brief Machine module info.
126   const AMDGPUMachineModuleInfo *MMI = nullptr;
127 
128   /// \brief Instruction info.
129   const SIInstrInfo *TII = nullptr;
130 
131   /// \brief Immediate for "vmcnt(0)".
132   unsigned Vmcnt0Immediate = 0;
133 
134   /// \brief Opcode for cache invalidation instruction (L1).
135   unsigned Wbinvl1Opcode = 0;
136 
137   /// \brief List of atomic pseudo instructions.
138   std::list<MachineBasicBlock::iterator> AtomicPseudoMIs;
139 
140   /// \brief Sets named bit (BitName) to "true" if present in \p MI. Returns
141   /// true if \p MI is modified, false otherwise.
142   template <uint16_t BitName>
143   bool enableNamedBit(const MachineBasicBlock::iterator &MI) const {
144     int BitIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), BitName);
145     if (BitIdx == -1)
146       return false;
147 
148     MachineOperand &Bit = MI->getOperand(BitIdx);
149     if (Bit.getImm() != 0)
150       return false;
151 
152     Bit.setImm(1);
153     return true;
154   }
155 
156   /// \brief Sets GLC bit to "true" if present in \p MI. Returns true if \p MI
157   /// is modified, false otherwise.
158   bool enableGLCBit(const MachineBasicBlock::iterator &MI) const {
159     return enableNamedBit<AMDGPU::OpName::glc>(MI);
160   }
161 
162   /// \brief Sets SLC bit to "true" if present in \p MI. Returns true if \p MI
163   /// is modified, false otherwise.
164   bool enableSLCBit(const MachineBasicBlock::iterator &MI) const {
165     return enableNamedBit<AMDGPU::OpName::slc>(MI);
166   }
167 
168   /// \brief Inserts "buffer_wbinvl1_vol" instruction \p Before or after \p MI.
169   /// Always returns true.
170   bool insertBufferWbinvl1Vol(MachineBasicBlock::iterator &MI,
171                               bool Before = true) const;
172   /// \brief Inserts "s_waitcnt vmcnt(0)" instruction \p Before or after \p MI.
173   /// Always returns true.
174   bool insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI,
175                            bool Before = true) const;
176 
177   /// \brief Removes all processed atomic pseudo instructions from the current
178   /// function. Returns true if current function is modified, false otherwise.
179   bool removeAtomicPseudoMIs();
180 
181   /// \brief Expands load operation \p MI. Returns true if instructions are
182   /// added/deleted or \p MI is modified, false otherwise.
183   bool expandLoad(const SIMemOpInfo &MOI,
184                   MachineBasicBlock::iterator &MI);
185   /// \brief Expands store operation \p MI. Returns true if instructions are
186   /// added/deleted or \p MI is modified, false otherwise.
187   bool expandStore(const SIMemOpInfo &MOI,
188                    MachineBasicBlock::iterator &MI);
189   /// \brief Expands atomic fence operation \p MI. Returns true if
190   /// instructions are added/deleted or \p MI is modified, false otherwise.
191   bool expandAtomicFence(const SIMemOpInfo &MOI,
192                          MachineBasicBlock::iterator &MI);
193   /// \brief Expands atomic cmpxchg operation \p MI. Returns true if
194   /// instructions are added/deleted or \p MI is modified, false otherwise.
195   bool expandAtomicCmpxchg(const SIMemOpInfo &MOI,
196                            MachineBasicBlock::iterator &MI);
197   /// \brief Expands atomic rmw operation \p MI. Returns true if
198   /// instructions are added/deleted or \p MI is modified, false otherwise.
199   bool expandAtomicRmw(const SIMemOpInfo &MOI,
200                        MachineBasicBlock::iterator &MI);
201 
202 public:
203   static char ID;
204 
205   SIMemoryLegalizer() : MachineFunctionPass(ID) {}
206 
207   void getAnalysisUsage(AnalysisUsage &AU) const override {
208     AU.setPreservesCFG();
209     MachineFunctionPass::getAnalysisUsage(AU);
210   }
211 
212   StringRef getPassName() const override {
213     return PASS_NAME;
214   }
215 
216   bool runOnMachineFunction(MachineFunction &MF) override;
217 };
218 
219 } // end namespace anonymous
220 
221 /* static */
222 Optional<SIMemOpInfo> SIMemOpInfo::constructFromMIWithMMO(
223     const MachineBasicBlock::iterator &MI) {
224   assert(MI->getNumMemOperands() > 0);
225 
226   const MachineFunction *MF = MI->getParent()->getParent();
227   const AMDGPUMachineModuleInfo *MMI =
228       &MF->getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>();
229 
230   SyncScope::ID SSID = SyncScope::SingleThread;
231   AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
232   AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic;
233   bool IsNonTemporal = true;
234 
235   // Validator should check whether or not MMOs cover the entire set of
236   // locations accessed by the memory instruction.
237   for (const auto &MMO : MI->memoperands()) {
238     const auto &IsSyncScopeInclusion =
239         MMI->isSyncScopeInclusion(SSID, MMO->getSyncScopeID());
240     if (!IsSyncScopeInclusion) {
241       reportUnknownSyncScope(MI);
242       return None;
243     }
244 
245     SSID = IsSyncScopeInclusion.getValue() ? SSID : MMO->getSyncScopeID();
246     Ordering =
247         isStrongerThan(Ordering, MMO->getOrdering()) ?
248             Ordering : MMO->getOrdering();
249     FailureOrdering =
250         isStrongerThan(FailureOrdering, MMO->getFailureOrdering()) ?
251             FailureOrdering : MMO->getFailureOrdering();
252 
253     if (!(MMO->getFlags() & MachineMemOperand::MONonTemporal))
254       IsNonTemporal = false;
255   }
256 
257   return SIMemOpInfo(SSID, Ordering, FailureOrdering, IsNonTemporal);
258 }
259 
260 /* static */
261 Optional<SIMemOpInfo> SIMemOpInfo::getLoadInfo(
262     const MachineBasicBlock::iterator &MI) {
263   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
264 
265   if (!(MI->mayLoad() && !MI->mayStore()))
266     return None;
267 
268   // Be conservative if there are no memory operands.
269   if (MI->getNumMemOperands() == 0)
270     return SIMemOpInfo(SyncScope::System,
271                        AtomicOrdering::SequentiallyConsistent);
272 
273   return SIMemOpInfo::constructFromMIWithMMO(MI);
274 }
275 
276 /* static */
277 Optional<SIMemOpInfo> SIMemOpInfo::getStoreInfo(
278     const MachineBasicBlock::iterator &MI) {
279   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
280 
281   if (!(!MI->mayLoad() && MI->mayStore()))
282     return None;
283 
284   // Be conservative if there are no memory operands.
285   if (MI->getNumMemOperands() == 0)
286     return SIMemOpInfo(SyncScope::System,
287                        AtomicOrdering::SequentiallyConsistent);
288 
289   return SIMemOpInfo::constructFromMIWithMMO(MI);
290 }
291 
292 /* static */
293 Optional<SIMemOpInfo> SIMemOpInfo::getAtomicFenceInfo(
294     const MachineBasicBlock::iterator &MI) {
295   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
296 
297   if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE)
298     return None;
299 
300   SyncScope::ID SSID =
301       static_cast<SyncScope::ID>(MI->getOperand(1).getImm());
302   AtomicOrdering Ordering =
303       static_cast<AtomicOrdering>(MI->getOperand(0).getImm());
304   return SIMemOpInfo(SSID, Ordering);
305 }
306 
307 /* static */
308 Optional<SIMemOpInfo> SIMemOpInfo::getAtomicCmpxchgInfo(
309     const MachineBasicBlock::iterator &MI) {
310   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
311 
312   if (!(MI->mayLoad() && MI->mayStore()))
313     return None;
314 
315   // Be conservative if there are no memory operands.
316   if (MI->getNumMemOperands() == 0)
317     return SIMemOpInfo(SyncScope::System,
318                        AtomicOrdering::SequentiallyConsistent,
319                        AtomicOrdering::SequentiallyConsistent);
320 
321   return SIMemOpInfo::constructFromMIWithMMO(MI);
322 }
323 
324 /* static */
325 Optional<SIMemOpInfo> SIMemOpInfo::getAtomicRmwInfo(
326     const MachineBasicBlock::iterator &MI) {
327   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
328 
329   if (!(MI->mayLoad() && MI->mayStore()))
330     return None;
331 
332   // Be conservative if there are no memory operands.
333   if (MI->getNumMemOperands() == 0)
334     return SIMemOpInfo(SyncScope::System,
335                        AtomicOrdering::SequentiallyConsistent);
336 
337   return SIMemOpInfo::constructFromMIWithMMO(MI);
338 }
339 
340 /* static */
341 void SIMemOpInfo::reportUnknownSyncScope(
342     const MachineBasicBlock::iterator &MI) {
343   DiagnosticInfoUnsupported Diag(MI->getParent()->getParent()->getFunction(),
344                                  "Unsupported synchronization scope");
345   LLVMContext *CTX = &MI->getParent()->getParent()->getFunction().getContext();
346   CTX->diagnose(Diag);
347 }
348 
349 bool SIMemoryLegalizer::insertBufferWbinvl1Vol(MachineBasicBlock::iterator &MI,
350                                                bool Before) const {
351   MachineBasicBlock &MBB = *MI->getParent();
352   DebugLoc DL = MI->getDebugLoc();
353 
354   if (!Before)
355     ++MI;
356 
357   BuildMI(MBB, MI, DL, TII->get(Wbinvl1Opcode));
358 
359   if (!Before)
360     --MI;
361 
362   return true;
363 }
364 
365 bool SIMemoryLegalizer::insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI,
366                                             bool Before) const {
367   MachineBasicBlock &MBB = *MI->getParent();
368   DebugLoc DL = MI->getDebugLoc();
369 
370   if (!Before)
371     ++MI;
372 
373   BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Vmcnt0Immediate);
374 
375   if (!Before)
376     --MI;
377 
378   return true;
379 }
380 
381 bool SIMemoryLegalizer::removeAtomicPseudoMIs() {
382   if (AtomicPseudoMIs.empty())
383     return false;
384 
385   for (auto &MI : AtomicPseudoMIs)
386     MI->eraseFromParent();
387 
388   AtomicPseudoMIs.clear();
389   return true;
390 }
391 
392 bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI,
393                                    MachineBasicBlock::iterator &MI) {
394   assert(MI->mayLoad() && !MI->mayStore());
395 
396   bool Changed = false;
397 
398   if (MOI.isAtomic()) {
399     if (MOI.getSSID() == SyncScope::System ||
400         MOI.getSSID() == MMI->getAgentSSID()) {
401       if (MOI.getOrdering() == AtomicOrdering::Monotonic ||
402           MOI.getOrdering() == AtomicOrdering::Acquire ||
403           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
404         Changed |= enableGLCBit(MI);
405 
406       if (MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
407         Changed |= insertWaitcntVmcnt0(MI);
408 
409       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
410           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
411         Changed |= insertWaitcntVmcnt0(MI, false);
412         Changed |= insertBufferWbinvl1Vol(MI, false);
413       }
414 
415       return Changed;
416     }
417 
418     if (MOI.getSSID() == SyncScope::SingleThread ||
419         MOI.getSSID() == MMI->getWorkgroupSSID() ||
420         MOI.getSSID() == MMI->getWavefrontSSID()) {
421       return Changed;
422     }
423 
424     llvm_unreachable("Unsupported synchronization scope");
425   }
426 
427   // Atomic instructions do not have the nontemporal attribute.
428   if (MOI.isNonTemporal()) {
429     Changed |= enableGLCBit(MI);
430     Changed |= enableSLCBit(MI);
431     return Changed;
432   }
433 
434   return Changed;
435 }
436 
437 bool SIMemoryLegalizer::expandStore(const SIMemOpInfo &MOI,
438                                     MachineBasicBlock::iterator &MI) {
439   assert(!MI->mayLoad() && MI->mayStore());
440 
441   bool Changed = false;
442 
443   if (MOI.isAtomic()) {
444     if (MOI.getSSID() == SyncScope::System ||
445         MOI.getSSID() == MMI->getAgentSSID()) {
446       if (MOI.getOrdering() == AtomicOrdering::Release ||
447           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
448         Changed |= insertWaitcntVmcnt0(MI);
449 
450       return Changed;
451     }
452 
453     if (MOI.getSSID() == SyncScope::SingleThread ||
454         MOI.getSSID() == MMI->getWorkgroupSSID() ||
455         MOI.getSSID() == MMI->getWavefrontSSID()) {
456       return Changed;
457     }
458 
459     llvm_unreachable("Unsupported synchronization scope");
460   }
461 
462   // Atomic instructions do not have the nontemporal attribute.
463   if (MOI.isNonTemporal()) {
464     Changed |= enableGLCBit(MI);
465     Changed |= enableSLCBit(MI);
466     return Changed;
467   }
468 
469   return Changed;
470 }
471 
472 bool SIMemoryLegalizer::expandAtomicFence(const SIMemOpInfo &MOI,
473                                           MachineBasicBlock::iterator &MI) {
474   assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE);
475 
476   bool Changed = false;
477 
478   if (MOI.isAtomic()) {
479     if (MOI.getSSID() == SyncScope::System ||
480         MOI.getSSID() == MMI->getAgentSSID()) {
481       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
482           MOI.getOrdering() == AtomicOrdering::Release ||
483           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
484           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
485         Changed |= insertWaitcntVmcnt0(MI);
486 
487       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
488           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
489           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
490         Changed |= insertBufferWbinvl1Vol(MI);
491 
492       AtomicPseudoMIs.push_back(MI);
493       return Changed;
494     }
495 
496     if (MOI.getSSID() == SyncScope::SingleThread ||
497         MOI.getSSID() == MMI->getWorkgroupSSID() ||
498         MOI.getSSID() == MMI->getWavefrontSSID()) {
499       AtomicPseudoMIs.push_back(MI);
500       return Changed;
501     }
502 
503     SIMemOpInfo::reportUnknownSyncScope(MI);
504   }
505 
506   return Changed;
507 }
508 
509 bool SIMemoryLegalizer::expandAtomicCmpxchg(const SIMemOpInfo &MOI,
510                                             MachineBasicBlock::iterator &MI) {
511   assert(MI->mayLoad() && MI->mayStore());
512 
513   bool Changed = false;
514 
515   if (MOI.isAtomic()) {
516     if (MOI.getSSID() == SyncScope::System ||
517         MOI.getSSID() == MMI->getAgentSSID()) {
518       if (MOI.getOrdering() == AtomicOrdering::Release ||
519           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
520           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent ||
521           MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)
522         Changed |= insertWaitcntVmcnt0(MI);
523 
524       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
525           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
526           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent ||
527           MOI.getFailureOrdering() == AtomicOrdering::Acquire ||
528           MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) {
529         Changed |= insertWaitcntVmcnt0(MI, false);
530         Changed |= insertBufferWbinvl1Vol(MI, false);
531       }
532 
533       return Changed;
534     }
535 
536     if (MOI.getSSID() == SyncScope::SingleThread ||
537         MOI.getSSID() == MMI->getWorkgroupSSID() ||
538         MOI.getSSID() == MMI->getWavefrontSSID()) {
539       Changed |= enableGLCBit(MI);
540       return Changed;
541     }
542 
543     llvm_unreachable("Unsupported synchronization scope");
544   }
545 
546   return Changed;
547 }
548 
549 bool SIMemoryLegalizer::expandAtomicRmw(const SIMemOpInfo &MOI,
550                                         MachineBasicBlock::iterator &MI) {
551   assert(MI->mayLoad() && MI->mayStore());
552 
553   bool Changed = false;
554 
555   if (MOI.isAtomic()) {
556     if (MOI.getSSID() == SyncScope::System ||
557         MOI.getSSID() == MMI->getAgentSSID()) {
558       if (MOI.getOrdering() == AtomicOrdering::Release ||
559           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
560           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
561         Changed |= insertWaitcntVmcnt0(MI);
562 
563       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
564           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
565           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
566         Changed |= insertWaitcntVmcnt0(MI, false);
567         Changed |= insertBufferWbinvl1Vol(MI, false);
568       }
569 
570       return Changed;
571     }
572 
573     if (MOI.getSSID() == SyncScope::SingleThread ||
574         MOI.getSSID() == MMI->getWorkgroupSSID() ||
575         MOI.getSSID() == MMI->getWavefrontSSID()) {
576       Changed |= enableGLCBit(MI);
577       return Changed;
578     }
579 
580     llvm_unreachable("Unsupported synchronization scope");
581   }
582 
583   return Changed;
584 }
585 
586 bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) {
587   bool Changed = false;
588   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
589   const IsaInfo::IsaVersion IV = IsaInfo::getIsaVersion(ST.getFeatureBits());
590 
591   MMI = &MF.getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>();
592   TII = ST.getInstrInfo();
593 
594   Vmcnt0Immediate =
595       AMDGPU::encodeWaitcnt(IV, 0, getExpcntBitMask(IV), getLgkmcntBitMask(IV));
596   Wbinvl1Opcode = ST.getGeneration() <= AMDGPUSubtarget::SOUTHERN_ISLANDS ?
597       AMDGPU::BUFFER_WBINVL1 : AMDGPU::BUFFER_WBINVL1_VOL;
598 
599   for (auto &MBB : MF) {
600     for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) {
601       if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic))
602         continue;
603 
604       if (const auto &MOI = SIMemOpInfo::getLoadInfo(MI))
605         Changed |= expandLoad(MOI.getValue(), MI);
606       else if (const auto &MOI = SIMemOpInfo::getStoreInfo(MI))
607         Changed |= expandStore(MOI.getValue(), MI);
608       else if (const auto &MOI = SIMemOpInfo::getAtomicFenceInfo(MI))
609         Changed |= expandAtomicFence(MOI.getValue(), MI);
610       else if (const auto &MOI = SIMemOpInfo::getAtomicCmpxchgInfo(MI))
611         Changed |= expandAtomicCmpxchg(MOI.getValue(), MI);
612       else if (const auto &MOI = SIMemOpInfo::getAtomicRmwInfo(MI))
613         Changed |= expandAtomicRmw(MOI.getValue(), MI);
614     }
615   }
616 
617   Changed |= removeAtomicPseudoMIs();
618   return Changed;
619 }
620 
621 INITIALIZE_PASS(SIMemoryLegalizer, DEBUG_TYPE, PASS_NAME, false, false)
622 
623 char SIMemoryLegalizer::ID = 0;
624 char &llvm::SIMemoryLegalizerID = SIMemoryLegalizer::ID;
625 
626 FunctionPass *llvm::createSIMemoryLegalizerPass() {
627   return new SIMemoryLegalizer();
628 }
629