Lines Matching full:scope
97 SIAtomicScope Scope = SIAtomicScope::SYSTEM;
107 SIAtomicScope Scope = SIAtomicScope::SYSTEM,
114 : Ordering(Ordering), FailureOrdering(FailureOrdering), Scope(Scope),
121 assert(Scope == SIAtomicScope::NONE &&
128 assert(Scope != SIAtomicScope::NONE &&
141 // Limit the scope to the maximum supported by the instruction's address
145 this->Scope = std::min(Scope, SIAtomicScope::SINGLETHREAD);
149 this->Scope = std::min(Scope, SIAtomicScope::WORKGROUP);
153 this->Scope = std::min(Scope, SIAtomicScope::AGENT);
158 /// \returns Atomic synchronization scope of the machine instruction used to
161 return Scope;
226 /// Inspects the target synchronization scope \p SSID and determines
227 /// the SI atomic scope it corresponds to, the address spaces it
293 /// the \p Scope memory scope for address spaces \p
296 SIAtomicScope Scope,
300 /// the \p Scope memory scope for address spaces \p
303 SIAtomicScope Scope,
307 /// to the \p Scope memory scope for address spaces \p AddrSpace. Return true
310 SIAtomicScope Scope,
330 /// observed by other memory instructions executing in memory scope \p Scope.
334 SIAtomicScope Scope,
343 /// operations by any thread for memory scopes up to memory scope \p Scope .
346 SIAtomicScope Scope,
353 /// subsequent memory instructions by any thread executing in memory scope \p
354 /// Scope. \p IsCrossAddrSpaceOrdering indicates if the memory ordering is
357 SIAtomicScope Scope,
391 SIAtomicScope Scope,
395 SIAtomicScope Scope,
399 SIAtomicScope Scope,
408 SIAtomicScope Scope,
415 SIAtomicScope Scope,
420 SIAtomicScope Scope,
432 SIAtomicScope Scope,
444 SIAtomicScope Scope,
448 SIAtomicScope Scope,
452 SIAtomicScope Scope,
461 SIAtomicScope Scope,
468 SIAtomicScope Scope,
473 SIAtomicScope Scope,
505 SIAtomicScope Scope,
509 SIAtomicScope Scope,
513 SIAtomicScope Scope,
521 bool insertAcquire(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
524 bool insertRelease(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
557 SIAtomicScope Scope,
566 SIAtomicScope Scope,
573 SIAtomicScope Scope,
583 SIAtomicScope Scope,
598 // Sets Scope policy to \p Value if CPol operand is present in instruction \p
603 // Stores with system scope (SCOPE_SYS) need to wait for:
613 SIAtomicScope Scope, SIAtomicAddrSpace AddrSpace) const;
618 bool insertWait(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
622 bool insertAcquire(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
632 bool insertRelease(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
637 SIAtomicScope Scope,
639 return setAtomicScope(MI, Scope, AddrSpace);
643 SIAtomicScope Scope,
645 return setAtomicScope(MI, Scope, AddrSpace);
649 SIAtomicScope Scope,
651 return setAtomicScope(MI, Scope, AddrSpace);
838 "Unsupported non-inclusive atomic synchronization scope");
851 SIAtomicScope Scope = SIAtomicScope::NONE;
857 reportUnsupported(MI, "Unsupported atomic synchronization scope");
860 std::tie(Scope, OrderingAddrSpace, IsCrossAddressSpaceOrdering) =
869 return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, InstrAddrSpace,
915 reportUnsupported(MI, "Unsupported atomic synchronization scope");
919 SIAtomicScope Scope = SIAtomicScope::NONE;
922 std::tie(Scope, OrderingAddrSpace, IsCrossAddressSpaceOrdering) =
931 return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, SIAtomicAddrSpace::ATOMIC,
985 SIAtomicScope Scope,
991 switch (Scope) {
1004 llvm_unreachable("Unsupported synchronization scope");
1020 SIAtomicScope Scope,
1033 SIAtomicScope Scope,
1069 // Ensure operation has completed at system scope to cause all volatile
1092 SIAtomicScope Scope,
1110 switch (Scope) {
1122 llvm_unreachable("Unsupported synchronization scope");
1127 switch (Scope) {
1145 llvm_unreachable("Unsupported synchronization scope");
1150 switch (Scope) {
1168 llvm_unreachable("Unsupported synchronization scope");
1190 SIAtomicScope Scope,
1205 switch (Scope) {
1217 llvm_unreachable("Unsupported synchronization scope");
1235 SIAtomicScope Scope,
1239 return insertWait(MI, Scope, AddrSpace, SIMemOp::LOAD | SIMemOp::STORE,
1244 SIAtomicScope Scope,
1265 switch (Scope) {
1277 llvm_unreachable("Unsupported synchronization scope");
1296 SIAtomicScope Scope,
1302 switch (Scope) {
1322 llvm_unreachable("Unsupported synchronization scope");
1338 SIAtomicScope Scope,
1344 switch (Scope) {
1357 llvm_unreachable("Unsupported synchronization scope");
1373 SIAtomicScope Scope,
1379 switch (Scope) {
1392 llvm_unreachable("Unsupported synchronization scope");
1422 // Ensure operation has completed at system scope to cause all volatile
1445 SIAtomicScope Scope,
1460 (Scope == SIAtomicScope::WORKGROUP)) {
1461 // Same as GFX7 using agent scope.
1462 Scope = SIAtomicScope::AGENT;
1468 return SIGfx7CacheControl::insertWait(MI, Scope, AddrSpace, Op,
1473 SIAtomicScope Scope,
1488 switch (Scope) {
1510 // Same as GFX7 using agent scope.
1511 Scope = SIAtomicScope::AGENT;
1519 llvm_unreachable("Unsupported synchronization scope");
1533 Changed |= SIGfx7CacheControl::insertAcquire(MI, Scope, AddrSpace, Pos);
1539 SIAtomicScope Scope,
1552 switch (Scope) {
1561 // Set SC bits to indicate system scope.
1574 llvm_unreachable("Unsupported synchronization scope");
1582 SIGfx7CacheControl::insertRelease(MI, Scope, AddrSpace,
1589 const MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
1595 switch (Scope) {
1597 // Set SC bits to indicate system scope.
1602 // Set SC bits to indicate agent scope.
1610 // bits to indicate work-group scope will do this automatically.
1615 // Leave SC bits unset to indicate wavefront scope.
1618 llvm_unreachable("Unsupported synchronization scope");
1634 SIAtomicScope Scope, SIAtomicAddrSpace AddrSpace) const {
1639 switch (Scope) {
1641 // Set SC bits to indicate system scope.
1646 // Set SC bits to indicate agent scope.
1650 // Set SC bits to indicate workgroup scope.
1655 // Leave SC bits unset to indicate wavefront scope.
1658 llvm_unreachable("Unsupported synchronization scope");
1673 const MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
1679 switch (Scope) {
1681 // Set SC1 bit to indicate system scope.
1689 // to indicate system or agent scope. The SC0 bit is used to indicate if
1691 // scope.
1694 llvm_unreachable("Unsupported synchronization scope");
1718 // Set SC bits to indicate system scope.
1722 // Ensure operation has completed at system scope to cause all volatile
1742 SIAtomicScope Scope,
1757 switch (Scope) {
1763 // Set SC bits to indicate system scope.
1777 // Set SC bits to indicate agent scope.
1795 // Set SC bits to indicate work-group scope.
1810 llvm_unreachable("Unsupported synchronization scope");
1828 SIAtomicScope Scope,
1841 switch (Scope) {
1850 // Set SC bits to indicate system scope.
1852 // Since AddrSpace contains SIAtomicAddrSpace::GLOBAL and Scope is
1859 // Set SC bits to indicate agent scope.
1862 // Since AddrSpace contains SIAtomicAddrSpace::GLOBAL and Scope is
1875 llvm_unreachable("Unsupported synchronization scope");
1884 Changed |= insertWait(MI, Scope, AddrSpace, SIMemOp::LOAD | SIMemOp::STORE,
1892 SIAtomicScope Scope,
1898 switch (Scope) {
1919 llvm_unreachable("Unsupported synchronization scope");
1959 // Ensure operation has completed at system scope to cause all volatile
1985 SIAtomicScope Scope,
2004 switch (Scope) {
2031 llvm_unreachable("Unsupported synchronization scope");
2036 switch (Scope) {
2054 llvm_unreachable("Unsupported synchronization scope");
2059 switch (Scope) {
2077 llvm_unreachable("Unsupported synchronization scope");
2106 SIAtomicScope Scope,
2121 switch (Scope) {
2146 llvm_unreachable("Unsupported synchronization scope");
2164 const MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
2170 switch (Scope) {
2190 llvm_unreachable("Unsupported synchronization scope");
2231 // Ensure operation has completed at system scope to cause all volatile
2279 uint64_t NewScope = Value & AMDGPU::CPol::SCOPE;
2280 if ((CPol->getImm() & AMDGPU::CPol::SCOPE) != NewScope) {
2281 CPol->setImm((CPol->getImm() & ~AMDGPU::CPol::SCOPE) | NewScope);
2305 SIAtomicScope Scope,
2323 switch (Scope) {
2350 llvm_unreachable("Unsupported synchronization scope");
2355 switch (Scope) {
2373 llvm_unreachable("Unsupported synchronization scope");
2401 SIAtomicScope Scope,
2420 switch (Scope) {
2442 llvm_unreachable("Unsupported synchronization scope");
2457 SIAtomicScope Scope,
2480 switch (Scope) {
2504 llvm_unreachable("Unsupported synchronization scope");
2516 insertWait(MI, Scope, AddrSpace, SIMemOp::LOAD | SIMemOp::STORE,
2551 // Ensure operation has completed at system scope to cause all volatile
2566 if (CPol && ((CPol->getImm() & CPol::SCOPE) == CPol::SCOPE_SYS))
2573 SIAtomicScope Scope,
2578 switch (Scope) {
2596 llvm_unreachable("Unsupported synchronization scope");
2657 // Atomic instructions already bypass caches to the scope specified by the
2691 // Atomic instructions already bypass caches to the scope specified by the
2698 // GFX12 specific, scope(desired coherence domain in cache hierarchy) is
2699 // instruction field, do not confuse it with atomic scope.
2729 /// ordering and memory scope, then library does not need to