; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=+precise-memory < %s | FileCheck %s -check-prefixes=GFX9
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -mattr=+precise-memory < %s | FileCheck %s -check-prefixes=GFX90A
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+precise-memory < %s | FileCheck %s -check-prefixes=GFX10
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=+enable-flat-scratch,+precise-memory < %s | FileCheck --check-prefixes=GFX9-FLATSCR %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+precise-memory < %s | FileCheck %s -check-prefixes=GFX11
; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+precise-memory < %s | FileCheck %s -check-prefixes=GFX12

; from atomicrmw-expand.ll
; covers flat_load, flat_atomic (atomic with return)
;
define void @syncscope_workgroup_nortn(ptr %addr, float %val) {
; GFX9-LABEL: syncscope_workgroup_nortn:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    flat_load_dword v4, v[0:1]
; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b64 s[4:5], 0
; GFX9-NEXT:  .LBB0_1: ; %atomicrmw.start
; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT:    v_add_f32_e32 v3, v4, v2
; GFX9-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT:    v_mov_b32_e32 v4, v3
; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT:    s_cbranch_execnz .LBB0_1
; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: syncscope_workgroup_nortn:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT:    flat_load_dword v5, v[0:1]
; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
; GFX90A-NEXT:  .LBB0_1: ; %atomicrmw.start
; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX90A-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT:    v_mov_b32_e32 v5, v3
; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT:    s_cbranch_execnz .LBB0_1
; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: syncscope_workgroup_nortn:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    flat_load_dword v4, v[0:1]
; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 s4, 0
; GFX10-NEXT:  .LBB0_1: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_add_f32_e32 v3, v4, v2
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT:    v_mov_b32_e32 v4, v3
; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT:    s_cbranch_execnz .LBB0_1
; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-FLATSCR-LABEL: syncscope_workgroup_nortn:
; GFX9-FLATSCR:       ; %bb.0:
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    flat_load_dword v4, v[0:1]
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    s_mov_b64 s[0:1], 0
; GFX9-FLATSCR-NEXT:  .LBB0_1: ; %atomicrmw.start
; GFX9-FLATSCR-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX9-FLATSCR-NEXT:    v_add_f32_e32 v3, v4, v2
; GFX9-FLATSCR-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-FLATSCR-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v4, v3
; GFX9-FLATSCR-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX9-FLATSCR-NEXT:    s_cbranch_execnz .LBB0_1
; GFX9-FLATSCR-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX9-FLATSCR-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX9-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: syncscope_workgroup_nortn:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    flat_load_b32 v4, v[0:1]
; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 s0, 0
; GFX11-NEXT:  .LBB0_1: ; %atomicrmw.start
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_add_f32_e32 v3, v4, v2
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-NEXT:    v_mov_b32_e32 v4, v3
; GFX11-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT:    s_cbranch_execnz .LBB0_1
; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX12-LABEL: syncscope_workgroup_nortn:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_atomic_add_f32 v[0:1], v2 scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %res = atomicrmw fadd ptr %addr, float %val syncscope("workgroup") seq_cst
  ret void
}

; from atomicrmw-nand.ll
; covers global_atomic (atomic with return), global_load
;
define i32 @atomic_nand_i32_global(ptr addrspace(1) %ptr) nounwind {
; GFX9-LABEL: atomic_nand_i32_global:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v2, v[0:1], off
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_mov_b64 s[4:5], 0
; GFX9-NEXT:  .LBB1_1: ; %atomicrmw.start
; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT:    v_mov_b32_e32 v3, v2
; GFX9-NEXT:    v_not_b32_e32 v2, v3
; GFX9-NEXT:    v_or_b32_e32 v2, -5, v2
; GFX9-NEXT:    global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT:    s_cbranch_execnz .LBB1_1
; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT:    v_mov_b32_e32 v0, v2
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: atomic_nand_i32_global:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT:    global_load_dword v2, v[0:1], off
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
; GFX90A-NEXT:  .LBB1_1: ; %atomicrmw.start
; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
; GFX90A-NEXT:    v_not_b32_e32 v2, v3
; GFX90A-NEXT:    v_or_b32_e32 v2, -5, v2
; GFX90A-NEXT:    buffer_wbl2
; GFX90A-NEXT:    global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_invl2
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT:    s_cbranch_execnz .LBB1_1
; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
; GFX90A-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_nand_i32_global:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    global_load_dword v2, v[0:1], off
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    s_mov_b32 s4, 0
; GFX10-NEXT:  .LBB1_1: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_mov_b32_e32 v3, v2
; GFX10-NEXT:    v_not_b32_e32 v2, v3
; GFX10-NEXT:    v_or_b32_e32 v2, -5, v2
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT:    s_cbranch_execnz .LBB1_1
; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT:    v_mov_b32_e32 v0, v2
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-FLATSCR-LABEL: atomic_nand_i32_global:
; GFX9-FLATSCR:       ; %bb.0:
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    global_load_dword v2, v[0:1], off
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    s_mov_b64 s[0:1], 0
; GFX9-FLATSCR-NEXT:  .LBB1_1: ; %atomicrmw.start
; GFX9-FLATSCR-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v3, v2
; GFX9-FLATSCR-NEXT:    v_not_b32_e32 v2, v3
; GFX9-FLATSCR-NEXT:    v_or_b32_e32 v2, -5, v2
; GFX9-FLATSCR-NEXT:    global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    buffer_wbinvl1_vol
; GFX9-FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GFX9-FLATSCR-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-FLATSCR-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX9-FLATSCR-NEXT:    s_cbranch_execnz .LBB1_1
; GFX9-FLATSCR-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX9-FLATSCR-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v0, v2
; GFX9-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_nand_i32_global:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    global_load_b32 v2, v[0:1], off
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    s_mov_b32 s0, 0
; GFX11-NEXT:  .LBB1_1: ; %atomicrmw.start
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    v_mov_b32_e32 v3, v2
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT:    v_not_b32_e32 v2, v3
; GFX11-NEXT:    v_or_b32_e32 v2, -5, v2
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
; GFX11-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT:    s_cbranch_execnz .LBB1_1
; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT:    v_mov_b32_e32 v0, v2
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX12-LABEL: atomic_nand_i32_global:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v2, v[0:1], off
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    s_mov_b32 s0, 0
; GFX12-NEXT:  .LBB1_1: ; %atomicrmw.start
; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT:    v_mov_b32_e32 v3, v2
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT:    v_not_b32_e32 v2, v3
; GFX12-NEXT:    v_or_b32_e32 v2, -5, v2
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-NEXT:    s_cbranch_execnz .LBB1_1
; GFX12-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT:    v_mov_b32_e32 v0, v2
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw nand ptr addrspace(1) %ptr, i32 4 seq_cst
  ret i32 %result
}

; from call-argument-types.ll
; covers scratch_load, scratch_store, buffer_load, buffer_store
;
declare hidden void @byval_align16_f64_arg(<32 x i32>, ptr addrspace(5) byval(double) align 16)
define void @tail_call_byval_align16(<32 x i32> %val, double %tmp)  {
; GFX9-LABEL: tail_call_byval_align16:
; GFX9:       ; %bb.0: ; %entry
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_getpc_b64 s[16:17]
; GFX9-NEXT:    s_add_u32 s16, s16, byval_align16_f64_arg@rel32@lo+4
; GFX9-NEXT:    s_addc_u32 s17, s17, byval_align16_f64_arg@rel32@hi+12
; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:20
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:16
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[16:17]
;
; GFX90A-LABEL: tail_call_byval_align16:
; GFX90A:       ; %bb.0: ; %entry
; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_load_dword v34, off, s[0:3], s32
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    s_getpc_b64 s[16:17]
; GFX90A-NEXT:    s_add_u32 s16, s16, byval_align16_f64_arg@rel32@lo+4
; GFX90A-NEXT:    s_addc_u32 s17, s17, byval_align16_f64_arg@rel32@hi+12
; GFX90A-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:20
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:16
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_store_dword v34, off, s[0:3], s32
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    s_setpc_b64 s[16:17]
;
; GFX10-LABEL: tail_call_byval_align16:
; GFX10:       ; %bb.0: ; %entry
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_load_dword v34, off, s[0:3], s32
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    s_getpc_b64 s[16:17]
; GFX10-NEXT:    s_add_u32 s16, s16, byval_align16_f64_arg@rel32@lo+4
; GFX10-NEXT:    s_addc_u32 s17, s17, byval_align16_f64_arg@rel32@hi+12
; GFX10-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:20
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:16
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    buffer_store_dword v34, off, s[0:3], s32
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    s_setpc_b64 s[16:17]
;
; GFX9-FLATSCR-LABEL: tail_call_byval_align16:
; GFX9-FLATSCR:       ; %bb.0: ; %entry
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    scratch_load_dword v32, off, s32
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    s_getpc_b64 s[0:1]
; GFX9-FLATSCR-NEXT:    s_add_u32 s0, s0, byval_align16_f64_arg@rel32@lo+4
; GFX9-FLATSCR-NEXT:    s_addc_u32 s1, s1, byval_align16_f64_arg@rel32@hi+12
; GFX9-FLATSCR-NEXT:    scratch_store_dword off, v32, s32
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    scratch_load_dwordx2 v[32:33], off, s32 offset:24
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    scratch_store_dwordx2 off, v[32:33], s32 offset:16
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    s_setpc_b64 s[0:1]
;
; GFX11-LABEL: tail_call_byval_align16:
; GFX11:       ; %bb.0: ; %entry
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    scratch_load_b32 v32, off, s32
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    s_getpc_b64 s[0:1]
; GFX11-NEXT:    s_add_u32 s0, s0, byval_align16_f64_arg@rel32@lo+4
; GFX11-NEXT:    s_addc_u32 s1, s1, byval_align16_f64_arg@rel32@hi+12
; GFX11-NEXT:    scratch_store_b32 off, v32, s32
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    scratch_load_b64 v[32:33], off, s32 offset:24
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    scratch_store_b64 off, v[32:33], s32 offset:16
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    s_setpc_b64 s[0:1]
;
; GFX12-LABEL: tail_call_byval_align16:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    scratch_load_b32 v32, off, s32
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    s_getpc_b64 s[0:1]
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_sext_i32_i16 s1, s1
; GFX12-NEXT:    s_add_co_u32 s0, s0, byval_align16_f64_arg@rel32@lo+12
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_add_co_ci_u32 s1, s1, byval_align16_f64_arg@rel32@hi+24
; GFX12-NEXT:    scratch_store_b32 off, v32, s32
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    scratch_load_b64 v[32:33], off, s32 offset:24
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    scratch_store_b64 off, v[32:33], s32 offset:16
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_setpc_b64 s[0:1]
entry:
  %alloca = alloca double, align 8, addrspace(5)
  tail call void @byval_align16_f64_arg(<32 x i32> %val, ptr addrspace(5) byval(double) align 16 %alloca)
  ret void
}

; from udiv.ll
; covers s_load
;
define amdgpu_kernel void @udiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
; GFX9-LABEL: udiv_i32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_mov_b32_e32 v1, 0
; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
; GFX9-NEXT:    s_sub_i32 s4, 0, s3
; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT:    v_readfirstlane_b32 s5, v0
; GFX9-NEXT:    s_mul_i32 s4, s4, s5
; GFX9-NEXT:    s_mul_hi_u32 s4, s5, s4
; GFX9-NEXT:    s_add_i32 s5, s5, s4
; GFX9-NEXT:    s_mul_hi_u32 s4, s2, s5
; GFX9-NEXT:    s_mul_i32 s5, s4, s3
; GFX9-NEXT:    s_sub_i32 s2, s2, s5
; GFX9-NEXT:    s_add_i32 s6, s4, 1
; GFX9-NEXT:    s_sub_i32 s5, s2, s3
; GFX9-NEXT:    s_cmp_ge_u32 s2, s3
; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
; GFX9-NEXT:    s_add_i32 s5, s4, 1
; GFX9-NEXT:    s_cmp_ge_u32 s2, s3
; GFX9-NEXT:    s_cselect_b32 s2, s5, s4
; GFX9-NEXT:    v_mov_b32_e32 v0, s2
; GFX9-NEXT:    global_store_dword v1, v0, s[0:1]
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_endpgm
;
; GFX90A-LABEL: udiv_i32:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_mov_b32_e32 v1, 0
; GFX90A-NEXT:    v_cvt_f32_u32_e32 v0, s3
; GFX90A-NEXT:    s_sub_i32 s4, 0, s3
; GFX90A-NEXT:    v_rcp_iflag_f32_e32 v0, v0
; GFX90A-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX90A-NEXT:    v_cvt_u32_f32_e32 v0, v0
; GFX90A-NEXT:    v_readfirstlane_b32 s5, v0
; GFX90A-NEXT:    s_mul_i32 s4, s4, s5
; GFX90A-NEXT:    s_mul_hi_u32 s4, s5, s4
; GFX90A-NEXT:    s_add_i32 s5, s5, s4
; GFX90A-NEXT:    s_mul_hi_u32 s4, s2, s5
; GFX90A-NEXT:    s_mul_i32 s5, s4, s3
; GFX90A-NEXT:    s_sub_i32 s2, s2, s5
; GFX90A-NEXT:    s_add_i32 s6, s4, 1
; GFX90A-NEXT:    s_sub_i32 s5, s2, s3
; GFX90A-NEXT:    s_cmp_ge_u32 s2, s3
; GFX90A-NEXT:    s_cselect_b32 s4, s6, s4
; GFX90A-NEXT:    s_cselect_b32 s2, s5, s2
; GFX90A-NEXT:    s_add_i32 s5, s4, 1
; GFX90A-NEXT:    s_cmp_ge_u32 s2, s3
; GFX90A-NEXT:    s_cselect_b32 s2, s5, s4
; GFX90A-NEXT:    v_mov_b32_e32 v0, s2
; GFX90A-NEXT:    global_store_dword v1, v0, s[0:1]
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: udiv_i32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s3
; GFX10-NEXT:    s_sub_i32 s5, 0, s3
; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
; GFX10-NEXT:    v_readfirstlane_b32 s4, v0
; GFX10-NEXT:    v_mov_b32_e32 v0, 0
; GFX10-NEXT:    s_mul_i32 s5, s5, s4
; GFX10-NEXT:    s_mul_hi_u32 s5, s4, s5
; GFX10-NEXT:    s_add_i32 s4, s4, s5
; GFX10-NEXT:    s_mul_hi_u32 s4, s2, s4
; GFX10-NEXT:    s_mul_i32 s5, s4, s3
; GFX10-NEXT:    s_sub_i32 s2, s2, s5
; GFX10-NEXT:    s_add_i32 s5, s4, 1
; GFX10-NEXT:    s_sub_i32 s6, s2, s3
; GFX10-NEXT:    s_cmp_ge_u32 s2, s3
; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
; GFX10-NEXT:    s_cselect_b32 s2, s6, s2
; GFX10-NEXT:    s_add_i32 s5, s4, 1
; GFX10-NEXT:    s_cmp_ge_u32 s2, s3
; GFX10-NEXT:    s_cselect_b32 s2, s5, s4
; GFX10-NEXT:    v_mov_b32_e32 v1, s2
; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    s_endpgm
;
; GFX9-FLATSCR-LABEL: udiv_i32:
; GFX9-FLATSCR:       ; %bb.0:
; GFX9-FLATSCR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v1, 0
; GFX9-FLATSCR-NEXT:    v_cvt_f32_u32_e32 v0, s3
; GFX9-FLATSCR-NEXT:    s_sub_i32 s4, 0, s3
; GFX9-FLATSCR-NEXT:    v_rcp_iflag_f32_e32 v0, v0
; GFX9-FLATSCR-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-FLATSCR-NEXT:    v_cvt_u32_f32_e32 v0, v0
; GFX9-FLATSCR-NEXT:    v_readfirstlane_b32 s5, v0
; GFX9-FLATSCR-NEXT:    s_mul_i32 s4, s4, s5
; GFX9-FLATSCR-NEXT:    s_mul_hi_u32 s4, s5, s4
; GFX9-FLATSCR-NEXT:    s_add_i32 s5, s5, s4
; GFX9-FLATSCR-NEXT:    s_mul_hi_u32 s4, s2, s5
; GFX9-FLATSCR-NEXT:    s_mul_i32 s5, s4, s3
; GFX9-FLATSCR-NEXT:    s_sub_i32 s2, s2, s5
; GFX9-FLATSCR-NEXT:    s_add_i32 s6, s4, 1
; GFX9-FLATSCR-NEXT:    s_sub_i32 s5, s2, s3
; GFX9-FLATSCR-NEXT:    s_cmp_ge_u32 s2, s3
; GFX9-FLATSCR-NEXT:    s_cselect_b32 s4, s6, s4
; GFX9-FLATSCR-NEXT:    s_cselect_b32 s2, s5, s2
; GFX9-FLATSCR-NEXT:    s_add_i32 s5, s4, 1
; GFX9-FLATSCR-NEXT:    s_cmp_ge_u32 s2, s3
; GFX9-FLATSCR-NEXT:    s_cselect_b32 s2, s5, s4
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v0, s2
; GFX9-FLATSCR-NEXT:    global_store_dword v1, v0, s[0:1]
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    s_endpgm
;
; GFX11-LABEL: udiv_i32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s3
; GFX11-NEXT:    s_sub_i32 s5, 0, s3
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
; GFX11-NEXT:    s_waitcnt_depctr 0xfff
; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT:    v_readfirstlane_b32 s4, v0
; GFX11-NEXT:    v_mov_b32_e32 v0, 0
; GFX11-NEXT:    s_mul_i32 s5, s5, s4
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT:    s_mul_hi_u32 s5, s4, s5
; GFX11-NEXT:    s_add_i32 s4, s4, s5
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT:    s_mul_hi_u32 s4, s2, s4
; GFX11-NEXT:    s_mul_i32 s5, s4, s3
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_sub_i32 s2, s2, s5
; GFX11-NEXT:    s_add_i32 s5, s4, 1
; GFX11-NEXT:    s_sub_i32 s6, s2, s3
; GFX11-NEXT:    s_cmp_ge_u32 s2, s3
; GFX11-NEXT:    s_cselect_b32 s4, s5, s4
; GFX11-NEXT:    s_cselect_b32 s2, s6, s2
; GFX11-NEXT:    s_add_i32 s5, s4, 1
; GFX11-NEXT:    s_cmp_ge_u32 s2, s3
; GFX11-NEXT:    s_cselect_b32 s2, s5, s4
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    v_mov_b32_e32 v1, s2
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: udiv_i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cvt_f32_u32 s4, s3
; GFX12-NEXT:    s_sub_co_i32 s5, 0, s3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX12-NEXT:    v_rcp_iflag_f32_e32 v0, s4
; GFX12-NEXT:    v_readfirstlane_b32 s4, v0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
; GFX12-NEXT:    s_mul_f32 s4, s4, 0x4f7ffffe
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_cvt_u32_f32 s4, s4
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
; GFX12-NEXT:    s_mul_i32 s5, s5, s4
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_mul_hi_u32 s5, s4, s5
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_add_co_i32 s4, s4, s5
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_mul_hi_u32 s4, s2, s4
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_mul_i32 s5, s4, s3
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_sub_co_i32 s2, s2, s5
; GFX12-NEXT:    s_add_co_i32 s5, s4, 1
; GFX12-NEXT:    s_sub_co_i32 s6, s2, s3
; GFX12-NEXT:    s_cmp_ge_u32 s2, s3
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_cselect_b32 s4, s5, s4
; GFX12-NEXT:    s_cselect_b32 s2, s6, s2
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_add_co_i32 s5, s4, 1
; GFX12-NEXT:    s_cmp_ge_u32 s2, s3
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_cselect_b32 s2, s5, s4
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    s_endpgm
  %r = udiv i32 %x, %y
  store i32 %r, ptr addrspace(1) %out
  ret void
}

declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32)

; from smrd.ll
; covers s_buffer_load
;
define amdgpu_ps float @smrd_sgpr_offset(<4 x i32> inreg %desc, i32 inreg %offset) #0 {
; GFX9-LABEL: smrd_sgpr_offset:
; GFX9:       ; %bb.0: ; %main_body
; GFX9-NEXT:    s_buffer_load_dword s0, s[0:3], s4 offset:0x0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_mov_b32_e32 v0, s0
; GFX9-NEXT:    ; return to shader part epilog
;
; GFX90A-LABEL: smrd_sgpr_offset:
; GFX90A:       ; %bb.0: ; %main_body
; GFX90A-NEXT:    s_buffer_load_dword s0, s[0:3], s4 offset:0x0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_mov_b32_e32 v0, s0
; GFX90A-NEXT:    ; return to shader part epilog
;
; GFX10-LABEL: smrd_sgpr_offset:
; GFX10:       ; %bb.0: ; %main_body
; GFX10-NEXT:    s_buffer_load_dword s0, s[0:3], s4 offset:0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v0, s0
; GFX10-NEXT:    ; return to shader part epilog
;
; GFX9-FLATSCR-LABEL: smrd_sgpr_offset:
; GFX9-FLATSCR:       ; %bb.0: ; %main_body
; GFX9-FLATSCR-NEXT:    s_mov_b32 s11, s5
; GFX9-FLATSCR-NEXT:    s_mov_b32 s10, s4
; GFX9-FLATSCR-NEXT:    s_mov_b32 s9, s3
; GFX9-FLATSCR-NEXT:    s_mov_b32 s8, s2
; GFX9-FLATSCR-NEXT:    s_buffer_load_dword s0, s[8:11], s6 offset:0x0
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v0, s0
; GFX9-FLATSCR-NEXT:    ; return to shader part epilog
;
; GFX11-LABEL: smrd_sgpr_offset:
; GFX11:       ; %bb.0: ; %main_body
; GFX11-NEXT:    s_buffer_load_b32 s0, s[0:3], s4 offset:0x0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_mov_b32_e32 v0, s0
; GFX11-NEXT:    ; return to shader part epilog
;
; GFX12-LABEL: smrd_sgpr_offset:
; GFX12:       ; %bb.0: ; %main_body
; GFX12-NEXT:    s_buffer_load_b32 s0, s[0:3], s4 offset:0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_mov_b32_e32 v0, s0
; GFX12-NEXT:    ; return to shader part epilog
main_body:
  %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0)
  ret float %r
}

; from atomic_load_add.ll
; covers s_load, ds_add (atomic without return)
;
define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
; GFX9-LABEL: atomic_add_local:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_mov_b64 s[0:1], exec
; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT:    s_cbranch_execz .LBB5_2
; GFX9-NEXT:  ; %bb.1:
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x24
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
; GFX9-NEXT:    s_mul_i32 s0, s0, 5
; GFX9-NEXT:    v_mov_b32_e32 v1, s0
; GFX9-NEXT:    v_mov_b32_e32 v0, s2
; GFX9-NEXT:    ds_add_u32 v0, v1
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:  .LBB5_2:
; GFX9-NEXT:    s_endpgm
;
; GFX90A-LABEL: atomic_add_local:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_mov_b64 s[0:1], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB5_2
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dword s2, s[4:5], 0x24
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
; GFX90A-NEXT:    s_mul_i32 s0, s0, 5
; GFX90A-NEXT:    v_mov_b32_e32 v1, s0
; GFX90A-NEXT:    v_mov_b32_e32 v0, s2
; GFX90A-NEXT:    ds_add_u32 v0, v1
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:  .LBB5_2:
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_add_local:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_mov_b32 s0, exec_lo
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB5_2
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dword s1, s[4:5], 0x24
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_bcnt1_i32_b32 s0, s0
; GFX10-NEXT:    s_mul_i32 s0, s0, 5
; GFX10-NEXT:    v_mov_b32_e32 v1, s0
; GFX10-NEXT:    v_mov_b32_e32 v0, s1
; GFX10-NEXT:    ds_add_u32 v0, v1
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:  .LBB5_2:
; GFX10-NEXT:    s_endpgm
;
; GFX9-FLATSCR-LABEL: atomic_add_local:
; GFX9-FLATSCR:       ; %bb.0:
; GFX9-FLATSCR-NEXT:    s_mov_b64 s[0:1], exec
; GFX9-FLATSCR-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX9-FLATSCR-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX9-FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX9-FLATSCR-NEXT:    s_cbranch_execz .LBB5_2
; GFX9-FLATSCR-NEXT:  ; %bb.1:
; GFX9-FLATSCR-NEXT:    s_load_dword s2, s[4:5], 0x24
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
; GFX9-FLATSCR-NEXT:    s_mul_i32 s0, s0, 5
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v1, s0
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v0, s2
; GFX9-FLATSCR-NEXT:    ds_add_u32 v0, v1
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:  .LBB5_2:
; GFX9-FLATSCR-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_add_local:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b32 s0, exec_lo
; GFX11-NEXT:    s_mov_b32 s1, exec_lo
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB5_2
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b32 s1, s[4:5], 0x24
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_bcnt1_i32_b32 s0, s0
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT:    s_mul_i32 s0, s0, 5
; GFX11-NEXT:    v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v0, s1
; GFX11-NEXT:    ds_add_u32 v0, v1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:  .LBB5_2:
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_local:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_mov_b32 s0, exec_lo
; GFX12-NEXT:    s_mov_b32 s1, exec_lo
; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX12-NEXT:    s_cbranch_execz .LBB5_2
; GFX12-NEXT:  ; %bb.1:
; GFX12-NEXT:    s_load_b32 s1, s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_bcnt1_i32_b32 s0, s0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_mul_i32 s0, s0, 5
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v0, s1
; GFX12-NEXT:    ds_add_u32 v0, v1
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB5_2:
; GFX12-NEXT:    s_endpgm
   %unused = atomicrmw volatile add ptr addrspace(3) %local, i32 5 seq_cst
   ret void
}

; from flat_atomics_i32_system.ll
; covers flat_atomic_swap (atomic without return)
;
define void @flat_atomic_xchg_i32_noret(ptr %ptr, i32 %in) {
; GFX9-LABEL: flat_atomic_xchg_i32_noret:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    flat_atomic_swap v[0:1], v2
; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: flat_atomic_xchg_i32_noret:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT:    buffer_wbl2
; GFX90A-NEXT:    flat_atomic_swap v[0:1], v2
; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT:    buffer_invl2
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_atomic_xchg_i32_noret:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    flat_atomic_swap v[0:1], v2
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-FLATSCR-LABEL: flat_atomic_xchg_i32_noret:
; GFX9-FLATSCR:       ; %bb.0:
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    flat_atomic_swap v[0:1], v2
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    buffer_wbinvl1_vol
; GFX9-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_atomic_xchg_i32_noret:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    flat_atomic_swap_b32 v[0:1], v2
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX12-LABEL: flat_atomic_xchg_i32_noret:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw xchg ptr %ptr, i32 %in seq_cst
  ret void
}

; from atomic_load_add.ll
; covers s_load, ds_add_rtn (atomic with return)
;
define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrspace(3) %local) {
; GFX9-LABEL: atomic_add_ret_local:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_mov_b64 s[2:3], exec
; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT:    ; implicit-def: $vgpr1
; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT:    s_cbranch_execz .LBB7_2
; GFX9-NEXT:  ; %bb.1:
; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT:    s_mul_i32 s2, s2, 5
; GFX9-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-NEXT:    v_mov_b32_e32 v1, s6
; GFX9-NEXT:    ds_add_rtn_u32 v1, v1, v2
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:  .LBB7_2:
; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
; GFX9-NEXT:    v_mov_b32_e32 v2, 0
; GFX9-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_endpgm
;
; GFX90A-LABEL: atomic_add_ret_local:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_mov_b64 s[2:3], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    ; implicit-def: $vgpr1
; GFX90A-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB7_2
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dword s6, s[4:5], 0x2c
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT:    s_mul_i32 s2, s2, 5
; GFX90A-NEXT:    v_mov_b32_e32 v2, s2
; GFX90A-NEXT:    v_mov_b32_e32 v1, s6
; GFX90A-NEXT:    ds_add_rtn_u32 v1, v1, v2
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:  .LBB7_2:
; GFX90A-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_readfirstlane_b32 s2, v1
; GFX90A-NEXT:    v_mov_b32_e32 v2, 0
; GFX90A-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX90A-NEXT:    global_store_dword v2, v0, s[0:1]
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_add_ret_local:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_mov_b32 s1, exec_lo
; GFX10-NEXT:    ; implicit-def: $vgpr1
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s0, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB7_2
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_bcnt1_i32_b32 s1, s1
; GFX10-NEXT:    s_mul_i32 s1, s1, 5
; GFX10-NEXT:    v_mov_b32_e32 v2, s1
; GFX10-NEXT:    v_mov_b32_e32 v1, s2
; GFX10-NEXT:    ds_add_rtn_u32 v1, v1, v2
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:  .LBB7_2:
; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_readfirstlane_b32 s2, v1
; GFX10-NEXT:    v_mov_b32_e32 v1, 0
; GFX10-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    s_endpgm
;
; GFX9-FLATSCR-LABEL: atomic_add_ret_local:
; GFX9-FLATSCR:       ; %bb.0:
; GFX9-FLATSCR-NEXT:    s_mov_b64 s[2:3], exec
; GFX9-FLATSCR-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX9-FLATSCR-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX9-FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT:    ; implicit-def: $vgpr1
; GFX9-FLATSCR-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX9-FLATSCR-NEXT:    s_cbranch_execz .LBB7_2
; GFX9-FLATSCR-NEXT:  ; %bb.1:
; GFX9-FLATSCR-NEXT:    s_load_dword s6, s[4:5], 0x2c
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-FLATSCR-NEXT:    s_mul_i32 s2, s2, 5
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v1, s6
; GFX9-FLATSCR-NEXT:    ds_add_rtn_u32 v1, v1, v2
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:  .LBB7_2:
; GFX9-FLATSCR-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX9-FLATSCR-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    v_readfirstlane_b32 s2, v1
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v2, 0
; GFX9-FLATSCR-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX9-FLATSCR-NEXT:    global_store_dword v2, v0, s[0:1]
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_add_ret_local:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b32 s1, exec_lo
; GFX11-NEXT:    s_mov_b32 s0, exec_lo
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11-NEXT:    ; implicit-def: $vgpr1
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB7_2
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_bcnt1_i32_b32 s1, s1
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT:    s_mul_i32 s1, s1, 5
; GFX11-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT:    ds_add_rtn_u32 v1, v1, v2
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:  .LBB7_2:
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_readfirstlane_b32 s2, v1
; GFX11-NEXT:    v_mov_b32_e32 v1, 0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_ret_local:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_mov_b32 s1, exec_lo
; GFX12-NEXT:    s_mov_b32 s0, exec_lo
; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12-NEXT:    ; implicit-def: $vgpr1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX12-NEXT:    s_cbranch_execz .LBB7_2
; GFX12-NEXT:  ; %bb.1:
; GFX12-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_bcnt1_i32_b32 s1, s1
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_mul_i32 s1, s1, 5
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s2
; GFX12-NEXT:    ds_add_rtn_u32 v1, v1, v2
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB7_2:
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
; GFX12-NEXT:    v_mov_b32_e32 v1, 0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX12-NEXT:    global_store_b32 v1, v0, s[0:1]
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    s_endpgm
  %val = atomicrmw volatile add ptr addrspace(3) %local, i32 5 seq_cst
  store i32 %val, ptr addrspace(1) %out
  ret void
}

declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32, ptr addrspace(8), i32, i32, i32 immarg)

; from atomic_optimizations_buffer.ll
; covers buffer_atomic (atomic with return)
;
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX9-LABEL: add_i32_constant:
; GFX9:       ; %bb.0: ; %entry
; GFX9-NEXT:    s_mov_b64 s[2:3], exec
; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT:    ; implicit-def: $vgpr1
; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT:    s_cbranch_execz .LBB8_2
; GFX9-NEXT:  ; %bb.1:
; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT:    s_mul_i32 s2, s2, 5
; GFX9-NEXT:    v_mov_b32_e32 v1, s2
; GFX9-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:  .LBB8_2:
; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
; GFX9-NEXT:    v_mov_b32_e32 v2, 0
; GFX9-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_endpgm
;
; GFX90A-LABEL: add_i32_constant:
; GFX90A:       ; %bb.0: ; %entry
; GFX90A-NEXT:    s_mov_b64 s[2:3], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    ; implicit-def: $vgpr1
; GFX90A-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB8_2
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT:    s_mul_i32 s2, s2, 5
; GFX90A-NEXT:    v_mov_b32_e32 v1, s2
; GFX90A-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:  .LBB8_2:
; GFX90A-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_readfirstlane_b32 s2, v1
; GFX90A-NEXT:    v_mov_b32_e32 v2, 0
; GFX90A-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX90A-NEXT:    global_store_dword v2, v0, s[0:1]
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: add_i32_constant:
; GFX10:       ; %bb.0: ; %entry
; GFX10-NEXT:    s_mov_b32 s1, exec_lo
; GFX10-NEXT:    ; implicit-def: $vgpr1
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s0, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB8_2
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_bcnt1_i32_b32 s1, s1
; GFX10-NEXT:    s_mul_i32 s1, s1, 5
; GFX10-NEXT:    v_mov_b32_e32 v1, s1
; GFX10-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:  .LBB8_2:
; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_readfirstlane_b32 s2, v1
; GFX10-NEXT:    v_mov_b32_e32 v1, 0
; GFX10-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    s_endpgm
;
; GFX9-FLATSCR-LABEL: add_i32_constant:
; GFX9-FLATSCR:       ; %bb.0: ; %entry
; GFX9-FLATSCR-NEXT:    s_mov_b64 s[2:3], exec
; GFX9-FLATSCR-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX9-FLATSCR-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX9-FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT:    ; implicit-def: $vgpr1
; GFX9-FLATSCR-NEXT:    s_and_saveexec_b64 s[0:1], vcc
; GFX9-FLATSCR-NEXT:    s_cbranch_execz .LBB8_2
; GFX9-FLATSCR-NEXT:  ; %bb.1:
; GFX9-FLATSCR-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-FLATSCR-NEXT:    s_mul_i32 s2, s2, 5
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v1, s2
; GFX9-FLATSCR-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:  .LBB8_2:
; GFX9-FLATSCR-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX9-FLATSCR-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    v_readfirstlane_b32 s2, v1
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v2, 0
; GFX9-FLATSCR-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX9-FLATSCR-NEXT:    global_store_dword v2, v0, s[0:1]
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    s_endpgm
;
; GFX11-LABEL: add_i32_constant:
; GFX11:       ; %bb.0: ; %entry
; GFX11-NEXT:    s_mov_b32 s1, exec_lo
; GFX11-NEXT:    s_mov_b32 s0, exec_lo
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11-NEXT:    ; implicit-def: $vgpr1
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB8_2
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_bcnt1_i32_b32 s1, s1
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT:    s_mul_i32 s1, s1, 5
; GFX11-NEXT:    v_mov_b32_e32 v1, s1
; GFX11-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:  .LBB8_2:
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_readfirstlane_b32 s2, v1
; GFX11-NEXT:    v_mov_b32_e32 v1, 0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: add_i32_constant:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_mov_b32 s1, exec_lo
; GFX12-NEXT:    s_mov_b32 s0, exec_lo
; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12-NEXT:    ; implicit-def: $vgpr1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX12-NEXT:    s_cbranch_execz .LBB8_2
; GFX12-NEXT:  ; %bb.1:
; GFX12-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_bcnt1_i32_b32 s1, s1
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_mul_i32 s1, s1, 5
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_mov_b32_e32 v1, s1
; GFX12-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:  .LBB8_2:
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
; GFX12-NEXT:    v_mov_b32_e32 v1, 0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
; GFX12-NEXT:    global_store_b32 v1, v0, s[0:1]
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    s_endpgm
entry:
  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)
  store i32 %old, ptr addrspace(1) %out
  ret void
}

declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32, i16, <8 x i32>, i32, i32)

; from llvm.amdgcn.image.load.a16.ll
; covers image_load
;
define amdgpu_ps <4 x float> @load.f32.1d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
; GFX9-LABEL: load.f32.1d:
; GFX9:       ; %bb.0: ; %main_body
; GFX9-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 unorm a16
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    ; return to shader part epilog
;
; GFX90A-LABEL: load.f32.1d:
; GFX90A:       ; %bb.0: ; %main_body
; GFX90A-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 unorm a16
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    ; return to shader part epilog
;
; GFX10-LABEL: load.f32.1d:
; GFX10:       ; %bb.0: ; %main_body
; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm a16
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    ; return to shader part epilog
;
; GFX9-FLATSCR-LABEL: load.f32.1d:
; GFX9-FLATSCR:       ; %bb.0: ; %main_body
; GFX9-FLATSCR-NEXT:    s_mov_b32 s11, s9
; GFX9-FLATSCR-NEXT:    s_mov_b32 s10, s8
; GFX9-FLATSCR-NEXT:    s_mov_b32 s9, s7
; GFX9-FLATSCR-NEXT:    s_mov_b32 s8, s6
; GFX9-FLATSCR-NEXT:    s_mov_b32 s7, s5
; GFX9-FLATSCR-NEXT:    s_mov_b32 s6, s4
; GFX9-FLATSCR-NEXT:    s_mov_b32 s5, s3
; GFX9-FLATSCR-NEXT:    s_mov_b32 s4, s2
; GFX9-FLATSCR-NEXT:    image_load v0, v0, s[4:11] dmask:0x1 unorm a16
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    ; return to shader part epilog
;
; GFX11-LABEL: load.f32.1d:
; GFX11:       ; %bb.0: ; %main_body
; GFX11-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm a16
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    ; return to shader part epilog
;
; GFX12-LABEL: load.f32.1d:
; GFX12:       ; %bb.0: ; %main_body
; GFX12-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D a16
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    ; return to shader part epilog
main_body:
  %x = extractelement <2 x i16> %coords, i32 0
  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 1, i16 %x, <8 x i32> %rsrc, i32 0, i32 0)
  ret <4 x float> %v
}

declare void @llvm.amdgcn.image.store.1d.v4f32.i16(<4 x float>, i32, i16, <8 x i32>, i32, i32)

; from llvm.amdgcn.image.store.a16.ll
; covers image_store
;
define amdgpu_ps void @store_f32_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords, <4 x float> %val) {
; GFX9-LABEL: store_f32_1d:
; GFX9:       ; %bb.0: ; %main_body
; GFX9-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0x1 unorm a16
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_endpgm
;
; GFX90A-LABEL: store_f32_1d:
; GFX90A:       ; %bb.0: ; %main_body
; GFX90A-NEXT:    v_mov_b32_e32 v5, v4
; GFX90A-NEXT:    v_mov_b32_e32 v4, v3
; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
; GFX90A-NEXT:    image_store v[2:5], v0, s[0:7] dmask:0x1 unorm a16
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: store_f32_1d:
; GFX10:       ; %bb.0: ; %main_body
; GFX10-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm a16
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    s_endpgm
;
; GFX9-FLATSCR-LABEL: store_f32_1d:
; GFX9-FLATSCR:       ; %bb.0: ; %main_body
; GFX9-FLATSCR-NEXT:    s_mov_b32 s11, s9
; GFX9-FLATSCR-NEXT:    s_mov_b32 s10, s8
; GFX9-FLATSCR-NEXT:    s_mov_b32 s9, s7
; GFX9-FLATSCR-NEXT:    s_mov_b32 s8, s6
; GFX9-FLATSCR-NEXT:    s_mov_b32 s7, s5
; GFX9-FLATSCR-NEXT:    s_mov_b32 s6, s4
; GFX9-FLATSCR-NEXT:    s_mov_b32 s5, s3
; GFX9-FLATSCR-NEXT:    s_mov_b32 s4, s2
; GFX9-FLATSCR-NEXT:    image_store v[1:4], v0, s[4:11] dmask:0x1 unorm a16
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    s_endpgm
;
; GFX11-LABEL: store_f32_1d:
; GFX11:       ; %bb.0: ; %main_body
; GFX11-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm a16
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: store_f32_1d:
; GFX12:       ; %bb.0: ; %main_body
; GFX12-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D a16
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    s_endpgm

main_body:
  %x = extractelement <2 x i16> %coords, i32 0
  call void @llvm.amdgcn.image.store.1d.v4f32.i16(<4 x float> %val, i32 1, i16 %x, <8 x i32> %rsrc, i32 0, i32 0)
  ret void
}

declare i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32, i32, <8 x i32>, i32, i32)

; from llvm.amdgcn.image.atomic.dim.ll
; covers image_atomic (atomic with return)
;
define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
; GFX9-LABEL: atomic_swap_1d:
; GFX9:       ; %bb.0: ; %main_body
; GFX9-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    ; return to shader part epilog
;
; GFX90A-LABEL: atomic_swap_1d:
; GFX90A:       ; %bb.0: ; %main_body
; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
; GFX90A-NEXT:    image_atomic_swap v0, v2, s[0:7] dmask:0x1 unorm glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    ; return to shader part epilog
;
; GFX10-LABEL: atomic_swap_1d:
; GFX10:       ; %bb.0: ; %main_body
; GFX10-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    ; return to shader part epilog
;
; GFX9-FLATSCR-LABEL: atomic_swap_1d:
; GFX9-FLATSCR:       ; %bb.0: ; %main_body
; GFX9-FLATSCR-NEXT:    s_mov_b32 s11, s9
; GFX9-FLATSCR-NEXT:    s_mov_b32 s10, s8
; GFX9-FLATSCR-NEXT:    s_mov_b32 s9, s7
; GFX9-FLATSCR-NEXT:    s_mov_b32 s8, s6
; GFX9-FLATSCR-NEXT:    s_mov_b32 s7, s5
; GFX9-FLATSCR-NEXT:    s_mov_b32 s6, s4
; GFX9-FLATSCR-NEXT:    s_mov_b32 s5, s3
; GFX9-FLATSCR-NEXT:    s_mov_b32 s4, s2
; GFX9-FLATSCR-NEXT:    image_atomic_swap v0, v1, s[4:11] dmask:0x1 unorm glc
; GFX9-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT:    ; return to shader part epilog
;
; GFX11-LABEL: atomic_swap_1d:
; GFX11:       ; %bb.0: ; %main_body
; GFX11-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    ; return to shader part epilog
;
; GFX12-LABEL: atomic_swap_1d:
; GFX12:       ; %bb.0: ; %main_body
; GFX12-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    ; return to shader part epilog
main_body:
  %v = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
  %out = bitcast i32 %v to float
  ret float %out
}

; from lds-bounds.ll
; covers ds_write_b64 (atomic without return)
@compute_lds = external addrspace(3) global [512 x i32], align 16
;
define amdgpu_cs void @store_aligned(ptr addrspace(3) %ptr) #0 {
; GFX9-LABEL: store_aligned:
; GFX9:       ; %bb.0: ; %entry
; GFX9-NEXT:    v_mov_b32_e32 v1, 42
; GFX9-NEXT:    v_mov_b32_e32 v2, 43
; GFX9-NEXT:    ds_write_b64 v0, v[1:2]
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_endpgm
;
; GFX90A-LABEL: store_aligned:
; GFX90A:       ; %bb.0: ; %entry
; GFX90A-NEXT:    v_mov_b32_e32 v2, 42
; GFX90A-NEXT:    v_mov_b32_e32 v3, 43
; GFX90A-NEXT:    ds_write_b64 v0, v[2:3]
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: store_aligned:
; GFX10:       ; %bb.0: ; %entry
; GFX10-NEXT:    v_mov_b32_e32 v1, 42
; GFX10-NEXT:    v_mov_b32_e32 v2, 43
; GFX10-NEXT:    ds_write_b64 v0, v[1:2]
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_endpgm
;
; GFX9-FLATSCR-LABEL: store_aligned:
; GFX9-FLATSCR:       ; %bb.0: ; %entry
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v1, 42
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v2, 43
; GFX9-FLATSCR-NEXT:    ds_write_b64 v0, v[1:2]
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    s_endpgm
;
; GFX11-LABEL: store_aligned:
; GFX11:       ; %bb.0: ; %entry
; GFX11-NEXT:    v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v2, 43
; GFX11-NEXT:    ds_store_b64 v0, v[1:2]
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: store_aligned:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v2, 43
; GFX12-NEXT:    ds_store_b64 v0, v[1:2]
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr.gep.1 = getelementptr i32, ptr addrspace(3) %ptr, i32 1

  store i32 42, ptr addrspace(3) %ptr, align 8
  store i32 43, ptr addrspace(3) %ptr.gep.1
  ret void
}


; from lds-bounds.ll
; covers ds_read_b64
;
define amdgpu_cs <2 x float> @load_aligned(ptr addrspace(3) %ptr) #0 {
; GFX9-LABEL: load_aligned:
; GFX9:       ; %bb.0: ; %entry
; GFX9-NEXT:    ds_read_b64 v[0:1], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    ; return to shader part epilog
;
; GFX90A-LABEL: load_aligned:
; GFX90A:       ; %bb.0: ; %entry
; GFX90A-NEXT:    ds_read_b64 v[0:1], v0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    ; return to shader part epilog
;
; GFX10-LABEL: load_aligned:
; GFX10:       ; %bb.0: ; %entry
; GFX10-NEXT:    ds_read_b64 v[0:1], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    ; return to shader part epilog
;
; GFX9-FLATSCR-LABEL: load_aligned:
; GFX9-FLATSCR:       ; %bb.0: ; %entry
; GFX9-FLATSCR-NEXT:    ds_read_b64 v[0:1], v0
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    ; return to shader part epilog
;
; GFX11-LABEL: load_aligned:
; GFX11:       ; %bb.0: ; %entry
; GFX11-NEXT:    ds_load_b64 v[0:1], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    ; return to shader part epilog
;
; GFX12-LABEL: load_aligned:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    ds_load_b64 v[0:1], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    ; return to shader part epilog
entry:
  %ptr.gep.1 = getelementptr i32, ptr addrspace(3) %ptr, i32 1

  %v.0 = load i32, ptr addrspace(3) %ptr, align 8
  %v.1 = load i32, ptr addrspace(3) %ptr.gep.1

  %r.0 = insertelement <2 x i32> poison, i32 %v.0, i32 0
  %r.1 = insertelement <2 x i32> %r.0, i32 %v.1, i32 1
  %bc = bitcast <2 x i32> %r.1 to <2 x float>
  ret <2 x float> %bc
}

; from lds-bounds.ll
; covers ds_write2_b32
;
define amdgpu_cs void @store_global_const_idx() #0 {
; GFX9-LABEL: store_global_const_idx:
; GFX9:       ; %bb.0: ; %entry
; GFX9-NEXT:    v_mov_b32_e32 v0, compute_lds@abs32@lo
; GFX9-NEXT:    v_mov_b32_e32 v1, 42
; GFX9-NEXT:    v_mov_b32_e32 v2, 43
; GFX9-NEXT:    ds_write2_b32 v0, v1, v2 offset0:3 offset1:4
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_endpgm
;
; GFX90A-LABEL: store_global_const_idx:
; GFX90A:       ; %bb.0: ; %entry
; GFX90A-NEXT:    v_mov_b32_e32 v0, compute_lds@abs32@lo
; GFX90A-NEXT:    v_mov_b32_e32 v1, 42
; GFX90A-NEXT:    v_mov_b32_e32 v2, 43
; GFX90A-NEXT:    ds_write2_b32 v0, v1, v2 offset0:3 offset1:4
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: store_global_const_idx:
; GFX10:       ; %bb.0: ; %entry
; GFX10-NEXT:    v_mov_b32_e32 v0, compute_lds@abs32@lo
; GFX10-NEXT:    v_mov_b32_e32 v1, 42
; GFX10-NEXT:    v_mov_b32_e32 v2, 43
; GFX10-NEXT:    ds_write2_b32 v0, v1, v2 offset0:3 offset1:4
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_endpgm
;
; GFX9-FLATSCR-LABEL: store_global_const_idx:
; GFX9-FLATSCR:       ; %bb.0: ; %entry
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v0, compute_lds@abs32@lo
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v1, 42
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v2, 43
; GFX9-FLATSCR-NEXT:    ds_write2_b32 v0, v1, v2 offset0:3 offset1:4
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    s_endpgm
;
; GFX11-LABEL: store_global_const_idx:
; GFX11:       ; %bb.0: ; %entry
; GFX11-NEXT:    v_dual_mov_b32 v0, compute_lds@abs32@lo :: v_dual_mov_b32 v1, 42
; GFX11-NEXT:    v_mov_b32_e32 v2, 43
; GFX11-NEXT:    ds_store_2addr_b32 v0, v1, v2 offset0:3 offset1:4
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: store_global_const_idx:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    v_dual_mov_b32 v0, compute_lds@abs32@lo :: v_dual_mov_b32 v1, 42
; GFX12-NEXT:    v_mov_b32_e32 v2, 43
; GFX12-NEXT:    ds_store_2addr_b32 v0, v1, v2 offset0:3 offset1:4
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr.a = getelementptr [512 x i32], ptr addrspace(3) @compute_lds, i32 0, i32 3
  %ptr.b = getelementptr [512 x i32], ptr addrspace(3) @compute_lds, i32 0, i32 4

  store i32 42, ptr addrspace(3) %ptr.a
  store i32 43, ptr addrspace(3) %ptr.b
  ret void
}

; from lds-bounds.ll
; covers ds_read2_b32
;
define amdgpu_cs <2 x float> @load_global_const_idx() #0 {
; GFX9-LABEL: load_global_const_idx:
; GFX9:       ; %bb.0: ; %entry
; GFX9-NEXT:    v_mov_b32_e32 v0, compute_lds@abs32@lo
; GFX9-NEXT:    ds_read2_b32 v[0:1], v0 offset0:3 offset1:4
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    ; return to shader part epilog
;
; GFX90A-LABEL: load_global_const_idx:
; GFX90A:       ; %bb.0: ; %entry
; GFX90A-NEXT:    v_mov_b32_e32 v0, compute_lds@abs32@lo
; GFX90A-NEXT:    ds_read2_b32 v[0:1], v0 offset0:3 offset1:4
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    ; return to shader part epilog
;
; GFX10-LABEL: load_global_const_idx:
; GFX10:       ; %bb.0: ; %entry
; GFX10-NEXT:    v_mov_b32_e32 v0, compute_lds@abs32@lo
; GFX10-NEXT:    ds_read2_b32 v[0:1], v0 offset0:3 offset1:4
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    ; return to shader part epilog
;
; GFX9-FLATSCR-LABEL: load_global_const_idx:
; GFX9-FLATSCR:       ; %bb.0: ; %entry
; GFX9-FLATSCR-NEXT:    v_mov_b32_e32 v0, compute_lds@abs32@lo
; GFX9-FLATSCR-NEXT:    ds_read2_b32 v[0:1], v0 offset0:3 offset1:4
; GFX9-FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT:    ; return to shader part epilog
;
; GFX11-LABEL: load_global_const_idx:
; GFX11:       ; %bb.0: ; %entry
; GFX11-NEXT:    v_mov_b32_e32 v0, compute_lds@abs32@lo
; GFX11-NEXT:    ds_load_2addr_b32 v[0:1], v0 offset0:3 offset1:4
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    ; return to shader part epilog
;
; GFX12-LABEL: load_global_const_idx:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    v_mov_b32_e32 v0, compute_lds@abs32@lo
; GFX12-NEXT:    ds_load_2addr_b32 v[0:1], v0 offset0:3 offset1:4
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    ; return to shader part epilog
entry:
  %ptr.a = getelementptr [512 x i32], ptr addrspace(3) @compute_lds, i32 0, i32 3
  %ptr.b = getelementptr [512 x i32], ptr addrspace(3) @compute_lds, i32 0, i32 4

  %v.0 = load i32, ptr addrspace(3) %ptr.a
  %v.1 = load i32, ptr addrspace(3) %ptr.b

  %r.0 = insertelement <2 x i32> poison, i32 %v.0, i32 0
  %r.1 = insertelement <2 x i32> %r.0, i32 %v.1, i32 1
  %bc = bitcast <2 x i32> %r.1 to <2 x float>
  ret <2 x float> %bc
}