1; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,MUBUF %s 2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-MUBUF,MUBUF %s 3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-promote-alloca,+enable-flat-scratch -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-FLATSCR %s 4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11 %s 5 6; Test that non-entry function frame indices are expanded properly to 7; give an index relative to the scratch wave offset register 8 9; Materialize into a mov. Make sure there isn't an unnecessary copy. 10; GCN-LABEL: {{^}}func_mov_fi_i32: 11; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 12 13; CI-NEXT: v_lshr_b32_e64 v0, s32, 6 14; GFX9-MUBUF-NEXT: v_lshrrev_b32_e64 v0, 6, s32 15 16; GFX9-FLATSCR: v_mov_b32_e32 v0, s32 17; GFX9-FLATSCR-NOT: v_lshrrev_b32_e64 18 19; MUBUF-NOT: v_mov 20 21; GCN: ds_write_b32 v0, v0 22define void @func_mov_fi_i32() #0 { 23 %alloca = alloca i32, addrspace(5) 24 store volatile ptr addrspace(5) %alloca, ptr addrspace(3) undef 25 ret void 26} 27 28; Offset due to different objects 29; GCN-LABEL: {{^}}func_mov_fi_i32_offset: 30; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 31 32; CI-DAG: v_lshr_b32_e64 v0, s32, 6 33; CI-NOT: v_mov 34; CI: ds_write_b32 v0, v0 35; CI-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s32, 6 36; CI-NEXT: v_add_i32_e{{32|64}} v0, {{s\[[0-9]+:[0-9]+\]|vcc}}, 4, [[SCALED]] 37; CI-NEXT: ds_write_b32 v0, v0 38 39; GFX9-MUBUF-NEXT: v_lshrrev_b32_e64 v0, 6, s32 40; GFX9-FLATSCR: v_mov_b32_e32 v0, s32 41; GFX9-FLATSCR: s_add_i32 [[ADD:[^,]+]], s32, 4 42; GFX9-NEXT: ds_write_b32 v0, v0 43; GFX9-MUBUF-NEXT: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s32 44; GFX9-MUBUF-NEXT: v_add_u32_e32 v0, 4, [[SCALED]] 45; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, [[ADD]] 46; GFX9-NEXT: ds_write_b32 v0, v0 47define void @func_mov_fi_i32_offset() #0 { 48 %alloca0 = alloca i32, addrspace(5) 49 %alloca1 = alloca i32, addrspace(5) 50 store volatile ptr addrspace(5) %alloca0, ptr addrspace(3) undef 51 store volatile ptr addrspace(5) %alloca1, ptr addrspace(3) undef 52 ret void 53} 54 55; Materialize into an add of a constant offset from the FI. 56; FIXME: Should be able to merge adds 57 58; GCN-LABEL: {{^}}func_add_constant_to_fi_i32: 59; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 60 61; CI: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s32, 6 62; CI-NEXT: v_add_i32_e32 v0, vcc, 4, [[SCALED]] 63 64; GFX9-MUBUF: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s32 65; GFX9-MUBUF-NEXT: v_add_u32_e32 v0, 4, [[SCALED]] 66 67; FIXME: Should commute and shrink 68; GFX9-FLATSCR: v_add_u32_e64 v0, 4, s32 69 70; GCN-NOT: v_mov 71; GCN: ds_write_b32 v0, v0 72define void @func_add_constant_to_fi_i32() #0 { 73 %alloca = alloca [2 x i32], align 4, addrspace(5) 74 %gep0 = getelementptr inbounds [2 x i32], ptr addrspace(5) %alloca, i32 0, i32 1 75 store volatile ptr addrspace(5) %gep0, ptr addrspace(3) undef 76 ret void 77} 78 79; A user the materialized frame index can't be meaningfully folded 80; into. 81; FIXME: Should use s_mul but the frame index always gets materialized into a 82; vgpr 83 84; GCN-LABEL: {{^}}func_other_fi_user_i32: 85; MUBUF: s_lshr_b32 [[SCALED:s[0-9]+]], s32, 6 86; MUBUF: s_mul_i32 [[MUL:s[0-9]+]], [[SCALED]], 9 87; MUBUF: v_mov_b32_e32 v0, [[MUL]] 88 89; GFX9-FLATSCR: s_mul_i32 [[MUL:s[0-9]+]], s32, 9 90; GFX9-FLATSCR: v_mov_b32_e32 v0, [[MUL]] 91 92; GCN-NOT: v_mov 93; GCN: ds_write_b32 v0, v0 94define void @func_other_fi_user_i32() #0 { 95 %alloca = alloca [2 x i32], align 4, addrspace(5) 96 %ptrtoint = ptrtoint ptr addrspace(5) %alloca to i32 97 %mul = mul i32 %ptrtoint, 9 98 store volatile i32 %mul, ptr addrspace(3) undef 99 ret void 100} 101 102; GCN-LABEL: {{^}}func_store_private_arg_i32_ptr: 103; GCN: v_mov_b32_e32 v1, 15{{$}} 104; MUBUF: buffer_store_dword v1, v0, s[0:3], 0 offen{{$}} 105; GFX9-FLATSCR: scratch_store_dword v0, v1, off{{$}} 106define void @func_store_private_arg_i32_ptr(ptr addrspace(5) %ptr) #0 { 107 store volatile i32 15, ptr addrspace(5) %ptr 108 ret void 109} 110 111; GCN-LABEL: {{^}}func_load_private_arg_i32_ptr: 112; GCN: s_waitcnt 113; MUBUF-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen glc{{$}} 114; GFX9-FLATSCR-NEXT: scratch_load_dword v0, v0, off glc{{$}} 115define void @func_load_private_arg_i32_ptr(ptr addrspace(5) %ptr) #0 { 116 %val = load volatile i32, ptr addrspace(5) %ptr 117 ret void 118} 119 120; GCN-LABEL: {{^}}void_func_byval_struct_i8_i32_ptr: 121; GCN: s_waitcnt 122 123; CI: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], s32, 6 124; CI-NEXT: v_or_b32_e32 v0, 4, [[SHIFT]] 125 126; GFX9-MUBUF: v_lshrrev_b32_e64 [[SHIFT:v[0-9]+]], 6, s32 127; GFX9-MUBUF-NEXT: v_or_b32_e32 v0, 4, [[SHIFT]] 128 129; GFX9-FLATSCR: v_or_b32_e64 v0, s32, 4 130 131; GCN-NOT: v_mov 132; GCN: ds_write_b32 v0, v0 133define void @void_func_byval_struct_i8_i32_ptr(ptr addrspace(5) byval({ i8, i32 }) %arg0) #0 { 134 %gep0 = getelementptr inbounds { i8, i32 }, ptr addrspace(5) %arg0, i32 0, i32 0 135 %gep1 = getelementptr inbounds { i8, i32 }, ptr addrspace(5) %arg0, i32 0, i32 1 136 %load1 = load i32, ptr addrspace(5) %gep1 137 store volatile ptr addrspace(5) %gep1, ptr addrspace(3) undef 138 ret void 139} 140 141; GCN-LABEL: {{^}}void_func_byval_struct_i8_i32_ptr_value: 142; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 143; MUBUF-NEXT: buffer_load_ubyte v0, off, s[0:3], s32 144; MUBUF-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4 145; GFX9-FLATSCR-NEXT: scratch_load_ubyte v0, off, s32 146; GFX9-FLATSCR-NEXT: scratch_load_dword v1, off, s32 offset:4 147define void @void_func_byval_struct_i8_i32_ptr_value(ptr addrspace(5) byval({ i8, i32 }) %arg0) #0 { 148 %gep0 = getelementptr inbounds { i8, i32 }, ptr addrspace(5) %arg0, i32 0, i32 0 149 %gep1 = getelementptr inbounds { i8, i32 }, ptr addrspace(5) %arg0, i32 0, i32 1 150 %load0 = load i8, ptr addrspace(5) %gep0 151 %load1 = load i32, ptr addrspace(5) %gep1 152 store volatile i8 %load0, ptr addrspace(3) undef 153 store volatile i32 %load1, ptr addrspace(3) undef 154 ret void 155} 156 157; GCN-LABEL: {{^}}void_func_byval_struct_i8_i32_ptr_nonentry_block: 158 159; GCN: s_and_saveexec_b64 160 161; CI: buffer_load_dword v{{[0-9]+}}, off, s[0:3], s32 offset:4 glc{{$}} 162; GFX9-MUBUF: buffer_load_dword v{{[0-9]+}}, off, s[0:3], s32 offset:4 glc{{$}} 163; GFX9-FLATSCR: scratch_load_dword v{{[0-9]+}}, off, s32 offset:4 glc{{$}} 164 165; CI: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], s32, 6 166; CI: v_add_i32_e64 [[GEP:v[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 4, [[SHIFT]] 167 168; GFX9-MUBUF: v_lshrrev_b32_e64 [[SP:v[0-9]+]], 6, s32 169; GFX9-MUBUF: v_add_u32_e32 [[GEP:v[0-9]+]], 4, [[SP]] 170 171; GFX9-FLATSCR: v_add_u32_e64 [[GEP:v[0-9]+]], 4, s32 172 173; GCN: ds_write_b32 v{{[0-9]+}}, [[GEP]] 174define void @void_func_byval_struct_i8_i32_ptr_nonentry_block(ptr addrspace(5) byval({ i8, i32 }) %arg0, i32 %arg2) #0 { 175 %cmp = icmp eq i32 %arg2, 0 176 br i1 %cmp, label %bb, label %ret 177 178bb: 179 %gep0 = getelementptr inbounds { i8, i32 }, ptr addrspace(5) %arg0, i32 0, i32 0 180 %gep1 = getelementptr inbounds { i8, i32 }, ptr addrspace(5) %arg0, i32 0, i32 1 181 %load1 = load volatile i32, ptr addrspace(5) %gep1 182 store volatile ptr addrspace(5) %gep1, ptr addrspace(3) undef 183 br label %ret 184 185ret: 186 ret void 187} 188 189; Added offset can't be used with VOP3 add 190; GCN-LABEL: {{^}}func_other_fi_user_non_inline_imm_offset_i32: 191 192; MUBUF: s_lshr_b32 [[SCALED:s[0-9]+]], s32, 6 193; MUBUF: s_addk_i32 [[SCALED]], 0x200 194 195; MUBUF: s_mul_i32 [[Z:s[0-9]+]], [[SCALED]], 9 196; MUBUF: v_mov_b32_e32 [[VZ:v[0-9]+]], [[Z]] 197 198; GFX9-FLATSCR: s_add_i32 [[SZ:[^,]+]], s32, 0x200 199; GFX9-FLATSCR: s_mul_i32 [[MUL:s[0-9]+]], [[SZ]], 9 200; GFX9-FLATSCR: v_mov_b32_e32 [[VZ:v[0-9]+]], [[MUL]] 201 202; GCN: ds_write_b32 v0, [[VZ]] 203define void @func_other_fi_user_non_inline_imm_offset_i32() #0 { 204 %alloca0 = alloca [128 x i32], align 4, addrspace(5) 205 %alloca1 = alloca [8 x i32], align 4, addrspace(5) 206 %gep0 = getelementptr inbounds [128 x i32], ptr addrspace(5) %alloca0, i32 0, i32 65 207 store volatile i32 7, ptr addrspace(5) %gep0 208 %ptrtoint = ptrtoint ptr addrspace(5) %alloca1 to i32 209 %mul = mul i32 %ptrtoint, 9 210 store volatile i32 %mul, ptr addrspace(3) undef 211 ret void 212} 213 214; GCN-LABEL: {{^}}func_other_fi_user_non_inline_imm_offset_i32_vcc_live: 215 216; MUBUF: s_lshr_b32 [[SCALED:s[0-9]+]], s32, 6 217; MUBUF: s_addk_i32 [[SCALED]], 0x200 218; MUBUF: s_mul_i32 [[Z:s[0-9]+]], [[SCALED]], 9 219; MUBUF: v_mov_b32_e32 [[VZ:v[0-9]+]], [[Z]] 220 221; GFX9-FLATSCR: s_add_i32 [[SZ:[^,]+]], s32, 0x200 222; GFX9-FLATSCR: s_mul_i32 [[MUL:s[0-9]+]], [[SZ]], 9 223; GFX9-FLATSCR: v_mov_b32_e32 [[VZ:v[0-9]+]], [[MUL]] 224 225; GCN: ds_write_b32 v0, [[VZ]] 226define void @func_other_fi_user_non_inline_imm_offset_i32_vcc_live() #0 { 227 %alloca0 = alloca [128 x i32], align 4, addrspace(5) 228 %alloca1 = alloca [8 x i32], align 4, addrspace(5) 229 %vcc = call i64 asm sideeffect "; def $0", "={vcc}"() 230 %gep0 = getelementptr inbounds [128 x i32], ptr addrspace(5) %alloca0, i32 0, i32 65 231 store volatile i32 7, ptr addrspace(5) %gep0 232 call void asm sideeffect "; use $0", "{vcc}"(i64 %vcc) 233 %ptrtoint = ptrtoint ptr addrspace(5) %alloca1 to i32 234 %mul = mul i32 %ptrtoint, 9 235 store volatile i32 %mul, ptr addrspace(3) undef 236 ret void 237} 238 239declare void @func(ptr addrspace(5) nocapture) #0 240 241; undef flag not preserved in eliminateFrameIndex when handling the 242; stores in the middle block. 243 244; GCN-LABEL: {{^}}undefined_stack_store_reg: 245; GCN: s_and_saveexec_b64 246; MUBUF: buffer_store_dword v0, off, s[0:3], s33 offset: 247; MUBUF: buffer_store_dword v0, off, s[0:3], s33 offset: 248; MUBUF: buffer_store_dword v0, off, s[0:3], s33 offset: 249; MUBUF: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s33 offset: 250; FLATSCR: scratch_store_dword v0, off, s33 offset: 251; FLATSCR: scratch_store_dword v0, off, s33 offset: 252; FLATSCR: scratch_store_dword v0, off, s33 offset: 253; FLATSCR: scratch_store_dword v{{[0-9]+}}, off, s33 offset: 254define void @undefined_stack_store_reg(float %arg, i32 %arg1) #0 { 255bb: 256 %tmp = alloca <4 x float>, align 16, addrspace(5) 257 %tmp2 = insertelement <4 x float> undef, float %arg, i32 0 258 store <4 x float> %tmp2, ptr addrspace(5) undef 259 %tmp3 = icmp eq i32 %arg1, 0 260 br i1 %tmp3, label %bb4, label %bb5 261 262bb4: 263 call void @func(ptr addrspace(5) nonnull undef) 264 store <4 x float> %tmp2, ptr addrspace(5) %tmp, align 16 265 call void @func(ptr addrspace(5) nonnull %tmp) 266 br label %bb5 267 268bb5: 269 ret void 270} 271 272; GCN-LABEL: {{^}}alloca_ptr_nonentry_block: 273; GCN: s_and_saveexec_b64 274; MUBUF: buffer_load_dword v{{[0-9]+}}, off, s[0:3], s32 offset:4 275; FLATSCR: scratch_load_dword v{{[0-9]+}}, off, s32 offset:4 276 277; CI: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], s32, 6 278; CI-NEXT: v_or_b32_e32 [[PTR:v[0-9]+]], 4, [[SHIFT]] 279 280; GFX9-MUBUF: v_lshrrev_b32_e64 [[SHIFT:v[0-9]+]], 6, s32 281; GFX9-MUBUF-NEXT: v_or_b32_e32 [[PTR:v[0-9]+]], 4, [[SHIFT]] 282 283; GFX9-FLATSCR: v_or_b32_e64 [[PTR:v[0-9]+]], s32, 4 284 285; GCN: ds_write_b32 v{{[0-9]+}}, [[PTR]] 286define void @alloca_ptr_nonentry_block(i32 %arg0) #0 { 287 %alloca0 = alloca { i8, i32 }, align 8, addrspace(5) 288 %cmp = icmp eq i32 %arg0, 0 289 br i1 %cmp, label %bb, label %ret 290 291bb: 292 %gep0 = getelementptr inbounds { i8, i32 }, ptr addrspace(5) %alloca0, i32 0, i32 0 293 %gep1 = getelementptr inbounds { i8, i32 }, ptr addrspace(5) %alloca0, i32 0, i32 1 294 %load1 = load volatile i32, ptr addrspace(5) %gep1 295 store volatile ptr addrspace(5) %gep1, ptr addrspace(3) undef 296 br label %ret 297 298ret: 299 ret void 300} 301 302%struct0 = type { [4224 x %type.i16] } 303%type.i16 = type { i16 } 304@_ZZN0 = external hidden addrspace(3) global %struct0, align 8 305 306; GFX11-LABEL: tied_operand_test: 307; GFX11: ; %bb.0: ; %entry 308; GFX11: scratch_load_u16 [[LDRESULT:v[0-9]+]], off, off 309; GFX11: v_dual_mov_b32 [[C:v[0-9]+]], 0x7b :: v_dual_mov_b32 v{{[0-9]+}}, s{{[0-9]+}} 310; GFX11-DAG: ds_store_b16 v{{[0-9]+}}, [[LDRESULT]] offset:10 311; GFX11-DAG: ds_store_b16 v{{[0-9]+}}, [[C]] offset:8 312; GFX11-NEXT: s_endpgm 313define protected amdgpu_kernel void @tied_operand_test(i1 %c1, i1 %c2, i32 %val) { 314entry: 315 %scratch0 = alloca i16, align 4, addrspace(5) 316 %scratch1 = alloca i16, align 4, addrspace(5) 317 %first = select i1 %c1, ptr addrspace(5) %scratch0, ptr addrspace(5) %scratch1 318 %spec.select = select i1 %c2, ptr addrspace(5) %first, ptr addrspace(5) %scratch0 319 %dead.load = load i16, ptr addrspace(5) %spec.select, align 2 320 %scratch0.load = load i16, ptr addrspace(5) %scratch0, align 4 321 %add4 = add nuw nsw i32 %val, 4 322 %addr0 = getelementptr inbounds %struct0, ptr addrspace(3) @_ZZN0, i32 0, i32 0, i32 %add4, i32 0 323 store i16 123, ptr addrspace(3) %addr0, align 2 324 %add5 = add nuw nsw i32 %val, 5 325 %addr1 = getelementptr inbounds %struct0, ptr addrspace(3) @_ZZN0, i32 0, i32 0, i32 %add5, i32 0 326 store i16 %scratch0.load, ptr addrspace(3) %addr1, align 2 327 ret void 328} 329 330; GCN-LABEL: {{^}}fi_vop3_literal_error: 331; CI: v_lshr_b32_e64 [[SCALED_FP:v[0-9]+]], s33, 6 332; CI: s_movk_i32 vcc_lo, 0x3000 333; CI-NEXT: v_add_i32_e32 [[SCALED_FP]], vcc, vcc_lo, [[SCALED_FP]] 334; CI-NEXT: v_add_i32_e32 v0, vcc, 64, [[SCALED_FP]] 335 336; GFX9-MUBUF: v_lshrrev_b32_e64 [[SCALED_FP:v[0-9]+]], 6, s33 337; GFX9-MUBUF-NEXT: v_add_u32_e32 [[SCALED_FP]], 0x3000, [[SCALED_FP]] 338; GFX9-MUBUF-NEXT: v_add_u32_e32 v0, 64, [[SCALED_FP]] 339define void @fi_vop3_literal_error() { 340entry: 341 %pin.low = alloca i32, align 8192, addrspace(5) 342 %local.area = alloca [1060 x i64], align 4096, addrspace(5) 343 store i32 0, ptr addrspace(5) %pin.low, align 4 344 %gep.small.offset = getelementptr i8, ptr addrspace(5) %local.area, i64 64 345 %load1 = load volatile i64, ptr addrspace(5) %gep.small.offset, align 4 346 ret void 347} 348 349attributes #0 = { nounwind } 350