1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def smrd_offset_8 : ImmOperand<i32, "SMRDOffset8", 1>; 10 11let EncoderMethod = "getSMEMOffsetEncoding", 12 DecoderMethod = "decodeSMEMOffset" in { 13def SMEMOffset : ImmOperand<i32, "SMEMOffset", 1>; 14def SMEMOffsetMod : NamedIntOperand<"offset", 0> { 15 let AlwaysPrint = 1; 16 let PrintInHex = 1; 17} 18def OptSMEMOffsetMod : NamedIntOperand<"offset"> { 19 let ImmTy = SMEMOffsetMod.ImmTy; 20 let PredicateMethod = SMEMOffsetMod.PredicateMethod; 21 let PrintMethod = SMEMOffsetMod.PrintMethod; 22} 23} 24 25//===----------------------------------------------------------------------===// 26// Scalar Memory classes 27//===----------------------------------------------------------------------===// 28 29class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> : 30 InstSI <outs, ins, "", pattern>, 31 SIMCInstr<NAME, SIEncodingFamily.NONE> { 32 let isPseudo = 1; 33 let isCodeGenOnly = 1; 34 35 let LGKM_CNT = 1; 36 let SMRD = 1; 37 let mayStore = 0; 38 let mayLoad = 1; 39 let hasSideEffects = 0; 40 let maybeAtomic = 0; 41 let UseNamedOperandTable = 1; 42 let SchedRW = [WriteSMEM]; 43 44 string Mnemonic = opName; 45 string AsmOperands = asmOps; 46 47 bits<1> has_sbase = 1; 48 bits<1> has_sdst = 1; 49 bit has_glc = 0; 50 bit has_dlc = 0; 51 bit has_offset = 0; 52 bit has_soffset = 0; 53 bit is_buffer = 0; 54} 55 56class SM_Real <SM_Pseudo ps, string opName = ps.Mnemonic> 57 : InstSI<ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands> { 58 59 let isPseudo = 0; 60 let isCodeGenOnly = 0; 61 62 Instruction Opcode = !cast<Instruction>(NAME); 63 64 // copy relevant pseudo op flags 65 let LGKM_CNT = ps.LGKM_CNT; 66 let SMRD = ps.SMRD; 67 let mayStore = ps.mayStore; 68 let mayLoad = ps.mayLoad; 69 let hasSideEffects = ps.hasSideEffects; 70 let UseNamedOperandTable = ps.UseNamedOperandTable; 71 let SchedRW = ps.SchedRW; 72 let SubtargetPredicate = ps.SubtargetPredicate; 73 let OtherPredicates = ps.OtherPredicates; 74 let AsmMatchConverter = ps.AsmMatchConverter; 75 let IsAtomicRet = ps.IsAtomicRet; 76 let IsAtomicNoRet = ps.IsAtomicNoRet; 77 let Uses = ps.Uses; 78 let Defs = ps.Defs; 79 let isConvergent = ps.isConvergent; 80 81 let TSFlags = ps.TSFlags; 82 83 bit is_buffer = ps.is_buffer; 84 85 // encoding 86 bits<7> sbase; 87 bits<7> sdst; 88 bits<32> offset; 89 bits<8> soffset; 90 bits<5> cpol; 91} 92 93class OffsetMode<bit hasOffset, bit hasSOffset, string variant, 94 dag ins, string asm> { 95 bit HasOffset = hasOffset; 96 bit HasSOffset = hasSOffset; 97 string Variant = variant; 98 dag Ins = ins; 99 string Asm = asm; 100} 101 102def IMM_Offset : OffsetMode<1, 0, "_IMM", (ins SMEMOffset:$offset), "$offset">; 103def SGPR_Offset : OffsetMode<0, 1, "_SGPR", (ins SReg_32:$soffset), "$soffset">; 104def SGPR_IMM_Offset : OffsetMode<1, 1, "_SGPR_IMM", 105 (ins SReg_32:$soffset, SMEMOffsetMod:$offset), 106 "$soffset$offset">; 107def SGPR_IMM_OptOffset : OffsetMode<1, 1, "_SGPR_IMM", 108 (ins SReg_32:$soffset, OptSMEMOffsetMod:$offset), 109 "$soffset$offset">; 110 111class SM_Probe_Pseudo <string opName, RegisterClass baseClass, OffsetMode offsets> 112 : SM_Pseudo<opName, (outs), 113 !con((ins i8imm:$sdata, baseClass:$sbase), offsets.Ins), 114 " $sdata, $sbase, " # offsets.Asm> { 115 let mayLoad = 0; 116 let mayStore = 0; 117 let has_glc = 0; 118 let LGKM_CNT = 0; 119 let ScalarStore = 0; 120 let hasSideEffects = 1; 121 let has_offset = offsets.HasOffset; 122 let has_soffset = offsets.HasSOffset; 123} 124 125class SM_Load_Pseudo <string opName, RegisterClass baseClass, 126 RegisterClass dstClass, OffsetMode offsets> 127 : SM_Pseudo<opName, (outs dstClass:$sdst), 128 !con((ins baseClass:$sbase), offsets.Ins, (ins CPol:$cpol)), 129 " $sdst, $sbase, " # offsets.Asm # "$cpol", []> { 130 RegisterClass BaseClass = baseClass; 131 let mayLoad = 1; 132 let isReMaterializable = 1; 133 let mayStore = 0; 134 let has_glc = 1; 135 let has_dlc = 1; 136 let has_offset = offsets.HasOffset; 137 let has_soffset = offsets.HasSOffset; 138} 139 140class SM_Store_Pseudo <string opName, RegisterClass baseClass, 141 RegisterClass srcClass, OffsetMode offsets> 142 : SM_Pseudo<opName, (outs), !con((ins srcClass:$sdata, baseClass:$sbase), 143 offsets.Ins, (ins CPol:$cpol)), 144 " $sdata, $sbase, " # offsets.Asm # "$cpol"> { 145 RegisterClass BaseClass = baseClass; 146 let mayLoad = 0; 147 let mayStore = 1; 148 let has_glc = 1; 149 let has_dlc = 1; 150 let has_offset = offsets.HasOffset; 151 let has_soffset = offsets.HasSOffset; 152 let ScalarStore = 1; 153} 154 155class SM_Discard_Pseudo <string opName, OffsetMode offsets> 156 : SM_Pseudo<opName, (outs), !con((ins SReg_64:$sbase), offsets.Ins), 157 " $sbase, " # offsets.Asm> { 158 let mayLoad = 0; 159 let mayStore = 0; 160 let has_glc = 0; 161 let has_sdst = 0; 162 let ScalarStore = 0; 163 let hasSideEffects = 1; 164 let has_offset = offsets.HasOffset; 165 let has_soffset = offsets.HasSOffset; 166} 167 168multiclass SM_Load_Pseudos<string op, RegisterClass baseClass, 169 RegisterClass dstClass, OffsetMode offsets> { 170 defvar opName = !tolower(op); 171 def "" : SM_Load_Pseudo <opName, baseClass, dstClass, offsets>; 172 173 // The constrained multi-dword load equivalents with early clobber flag at 174 // the dst operands. They are needed only for codegen and there is no need 175 // for their real opcodes. 176 if !gt(dstClass.RegTypes[0].Size, 32) then 177 let Constraints = "@earlyclobber $sdst", 178 PseudoInstr = op # offsets.Variant in 179 def "" # _ec : SM_Load_Pseudo <opName, baseClass, dstClass, offsets>; 180} 181 182multiclass SM_Pseudo_Loads<RegisterClass baseClass, 183 RegisterClass dstClass> { 184 defm _IMM : SM_Load_Pseudos <NAME, baseClass, dstClass, IMM_Offset>; 185 defm _SGPR : SM_Load_Pseudos <NAME, baseClass, dstClass, SGPR_Offset>; 186 defm _SGPR_IMM : SM_Load_Pseudos <NAME, baseClass, dstClass, SGPR_IMM_Offset>; 187} 188 189multiclass SM_Pseudo_Stores<RegisterClass baseClass, 190 RegisterClass srcClass> { 191 defvar opName = !tolower(NAME); 192 def _IMM : SM_Store_Pseudo <opName, baseClass, srcClass, IMM_Offset>; 193 def _SGPR : SM_Store_Pseudo <opName, baseClass, srcClass, SGPR_Offset>; 194 def _SGPR_IMM : SM_Store_Pseudo <opName, baseClass, srcClass, SGPR_IMM_Offset>; 195} 196 197multiclass SM_Pseudo_Discards { 198 defvar opName = !tolower(NAME); 199 def _IMM : SM_Discard_Pseudo <opName, IMM_Offset>; 200 def _SGPR : SM_Discard_Pseudo <opName, SGPR_Offset>; 201 def _SGPR_IMM : SM_Discard_Pseudo <opName, SGPR_IMM_Offset>; 202} 203 204class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 205 opName, (outs SReg_64_XEXEC:$sdst), (ins), 206 " $sdst", [(set i64:$sdst, (node))]> { 207 let hasSideEffects = 1; 208 209 let mayStore = 0; 210 let mayLoad = 0; 211 let has_sbase = 0; 212} 213 214class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 215 opName, (outs), (ins), "", [(node)]> { 216 let hasSideEffects = 1; 217 let mayLoad = 0; 218 let mayStore = 0; 219 let has_sdst = 0; 220 let has_sbase = 0; 221} 222 223multiclass SM_Pseudo_Probe<RegisterClass baseClass> { 224 defvar opName = !tolower(NAME); 225 def _IMM : SM_Probe_Pseudo <opName, baseClass, IMM_Offset>; 226 def _SGPR : SM_Probe_Pseudo <opName, baseClass, SGPR_Offset>; 227 def _SGPR_IMM : SM_Probe_Pseudo <opName, baseClass, SGPR_IMM_Offset>; 228 def _SGPR_OPT_IMM : SM_Probe_Pseudo <opName, baseClass, SGPR_IMM_OptOffset>; 229} 230 231class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo< 232 opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins), 233 " $sdst", [(set i32:$sdst, (node))]> { 234 let hasSideEffects = 1; 235 let mayStore = 0; 236 let mayLoad = 0; 237 let has_sbase = 0; 238} 239 240class SM_Prefetch_Pseudo <string opName, RegisterClass baseClass, bit hasSBase> 241 : SM_Pseudo<opName, (outs), !con(!if(hasSBase, (ins baseClass:$sbase), (ins)), 242 (ins SMEMOffset:$offset, SReg_32:$soffset, i8imm:$sdata)), 243 !if(hasSBase, " $sbase,", "") # " $offset, $soffset, $sdata"> { 244 // Mark prefetches as both load and store to prevent reordering with loads 245 // and stores. This is also needed for pattern to match prefetch intrinsic. 246 let mayLoad = 1; 247 let mayStore = 1; 248 let has_glc = 0; 249 let LGKM_CNT = 0; 250 let has_sbase = hasSBase; 251 let ScalarStore = 0; 252 let has_offset = 1; 253 let has_soffset = 1; 254} 255 256//===----------------------------------------------------------------------===// 257// Scalar Atomic Memory Classes 258//===----------------------------------------------------------------------===// 259 260class SM_Atomic_Pseudo <string opName, 261 dag outs, dag ins, string asmOps, bit isRet> 262 : SM_Pseudo<opName, outs, ins, asmOps, []> { 263 264 bit glc = isRet; 265 266 let mayLoad = 1; 267 let mayStore = 1; 268 let has_glc = 1; 269 let has_dlc = 1; 270 let has_soffset = 1; 271 272 // Should these be set? 273 let ScalarStore = 1; 274 let hasSideEffects = 1; 275 let maybeAtomic = 1; 276 277 let IsAtomicNoRet = !not(isRet); 278 let IsAtomicRet = isRet; 279} 280 281class SM_Pseudo_Atomic<string opName, 282 RegisterClass baseClass, 283 RegisterClass dataClass, 284 OffsetMode offsets, 285 bit isRet, 286 Operand CPolTy = !if(isRet, CPol_GLC, CPol_NonGLC)> : 287 SM_Atomic_Pseudo<opName, 288 !if(isRet, (outs dataClass:$sdst), (outs)), 289 !con((ins dataClass:$sdata, baseClass:$sbase), offsets.Ins, 290 (ins CPolTy:$cpol)), 291 !if(isRet, " $sdst", " $sdata") # 292 ", $sbase, " # offsets.Asm # "$cpol", 293 isRet> { 294 let has_offset = offsets.HasOffset; 295 let has_soffset = offsets.HasSOffset; 296 297 let Constraints = !if(isRet, "$sdst = $sdata", ""); 298 let DisableEncoding = !if(isRet, "$sdata", ""); 299} 300 301multiclass SM_Pseudo_Atomics<RegisterClass baseClass, 302 RegisterClass dataClass> { 303 defvar opName = !tolower(NAME); 304 def _IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 0>; 305 def _SGPR : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 0>; 306 def _SGPR_IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 0>; 307 def _IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 1>; 308 def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 1>; 309 def _SGPR_IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 1>; 310} 311 312//===----------------------------------------------------------------------===// 313// Scalar Memory Instructions 314//===----------------------------------------------------------------------===// 315 316// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit 317// SMRD instructions, because the SReg_32_XM0 register class does not include M0 318// and writing to M0 from an SMRD instruction will hang the GPU. 319 320// XXX - SMEM instructions do not allow exec for data operand, but 321// does sdst for SMRD on SI/CI? 322defm S_LOAD_DWORD : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>; 323defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <SReg_64, SReg_64_XEXEC>; 324let SubtargetPredicate = HasScalarDwordx3Loads in 325 defm S_LOAD_DWORDX3 : SM_Pseudo_Loads <SReg_64, SReg_96>; 326defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <SReg_64, SReg_128>; 327defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <SReg_64, SReg_256>; 328defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <SReg_64, SReg_512>; 329defm S_LOAD_I8 : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>; 330defm S_LOAD_U8 : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>; 331defm S_LOAD_I16 : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>; 332defm S_LOAD_U16 : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>; 333 334let is_buffer = 1 in { 335defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 336// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on 337// SI/CI, bit disallowed for SMEM on VI. 338defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_64_XEXEC>; 339let SubtargetPredicate = HasScalarDwordx3Loads in 340 defm S_BUFFER_LOAD_DWORDX3 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_96>; 341defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_128>; 342defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_256>; 343defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_512>; 344defm S_BUFFER_LOAD_I8 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 345defm S_BUFFER_LOAD_U8 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 346defm S_BUFFER_LOAD_I16 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 347defm S_BUFFER_LOAD_U16 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 348} 349 350let SubtargetPredicate = HasScalarStores in { 351defm S_STORE_DWORD : SM_Pseudo_Stores <SReg_64, SReg_32_XM0_XEXEC>; 352defm S_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_64, SReg_64_XEXEC>; 353defm S_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_64, SReg_128>; 354 355let is_buffer = 1 in { 356defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 357defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_128_XNULL, SReg_64_XEXEC>; 358defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_128_XNULL, SReg_128>; 359} 360} // End SubtargetPredicate = HasScalarStores 361 362let SubtargetPredicate = HasSMemTimeInst in 363def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>; 364def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>; 365 366let SubtargetPredicate = isGFX7GFX8GFX9 in { 367def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>; 368} // let SubtargetPredicate = isGFX7GFX8GFX9 369 370let SubtargetPredicate = isGFX8Plus in { 371let OtherPredicates = [HasScalarStores] in { 372def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>; 373def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>; 374} // End OtherPredicates = [HasScalarStores] 375 376defm S_ATC_PROBE : SM_Pseudo_Probe <SReg_64>; 377let is_buffer = 1 in { 378defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <SReg_128_XNULL>; 379} 380} // SubtargetPredicate = isGFX8Plus 381 382let SubtargetPredicate = HasSMemRealTime in 383def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>; 384 385let SubtargetPredicate = isGFX10Plus in 386def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">; 387let SubtargetPredicate = HasGetWaveIdInst in 388def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>; 389 390 391let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in { 392defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>; 393defm S_SCRATCH_LOAD_DWORDX2 : SM_Pseudo_Loads <SReg_64, SReg_64_XEXEC>; 394defm S_SCRATCH_LOAD_DWORDX4 : SM_Pseudo_Loads <SReg_64, SReg_128>; 395 396defm S_SCRATCH_STORE_DWORD : SM_Pseudo_Stores <SReg_64, SReg_32_XM0_XEXEC>; 397defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_64, SReg_64_XEXEC>; 398defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_64, SReg_128>; 399} // SubtargetPredicate = HasScalarFlatScratchInsts 400 401let SubtargetPredicate = HasScalarAtomics in { 402 403let is_buffer = 1 in { 404defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 405defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 406defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 407defm S_BUFFER_ATOMIC_SUB : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 408defm S_BUFFER_ATOMIC_SMIN : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 409defm S_BUFFER_ATOMIC_UMIN : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 410defm S_BUFFER_ATOMIC_SMAX : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 411defm S_BUFFER_ATOMIC_UMAX : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 412defm S_BUFFER_ATOMIC_AND : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 413defm S_BUFFER_ATOMIC_OR : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 414defm S_BUFFER_ATOMIC_XOR : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 415defm S_BUFFER_ATOMIC_INC : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 416defm S_BUFFER_ATOMIC_DEC : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>; 417 418defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 419defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_128>; 420defm S_BUFFER_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 421defm S_BUFFER_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 422defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 423defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 424defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 425defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 426defm S_BUFFER_ATOMIC_AND_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 427defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 428defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 429defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 430defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>; 431} 432 433defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 434defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 435defm S_ATOMIC_ADD : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 436defm S_ATOMIC_SUB : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 437defm S_ATOMIC_SMIN : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 438defm S_ATOMIC_UMIN : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 439defm S_ATOMIC_SMAX : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 440defm S_ATOMIC_UMAX : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 441defm S_ATOMIC_AND : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 442defm S_ATOMIC_OR : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 443defm S_ATOMIC_XOR : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 444defm S_ATOMIC_INC : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 445defm S_ATOMIC_DEC : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 446 447defm S_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 448defm S_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <SReg_64, SReg_128>; 449defm S_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 450defm S_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 451defm S_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 452defm S_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 453defm S_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 454defm S_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 455defm S_ATOMIC_AND_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 456defm S_ATOMIC_OR_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 457defm S_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 458defm S_ATOMIC_INC_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 459defm S_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 460 461} // let SubtargetPredicate = HasScalarAtomics 462 463let SubtargetPredicate = HasScalarAtomics in { 464defm S_DCACHE_DISCARD : SM_Pseudo_Discards; 465defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards; 466} 467 468let SubtargetPredicate = isGFX12Plus in { 469def S_PREFETCH_INST : SM_Prefetch_Pseudo <"s_prefetch_inst", SReg_64, 1>; 470def S_PREFETCH_INST_PC_REL : SM_Prefetch_Pseudo <"s_prefetch_inst_pc_rel", SReg_64, 0>; 471def S_PREFETCH_DATA : SM_Prefetch_Pseudo <"s_prefetch_data", SReg_64, 1>; 472def S_PREFETCH_DATA_PC_REL : SM_Prefetch_Pseudo <"s_prefetch_data_pc_rel", SReg_64, 0>; 473def S_BUFFER_PREFETCH_DATA : SM_Prefetch_Pseudo <"s_buffer_prefetch_data", SReg_128_XNULL, 1> { 474 let is_buffer = 1; 475} 476} // end let SubtargetPredicate = isGFX12Plus 477 478//===----------------------------------------------------------------------===// 479// Targets 480//===----------------------------------------------------------------------===// 481 482//===----------------------------------------------------------------------===// 483// SI 484//===----------------------------------------------------------------------===// 485 486class SMRD_Real_si <bits<5> op, SM_Pseudo ps> 487 : SM_Real<ps> 488 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 489 , Enc32 { 490 491 let AssemblerPredicate = isGFX6GFX7; 492 let DecoderNamespace = "GFX6GFX7"; 493 494 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?)); 495 let Inst{8} = ps.has_offset; 496 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 497 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 498 let Inst{26-22} = op; 499 let Inst{31-27} = 0x18; //encoding 500} 501 502multiclass SM_Real_Loads_si<bits<5> op> { 503 defvar ps = NAME; 504 defvar immPs = !cast<SM_Load_Pseudo>(ps#_IMM); 505 def _IMM_si : SMRD_Real_si <op, immPs> { 506 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, CPol:$cpol); 507 } 508 509 defvar sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR); 510 def _SGPR_si : SMRD_Real_si <op, sgprPs>; 511} 512 513defm S_LOAD_DWORD : SM_Real_Loads_si <0x00>; 514defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01>; 515defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02>; 516defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03>; 517defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04>; 518defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08>; 519defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09>; 520defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a>; 521defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b>; 522defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c>; 523 524def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>; 525def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>; 526 527 528//===----------------------------------------------------------------------===// 529// VI and GFX9. 530//===----------------------------------------------------------------------===// 531 532class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> 533 : SM_Real<ps> 534 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> 535 , Enc64 { 536 field bit IsGFX9SpecificEncoding = false; 537 let AssemblerPredicate = !if(IsGFX9SpecificEncoding, isGFX9Only, isGFX8GFX9); 538 let DecoderNamespace = "GFX8"; 539 540 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 541 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 542 543 // Note that for GFX9 instructions with immediate offsets, soffset_en 544 // must be defined, whereas in GFX8 it's undefined in all cases, 545 // meaning GFX9 is not perfectly backward-compatible with GFX8, despite 546 // documentation suggesting otherwise. 547 field bit SOffsetEn = !if(IsGFX9SpecificEncoding, 548 !if(ps.has_offset, ps.has_soffset, !if(ps.has_soffset, 0, ?)), 549 ?); 550 let Inst{14} = SOffsetEn; 551 552 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 553 554 // imm 555 // TODO: Shall not be defined if the instruction has no offset nor 556 // soffset. 557 let Inst{17} = ps.has_offset; 558 559 let Inst{25-18} = op; 560 let Inst{31-26} = 0x30; //encoding 561 562 // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed. 563 // Offset value is corrected accordingly when offset is encoded/decoded. 564 // TODO: Forbid non-M0 register offsets for GFX8 stores and atomics. 565 field bits<21> Offset; 566 let Offset{6-0} = !if(ps.has_offset, offset{6-0}, 567 !if(ps.has_soffset, soffset{6-0}, ?)); 568 let Offset{20-7} = !if(ps.has_offset, offset{20-7}, ?); 569 let Inst{52-32} = Offset; 570 571 // soffset 572 let Inst{63-57} = !if(!and(IsGFX9SpecificEncoding, ps.has_soffset), 573 soffset{6-0}, ?); 574} 575 576class SMEM_Real_Load_vi<bits<8> op, string ps> 577 : SMEM_Real_vi<op, !cast<SM_Pseudo>(ps)>; 578 579// The alternative GFX9 SGPR encoding using soffset to encode the 580// offset register. Not available in assembler and goes to the GFX9 581// encoding family to avoid conflicts with the primary SGPR variant. 582class SMEM_Real_SGPR_alt_gfx9 { 583 bit IsGFX9SpecificEncoding = true; 584 bit SOffsetEn = 1; 585 bit Offset = ?; 586 int Subtarget = SIEncodingFamily.GFX9; 587 string AsmVariantName = "NonParsable"; 588} 589 590multiclass SM_Real_Loads_vi<bits<8> op> { 591 defvar ps = NAME; 592 def _IMM_vi : SMEM_Real_Load_vi <op, ps#"_IMM">; 593 def _SGPR_vi : SMEM_Real_Load_vi <op, ps#"_SGPR">; 594 def _SGPR_alt_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR">, 595 SMEM_Real_SGPR_alt_gfx9; 596 let IsGFX9SpecificEncoding = true in 597 def _SGPR_IMM_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR_IMM">; 598} 599 600class SMEM_Real_Store_Base_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> { 601 // encoding 602 bits<7> sdata; 603 604 let sdst = ?; 605 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 606} 607 608class SMEM_Real_Store_vi <bits<8> op, string ps> 609 : SMEM_Real_Store_Base_vi <op, !cast<SM_Pseudo>(ps)>; 610 611multiclass SM_Real_Stores_vi<bits<8> op> { 612 defvar ps = NAME; 613 def _IMM_vi : SMEM_Real_Store_vi <op, ps#"_IMM">; 614 def _SGPR_vi : SMEM_Real_Store_vi <op, ps#"_SGPR">; 615 def _SGPR_alt_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR">, 616 SMEM_Real_SGPR_alt_gfx9; 617 let IsGFX9SpecificEncoding = true in 618 def _SGPR_IMM_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR_IMM">; 619} 620 621multiclass SM_Real_Probe_vi<bits<8> op> { 622 defvar ps = NAME; 623 def _IMM_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 624 def _SGPR_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 625 def _SGPR_alt_gfx9 626 : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>, 627 SMEM_Real_SGPR_alt_gfx9; 628 let IsGFX9SpecificEncoding = true in 629 def _SGPR_IMM_gfx9 630 : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>; 631} 632 633defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00>; 634defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01>; 635defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02>; 636defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03>; 637defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04>; 638defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08>; 639defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09>; 640defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a>; 641defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b>; 642defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c>; 643 644defm S_STORE_DWORD : SM_Real_Stores_vi <0x10>; 645defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11>; 646defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12>; 647 648defm S_BUFFER_STORE_DWORD : SM_Real_Stores_vi <0x18>; 649defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_vi <0x19>; 650defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_vi <0x1a>; 651 652// These instructions use same encoding 653def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>; 654def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>; 655def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>; 656def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>; 657def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>; 658def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>; 659 660defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_vi <0x05>; 661defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_vi <0x06>; 662defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_vi <0x07>; 663 664defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_vi <0x15>; 665defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16>; 666defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17>; 667 668defm S_ATC_PROBE : SM_Real_Probe_vi <0x26>; 669defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27>; 670 671//===----------------------------------------------------------------------===// 672// GFX9 673//===----------------------------------------------------------------------===// 674 675class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps> 676 : SMEM_Real_vi <op, ps> { 677 678 bits<7> sdata; 679 680 let Constraints = ps.Constraints; 681 let DisableEncoding = ps.DisableEncoding; 682 683 let cpol{CPolBit.GLC} = ps.glc; 684 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 685} 686 687multiclass SM_Real_Atomics_vi<bits<8> op> { 688 defvar ps = NAME; 689 def _IMM_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 690 def _SGPR_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 691 def _SGPR_alt_gfx9 692 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>, 693 SMEM_Real_SGPR_alt_gfx9; 694 let IsGFX9SpecificEncoding = true in 695 def _SGPR_IMM_gfx9 696 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>; 697 def _IMM_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 698 def _SGPR_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 699 def _SGPR_RTN_alt_gfx9 700 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>, 701 SMEM_Real_SGPR_alt_gfx9; 702 let IsGFX9SpecificEncoding = true in 703 def _SGPR_IMM_RTN_gfx9 704 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>; 705} 706 707defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_vi <0x40>; 708defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x41>; 709defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_vi <0x42>; 710defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_vi <0x43>; 711defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_vi <0x44>; 712defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_vi <0x45>; 713defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_vi <0x46>; 714defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_vi <0x47>; 715defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_vi <0x48>; 716defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_vi <0x49>; 717defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_vi <0x4a>; 718defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_vi <0x4b>; 719defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_vi <0x4c>; 720 721defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0x60>; 722defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0x61>; 723defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0x62>; 724defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0x63>; 725defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0x64>; 726defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0x65>; 727defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0x66>; 728defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0x67>; 729defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0x68>; 730defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0x69>; 731defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0x6a>; 732defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0x6b>; 733defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0x6c>; 734 735defm S_ATOMIC_SWAP : SM_Real_Atomics_vi <0x80>; 736defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x81>; 737defm S_ATOMIC_ADD : SM_Real_Atomics_vi <0x82>; 738defm S_ATOMIC_SUB : SM_Real_Atomics_vi <0x83>; 739defm S_ATOMIC_SMIN : SM_Real_Atomics_vi <0x84>; 740defm S_ATOMIC_UMIN : SM_Real_Atomics_vi <0x85>; 741defm S_ATOMIC_SMAX : SM_Real_Atomics_vi <0x86>; 742defm S_ATOMIC_UMAX : SM_Real_Atomics_vi <0x87>; 743defm S_ATOMIC_AND : SM_Real_Atomics_vi <0x88>; 744defm S_ATOMIC_OR : SM_Real_Atomics_vi <0x89>; 745defm S_ATOMIC_XOR : SM_Real_Atomics_vi <0x8a>; 746defm S_ATOMIC_INC : SM_Real_Atomics_vi <0x8b>; 747defm S_ATOMIC_DEC : SM_Real_Atomics_vi <0x8c>; 748 749defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0xa0>; 750defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0xa1>; 751defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0xa2>; 752defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0xa3>; 753defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0xa4>; 754defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0xa5>; 755defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0xa6>; 756defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0xa7>; 757defm S_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0xa8>; 758defm S_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0xa9>; 759defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0xaa>; 760defm S_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0xab>; 761defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0xac>; 762 763multiclass SM_Real_Discard_vi<bits<8> op> { 764 defvar ps = NAME; 765 def _IMM_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>; 766 def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>; 767 def _SGPR_alt_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>, 768 SMEM_Real_SGPR_alt_gfx9; 769 let IsGFX9SpecificEncoding = true in 770 def _SGPR_IMM_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR_IMM)>; 771} 772 773defm S_DCACHE_DISCARD : SM_Real_Discard_vi <0x28>; 774defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29>; 775 776//===----------------------------------------------------------------------===// 777// CI 778//===----------------------------------------------------------------------===// 779 780def smrd_literal_offset : ImmOperand<i32, "SMRDLiteralOffset">; 781 782class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : 783 SM_Real<ps>, 784 Enc64 { 785 786 let AssemblerPredicate = isGFX7Only; 787 let DecoderNamespace = "GFX7"; 788 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, CPol:$cpol); 789 790 let Inst{7-0} = 0xff; 791 let Inst{8} = 0; 792 let Inst{14-9} = sbase{6-1}; 793 let Inst{21-15} = sdst{6-0}; 794 let Inst{26-22} = op; 795 let Inst{31-27} = 0x18; //encoding 796 let Inst{63-32} = offset{31-0}; 797} 798 799def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>; 800def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>; 801def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>; 802def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>; 803def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>; 804def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>; 805def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>; 806def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>; 807def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>; 808def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>; 809 810class SMRD_Real_ci <bits<5> op, SM_Pseudo ps> 811 : SM_Real<ps> 812 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 813 , Enc32 { 814 815 let AssemblerPredicate = isGFX7Only; 816 let DecoderNamespace = "GFX7"; 817 818 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?)); 819 let Inst{8} = ps.has_offset; 820 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 821 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 822 let Inst{26-22} = op; 823 let Inst{31-27} = 0x18; //encoding 824} 825 826def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>; 827 828//===----------------------------------------------------------------------===// 829// Scalar Memory Patterns 830//===----------------------------------------------------------------------===// 831 832class SMRDLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr), [{ return isUniformLoad(N);}]> { 833 let GISelPredicateCode = [{ 834 if (!MI.hasOneMemOperand()) 835 return false; 836 if (!isInstrUniform(MI)) 837 return false; 838 839 // FIXME: We should probably be caching this. 840 SmallVector<GEPInfo, 4> AddrInfo; 841 getAddrModeInfo(MI, MRI, AddrInfo); 842 843 if (hasVgprParts(AddrInfo)) 844 return false; 845 return true; 846 }]; 847} 848 849def smrd_load : SMRDLoadPat<load>; 850def smrd_extloadi8 : SMRDLoadPat<extloadi8>; 851def smrd_zextloadi8 : SMRDLoadPat<zextloadi8>; 852def smrd_sextloadi8 : SMRDLoadPat<sextloadi8>; 853def smrd_extloadi16 : SMRDLoadPat<extloadi16>; 854def smrd_zextloadi16 : SMRDLoadPat<zextloadi16>; 855def smrd_sextloadi16 : SMRDLoadPat<sextloadi16>; 856 857def smrd_prefetch : PatFrag <(ops node:$ptr, node:$rw, node:$loc, node:$type), 858 (prefetch node:$ptr, node:$rw, node:$loc, node:$type), 859 [{ return !N->getOperand(1)->isDivergent();}]> { 860 let GISelPredicateCode = [{ 861 return isInstrUniform(MI); 862 }]; 863} 864 865def SMRDImm : ComplexPattern<iPTR, 2, "SelectSMRDImm">; 866def SMRDImm32 : ComplexPattern<iPTR, 2, "SelectSMRDImm32">; 867def SMRDSgpr : ComplexPattern<iPTR, 2, "SelectSMRDSgpr">; 868def SMRDSgprImm : ComplexPattern<iPTR, 3, "SelectSMRDSgprImm">; 869def SMRDBufferImm : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">; 870def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">; 871def SMRDBufferSgprImm : ComplexPattern<iPTR, 2, "SelectSMRDBufferSgprImm">; 872 873class SMRDAlignedLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr), [{ 874 // Returns true if it is a single dword load or naturally aligned multi-dword load. 875 LoadSDNode *Ld = cast<LoadSDNode>(N); 876 unsigned Size = Ld->getMemoryVT().getStoreSize(); 877 return Size <= 4 || Ld->getAlign().value() >= Size; 878}]> { 879 let GISelPredicateCode = [{ 880 auto &Ld = cast<GLoad>(MI); 881 TypeSize Size = Ld.getMMO().getSize().getValue(); 882 return Size <= 4 || Ld.getMMO().getAlign().value() >= Size; 883 }]; 884} 885 886def aligned_smrd_load : SMRDAlignedLoadPat<smrd_load>; 887 888multiclass SMRD_Patterns <string Instr, ValueType vt, PatFrag frag, 889 bit immci = true, string suffix = ""> { 890 // 1. IMM offset 891 def : GCNPat < 892 (frag (SMRDImm i64:$sbase, i32:$offset)), 893 (vt (!cast<SM_Pseudo>(Instr#"_IMM"#suffix) $sbase, $offset, 0))>; 894 895 // 2. 32-bit IMM offset on CI 896 if immci then def : GCNPat < 897 (frag (SMRDImm32 i64:$sbase, i32:$offset)), 898 (vt (!cast<InstSI>(Instr#"_IMM_ci"#suffix) $sbase, $offset, 0))> { 899 let SubtargetPredicate = isGFX7Only; 900 } 901 902 // 3. SGPR offset 903 def : GCNPat < 904 (frag (SMRDSgpr i64:$sbase, i32:$soffset)), 905 (vt (!cast<SM_Pseudo>(Instr#"_SGPR"#suffix) $sbase, $soffset, 0))> { 906 let SubtargetPredicate = isNotGFX9Plus; 907 } 908 def : GCNPat < 909 (frag (SMRDSgpr i64:$sbase, i32:$soffset)), 910 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM"#suffix) $sbase, $soffset, 0, 0))> { 911 let SubtargetPredicate = isGFX9Plus; 912 } 913 914 // 4. SGPR+IMM offset 915 def : GCNPat < 916 (frag (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)), 917 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM"#suffix) $sbase, $soffset, $offset, 0))> { 918 let SubtargetPredicate = isGFX9Plus; 919 } 920 921 // 5. No offset 922 def : GCNPat < 923 (vt (frag (i64 SReg_64:$sbase))), 924 (vt (!cast<SM_Pseudo>(Instr#"_IMM"#suffix) i64:$sbase, 0, 0))>; 925} 926 927multiclass SMRD_Pattern <string Instr, ValueType vt, bit immci = true> { 928 // High priority when XNACK is enabled and the load was naturally aligned. 929 let OtherPredicates = [HasXNACKEnabled], AddedComplexity = 102 in 930 defm: SMRD_Patterns <Instr, vt, aligned_smrd_load, immci>; 931 932 // XNACK is enabled and the load wasn't naturally aligned. The constrained sload variant. 933 if !gt(vt.Size, 32) then { 934 let OtherPredicates = [HasXNACKEnabled], AddedComplexity = 101 in 935 defm: SMRD_Patterns <Instr, vt, smrd_load, /*immci=*/false, /*suffix=*/"_ec">; 936 } 937 938 // XNACK is disabled. 939 let AddedComplexity = 100 in 940 defm: SMRD_Patterns <Instr, vt, smrd_load, immci>; 941} 942 943multiclass SMLoad_Pattern <string Instr, ValueType vt, bit immci = true> { 944 // 1. Offset as an immediate 945 def : GCNPat < 946 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy), 947 (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> { 948 let AddedComplexity = 2; 949 } 950 951 // 2. 32-bit IMM offset on CI 952 if immci then def : GCNPat < 953 (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)), 954 (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset, 955 (extract_cpol $cachepolicy))> { 956 let SubtargetPredicate = isGFX7Only; 957 let AddedComplexity = 1; 958 } 959 960 // 3. Offset loaded in an 32bit SGPR 961 def : GCNPat < 962 (SIsbuffer_load v4i32:$sbase, i32:$soffset, timm:$cachepolicy), 963 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$soffset, (extract_cpol $cachepolicy)))> { 964 let SubtargetPredicate = isNotGFX9Plus; 965 } 966 def : GCNPat < 967 (SIsbuffer_load v4i32:$sbase, i32:$soffset, timm:$cachepolicy), 968 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, 0, (extract_cpol $cachepolicy)))> { 969 let SubtargetPredicate = isGFX9Plus; 970 } 971 972 // 4. Offset as an 32-bit SGPR + immediate 973 def : GCNPat < 974 (SIsbuffer_load v4i32:$sbase, (SMRDBufferSgprImm i32:$soffset, i32:$offset), 975 timm:$cachepolicy), 976 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, i32imm:$offset, 977 (extract_cpol $cachepolicy)))> { 978 let SubtargetPredicate = isGFX9Plus; 979 } 980} 981 982multiclass ScalarLoadWithExtensionPat <string Instr, SDPatternOperator node, ValueType vt> { 983 // 1. IMM offset 984 def : GCNPat < 985 (node (SMRDImm i64:$sbase, i32:$offset)), 986 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))>{ 987 let SubtargetPredicate = isGFX12Plus; 988 } 989 990 // 2. SGPR offset 991 def : GCNPat < 992 (node (SMRDSgpr i64:$sbase, i32:$soffset)), 993 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, 0, 0))>{ 994 let SubtargetPredicate = isGFX12Plus; 995 } 996 997 // 3. SGPR+IMM offset 998 def : GCNPat < 999 (node (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)), 1000 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, $offset, 0))>{ 1001 let SubtargetPredicate = isGFX12Plus; 1002 } 1003 1004 // 4. No offset 1005 def : GCNPat < 1006 (vt (node (i64 SReg_64:$sbase))), 1007 (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0))>{ 1008 let SubtargetPredicate = isGFX12Plus; 1009 } 1010} 1011 1012multiclass ScalarBufferLoadIntrinsicPat <SDPatternOperator name, string Instr> { 1013 1014 // 1. Offset as an immediate 1015 def : GCNPat < 1016 (name v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy), 1017 (i32 (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> { 1018 let SubtargetPredicate = isGFX12Plus; 1019 } 1020 1021 // 2. Offset as an 32-bit SGPR 1022 def : GCNPat < 1023 (name v4i32:$sbase, i32:$soffset, timm:$cachepolicy), 1024 (i32 (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, 0, (extract_cpol $cachepolicy)))> { 1025 let SubtargetPredicate = isGFX12Plus; 1026 } 1027 1028 // 3. Offset as an 32-bit SGPR + immediate 1029 def : GCNPat < 1030 (name v4i32:$sbase, (SMRDBufferSgprImm i32:$soffset, i32:$offset), 1031 timm:$cachepolicy), 1032 (i32 (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, i32imm:$offset, 1033 (extract_cpol $cachepolicy)))> { 1034 let SubtargetPredicate = isGFX12Plus; 1035 } 1036} 1037 1038// Global and constant loads can be selected to either MUBUF or SMRD 1039// instructions, but SMRD instructions are faster so we want the instruction 1040// selector to prefer those. 1041let AddedComplexity = 100 in { 1042 1043defm : ScalarLoadWithExtensionPat <"S_LOAD_U8", smrd_extloadi8, i32>; 1044defm : ScalarLoadWithExtensionPat <"S_LOAD_U8", smrd_zextloadi8, i32>; 1045defm : ScalarLoadWithExtensionPat <"S_LOAD_I8", smrd_sextloadi8, i32>; 1046defm : ScalarLoadWithExtensionPat <"S_LOAD_U16", smrd_extloadi16, i32>; 1047defm : ScalarLoadWithExtensionPat <"S_LOAD_U16", smrd_zextloadi16, i32>; 1048defm : ScalarLoadWithExtensionPat <"S_LOAD_I16", smrd_sextloadi16, i32>; 1049defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_byte, "S_BUFFER_LOAD_I8">; 1050defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_ubyte, "S_BUFFER_LOAD_U8">; 1051defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_short, "S_BUFFER_LOAD_I16">; 1052defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_ushort, "S_BUFFER_LOAD_U16">; 1053 1054} // End let AddedComplexity = 100 1055 1056foreach vt = Reg32Types.types in { 1057defm : SMRD_Pattern <"S_LOAD_DWORD", vt>; 1058} 1059 1060foreach vt = SReg_64.RegTypes in { 1061defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>; 1062} 1063 1064foreach vt = SReg_96.RegTypes in { 1065defm : SMRD_Pattern <"S_LOAD_DWORDX3", vt, false>; 1066} 1067 1068foreach vt = SReg_128.RegTypes in { 1069defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>; 1070} 1071 1072foreach vt = SReg_256.RegTypes in { 1073defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>; 1074} 1075 1076foreach vt = SReg_512.RegTypes in { 1077defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>; 1078} 1079 1080 1081defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>; 1082defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>; 1083defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX3", v3i32, false>; 1084defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>; 1085defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>; 1086defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16i32>; 1087 1088defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", f32>; 1089defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2f32>; 1090defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX3", v3f32, false>; 1091defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4f32>; 1092defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8f32>; 1093defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16f32>; 1094 1095let OtherPredicates = [HasSMemTimeInst] in { 1096def : GCNPat < 1097 (i64 (readcyclecounter)), 1098 (S_MEMTIME) 1099>; 1100} // let OtherPredicates = [HasSMemTimeInst] 1101 1102let OtherPredicates = [HasShaderCyclesRegister] in { 1103def : GCNPat < 1104 (i64 (readcyclecounter)), 1105 (REG_SEQUENCE SReg_64, 1106 (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0, 1107 (S_MOV_B32 (i32 0)), sub1)> { 1108} 1109} // let OtherPredicates = [HasShaderCyclesRegister] 1110 1111let OtherPredicates = [HasSMemRealTime] in { 1112def : GCNPat < 1113 (i64 (readsteadycounter)), 1114 (S_MEMREALTIME) 1115>; 1116} // let OtherPredicates = [HasSMemRealTime] 1117 1118let SubtargetPredicate = isGFX11Plus in { 1119def : GCNPat < 1120 (i64 (readsteadycounter)), 1121 (S_SENDMSG_RTN_B64 (i32 /*MSG_RTN_GET_REALTIME=*/0x83)) 1122>; 1123} // let SubtargetPredicate = [isGFX11Plus] 1124 1125def i32imm_zero : TImmLeaf <i32, [{ 1126 return Imm == 0; 1127}]>; 1128 1129def i32imm_one : TImmLeaf <i32, [{ 1130 return Imm == 1; 1131}]>; 1132 1133multiclass SMPrefetchPat<string type, TImmLeaf cache_type> { 1134 def : GCNPat < 1135 (smrd_prefetch (SMRDImm i64:$sbase, i32:$offset), timm, timm, cache_type), 1136 (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) $sbase, $offset, (i32 SGPR_NULL), (i8 0)) 1137 >; 1138 1139 def : GCNPat < 1140 (smrd_prefetch (i64 SReg_64:$sbase), timm, timm, cache_type), 1141 (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) $sbase, 0, (i32 SGPR_NULL), (i8 0)) 1142 >; 1143 1144 def : GCNPat < 1145 (smrd_prefetch (i32 SReg_32:$sbase), timm, timm, cache_type), 1146 (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) 1147 (i64 (REG_SEQUENCE SReg_64, $sbase, sub0, (i32 (S_MOV_B32 (i32 0))), sub1)), 1148 0, (i32 SGPR_NULL), (i8 0)) 1149 >; 1150} 1151 1152defm : SMPrefetchPat<"INST", i32imm_zero>; 1153defm : SMPrefetchPat<"DATA", i32imm_one>; 1154 1155let SubtargetPredicate = isGFX12Plus in { 1156 def : GCNPat < 1157 (int_amdgcn_s_prefetch_data (SMRDImm i64:$sbase, i32:$offset), (i32 SReg_32:$len)), 1158 (S_PREFETCH_DATA $sbase, $offset, $len, 0) 1159 >; 1160 1161 def : GCNPat < 1162 (int_amdgcn_s_prefetch_data (i64 SReg_64:$sbase), (i32 SReg_32:$len)), 1163 (S_PREFETCH_DATA $sbase, 0, $len, 0) 1164 >; 1165 1166 def : GCNPat < 1167 (int_amdgcn_s_prefetch_data (SMRDImm i64:$sbase, i32:$offset), imm:$len), 1168 (S_PREFETCH_DATA $sbase, $offset, (i32 SGPR_NULL), (as_i8timm $len)) 1169 >; 1170 1171 def : GCNPat < 1172 (int_amdgcn_s_prefetch_data (i64 SReg_64:$sbase), imm:$len), 1173 (S_PREFETCH_DATA $sbase, 0, (i32 SGPR_NULL), (as_i8timm $len)) 1174 1175 >; 1176 1177 def : GCNPat < 1178 (SIsbuffer_prefetch v4i32:$sbase, (SMRDBufferImm i32:$offset), (i32 SReg_32:$len)), 1179 (S_BUFFER_PREFETCH_DATA SReg_128:$sbase, i32imm:$offset, $len, 0) 1180 >; 1181 1182 def : GCNPat < 1183 (SIsbuffer_prefetch v4i32:$sbase, (SMRDBufferImm i32:$offset), imm:$len), 1184 (S_BUFFER_PREFETCH_DATA SReg_128:$sbase, i32imm:$offset, (i32 SGPR_NULL), (as_i8timm $len)) 1185 >; 1186} // End let SubtargetPredicate = isGFX12Plus 1187 1188//===----------------------------------------------------------------------===// 1189// GFX10. 1190//===----------------------------------------------------------------------===// 1191 1192class SMEM_Real_10Plus_common<bits<8> op, SM_Pseudo ps, string opName, 1193 int subtarget, RegisterWithSubRegs sgpr_null> : 1194 SM_Real<ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>, Enc64 { 1195 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 1196 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 1197 let Inst{25-18} = op; 1198 let Inst{31-26} = 0x3d; 1199 // There are SMEM instructions that do not employ any of the offset 1200 // fields, in which case we need them to remain undefined. 1201 let Inst{52-32} = !if(ps.has_offset, offset{20-0}, !if(ps.has_soffset, 0, ?)); 1202 let Inst{63-57} = !if(ps.has_soffset, soffset{6-0}, 1203 !if(ps.has_offset, sgpr_null.HWEncoding{6-0}, ?)); 1204} 1205 1206class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> 1207 : SMEM_Real_10Plus_common<op, ps, ps.Mnemonic, SIEncodingFamily.GFX10, 1208 SGPR_NULL_gfxpre11> { 1209 let AssemblerPredicate = isGFX10Only; 1210 let DecoderNamespace = "GFX10"; 1211 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ?); 1212 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 1213} 1214 1215class SMEM_Real_Load_gfx10<bits<8> op, string ps> 1216 : SMEM_Real_gfx10<op, !cast<SM_Pseudo>(ps)>; 1217 1218multiclass SM_Real_Loads_gfx10<bits<8> op> { 1219 defvar ps = NAME; 1220 def _IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_IMM">; 1221 def _SGPR_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_SGPR">; 1222 def _SGPR_IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_SGPR_IMM">; 1223} 1224 1225class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> { 1226 bits<7> sdata; 1227 1228 let sdst = ?; 1229 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 1230} 1231 1232multiclass SM_Real_Stores_gfx10<bits<8> op> { 1233 defvar ps = NAME; 1234 defvar immPs = !cast<SM_Store_Pseudo>(ps#_IMM); 1235 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs>; 1236 1237 defvar sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR); 1238 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs>; 1239 1240 defvar sgprImmPs = !cast<SM_Store_Pseudo>(ps#_SGPR_IMM); 1241 def _SGPR_IMM_gfx10 : SMEM_Real_Store_gfx10 <op, sgprImmPs>; 1242} 1243 1244defm S_LOAD_DWORD : SM_Real_Loads_gfx10<0x000>; 1245defm S_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x001>; 1246defm S_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x002>; 1247defm S_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x003>; 1248defm S_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x004>; 1249 1250defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_gfx10<0x005>; 1251defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x006>; 1252defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x007>; 1253 1254defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_gfx10<0x008>; 1255defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x009>; 1256defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x00a>; 1257defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x00b>; 1258defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x00c>; 1259 1260defm S_STORE_DWORD : SM_Real_Stores_gfx10<0x010>; 1261defm S_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x011>; 1262defm S_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x012>; 1263defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_gfx10<0x015>; 1264defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016>; 1265defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017>; 1266defm S_BUFFER_STORE_DWORD : SM_Real_Stores_gfx10<0x018>; 1267defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x019>; 1268defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x01a>; 1269 1270def S_MEMREALTIME_gfx10 : SMEM_Real_gfx10<0x025, S_MEMREALTIME>; 1271def S_MEMTIME_gfx10 : SMEM_Real_gfx10<0x024, S_MEMTIME>; 1272def S_GL1_INV_gfx10 : SMEM_Real_gfx10<0x01f, S_GL1_INV>; 1273def S_GET_WAVEID_IN_WORKGROUP_gfx10 : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>; 1274def S_DCACHE_INV_gfx10 : SMEM_Real_gfx10<0x020, S_DCACHE_INV>; 1275 1276def S_DCACHE_WB_gfx10 : SMEM_Real_gfx10<0x021, S_DCACHE_WB>; 1277 1278multiclass SM_Real_Probe_gfx10<bits<8> op> { 1279 defvar ps = NAME; 1280 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1281 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1282 def _SGPR_IMM_gfx10 1283 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>; 1284} 1285 1286defm S_ATC_PROBE : SM_Real_Probe_gfx10 <0x26>; 1287defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27>; 1288 1289class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps> 1290 : SMEM_Real_gfx10 <op, ps> { 1291 1292 bits<7> sdata; 1293 1294 let Constraints = ps.Constraints; 1295 let DisableEncoding = ps.DisableEncoding; 1296 1297 let cpol{CPolBit.GLC} = ps.glc; 1298 1299 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 1300 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 1301} 1302 1303multiclass SM_Real_Atomics_gfx10<bits<8> op> { 1304 defvar ps = NAME; 1305 def _IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 1306 def _SGPR_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 1307 def _SGPR_IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>; 1308 def _IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 1309 def _SGPR_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 1310 def _SGPR_IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>; 1311} 1312 1313defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x40>; 1314defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x41>; 1315defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x42>; 1316defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x43>; 1317defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x44>; 1318defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x45>; 1319defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x46>; 1320defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x47>; 1321defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x48>; 1322defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x49>; 1323defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x4a>; 1324defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x4b>; 1325defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x4c>; 1326 1327defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0x60>; 1328defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0x61>; 1329defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0x62>; 1330defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0x63>; 1331defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0x64>; 1332defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0x65>; 1333defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0x66>; 1334defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0x67>; 1335defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0x68>; 1336defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0x69>; 1337defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0x6a>; 1338defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0x6b>; 1339defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0x6c>; 1340 1341defm S_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x80>; 1342defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x81>; 1343defm S_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x82>; 1344defm S_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x83>; 1345defm S_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x84>; 1346defm S_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x85>; 1347defm S_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x86>; 1348defm S_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x87>; 1349defm S_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x88>; 1350defm S_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x89>; 1351defm S_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x8a>; 1352defm S_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x8b>; 1353defm S_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x8c>; 1354 1355defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0xa0>; 1356defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0xa1>; 1357defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0xa2>; 1358defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0xa3>; 1359defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0xa4>; 1360defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0xa5>; 1361defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0xa6>; 1362defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0xa7>; 1363defm S_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0xa8>; 1364defm S_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0xa9>; 1365defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0xaa>; 1366defm S_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0xab>; 1367defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0xac>; 1368 1369multiclass SM_Real_Discard_gfx10<bits<8> op> { 1370 defvar ps = NAME; 1371 def _IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1372 def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1373 def _SGPR_IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>; 1374} 1375 1376defm S_DCACHE_DISCARD : SM_Real_Discard_gfx10 <0x28>; 1377defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29>; 1378 1379def SMInfoTable : GenericTable { 1380 let FilterClass = "SM_Real"; 1381 let CppTypeName = "SMInfo"; 1382 let Fields = ["Opcode", "is_buffer"]; 1383 1384 let PrimaryKey = ["Opcode"]; 1385 let PrimaryKeyName = "getSMEMOpcodeHelper"; 1386} 1387 1388//===----------------------------------------------------------------------===// 1389// GFX11. 1390//===----------------------------------------------------------------------===// 1391 1392class SMEM_Real_gfx11<bits<8> op, SM_Pseudo ps, string opName = ps.Mnemonic> : 1393 SMEM_Real_10Plus_common<op, ps, opName, SIEncodingFamily.GFX11, 1394 SGPR_NULL_gfx11plus> { 1395 let AssemblerPredicate = isGFX11Only; 1396 let DecoderNamespace = "GFX11"; 1397 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 1398 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, 0); 1399} 1400 1401class SMEM_Real_Load_gfx11<bits<8> op, string ps, string opName> : 1402 SMEM_Real_gfx11<op, !cast<SM_Pseudo>(ps), opName>; 1403 1404multiclass SM_Real_Loads_gfx11<bits<8> op, string ps> { 1405 defvar opName = !tolower(NAME); 1406 def _IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_IMM", opName>; 1407 def _SGPR_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR", opName>; 1408 def _SGPR_IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR_IMM", opName>; 1409 def : AMDGPUMnemonicAlias<!cast<SM_Pseudo>(ps#"_IMM").Mnemonic, opName> { 1410 let AssemblerPredicate = isGFX11Plus; 1411 } 1412} 1413 1414defm S_LOAD_B32 : SM_Real_Loads_gfx11<0x000, "S_LOAD_DWORD">; 1415defm S_LOAD_B64 : SM_Real_Loads_gfx11<0x001, "S_LOAD_DWORDX2">; 1416defm S_LOAD_B128 : SM_Real_Loads_gfx11<0x002, "S_LOAD_DWORDX4">; 1417defm S_LOAD_B256 : SM_Real_Loads_gfx11<0x003, "S_LOAD_DWORDX8">; 1418defm S_LOAD_B512 : SM_Real_Loads_gfx11<0x004, "S_LOAD_DWORDX16">; 1419 1420defm S_BUFFER_LOAD_B32 : SM_Real_Loads_gfx11<0x008, "S_BUFFER_LOAD_DWORD">; 1421defm S_BUFFER_LOAD_B64 : SM_Real_Loads_gfx11<0x009, "S_BUFFER_LOAD_DWORDX2">; 1422defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx11<0x00a, "S_BUFFER_LOAD_DWORDX4">; 1423defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx11<0x00b, "S_BUFFER_LOAD_DWORDX8">; 1424defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx11<0x00c, "S_BUFFER_LOAD_DWORDX16">; 1425 1426def S_GL1_INV_gfx11 : SMEM_Real_gfx11<0x020, S_GL1_INV>; 1427def S_DCACHE_INV_gfx11 : SMEM_Real_gfx11<0x021, S_DCACHE_INV>; 1428 1429class SMEM_Real_Store_gfx11 <bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx11<op, ps> { 1430 // encoding 1431 bits<7> sdata; 1432 1433 let sdst = ?; 1434 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 1435} 1436 1437multiclass SM_Real_Probe_gfx11<bits<8> op> { 1438 defvar ps = NAME; 1439 def _IMM_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 1440 def _SGPR_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 1441 def _SGPR_IMM_gfx11 1442 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>; 1443} 1444 1445defm S_ATC_PROBE : SM_Real_Probe_gfx11 <0x22>; 1446defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx11 <0x23>; 1447 1448//===----------------------------------------------------------------------===// 1449// GFX12. 1450//===----------------------------------------------------------------------===// 1451 1452class SMEM_Real_gfx12Plus<bits<6> op, SM_Pseudo ps, string opName, 1453 int subtarget, RegisterWithSubRegs sgpr_null> : 1454 SM_Real<ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>, Enc64 { 1455 1456 let Inst{18-13} = op; 1457 let Inst{31-26} = 0x3d; 1458 1459 let Inst{55-32} = !if(ps.has_offset, offset{23-0}, !if(ps.has_soffset, 0, ?)); 1460 let Inst{63-57} = !if(ps.has_soffset, soffset{6-0}, 1461 !if(ps.has_offset, sgpr_null.HWEncoding{6-0}, ?)); 1462} 1463 1464class SMEM_Real_gfx12<bits<6> op, SM_Pseudo ps, string opName = ps.Mnemonic> : 1465 SMEM_Real_gfx12Plus<op, ps, opName, SIEncodingFamily.GFX12, 1466 SGPR_NULL_gfx11plus> { 1467 let AssemblerPredicate = isGFX12Plus; 1468 let DecoderNamespace = "GFX12"; 1469 1470 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 1471 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 1472} 1473 1474class SMEM_Real_Prefetch_gfx12<bits<6> op, SM_Pseudo ps> : 1475 SMEM_Real_gfx12<op, ps> { 1476 bits<7> sdata; // Only 5 bits of sdata are supported. 1477 1478 let sdst = ?; 1479 let Inst{12-11} = 0; // Unused sdata bits. 1480 let Inst{10-6} = !if(ps.has_sdst, sdata{4-0}, ?); 1481} 1482 1483class SMEM_Real_Load_gfx12<bits<6> op, string ps, string opName, OffsetMode offsets> : 1484 SMEM_Real_gfx12<op, !cast<SM_Pseudo>(ps # offsets.Variant), opName> { 1485 RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps # offsets.Variant).BaseClass; 1486 let InOperandList = !con((ins BaseClass:$sbase), offsets.Ins, (ins CPol:$cpol)); 1487 1488 let Inst{22-21} = cpol{4-3}; // scope 1489 let Inst{24-23} = cpol{1-0}; // th - only lower 2 bits are supported 1490} 1491 1492multiclass SM_Real_Loads_gfx12<bits<6> op, string ps = NAME> { 1493 defvar opName = !tolower(NAME); 1494 def _IMM_gfx12 : SMEM_Real_Load_gfx12<op, ps, opName, IMM_Offset>; 1495 def _SGPR_IMM_gfx12 : SMEM_Real_Load_gfx12<op, ps, opName, SGPR_IMM_OptOffset>; 1496} 1497 1498defm S_LOAD_B32 : SM_Real_Loads_gfx12<0x00, "S_LOAD_DWORD">; 1499defm S_LOAD_B64 : SM_Real_Loads_gfx12<0x01, "S_LOAD_DWORDX2">; 1500defm S_LOAD_B96 : SM_Real_Loads_gfx12<0x05, "S_LOAD_DWORDX3">; 1501defm S_LOAD_B128 : SM_Real_Loads_gfx12<0x02, "S_LOAD_DWORDX4">; 1502defm S_LOAD_B256 : SM_Real_Loads_gfx12<0x03, "S_LOAD_DWORDX8">; 1503defm S_LOAD_B512 : SM_Real_Loads_gfx12<0x04, "S_LOAD_DWORDX16">; 1504 1505defm S_LOAD_I8 : SM_Real_Loads_gfx12<0x08>; 1506defm S_LOAD_U8 : SM_Real_Loads_gfx12<0x09>; 1507defm S_LOAD_I16 : SM_Real_Loads_gfx12<0x0a>; 1508defm S_LOAD_U16 : SM_Real_Loads_gfx12<0x0b>; 1509 1510defm S_BUFFER_LOAD_B32 : SM_Real_Loads_gfx12<0x10, "S_BUFFER_LOAD_DWORD">; 1511defm S_BUFFER_LOAD_B64 : SM_Real_Loads_gfx12<0x11, "S_BUFFER_LOAD_DWORDX2">; 1512defm S_BUFFER_LOAD_B96 : SM_Real_Loads_gfx12<0x15, "S_BUFFER_LOAD_DWORDX3">; 1513defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx12<0x12, "S_BUFFER_LOAD_DWORDX4">; 1514defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx12<0x13, "S_BUFFER_LOAD_DWORDX8">; 1515defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx12<0x14, "S_BUFFER_LOAD_DWORDX16">; 1516 1517defm S_BUFFER_LOAD_I8 : SM_Real_Loads_gfx12<0x18>; 1518defm S_BUFFER_LOAD_U8 : SM_Real_Loads_gfx12<0x19>; 1519defm S_BUFFER_LOAD_I16 : SM_Real_Loads_gfx12<0x1a>; 1520defm S_BUFFER_LOAD_U16 : SM_Real_Loads_gfx12<0x1b>; 1521 1522def S_DCACHE_INV_gfx12 : SMEM_Real_gfx12<0x021, S_DCACHE_INV>; 1523 1524def S_PREFETCH_INST_gfx12 : SMEM_Real_Prefetch_gfx12<0x24, S_PREFETCH_INST>; 1525def S_PREFETCH_INST_PC_REL_gfx12 : SMEM_Real_Prefetch_gfx12<0x25, S_PREFETCH_INST_PC_REL>; 1526def S_PREFETCH_DATA_gfx12 : SMEM_Real_Prefetch_gfx12<0x26, S_PREFETCH_DATA>; 1527def S_BUFFER_PREFETCH_DATA_gfx12 : SMEM_Real_Prefetch_gfx12<0x27, S_BUFFER_PREFETCH_DATA>; 1528def S_PREFETCH_DATA_PC_REL_gfx12 : SMEM_Real_Prefetch_gfx12<0x28, S_PREFETCH_DATA_PC_REL>; 1529 1530multiclass SMEM_Real_Probe_gfx12<bits<6> op> { 1531 defvar ps = NAME; 1532 def _IMM_gfx12 : SMEM_Real_Prefetch_gfx12<op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 1533 def _SGPR_IMM_gfx12 : SMEM_Real_Prefetch_gfx12<op, !cast<SM_Probe_Pseudo>(ps#_SGPR_OPT_IMM)>; 1534} 1535 1536defm S_ATC_PROBE : SMEM_Real_Probe_gfx12<0x22>; 1537defm S_ATC_PROBE_BUFFER : SMEM_Real_Probe_gfx12<0x23>; 1538