1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12let WantsParent = true in { 13 def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen">; 14 def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [], 20>; 15} 16 17def BUFSOffset : ComplexPattern<iPTR, 1, "SelectBUFSOffset">; 18 19def BUFAddrKind { 20 int Offset = 0; 21 int OffEn = 1; 22 int IdxEn = 2; 23 int BothEn = 3; 24 int Addr64 = 4; 25} 26 27class getAddrName<int addrKind> { 28 string ret = 29 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 30 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 31 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 32 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 33 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 34 ""))))); 35} 36 37class MUBUFAddr64Table <bit is_addr64, string Name> { 38 bit IsAddr64 = is_addr64; 39 string OpName = Name; 40} 41 42class MTBUFAddr64Table <bit is_addr64, string Name> { 43 bit IsAddr64 = is_addr64; 44 string OpName = Name; 45} 46 47 48//===----------------------------------------------------------------------===// 49// BUF class (base class for MTBUF and MUBUF pseudos) 50//===----------------------------------------------------------------------===// 51 52class BUF_Pseudo <string opName, dag outs, dag ins, 53 string asmOps, list<dag> pattern=[]> : 54 InstSI<outs, ins, "", pattern>, 55 SIMCInstr<opName, SIEncodingFamily.NONE> { 56 57 let isPseudo = 1; 58 let isCodeGenOnly = 1; 59 let Size = 8; 60 let UseNamedOperandTable = 1; 61 62 string Mnemonic = opName; 63 string AsmOperands = asmOps; 64 65 Instruction Opcode = !cast<Instruction>(NAME); 66 67 68 let VM_CNT = 1; 69 let EXP_CNT = 1; 70 71 let Uses = [EXEC]; 72 let hasSideEffects = 0; 73 let SchedRW = [WriteVMEM]; 74 75 76 77 bits<1> offen = 0; 78 bits<1> idxen = 0; 79 bits<1> addr64 = 0; 80 bits<1> lds = 0; 81 bits<1> has_vdata = !not(lds); 82 bits<1> has_vaddr = 1; 83 bits<1> has_glc = 1; 84 bits<1> has_dlc = 1; 85 bits<1> glc_value = 0; // the value for glc if no such operand 86 bits<1> dlc_value = 0; // the value for dlc if no such operand 87 bits<1> has_srsrc = 1; 88 bits<1> has_soffset = 1; 89 bits<1> has_offset = 1; 90 bits<1> has_slc = 1; 91 bits<1> tfe = 0; 92 bits<4> elements = 0; 93 bits<1> has_sccb = 1; 94 bits<1> sccb_value = 0; 95 bits<1> IsBufferInv = 0; 96} 97 98 99 100//===----------------------------------------------------------------------===// 101// MTBUF classes 102//===----------------------------------------------------------------------===// 103 104class MTBUFGetBaseOpcode<string Op> { 105 string ret = !subst("FORMAT_XY", "FORMAT_X", 106 !subst("FORMAT_XYZ", "FORMAT_X", 107 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 108} 109 110 111class MTBUF_Pseudo <string opName, dag outs, dag ins, 112 string asmOps, list<dag> pattern=[]> : 113 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 114 115 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 116 let MTBUF = 1; 117} 118 119class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> : 120 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 121 122 let isPseudo = 0; 123 let isCodeGenOnly = 0; 124 125 let VM_CNT = 1; 126 let EXP_CNT = 1; 127 let MTBUF = 1; 128 129 // copy relevant pseudo op flags 130 let UseNamedOperandTable = ps.UseNamedOperandTable; 131 let SubtargetPredicate = ps.SubtargetPredicate; 132 let OtherPredicates = ps.OtherPredicates; 133 let AsmMatchConverter = ps.AsmMatchConverter; 134 let Constraints = ps.Constraints; 135 let DisableEncoding = ps.DisableEncoding; 136 let TSFlags = ps.TSFlags; 137 let SchedRW = ps.SchedRW; 138 let mayLoad = ps.mayLoad; 139 let mayStore = ps.mayStore; 140 let IsAtomicRet = ps.IsAtomicRet; 141 let IsAtomicNoRet = ps.IsAtomicNoRet; 142 let Uses = ps.Uses; 143 let Defs = ps.Defs; 144 let isConvergent = ps.isConvergent; 145 146 bits<12> offset; 147 bits<5> cpol; 148 bits<7> format; 149 bits<8> vaddr; 150 bits<10> vdata; 151 bits<7> srsrc; 152 bits<8> soffset; 153 154 bits<4> dfmt = format{3-0}; 155 bits<3> nfmt = format{6-4}; 156 157 // GFX90A+ only: instruction uses AccVGPR for data 158 // Bit supersedes tfe. 159 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 160} 161 162class getMTBUFInsDA<list<RegisterClass> vdataList, 163 list<RegisterClass> vaddrList=[], bit hasRestrictedSOffset> { 164 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 165 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 166 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 167 168 dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), 169 (ins SCSrc_b32:$soffset)); 170 171 dag NonVaddrInputs = !con((ins SReg_128_XNULL:$srsrc), SOffset, 172 (ins Offset:$offset, FORMAT:$format, CPol_0:$cpol, i1imm_0:$swz)); 173 174 dag Inputs = !if(!empty(vaddrList), 175 NonVaddrInputs, 176 !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 177 dag ret = !if(!empty(vdataList), 178 Inputs, 179 !con((ins vdata_op:$vdata), Inputs)); 180} 181 182class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit hasRestrictedSOffset> { 183 dag ret = 184 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList, [], hasRestrictedSOffset>.ret, 185 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32], hasRestrictedSOffset>.ret, 186 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32], hasRestrictedSOffset>.ret, 187 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64], hasRestrictedSOffset>.ret, 188 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64], hasRestrictedSOffset>.ret, 189 (ins)))))); 190} 191 192class getMTBUFAsmOps<int addrKind> { 193 string Pfx = 194 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 195 !if(!eq(addrKind, BUFAddrKind.OffEn), 196 "$vaddr, $srsrc,$format $soffset offen", 197 !if(!eq(addrKind, BUFAddrKind.IdxEn), 198 "$vaddr, $srsrc,$format $soffset idxen", 199 !if(!eq(addrKind, BUFAddrKind.BothEn), 200 "$vaddr, $srsrc,$format $soffset idxen offen", 201 !if(!eq(addrKind, BUFAddrKind.Addr64), 202 "$vaddr, $srsrc,$format $soffset addr64", 203 ""))))); 204 string ret = " $vdata, " # Pfx # "$offset$cpol"; 205} 206 207class MTBUF_SetupAddr<int addrKind> { 208 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 209 !eq(addrKind, BUFAddrKind.BothEn)); 210 211 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 212 !eq(addrKind, BUFAddrKind.BothEn)); 213 214 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 215 216 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 217} 218 219class MTBUF_Load_Pseudo <string opName, 220 int addrKind, 221 RegisterClass vdataClass, 222 int elems, 223 bit hasRestrictedSOffset = 0, 224 list<dag> pattern=[], 225 // Workaround bug bz30254 226 int addrKindCopy = addrKind> 227 : MTBUF_Pseudo<opName, 228 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 229 getMTBUFIns<addrKindCopy, [], hasRestrictedSOffset>.ret, 230 getMTBUFAsmOps<addrKindCopy>.ret, 231 pattern>, 232 MTBUF_SetupAddr<addrKindCopy> { 233 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 234 let mayLoad = 1; 235 let mayStore = 0; 236 let elements = elems; 237} 238 239multiclass MTBUF_Pseudo_Loads_Helper<string opName, RegisterClass vdataClass, 240 int elems, bit hasRestrictedSOffset> { 241 242 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>, 243 MTBUFAddr64Table<0, NAME>; 244 245 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset>, 246 MTBUFAddr64Table<1, NAME>; 247 248 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>; 249 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>; 250 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>; 251 252 let DisableWQM = 1 in { 253 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>; 254 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>; 255 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>; 256 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>; 257 } 258} 259 260multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 261 int elems> { 262 defm NAME : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 0>; 263 defm _VBUFFER : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 1>; 264} 265 266class MTBUF_Store_Pseudo <string opName, 267 int addrKind, 268 RegisterClass vdataClass, 269 int elems, 270 bit hasRestrictedSOffset = 0, 271 list<dag> pattern=[], 272 // Workaround bug bz30254 273 int addrKindCopy = addrKind, 274 RegisterClass vdataClassCopy = vdataClass> 275 : MTBUF_Pseudo<opName, 276 (outs), 277 getMTBUFIns<addrKindCopy, [vdataClassCopy], hasRestrictedSOffset>.ret, 278 getMTBUFAsmOps<addrKindCopy>.ret, 279 pattern>, 280 MTBUF_SetupAddr<addrKindCopy> { 281 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 282 let mayLoad = 0; 283 let mayStore = 1; 284 let elements = elems; 285} 286 287multiclass MTBUF_Pseudo_Stores_Helper<string opName, RegisterClass vdataClass, 288 int elems, bit hasRestrictedSOffset> { 289 290 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>, 291 MTBUFAddr64Table<0, NAME>; 292 293 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset>, 294 MTBUFAddr64Table<1, NAME>; 295 296 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>; 297 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>; 298 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>; 299 300 let DisableWQM = 1 in { 301 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>; 302 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>; 303 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>; 304 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>; 305 } 306} 307 308multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 309 int elems> { 310 defm NAME : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 0>; 311 defm _VBUFFER : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 1>; 312} 313 314//===----------------------------------------------------------------------===// 315// MUBUF classes 316//===----------------------------------------------------------------------===// 317 318class MUBUFGetBaseOpcode<string Op> { 319 string ret = !subst("DWORDX2", "DWORD", 320 !subst("DWORDX3", "DWORD", 321 !subst("DWORDX4", "DWORD", Op))); 322} 323 324class MUBUF_Pseudo <string opName, dag outs, dag ins, 325 string asmOps, list<dag> pattern=[]> : 326 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 327 328 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 329 let MUBUF = 1; 330 let AsmMatchConverter = "cvtMubuf"; 331 let usesCustomInserter = 1; 332} 333 334class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> : 335 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 336 337 let isPseudo = 0; 338 let isCodeGenOnly = 0; 339 340 let VM_CNT = 1; 341 let EXP_CNT = 1; 342 let MUBUF = 1; 343 344 // copy relevant pseudo op flags 345 let SubtargetPredicate = ps.SubtargetPredicate; 346 let AsmMatchConverter = ps.AsmMatchConverter; 347 let OtherPredicates = ps.OtherPredicates; 348 let Constraints = ps.Constraints; 349 let DisableEncoding = ps.DisableEncoding; 350 let TSFlags = ps.TSFlags; 351 let UseNamedOperandTable = ps.UseNamedOperandTable; 352 let SchedRW = ps.SchedRW; 353 let mayLoad = ps.mayLoad; 354 let mayStore = ps.mayStore; 355 let IsAtomicRet = ps.IsAtomicRet; 356 let IsAtomicNoRet = ps.IsAtomicNoRet; 357 let VALU = ps.VALU; 358 let LGKM_CNT = ps.LGKM_CNT; 359 let Uses = ps.Uses; 360 let Defs = ps.Defs; 361 let isConvergent = ps.isConvergent; 362 363 bits<12> offset; 364 bits<5> cpol; 365 bits<8> vaddr; 366 bits<10> vdata; 367 bits<7> srsrc; 368 bits<8> soffset; 369 370 // GFX90A+ only: instruction uses AccVGPR for data 371 // Bit supersedes tfe. 372 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 373} 374 375// For cache invalidation instructions. 376class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 377 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 378 379 let AsmMatchConverter = ""; 380 381 let hasSideEffects = 1; 382 let mayLoad = 0; 383 let mayStore = 0; 384 385 let IsBufferInv = 1; 386 // Set everything else to 0. 387 let offen = 0; 388 let idxen = 0; 389 let addr64 = 0; 390 let has_vdata = 0; 391 let has_vaddr = 0; 392 let has_glc = 0; 393 let has_dlc = 0; 394 let glc_value = 0; 395 let dlc_value = 0; 396 let has_srsrc = 0; 397 let has_soffset = 0; 398 let has_offset = 0; 399 let has_slc = 0; 400 let has_sccb = 0; 401 let sccb_value = 0; 402} 403 404class getLdStVDataRegisterOperand<RegisterClass RC, bit isTFE> { 405 RegisterOperand tfeVDataOp = 406 !cond(!eq(RC.Size, 32) : AVLdSt_64, 407 !eq(RC.Size, 64) : AVLdSt_96, 408 !eq(RC.Size, 96) : AVLdSt_128, 409 !eq(RC.Size, 128) : AVLdSt_160); 410 411 RegisterOperand ret = !if(isTFE, tfeVDataOp, getLdStRegisterOperand<RC>.ret); 412} 413 414class getMUBUFInsDA<list<RegisterClass> vdataList, 415 list<RegisterClass> vaddrList, bit isTFE, bit hasRestrictedSOffset> { 416 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 417 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 418 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdataClass, isTFE>.ret; 419 420 dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); 421 dag NonVaddrInputs = !con((ins SReg_128_XNULL:$srsrc), SOffset, (ins Offset:$offset, CPol_0:$cpol, i1imm_0:$swz)); 422 423 dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 424 dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs)); 425} 426 427class getMUBUFElements<ValueType vt> { 428 int ret = 429 !if(!eq(vt, f16), 1, 430 !if(!eq(vt, v2f16), 2, 431 !if(!eq(vt, v3f16), 3, 432 !if(!eq(vt, v4f16), 4, 433 !if(!eq(vt.Size, 32), 1, 434 !if(!eq(vt.Size, 64), 2, 435 !if(!eq(vt.Size, 96), 3, 436 !if(!eq(vt.Size, 128), 4, 0) 437 ) 438 ) 439 ) 440 ) 441 ) 442 ) 443 ); 444} 445 446class getMUBUFIns<int addrKind, list<RegisterClass> vdataList, bit isTFE, bit hasRestrictedSOffset> { 447 dag ret = 448 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE, hasRestrictedSOffset>.ret, 449 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasRestrictedSOffset>.ret, 450 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasRestrictedSOffset>.ret, 451 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasRestrictedSOffset>.ret, 452 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasRestrictedSOffset>.ret, 453 (ins)))))); 454} 455 456class getMUBUFAsmOps<int addrKind, bit noVdata = 0, bit isLds = 0, bit isTFE = 0> { 457 string Vdata = !if(noVdata, " ", " $vdata, "); 458 string Lds = !if(isLds, " lds", ""); 459 string TFE = !if(isTFE, " tfe", ""); 460 string MainArgs = 461 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 462 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 463 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 464 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 465 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 466 ""))))); 467 string Offset = "$offset"; 468 string OtherArgs = "$cpol"; 469 470 string ret = Vdata # MainArgs # Offset # OtherArgs # Lds # TFE; 471} 472 473class MUBUF_SetupAddr<int addrKind> { 474 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 475 !eq(addrKind, BUFAddrKind.BothEn)); 476 477 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 478 !eq(addrKind, BUFAddrKind.BothEn)); 479 480 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 481 482 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 483} 484 485class MUBUF_Load_Pseudo <string opName, 486 int addrKind, 487 ValueType vdata_vt, 488 bit HasTiedDest = 0, 489 bit isLds = 0, 490 bit isLdsOpc = 0, 491 bit isTFE = 0, 492 bit hasRestrictedSOffset = 0, 493 list<dag> pattern=[], 494 // Workaround bug bz30254 495 int addrKindCopy = addrKind, 496 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret.RegClass, 497 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdata_rc, isTFE>.ret> 498 : MUBUF_Pseudo<opName, 499 !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)), 500 !con(getMUBUFIns<addrKindCopy, [], isTFE, hasRestrictedSOffset>.ret, 501 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 502 getMUBUFAsmOps<addrKindCopy, !or(isLds, isLdsOpc), isLds, isTFE>.ret, 503 pattern>, 504 MUBUF_SetupAddr<addrKindCopy> { 505 let PseudoInstr = opName # !if(isLds, "_lds", "") # !if(isTFE, "_tfe", "") # 506 "_" # getAddrName<addrKindCopy>.ret; 507 let AsmMatchConverter = "cvtMubuf"; 508 509 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 510 let LGKM_CNT = isLds; 511 let has_vdata = !not(!or(isLds, isLdsOpc)); 512 let mayLoad = 1; 513 let mayStore = isLds; 514 let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]); 515 let tfe = isTFE; 516 let lds = isLds; 517 let elements = getMUBUFElements<vdata_vt>.ret; 518 let VALU = isLds; 519} 520 521class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : GCNPat < 522 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 523 (load_vt (inst v4i32:$srsrc, i32:$soffset, i32:$offset)) 524>; 525 526class MUBUF_Addr64_Load_Pat <Instruction inst, 527 ValueType load_vt = i32, 528 SDPatternOperator ld = null_frag> : GCNPat < 529 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), 530 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)) 531>; 532 533multiclass MUBUF_Pseudo_Load_Pats_Common<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 534 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 535 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 536} 537 538multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag>{ 539 let OtherPredicates = [HasUnrestrictedSOffset] in { 540 defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst, load_vt, ld>; 541 } 542 defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst # "_VBUFFER", load_vt, ld>; 543} 544 545multiclass MUBUF_Pseudo_Loads_Helper<string opName, ValueType load_vt, 546 bit TiedDest, bit isLds, bit isTFE, bit hasRestrictedSOffset> { 547 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 548 549 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>, 550 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 551 552 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>, 553 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 554 555 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 556 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 557 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 558 559 let DisableWQM = 1 in { 560 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 561 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 562 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 563 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 564 } 565} 566 567multiclass MUBUF_Pseudo_Loads<string opName, ValueType load_vt = i32, 568 bit TiedDest = 0, bit isLds = 0> { 569 defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 0>; 570 defm _VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1>; 571 572 if !not(isLds) then { 573 defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 0>; 574 defm _TFE_VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1>; 575 } 576} 577 578multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32, Predicate LDSPred = TruePredicate> { 579 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 580 581 if !ne(LDSPred, TruePredicate) then { 582 let SubtargetPredicate = LDSPred in { 583 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 584 } 585 } else { 586 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 587 } 588 589} 590 591multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName, 592 ValueType load_vt = i32, 593 bit TiedDest = 0, 594 bit isLds = 0, 595 bit isLdsOpc = 1> { 596 597 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 598 599 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc>; 600 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 601 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 602 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 603 604 def _VBUFFER_OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 605 def _VBUFFER_OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 606 def _VBUFFER_IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 607 def _VBUFFER_BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 608} 609 610class MUBUF_Store_Pseudo <string opName, 611 int addrKind, 612 ValueType store_vt, 613 bit isTFE = 0, 614 bit hasRestrictedSOffset = 0, 615 list<dag> pattern=[], 616 // Workaround bug bz30254 617 int addrKindCopy = addrKind> 618 : MUBUF_Pseudo<opName, 619 (outs), 620 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret.RegClass], isTFE, hasRestrictedSOffset>.ret, 621 getMUBUFAsmOps<addrKindCopy, 0, 0, isTFE>.ret, 622 pattern>, 623 MUBUF_SetupAddr<addrKindCopy> { 624 let PseudoInstr = opName # "_" # !if(isTFE, "_tfe", "") # 625 getAddrName<addrKindCopy>.ret; 626 let mayLoad = 0; 627 let mayStore = 1; 628 let elements = getMUBUFElements<store_vt>.ret; 629 let tfe = isTFE; 630} 631 632multiclass MUBUF_Pseudo_Store_Pats_Common<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> { 633 634 def : GCNPat < 635 (st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)), 636 (!cast<MUBUF_Pseudo>(BaseInst # _OFFSET) store_vt:$vdata, v4i32:$srsrc, i32:$soffset, i32:$offset)>; 637 638 def : GCNPat < 639 (st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)), 640 (!cast<MUBUF_Pseudo>(BaseInst # _ADDR64) store_vt:$vdata, i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)>; 641} 642 643multiclass MUBUF_Pseudo_Store_Pats<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> { 644 let OtherPredicates = [HasUnrestrictedSOffset] in { 645 defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst, store_vt, st>; 646 } 647 defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst # "_VBUFFER", store_vt, st>; 648} 649 650multiclass MUBUF_Pseudo_Stores_Helper<string opName, ValueType store_vt, 651 bit isTFE, bit hasRestrictedSOffset> { 652 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 653 654 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset>, 655 MUBUFAddr64Table<0, NAME>; 656 657 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, hasRestrictedSOffset>, 658 MUBUFAddr64Table<1, NAME>; 659 660 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 661 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 662 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 663 664 let DisableWQM = 1 in { 665 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset>; 666 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 667 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 668 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 669 } 670} 671 672multiclass MUBUF_Pseudo_Stores<string opName, ValueType store_vt = i32> { 673 defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 0>; 674 defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 0>; 675 676 defm _VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1>; 677 defm _TFE_VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1>; 678} 679 680class MUBUF_Pseudo_Store_Lds<string opName> 681 : MUBUF_Pseudo<opName, 682 (outs), 683 (ins SReg_128_XNULL:$srsrc, SCSrc_b32:$soffset, Offset:$offset, CPol:$cpol, i1imm:$swz), 684 " $srsrc, $soffset$offset lds$cpol"> { 685 let LGKM_CNT = 1; 686 let mayLoad = 1; 687 let mayStore = 1; 688 689 let has_vdata = 0; 690 let has_vaddr = 0; 691 let lds = 1; 692 let VALU = 1; 693 694 let Uses = [EXEC, M0]; 695 let AsmMatchConverter = "cvtMubuf"; 696} 697 698class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, bit hasRestrictedSOffset, 699 list<RegisterClass> vaddrList=[]> { 700 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 701 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 702 703 dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata)); 704 dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddrClass:$vaddr))); 705 dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); 706 dag MainInputs = !con((ins SReg_128_XNULL:$srsrc), SOffset, (ins Offset:$offset)); 707 dag CPol = !if(vdata_in, (ins CPol_GLC_WithDefault:$cpol), 708 (ins CPol_NonGLC_WithDefault:$cpol)); 709 710 dag ret = !con(Data, MainInputs, CPol); 711} 712 713class getMUBUFAtomicIns<int addrKind, 714 RegisterClass vdataClass, 715 bit vdata_in, 716 bit hasRestrictedSOffset, 717 // Workaround bug bz30254 718 RegisterClass vdataClassCopy=vdataClass> { 719 dag ret = 720 !if(!eq(addrKind, BUFAddrKind.Offset), 721 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset>.ret, 722 !if(!eq(addrKind, BUFAddrKind.OffEn), 723 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VGPR_32]>.ret, 724 !if(!eq(addrKind, BUFAddrKind.IdxEn), 725 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VGPR_32]>.ret, 726 !if(!eq(addrKind, BUFAddrKind.BothEn), 727 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VReg_64]>.ret, 728 !if(!eq(addrKind, BUFAddrKind.Addr64), 729 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VReg_64]>.ret, 730 (ins)))))); 731} 732 733class MUBUF_Atomic_Pseudo<string opName, 734 int addrKind, 735 dag outs, 736 dag ins, 737 string asmOps, 738 list<dag> pattern=[], 739 // Workaround bug bz30254 740 int addrKindCopy = addrKind> 741 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 742 MUBUF_SetupAddr<addrKindCopy> { 743 let mayStore = 1; 744 let mayLoad = 1; 745 let hasSideEffects = 1; 746 let DisableWQM = 1; 747 let has_glc = 0; 748 let has_dlc = 0; 749 let has_sccb = 1; 750 let AsmMatchConverter = "cvtMubufAtomic"; 751} 752 753class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 754 RegisterClass vdataClass, 755 bit hasRestrictedSOffset = 0, 756 list<dag> pattern=[], 757 // Workaround bug bz30254 758 int addrKindCopy = addrKind, 759 RegisterClass vdataClassCopy = vdataClass> 760 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 761 (outs), 762 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0, hasRestrictedSOffset>.ret, 763 getMUBUFAsmOps<addrKindCopy>.ret, 764 pattern> { 765 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 766 let glc_value = 0; 767 let dlc_value = 0; 768 let sccb_value = 0; 769 let IsAtomicNoRet = 1; 770} 771 772class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 773 RegisterClass vdataClass, 774 bit hasRestrictedSOffset = 0, 775 list<dag> pattern=[], 776 // Workaround bug bz30254 777 int addrKindCopy = addrKind, 778 RegisterClass vdataClassCopy = vdataClass, 779 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 780 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 781 (outs vdata_op:$vdata), 782 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1, hasRestrictedSOffset>.ret, 783 getMUBUFAsmOps<addrKindCopy>.ret, 784 pattern> { 785 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 786 let glc_value = 1; 787 let dlc_value = 0; 788 let sccb_value = 0; 789 let IsAtomicRet = 1; 790 let Constraints = "$vdata = $vdata_in"; 791 let DisableEncoding = "$vdata_in"; 792} 793 794multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 795 RegisterClass vdataClass, 796 ValueType vdataType> { 797 let FPAtomic = vdataType.isFP in { 798 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0>, 799 MUBUFAddr64Table <0, NAME>; 800 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0>, 801 MUBUFAddr64Table <1, NAME>; 802 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>; 803 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>; 804 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>; 805 806 def _VBUFFER_OFFSET : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1>, 807 MUBUFAddr64Table <0, NAME # "_VBUFFER">; 808 def _VBUFFER_ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1>, 809 MUBUFAddr64Table <1, NAME # "_VBUFFER">; 810 def _VBUFFER_OFFEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>; 811 def _VBUFFER_IDXEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>; 812 def _VBUFFER_BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>; 813 } 814} 815 816multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 817 RegisterClass vdataClass, 818 ValueType vdataType, 819 SDPatternOperator atomic> { 820 let FPAtomic = vdataType.isFP in { 821 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0, 822 [(set vdataType:$vdata, 823 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), 824 vdataType:$vdata_in))]>, 825 MUBUFAddr64Table <0, NAME # "_RTN">; 826 827 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0, 828 [(set vdataType:$vdata, 829 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 830 vdataType:$vdata_in))]>, 831 MUBUFAddr64Table <1, NAME # "_RTN">; 832 833 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>; 834 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>; 835 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>; 836 837 def _VBUFFER_OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1, 838 [(set vdataType:$vdata, 839 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), 840 vdataType:$vdata_in))]>, 841 MUBUFAddr64Table <0, NAME # "_VBUFFER_RTN">; 842 843 def _VBUFFER_ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1, 844 [(set vdataType:$vdata, 845 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 846 vdataType:$vdata_in))]>, 847 MUBUFAddr64Table <1, NAME # "_VBUFFER_RTN">; 848 849 def _VBUFFER_OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>; 850 def _VBUFFER_IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>; 851 def _VBUFFER_BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>; 852 } 853} 854 855multiclass MUBUF_Pseudo_Atomics <string opName, 856 RegisterClass vdataClass, 857 ValueType vdataType, 858 SDPatternOperator atomic = null_frag> : 859 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 860 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 861 862 863//===----------------------------------------------------------------------===// 864// MUBUF Instructions 865//===----------------------------------------------------------------------===// 866 867defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 868 "buffer_load_format_x", f32 869>; 870defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 871 "buffer_load_format_xy", v2f32 872>; 873defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 874 "buffer_load_format_xyz", v3f32 875>; 876defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 877 "buffer_load_format_xyzw", v4f32 878>; 879defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 880 "buffer_store_format_x", f32 881>; 882defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 883 "buffer_store_format_xy", v2f32 884>; 885defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 886 "buffer_store_format_xyz", v3f32 887>; 888defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 889 "buffer_store_format_xyzw", v4f32 890>; 891 892let OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1 in { 893let TiedSourceNotRead = 1 in { 894 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 895 "buffer_load_format_d16_x", i32 896 >; 897 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 898 "buffer_load_format_d16_xy", v2i32 899 >; 900 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 901 "buffer_load_format_d16_xyz", v3i32 902 >; 903 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 904 "buffer_load_format_d16_xyzw", v4i32 905 >; 906} 907 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 908 "buffer_store_format_d16_x", i32 909 >; 910 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 911 "buffer_store_format_d16_xy", v2i32 912 >; 913 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 914 "buffer_store_format_d16_xyz", v3i32 915 >; 916 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 917 "buffer_store_format_d16_xyzw", v4i32 918 >; 919} // End OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1. 920 921let OtherPredicates = [HasPackedD16VMem], D16Buf = 1 in { 922let TiedSourceNotRead = 1 in { 923 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 924 "buffer_load_format_d16_x", f16 925 >; 926 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 927 "buffer_load_format_d16_xy", v2f16 928 >; 929 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 930 "buffer_load_format_d16_xyz", v3f16 931 >; 932 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 933 "buffer_load_format_d16_xyzw", v4f16 934 >; 935} 936 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 937 "buffer_store_format_d16_x", f16 938 >; 939 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 940 "buffer_store_format_d16_xy", v2f16 941 >; 942 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 943 "buffer_store_format_d16_xyz", v3f16 944 >; 945 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 946 "buffer_store_format_d16_xyzw", v4f16 947 >; 948} // End OtherPredicates = [HasPackedD16VMem], D16Buf = 1. 949 950defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 951 "buffer_load_ubyte", i32 952>; 953defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 954 "buffer_load_sbyte", i32 955>; 956defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 957 "buffer_load_ushort", i32 958>; 959defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 960 "buffer_load_sshort", i32 961>; 962defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 963 "buffer_load_dword", i32 964>; 965defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 966 "buffer_load_dwordx2", v2i32 967>; 968defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads_Lds < 969 "buffer_load_dwordx3", v3i32, /*LDSPred=*/HasGFX950Insts 970>; 971defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads_Lds < 972 "buffer_load_dwordx4", v4i32, /*LDSPred=*/HasGFX950Insts 973>; 974 975defm BUFFER_LOAD_LDS_B32 : MUBUF_Pseudo_Loads_LDSOpc < 976 "buffer_load_lds_b32", i32 977>; 978defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Pseudo_Loads_LDSOpc < 979 "buffer_load_lds_format_x", f32 980>; 981defm BUFFER_LOAD_LDS_I8 : MUBUF_Pseudo_Loads_LDSOpc < 982 "buffer_load_lds_i8", i32 983>; 984defm BUFFER_LOAD_LDS_I16 : MUBUF_Pseudo_Loads_LDSOpc < 985 "buffer_load_lds_i16", i32 986>; 987defm BUFFER_LOAD_LDS_U8 : MUBUF_Pseudo_Loads_LDSOpc < 988 "buffer_load_lds_u8", i32 989>; 990defm BUFFER_LOAD_LDS_U16 : MUBUF_Pseudo_Loads_LDSOpc < 991 "buffer_load_lds_u16", i32 992>; 993 994defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 995defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_zext_8_global>; 996defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 997defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_zext_16_global>; 998defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 999defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 1000defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 1001defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 1002defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 1003defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, atomic_load_sext_8_global>; 1004defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, atomic_load_sext_16_global>; 1005defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 1006defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 1007defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 1008defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, atomic_load_sext_16_global>; 1009 1010foreach vt = Reg32Types.types in { 1011defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, load_global>; 1012} 1013 1014foreach vt = VReg_64.RegTypes in { 1015defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, load_global>; 1016} 1017 1018foreach vt = VReg_96.RegTypes in { 1019defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", vt, load_global>; 1020} 1021 1022foreach vt = VReg_128.RegTypes in { 1023defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", vt, load_global>; 1024} 1025 1026defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 1027 "buffer_store_byte", i32 1028>; 1029defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 1030 "buffer_store_short", i32 1031>; 1032defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 1033 "buffer_store_dword", i32 1034>; 1035defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 1036 "buffer_store_dwordx2", v2i32 1037>; 1038defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 1039 "buffer_store_dwordx3", v3i32 1040>; 1041defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 1042 "buffer_store_dwordx4", v4i32 1043>; 1044 1045defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_BYTE", i32, truncstorei8_global>; 1046defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_SHORT", i32, truncstorei16_global>; 1047 1048foreach vt = Reg32Types.types in { 1049defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, store_global>; 1050} 1051 1052foreach vt = VReg_64.RegTypes in { 1053defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, store_global>; 1054} 1055 1056foreach vt = VReg_96.RegTypes in { 1057defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX3", vt, store_global>; 1058} 1059 1060foreach vt = VReg_128.RegTypes in { 1061defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX4", vt, store_global>; 1062} 1063 1064defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 1065 "buffer_atomic_swap", VGPR_32, i32 1066>; 1067defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 1068 "buffer_atomic_cmpswap", VReg_64, v2i32 1069>; 1070defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 1071 "buffer_atomic_add", VGPR_32, i32 1072>; 1073defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 1074 "buffer_atomic_sub", VGPR_32, i32 1075>; 1076defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 1077 "buffer_atomic_smin", VGPR_32, i32 1078>; 1079defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 1080 "buffer_atomic_umin", VGPR_32, i32 1081>; 1082defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 1083 "buffer_atomic_smax", VGPR_32, i32 1084>; 1085defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 1086 "buffer_atomic_umax", VGPR_32, i32 1087>; 1088defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 1089 "buffer_atomic_and", VGPR_32, i32 1090>; 1091defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 1092 "buffer_atomic_or", VGPR_32, i32 1093>; 1094defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 1095 "buffer_atomic_xor", VGPR_32, i32 1096>; 1097defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 1098 "buffer_atomic_inc", VGPR_32, i32 1099>; 1100defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 1101 "buffer_atomic_dec", VGPR_32, i32 1102>; 1103defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 1104 "buffer_atomic_swap_x2", VReg_64, i64 1105>; 1106defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1107 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 1108>; 1109defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1110 "buffer_atomic_add_x2", VReg_64, i64 1111>; 1112defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1113 "buffer_atomic_sub_x2", VReg_64, i64 1114>; 1115defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1116 "buffer_atomic_smin_x2", VReg_64, i64 1117>; 1118defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1119 "buffer_atomic_umin_x2", VReg_64, i64 1120>; 1121defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1122 "buffer_atomic_smax_x2", VReg_64, i64 1123>; 1124defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1125 "buffer_atomic_umax_x2", VReg_64, i64 1126>; 1127defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1128 "buffer_atomic_and_x2", VReg_64, i64 1129>; 1130defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1131 "buffer_atomic_or_x2", VReg_64, i64 1132>; 1133defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1134 "buffer_atomic_xor_x2", VReg_64, i64 1135>; 1136defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1137 "buffer_atomic_inc_x2", VReg_64, i64 1138>; 1139defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1140 "buffer_atomic_dec_x2", VReg_64, i64 1141>; 1142 1143let OtherPredicates = [HasGFX10_BEncoding] in { 1144 defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics < 1145 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1146 >; 1147} 1148 1149let SubtargetPredicate = isGFX8GFX9NotGFX940 in { 1150def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1151} 1152 1153let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1154/* 1155defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1156defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1157*/ 1158 1159def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1160 int_amdgcn_buffer_wbinvl1_sc>; 1161} 1162 1163let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1164 1165defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1166 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1167>; 1168} 1169 1170let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts in { 1171defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1172 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1173>; 1174defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1175 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1176>; 1177} 1178 1179let SubtargetPredicate = isGFX6GFX7GFX10 in { 1180defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1181 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1182>; 1183} 1184 1185let SubtargetPredicate = HasD16LoadStore in { 1186let TiedSourceNotRead = 1 in { 1187 1188defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1189 "buffer_load_ubyte_d16", i32, 1 1190>; 1191 1192defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1193 "buffer_load_ubyte_d16_hi", i32, 1 1194>; 1195 1196defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1197 "buffer_load_sbyte_d16", i32, 1 1198>; 1199 1200defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1201 "buffer_load_sbyte_d16_hi", i32, 1 1202>; 1203 1204defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1205 "buffer_load_short_d16", i32, 1 1206>; 1207 1208defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1209 "buffer_load_short_d16_hi", i32, 1 1210>; 1211 1212defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1213 "buffer_load_format_d16_hi_x", i32 1214>; 1215} // End TiedSourceNotRead 1216 1217defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1218 "buffer_store_byte_d16_hi", i32 1219>; 1220 1221defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1222 "buffer_store_short_d16_hi", i32 1223>; 1224 1225defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1226 "buffer_store_format_d16_hi_x", i32 1227>; 1228 1229} // End HasD16LoadStore 1230 1231let SubtargetPredicate = isNotGFX940Plus in 1232def BUFFER_WBINVL1 : MUBUF_Invalidate < 1233 "buffer_wbinvl1", int_amdgcn_buffer_wbinvl1 1234>; 1235 1236let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1237defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN< 1238 "buffer_atomic_add_f32", VGPR_32, f32 1239>; 1240 1241let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in 1242defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1243 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1244>; 1245 1246let SubtargetPredicate = HasAtomicFaddRtnInsts in 1247defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN< 1248 "buffer_atomic_add_f32", VGPR_32, f32, null_frag 1249>; 1250 1251let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts in 1252defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1253 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, null_frag 1254>; 1255 1256let SubtargetPredicate = isGFX12Plus in { 1257defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Pseudo_Atomics < 1258 "buffer_atomic_cond_sub_u32", VGPR_32, i32 1259>; 1260} 1261 1262let SubtargetPredicate = HasAtomicBufferPkAddBF16Inst in { 1263let FPAtomic = 1 in 1264defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Pseudo_Atomics < 1265 "buffer_atomic_pk_add_bf16", VGPR_32, v2bf16 1266>; 1267} 1268 1269//===----------------------------------------------------------------------===// 1270// MTBUF Instructions 1271//===----------------------------------------------------------------------===// 1272 1273defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1274defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1275defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1276defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1277defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1278defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1279defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1280defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1281 1282let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1283let TiedSourceNotRead = 1 in { 1284 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1285 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1286 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1287 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1288} 1289 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1290 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1291 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1292 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1293} // End HasUnpackedD16VMem. 1294 1295let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1296let TiedSourceNotRead = 1 in { 1297 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1298 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1299 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1300 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1301} 1302 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1303 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1304 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1305 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1306} // End HasPackedD16VMem. 1307 1308let SubtargetPredicate = isGFX7Plus in { 1309 1310//===----------------------------------------------------------------------===// 1311// Instruction definitions for CI and newer. 1312//===----------------------------------------------------------------------===// 1313 1314let SubtargetPredicate = isNotGFX940Plus in 1315def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1316 int_amdgcn_buffer_wbinvl1_vol>; 1317 1318} // End let SubtargetPredicate = isGFX7Plus 1319 1320let SubtargetPredicate = isGFX90APlus in { 1321 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1322 let has_glc = 1; 1323 let has_sccb = 1; 1324 let InOperandList = (ins CPol_0:$cpol); 1325 let AsmOperands = "$cpol"; 1326 } 1327 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1328 let SubtargetPredicate = isGFX90AOnly; 1329 } 1330} // End SubtargetPredicate = isGFX90APlus 1331 1332let SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst in { 1333 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1334} // End SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst 1335 1336let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in { 1337 // Note the names can be buffer_atomic_fmin_x2/buffer_atomic_fmax_x2 1338 // depending on some subtargets. 1339 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1340 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1341} 1342 1343def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1344 let SubtargetPredicate = isGFX940Plus; 1345 let has_glc = 1; 1346 let has_sccb = 1; 1347 let InOperandList = (ins CPol_0:$cpol); 1348 let AsmOperands = "$cpol"; 1349} 1350 1351def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1352def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1353 1354//===----------------------------------------------------------------------===// 1355// MUBUF Patterns 1356//===----------------------------------------------------------------------===// 1357 1358//===----------------------------------------------------------------------===// 1359// buffer_load/store_format patterns 1360//===----------------------------------------------------------------------===// 1361 1362multiclass MUBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 1363 string opcode, ValueType memoryVt = vt> { 1364 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1365 1366 def : GCNPat< 1367 (vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 1368 timm:$auxiliary, 0)), 1369 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1370 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1371 >; 1372 1373 def : GCNPat< 1374 (vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1375 timm:$auxiliary, 0)), 1376 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1377 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1378 >; 1379 1380 def : GCNPat< 1381 (vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 1382 timm:$auxiliary, timm)), 1383 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1384 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1385 >; 1386 1387 def : GCNPat< 1388 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1389 timm:$auxiliary, timm)), 1390 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1391 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1392 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1393 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1394 >; 1395} 1396 1397multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1398 string opcode, ValueType memoryVt = vt>{ 1399 let SubtargetPredicate = HasUnrestrictedSOffset in { 1400 defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>; 1401 } 1402 defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 1403} 1404 1405defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1406defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1407defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1408defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1409defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1410defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1411defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1412defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1413 1414defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v2i32, "BUFFER_LOAD_FORMAT_X_TFE">; 1415defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v3i32, "BUFFER_LOAD_FORMAT_XY_TFE">; 1416defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v4i32, "BUFFER_LOAD_FORMAT_XYZ_TFE">; 1417defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v5i32, "BUFFER_LOAD_FORMAT_XYZW_TFE">; 1418 1419let OtherPredicates = [HasUnpackedD16VMem] in { 1420 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1421 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1422 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1423 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1424 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1425 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1426} // End HasUnpackedD16VMem. 1427 1428let OtherPredicates = [HasPackedD16VMem] in { 1429 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1430 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1431 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1432 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1433 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1434 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1435 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1436 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1437 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1438} // End HasPackedD16VMem. 1439 1440foreach vt = Reg32Types.types in { 1441defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORD">; 1442} 1443 1444foreach vt = Reg64Types.types in { 1445defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORDX2">; 1446} 1447 1448foreach vt = Reg96Types.types in { 1449defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORDX3">; 1450} 1451 1452foreach vt = Reg128Types.types in { 1453defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORDX4">; 1454} 1455 1456defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1457defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1458defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1459defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1460 1461defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v2i32, "BUFFER_LOAD_DWORD_TFE">; 1462defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v3i32, "BUFFER_LOAD_DWORDX2_TFE">; 1463defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v4i32, "BUFFER_LOAD_DWORDX3_TFE">; 1464defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v5i32, "BUFFER_LOAD_DWORDX4_TFE">; 1465defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte_tfe, v2i32, "BUFFER_LOAD_SBYTE_TFE">; 1466defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short_tfe, v2i32, "BUFFER_LOAD_SSHORT_TFE">; 1467defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte_tfe, v2i32, "BUFFER_LOAD_UBYTE_TFE">; 1468defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort_tfe, v2i32, "BUFFER_LOAD_USHORT_TFE">; 1469 1470multiclass MUBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 1471 string opcode, ValueType memoryVt = vt> { 1472 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1473 1474 def : GCNPat< 1475 (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 1476 timm:$auxiliary, 0), 1477 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1478 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1479 >; 1480 1481 def : GCNPat< 1482 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1483 timm:$auxiliary, 0), 1484 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1485 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1486 >; 1487 1488 def : GCNPat< 1489 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 1490 timm:$auxiliary, timm), 1491 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1492 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1493 >; 1494 1495 def : GCNPat< 1496 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1497 timm:$auxiliary, timm), 1498 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1499 getVregSrcForVT<vt>.ret:$vdata, 1500 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1501 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary), 1502 (extract_swz $auxiliary)) 1503 >; 1504} 1505 1506multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1507 string opcode, ValueType memoryVt = vt> { 1508 let SubtargetPredicate = HasUnrestrictedSOffset in { 1509 defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>; 1510 } 1511 defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 1512} 1513 1514defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1515defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1516defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1517defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1518defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1519defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1520defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1521defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1522defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1523 1524let OtherPredicates = [HasUnpackedD16VMem] in { 1525 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1526 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1527 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1528 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1529 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1530 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1531} // End HasUnpackedD16VMem. 1532 1533let OtherPredicates = [HasPackedD16VMem] in { 1534 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1535 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1536 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1537 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1538 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1539 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1540 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1541 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1542 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1543} // End HasPackedD16VMem. 1544 1545foreach vt = Reg32Types.types in { 1546defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORD">; 1547} 1548 1549foreach vt = Reg64Types.types in { 1550defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORDX2">; 1551} 1552 1553foreach vt = Reg96Types.types in { 1554defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORDX3">; 1555} 1556 1557foreach vt = Reg128Types.types in { 1558defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORDX4">; 1559} 1560 1561defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1562defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1563 1564//===----------------------------------------------------------------------===// 1565// buffer_atomic patterns 1566//===----------------------------------------------------------------------===// 1567 1568multiclass BufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1569 foreach RtnMode = ["ret", "noret"] in { 1570 1571 defvar Op = !cast<SDPatternOperator>(OpPrefix 1572 # !if(!eq(RtnMode, "ret"), "", "_noret") 1573 # !if(isIntr, "", "_" # vt)); 1574 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1575 1576 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1577 def : GCNPat< 1578 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$vdata_in)), 1579 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1580 SReg_128:$srsrc, SCSrc_b32:$soffset, Offset:$offset) 1581 >; 1582 1583 def : GCNPat< 1584 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 1585 vt:$vdata_in)), 1586 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1587 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, Offset:$offset) 1588 >; 1589 } // end let AddedComplexity 1590 1591 } // end foreach RtnMode 1592} 1593 1594multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1595 let SubtargetPredicate = HasUnrestrictedSOffset in { 1596 defm : BufferAtomicPat_Common<OpPrefix, vt, Inst, isIntr>; 1597 } 1598 defm : BufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", isIntr>; 1599} 1600 1601multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> : 1602 BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1603 1604multiclass BufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> { 1605 foreach RtnMode = ["ret", "noret"] in { 1606 1607 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global" 1608 # !if(!eq(RtnMode, "ret"), "", "_noret") 1609 # "_" # vt); 1610 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1611 defvar data_vt_RC = getVregSrcForVT<data_vt>.ret.RegClass; 1612 1613 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1614 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1615 data_vt_RC:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1616 Offset:$offset); 1617 def : GCNPat< 1618 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), data_vt:$vdata_in)), 1619 !if(!eq(RtnMode, "ret"), 1620 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, data_vt_RC)), 1621 !if(!eq(vt, i32), sub0, sub0_sub1)), 1622 OffsetResDag) 1623 >; 1624 1625 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1626 data_vt_RC:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1627 SCSrc_b32:$soffset, Offset:$offset); 1628 def : GCNPat< 1629 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 1630 data_vt:$vdata_in)), 1631 !if(!eq(RtnMode, "ret"), 1632 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, data_vt_RC)), 1633 !if(!eq(vt, i32), sub0, sub0_sub1)), 1634 Addr64ResDag) 1635 >; 1636 } // end let AddedComplexity 1637 1638 } // end foreach RtnMode 1639} 1640 1641multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1642 let SubtargetPredicate = HasUnrestrictedSOffset in { 1643 defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>; 1644 } 1645 defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">; 1646} 1647 1648 1649foreach Ty = [i32, i64] in { 1650 1651defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1652 1653defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1654defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1655defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1656defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1657defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1658defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1659defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1660defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1661defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1662defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1663defm : BufferAtomicPat<"atomic_load_uinc_wrap_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1664defm : BufferAtomicPat<"atomic_load_udec_wrap_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1665 1666} // end foreach Ty 1667 1668let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts in { 1669defm : BufferAtomicPat<"atomic_load_fmin_global", f32, "BUFFER_ATOMIC_FMIN">; 1670defm : BufferAtomicPat<"atomic_load_fmax_global", f32, "BUFFER_ATOMIC_FMAX">; 1671} 1672 1673let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in { 1674defm : BufferAtomicPat<"atomic_load_fmin_global", f64, "BUFFER_ATOMIC_MIN_F64">; 1675defm : BufferAtomicPat<"atomic_load_fmax_global", f64, "BUFFER_ATOMIC_MAX_F64">; 1676} 1677 1678defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1679defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1680 1681multiclass SIBufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, 1682 list<string> RtnModes = ["ret", "noret"]> { 1683 foreach RtnMode = RtnModes in { 1684 1685 defvar Op = !cast<SDPatternOperator>(OpPrefix 1686 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1687 1688 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1689 defvar CachePolicy = !if(!eq(RtnMode, "ret"), 1690 (extract_cpol_set_glc $auxiliary), (extract_cpol $auxiliary)); 1691 1692 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1693 def : GCNPat< 1694 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), 1695 timm:$offset, timm:$auxiliary, 0)), 1696 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1697 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1698 timm:$offset, CachePolicy) 1699 >; 1700 1701 def : GCNPat< 1702 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), 1703 timm:$offset, timm:$auxiliary, timm)), 1704 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1705 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1706 SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1707 >; 1708 1709 def : GCNPat< 1710 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1711 (BUFSOffset i32:$soffset), timm:$offset, timm:$auxiliary, 0)), 1712 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1713 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1714 SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1715 >; 1716 1717 def : GCNPat< 1718 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1719 (BUFSOffset i32:$soffset), timm:$offset, timm:$auxiliary, timm)), 1720 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1721 getVregSrcForVT<vt>.ret:$vdata_in, 1722 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1723 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1724 >; 1725 } // end let AddedComplexity 1726 1727 } // end foreach RtnMode 1728} 1729 1730multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1731 list<string> RtnModes = ["ret", "noret"]> { 1732 let OtherPredicates = [HasUnrestrictedSOffset] in { 1733 defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst, RtnModes>; 1734 } 1735 1736 // FIXME: This needs a !HasUnrestrictedSOffset predicate 1737 defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", RtnModes>; 1738} 1739 1740defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1741defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1742defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1743defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1744defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1745defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1746defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1747defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1748defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1749defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1750defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1751defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1752defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1753defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["ret"]>; 1754defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1755defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1756defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1757defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1758defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1759defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1760defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1761defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1762defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1763defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1764defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1765defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1766 1767let SubtargetPredicate = HasAtomicCSubNoRtnInsts in 1768defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["noret"]>; 1769 1770let SubtargetPredicate = HasAtomicBufferPkAddBF16Inst in { 1771 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2bf16, "BUFFER_ATOMIC_PK_ADD_BF16">; 1772} 1773 1774let SubtargetPredicate = isGFX12Plus in { 1775 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["ret"]>; 1776} 1777 1778let SubtargetPredicate = HasAtomicCSubNoRtnInsts in { 1779defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["noret"]>; 1780} 1781 1782let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts in { 1783 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1784 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1785} 1786 1787let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in { 1788 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1789 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1790} 1791 1792class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1793 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1794 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7))> { 1795 let HasNoUse = true; 1796} 1797 1798multiclass BufferAtomicPatterns_NO_RTN_Common<SDPatternOperator name, ValueType vt, 1799 string opcode> { 1800 def : GCNPat< 1801 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1802 0, (BUFSOffset i32:$soffset), timm:$offset, 1803 timm:$auxiliary, 0), 1804 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1805 timm:$offset, (extract_cpol $auxiliary)) 1806 >; 1807 1808 def : GCNPat< 1809 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1810 0, (BUFSOffset i32:$soffset), timm:$offset, 1811 timm:$auxiliary, timm), 1812 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1813 timm:$offset, (extract_cpol $auxiliary)) 1814 >; 1815 1816 def : GCNPat< 1817 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1818 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1819 timm:$auxiliary, 0), 1820 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1821 timm:$offset, (extract_cpol $auxiliary)) 1822 >; 1823 1824 def : GCNPat< 1825 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1826 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1827 timm:$auxiliary, timm), 1828 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1829 getVregSrcForVT<vt>.ret:$vdata_in, 1830 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1831 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary)) 1832 >; 1833} 1834 1835multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1836 string opcode> { 1837 let SubtargetPredicate = HasUnrestrictedSOffset in { 1838 defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode>; 1839 } 1840 defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode # "_VBUFFER">; 1841} 1842 1843let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1844 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["noret"]>; 1845 1846let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in { 1847 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["noret"]>; 1848} // End SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts 1849 1850let SubtargetPredicate = HasAtomicFaddRtnInsts in 1851 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["ret"]>; 1852 1853let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts in { 1854 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["ret"]>; 1855} // End SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts 1856 1857let SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst in { 1858 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1859} // End SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst 1860 1861let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in { 1862 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1863 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1864} //End let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts 1865 1866multiclass SIBufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> { 1867 foreach RtnMode = ["ret", "noret"] in { 1868 defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap 1869 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1870 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1871 defvar CachePolicy = !if(!eq(RtnMode, "ret"), 1872 (extract_cpol_set_glc $auxiliary), 1873 (extract_cpol $auxiliary)); 1874 defvar SrcRC = getVregSrcForVT<vt>.ret; 1875 defvar DataRC = getVregSrcForVT<data_vt>.ret.RegClass; 1876 defvar SubLo = !if(!eq(vt, i32), sub0, sub0_sub1); 1877 defvar SubHi = !if(!eq(vt, i32), sub1, sub2_sub3); 1878 1879 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1880 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1881 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy); 1882 def : GCNPat< 1883 (vt (Op 1884 vt:$data, vt:$cmp, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), 1885 timm:$offset, timm:$auxiliary, 0)), 1886 !if(!eq(RtnMode, "ret"), 1887 (EXTRACT_SUBREG OffsetResDag, SubLo), 1888 OffsetResDag) 1889 >; 1890 1891 defvar IdxenResDag = (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1892 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1893 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1894 CachePolicy); 1895 def : GCNPat< 1896 (vt (Op 1897 vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex, 1898 0, (BUFSOffset i32:$soffset), timm:$offset, 1899 timm:$auxiliary, timm)), 1900 !if(!eq(RtnMode, "ret"), 1901 (EXTRACT_SUBREG IdxenResDag, SubLo), 1902 IdxenResDag) 1903 >; 1904 1905 defvar OffenResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1906 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1907 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1908 CachePolicy); 1909 def : GCNPat< 1910 (vt (Op 1911 vt:$data, vt:$cmp, v4i32:$rsrc, 0, 1912 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1913 timm:$auxiliary, 0)), 1914 !if(!eq(RtnMode, "ret"), 1915 (EXTRACT_SUBREG OffenResDag, SubLo), 1916 OffenResDag) 1917 >; 1918 1919 defvar BothenResDag = (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1920 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1921 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1922 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy); 1923 def : GCNPat< 1924 (vt (Op 1925 vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex, 1926 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1927 timm:$auxiliary, timm)), 1928 !if(!eq(RtnMode, "ret"), 1929 (EXTRACT_SUBREG BothenResDag, SubLo), 1930 BothenResDag) 1931 >; 1932 } // end foreach RtnMode 1933} 1934 1935multiclass SIBufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1936 let OtherPredicates = [HasUnrestrictedSOffset] in { 1937 defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>; 1938 } 1939 defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">; 1940} 1941 1942defm : SIBufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1943defm : SIBufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1944 1945class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1946 PatFrag constant_ld> : GCNPat < 1947 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1948 i32:$offset))), 1949 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1950 >; 1951 1952multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1953 ValueType vt, PatFrag atomic_ld> { 1954 def : GCNPat < 1955 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), 1956 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1957 >; 1958 1959 def : GCNPat < 1960 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))), 1961 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1962 >; 1963} 1964 1965let SubtargetPredicate = isGFX6GFX7 in { 1966def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1967def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1968def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1969def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1970def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1971def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1972 1973defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1974defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1975} // End SubtargetPredicate = isGFX6GFX7 1976 1977multiclass MUBUFLoad_PatternOffset_Common <string Instr, ValueType vt, 1978 PatFrag ld> { 1979 def : GCNPat < 1980 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 1981 (!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $srsrc, $soffset, $offset) 1982 >; 1983} 1984 1985multiclass MUBUFLoad_PatternOffset <string Instr, ValueType vt, 1986 PatFrag ld> { 1987 let OtherPredicates = [HasUnrestrictedSOffset] in { 1988 defm : MUBUFLoad_PatternOffset_Common<Instr, vt, ld>; 1989 } 1990 defm : MUBUFLoad_PatternOffset_Common<Instr # "_VBUFFER", vt, ld>; 1991} 1992 1993let OtherPredicates = [Has16BitInsts] in { 1994 1995defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_constant>; 1996defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_constant>; 1997defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_constant>; 1998defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_global>; 1999defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_global>; 2000defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_global>; 2001 2002defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_USHORT", i16, load_global>; 2003 2004} // End OtherPredicates = [Has16BitInsts] 2005 2006multiclass MUBUFScratchLoadPat_Common <string Instr, 2007 ValueType vt, PatFrag ld> { 2008 def : GCNPat < 2009 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 2010 i32:$soffset, i32:$offset))), 2011 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, 0, 0) 2012 >; 2013 2014 def : GCNPat < 2015 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 2016 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, 0, 0) 2017 >; 2018} 2019 2020multiclass MUBUFScratchLoadPat <string Instr, 2021 ValueType vt, PatFrag ld> { 2022 let SubtargetPredicate = HasUnrestrictedSOffset in { 2023 defm : MUBUFScratchLoadPat_Common<Instr, vt, ld>; 2024 } 2025 defm : MUBUFScratchLoadPat_Common<Instr # "_VBUFFER", vt, ld>; 2026} 2027 2028// XXX - Is it possible to have a complex pattern in a PatFrag? 2029multiclass MUBUFScratchLoadPat_D16_Common <string Instr, 2030 ValueType vt, PatFrag ld_frag> { 2031 def : GCNPat < 2032 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, i32:$offset), vt:$in), 2033 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, $in) 2034 >; 2035 2036 def : GCNPat < 2037 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$in), 2038 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, $in) 2039 >; 2040} 2041 2042multiclass MUBUFScratchLoadPat_D16 <string Instr, 2043 ValueType vt, PatFrag ld_frag> { 2044 let SubtargetPredicate = HasUnrestrictedSOffset in { 2045 defm : MUBUFScratchLoadPat_D16_Common<Instr, vt, ld_frag>; 2046 } 2047 defm : MUBUFScratchLoadPat_D16_Common<Instr # "_VBUFFER", vt, ld_frag>; 2048} 2049 2050let OtherPredicates = [DisableFlatScratch] in { 2051defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i32, sextloadi8_private>; 2052defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, extloadi8_private>; 2053defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, zextloadi8_private>; 2054defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i16, sextloadi8_private>; 2055defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, extloadi8_private>; 2056defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, zextloadi8_private>; 2057defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SSHORT", i32, sextloadi16_private>; 2058defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, extloadi16_private>; 2059defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, zextloadi16_private>; 2060defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i16, load_private>; 2061 2062foreach vt = Reg32Types.types in { 2063defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORD", vt, load_private>; 2064} 2065defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX2", v2i32, load_private>; 2066defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX3", v3i32, load_private>; 2067defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX4", v4i32, load_private>; 2068 2069let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 2070defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2i16, load_d16_hi_private>; 2071defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2i16, az_extloadi8_d16_hi_private>; 2072defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2i16, sextloadi8_d16_hi_private>; 2073defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2f16, load_d16_hi_private>; 2074defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2f16, az_extloadi8_d16_hi_private>; 2075defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2f16, sextloadi8_d16_hi_private>; 2076 2077defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2i16, load_d16_lo_private>; 2078defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2i16, az_extloadi8_d16_lo_private>; 2079defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2i16, sextloadi8_d16_lo_private>; 2080defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2f16, load_d16_lo_private>; 2081defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2f16, az_extloadi8_d16_lo_private>; 2082defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2f16, sextloadi8_d16_lo_private>; 2083} 2084 2085} // End OtherPredicates = [DisableFlatScratch] 2086 2087multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 2088 ValueType vt, PatFrag atomic_st> { 2089 def : GCNPat < 2090 (atomic_st vt:$val, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)), 2091 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 2092 >; 2093 2094 def : GCNPat < 2095 (atomic_st vt:$val, (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset)), 2096 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 2097 >; 2098} 2099let SubtargetPredicate = isGFX6GFX7 in { 2100defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 2101defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 2102defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 2103defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 2104defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 2105defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 2106} // End Predicates = isGFX6GFX7 2107 2108 2109multiclass MUBUFStore_PatternOffset_Common <string Instr, ValueType vt, 2110 PatFrag st> { 2111 2112 def : GCNPat < 2113 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)), 2114 (!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $vdata, $srsrc, $soffset, $offset) 2115 >; 2116} 2117 2118multiclass MUBUFStore_PatternOffset <string Instr, ValueType vt, 2119 PatFrag st> { 2120 let SubtargetPredicate = HasUnrestrictedSOffset in { 2121 defm : MUBUFStore_PatternOffset_Common<Instr, vt, st>; 2122 } 2123 defm : MUBUFStore_PatternOffset_Common<Instr # "_VBUFFER", vt, st>; 2124} 2125 2126defm : MUBUFStore_PatternOffset <"BUFFER_STORE_BYTE", i16, truncstorei8_global>; 2127defm : MUBUFStore_PatternOffset <"BUFFER_STORE_SHORT", i16, store_global>; 2128 2129multiclass MUBUFScratchStorePat_Common <string Instr, 2130 ValueType vt, PatFrag st, 2131 RegisterClass rc = VGPR_32> { 2132 def : GCNPat < 2133 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 2134 i32:$soffset, i32:$offset)), 2135 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0) 2136 >; 2137 2138 def : GCNPat < 2139 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 2140 i32:$offset)), 2141 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) rc:$value, $srsrc, $soffset, $offset, 0, 0) 2142 >; 2143} 2144 2145multiclass MUBUFScratchStorePat <string Instr, 2146 ValueType vt, PatFrag st, 2147 RegisterClass rc = VGPR_32> { 2148 let SubtargetPredicate = HasUnrestrictedSOffset in { 2149 defm : MUBUFScratchStorePat_Common<Instr, vt, st, rc>; 2150 } 2151 defm : MUBUFScratchStorePat_Common<Instr # "_VBUFFER", vt, st, rc>; 2152} 2153 2154let OtherPredicates = [DisableFlatScratch] in { 2155defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i32, truncstorei8_private>; 2156defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i32, truncstorei16_private>; 2157defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i16, truncstorei8_private>; 2158defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i16, store_private>; 2159 2160foreach vt = Reg32Types.types in { 2161defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORD", vt, store_private>; 2162} 2163 2164defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX2", v2i32, store_private, VReg_64>; 2165defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX3", v3i32, store_private, VReg_96>; 2166defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX4", v4i32, store_private, VReg_128>; 2167 2168 2169let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 2170 // Hiding the extract high pattern in the PatFrag seems to not 2171 // automatically increase the complexity. 2172let AddedComplexity = 1 in { 2173defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT_D16_HI", i32, store_hi16_private>; 2174defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE_D16_HI", i32, truncstorei8_hi16_private>; 2175} 2176} 2177} // End OtherPredicates = [DisableFlatScratch] 2178 2179//===----------------------------------------------------------------------===// 2180// MTBUF Patterns 2181//===----------------------------------------------------------------------===// 2182 2183//===----------------------------------------------------------------------===// 2184// tbuffer_load/store_format patterns 2185//===----------------------------------------------------------------------===// 2186 2187multiclass MTBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 2188 string opcode, ValueType memoryVt = vt> { 2189 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 2190 2191 def : GCNPat< 2192 (vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 2193 timm:$format, timm:$auxiliary, 0)), 2194 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2195 (as_i8timm $format), 2196 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2197 >; 2198 2199 def : GCNPat< 2200 (vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 2201 timm:$format, timm:$auxiliary, timm)), 2202 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2203 (as_i8timm $format), 2204 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2205 >; 2206 2207 def : GCNPat< 2208 (vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2209 timm:$format, timm:$auxiliary, 0)), 2210 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2211 (as_i8timm $format), 2212 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2213 >; 2214 2215 def : GCNPat< 2216 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2217 timm:$format, timm:$auxiliary, timm)), 2218 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 2219 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2220 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2221 (as_i8timm $format), 2222 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2223 >; 2224} 2225 2226multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 2227 string opcode, ValueType memoryVt = vt> { 2228 let OtherPredicates = [HasUnrestrictedSOffset] in { 2229 defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>; 2230 } 2231 defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 2232} 2233 2234defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 2235defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 2236defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 2237defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 2238defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 2239defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 2240defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 2241defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 2242 2243let SubtargetPredicate = HasUnpackedD16VMem in { 2244 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 2245 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 2246 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 2247 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 2248 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 2249} // End HasUnpackedD16VMem. 2250 2251let SubtargetPredicate = HasPackedD16VMem in { 2252 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 2253 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 2254 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 2255 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 2256 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 2257} // End HasPackedD16VMem. 2258 2259multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 2260 string opcode, ValueType memoryVt = vt> { 2261 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 2262 2263 def : GCNPat< 2264 (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 2265 timm:$format, timm:$auxiliary, 0), 2266 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 2267 timm:$offset, (as_i8timm $format), 2268 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2269 >; 2270 2271 def : GCNPat< 2272 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 2273 timm:$format, timm:$auxiliary, timm), 2274 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 2275 timm:$offset, (as_i8timm $format), 2276 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2277 >; 2278 2279 def : GCNPat< 2280 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2281 timm:$format, timm:$auxiliary, 0), 2282 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 2283 timm:$offset, (as_i8timm $format), 2284 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2285 >; 2286 2287 def : GCNPat< 2288 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), 2289 timm:$offset, timm:$format, timm:$auxiliary, timm), 2290 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 2291 getVregSrcForVT<vt>.ret:$vdata, 2292 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2293 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format), 2294 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2295 >; 2296} 2297 2298multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 2299 string opcode, ValueType memoryVt = vt> { 2300 let OtherPredicates = [HasUnrestrictedSOffset] in { 2301 defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>; 2302 } 2303 defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 2304} 2305 2306defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 2307defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 2308defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 2309defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 2310defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 2311defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 2312defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 2313defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 2314 2315let SubtargetPredicate = HasUnpackedD16VMem in { 2316 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2317 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2318 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2319 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2320 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2321} // End HasUnpackedD16VMem. 2322 2323let SubtargetPredicate = HasPackedD16VMem in { 2324 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2325 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2326 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2327 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2328 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2329} // End HasPackedD16VMem. 2330 2331//===----------------------------------------------------------------------===// 2332// Target-specific instruction encodings. 2333//===----------------------------------------------------------------------===// 2334 2335// Shortcut to default Mnemonic from BUF_Pseudo. Hides the cast to the 2336// specific pseudo (bothen in this case) since any of them will work. 2337class get_BUF_ps<string name> { 2338 string Mnemonic = !cast<BUF_Pseudo>(name # "_OFFSET").Mnemonic; 2339} 2340 2341//===----------------------------------------------------------------------===// 2342// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11. 2343//===----------------------------------------------------------------------===// 2344 2345class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef, 2346 string real_name = ps.Mnemonic> : 2347 MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2348 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2349 let Inst{31-26} = 0x38; 2350 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2351 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2352 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2353 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2354} 2355 2356multiclass MUBUF_Real_gfx11<bits<8> op, string real_name = !cast<MUBUF_Pseudo>(NAME).Mnemonic> { 2357 defvar ps = !cast<MUBUF_Pseudo>(NAME); 2358 def _gfx11 : Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2359 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2360 // In GFX11 dlc is applicable to all loads/stores/atomics. 2361 let Inst{13} = !if(!or(ps.mayLoad, ps.mayStore), cpol{CPolBit.DLC}, ps.dlc_value); 2362 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2363 let Inst{25-18} = op; 2364 let Inst{53} = ps.tfe; 2365 let Inst{54} = ps.offen; 2366 let Inst{55} = ps.idxen; 2367 let AssemblerPredicate = isGFX11Only; 2368 let DecoderNamespace = "GFX11"; 2369 } 2370} 2371 2372class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef, string asmName> : 2373 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef, asmName> { 2374 let Inst{12} = ps.offen; 2375 let Inst{13} = ps.idxen; 2376 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2377 let Inst{16} = ps.lds; 2378 let Inst{24-18} = op; 2379 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2380 let Inst{55} = ps.tfe; 2381} 2382 2383multiclass MUBUF_Real_gfx10<bits<8> op, string psName = NAME, 2384 string asmName = !cast<MUBUF_Pseudo>(psName).Mnemonic> { 2385 defvar ps = !cast<MUBUF_Pseudo>(psName); 2386 def _gfx10 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10, asmName> { 2387 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2388 let Inst{25} = op{7}; 2389 let AssemblerPredicate = isGFX10Only; 2390 let DecoderNamespace = "GFX10"; 2391 } 2392} 2393 2394multiclass MUBUF_Real_gfx6_gfx7<bits<8> op, string psName = NAME, 2395 string asmName = !cast<MUBUF_Pseudo>(psName).Mnemonic> { 2396 defvar ps = !cast<MUBUF_Pseudo>(psName); 2397 def _gfx6_gfx7 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI, asmName> { 2398 let Inst{15} = ps.addr64; 2399 let AssemblerPredicate = isGFX6GFX7; 2400 let DecoderNamespace = "GFX6GFX7"; 2401 } 2402} 2403 2404multiclass MUBUF_Real_gfx6<bits<8> op> { 2405 defvar ps = !cast<MUBUF_Pseudo>(NAME); 2406 def _gfx6 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI, ps.Mnemonic> { 2407 let Inst{15} = ps.addr64; 2408 let AssemblerPredicate = isGFX6; 2409 let DecoderNamespace = "GFX6"; 2410 } 2411} 2412 2413multiclass MUBUF_Real_gfx7<bits<8> op> { 2414 defvar ps = !cast<MUBUF_Pseudo>(NAME); 2415 def _gfx7 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI, ps.Mnemonic> { 2416 let Inst{15} = ps.addr64; 2417 let AssemblerPredicate = isGFX7Only; 2418 let DecoderNamespace = "GFX7"; 2419 } 2420} 2421 2422//===----------------------------------------------------------------------===// 2423// Base ENC_VBUFFER for GFX12. 2424//===----------------------------------------------------------------------===// 2425 2426class VBUFFER_Real <bits<8> op, BUF_Pseudo ps, string real_name> : 2427 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []>, Enc96 { 2428 2429 let isPseudo = 0; 2430 let isCodeGenOnly = 0; 2431 2432 let VM_CNT = 1; 2433 let EXP_CNT = 1; 2434 2435 // copy relevant pseudo op flags 2436 let SubtargetPredicate = ps.SubtargetPredicate; 2437 let AsmMatchConverter = ps.AsmMatchConverter; 2438 let OtherPredicates = ps.OtherPredicates; 2439 let Constraints = ps.Constraints; 2440 let DisableEncoding = ps.DisableEncoding; 2441 let TSFlags = ps.TSFlags; 2442 let UseNamedOperandTable = ps.UseNamedOperandTable; 2443 let SchedRW = ps.SchedRW; 2444 let mayLoad = ps.mayLoad; 2445 let mayStore = ps.mayStore; 2446 let IsAtomicRet = ps.IsAtomicRet; 2447 let IsAtomicNoRet = ps.IsAtomicNoRet; 2448 let VALU = ps.VALU; 2449 let LGKM_CNT = ps.LGKM_CNT; 2450 let MUBUF = ps.MUBUF; 2451 let MTBUF = ps.MTBUF; 2452 let Uses = ps.Uses; 2453 let Defs = ps.Defs; 2454 let isConvergent = ps.isConvergent; 2455 2456 bits<24> offset; 2457 bits<8> vaddr; 2458 bits<10> vdata; 2459 2460 bits<7> srsrc; 2461 bits<7> soffset; 2462 bits<6> cpol; 2463 2464 let Inst{95-72} = !if(ps.has_offset, offset, ?); 2465 let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?); 2466 let Inst{39-32} = !if(ps.has_vdata, vdata{7-0}, ?); 2467 2468 let Inst{47-41} = !if(ps.has_srsrc, srsrc, ?); 2469 let Inst{49-48} = 0b00; 2470 let Inst{6-0} = !if(ps.has_soffset, soffset, ?); 2471 let Inst{21-14} = op; 2472 let Inst{22} = ps.tfe; 2473 let Inst{62} = ps.offen; 2474 let Inst{63} = ps.idxen; 2475 2476 let Inst{54-53} = cpol{2-1}; // th{2-1} 2477 let Inst{52} = !if(ps.IsAtomicRet, 1, cpol{0}); // th{0} 2478 let Inst{51-50} = cpol{4-3}; // scope 2479 2480 let Inst{31-26} = 0b110001; 2481} 2482 2483class VBUFFER_Real_gfx12<bits<8> op, BUF_Pseudo ps, string real_name> : 2484 VBUFFER_Real<op, ps, real_name>, 2485 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX12> { 2486 let AssemblerPredicate = isGFX12Only; 2487 let DecoderNamespace = "GFX12"; 2488} 2489 2490multiclass VBUFFER_MUBUF_Real_gfx12<bits<8> op, string real_name> { 2491 defvar ps = !cast<MUBUF_Pseudo>(NAME); 2492 def _gfx12 : VBUFFER_Real_gfx12<op, ps, real_name> { 2493 // Set the format field to be 1 to avoid round-trip issues, as some tools 2494 // print BUF_FMT_INVALID for format 0. 2495 let Inst{61-55} = 0b0000001; 2496 } 2497 // Have a version of the instruction to disassemble to for any other 2498 // format field values. 2499 def _gfx12_format : VBUFFER_Real<op, ps, real_name> { 2500 let AsmVariantName = "NonParsable"; 2501 let DecoderNamespace = "GFX12"; 2502 } 2503} 2504 2505multiclass VBUFFER_MTBUF_Real_gfx12<bits<4> op, string real_name> { 2506 defvar ps = !cast<MTBUF_Pseudo>(NAME); 2507 def _gfx12 : VBUFFER_Real_gfx12<{0b1000, op}, ps, real_name> { 2508 bits<7> format; 2509 let Inst{61-55} = format; 2510 } 2511} 2512 2513//===----------------------------------------------------------------------===// 2514// MUBUF - GFX11, GFX12. 2515//===----------------------------------------------------------------------===// 2516 2517// gfx11 instruction that accept both old and new assembler name. 2518class Mnem_gfx11_gfx12 <string mnemonic, string real_name> : 2519 AMDGPUMnemonicAlias<mnemonic, real_name> { 2520 let AssemblerPredicate = isGFX11Plus; 2521} 2522 2523class Mnem_gfx11 <string mnemonic, string real_name> : 2524 AMDGPUMnemonicAlias<mnemonic, real_name> { 2525 let AssemblerPredicate = isGFX11Only; 2526} 2527 2528class Mnem_gfx12 <string mnemonic, string real_name> : 2529 AMDGPUMnemonicAlias<mnemonic, real_name> { 2530 let AssemblerPredicate = isGFX12Plus; 2531} 2532 2533multiclass MUBUF_Real_AllAddr_gfx11_Impl2<bits<8> op, string real_name> { 2534 defm _BOTHEN : MUBUF_Real_gfx11<op, real_name>; 2535 defm _IDXEN : MUBUF_Real_gfx11<op, real_name>; 2536 defm _OFFEN : MUBUF_Real_gfx11<op, real_name>; 2537 defm _OFFSET : MUBUF_Real_gfx11<op, real_name>; 2538} 2539 2540multiclass MUBUF_Real_AllAddr_gfx12_Impl2<bits<8> op, string real_name> { 2541 defm _VBUFFER_BOTHEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2542 defm _VBUFFER_IDXEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2543 defm _VBUFFER_OFFEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2544 defm _VBUFFER_OFFSET : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2545} 2546 2547multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<bits<8> op, string real_name> : 2548 MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>, 2549 MUBUF_Real_AllAddr_gfx12_Impl2<op, real_name>; 2550 2551multiclass MUBUF_Real_AllAddr_gfx11_Impl<bits<8> op, bit hasTFE, 2552 string real_name = get_BUF_ps<NAME>.Mnemonic> { 2553 defm NAME : MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>; 2554 if hasTFE then 2555 defm _TFE : MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>; 2556} 2557 2558multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<8> op, string real_name> { 2559 defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<op, real_name>; 2560 defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<op, real_name>; 2561} 2562 2563// Non-renamed, non-atomic gfx11/gfx12 mubuf instructions. 2564multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op, bit hasTFE = 1> : 2565 MUBUF_Real_AllAddr_gfx11_Impl<op, hasTFE>; 2566 2567multiclass MUBUF_Real_AllAddr_gfx11_gfx12<bits<8> op, 2568 string real_name = get_BUF_ps<NAME>.Mnemonic> : 2569 MUBUF_Real_AllAddr_gfx11_gfx12_Impl<op, real_name> { 2570 defvar ps = get_BUF_ps<NAME>; 2571 if !ne(ps.Mnemonic, real_name) then 2572 def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>; 2573} 2574 2575multiclass MUBUF_Real_Atomic_gfx11_impl<bits<8> op, bit is_return, 2576 string real_name> { 2577 defvar Rtn = !if(is_return, "_RTN", ""); 2578 defm _BOTHEN#Rtn : MUBUF_Real_gfx11<op, real_name>; 2579 defm _IDXEN#Rtn : MUBUF_Real_gfx11<op, real_name>; 2580 defm _OFFEN#Rtn : MUBUF_Real_gfx11<op, real_name>; 2581 defm _OFFSET#Rtn : MUBUF_Real_gfx11<op, real_name>; 2582} 2583 2584multiclass MUBUF_Real_Atomic_gfx12_impl<bits<8> op, bit is_return, 2585 string real_name = get_BUF_ps<NAME>.Mnemonic> { 2586 defvar Rtn = !if(is_return, "_RTN", ""); 2587 defm _VBUFFER_BOTHEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2588 defm _VBUFFER_IDXEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2589 defm _VBUFFER_OFFEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2590 defm _VBUFFER_OFFSET#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2591} 2592 2593multiclass MUBUF_Real_Atomic_gfx11_gfx12_impl<bits<8> op, bit is_return, 2594 string real_name> : 2595 MUBUF_Real_Atomic_gfx11_impl<op, is_return, real_name>, 2596 MUBUF_Real_Atomic_gfx12_impl<op, is_return, real_name>; 2597 2598multiclass MUBUF_Real_Atomic_gfx12<bits<8> op> : 2599 MUBUF_Real_Atomic_gfx12_impl<op, 0>, 2600 MUBUF_Real_Atomic_gfx12_impl<op, 1>; 2601 2602multiclass MUBUF_Real_Atomic_gfx11<bits<8> op, string real_name> : 2603 MUBUF_Real_Atomic_gfx11_impl<op, 0, real_name>, 2604 MUBUF_Real_Atomic_gfx11_impl<op, 1, real_name> { 2605 defvar ps = get_BUF_ps<NAME>; 2606 def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>; 2607} 2608 2609multiclass MUBUF_Real_Atomic_gfx11_gfx12<bits<8> op, 2610 string gfx12_name = get_BUF_ps<NAME>.Mnemonic, 2611 string gfx11_name = gfx12_name> : 2612 MUBUF_Real_Atomic_gfx11_impl<op, 0, gfx11_name>, 2613 MUBUF_Real_Atomic_gfx11_impl<op, 1, gfx11_name>, 2614 MUBUF_Real_Atomic_gfx12_impl<op, 0, gfx12_name>, 2615 MUBUF_Real_Atomic_gfx12_impl<op, 1, gfx12_name> { 2616 defvar ps = get_BUF_ps<NAME>; 2617 if !ne(ps.Mnemonic, gfx11_name) then 2618 def : Mnem_gfx11<ps.Mnemonic, gfx11_name>; 2619 if !ne(ps.Mnemonic, gfx12_name) then 2620 def : Mnem_gfx12<ps.Mnemonic, gfx12_name>; 2621 if !ne(gfx11_name, gfx12_name) then 2622 def : Mnem_gfx12<gfx11_name, gfx12_name>; 2623} 2624 2625defm BUFFER_GL0_INV : MUBUF_Real_gfx11<0x02B>; 2626defm BUFFER_GL1_INV : MUBUF_Real_gfx11<0x02C>; 2627 2628defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12<0x014, "buffer_load_b32">; 2629defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12<0x015, "buffer_load_b64">; 2630defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12<0x016, "buffer_load_b96">; 2631defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12<0x017, "buffer_load_b128">; 2632defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x020, "buffer_load_d16_b16">; 2633defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x008, "buffer_load_d16_format_x">; 2634defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x009, "buffer_load_d16_format_xy">; 2635defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x00a, "buffer_load_d16_format_xyz">; 2636defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x00b, "buffer_load_d16_format_xyzw">; 2637defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x023, "buffer_load_d16_hi_b16">; 2638defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x026, "buffer_load_d16_hi_format_x">; 2639defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x022, "buffer_load_d16_hi_i8">; 2640defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x021, "buffer_load_d16_hi_u8">; 2641defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01f, "buffer_load_d16_i8">; 2642defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01e, "buffer_load_d16_u8">; 2643defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x000>; 2644defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x001>; 2645defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x002>; 2646defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x003>; 2647defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x011, "buffer_load_i8">; 2648defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x013, "buffer_load_i16">; 2649defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x010, "buffer_load_u8">; 2650defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x012, "buffer_load_u16">; 2651defm BUFFER_LOAD_LDS_B32 : MUBUF_Real_AllAddr_gfx11<0x031, 0>; 2652defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x032, 0>; 2653defm BUFFER_LOAD_LDS_I8 : MUBUF_Real_AllAddr_gfx11<0x02e, 0>; 2654defm BUFFER_LOAD_LDS_I16 : MUBUF_Real_AllAddr_gfx11<0x030, 0>; 2655defm BUFFER_LOAD_LDS_U8 : MUBUF_Real_AllAddr_gfx11<0x02d, 0>; 2656defm BUFFER_LOAD_LDS_U16 : MUBUF_Real_AllAddr_gfx11<0x02f, 0>; 2657defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x018, "buffer_store_b8">; 2658defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x019, "buffer_store_b16">; 2659defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12<0x01A, "buffer_store_b32">; 2660defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01B, "buffer_store_b64">; 2661defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01C, "buffer_store_b96">; 2662defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01D, "buffer_store_b128">; 2663defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x00C, "buffer_store_d16_format_x">; 2664defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x00D, "buffer_store_d16_format_xy">; 2665defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x00E, "buffer_store_d16_format_xyz">; 2666defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x00F, "buffer_store_d16_format_xyzw">; 2667defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x024, "buffer_store_d16_hi_b8">; 2668defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x025, "buffer_store_d16_hi_b16">; 2669defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x027, "buffer_store_d16_hi_format_x">; 2670defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x004>; 2671defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x005>; 2672defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x006>; 2673defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x007>; 2674defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_gfx11_gfx12<0x056>; 2675defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_gfx11_gfx12<0x035, "buffer_atomic_add_u32">; 2676defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x043, "buffer_atomic_add_u64">; 2677defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_gfx11_gfx12<0x03C, "buffer_atomic_and_b32">; 2678defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x049, "buffer_atomic_and_b64">; 2679defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_gfx11_gfx12<0x034, "buffer_atomic_cmpswap_b32">; 2680defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x042, "buffer_atomic_cmpswap_b64">; 2681defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_gfx11<0x050, "buffer_atomic_cmpswap_f32">; 2682defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Real_Atomic_gfx12<0x050>; 2683defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomic_gfx11_gfx12<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">; 2684defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_gfx11_gfx12<0x040, "buffer_atomic_dec_u32">; 2685defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04D, "buffer_atomic_dec_u64">; 2686defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_gfx11_gfx12<0x03F, "buffer_atomic_inc_u32">; 2687defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04C, "buffer_atomic_inc_u64">; 2688defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">; 2689defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x03A, "buffer_atomic_max_i32">; 2690defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x047, "buffer_atomic_max_i64">; 2691defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x03B, "buffer_atomic_max_u32">; 2692defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x048, "buffer_atomic_max_u64">; 2693defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">; 2694defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x038, "buffer_atomic_min_i32">; 2695defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x045, "buffer_atomic_min_i64">; 2696defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x039, "buffer_atomic_min_u32">; 2697defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x046, "buffer_atomic_min_u64">; 2698defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_gfx11_gfx12<0x03D, "buffer_atomic_or_b32">; 2699defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04A, "buffer_atomic_or_b64">; 2700defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_gfx11_gfx12<0x036, "buffer_atomic_sub_u32">; 2701defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x044, "buffer_atomic_sub_u64">; 2702defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_gfx11_gfx12<0x033, "buffer_atomic_swap_b32">; 2703defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x041, "buffer_atomic_swap_b64">; 2704defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_gfx11_gfx12<0x03E, "buffer_atomic_xor_b32">; 2705defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04B, "buffer_atomic_xor_b64">; 2706defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_gfx12<0x059>; 2707defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Real_Atomic_gfx12<0x05a>; 2708 2709//===----------------------------------------------------------------------===// 2710// MUBUF - GFX10. 2711//===----------------------------------------------------------------------===// 2712 2713multiclass MUBUF_Real_AllAddr_Helper_gfx10<bits<8> op> { 2714 defm _BOTHEN : MUBUF_Real_gfx10<op>; 2715 defm _IDXEN : MUBUF_Real_gfx10<op>; 2716 defm _OFFEN : MUBUF_Real_gfx10<op>; 2717 defm _OFFSET : MUBUF_Real_gfx10<op>; 2718} 2719multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2720 defm NAME : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2721 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2722} 2723multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op, bit isTFE = 0> { 2724 defm _OFFSET : MUBUF_Real_gfx10<op>; 2725 defm _OFFEN : MUBUF_Real_gfx10<op>; 2726 defm _IDXEN : MUBUF_Real_gfx10<op>; 2727 defm _BOTHEN : MUBUF_Real_gfx10<op>; 2728 2729 if !not(isTFE) then { 2730 defm _LDS_OFFSET : MUBUF_Real_gfx10<op>; 2731 defm _LDS_OFFEN : MUBUF_Real_gfx10<op>; 2732 defm _LDS_IDXEN : MUBUF_Real_gfx10<op>; 2733 defm _LDS_BOTHEN : MUBUF_Real_gfx10<op>; 2734 } 2735} 2736multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op, string psName = NAME, 2737 string asmName = !cast<MUBUF_Pseudo>(psName).Mnemonic> { 2738 defm _BOTHEN_RTN : MUBUF_Real_gfx10<op, psName#"_BOTHEN_RTN", asmName>; 2739 defm _IDXEN_RTN : MUBUF_Real_gfx10<op, psName#"_IDXEN_RTN", asmName>; 2740 defm _OFFEN_RTN : MUBUF_Real_gfx10<op, psName#"_OFFEN_RTN", asmName>; 2741 defm _OFFSET_RTN : MUBUF_Real_gfx10<op, psName#"_OFFSET_RTN", asmName>; 2742} 2743multiclass MUBUF_Real_Atomics_gfx10<bits<8> op, string psName = NAME, 2744 string asmName = get_BUF_ps<psName>.Mnemonic> : 2745 MUBUF_Real_Atomics_RTN_gfx10<op, psName, asmName> { 2746 defm _BOTHEN : MUBUF_Real_gfx10<op, psName#"_BOTHEN", asmName>; 2747 defm _IDXEN : MUBUF_Real_gfx10<op, psName#"_IDXEN", asmName>; 2748 defm _OFFEN : MUBUF_Real_gfx10<op, psName#"_OFFEN", asmName>; 2749 defm _OFFSET : MUBUF_Real_gfx10<op, psName#"_OFFSET", asmName>; 2750} 2751 2752defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2753defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2754defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2755defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2756defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2757defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2758defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2759defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2760defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2761defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2762defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2763defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2764defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2765defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2766defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2767defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2768defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2769defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2770 2771defm BUFFER_GL0_INV : MUBUF_Real_gfx10<0x071>; 2772defm BUFFER_GL1_INV : MUBUF_Real_gfx10<0x072>; 2773 2774//===----------------------------------------------------------------------===// 2775// MUBUF - GFX6, GFX7, GFX10. 2776//===----------------------------------------------------------------------===// 2777 2778multiclass MUBUF_Real_AllAddr_Helper_gfx6_gfx7<bits<8> op> { 2779 defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op>; 2780 defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op>; 2781 defm _IDXEN : MUBUF_Real_gfx6_gfx7<op>; 2782 defm _OFFEN : MUBUF_Real_gfx6_gfx7<op>; 2783 defm _OFFSET : MUBUF_Real_gfx6_gfx7<op>; 2784} 2785multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2786 defm NAME : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2787 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2788} 2789multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op, bit isTFE = 0> { 2790 defm _OFFSET : MUBUF_Real_gfx6_gfx7<op>; 2791 defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op>; 2792 defm _OFFEN : MUBUF_Real_gfx6_gfx7<op>; 2793 defm _IDXEN : MUBUF_Real_gfx6_gfx7<op>; 2794 defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op>; 2795 2796 if !not(isTFE) then { 2797 defm _LDS_OFFSET : MUBUF_Real_gfx6_gfx7<op>; 2798 defm _LDS_ADDR64 : MUBUF_Real_gfx6_gfx7<op>; 2799 defm _LDS_OFFEN : MUBUF_Real_gfx6_gfx7<op>; 2800 defm _LDS_IDXEN : MUBUF_Real_gfx6_gfx7<op>; 2801 defm _LDS_BOTHEN : MUBUF_Real_gfx6_gfx7<op>; 2802 } 2803} 2804multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op, string psName, string asmName> { 2805 defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op, psName#"_ADDR64", asmName>; 2806 defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op, psName#"_BOTHEN", asmName>; 2807 defm _IDXEN : MUBUF_Real_gfx6_gfx7<op, psName#"_IDXEN", asmName>; 2808 defm _OFFEN : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFEN", asmName>; 2809 defm _OFFSET : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFSET", asmName>; 2810 2811 defm _ADDR64_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_ADDR64_RTN", asmName>; 2812 defm _BOTHEN_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_BOTHEN_RTN", asmName>; 2813 defm _IDXEN_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_IDXEN_RTN", asmName>; 2814 defm _OFFEN_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFEN_RTN", asmName>; 2815 defm _OFFSET_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFSET_RTN", asmName>; 2816} 2817 2818multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2819 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2820 2821multiclass MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<bits<8> op, bit isTFE = 0> : 2822 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op, isTFE>, 2823 MUBUF_Real_AllAddr_Lds_gfx10<op, isTFE>; 2824 2825multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> { 2826 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op>; 2827 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op, 1>; 2828} 2829 2830multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op, string psName = NAME, 2831 string asmName = get_BUF_ps<psName>.Mnemonic> : 2832 MUBUF_Real_Atomics_gfx6_gfx7<op, psName, asmName>, 2833 MUBUF_Real_Atomics_gfx10<op, psName, asmName>; 2834 2835// FIXME-GFX6: Following instructions are available only on GFX6. 2836//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2837//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2838 2839defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2840defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2841defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2842defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2843defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2844defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2845defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2846defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2847defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2848defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2849defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2850defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2851defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2852defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2853defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2854defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2855defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2856defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2857defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2858defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2859defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2860defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2861 2862defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2863defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2864defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2865defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2866defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2867defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2868defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2869defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2870defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2871defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2872defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2873defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2874defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2875defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2876defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2877defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2878defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2879defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2880defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2881defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2882defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2883defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2884defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2885defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2886defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2887defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2888defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2889defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2890defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2891// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2892defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2893defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f, "BUFFER_ATOMIC_MIN_F64", "buffer_atomic_fmin_x2">; 2894defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060, "BUFFER_ATOMIC_MAX_F64", "buffer_atomic_fmax_x2">; 2895 2896defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_gfx10<0x034>; 2897 2898defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2899defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2900defm BUFFER_WBINVL1 : MUBUF_Real_gfx6_gfx7<0x071>; 2901 2902//===----------------------------------------------------------------------===// 2903// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11. 2904//===----------------------------------------------------------------------===// 2905 2906class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef, 2907 string real_name = ps.Mnemonic> : 2908 MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2909 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2910 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2911 let Inst{31-26} = 0x3a; //encoding 2912 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2913 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2914 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2915 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2916} 2917 2918multiclass MTBUF_Real_gfx11<bits<4> op, string real_name> { 2919 defvar ps = !cast<MTBUF_Pseudo>(NAME); 2920 def _gfx11 : Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2921 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2922 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2923 let Inst{18-15} = op; 2924 let Inst{25-19} = format; 2925 let Inst{53} = ps.tfe; 2926 let Inst{54} = ps.offen; 2927 let Inst{55} = ps.idxen; 2928 let AssemblerPredicate = isGFX11Only; 2929 let DecoderNamespace = "GFX11"; 2930 } 2931} 2932 2933class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2934 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2935 let Inst{12} = ps.offen; 2936 let Inst{13} = ps.idxen; 2937 let Inst{18-16} = op; 2938 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2939 let Inst{55} = ps.tfe; 2940} 2941 2942//===----------------------------------------------------------------------===// 2943// MTBUF - GFX11. 2944//===----------------------------------------------------------------------===// 2945 2946multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<4> op, string real_name> { 2947 defm _BOTHEN : MTBUF_Real_gfx11<op, real_name>; 2948 defm _IDXEN : MTBUF_Real_gfx11<op, real_name>; 2949 defm _OFFEN : MTBUF_Real_gfx11<op, real_name>; 2950 defm _OFFSET : MTBUF_Real_gfx11<op, real_name>; 2951 2952 defm _VBUFFER_BOTHEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>; 2953 defm _VBUFFER_IDXEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>; 2954 defm _VBUFFER_OFFEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>; 2955 defm _VBUFFER_OFFSET : VBUFFER_MTBUF_Real_gfx12<op, real_name>; 2956} 2957 2958multiclass MTBUF_Real_AllAddr_gfx11_gfx12<bits<4> op, 2959 string real_name = get_BUF_ps<NAME>.Mnemonic> 2960 : MTBUF_Real_AllAddr_gfx11_gfx12_Impl<op, real_name> { 2961 defvar ps = get_BUF_ps<NAME>; 2962 if !ne(ps.Mnemonic, real_name) then 2963 def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>; 2964} 2965 2966defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x008, "tbuffer_load_d16_format_x">; 2967defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x009, "tbuffer_load_d16_format_xy">; 2968defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x00a, "tbuffer_load_d16_format_xyz">; 2969defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x00b, "tbuffer_load_d16_format_xyzw">; 2970defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x000>; 2971defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x001>; 2972defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x002>; 2973defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x003>; 2974defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x00c, "tbuffer_store_d16_format_x">; 2975defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x00d, "tbuffer_store_d16_format_xy">; 2976defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x00e, "tbuffer_store_d16_format_xyz">; 2977defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x00f, "tbuffer_store_d16_format_xyzw">; 2978defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x004>; 2979defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x005>; 2980defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x006>; 2981defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x007>; 2982 2983//===----------------------------------------------------------------------===// 2984// MTBUF - GFX10. 2985//===----------------------------------------------------------------------===// 2986 2987multiclass MTBUF_Real_gfx10<bits<4> op> { 2988 defvar ps = !cast<MTBUF_Pseudo>(NAME); 2989 def _gfx10 : Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2990 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2991 let Inst{25-19} = format; 2992 let Inst{53} = op{3}; 2993 let AssemblerPredicate = isGFX10Only; 2994 let DecoderNamespace = "GFX10"; 2995 } 2996} 2997 2998multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2999 defm _BOTHEN : MTBUF_Real_gfx10<op>; 3000 defm _IDXEN : MTBUF_Real_gfx10<op>; 3001 defm _OFFEN : MTBUF_Real_gfx10<op>; 3002 defm _OFFSET : MTBUF_Real_gfx10<op>; 3003} 3004 3005defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 3006defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 3007defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 3008defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 3009defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 3010defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 3011defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 3012defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 3013 3014//===----------------------------------------------------------------------===// 3015// MTBUF - GFX6, GFX7, GFX10. 3016//===----------------------------------------------------------------------===// 3017 3018multiclass MTBUF_Real_gfx6_gfx7<bits<4> op> { 3019 defvar ps = !cast<MTBUF_Pseudo>(NAME); 3020 def _gfx6_gfx7 : Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 3021 let Inst{15} = ps.addr64; 3022 let Inst{22-19} = dfmt; 3023 let Inst{25-23} = nfmt; 3024 let AssemblerPredicate = isGFX6GFX7; 3025 let DecoderNamespace = "GFX6GFX7"; 3026 } 3027} 3028 3029multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 3030 defm _ADDR64 : MTBUF_Real_gfx6_gfx7<op>; 3031 defm _BOTHEN : MTBUF_Real_gfx6_gfx7<op>; 3032 defm _IDXEN : MTBUF_Real_gfx6_gfx7<op>; 3033 defm _OFFEN : MTBUF_Real_gfx6_gfx7<op>; 3034 defm _OFFSET : MTBUF_Real_gfx6_gfx7<op>; 3035} 3036 3037multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 3038 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 3039 3040defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 3041defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 3042defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 3043defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 3044defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 3045defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 3046defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 3047defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 3048 3049//===----------------------------------------------------------------------===// 3050// GFX8, GFX9 (VI). 3051//===----------------------------------------------------------------------===// 3052 3053class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 3054 bit has_sccb = ps.has_sccb> : 3055 MUBUF_Real<ps>, 3056 Enc64, 3057 SIMCInstr<ps.PseudoInstr, Enc> { 3058 3059 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3060 let Inst{12} = ps.offen; 3061 let Inst{13} = ps.idxen; 3062 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3063 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 3064 let Inst{16} = ps.lds; 3065 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3066 let Inst{24-18} = op; 3067 let Inst{31-26} = 0x38; //encoding 3068 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3069 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3070 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3071 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3072} 3073 3074multiclass MUBUF_Real_vi <bits<7> op, 3075 bit has_sccb = !cast<MUBUF_Pseudo>(NAME).has_sccb> { 3076 defvar ps = !cast<MUBUF_Pseudo>(NAME); 3077 def _vi : MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 3078 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 3079 let DecoderNamespace = "GFX8"; 3080 3081 let Inst{55} = ps.tfe; 3082 } 3083} 3084 3085multiclass MUBUF_Real_gfx90a <bits<7> op, 3086 bit has_sccb = !cast<MUBUF_Pseudo>(NAME).has_sccb> { 3087 defvar ps = !cast<MUBUF_Pseudo>(NAME); 3088 def _gfx90a : MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 3089 let AssemblerPredicate = isGFX90APlus; 3090 let DecoderNamespace = "GFX90A"; 3091 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 3092 ps.AsmOperands); 3093 3094 let Inst{55} = acc; 3095 } 3096} 3097 3098class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 3099 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 3100 let AssemblerPredicate = isGFX940Plus; 3101 let DecoderNamespace = "GFX9"; 3102 let AsmString = ps.Mnemonic # ps.AsmOperands; 3103 3104 let Inst{55} = acc; 3105} 3106 3107multiclass MUBUF_Real_vi_gfx90a<bits<7> op, bit isTFE = 0> : MUBUF_Real_vi<op> { 3108 defvar ps = !cast<MUBUF_Pseudo>(NAME); 3109 3110 if !not(isTFE) then { 3111 if !not(ps.FPAtomic) then 3112 defm NAME : MUBUF_Real_gfx90a<op>; 3113 } 3114 3115 if ps.FPAtomic then { 3116 let AssemblerPredicate = isGFX90AOnly in 3117 defm NAME : MUBUF_Real_gfx90a<op, 0>; 3118 3119 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 3120 } 3121} 3122 3123multiclass MUBUF_Real_AllAddr_Helper_vi<bits<7> op, bit isTFE = 0> { 3124 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, isTFE>; 3125 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, isTFE>; 3126 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, isTFE>; 3127 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, isTFE>; 3128} 3129 3130multiclass MUBUF_Real_AllAddr_vi<bits<7> op, bit hasTFE = 1> { 3131 defm NAME : MUBUF_Real_AllAddr_Helper_vi<op>; 3132 if hasTFE then 3133 defm _TFE : MUBUF_Real_AllAddr_Helper_vi<op, 1>; 3134} 3135 3136multiclass MUBUF_Real_AllAddr_Lds_Helper_vi<bits<7> op, bit isTFE = 0> { 3137 defm _OFFSET : MUBUF_Real_vi <op>; 3138 defm _OFFEN : MUBUF_Real_vi <op>; 3139 defm _IDXEN : MUBUF_Real_vi <op>; 3140 defm _BOTHEN : MUBUF_Real_vi <op>; 3141 3142 if !not(isTFE) then { 3143 defm _LDS_OFFSET : MUBUF_Real_vi <op>; 3144 defm _LDS_OFFEN : MUBUF_Real_vi <op>; 3145 defm _LDS_IDXEN : MUBUF_Real_vi <op>; 3146 defm _LDS_BOTHEN : MUBUF_Real_vi <op>; 3147 3148 defm _OFFSET : MUBUF_Real_gfx90a <op>; 3149 defm _OFFEN : MUBUF_Real_gfx90a <op>; 3150 defm _IDXEN : MUBUF_Real_gfx90a <op>; 3151 defm _BOTHEN : MUBUF_Real_gfx90a <op>; 3152 3153 defm _LDS_OFFSET : MUBUF_Real_gfx90a <op>; 3154 defm _LDS_OFFEN : MUBUF_Real_gfx90a <op>; 3155 defm _LDS_IDXEN : MUBUF_Real_gfx90a <op>; 3156 defm _LDS_BOTHEN : MUBUF_Real_gfx90a <op>; 3157 } 3158} 3159 3160multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 3161 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_vi<op>; 3162 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_vi<op, 1>; 3163} 3164 3165multiclass MUBUF_Real_gfx80 <bits<7> op> { 3166 defvar ps = !cast<MUBUF_Pseudo>(NAME); 3167 def _gfx80 : MUBUF_Real<ps>, 3168 Enc64, 3169 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 3170 let AssemblerPredicate=HasUnpackedD16VMem; 3171 let DecoderNamespace="GFX80_UNPACKED"; 3172 3173 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3174 let Inst{12} = ps.offen; 3175 let Inst{13} = ps.idxen; 3176 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3177 let Inst{16} = ps.lds; 3178 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3179 let Inst{24-18} = op; 3180 let Inst{31-26} = 0x38; //encoding 3181 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3182 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3183 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3184 let Inst{55} = ps.tfe; 3185 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3186 } 3187} 3188 3189multiclass MUBUF_Real_AllAddr_Helper_gfx80<bits<7> op> { 3190 defm _OFFSET : MUBUF_Real_gfx80 <op>; 3191 defm _OFFEN : MUBUF_Real_gfx80 <op>; 3192 defm _IDXEN : MUBUF_Real_gfx80 <op>; 3193 defm _BOTHEN : MUBUF_Real_gfx80 <op>; 3194} 3195 3196multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 3197 defm NAME : MUBUF_Real_AllAddr_Helper_gfx80<op>; 3198 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx80<op>; 3199} 3200 3201multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 3202 MUBUF_Real_AllAddr_vi<op, 0> { 3203 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op>; 3204 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op>; 3205 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op>; 3206 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op>; 3207} 3208 3209defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 3210defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 3211defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 3212defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 3213defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 3214defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 3215defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 3216defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 3217let SubtargetPredicate = HasUnpackedD16VMem in { 3218 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 3219 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 3220 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 3221 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 3222 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 3223 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 3224 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 3225 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 3226} // End HasUnpackedD16VMem. 3227let SubtargetPredicate = HasPackedD16VMem in { 3228 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 3229 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 3230 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 3231 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 3232 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 3233 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 3234 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 3235 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 3236} // End HasPackedD16VMem. 3237defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 3238defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 3239defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 3240defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 3241defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 3242defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 3243defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 3244defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 3245defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 3246defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 3247defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 3248defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 3249defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 3250defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 3251defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 3252defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 3253 3254defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 3255defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 3256defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 3257defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 3258defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 3259defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 3260 3261defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 3262defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 3263 3264defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 3265defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 3266defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 3267defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 3268defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 3269defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 3270defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 3271defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 3272defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 3273defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 3274defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 3275defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 3276defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 3277 3278defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 3279defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 3280defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 3281defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 3282defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 3283defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 3284defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 3285defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 3286defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 3287defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 3288defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 3289defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 3290defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 3291 3292defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d>; 3293 3294let AssemblerPredicate = isGFX8GFX9 in { 3295defm BUFFER_WBINVL1 : MUBUF_Real_vi <0x3e>; 3296defm BUFFER_WBINVL1_VOL : MUBUF_Real_vi <0x3f>; 3297} // End AssemblerPredicate = isGFX8GFX9 3298 3299 3300defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 3301defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Real_Atomic_vi <0x52>; 3302 3303defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 3304 3305let SubtargetPredicate = isGFX90APlus in { 3306 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 3307 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 3308 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 3309} // End SubtargetPredicate = isGFX90APlus 3310 3311let AsmString = BUFFER_WBL2.Mnemonic, // drop flags 3312 AssemblerPredicate = isGFX90AOnly, 3313 SubtargetPredicate = isGFX90AOnly in 3314defm BUFFER_WBL2 : MUBUF_Real_gfx90a<0x28>; 3315defm BUFFER_INVL2 : MUBUF_Real_gfx90a<0x29>; 3316 3317let SubtargetPredicate = isGFX940Plus in { 3318def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 3319def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 3320} 3321 3322class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 3323 MTBUF_Real<ps>, 3324 Enc64, 3325 SIMCInstr<ps.PseudoInstr, Enc> { 3326 3327 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3328 let Inst{12} = ps.offen; 3329 let Inst{13} = ps.idxen; 3330 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3331 let Inst{18-15} = op; 3332 let Inst{22-19} = dfmt; 3333 let Inst{25-23} = nfmt; 3334 let Inst{31-26} = 0x3a; //encoding 3335 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3336 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3337 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3338 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 3339 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3340 let Inst{55} = ps.tfe; 3341 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3342} 3343 3344class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 3345 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 3346 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 3347 let DecoderNamespace = "GFX8"; 3348 3349 let Inst{55} = ps.tfe; 3350} 3351 3352class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 3353 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 3354 let AssemblerPredicate = isGFX90APlus; 3355 let DecoderNamespace = "GFX90A"; 3356 let AsmString = ps.Mnemonic # ps.AsmOperands; 3357 3358 let Inst{55} = acc; 3359} 3360 3361multiclass MTBUF_Real_vi_gfx90a<bits<4> op> { 3362 defvar ps = !cast<MTBUF_Pseudo>(NAME); 3363 def _vi : MTBUF_Real_vi<op, ps>; 3364 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 3365} 3366 3367multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 3368 defm _OFFSET : MTBUF_Real_vi_gfx90a <op>; 3369 defm _OFFEN : MTBUF_Real_vi_gfx90a <op>; 3370 defm _IDXEN : MTBUF_Real_vi_gfx90a <op>; 3371 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op>; 3372} 3373 3374multiclass MTBUF_Real_gfx80 <bits<4> op> { 3375 defvar ps = !cast<MTBUF_Pseudo>(NAME); 3376 def _gfx80 : MTBUF_Real<ps>, 3377 Enc64, 3378 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 3379 let AssemblerPredicate=HasUnpackedD16VMem; 3380 let DecoderNamespace="GFX80_UNPACKED"; 3381 3382 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3383 let Inst{12} = ps.offen; 3384 let Inst{13} = ps.idxen; 3385 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3386 let Inst{18-15} = op; 3387 let Inst{22-19} = dfmt; 3388 let Inst{25-23} = nfmt; 3389 let Inst{31-26} = 0x3a; //encoding 3390 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3391 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3392 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3393 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3394 let Inst{55} = ps.tfe; 3395 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3396 } 3397} 3398 3399multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 3400 defm _OFFSET : MTBUF_Real_gfx80 <op>; 3401 defm _OFFEN : MTBUF_Real_gfx80 <op>; 3402 defm _IDXEN : MTBUF_Real_gfx80 <op>; 3403 defm _BOTHEN : MTBUF_Real_gfx80 <op>; 3404} 3405 3406defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 3407defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 3408defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 3409defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 3410defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 3411defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 3412defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 3413defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 3414let SubtargetPredicate = HasUnpackedD16VMem in { 3415 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 3416 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 3417 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 3418 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 3419 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 3420 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 3421 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 3422 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 3423} // End HasUnpackedD16VMem. 3424let SubtargetPredicate = HasPackedD16VMem in { 3425 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 3426 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 3427 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 3428 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 3429 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 3430 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 3431 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 3432 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 3433} // End HasUnpackedD16VMem. 3434 3435def MUBUFInfoTable : GenericTable { 3436 let FilterClass = "MUBUF_Pseudo"; 3437 let CppTypeName = "MUBUFInfo"; 3438 let Fields = [ 3439 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 3440 "IsBufferInv", "tfe" 3441 ]; 3442 3443 let PrimaryKey = ["Opcode"]; 3444 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 3445} 3446 3447def getMUBUFInfoFromOpcode : SearchIndex { 3448 let Table = MUBUFInfoTable; 3449 let Key = ["Opcode"]; 3450} 3451 3452def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3453 let Table = MUBUFInfoTable; 3454 let Key = ["BaseOpcode", "elements"]; 3455} 3456 3457def MTBUFInfoTable : GenericTable { 3458 let FilterClass = "MTBUF_Pseudo"; 3459 let CppTypeName = "MTBUFInfo"; 3460 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 3461 3462 let PrimaryKey = ["Opcode"]; 3463 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 3464} 3465 3466def getMTBUFInfoFromOpcode : SearchIndex { 3467 let Table = MTBUFInfoTable; 3468 let Key = ["Opcode"]; 3469} 3470 3471def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3472 let Table = MTBUFInfoTable; 3473 let Key = ["BaseOpcode", "elements"]; 3474} 3475