1 //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// This file implements the MachineIRBuidler class. 10 //===----------------------------------------------------------------------===// 11 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 12 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" 13 14 #include "llvm/CodeGen/MachineFunction.h" 15 #include "llvm/CodeGen/MachineInstr.h" 16 #include "llvm/CodeGen/MachineInstrBuilder.h" 17 #include "llvm/CodeGen/MachineRegisterInfo.h" 18 #include "llvm/CodeGen/TargetInstrInfo.h" 19 #include "llvm/CodeGen/TargetLowering.h" 20 #include "llvm/CodeGen/TargetOpcodes.h" 21 #include "llvm/CodeGen/TargetSubtargetInfo.h" 22 #include "llvm/IR/DebugInfo.h" 23 24 using namespace llvm; 25 26 void MachineIRBuilder::setMF(MachineFunction &MF) { 27 State.MF = &MF; 28 State.MBB = nullptr; 29 State.MRI = &MF.getRegInfo(); 30 State.TII = MF.getSubtarget().getInstrInfo(); 31 State.DL = DebugLoc(); 32 State.II = MachineBasicBlock::iterator(); 33 State.Observer = nullptr; 34 } 35 36 void MachineIRBuilder::setMBB(MachineBasicBlock &MBB) { 37 State.MBB = &MBB; 38 State.II = MBB.end(); 39 assert(&getMF() == MBB.getParent() && 40 "Basic block is in a different function"); 41 } 42 43 void MachineIRBuilder::setInstr(MachineInstr &MI) { 44 assert(MI.getParent() && "Instruction is not part of a basic block"); 45 setMBB(*MI.getParent()); 46 State.II = MI.getIterator(); 47 } 48 49 void MachineIRBuilder::setCSEInfo(GISelCSEInfo *Info) { State.CSEInfo = Info; } 50 51 void MachineIRBuilder::setInsertPt(MachineBasicBlock &MBB, 52 MachineBasicBlock::iterator II) { 53 assert(MBB.getParent() == &getMF() && 54 "Basic block is in a different function"); 55 State.MBB = &MBB; 56 State.II = II; 57 } 58 59 void MachineIRBuilder::recordInsertion(MachineInstr *InsertedInstr) const { 60 if (State.Observer) 61 State.Observer->createdInstr(*InsertedInstr); 62 } 63 64 void MachineIRBuilder::setChangeObserver(GISelChangeObserver &Observer) { 65 State.Observer = &Observer; 66 } 67 68 void MachineIRBuilder::stopObservingChanges() { State.Observer = nullptr; } 69 70 //------------------------------------------------------------------------------ 71 // Build instruction variants. 72 //------------------------------------------------------------------------------ 73 74 MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opcode) { 75 return insertInstr(buildInstrNoInsert(Opcode)); 76 } 77 78 MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) { 79 MachineInstrBuilder MIB = BuildMI(getMF(), getDL(), getTII().get(Opcode)); 80 return MIB; 81 } 82 83 MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) { 84 getMBB().insert(getInsertPt(), MIB); 85 recordInsertion(MIB); 86 return MIB; 87 } 88 89 MachineInstrBuilder 90 MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable, 91 const MDNode *Expr) { 92 assert(isa<DILocalVariable>(Variable) && "not a variable"); 93 assert(cast<DIExpression>(Expr)->isValid() && "not an expression"); 94 assert( 95 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) && 96 "Expected inlined-at fields to agree"); 97 return insertInstr(BuildMI(getMF(), getDL(), 98 getTII().get(TargetOpcode::DBG_VALUE), 99 /*IsIndirect*/ false, Reg, Variable, Expr)); 100 } 101 102 MachineInstrBuilder 103 MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable, 104 const MDNode *Expr) { 105 assert(isa<DILocalVariable>(Variable) && "not a variable"); 106 assert(cast<DIExpression>(Expr)->isValid() && "not an expression"); 107 assert( 108 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) && 109 "Expected inlined-at fields to agree"); 110 // DBG_VALUE insts now carry IR-level indirection in their DIExpression 111 // rather than encoding it in the instruction itself. 112 const DIExpression *DIExpr = cast<DIExpression>(Expr); 113 DIExpr = DIExpression::append(DIExpr, {dwarf::DW_OP_deref}); 114 return insertInstr(BuildMI(getMF(), getDL(), 115 getTII().get(TargetOpcode::DBG_VALUE), 116 /*IsIndirect*/ false, Reg, Variable, DIExpr)); 117 } 118 119 MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI, 120 const MDNode *Variable, 121 const MDNode *Expr) { 122 assert(isa<DILocalVariable>(Variable) && "not a variable"); 123 assert(cast<DIExpression>(Expr)->isValid() && "not an expression"); 124 assert( 125 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) && 126 "Expected inlined-at fields to agree"); 127 // DBG_VALUE insts now carry IR-level indirection in their DIExpression 128 // rather than encoding it in the instruction itself. 129 const DIExpression *DIExpr = cast<DIExpression>(Expr); 130 DIExpr = DIExpression::append(DIExpr, {dwarf::DW_OP_deref}); 131 return buildInstr(TargetOpcode::DBG_VALUE) 132 .addFrameIndex(FI) 133 .addReg(0) 134 .addMetadata(Variable) 135 .addMetadata(DIExpr); 136 } 137 138 MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C, 139 const MDNode *Variable, 140 const MDNode *Expr) { 141 assert(isa<DILocalVariable>(Variable) && "not a variable"); 142 assert(cast<DIExpression>(Expr)->isValid() && "not an expression"); 143 assert( 144 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) && 145 "Expected inlined-at fields to agree"); 146 auto MIB = buildInstr(TargetOpcode::DBG_VALUE); 147 if (auto *CI = dyn_cast<ConstantInt>(&C)) { 148 if (CI->getBitWidth() > 64) 149 MIB.addCImm(CI); 150 else 151 MIB.addImm(CI->getZExtValue()); 152 } else if (auto *CFP = dyn_cast<ConstantFP>(&C)) { 153 MIB.addFPImm(CFP); 154 } else { 155 // Insert %noreg if we didn't find a usable constant and had to drop it. 156 MIB.addReg(0U); 157 } 158 159 return MIB.addReg(0).addMetadata(Variable).addMetadata(Expr); 160 } 161 162 MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) { 163 assert(isa<DILabel>(Label) && "not a label"); 164 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) && 165 "Expected inlined-at fields to agree"); 166 auto MIB = buildInstr(TargetOpcode::DBG_LABEL); 167 168 return MIB.addMetadata(Label); 169 } 170 171 MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res, 172 const SrcOp &Size, 173 unsigned Align) { 174 assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type"); 175 auto MIB = buildInstr(TargetOpcode::G_DYN_STACKALLOC); 176 Res.addDefToMIB(*getMRI(), MIB); 177 Size.addSrcToMIB(MIB); 178 MIB.addImm(Align); 179 return MIB; 180 } 181 182 MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res, 183 int Idx) { 184 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type"); 185 auto MIB = buildInstr(TargetOpcode::G_FRAME_INDEX); 186 Res.addDefToMIB(*getMRI(), MIB); 187 MIB.addFrameIndex(Idx); 188 return MIB; 189 } 190 191 MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res, 192 const GlobalValue *GV) { 193 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type"); 194 assert(Res.getLLTTy(*getMRI()).getAddressSpace() == 195 GV->getType()->getAddressSpace() && 196 "address space mismatch"); 197 198 auto MIB = buildInstr(TargetOpcode::G_GLOBAL_VALUE); 199 Res.addDefToMIB(*getMRI(), MIB); 200 MIB.addGlobalAddress(GV); 201 return MIB; 202 } 203 204 MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy, 205 unsigned JTI) { 206 return buildInstr(TargetOpcode::G_JUMP_TABLE, {PtrTy}, {}) 207 .addJumpTableIndex(JTI); 208 } 209 210 void MachineIRBuilder::validateBinaryOp(const LLT &Res, const LLT &Op0, 211 const LLT &Op1) { 212 assert((Res.isScalar() || Res.isVector()) && "invalid operand type"); 213 assert((Res == Op0 && Res == Op1) && "type mismatch"); 214 } 215 216 void MachineIRBuilder::validateShiftOp(const LLT &Res, const LLT &Op0, 217 const LLT &Op1) { 218 assert((Res.isScalar() || Res.isVector()) && "invalid operand type"); 219 assert((Res == Op0) && "type mismatch"); 220 } 221 222 MachineInstrBuilder MachineIRBuilder::buildPtrAdd(const DstOp &Res, 223 const SrcOp &Op0, 224 const SrcOp &Op1) { 225 assert(Res.getLLTTy(*getMRI()).isPointer() && 226 Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch"); 227 assert(Op1.getLLTTy(*getMRI()).isScalar() && "invalid offset type"); 228 229 return buildInstr(TargetOpcode::G_PTR_ADD, {Res}, {Op0, Op1}); 230 } 231 232 Optional<MachineInstrBuilder> 233 MachineIRBuilder::materializePtrAdd(Register &Res, Register Op0, 234 const LLT &ValueTy, uint64_t Value) { 235 assert(Res == 0 && "Res is a result argument"); 236 assert(ValueTy.isScalar() && "invalid offset type"); 237 238 if (Value == 0) { 239 Res = Op0; 240 return None; 241 } 242 243 Res = getMRI()->createGenericVirtualRegister(getMRI()->getType(Op0)); 244 auto Cst = buildConstant(ValueTy, Value); 245 return buildPtrAdd(Res, Op0, Cst.getReg(0)); 246 } 247 248 MachineInstrBuilder MachineIRBuilder::buildPtrMask(const DstOp &Res, 249 const SrcOp &Op0, 250 uint32_t NumBits) { 251 assert(Res.getLLTTy(*getMRI()).isPointer() && 252 Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch"); 253 254 auto MIB = buildInstr(TargetOpcode::G_PTR_MASK); 255 Res.addDefToMIB(*getMRI(), MIB); 256 Op0.addSrcToMIB(MIB); 257 MIB.addImm(NumBits); 258 return MIB; 259 } 260 261 MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) { 262 return buildInstr(TargetOpcode::G_BR).addMBB(&Dest); 263 } 264 265 MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) { 266 assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination"); 267 return buildInstr(TargetOpcode::G_BRINDIRECT).addUse(Tgt); 268 } 269 270 MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr, 271 unsigned JTI, 272 Register IndexReg) { 273 assert(getMRI()->getType(TablePtr).isPointer() && 274 "Table reg must be a pointer"); 275 return buildInstr(TargetOpcode::G_BRJT) 276 .addUse(TablePtr) 277 .addJumpTableIndex(JTI) 278 .addUse(IndexReg); 279 } 280 281 MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res, 282 const SrcOp &Op) { 283 return buildInstr(TargetOpcode::COPY, Res, Op); 284 } 285 286 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res, 287 const ConstantInt &Val) { 288 LLT Ty = Res.getLLTTy(*getMRI()); 289 LLT EltTy = Ty.getScalarType(); 290 assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() && 291 "creating constant with the wrong size"); 292 293 if (Ty.isVector()) { 294 auto Const = buildInstr(TargetOpcode::G_CONSTANT) 295 .addDef(getMRI()->createGenericVirtualRegister(EltTy)) 296 .addCImm(&Val); 297 return buildSplatVector(Res, Const); 298 } 299 300 auto Const = buildInstr(TargetOpcode::G_CONSTANT); 301 Res.addDefToMIB(*getMRI(), Const); 302 Const.addCImm(&Val); 303 return Const; 304 } 305 306 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res, 307 int64_t Val) { 308 auto IntN = IntegerType::get(getMF().getFunction().getContext(), 309 Res.getLLTTy(*getMRI()).getScalarSizeInBits()); 310 ConstantInt *CI = ConstantInt::get(IntN, Val, true); 311 return buildConstant(Res, *CI); 312 } 313 314 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, 315 const ConstantFP &Val) { 316 LLT Ty = Res.getLLTTy(*getMRI()); 317 LLT EltTy = Ty.getScalarType(); 318 319 assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics()) 320 == EltTy.getSizeInBits() && 321 "creating fconstant with the wrong size"); 322 323 assert(!Ty.isPointer() && "invalid operand type"); 324 325 if (Ty.isVector()) { 326 auto Const = buildInstr(TargetOpcode::G_FCONSTANT) 327 .addDef(getMRI()->createGenericVirtualRegister(EltTy)) 328 .addFPImm(&Val); 329 330 return buildSplatVector(Res, Const); 331 } 332 333 auto Const = buildInstr(TargetOpcode::G_FCONSTANT); 334 Res.addDefToMIB(*getMRI(), Const); 335 Const.addFPImm(&Val); 336 return Const; 337 } 338 339 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res, 340 const APInt &Val) { 341 ConstantInt *CI = ConstantInt::get(getMF().getFunction().getContext(), Val); 342 return buildConstant(Res, *CI); 343 } 344 345 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, 346 double Val) { 347 LLT DstTy = Res.getLLTTy(*getMRI()); 348 auto &Ctx = getMF().getFunction().getContext(); 349 auto *CFP = 350 ConstantFP::get(Ctx, getAPFloatFromSize(Val, DstTy.getScalarSizeInBits())); 351 return buildFConstant(Res, *CFP); 352 } 353 354 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, 355 const APFloat &Val) { 356 auto &Ctx = getMF().getFunction().getContext(); 357 auto *CFP = ConstantFP::get(Ctx, Val); 358 return buildFConstant(Res, *CFP); 359 } 360 361 MachineInstrBuilder MachineIRBuilder::buildBrCond(Register Tst, 362 MachineBasicBlock &Dest) { 363 assert(getMRI()->getType(Tst).isScalar() && "invalid operand type"); 364 365 return buildInstr(TargetOpcode::G_BRCOND).addUse(Tst).addMBB(&Dest); 366 } 367 368 MachineInstrBuilder MachineIRBuilder::buildLoad(const DstOp &Res, 369 const SrcOp &Addr, 370 MachineMemOperand &MMO) { 371 return buildLoadInstr(TargetOpcode::G_LOAD, Res, Addr, MMO); 372 } 373 374 MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode, 375 const DstOp &Res, 376 const SrcOp &Addr, 377 MachineMemOperand &MMO) { 378 assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type"); 379 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type"); 380 381 auto MIB = buildInstr(Opcode); 382 Res.addDefToMIB(*getMRI(), MIB); 383 Addr.addSrcToMIB(MIB); 384 MIB.addMemOperand(&MMO); 385 return MIB; 386 } 387 388 MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val, 389 const SrcOp &Addr, 390 MachineMemOperand &MMO) { 391 assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type"); 392 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type"); 393 394 auto MIB = buildInstr(TargetOpcode::G_STORE); 395 Val.addSrcToMIB(MIB); 396 Addr.addSrcToMIB(MIB); 397 MIB.addMemOperand(&MMO); 398 return MIB; 399 } 400 401 MachineInstrBuilder MachineIRBuilder::buildUAddo(const DstOp &Res, 402 const DstOp &CarryOut, 403 const SrcOp &Op0, 404 const SrcOp &Op1) { 405 return buildInstr(TargetOpcode::G_UADDO, {Res, CarryOut}, {Op0, Op1}); 406 } 407 408 MachineInstrBuilder MachineIRBuilder::buildUAdde(const DstOp &Res, 409 const DstOp &CarryOut, 410 const SrcOp &Op0, 411 const SrcOp &Op1, 412 const SrcOp &CarryIn) { 413 return buildInstr(TargetOpcode::G_UADDE, {Res, CarryOut}, 414 {Op0, Op1, CarryIn}); 415 } 416 417 MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res, 418 const SrcOp &Op) { 419 return buildInstr(TargetOpcode::G_ANYEXT, Res, Op); 420 } 421 422 MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res, 423 const SrcOp &Op) { 424 return buildInstr(TargetOpcode::G_SEXT, Res, Op); 425 } 426 427 MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res, 428 const SrcOp &Op) { 429 return buildInstr(TargetOpcode::G_ZEXT, Res, Op); 430 } 431 432 unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const { 433 const auto *TLI = getMF().getSubtarget().getTargetLowering(); 434 switch (TLI->getBooleanContents(IsVec, IsFP)) { 435 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 436 return TargetOpcode::G_SEXT; 437 case TargetLoweringBase::ZeroOrOneBooleanContent: 438 return TargetOpcode::G_ZEXT; 439 default: 440 return TargetOpcode::G_ANYEXT; 441 } 442 } 443 444 MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res, 445 const SrcOp &Op, 446 bool IsFP) { 447 unsigned ExtOp = getBoolExtOp(getMRI()->getType(Op.getReg()).isVector(), IsFP); 448 return buildInstr(ExtOp, Res, Op); 449 } 450 451 MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc, 452 const DstOp &Res, 453 const SrcOp &Op) { 454 assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc || 455 TargetOpcode::G_SEXT == ExtOpc) && 456 "Expecting Extending Opc"); 457 assert(Res.getLLTTy(*getMRI()).isScalar() || 458 Res.getLLTTy(*getMRI()).isVector()); 459 assert(Res.getLLTTy(*getMRI()).isScalar() == 460 Op.getLLTTy(*getMRI()).isScalar()); 461 462 unsigned Opcode = TargetOpcode::COPY; 463 if (Res.getLLTTy(*getMRI()).getSizeInBits() > 464 Op.getLLTTy(*getMRI()).getSizeInBits()) 465 Opcode = ExtOpc; 466 else if (Res.getLLTTy(*getMRI()).getSizeInBits() < 467 Op.getLLTTy(*getMRI()).getSizeInBits()) 468 Opcode = TargetOpcode::G_TRUNC; 469 else 470 assert(Res.getLLTTy(*getMRI()) == Op.getLLTTy(*getMRI())); 471 472 return buildInstr(Opcode, Res, Op); 473 } 474 475 MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res, 476 const SrcOp &Op) { 477 return buildExtOrTrunc(TargetOpcode::G_SEXT, Res, Op); 478 } 479 480 MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res, 481 const SrcOp &Op) { 482 return buildExtOrTrunc(TargetOpcode::G_ZEXT, Res, Op); 483 } 484 485 MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res, 486 const SrcOp &Op) { 487 return buildExtOrTrunc(TargetOpcode::G_ANYEXT, Res, Op); 488 } 489 490 MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst, 491 const SrcOp &Src) { 492 LLT SrcTy = Src.getLLTTy(*getMRI()); 493 LLT DstTy = Dst.getLLTTy(*getMRI()); 494 if (SrcTy == DstTy) 495 return buildCopy(Dst, Src); 496 497 unsigned Opcode; 498 if (SrcTy.isPointer() && DstTy.isScalar()) 499 Opcode = TargetOpcode::G_PTRTOINT; 500 else if (DstTy.isPointer() && SrcTy.isScalar()) 501 Opcode = TargetOpcode::G_INTTOPTR; 502 else { 503 assert(!SrcTy.isPointer() && !DstTy.isPointer() && "n G_ADDRCAST yet"); 504 Opcode = TargetOpcode::G_BITCAST; 505 } 506 507 return buildInstr(Opcode, Dst, Src); 508 } 509 510 MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst, 511 const SrcOp &Src, 512 uint64_t Index) { 513 LLT SrcTy = Src.getLLTTy(*getMRI()); 514 LLT DstTy = Dst.getLLTTy(*getMRI()); 515 516 #ifndef NDEBUG 517 assert(SrcTy.isValid() && "invalid operand type"); 518 assert(DstTy.isValid() && "invalid operand type"); 519 assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() && 520 "extracting off end of register"); 521 #endif 522 523 if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) { 524 assert(Index == 0 && "insertion past the end of a register"); 525 return buildCast(Dst, Src); 526 } 527 528 auto Extract = buildInstr(TargetOpcode::G_EXTRACT); 529 Dst.addDefToMIB(*getMRI(), Extract); 530 Src.addSrcToMIB(Extract); 531 Extract.addImm(Index); 532 return Extract; 533 } 534 535 void MachineIRBuilder::buildSequence(Register Res, ArrayRef<Register> Ops, 536 ArrayRef<uint64_t> Indices) { 537 #ifndef NDEBUG 538 assert(Ops.size() == Indices.size() && "incompatible args"); 539 assert(!Ops.empty() && "invalid trivial sequence"); 540 assert(std::is_sorted(Indices.begin(), Indices.end()) && 541 "sequence offsets must be in ascending order"); 542 543 assert(getMRI()->getType(Res).isValid() && "invalid operand type"); 544 for (auto Op : Ops) 545 assert(getMRI()->getType(Op).isValid() && "invalid operand type"); 546 #endif 547 548 LLT ResTy = getMRI()->getType(Res); 549 LLT OpTy = getMRI()->getType(Ops[0]); 550 unsigned OpSize = OpTy.getSizeInBits(); 551 bool MaybeMerge = true; 552 for (unsigned i = 0; i < Ops.size(); ++i) { 553 if (getMRI()->getType(Ops[i]) != OpTy || Indices[i] != i * OpSize) { 554 MaybeMerge = false; 555 break; 556 } 557 } 558 559 if (MaybeMerge && Ops.size() * OpSize == ResTy.getSizeInBits()) { 560 buildMerge(Res, Ops); 561 return; 562 } 563 564 Register ResIn = getMRI()->createGenericVirtualRegister(ResTy); 565 buildUndef(ResIn); 566 567 for (unsigned i = 0; i < Ops.size(); ++i) { 568 Register ResOut = i + 1 == Ops.size() 569 ? Res 570 : getMRI()->createGenericVirtualRegister(ResTy); 571 buildInsert(ResOut, ResIn, Ops[i], Indices[i]); 572 ResIn = ResOut; 573 } 574 } 575 576 MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) { 577 return buildInstr(TargetOpcode::G_IMPLICIT_DEF, {Res}, {}); 578 } 579 580 MachineInstrBuilder MachineIRBuilder::buildMerge(const DstOp &Res, 581 ArrayRef<Register> Ops) { 582 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>, 583 // we need some temporary storage for the DstOp objects. Here we use a 584 // sufficiently large SmallVector to not go through the heap. 585 SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end()); 586 assert(TmpVec.size() > 1); 587 return buildInstr(TargetOpcode::G_MERGE_VALUES, Res, TmpVec); 588 } 589 590 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res, 591 const SrcOp &Op) { 592 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>, 593 // we need some temporary storage for the DstOp objects. Here we use a 594 // sufficiently large SmallVector to not go through the heap. 595 SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end()); 596 assert(TmpVec.size() > 1); 597 return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op); 598 } 599 600 MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res, 601 const SrcOp &Op) { 602 unsigned NumReg = Op.getLLTTy(*getMRI()).getSizeInBits() / Res.getSizeInBits(); 603 SmallVector<Register, 8> TmpVec; 604 for (unsigned I = 0; I != NumReg; ++I) 605 TmpVec.push_back(getMRI()->createGenericVirtualRegister(Res)); 606 return buildUnmerge(TmpVec, Op); 607 } 608 609 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res, 610 const SrcOp &Op) { 611 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>, 612 // we need some temporary storage for the DstOp objects. Here we use a 613 // sufficiently large SmallVector to not go through the heap. 614 SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end()); 615 assert(TmpVec.size() > 1); 616 return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op); 617 } 618 619 MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res, 620 ArrayRef<Register> Ops) { 621 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>, 622 // we need some temporary storage for the DstOp objects. Here we use a 623 // sufficiently large SmallVector to not go through the heap. 624 SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end()); 625 return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); 626 } 627 628 MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res, 629 const SrcOp &Src) { 630 SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src); 631 return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); 632 } 633 634 MachineInstrBuilder 635 MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res, 636 ArrayRef<Register> Ops) { 637 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>, 638 // we need some temporary storage for the DstOp objects. Here we use a 639 // sufficiently large SmallVector to not go through the heap. 640 SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end()); 641 return buildInstr(TargetOpcode::G_BUILD_VECTOR_TRUNC, Res, TmpVec); 642 } 643 644 MachineInstrBuilder 645 MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) { 646 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>, 647 // we need some temporary storage for the DstOp objects. Here we use a 648 // sufficiently large SmallVector to not go through the heap. 649 SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end()); 650 return buildInstr(TargetOpcode::G_CONCAT_VECTORS, Res, TmpVec); 651 } 652 653 MachineInstrBuilder MachineIRBuilder::buildInsert(Register Res, Register Src, 654 Register Op, unsigned Index) { 655 assert(Index + getMRI()->getType(Op).getSizeInBits() <= 656 getMRI()->getType(Res).getSizeInBits() && 657 "insertion past the end of a register"); 658 659 if (getMRI()->getType(Res).getSizeInBits() == 660 getMRI()->getType(Op).getSizeInBits()) { 661 return buildCast(Res, Op); 662 } 663 664 return buildInstr(TargetOpcode::G_INSERT) 665 .addDef(Res) 666 .addUse(Src) 667 .addUse(Op) 668 .addImm(Index); 669 } 670 671 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID, 672 ArrayRef<Register> ResultRegs, 673 bool HasSideEffects) { 674 auto MIB = 675 buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS 676 : TargetOpcode::G_INTRINSIC); 677 for (unsigned ResultReg : ResultRegs) 678 MIB.addDef(ResultReg); 679 MIB.addIntrinsicID(ID); 680 return MIB; 681 } 682 683 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID, 684 ArrayRef<DstOp> Results, 685 bool HasSideEffects) { 686 auto MIB = 687 buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS 688 : TargetOpcode::G_INTRINSIC); 689 for (DstOp Result : Results) 690 Result.addDefToMIB(*getMRI(), MIB); 691 MIB.addIntrinsicID(ID); 692 return MIB; 693 } 694 695 MachineInstrBuilder MachineIRBuilder::buildTrunc(const DstOp &Res, 696 const SrcOp &Op) { 697 return buildInstr(TargetOpcode::G_TRUNC, Res, Op); 698 } 699 700 MachineInstrBuilder MachineIRBuilder::buildFPTrunc(const DstOp &Res, 701 const SrcOp &Op, 702 Optional<unsigned> Flags) { 703 return buildInstr(TargetOpcode::G_FPTRUNC, Res, Op, Flags); 704 } 705 706 MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred, 707 const DstOp &Res, 708 const SrcOp &Op0, 709 const SrcOp &Op1) { 710 return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1}); 711 } 712 713 MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred, 714 const DstOp &Res, 715 const SrcOp &Op0, 716 const SrcOp &Op1, 717 Optional<unsigned> Flags) { 718 719 return buildInstr(TargetOpcode::G_FCMP, Res, {Pred, Op0, Op1}, Flags); 720 } 721 722 MachineInstrBuilder MachineIRBuilder::buildSelect(const DstOp &Res, 723 const SrcOp &Tst, 724 const SrcOp &Op0, 725 const SrcOp &Op1, 726 Optional<unsigned> Flags) { 727 728 return buildInstr(TargetOpcode::G_SELECT, {Res}, {Tst, Op0, Op1}, Flags); 729 } 730 731 MachineInstrBuilder 732 MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, 733 const SrcOp &Elt, const SrcOp &Idx) { 734 return buildInstr(TargetOpcode::G_INSERT_VECTOR_ELT, Res, {Val, Elt, Idx}); 735 } 736 737 MachineInstrBuilder 738 MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, 739 const SrcOp &Idx) { 740 return buildInstr(TargetOpcode::G_EXTRACT_VECTOR_ELT, Res, {Val, Idx}); 741 } 742 743 MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess( 744 Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, 745 Register NewVal, MachineMemOperand &MMO) { 746 #ifndef NDEBUG 747 LLT OldValResTy = getMRI()->getType(OldValRes); 748 LLT SuccessResTy = getMRI()->getType(SuccessRes); 749 LLT AddrTy = getMRI()->getType(Addr); 750 LLT CmpValTy = getMRI()->getType(CmpVal); 751 LLT NewValTy = getMRI()->getType(NewVal); 752 assert(OldValResTy.isScalar() && "invalid operand type"); 753 assert(SuccessResTy.isScalar() && "invalid operand type"); 754 assert(AddrTy.isPointer() && "invalid operand type"); 755 assert(CmpValTy.isValid() && "invalid operand type"); 756 assert(NewValTy.isValid() && "invalid operand type"); 757 assert(OldValResTy == CmpValTy && "type mismatch"); 758 assert(OldValResTy == NewValTy && "type mismatch"); 759 #endif 760 761 return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) 762 .addDef(OldValRes) 763 .addDef(SuccessRes) 764 .addUse(Addr) 765 .addUse(CmpVal) 766 .addUse(NewVal) 767 .addMemOperand(&MMO); 768 } 769 770 MachineInstrBuilder 771 MachineIRBuilder::buildAtomicCmpXchg(Register OldValRes, Register Addr, 772 Register CmpVal, Register NewVal, 773 MachineMemOperand &MMO) { 774 #ifndef NDEBUG 775 LLT OldValResTy = getMRI()->getType(OldValRes); 776 LLT AddrTy = getMRI()->getType(Addr); 777 LLT CmpValTy = getMRI()->getType(CmpVal); 778 LLT NewValTy = getMRI()->getType(NewVal); 779 assert(OldValResTy.isScalar() && "invalid operand type"); 780 assert(AddrTy.isPointer() && "invalid operand type"); 781 assert(CmpValTy.isValid() && "invalid operand type"); 782 assert(NewValTy.isValid() && "invalid operand type"); 783 assert(OldValResTy == CmpValTy && "type mismatch"); 784 assert(OldValResTy == NewValTy && "type mismatch"); 785 #endif 786 787 return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG) 788 .addDef(OldValRes) 789 .addUse(Addr) 790 .addUse(CmpVal) 791 .addUse(NewVal) 792 .addMemOperand(&MMO); 793 } 794 795 MachineInstrBuilder MachineIRBuilder::buildAtomicRMW( 796 unsigned Opcode, const DstOp &OldValRes, 797 const SrcOp &Addr, const SrcOp &Val, 798 MachineMemOperand &MMO) { 799 800 #ifndef NDEBUG 801 LLT OldValResTy = OldValRes.getLLTTy(*getMRI()); 802 LLT AddrTy = Addr.getLLTTy(*getMRI()); 803 LLT ValTy = Val.getLLTTy(*getMRI()); 804 assert(OldValResTy.isScalar() && "invalid operand type"); 805 assert(AddrTy.isPointer() && "invalid operand type"); 806 assert(ValTy.isValid() && "invalid operand type"); 807 assert(OldValResTy == ValTy && "type mismatch"); 808 assert(MMO.isAtomic() && "not atomic mem operand"); 809 #endif 810 811 auto MIB = buildInstr(Opcode); 812 OldValRes.addDefToMIB(*getMRI(), MIB); 813 Addr.addSrcToMIB(MIB); 814 Val.addSrcToMIB(MIB); 815 MIB.addMemOperand(&MMO); 816 return MIB; 817 } 818 819 MachineInstrBuilder 820 MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr, 821 Register Val, MachineMemOperand &MMO) { 822 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val, 823 MMO); 824 } 825 MachineInstrBuilder 826 MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr, 827 Register Val, MachineMemOperand &MMO) { 828 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val, 829 MMO); 830 } 831 MachineInstrBuilder 832 MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr, 833 Register Val, MachineMemOperand &MMO) { 834 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val, 835 MMO); 836 } 837 MachineInstrBuilder 838 MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr, 839 Register Val, MachineMemOperand &MMO) { 840 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val, 841 MMO); 842 } 843 MachineInstrBuilder 844 MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr, 845 Register Val, MachineMemOperand &MMO) { 846 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val, 847 MMO); 848 } 849 MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes, 850 Register Addr, 851 Register Val, 852 MachineMemOperand &MMO) { 853 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val, 854 MMO); 855 } 856 MachineInstrBuilder 857 MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr, 858 Register Val, MachineMemOperand &MMO) { 859 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val, 860 MMO); 861 } 862 MachineInstrBuilder 863 MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr, 864 Register Val, MachineMemOperand &MMO) { 865 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val, 866 MMO); 867 } 868 MachineInstrBuilder 869 MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr, 870 Register Val, MachineMemOperand &MMO) { 871 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val, 872 MMO); 873 } 874 MachineInstrBuilder 875 MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr, 876 Register Val, MachineMemOperand &MMO) { 877 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val, 878 MMO); 879 } 880 MachineInstrBuilder 881 MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr, 882 Register Val, MachineMemOperand &MMO) { 883 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val, 884 MMO); 885 } 886 887 MachineInstrBuilder 888 MachineIRBuilder::buildAtomicRMWFAdd( 889 const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, 890 MachineMemOperand &MMO) { 891 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val, 892 MMO); 893 } 894 895 MachineInstrBuilder 896 MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, 897 MachineMemOperand &MMO) { 898 return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val, 899 MMO); 900 } 901 902 MachineInstrBuilder 903 MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) { 904 return buildInstr(TargetOpcode::G_FENCE) 905 .addImm(Ordering) 906 .addImm(Scope); 907 } 908 909 MachineInstrBuilder 910 MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) { 911 #ifndef NDEBUG 912 assert(getMRI()->getType(Res).isPointer() && "invalid res type"); 913 #endif 914 915 return buildInstr(TargetOpcode::G_BLOCK_ADDR).addDef(Res).addBlockAddress(BA); 916 } 917 918 void MachineIRBuilder::validateTruncExt(const LLT &DstTy, const LLT &SrcTy, 919 bool IsExtend) { 920 #ifndef NDEBUG 921 if (DstTy.isVector()) { 922 assert(SrcTy.isVector() && "mismatched cast between vector and non-vector"); 923 assert(SrcTy.getNumElements() == DstTy.getNumElements() && 924 "different number of elements in a trunc/ext"); 925 } else 926 assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc"); 927 928 if (IsExtend) 929 assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() && 930 "invalid narrowing extend"); 931 else 932 assert(DstTy.getSizeInBits() < SrcTy.getSizeInBits() && 933 "invalid widening trunc"); 934 #endif 935 } 936 937 void MachineIRBuilder::validateSelectOp(const LLT &ResTy, const LLT &TstTy, 938 const LLT &Op0Ty, const LLT &Op1Ty) { 939 #ifndef NDEBUG 940 assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) && 941 "invalid operand type"); 942 assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch"); 943 if (ResTy.isScalar() || ResTy.isPointer()) 944 assert(TstTy.isScalar() && "type mismatch"); 945 else 946 assert((TstTy.isScalar() || 947 (TstTy.isVector() && 948 TstTy.getNumElements() == Op0Ty.getNumElements())) && 949 "type mismatch"); 950 #endif 951 } 952 953 MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc, 954 ArrayRef<DstOp> DstOps, 955 ArrayRef<SrcOp> SrcOps, 956 Optional<unsigned> Flags) { 957 switch (Opc) { 958 default: 959 break; 960 case TargetOpcode::G_SELECT: { 961 assert(DstOps.size() == 1 && "Invalid select"); 962 assert(SrcOps.size() == 3 && "Invalid select"); 963 validateSelectOp( 964 DstOps[0].getLLTTy(*getMRI()), SrcOps[0].getLLTTy(*getMRI()), 965 SrcOps[1].getLLTTy(*getMRI()), SrcOps[2].getLLTTy(*getMRI())); 966 break; 967 } 968 case TargetOpcode::G_ADD: 969 case TargetOpcode::G_AND: 970 case TargetOpcode::G_MUL: 971 case TargetOpcode::G_OR: 972 case TargetOpcode::G_SUB: 973 case TargetOpcode::G_XOR: 974 case TargetOpcode::G_UDIV: 975 case TargetOpcode::G_SDIV: 976 case TargetOpcode::G_UREM: 977 case TargetOpcode::G_SREM: 978 case TargetOpcode::G_SMIN: 979 case TargetOpcode::G_SMAX: 980 case TargetOpcode::G_UMIN: 981 case TargetOpcode::G_UMAX: { 982 // All these are binary ops. 983 assert(DstOps.size() == 1 && "Invalid Dst"); 984 assert(SrcOps.size() == 2 && "Invalid Srcs"); 985 validateBinaryOp(DstOps[0].getLLTTy(*getMRI()), 986 SrcOps[0].getLLTTy(*getMRI()), 987 SrcOps[1].getLLTTy(*getMRI())); 988 break; 989 } 990 case TargetOpcode::G_SHL: 991 case TargetOpcode::G_ASHR: 992 case TargetOpcode::G_LSHR: { 993 assert(DstOps.size() == 1 && "Invalid Dst"); 994 assert(SrcOps.size() == 2 && "Invalid Srcs"); 995 validateShiftOp(DstOps[0].getLLTTy(*getMRI()), 996 SrcOps[0].getLLTTy(*getMRI()), 997 SrcOps[1].getLLTTy(*getMRI())); 998 break; 999 } 1000 case TargetOpcode::G_SEXT: 1001 case TargetOpcode::G_ZEXT: 1002 case TargetOpcode::G_ANYEXT: 1003 assert(DstOps.size() == 1 && "Invalid Dst"); 1004 assert(SrcOps.size() == 1 && "Invalid Srcs"); 1005 validateTruncExt(DstOps[0].getLLTTy(*getMRI()), 1006 SrcOps[0].getLLTTy(*getMRI()), true); 1007 break; 1008 case TargetOpcode::G_TRUNC: 1009 case TargetOpcode::G_FPTRUNC: { 1010 assert(DstOps.size() == 1 && "Invalid Dst"); 1011 assert(SrcOps.size() == 1 && "Invalid Srcs"); 1012 validateTruncExt(DstOps[0].getLLTTy(*getMRI()), 1013 SrcOps[0].getLLTTy(*getMRI()), false); 1014 break; 1015 } 1016 case TargetOpcode::COPY: 1017 assert(DstOps.size() == 1 && "Invalid Dst"); 1018 // If the caller wants to add a subreg source it has to be done separately 1019 // so we may not have any SrcOps at this point yet. 1020 break; 1021 case TargetOpcode::G_FCMP: 1022 case TargetOpcode::G_ICMP: { 1023 assert(DstOps.size() == 1 && "Invalid Dst Operands"); 1024 assert(SrcOps.size() == 3 && "Invalid Src Operands"); 1025 // For F/ICMP, the first src operand is the predicate, followed by 1026 // the two comparands. 1027 assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate && 1028 "Expecting predicate"); 1029 assert([&]() -> bool { 1030 CmpInst::Predicate Pred = SrcOps[0].getPredicate(); 1031 return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred) 1032 : CmpInst::isFPPredicate(Pred); 1033 }() && "Invalid predicate"); 1034 assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) && 1035 "Type mismatch"); 1036 assert([&]() -> bool { 1037 LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI()); 1038 LLT DstTy = DstOps[0].getLLTTy(*getMRI()); 1039 if (Op0Ty.isScalar() || Op0Ty.isPointer()) 1040 return DstTy.isScalar(); 1041 else 1042 return DstTy.isVector() && 1043 DstTy.getNumElements() == Op0Ty.getNumElements(); 1044 }() && "Type Mismatch"); 1045 break; 1046 } 1047 case TargetOpcode::G_UNMERGE_VALUES: { 1048 assert(!DstOps.empty() && "Invalid trivial sequence"); 1049 assert(SrcOps.size() == 1 && "Invalid src for Unmerge"); 1050 assert(std::all_of(DstOps.begin(), DstOps.end(), 1051 [&, this](const DstOp &Op) { 1052 return Op.getLLTTy(*getMRI()) == 1053 DstOps[0].getLLTTy(*getMRI()); 1054 }) && 1055 "type mismatch in output list"); 1056 assert(DstOps.size() * DstOps[0].getLLTTy(*getMRI()).getSizeInBits() == 1057 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() && 1058 "input operands do not cover output register"); 1059 break; 1060 } 1061 case TargetOpcode::G_MERGE_VALUES: { 1062 assert(!SrcOps.empty() && "invalid trivial sequence"); 1063 assert(DstOps.size() == 1 && "Invalid Dst"); 1064 assert(std::all_of(SrcOps.begin(), SrcOps.end(), 1065 [&, this](const SrcOp &Op) { 1066 return Op.getLLTTy(*getMRI()) == 1067 SrcOps[0].getLLTTy(*getMRI()); 1068 }) && 1069 "type mismatch in input list"); 1070 assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() == 1071 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() && 1072 "input operands do not cover output register"); 1073 if (SrcOps.size() == 1) 1074 return buildCast(DstOps[0], SrcOps[0]); 1075 if (DstOps[0].getLLTTy(*getMRI()).isVector()) { 1076 if (SrcOps[0].getLLTTy(*getMRI()).isVector()) 1077 return buildInstr(TargetOpcode::G_CONCAT_VECTORS, DstOps, SrcOps); 1078 return buildInstr(TargetOpcode::G_BUILD_VECTOR, DstOps, SrcOps); 1079 } 1080 break; 1081 } 1082 case TargetOpcode::G_EXTRACT_VECTOR_ELT: { 1083 assert(DstOps.size() == 1 && "Invalid Dst size"); 1084 assert(SrcOps.size() == 2 && "Invalid Src size"); 1085 assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type"); 1086 assert((DstOps[0].getLLTTy(*getMRI()).isScalar() || 1087 DstOps[0].getLLTTy(*getMRI()).isPointer()) && 1088 "Invalid operand type"); 1089 assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type"); 1090 assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() == 1091 DstOps[0].getLLTTy(*getMRI()) && 1092 "Type mismatch"); 1093 break; 1094 } 1095 case TargetOpcode::G_INSERT_VECTOR_ELT: { 1096 assert(DstOps.size() == 1 && "Invalid dst size"); 1097 assert(SrcOps.size() == 3 && "Invalid src size"); 1098 assert(DstOps[0].getLLTTy(*getMRI()).isVector() && 1099 SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type"); 1100 assert(DstOps[0].getLLTTy(*getMRI()).getElementType() == 1101 SrcOps[1].getLLTTy(*getMRI()) && 1102 "Type mismatch"); 1103 assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index"); 1104 assert(DstOps[0].getLLTTy(*getMRI()).getNumElements() == 1105 SrcOps[0].getLLTTy(*getMRI()).getNumElements() && 1106 "Type mismatch"); 1107 break; 1108 } 1109 case TargetOpcode::G_BUILD_VECTOR: { 1110 assert((!SrcOps.empty() || SrcOps.size() < 2) && 1111 "Must have at least 2 operands"); 1112 assert(DstOps.size() == 1 && "Invalid DstOps"); 1113 assert(DstOps[0].getLLTTy(*getMRI()).isVector() && 1114 "Res type must be a vector"); 1115 assert(std::all_of(SrcOps.begin(), SrcOps.end(), 1116 [&, this](const SrcOp &Op) { 1117 return Op.getLLTTy(*getMRI()) == 1118 SrcOps[0].getLLTTy(*getMRI()); 1119 }) && 1120 "type mismatch in input list"); 1121 assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() == 1122 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() && 1123 "input scalars do not exactly cover the output vector register"); 1124 break; 1125 } 1126 case TargetOpcode::G_BUILD_VECTOR_TRUNC: { 1127 assert((!SrcOps.empty() || SrcOps.size() < 2) && 1128 "Must have at least 2 operands"); 1129 assert(DstOps.size() == 1 && "Invalid DstOps"); 1130 assert(DstOps[0].getLLTTy(*getMRI()).isVector() && 1131 "Res type must be a vector"); 1132 assert(std::all_of(SrcOps.begin(), SrcOps.end(), 1133 [&, this](const SrcOp &Op) { 1134 return Op.getLLTTy(*getMRI()) == 1135 SrcOps[0].getLLTTy(*getMRI()); 1136 }) && 1137 "type mismatch in input list"); 1138 if (SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() == 1139 DstOps[0].getLLTTy(*getMRI()).getElementType().getSizeInBits()) 1140 return buildInstr(TargetOpcode::G_BUILD_VECTOR, DstOps, SrcOps); 1141 break; 1142 } 1143 case TargetOpcode::G_CONCAT_VECTORS: { 1144 assert(DstOps.size() == 1 && "Invalid DstOps"); 1145 assert((!SrcOps.empty() || SrcOps.size() < 2) && 1146 "Must have at least 2 operands"); 1147 assert(std::all_of(SrcOps.begin(), SrcOps.end(), 1148 [&, this](const SrcOp &Op) { 1149 return (Op.getLLTTy(*getMRI()).isVector() && 1150 Op.getLLTTy(*getMRI()) == 1151 SrcOps[0].getLLTTy(*getMRI())); 1152 }) && 1153 "type mismatch in input list"); 1154 assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() == 1155 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() && 1156 "input vectors do not exactly cover the output vector register"); 1157 break; 1158 } 1159 case TargetOpcode::G_UADDE: { 1160 assert(DstOps.size() == 2 && "Invalid no of dst operands"); 1161 assert(SrcOps.size() == 3 && "Invalid no of src operands"); 1162 assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand"); 1163 assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) && 1164 (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) && 1165 "Invalid operand"); 1166 assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand"); 1167 assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) && 1168 "type mismatch"); 1169 break; 1170 } 1171 } 1172 1173 auto MIB = buildInstr(Opc); 1174 for (const DstOp &Op : DstOps) 1175 Op.addDefToMIB(*getMRI(), MIB); 1176 for (const SrcOp &Op : SrcOps) 1177 Op.addSrcToMIB(MIB); 1178 if (Flags) 1179 MIB->setFlags(*Flags); 1180 return MIB; 1181 } 1182