1 //===- llvm/CodeGen/GlobalISel/Utils.cpp -------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file This file implements the utility functions used by the GlobalISel 9 /// pipeline. 10 //===----------------------------------------------------------------------===// 11 12 #include "llvm/CodeGen/GlobalISel/Utils.h" 13 #include "llvm/ADT/APFloat.h" 14 #include "llvm/ADT/APInt.h" 15 #include "llvm/Analysis/ValueTracking.h" 16 #include "llvm/CodeGen/CodeGenCommonISel.h" 17 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" 18 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" 19 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 20 #include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h" 21 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" 22 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 23 #include "llvm/CodeGen/MachineInstr.h" 24 #include "llvm/CodeGen/MachineInstrBuilder.h" 25 #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" 26 #include "llvm/CodeGen/MachineRegisterInfo.h" 27 #include "llvm/CodeGen/MachineSizeOpts.h" 28 #include "llvm/CodeGen/RegisterBankInfo.h" 29 #include "llvm/CodeGen/StackProtector.h" 30 #include "llvm/CodeGen/TargetInstrInfo.h" 31 #include "llvm/CodeGen/TargetLowering.h" 32 #include "llvm/CodeGen/TargetOpcodes.h" 33 #include "llvm/CodeGen/TargetPassConfig.h" 34 #include "llvm/CodeGen/TargetRegisterInfo.h" 35 #include "llvm/IR/Constants.h" 36 #include "llvm/Target/TargetMachine.h" 37 #include "llvm/Transforms/Utils/SizeOpts.h" 38 #include <numeric> 39 #include <optional> 40 41 #define DEBUG_TYPE "globalisel-utils" 42 43 using namespace llvm; 44 using namespace MIPatternMatch; 45 46 Register llvm::constrainRegToClass(MachineRegisterInfo &MRI, 47 const TargetInstrInfo &TII, 48 const RegisterBankInfo &RBI, Register Reg, 49 const TargetRegisterClass &RegClass) { 50 if (!RBI.constrainGenericRegister(Reg, RegClass, MRI)) 51 return MRI.createVirtualRegister(&RegClass); 52 53 return Reg; 54 } 55 56 Register llvm::constrainOperandRegClass( 57 const MachineFunction &MF, const TargetRegisterInfo &TRI, 58 MachineRegisterInfo &MRI, const TargetInstrInfo &TII, 59 const RegisterBankInfo &RBI, MachineInstr &InsertPt, 60 const TargetRegisterClass &RegClass, MachineOperand &RegMO) { 61 Register Reg = RegMO.getReg(); 62 // Assume physical registers are properly constrained. 63 assert(Reg.isVirtual() && "PhysReg not implemented"); 64 65 // Save the old register class to check whether 66 // the change notifications will be required. 67 // TODO: A better approach would be to pass 68 // the observers to constrainRegToClass(). 69 auto *OldRegClass = MRI.getRegClassOrNull(Reg); 70 Register ConstrainedReg = constrainRegToClass(MRI, TII, RBI, Reg, RegClass); 71 // If we created a new virtual register because the class is not compatible 72 // then create a copy between the new and the old register. 73 if (ConstrainedReg != Reg) { 74 MachineBasicBlock::iterator InsertIt(&InsertPt); 75 MachineBasicBlock &MBB = *InsertPt.getParent(); 76 // FIXME: The copy needs to have the classes constrained for its operands. 77 // Use operand's regbank to get the class for old register (Reg). 78 if (RegMO.isUse()) { 79 BuildMI(MBB, InsertIt, InsertPt.getDebugLoc(), 80 TII.get(TargetOpcode::COPY), ConstrainedReg) 81 .addReg(Reg); 82 } else { 83 assert(RegMO.isDef() && "Must be a definition"); 84 BuildMI(MBB, std::next(InsertIt), InsertPt.getDebugLoc(), 85 TII.get(TargetOpcode::COPY), Reg) 86 .addReg(ConstrainedReg); 87 } 88 if (GISelChangeObserver *Observer = MF.getObserver()) { 89 Observer->changingInstr(*RegMO.getParent()); 90 } 91 RegMO.setReg(ConstrainedReg); 92 if (GISelChangeObserver *Observer = MF.getObserver()) { 93 Observer->changedInstr(*RegMO.getParent()); 94 } 95 } else if (OldRegClass != MRI.getRegClassOrNull(Reg)) { 96 if (GISelChangeObserver *Observer = MF.getObserver()) { 97 if (!RegMO.isDef()) { 98 MachineInstr *RegDef = MRI.getVRegDef(Reg); 99 Observer->changedInstr(*RegDef); 100 } 101 Observer->changingAllUsesOfReg(MRI, Reg); 102 Observer->finishedChangingAllUsesOfReg(); 103 } 104 } 105 return ConstrainedReg; 106 } 107 108 Register llvm::constrainOperandRegClass( 109 const MachineFunction &MF, const TargetRegisterInfo &TRI, 110 MachineRegisterInfo &MRI, const TargetInstrInfo &TII, 111 const RegisterBankInfo &RBI, MachineInstr &InsertPt, const MCInstrDesc &II, 112 MachineOperand &RegMO, unsigned OpIdx) { 113 Register Reg = RegMO.getReg(); 114 // Assume physical registers are properly constrained. 115 assert(Reg.isVirtual() && "PhysReg not implemented"); 116 117 const TargetRegisterClass *OpRC = TII.getRegClass(II, OpIdx, &TRI, MF); 118 // Some of the target independent instructions, like COPY, may not impose any 119 // register class constraints on some of their operands: If it's a use, we can 120 // skip constraining as the instruction defining the register would constrain 121 // it. 122 123 if (OpRC) { 124 // Obtain the RC from incoming regbank if it is a proper sub-class. Operands 125 // can have multiple regbanks for a superclass that combine different 126 // register types (E.g., AMDGPU's VGPR and AGPR). The regbank ambiguity 127 // resolved by targets during regbankselect should not be overridden. 128 if (const auto *SubRC = TRI.getCommonSubClass( 129 OpRC, TRI.getConstrainedRegClassForOperand(RegMO, MRI))) 130 OpRC = SubRC; 131 132 OpRC = TRI.getAllocatableClass(OpRC); 133 } 134 135 if (!OpRC) { 136 assert((!isTargetSpecificOpcode(II.getOpcode()) || RegMO.isUse()) && 137 "Register class constraint is required unless either the " 138 "instruction is target independent or the operand is a use"); 139 // FIXME: Just bailing out like this here could be not enough, unless we 140 // expect the users of this function to do the right thing for PHIs and 141 // COPY: 142 // v1 = COPY v0 143 // v2 = COPY v1 144 // v1 here may end up not being constrained at all. Please notice that to 145 // reproduce the issue we likely need a destination pattern of a selection 146 // rule producing such extra copies, not just an input GMIR with them as 147 // every existing target using selectImpl handles copies before calling it 148 // and they never reach this function. 149 return Reg; 150 } 151 return constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt, *OpRC, 152 RegMO); 153 } 154 155 bool llvm::constrainSelectedInstRegOperands(MachineInstr &I, 156 const TargetInstrInfo &TII, 157 const TargetRegisterInfo &TRI, 158 const RegisterBankInfo &RBI) { 159 assert(!isPreISelGenericOpcode(I.getOpcode()) && 160 "A selected instruction is expected"); 161 MachineBasicBlock &MBB = *I.getParent(); 162 MachineFunction &MF = *MBB.getParent(); 163 MachineRegisterInfo &MRI = MF.getRegInfo(); 164 165 for (unsigned OpI = 0, OpE = I.getNumExplicitOperands(); OpI != OpE; ++OpI) { 166 MachineOperand &MO = I.getOperand(OpI); 167 168 // There's nothing to be done on non-register operands. 169 if (!MO.isReg()) 170 continue; 171 172 LLVM_DEBUG(dbgs() << "Converting operand: " << MO << '\n'); 173 assert(MO.isReg() && "Unsupported non-reg operand"); 174 175 Register Reg = MO.getReg(); 176 // Physical registers don't need to be constrained. 177 if (Reg.isPhysical()) 178 continue; 179 180 // Register operands with a value of 0 (e.g. predicate operands) don't need 181 // to be constrained. 182 if (Reg == 0) 183 continue; 184 185 // If the operand is a vreg, we should constrain its regclass, and only 186 // insert COPYs if that's impossible. 187 // constrainOperandRegClass does that for us. 188 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, I.getDesc(), MO, OpI); 189 190 // Tie uses to defs as indicated in MCInstrDesc if this hasn't already been 191 // done. 192 if (MO.isUse()) { 193 int DefIdx = I.getDesc().getOperandConstraint(OpI, MCOI::TIED_TO); 194 if (DefIdx != -1 && !I.isRegTiedToUseOperand(DefIdx)) 195 I.tieOperands(DefIdx, OpI); 196 } 197 } 198 return true; 199 } 200 201 bool llvm::canReplaceReg(Register DstReg, Register SrcReg, 202 MachineRegisterInfo &MRI) { 203 // Give up if either DstReg or SrcReg is a physical register. 204 if (DstReg.isPhysical() || SrcReg.isPhysical()) 205 return false; 206 // Give up if the types don't match. 207 if (MRI.getType(DstReg) != MRI.getType(SrcReg)) 208 return false; 209 // Replace if either DstReg has no constraints or the register 210 // constraints match. 211 const auto &DstRBC = MRI.getRegClassOrRegBank(DstReg); 212 if (!DstRBC || DstRBC == MRI.getRegClassOrRegBank(SrcReg)) 213 return true; 214 215 // Otherwise match if the Src is already a regclass that is covered by the Dst 216 // RegBank. 217 return DstRBC.is<const RegisterBank *>() && MRI.getRegClassOrNull(SrcReg) && 218 DstRBC.get<const RegisterBank *>()->covers( 219 *MRI.getRegClassOrNull(SrcReg)); 220 } 221 222 bool llvm::isTriviallyDead(const MachineInstr &MI, 223 const MachineRegisterInfo &MRI) { 224 // FIXME: This logical is mostly duplicated with 225 // DeadMachineInstructionElim::isDead. Why is LOCAL_ESCAPE not considered in 226 // MachineInstr::isLabel? 227 228 // Don't delete frame allocation labels. 229 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) 230 return false; 231 // LIFETIME markers should be preserved even if they seem dead. 232 if (MI.getOpcode() == TargetOpcode::LIFETIME_START || 233 MI.getOpcode() == TargetOpcode::LIFETIME_END) 234 return false; 235 236 // If we can move an instruction, we can remove it. Otherwise, it has 237 // a side-effect of some sort. 238 bool SawStore = false; 239 if (!MI.isSafeToMove(/*AA=*/nullptr, SawStore) && !MI.isPHI()) 240 return false; 241 242 // Instructions without side-effects are dead iff they only define dead vregs. 243 for (const auto &MO : MI.all_defs()) { 244 Register Reg = MO.getReg(); 245 if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg)) 246 return false; 247 } 248 return true; 249 } 250 251 static void reportGISelDiagnostic(DiagnosticSeverity Severity, 252 MachineFunction &MF, 253 const TargetPassConfig &TPC, 254 MachineOptimizationRemarkEmitter &MORE, 255 MachineOptimizationRemarkMissed &R) { 256 bool IsFatal = Severity == DS_Error && 257 TPC.isGlobalISelAbortEnabled(); 258 // Print the function name explicitly if we don't have a debug location (which 259 // makes the diagnostic less useful) or if we're going to emit a raw error. 260 if (!R.getLocation().isValid() || IsFatal) 261 R << (" (in function: " + MF.getName() + ")").str(); 262 263 if (IsFatal) 264 report_fatal_error(Twine(R.getMsg())); 265 else 266 MORE.emit(R); 267 } 268 269 void llvm::reportGISelWarning(MachineFunction &MF, const TargetPassConfig &TPC, 270 MachineOptimizationRemarkEmitter &MORE, 271 MachineOptimizationRemarkMissed &R) { 272 reportGISelDiagnostic(DS_Warning, MF, TPC, MORE, R); 273 } 274 275 void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC, 276 MachineOptimizationRemarkEmitter &MORE, 277 MachineOptimizationRemarkMissed &R) { 278 MF.getProperties().set(MachineFunctionProperties::Property::FailedISel); 279 reportGISelDiagnostic(DS_Error, MF, TPC, MORE, R); 280 } 281 282 void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC, 283 MachineOptimizationRemarkEmitter &MORE, 284 const char *PassName, StringRef Msg, 285 const MachineInstr &MI) { 286 MachineOptimizationRemarkMissed R(PassName, "GISelFailure: ", 287 MI.getDebugLoc(), MI.getParent()); 288 R << Msg; 289 // Printing MI is expensive; only do it if expensive remarks are enabled. 290 if (TPC.isGlobalISelAbortEnabled() || MORE.allowExtraAnalysis(PassName)) 291 R << ": " << ore::MNV("Inst", MI); 292 reportGISelFailure(MF, TPC, MORE, R); 293 } 294 295 std::optional<APInt> llvm::getIConstantVRegVal(Register VReg, 296 const MachineRegisterInfo &MRI) { 297 std::optional<ValueAndVReg> ValAndVReg = getIConstantVRegValWithLookThrough( 298 VReg, MRI, /*LookThroughInstrs*/ false); 299 assert((!ValAndVReg || ValAndVReg->VReg == VReg) && 300 "Value found while looking through instrs"); 301 if (!ValAndVReg) 302 return std::nullopt; 303 return ValAndVReg->Value; 304 } 305 306 std::optional<int64_t> 307 llvm::getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI) { 308 std::optional<APInt> Val = getIConstantVRegVal(VReg, MRI); 309 if (Val && Val->getBitWidth() <= 64) 310 return Val->getSExtValue(); 311 return std::nullopt; 312 } 313 314 namespace { 315 316 // This function is used in many places, and as such, it has some 317 // micro-optimizations to try and make it as fast as it can be. 318 // 319 // - We use template arguments to avoid an indirect call caused by passing a 320 // function_ref/std::function 321 // - GetAPCstValue does not return std::optional<APInt> as that's expensive. 322 // Instead it returns true/false and places the result in a pre-constructed 323 // APInt. 324 // 325 // Please change this function carefully and benchmark your changes. 326 template <bool (*IsConstantOpcode)(const MachineInstr *), 327 bool (*GetAPCstValue)(const MachineInstr *MI, APInt &)> 328 std::optional<ValueAndVReg> 329 getConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, 330 bool LookThroughInstrs = true, 331 bool LookThroughAnyExt = false) { 332 SmallVector<std::pair<unsigned, unsigned>, 4> SeenOpcodes; 333 MachineInstr *MI; 334 335 while ((MI = MRI.getVRegDef(VReg)) && !IsConstantOpcode(MI) && 336 LookThroughInstrs) { 337 switch (MI->getOpcode()) { 338 case TargetOpcode::G_ANYEXT: 339 if (!LookThroughAnyExt) 340 return std::nullopt; 341 [[fallthrough]]; 342 case TargetOpcode::G_TRUNC: 343 case TargetOpcode::G_SEXT: 344 case TargetOpcode::G_ZEXT: 345 SeenOpcodes.push_back(std::make_pair( 346 MI->getOpcode(), 347 MRI.getType(MI->getOperand(0).getReg()).getSizeInBits())); 348 VReg = MI->getOperand(1).getReg(); 349 break; 350 case TargetOpcode::COPY: 351 VReg = MI->getOperand(1).getReg(); 352 if (VReg.isPhysical()) 353 return std::nullopt; 354 break; 355 case TargetOpcode::G_INTTOPTR: 356 VReg = MI->getOperand(1).getReg(); 357 break; 358 default: 359 return std::nullopt; 360 } 361 } 362 if (!MI || !IsConstantOpcode(MI)) 363 return std::nullopt; 364 365 APInt Val; 366 if (!GetAPCstValue(MI, Val)) 367 return std::nullopt; 368 for (auto &Pair : reverse(SeenOpcodes)) { 369 switch (Pair.first) { 370 case TargetOpcode::G_TRUNC: 371 Val = Val.trunc(Pair.second); 372 break; 373 case TargetOpcode::G_ANYEXT: 374 case TargetOpcode::G_SEXT: 375 Val = Val.sext(Pair.second); 376 break; 377 case TargetOpcode::G_ZEXT: 378 Val = Val.zext(Pair.second); 379 break; 380 } 381 } 382 383 return ValueAndVReg{std::move(Val), VReg}; 384 } 385 386 bool isIConstant(const MachineInstr *MI) { 387 if (!MI) 388 return false; 389 return MI->getOpcode() == TargetOpcode::G_CONSTANT; 390 } 391 392 bool isFConstant(const MachineInstr *MI) { 393 if (!MI) 394 return false; 395 return MI->getOpcode() == TargetOpcode::G_FCONSTANT; 396 } 397 398 bool isAnyConstant(const MachineInstr *MI) { 399 if (!MI) 400 return false; 401 unsigned Opc = MI->getOpcode(); 402 return Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::G_FCONSTANT; 403 } 404 405 bool getCImmAsAPInt(const MachineInstr *MI, APInt &Result) { 406 const MachineOperand &CstVal = MI->getOperand(1); 407 if (!CstVal.isCImm()) 408 return false; 409 Result = CstVal.getCImm()->getValue(); 410 return true; 411 } 412 413 bool getCImmOrFPImmAsAPInt(const MachineInstr *MI, APInt &Result) { 414 const MachineOperand &CstVal = MI->getOperand(1); 415 if (CstVal.isCImm()) 416 Result = CstVal.getCImm()->getValue(); 417 else if (CstVal.isFPImm()) 418 Result = CstVal.getFPImm()->getValueAPF().bitcastToAPInt(); 419 else 420 return false; 421 return true; 422 } 423 424 } // end anonymous namespace 425 426 std::optional<ValueAndVReg> llvm::getIConstantVRegValWithLookThrough( 427 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) { 428 return getConstantVRegValWithLookThrough<isIConstant, getCImmAsAPInt>( 429 VReg, MRI, LookThroughInstrs); 430 } 431 432 std::optional<ValueAndVReg> llvm::getAnyConstantVRegValWithLookThrough( 433 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs, 434 bool LookThroughAnyExt) { 435 return getConstantVRegValWithLookThrough<isAnyConstant, 436 getCImmOrFPImmAsAPInt>( 437 VReg, MRI, LookThroughInstrs, LookThroughAnyExt); 438 } 439 440 std::optional<FPValueAndVReg> llvm::getFConstantVRegValWithLookThrough( 441 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) { 442 auto Reg = 443 getConstantVRegValWithLookThrough<isFConstant, getCImmOrFPImmAsAPInt>( 444 VReg, MRI, LookThroughInstrs); 445 if (!Reg) 446 return std::nullopt; 447 return FPValueAndVReg{getConstantFPVRegVal(Reg->VReg, MRI)->getValueAPF(), 448 Reg->VReg}; 449 } 450 451 const ConstantFP * 452 llvm::getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI) { 453 MachineInstr *MI = MRI.getVRegDef(VReg); 454 if (TargetOpcode::G_FCONSTANT != MI->getOpcode()) 455 return nullptr; 456 return MI->getOperand(1).getFPImm(); 457 } 458 459 std::optional<DefinitionAndSourceRegister> 460 llvm::getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI) { 461 Register DefSrcReg = Reg; 462 auto *DefMI = MRI.getVRegDef(Reg); 463 auto DstTy = MRI.getType(DefMI->getOperand(0).getReg()); 464 if (!DstTy.isValid()) 465 return std::nullopt; 466 unsigned Opc = DefMI->getOpcode(); 467 while (Opc == TargetOpcode::COPY || isPreISelGenericOptimizationHint(Opc)) { 468 Register SrcReg = DefMI->getOperand(1).getReg(); 469 auto SrcTy = MRI.getType(SrcReg); 470 if (!SrcTy.isValid()) 471 break; 472 DefMI = MRI.getVRegDef(SrcReg); 473 DefSrcReg = SrcReg; 474 Opc = DefMI->getOpcode(); 475 } 476 return DefinitionAndSourceRegister{DefMI, DefSrcReg}; 477 } 478 479 MachineInstr *llvm::getDefIgnoringCopies(Register Reg, 480 const MachineRegisterInfo &MRI) { 481 std::optional<DefinitionAndSourceRegister> DefSrcReg = 482 getDefSrcRegIgnoringCopies(Reg, MRI); 483 return DefSrcReg ? DefSrcReg->MI : nullptr; 484 } 485 486 Register llvm::getSrcRegIgnoringCopies(Register Reg, 487 const MachineRegisterInfo &MRI) { 488 std::optional<DefinitionAndSourceRegister> DefSrcReg = 489 getDefSrcRegIgnoringCopies(Reg, MRI); 490 return DefSrcReg ? DefSrcReg->Reg : Register(); 491 } 492 493 void llvm::extractParts(Register Reg, LLT Ty, int NumParts, 494 SmallVectorImpl<Register> &VRegs, 495 MachineIRBuilder &MIRBuilder, 496 MachineRegisterInfo &MRI) { 497 for (int i = 0; i < NumParts; ++i) 498 VRegs.push_back(MRI.createGenericVirtualRegister(Ty)); 499 MIRBuilder.buildUnmerge(VRegs, Reg); 500 } 501 502 bool llvm::extractParts(Register Reg, LLT RegTy, LLT MainTy, LLT &LeftoverTy, 503 SmallVectorImpl<Register> &VRegs, 504 SmallVectorImpl<Register> &LeftoverRegs, 505 MachineIRBuilder &MIRBuilder, 506 MachineRegisterInfo &MRI) { 507 assert(!LeftoverTy.isValid() && "this is an out argument"); 508 509 unsigned RegSize = RegTy.getSizeInBits(); 510 unsigned MainSize = MainTy.getSizeInBits(); 511 unsigned NumParts = RegSize / MainSize; 512 unsigned LeftoverSize = RegSize - NumParts * MainSize; 513 514 // Use an unmerge when possible. 515 if (LeftoverSize == 0) { 516 for (unsigned I = 0; I < NumParts; ++I) 517 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy)); 518 MIRBuilder.buildUnmerge(VRegs, Reg); 519 return true; 520 } 521 522 // Try to use unmerge for irregular vector split where possible 523 // For example when splitting a <6 x i32> into <4 x i32> with <2 x i32> 524 // leftover, it becomes: 525 // <2 x i32> %2, <2 x i32>%3, <2 x i32> %4 = G_UNMERGE_VALUE <6 x i32> %1 526 // <4 x i32> %5 = G_CONCAT_VECTOR <2 x i32> %2, <2 x i32> %3 527 if (RegTy.isVector() && MainTy.isVector()) { 528 unsigned RegNumElts = RegTy.getNumElements(); 529 unsigned MainNumElts = MainTy.getNumElements(); 530 unsigned LeftoverNumElts = RegNumElts % MainNumElts; 531 // If can unmerge to LeftoverTy, do it 532 if (MainNumElts % LeftoverNumElts == 0 && 533 RegNumElts % LeftoverNumElts == 0 && 534 RegTy.getScalarSizeInBits() == MainTy.getScalarSizeInBits() && 535 LeftoverNumElts > 1) { 536 LeftoverTy = 537 LLT::fixed_vector(LeftoverNumElts, RegTy.getScalarSizeInBits()); 538 539 // Unmerge the SrcReg to LeftoverTy vectors 540 SmallVector<Register, 4> UnmergeValues; 541 extractParts(Reg, LeftoverTy, RegNumElts / LeftoverNumElts, UnmergeValues, 542 MIRBuilder, MRI); 543 544 // Find how many LeftoverTy makes one MainTy 545 unsigned LeftoverPerMain = MainNumElts / LeftoverNumElts; 546 unsigned NumOfLeftoverVal = 547 ((RegNumElts % MainNumElts) / LeftoverNumElts); 548 549 // Create as many MainTy as possible using unmerged value 550 SmallVector<Register, 4> MergeValues; 551 for (unsigned I = 0; I < UnmergeValues.size() - NumOfLeftoverVal; I++) { 552 MergeValues.push_back(UnmergeValues[I]); 553 if (MergeValues.size() == LeftoverPerMain) { 554 VRegs.push_back( 555 MIRBuilder.buildMergeLikeInstr(MainTy, MergeValues).getReg(0)); 556 MergeValues.clear(); 557 } 558 } 559 // Populate LeftoverRegs with the leftovers 560 for (unsigned I = UnmergeValues.size() - NumOfLeftoverVal; 561 I < UnmergeValues.size(); I++) { 562 LeftoverRegs.push_back(UnmergeValues[I]); 563 } 564 return true; 565 } 566 } 567 // Perform irregular split. Leftover is last element of RegPieces. 568 if (MainTy.isVector()) { 569 SmallVector<Register, 8> RegPieces; 570 extractVectorParts(Reg, MainTy.getNumElements(), RegPieces, MIRBuilder, 571 MRI); 572 for (unsigned i = 0; i < RegPieces.size() - 1; ++i) 573 VRegs.push_back(RegPieces[i]); 574 LeftoverRegs.push_back(RegPieces[RegPieces.size() - 1]); 575 LeftoverTy = MRI.getType(LeftoverRegs[0]); 576 return true; 577 } 578 579 LeftoverTy = LLT::scalar(LeftoverSize); 580 // For irregular sizes, extract the individual parts. 581 for (unsigned I = 0; I != NumParts; ++I) { 582 Register NewReg = MRI.createGenericVirtualRegister(MainTy); 583 VRegs.push_back(NewReg); 584 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I); 585 } 586 587 for (unsigned Offset = MainSize * NumParts; Offset < RegSize; 588 Offset += LeftoverSize) { 589 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy); 590 LeftoverRegs.push_back(NewReg); 591 MIRBuilder.buildExtract(NewReg, Reg, Offset); 592 } 593 594 return true; 595 } 596 597 void llvm::extractVectorParts(Register Reg, unsigned NumElts, 598 SmallVectorImpl<Register> &VRegs, 599 MachineIRBuilder &MIRBuilder, 600 MachineRegisterInfo &MRI) { 601 LLT RegTy = MRI.getType(Reg); 602 assert(RegTy.isVector() && "Expected a vector type"); 603 604 LLT EltTy = RegTy.getElementType(); 605 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElts, EltTy); 606 unsigned RegNumElts = RegTy.getNumElements(); 607 unsigned LeftoverNumElts = RegNumElts % NumElts; 608 unsigned NumNarrowTyPieces = RegNumElts / NumElts; 609 610 // Perfect split without leftover 611 if (LeftoverNumElts == 0) 612 return extractParts(Reg, NarrowTy, NumNarrowTyPieces, VRegs, MIRBuilder, 613 MRI); 614 615 // Irregular split. Provide direct access to all elements for artifact 616 // combiner using unmerge to elements. Then build vectors with NumElts 617 // elements. Remaining element(s) will be (used to build vector) Leftover. 618 SmallVector<Register, 8> Elts; 619 extractParts(Reg, EltTy, RegNumElts, Elts, MIRBuilder, MRI); 620 621 unsigned Offset = 0; 622 // Requested sub-vectors of NarrowTy. 623 for (unsigned i = 0; i < NumNarrowTyPieces; ++i, Offset += NumElts) { 624 ArrayRef<Register> Pieces(&Elts[Offset], NumElts); 625 VRegs.push_back(MIRBuilder.buildMergeLikeInstr(NarrowTy, Pieces).getReg(0)); 626 } 627 628 // Leftover element(s). 629 if (LeftoverNumElts == 1) { 630 VRegs.push_back(Elts[Offset]); 631 } else { 632 LLT LeftoverTy = LLT::fixed_vector(LeftoverNumElts, EltTy); 633 ArrayRef<Register> Pieces(&Elts[Offset], LeftoverNumElts); 634 VRegs.push_back( 635 MIRBuilder.buildMergeLikeInstr(LeftoverTy, Pieces).getReg(0)); 636 } 637 } 638 639 MachineInstr *llvm::getOpcodeDef(unsigned Opcode, Register Reg, 640 const MachineRegisterInfo &MRI) { 641 MachineInstr *DefMI = getDefIgnoringCopies(Reg, MRI); 642 return DefMI && DefMI->getOpcode() == Opcode ? DefMI : nullptr; 643 } 644 645 APFloat llvm::getAPFloatFromSize(double Val, unsigned Size) { 646 if (Size == 32) 647 return APFloat(float(Val)); 648 if (Size == 64) 649 return APFloat(Val); 650 if (Size != 16) 651 llvm_unreachable("Unsupported FPConstant size"); 652 bool Ignored; 653 APFloat APF(Val); 654 APF.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &Ignored); 655 return APF; 656 } 657 658 std::optional<APInt> llvm::ConstantFoldBinOp(unsigned Opcode, 659 const Register Op1, 660 const Register Op2, 661 const MachineRegisterInfo &MRI) { 662 auto MaybeOp2Cst = getAnyConstantVRegValWithLookThrough(Op2, MRI, false); 663 if (!MaybeOp2Cst) 664 return std::nullopt; 665 666 auto MaybeOp1Cst = getAnyConstantVRegValWithLookThrough(Op1, MRI, false); 667 if (!MaybeOp1Cst) 668 return std::nullopt; 669 670 const APInt &C1 = MaybeOp1Cst->Value; 671 const APInt &C2 = MaybeOp2Cst->Value; 672 switch (Opcode) { 673 default: 674 break; 675 case TargetOpcode::G_ADD: 676 return C1 + C2; 677 case TargetOpcode::G_PTR_ADD: 678 // Types can be of different width here. 679 // Result needs to be the same width as C1, so trunc or sext C2. 680 return C1 + C2.sextOrTrunc(C1.getBitWidth()); 681 case TargetOpcode::G_AND: 682 return C1 & C2; 683 case TargetOpcode::G_ASHR: 684 return C1.ashr(C2); 685 case TargetOpcode::G_LSHR: 686 return C1.lshr(C2); 687 case TargetOpcode::G_MUL: 688 return C1 * C2; 689 case TargetOpcode::G_OR: 690 return C1 | C2; 691 case TargetOpcode::G_SHL: 692 return C1 << C2; 693 case TargetOpcode::G_SUB: 694 return C1 - C2; 695 case TargetOpcode::G_XOR: 696 return C1 ^ C2; 697 case TargetOpcode::G_UDIV: 698 if (!C2.getBoolValue()) 699 break; 700 return C1.udiv(C2); 701 case TargetOpcode::G_SDIV: 702 if (!C2.getBoolValue()) 703 break; 704 return C1.sdiv(C2); 705 case TargetOpcode::G_UREM: 706 if (!C2.getBoolValue()) 707 break; 708 return C1.urem(C2); 709 case TargetOpcode::G_SREM: 710 if (!C2.getBoolValue()) 711 break; 712 return C1.srem(C2); 713 case TargetOpcode::G_SMIN: 714 return APIntOps::smin(C1, C2); 715 case TargetOpcode::G_SMAX: 716 return APIntOps::smax(C1, C2); 717 case TargetOpcode::G_UMIN: 718 return APIntOps::umin(C1, C2); 719 case TargetOpcode::G_UMAX: 720 return APIntOps::umax(C1, C2); 721 } 722 723 return std::nullopt; 724 } 725 726 std::optional<APFloat> 727 llvm::ConstantFoldFPBinOp(unsigned Opcode, const Register Op1, 728 const Register Op2, const MachineRegisterInfo &MRI) { 729 const ConstantFP *Op2Cst = getConstantFPVRegVal(Op2, MRI); 730 if (!Op2Cst) 731 return std::nullopt; 732 733 const ConstantFP *Op1Cst = getConstantFPVRegVal(Op1, MRI); 734 if (!Op1Cst) 735 return std::nullopt; 736 737 APFloat C1 = Op1Cst->getValueAPF(); 738 const APFloat &C2 = Op2Cst->getValueAPF(); 739 switch (Opcode) { 740 case TargetOpcode::G_FADD: 741 C1.add(C2, APFloat::rmNearestTiesToEven); 742 return C1; 743 case TargetOpcode::G_FSUB: 744 C1.subtract(C2, APFloat::rmNearestTiesToEven); 745 return C1; 746 case TargetOpcode::G_FMUL: 747 C1.multiply(C2, APFloat::rmNearestTiesToEven); 748 return C1; 749 case TargetOpcode::G_FDIV: 750 C1.divide(C2, APFloat::rmNearestTiesToEven); 751 return C1; 752 case TargetOpcode::G_FREM: 753 C1.mod(C2); 754 return C1; 755 case TargetOpcode::G_FCOPYSIGN: 756 C1.copySign(C2); 757 return C1; 758 case TargetOpcode::G_FMINNUM: 759 return minnum(C1, C2); 760 case TargetOpcode::G_FMAXNUM: 761 return maxnum(C1, C2); 762 case TargetOpcode::G_FMINIMUM: 763 return minimum(C1, C2); 764 case TargetOpcode::G_FMAXIMUM: 765 return maximum(C1, C2); 766 case TargetOpcode::G_FMINNUM_IEEE: 767 case TargetOpcode::G_FMAXNUM_IEEE: 768 // FIXME: These operations were unfortunately named. fminnum/fmaxnum do not 769 // follow the IEEE behavior for signaling nans and follow libm's fmin/fmax, 770 // and currently there isn't a nice wrapper in APFloat for the version with 771 // correct snan handling. 772 break; 773 default: 774 break; 775 } 776 777 return std::nullopt; 778 } 779 780 SmallVector<APInt> 781 llvm::ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, 782 const Register Op2, 783 const MachineRegisterInfo &MRI) { 784 auto *SrcVec2 = getOpcodeDef<GBuildVector>(Op2, MRI); 785 if (!SrcVec2) 786 return SmallVector<APInt>(); 787 788 auto *SrcVec1 = getOpcodeDef<GBuildVector>(Op1, MRI); 789 if (!SrcVec1) 790 return SmallVector<APInt>(); 791 792 SmallVector<APInt> FoldedElements; 793 for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) { 794 auto MaybeCst = ConstantFoldBinOp(Opcode, SrcVec1->getSourceReg(Idx), 795 SrcVec2->getSourceReg(Idx), MRI); 796 if (!MaybeCst) 797 return SmallVector<APInt>(); 798 FoldedElements.push_back(*MaybeCst); 799 } 800 return FoldedElements; 801 } 802 803 bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI, 804 bool SNaN) { 805 const MachineInstr *DefMI = MRI.getVRegDef(Val); 806 if (!DefMI) 807 return false; 808 809 const TargetMachine& TM = DefMI->getMF()->getTarget(); 810 if (DefMI->getFlag(MachineInstr::FmNoNans) || TM.Options.NoNaNsFPMath) 811 return true; 812 813 // If the value is a constant, we can obviously see if it is a NaN or not. 814 if (const ConstantFP *FPVal = getConstantFPVRegVal(Val, MRI)) { 815 return !FPVal->getValueAPF().isNaN() || 816 (SNaN && !FPVal->getValueAPF().isSignaling()); 817 } 818 819 if (DefMI->getOpcode() == TargetOpcode::G_BUILD_VECTOR) { 820 for (const auto &Op : DefMI->uses()) 821 if (!isKnownNeverNaN(Op.getReg(), MRI, SNaN)) 822 return false; 823 return true; 824 } 825 826 switch (DefMI->getOpcode()) { 827 default: 828 break; 829 case TargetOpcode::G_FADD: 830 case TargetOpcode::G_FSUB: 831 case TargetOpcode::G_FMUL: 832 case TargetOpcode::G_FDIV: 833 case TargetOpcode::G_FREM: 834 case TargetOpcode::G_FSIN: 835 case TargetOpcode::G_FCOS: 836 case TargetOpcode::G_FTAN: 837 case TargetOpcode::G_FMA: 838 case TargetOpcode::G_FMAD: 839 if (SNaN) 840 return true; 841 842 // TODO: Need isKnownNeverInfinity 843 return false; 844 case TargetOpcode::G_FMINNUM_IEEE: 845 case TargetOpcode::G_FMAXNUM_IEEE: { 846 if (SNaN) 847 return true; 848 // This can return a NaN if either operand is an sNaN, or if both operands 849 // are NaN. 850 return (isKnownNeverNaN(DefMI->getOperand(1).getReg(), MRI) && 851 isKnownNeverSNaN(DefMI->getOperand(2).getReg(), MRI)) || 852 (isKnownNeverSNaN(DefMI->getOperand(1).getReg(), MRI) && 853 isKnownNeverNaN(DefMI->getOperand(2).getReg(), MRI)); 854 } 855 case TargetOpcode::G_FMINNUM: 856 case TargetOpcode::G_FMAXNUM: { 857 // Only one needs to be known not-nan, since it will be returned if the 858 // other ends up being one. 859 return isKnownNeverNaN(DefMI->getOperand(1).getReg(), MRI, SNaN) || 860 isKnownNeverNaN(DefMI->getOperand(2).getReg(), MRI, SNaN); 861 } 862 } 863 864 if (SNaN) { 865 // FP operations quiet. For now, just handle the ones inserted during 866 // legalization. 867 switch (DefMI->getOpcode()) { 868 case TargetOpcode::G_FPEXT: 869 case TargetOpcode::G_FPTRUNC: 870 case TargetOpcode::G_FCANONICALIZE: 871 return true; 872 default: 873 return false; 874 } 875 } 876 877 return false; 878 } 879 880 Align llvm::inferAlignFromPtrInfo(MachineFunction &MF, 881 const MachinePointerInfo &MPO) { 882 auto PSV = dyn_cast_if_present<const PseudoSourceValue *>(MPO.V); 883 if (auto FSPV = dyn_cast_or_null<FixedStackPseudoSourceValue>(PSV)) { 884 MachineFrameInfo &MFI = MF.getFrameInfo(); 885 return commonAlignment(MFI.getObjectAlign(FSPV->getFrameIndex()), 886 MPO.Offset); 887 } 888 889 if (const Value *V = dyn_cast_if_present<const Value *>(MPO.V)) { 890 const Module *M = MF.getFunction().getParent(); 891 return V->getPointerAlignment(M->getDataLayout()); 892 } 893 894 return Align(1); 895 } 896 897 Register llvm::getFunctionLiveInPhysReg(MachineFunction &MF, 898 const TargetInstrInfo &TII, 899 MCRegister PhysReg, 900 const TargetRegisterClass &RC, 901 const DebugLoc &DL, LLT RegTy) { 902 MachineBasicBlock &EntryMBB = MF.front(); 903 MachineRegisterInfo &MRI = MF.getRegInfo(); 904 Register LiveIn = MRI.getLiveInVirtReg(PhysReg); 905 if (LiveIn) { 906 MachineInstr *Def = MRI.getVRegDef(LiveIn); 907 if (Def) { 908 // FIXME: Should the verifier check this is in the entry block? 909 assert(Def->getParent() == &EntryMBB && "live-in copy not in entry block"); 910 return LiveIn; 911 } 912 913 // It's possible the incoming argument register and copy was added during 914 // lowering, but later deleted due to being/becoming dead. If this happens, 915 // re-insert the copy. 916 } else { 917 // The live in register was not present, so add it. 918 LiveIn = MF.addLiveIn(PhysReg, &RC); 919 if (RegTy.isValid()) 920 MRI.setType(LiveIn, RegTy); 921 } 922 923 BuildMI(EntryMBB, EntryMBB.begin(), DL, TII.get(TargetOpcode::COPY), LiveIn) 924 .addReg(PhysReg); 925 if (!EntryMBB.isLiveIn(PhysReg)) 926 EntryMBB.addLiveIn(PhysReg); 927 return LiveIn; 928 } 929 930 std::optional<APInt> llvm::ConstantFoldExtOp(unsigned Opcode, 931 const Register Op1, uint64_t Imm, 932 const MachineRegisterInfo &MRI) { 933 auto MaybeOp1Cst = getIConstantVRegVal(Op1, MRI); 934 if (MaybeOp1Cst) { 935 switch (Opcode) { 936 default: 937 break; 938 case TargetOpcode::G_SEXT_INREG: { 939 LLT Ty = MRI.getType(Op1); 940 return MaybeOp1Cst->trunc(Imm).sext(Ty.getScalarSizeInBits()); 941 } 942 } 943 } 944 return std::nullopt; 945 } 946 947 std::optional<APInt> llvm::ConstantFoldCastOp(unsigned Opcode, LLT DstTy, 948 const Register Op0, 949 const MachineRegisterInfo &MRI) { 950 std::optional<APInt> Val = getIConstantVRegVal(Op0, MRI); 951 if (!Val) 952 return Val; 953 954 const unsigned DstSize = DstTy.getScalarSizeInBits(); 955 956 switch (Opcode) { 957 case TargetOpcode::G_SEXT: 958 return Val->sext(DstSize); 959 case TargetOpcode::G_ZEXT: 960 case TargetOpcode::G_ANYEXT: 961 // TODO: DAG considers target preference when constant folding any_extend. 962 return Val->zext(DstSize); 963 default: 964 break; 965 } 966 967 llvm_unreachable("unexpected cast opcode to constant fold"); 968 } 969 970 std::optional<APFloat> 971 llvm::ConstantFoldIntToFloat(unsigned Opcode, LLT DstTy, Register Src, 972 const MachineRegisterInfo &MRI) { 973 assert(Opcode == TargetOpcode::G_SITOFP || Opcode == TargetOpcode::G_UITOFP); 974 if (auto MaybeSrcVal = getIConstantVRegVal(Src, MRI)) { 975 APFloat DstVal(getFltSemanticForLLT(DstTy)); 976 DstVal.convertFromAPInt(*MaybeSrcVal, Opcode == TargetOpcode::G_SITOFP, 977 APFloat::rmNearestTiesToEven); 978 return DstVal; 979 } 980 return std::nullopt; 981 } 982 983 std::optional<SmallVector<unsigned>> 984 llvm::ConstantFoldCountZeros(Register Src, const MachineRegisterInfo &MRI, 985 std::function<unsigned(APInt)> CB) { 986 LLT Ty = MRI.getType(Src); 987 SmallVector<unsigned> FoldedCTLZs; 988 auto tryFoldScalar = [&](Register R) -> std::optional<unsigned> { 989 auto MaybeCst = getIConstantVRegVal(R, MRI); 990 if (!MaybeCst) 991 return std::nullopt; 992 return CB(*MaybeCst); 993 }; 994 if (Ty.isVector()) { 995 // Try to constant fold each element. 996 auto *BV = getOpcodeDef<GBuildVector>(Src, MRI); 997 if (!BV) 998 return std::nullopt; 999 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) { 1000 if (auto MaybeFold = tryFoldScalar(BV->getSourceReg(SrcIdx))) { 1001 FoldedCTLZs.emplace_back(*MaybeFold); 1002 continue; 1003 } 1004 return std::nullopt; 1005 } 1006 return FoldedCTLZs; 1007 } 1008 if (auto MaybeCst = tryFoldScalar(Src)) { 1009 FoldedCTLZs.emplace_back(*MaybeCst); 1010 return FoldedCTLZs; 1011 } 1012 return std::nullopt; 1013 } 1014 1015 std::optional<SmallVector<APInt>> 1016 llvm::ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2, 1017 const MachineRegisterInfo &MRI) { 1018 LLT Ty = MRI.getType(Op1); 1019 if (Ty != MRI.getType(Op2)) 1020 return std::nullopt; 1021 1022 auto TryFoldScalar = [&MRI, Pred](Register LHS, 1023 Register RHS) -> std::optional<APInt> { 1024 auto LHSCst = getIConstantVRegVal(LHS, MRI); 1025 auto RHSCst = getIConstantVRegVal(RHS, MRI); 1026 if (!LHSCst || !RHSCst) 1027 return std::nullopt; 1028 1029 switch (Pred) { 1030 case CmpInst::Predicate::ICMP_EQ: 1031 return APInt(/*numBits=*/1, LHSCst->eq(*RHSCst)); 1032 case CmpInst::Predicate::ICMP_NE: 1033 return APInt(/*numBits=*/1, LHSCst->ne(*RHSCst)); 1034 case CmpInst::Predicate::ICMP_UGT: 1035 return APInt(/*numBits=*/1, LHSCst->ugt(*RHSCst)); 1036 case CmpInst::Predicate::ICMP_UGE: 1037 return APInt(/*numBits=*/1, LHSCst->uge(*RHSCst)); 1038 case CmpInst::Predicate::ICMP_ULT: 1039 return APInt(/*numBits=*/1, LHSCst->ult(*RHSCst)); 1040 case CmpInst::Predicate::ICMP_ULE: 1041 return APInt(/*numBits=*/1, LHSCst->ule(*RHSCst)); 1042 case CmpInst::Predicate::ICMP_SGT: 1043 return APInt(/*numBits=*/1, LHSCst->sgt(*RHSCst)); 1044 case CmpInst::Predicate::ICMP_SGE: 1045 return APInt(/*numBits=*/1, LHSCst->sge(*RHSCst)); 1046 case CmpInst::Predicate::ICMP_SLT: 1047 return APInt(/*numBits=*/1, LHSCst->slt(*RHSCst)); 1048 case CmpInst::Predicate::ICMP_SLE: 1049 return APInt(/*numBits=*/1, LHSCst->sle(*RHSCst)); 1050 default: 1051 return std::nullopt; 1052 } 1053 }; 1054 1055 SmallVector<APInt> FoldedICmps; 1056 1057 if (Ty.isVector()) { 1058 // Try to constant fold each element. 1059 auto *BV1 = getOpcodeDef<GBuildVector>(Op1, MRI); 1060 auto *BV2 = getOpcodeDef<GBuildVector>(Op2, MRI); 1061 if (!BV1 || !BV2) 1062 return std::nullopt; 1063 assert(BV1->getNumSources() == BV2->getNumSources() && "Invalid vectors"); 1064 for (unsigned I = 0; I < BV1->getNumSources(); ++I) { 1065 if (auto MaybeFold = 1066 TryFoldScalar(BV1->getSourceReg(I), BV2->getSourceReg(I))) { 1067 FoldedICmps.emplace_back(*MaybeFold); 1068 continue; 1069 } 1070 return std::nullopt; 1071 } 1072 return FoldedICmps; 1073 } 1074 1075 if (auto MaybeCst = TryFoldScalar(Op1, Op2)) { 1076 FoldedICmps.emplace_back(*MaybeCst); 1077 return FoldedICmps; 1078 } 1079 1080 return std::nullopt; 1081 } 1082 1083 bool llvm::isKnownToBeAPowerOfTwo(Register Reg, const MachineRegisterInfo &MRI, 1084 GISelKnownBits *KB) { 1085 std::optional<DefinitionAndSourceRegister> DefSrcReg = 1086 getDefSrcRegIgnoringCopies(Reg, MRI); 1087 if (!DefSrcReg) 1088 return false; 1089 1090 const MachineInstr &MI = *DefSrcReg->MI; 1091 const LLT Ty = MRI.getType(Reg); 1092 1093 switch (MI.getOpcode()) { 1094 case TargetOpcode::G_CONSTANT: { 1095 unsigned BitWidth = Ty.getScalarSizeInBits(); 1096 const ConstantInt *CI = MI.getOperand(1).getCImm(); 1097 return CI->getValue().zextOrTrunc(BitWidth).isPowerOf2(); 1098 } 1099 case TargetOpcode::G_SHL: { 1100 // A left-shift of a constant one will have exactly one bit set because 1101 // shifting the bit off the end is undefined. 1102 1103 // TODO: Constant splat 1104 if (auto ConstLHS = getIConstantVRegVal(MI.getOperand(1).getReg(), MRI)) { 1105 if (*ConstLHS == 1) 1106 return true; 1107 } 1108 1109 break; 1110 } 1111 case TargetOpcode::G_LSHR: { 1112 if (auto ConstLHS = getIConstantVRegVal(MI.getOperand(1).getReg(), MRI)) { 1113 if (ConstLHS->isSignMask()) 1114 return true; 1115 } 1116 1117 break; 1118 } 1119 case TargetOpcode::G_BUILD_VECTOR: { 1120 // TODO: Probably should have a recursion depth guard since you could have 1121 // bitcasted vector elements. 1122 for (const MachineOperand &MO : llvm::drop_begin(MI.operands())) 1123 if (!isKnownToBeAPowerOfTwo(MO.getReg(), MRI, KB)) 1124 return false; 1125 1126 return true; 1127 } 1128 case TargetOpcode::G_BUILD_VECTOR_TRUNC: { 1129 // Only handle constants since we would need to know if number of leading 1130 // zeros is greater than the truncation amount. 1131 const unsigned BitWidth = Ty.getScalarSizeInBits(); 1132 for (const MachineOperand &MO : llvm::drop_begin(MI.operands())) { 1133 auto Const = getIConstantVRegVal(MO.getReg(), MRI); 1134 if (!Const || !Const->zextOrTrunc(BitWidth).isPowerOf2()) 1135 return false; 1136 } 1137 1138 return true; 1139 } 1140 default: 1141 break; 1142 } 1143 1144 if (!KB) 1145 return false; 1146 1147 // More could be done here, though the above checks are enough 1148 // to handle some common cases. 1149 1150 // Fall back to computeKnownBits to catch other known cases. 1151 KnownBits Known = KB->getKnownBits(Reg); 1152 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1); 1153 } 1154 1155 void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) { 1156 AU.addPreserved<StackProtector>(); 1157 } 1158 1159 LLT llvm::getLCMType(LLT OrigTy, LLT TargetTy) { 1160 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits()) 1161 return OrigTy; 1162 1163 if (OrigTy.isVector() && TargetTy.isVector()) { 1164 LLT OrigElt = OrigTy.getElementType(); 1165 LLT TargetElt = TargetTy.getElementType(); 1166 1167 // TODO: The docstring for this function says the intention is to use this 1168 // function to build MERGE/UNMERGE instructions. It won't be the case that 1169 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We 1170 // could implement getLCMType between the two in the future if there was a 1171 // need, but it is not worth it now as this function should not be used in 1172 // that way. 1173 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) || 1174 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) && 1175 "getLCMType not implemented between fixed and scalable vectors."); 1176 1177 if (OrigElt.getSizeInBits() == TargetElt.getSizeInBits()) { 1178 int GCDMinElts = std::gcd(OrigTy.getElementCount().getKnownMinValue(), 1179 TargetTy.getElementCount().getKnownMinValue()); 1180 // Prefer the original element type. 1181 ElementCount Mul = OrigTy.getElementCount().multiplyCoefficientBy( 1182 TargetTy.getElementCount().getKnownMinValue()); 1183 return LLT::vector(Mul.divideCoefficientBy(GCDMinElts), 1184 OrigTy.getElementType()); 1185 } 1186 unsigned LCM = std::lcm(OrigTy.getSizeInBits().getKnownMinValue(), 1187 TargetTy.getSizeInBits().getKnownMinValue()); 1188 return LLT::vector( 1189 ElementCount::get(LCM / OrigElt.getSizeInBits(), OrigTy.isScalable()), 1190 OrigElt); 1191 } 1192 1193 // One type is scalar, one type is vector 1194 if (OrigTy.isVector() || TargetTy.isVector()) { 1195 LLT VecTy = OrigTy.isVector() ? OrigTy : TargetTy; 1196 LLT ScalarTy = OrigTy.isVector() ? TargetTy : OrigTy; 1197 LLT EltTy = VecTy.getElementType(); 1198 LLT OrigEltTy = OrigTy.isVector() ? OrigTy.getElementType() : OrigTy; 1199 1200 // Prefer scalar type from OrigTy. 1201 if (EltTy.getSizeInBits() == ScalarTy.getSizeInBits()) 1202 return LLT::vector(VecTy.getElementCount(), OrigEltTy); 1203 1204 // Different size scalars. Create vector with the same total size. 1205 // LCM will take fixed/scalable from VecTy. 1206 unsigned LCM = std::lcm(EltTy.getSizeInBits().getFixedValue() * 1207 VecTy.getElementCount().getKnownMinValue(), 1208 ScalarTy.getSizeInBits().getFixedValue()); 1209 // Prefer type from OrigTy 1210 return LLT::vector(ElementCount::get(LCM / OrigEltTy.getSizeInBits(), 1211 VecTy.getElementCount().isScalable()), 1212 OrigEltTy); 1213 } 1214 1215 // At this point, both types are scalars of different size 1216 unsigned LCM = std::lcm(OrigTy.getSizeInBits().getFixedValue(), 1217 TargetTy.getSizeInBits().getFixedValue()); 1218 // Preserve pointer types. 1219 if (LCM == OrigTy.getSizeInBits()) 1220 return OrigTy; 1221 if (LCM == TargetTy.getSizeInBits()) 1222 return TargetTy; 1223 return LLT::scalar(LCM); 1224 } 1225 1226 LLT llvm::getCoverTy(LLT OrigTy, LLT TargetTy) { 1227 1228 if ((OrigTy.isScalableVector() && TargetTy.isFixedVector()) || 1229 (OrigTy.isFixedVector() && TargetTy.isScalableVector())) 1230 llvm_unreachable( 1231 "getCoverTy not implemented between fixed and scalable vectors."); 1232 1233 if (!OrigTy.isVector() || !TargetTy.isVector() || OrigTy == TargetTy || 1234 (OrigTy.getScalarSizeInBits() != TargetTy.getScalarSizeInBits())) 1235 return getLCMType(OrigTy, TargetTy); 1236 1237 unsigned OrigTyNumElts = OrigTy.getElementCount().getKnownMinValue(); 1238 unsigned TargetTyNumElts = TargetTy.getElementCount().getKnownMinValue(); 1239 if (OrigTyNumElts % TargetTyNumElts == 0) 1240 return OrigTy; 1241 1242 unsigned NumElts = alignTo(OrigTyNumElts, TargetTyNumElts); 1243 return LLT::scalarOrVector(ElementCount::getFixed(NumElts), 1244 OrigTy.getElementType()); 1245 } 1246 1247 LLT llvm::getGCDType(LLT OrigTy, LLT TargetTy) { 1248 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits()) 1249 return OrigTy; 1250 1251 if (OrigTy.isVector() && TargetTy.isVector()) { 1252 LLT OrigElt = OrigTy.getElementType(); 1253 1254 // TODO: The docstring for this function says the intention is to use this 1255 // function to build MERGE/UNMERGE instructions. It won't be the case that 1256 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We 1257 // could implement getGCDType between the two in the future if there was a 1258 // need, but it is not worth it now as this function should not be used in 1259 // that way. 1260 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) || 1261 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) && 1262 "getGCDType not implemented between fixed and scalable vectors."); 1263 1264 unsigned GCD = std::gcd(OrigTy.getSizeInBits().getKnownMinValue(), 1265 TargetTy.getSizeInBits().getKnownMinValue()); 1266 if (GCD == OrigElt.getSizeInBits()) 1267 return LLT::scalarOrVector(ElementCount::get(1, OrigTy.isScalable()), 1268 OrigElt); 1269 1270 // Cannot produce original element type, but both have vscale in common. 1271 if (GCD < OrigElt.getSizeInBits()) 1272 return LLT::scalarOrVector(ElementCount::get(1, OrigTy.isScalable()), 1273 GCD); 1274 1275 return LLT::vector( 1276 ElementCount::get(GCD / OrigElt.getSizeInBits().getFixedValue(), 1277 OrigTy.isScalable()), 1278 OrigElt); 1279 } 1280 1281 // If one type is vector and the element size matches the scalar size, then 1282 // the gcd is the scalar type. 1283 if (OrigTy.isVector() && 1284 OrigTy.getElementType().getSizeInBits() == TargetTy.getSizeInBits()) 1285 return OrigTy.getElementType(); 1286 if (TargetTy.isVector() && 1287 TargetTy.getElementType().getSizeInBits() == OrigTy.getSizeInBits()) 1288 return OrigTy; 1289 1290 // At this point, both types are either scalars of different type or one is a 1291 // vector and one is a scalar. If both types are scalars, the GCD type is the 1292 // GCD between the two scalar sizes. If one is vector and one is scalar, then 1293 // the GCD type is the GCD between the scalar and the vector element size. 1294 LLT OrigScalar = OrigTy.getScalarType(); 1295 LLT TargetScalar = TargetTy.getScalarType(); 1296 unsigned GCD = std::gcd(OrigScalar.getSizeInBits().getFixedValue(), 1297 TargetScalar.getSizeInBits().getFixedValue()); 1298 return LLT::scalar(GCD); 1299 } 1300 1301 std::optional<int> llvm::getSplatIndex(MachineInstr &MI) { 1302 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR && 1303 "Only G_SHUFFLE_VECTOR can have a splat index!"); 1304 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask(); 1305 auto FirstDefinedIdx = find_if(Mask, [](int Elt) { return Elt >= 0; }); 1306 1307 // If all elements are undefined, this shuffle can be considered a splat. 1308 // Return 0 for better potential for callers to simplify. 1309 if (FirstDefinedIdx == Mask.end()) 1310 return 0; 1311 1312 // Make sure all remaining elements are either undef or the same 1313 // as the first non-undef value. 1314 int SplatValue = *FirstDefinedIdx; 1315 if (any_of(make_range(std::next(FirstDefinedIdx), Mask.end()), 1316 [&SplatValue](int Elt) { return Elt >= 0 && Elt != SplatValue; })) 1317 return std::nullopt; 1318 1319 return SplatValue; 1320 } 1321 1322 static bool isBuildVectorOp(unsigned Opcode) { 1323 return Opcode == TargetOpcode::G_BUILD_VECTOR || 1324 Opcode == TargetOpcode::G_BUILD_VECTOR_TRUNC; 1325 } 1326 1327 namespace { 1328 1329 std::optional<ValueAndVReg> getAnyConstantSplat(Register VReg, 1330 const MachineRegisterInfo &MRI, 1331 bool AllowUndef) { 1332 MachineInstr *MI = getDefIgnoringCopies(VReg, MRI); 1333 if (!MI) 1334 return std::nullopt; 1335 1336 bool isConcatVectorsOp = MI->getOpcode() == TargetOpcode::G_CONCAT_VECTORS; 1337 if (!isBuildVectorOp(MI->getOpcode()) && !isConcatVectorsOp) 1338 return std::nullopt; 1339 1340 std::optional<ValueAndVReg> SplatValAndReg; 1341 for (MachineOperand &Op : MI->uses()) { 1342 Register Element = Op.getReg(); 1343 // If we have a G_CONCAT_VECTOR, we recursively look into the 1344 // vectors that we're concatenating to see if they're splats. 1345 auto ElementValAndReg = 1346 isConcatVectorsOp 1347 ? getAnyConstantSplat(Element, MRI, AllowUndef) 1348 : getAnyConstantVRegValWithLookThrough(Element, MRI, true, true); 1349 1350 // If AllowUndef, treat undef as value that will result in a constant splat. 1351 if (!ElementValAndReg) { 1352 if (AllowUndef && isa<GImplicitDef>(MRI.getVRegDef(Element))) 1353 continue; 1354 return std::nullopt; 1355 } 1356 1357 // Record splat value 1358 if (!SplatValAndReg) 1359 SplatValAndReg = ElementValAndReg; 1360 1361 // Different constant than the one already recorded, not a constant splat. 1362 if (SplatValAndReg->Value != ElementValAndReg->Value) 1363 return std::nullopt; 1364 } 1365 1366 return SplatValAndReg; 1367 } 1368 1369 } // end anonymous namespace 1370 1371 bool llvm::isBuildVectorConstantSplat(const Register Reg, 1372 const MachineRegisterInfo &MRI, 1373 int64_t SplatValue, bool AllowUndef) { 1374 if (auto SplatValAndReg = getAnyConstantSplat(Reg, MRI, AllowUndef)) 1375 return mi_match(SplatValAndReg->VReg, MRI, m_SpecificICst(SplatValue)); 1376 return false; 1377 } 1378 1379 bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI, 1380 const MachineRegisterInfo &MRI, 1381 int64_t SplatValue, bool AllowUndef) { 1382 return isBuildVectorConstantSplat(MI.getOperand(0).getReg(), MRI, SplatValue, 1383 AllowUndef); 1384 } 1385 1386 std::optional<APInt> 1387 llvm::getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI) { 1388 if (auto SplatValAndReg = 1389 getAnyConstantSplat(Reg, MRI, /* AllowUndef */ false)) { 1390 if (std::optional<ValueAndVReg> ValAndVReg = 1391 getIConstantVRegValWithLookThrough(SplatValAndReg->VReg, MRI)) 1392 return ValAndVReg->Value; 1393 } 1394 1395 return std::nullopt; 1396 } 1397 1398 std::optional<APInt> 1399 llvm::getIConstantSplatVal(const MachineInstr &MI, 1400 const MachineRegisterInfo &MRI) { 1401 return getIConstantSplatVal(MI.getOperand(0).getReg(), MRI); 1402 } 1403 1404 std::optional<int64_t> 1405 llvm::getIConstantSplatSExtVal(const Register Reg, 1406 const MachineRegisterInfo &MRI) { 1407 if (auto SplatValAndReg = 1408 getAnyConstantSplat(Reg, MRI, /* AllowUndef */ false)) 1409 return getIConstantVRegSExtVal(SplatValAndReg->VReg, MRI); 1410 return std::nullopt; 1411 } 1412 1413 std::optional<int64_t> 1414 llvm::getIConstantSplatSExtVal(const MachineInstr &MI, 1415 const MachineRegisterInfo &MRI) { 1416 return getIConstantSplatSExtVal(MI.getOperand(0).getReg(), MRI); 1417 } 1418 1419 std::optional<FPValueAndVReg> 1420 llvm::getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI, 1421 bool AllowUndef) { 1422 if (auto SplatValAndReg = getAnyConstantSplat(VReg, MRI, AllowUndef)) 1423 return getFConstantVRegValWithLookThrough(SplatValAndReg->VReg, MRI); 1424 return std::nullopt; 1425 } 1426 1427 bool llvm::isBuildVectorAllZeros(const MachineInstr &MI, 1428 const MachineRegisterInfo &MRI, 1429 bool AllowUndef) { 1430 return isBuildVectorConstantSplat(MI, MRI, 0, AllowUndef); 1431 } 1432 1433 bool llvm::isBuildVectorAllOnes(const MachineInstr &MI, 1434 const MachineRegisterInfo &MRI, 1435 bool AllowUndef) { 1436 return isBuildVectorConstantSplat(MI, MRI, -1, AllowUndef); 1437 } 1438 1439 std::optional<RegOrConstant> 1440 llvm::getVectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI) { 1441 unsigned Opc = MI.getOpcode(); 1442 if (!isBuildVectorOp(Opc)) 1443 return std::nullopt; 1444 if (auto Splat = getIConstantSplatSExtVal(MI, MRI)) 1445 return RegOrConstant(*Splat); 1446 auto Reg = MI.getOperand(1).getReg(); 1447 if (any_of(drop_begin(MI.operands(), 2), 1448 [&Reg](const MachineOperand &Op) { return Op.getReg() != Reg; })) 1449 return std::nullopt; 1450 return RegOrConstant(Reg); 1451 } 1452 1453 static bool isConstantScalar(const MachineInstr &MI, 1454 const MachineRegisterInfo &MRI, 1455 bool AllowFP = true, 1456 bool AllowOpaqueConstants = true) { 1457 switch (MI.getOpcode()) { 1458 case TargetOpcode::G_CONSTANT: 1459 case TargetOpcode::G_IMPLICIT_DEF: 1460 return true; 1461 case TargetOpcode::G_FCONSTANT: 1462 return AllowFP; 1463 case TargetOpcode::G_GLOBAL_VALUE: 1464 case TargetOpcode::G_FRAME_INDEX: 1465 case TargetOpcode::G_BLOCK_ADDR: 1466 case TargetOpcode::G_JUMP_TABLE: 1467 return AllowOpaqueConstants; 1468 default: 1469 return false; 1470 } 1471 } 1472 1473 bool llvm::isConstantOrConstantVector(MachineInstr &MI, 1474 const MachineRegisterInfo &MRI) { 1475 Register Def = MI.getOperand(0).getReg(); 1476 if (auto C = getIConstantVRegValWithLookThrough(Def, MRI)) 1477 return true; 1478 GBuildVector *BV = dyn_cast<GBuildVector>(&MI); 1479 if (!BV) 1480 return false; 1481 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) { 1482 if (getIConstantVRegValWithLookThrough(BV->getSourceReg(SrcIdx), MRI) || 1483 getOpcodeDef<GImplicitDef>(BV->getSourceReg(SrcIdx), MRI)) 1484 continue; 1485 return false; 1486 } 1487 return true; 1488 } 1489 1490 bool llvm::isConstantOrConstantVector(const MachineInstr &MI, 1491 const MachineRegisterInfo &MRI, 1492 bool AllowFP, bool AllowOpaqueConstants) { 1493 if (isConstantScalar(MI, MRI, AllowFP, AllowOpaqueConstants)) 1494 return true; 1495 1496 if (!isBuildVectorOp(MI.getOpcode())) 1497 return false; 1498 1499 const unsigned NumOps = MI.getNumOperands(); 1500 for (unsigned I = 1; I != NumOps; ++I) { 1501 const MachineInstr *ElementDef = MRI.getVRegDef(MI.getOperand(I).getReg()); 1502 if (!isConstantScalar(*ElementDef, MRI, AllowFP, AllowOpaqueConstants)) 1503 return false; 1504 } 1505 1506 return true; 1507 } 1508 1509 std::optional<APInt> 1510 llvm::isConstantOrConstantSplatVector(MachineInstr &MI, 1511 const MachineRegisterInfo &MRI) { 1512 Register Def = MI.getOperand(0).getReg(); 1513 if (auto C = getIConstantVRegValWithLookThrough(Def, MRI)) 1514 return C->Value; 1515 auto MaybeCst = getIConstantSplatSExtVal(MI, MRI); 1516 if (!MaybeCst) 1517 return std::nullopt; 1518 const unsigned ScalarSize = MRI.getType(Def).getScalarSizeInBits(); 1519 return APInt(ScalarSize, *MaybeCst, true); 1520 } 1521 1522 bool llvm::isNullOrNullSplat(const MachineInstr &MI, 1523 const MachineRegisterInfo &MRI, bool AllowUndefs) { 1524 switch (MI.getOpcode()) { 1525 case TargetOpcode::G_IMPLICIT_DEF: 1526 return AllowUndefs; 1527 case TargetOpcode::G_CONSTANT: 1528 return MI.getOperand(1).getCImm()->isNullValue(); 1529 case TargetOpcode::G_FCONSTANT: { 1530 const ConstantFP *FPImm = MI.getOperand(1).getFPImm(); 1531 return FPImm->isZero() && !FPImm->isNegative(); 1532 } 1533 default: 1534 if (!AllowUndefs) // TODO: isBuildVectorAllZeros assumes undef is OK already 1535 return false; 1536 return isBuildVectorAllZeros(MI, MRI); 1537 } 1538 } 1539 1540 bool llvm::isAllOnesOrAllOnesSplat(const MachineInstr &MI, 1541 const MachineRegisterInfo &MRI, 1542 bool AllowUndefs) { 1543 switch (MI.getOpcode()) { 1544 case TargetOpcode::G_IMPLICIT_DEF: 1545 return AllowUndefs; 1546 case TargetOpcode::G_CONSTANT: 1547 return MI.getOperand(1).getCImm()->isAllOnesValue(); 1548 default: 1549 if (!AllowUndefs) // TODO: isBuildVectorAllOnes assumes undef is OK already 1550 return false; 1551 return isBuildVectorAllOnes(MI, MRI); 1552 } 1553 } 1554 1555 bool llvm::matchUnaryPredicate( 1556 const MachineRegisterInfo &MRI, Register Reg, 1557 std::function<bool(const Constant *ConstVal)> Match, bool AllowUndefs) { 1558 1559 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI); 1560 if (AllowUndefs && Def->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1561 return Match(nullptr); 1562 1563 // TODO: Also handle fconstant 1564 if (Def->getOpcode() == TargetOpcode::G_CONSTANT) 1565 return Match(Def->getOperand(1).getCImm()); 1566 1567 if (Def->getOpcode() != TargetOpcode::G_BUILD_VECTOR) 1568 return false; 1569 1570 for (unsigned I = 1, E = Def->getNumOperands(); I != E; ++I) { 1571 Register SrcElt = Def->getOperand(I).getReg(); 1572 const MachineInstr *SrcDef = getDefIgnoringCopies(SrcElt, MRI); 1573 if (AllowUndefs && SrcDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) { 1574 if (!Match(nullptr)) 1575 return false; 1576 continue; 1577 } 1578 1579 if (SrcDef->getOpcode() != TargetOpcode::G_CONSTANT || 1580 !Match(SrcDef->getOperand(1).getCImm())) 1581 return false; 1582 } 1583 1584 return true; 1585 } 1586 1587 bool llvm::isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector, 1588 bool IsFP) { 1589 switch (TLI.getBooleanContents(IsVector, IsFP)) { 1590 case TargetLowering::UndefinedBooleanContent: 1591 return Val & 0x1; 1592 case TargetLowering::ZeroOrOneBooleanContent: 1593 return Val == 1; 1594 case TargetLowering::ZeroOrNegativeOneBooleanContent: 1595 return Val == -1; 1596 } 1597 llvm_unreachable("Invalid boolean contents"); 1598 } 1599 1600 bool llvm::isConstFalseVal(const TargetLowering &TLI, int64_t Val, 1601 bool IsVector, bool IsFP) { 1602 switch (TLI.getBooleanContents(IsVector, IsFP)) { 1603 case TargetLowering::UndefinedBooleanContent: 1604 return ~Val & 0x1; 1605 case TargetLowering::ZeroOrOneBooleanContent: 1606 case TargetLowering::ZeroOrNegativeOneBooleanContent: 1607 return Val == 0; 1608 } 1609 llvm_unreachable("Invalid boolean contents"); 1610 } 1611 1612 int64_t llvm::getICmpTrueVal(const TargetLowering &TLI, bool IsVector, 1613 bool IsFP) { 1614 switch (TLI.getBooleanContents(IsVector, IsFP)) { 1615 case TargetLowering::UndefinedBooleanContent: 1616 case TargetLowering::ZeroOrOneBooleanContent: 1617 return 1; 1618 case TargetLowering::ZeroOrNegativeOneBooleanContent: 1619 return -1; 1620 } 1621 llvm_unreachable("Invalid boolean contents"); 1622 } 1623 1624 bool llvm::shouldOptForSize(const MachineBasicBlock &MBB, 1625 ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) { 1626 const auto &F = MBB.getParent()->getFunction(); 1627 return F.hasOptSize() || F.hasMinSize() || 1628 llvm::shouldOptimizeForSize(MBB.getBasicBlock(), PSI, BFI); 1629 } 1630 1631 void llvm::saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI, 1632 LostDebugLocObserver *LocObserver, 1633 SmallInstListTy &DeadInstChain) { 1634 for (MachineOperand &Op : MI.uses()) { 1635 if (Op.isReg() && Op.getReg().isVirtual()) 1636 DeadInstChain.insert(MRI.getVRegDef(Op.getReg())); 1637 } 1638 LLVM_DEBUG(dbgs() << MI << "Is dead; erasing.\n"); 1639 DeadInstChain.remove(&MI); 1640 MI.eraseFromParent(); 1641 if (LocObserver) 1642 LocObserver->checkpoint(false); 1643 } 1644 1645 void llvm::eraseInstrs(ArrayRef<MachineInstr *> DeadInstrs, 1646 MachineRegisterInfo &MRI, 1647 LostDebugLocObserver *LocObserver) { 1648 SmallInstListTy DeadInstChain; 1649 for (MachineInstr *MI : DeadInstrs) 1650 saveUsesAndErase(*MI, MRI, LocObserver, DeadInstChain); 1651 1652 while (!DeadInstChain.empty()) { 1653 MachineInstr *Inst = DeadInstChain.pop_back_val(); 1654 if (!isTriviallyDead(*Inst, MRI)) 1655 continue; 1656 saveUsesAndErase(*Inst, MRI, LocObserver, DeadInstChain); 1657 } 1658 } 1659 1660 void llvm::eraseInstr(MachineInstr &MI, MachineRegisterInfo &MRI, 1661 LostDebugLocObserver *LocObserver) { 1662 return eraseInstrs({&MI}, MRI, LocObserver); 1663 } 1664 1665 void llvm::salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI) { 1666 for (auto &Def : MI.defs()) { 1667 assert(Def.isReg() && "Must be a reg"); 1668 1669 SmallVector<MachineOperand *, 16> DbgUsers; 1670 for (auto &MOUse : MRI.use_operands(Def.getReg())) { 1671 MachineInstr *DbgValue = MOUse.getParent(); 1672 // Ignore partially formed DBG_VALUEs. 1673 if (DbgValue->isNonListDebugValue() && DbgValue->getNumOperands() == 4) { 1674 DbgUsers.push_back(&MOUse); 1675 } 1676 } 1677 1678 if (!DbgUsers.empty()) { 1679 salvageDebugInfoForDbgValue(MRI, MI, DbgUsers); 1680 } 1681 } 1682 } 1683 1684 bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) { 1685 switch (Opc) { 1686 case TargetOpcode::G_FABS: 1687 case TargetOpcode::G_FADD: 1688 case TargetOpcode::G_FCANONICALIZE: 1689 case TargetOpcode::G_FCEIL: 1690 case TargetOpcode::G_FCONSTANT: 1691 case TargetOpcode::G_FCOPYSIGN: 1692 case TargetOpcode::G_FCOS: 1693 case TargetOpcode::G_FDIV: 1694 case TargetOpcode::G_FEXP2: 1695 case TargetOpcode::G_FEXP: 1696 case TargetOpcode::G_FFLOOR: 1697 case TargetOpcode::G_FLOG10: 1698 case TargetOpcode::G_FLOG2: 1699 case TargetOpcode::G_FLOG: 1700 case TargetOpcode::G_FMA: 1701 case TargetOpcode::G_FMAD: 1702 case TargetOpcode::G_FMAXIMUM: 1703 case TargetOpcode::G_FMAXNUM: 1704 case TargetOpcode::G_FMAXNUM_IEEE: 1705 case TargetOpcode::G_FMINIMUM: 1706 case TargetOpcode::G_FMINNUM: 1707 case TargetOpcode::G_FMINNUM_IEEE: 1708 case TargetOpcode::G_FMUL: 1709 case TargetOpcode::G_FNEARBYINT: 1710 case TargetOpcode::G_FNEG: 1711 case TargetOpcode::G_FPEXT: 1712 case TargetOpcode::G_FPOW: 1713 case TargetOpcode::G_FPTRUNC: 1714 case TargetOpcode::G_FREM: 1715 case TargetOpcode::G_FRINT: 1716 case TargetOpcode::G_FSIN: 1717 case TargetOpcode::G_FTAN: 1718 case TargetOpcode::G_FSQRT: 1719 case TargetOpcode::G_FSUB: 1720 case TargetOpcode::G_INTRINSIC_ROUND: 1721 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 1722 case TargetOpcode::G_INTRINSIC_TRUNC: 1723 return true; 1724 default: 1725 return false; 1726 } 1727 } 1728 1729 /// Shifts return poison if shiftwidth is larger than the bitwidth. 1730 static bool shiftAmountKnownInRange(Register ShiftAmount, 1731 const MachineRegisterInfo &MRI) { 1732 LLT Ty = MRI.getType(ShiftAmount); 1733 1734 if (Ty.isScalableVector()) 1735 return false; // Can't tell, just return false to be safe 1736 1737 if (Ty.isScalar()) { 1738 std::optional<ValueAndVReg> Val = 1739 getIConstantVRegValWithLookThrough(ShiftAmount, MRI); 1740 if (!Val) 1741 return false; 1742 return Val->Value.ult(Ty.getScalarSizeInBits()); 1743 } 1744 1745 GBuildVector *BV = getOpcodeDef<GBuildVector>(ShiftAmount, MRI); 1746 if (!BV) 1747 return false; 1748 1749 unsigned Sources = BV->getNumSources(); 1750 for (unsigned I = 0; I < Sources; ++I) { 1751 std::optional<ValueAndVReg> Val = 1752 getIConstantVRegValWithLookThrough(BV->getSourceReg(I), MRI); 1753 if (!Val) 1754 return false; 1755 if (!Val->Value.ult(Ty.getScalarSizeInBits())) 1756 return false; 1757 } 1758 1759 return true; 1760 } 1761 1762 namespace { 1763 enum class UndefPoisonKind { 1764 PoisonOnly = (1 << 0), 1765 UndefOnly = (1 << 1), 1766 UndefOrPoison = PoisonOnly | UndefOnly, 1767 }; 1768 } 1769 1770 static bool includesPoison(UndefPoisonKind Kind) { 1771 return (unsigned(Kind) & unsigned(UndefPoisonKind::PoisonOnly)) != 0; 1772 } 1773 1774 static bool includesUndef(UndefPoisonKind Kind) { 1775 return (unsigned(Kind) & unsigned(UndefPoisonKind::UndefOnly)) != 0; 1776 } 1777 1778 static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI, 1779 bool ConsiderFlagsAndMetadata, 1780 UndefPoisonKind Kind) { 1781 MachineInstr *RegDef = MRI.getVRegDef(Reg); 1782 1783 if (ConsiderFlagsAndMetadata && includesPoison(Kind)) 1784 if (auto *GMI = dyn_cast<GenericMachineInstr>(RegDef)) 1785 if (GMI->hasPoisonGeneratingFlags()) 1786 return true; 1787 1788 // Check whether opcode is a poison/undef-generating operation. 1789 switch (RegDef->getOpcode()) { 1790 case TargetOpcode::G_FREEZE: 1791 case TargetOpcode::G_BUILD_VECTOR: 1792 case TargetOpcode::G_CONSTANT_FOLD_BARRIER: 1793 return false; 1794 case TargetOpcode::G_SHL: 1795 case TargetOpcode::G_ASHR: 1796 case TargetOpcode::G_LSHR: 1797 return includesPoison(Kind) && 1798 !shiftAmountKnownInRange(RegDef->getOperand(2).getReg(), MRI); 1799 case TargetOpcode::G_FPTOSI: 1800 case TargetOpcode::G_FPTOUI: 1801 // fptosi/ui yields poison if the resulting value does not fit in the 1802 // destination type. 1803 return true; 1804 case TargetOpcode::G_CTLZ: 1805 case TargetOpcode::G_CTTZ: 1806 case TargetOpcode::G_ABS: 1807 case TargetOpcode::G_CTPOP: 1808 case TargetOpcode::G_BSWAP: 1809 case TargetOpcode::G_BITREVERSE: 1810 case TargetOpcode::G_FSHL: 1811 case TargetOpcode::G_FSHR: 1812 case TargetOpcode::G_SMAX: 1813 case TargetOpcode::G_SMIN: 1814 case TargetOpcode::G_UMAX: 1815 case TargetOpcode::G_UMIN: 1816 case TargetOpcode::G_PTRMASK: 1817 case TargetOpcode::G_SADDO: 1818 case TargetOpcode::G_SSUBO: 1819 case TargetOpcode::G_UADDO: 1820 case TargetOpcode::G_USUBO: 1821 case TargetOpcode::G_SMULO: 1822 case TargetOpcode::G_UMULO: 1823 case TargetOpcode::G_SADDSAT: 1824 case TargetOpcode::G_UADDSAT: 1825 case TargetOpcode::G_SSUBSAT: 1826 case TargetOpcode::G_USUBSAT: 1827 return false; 1828 case TargetOpcode::G_SSHLSAT: 1829 case TargetOpcode::G_USHLSAT: 1830 return includesPoison(Kind) && 1831 !shiftAmountKnownInRange(RegDef->getOperand(2).getReg(), MRI); 1832 default: 1833 return !isa<GCastOp>(RegDef) && !isa<GBinOp>(RegDef); 1834 } 1835 } 1836 1837 static bool isGuaranteedNotToBeUndefOrPoison(Register Reg, 1838 const MachineRegisterInfo &MRI, 1839 unsigned Depth, 1840 UndefPoisonKind Kind) { 1841 if (Depth >= MaxAnalysisRecursionDepth) 1842 return false; 1843 1844 MachineInstr *RegDef = MRI.getVRegDef(Reg); 1845 1846 switch (RegDef->getOpcode()) { 1847 case TargetOpcode::G_FREEZE: 1848 return true; 1849 case TargetOpcode::G_IMPLICIT_DEF: 1850 return !includesUndef(Kind); 1851 case TargetOpcode::G_CONSTANT: 1852 case TargetOpcode::G_FCONSTANT: 1853 return true; 1854 case TargetOpcode::G_BUILD_VECTOR: { 1855 GBuildVector *BV = cast<GBuildVector>(RegDef); 1856 unsigned NumSources = BV->getNumSources(); 1857 for (unsigned I = 0; I < NumSources; ++I) 1858 if (!::isGuaranteedNotToBeUndefOrPoison(BV->getSourceReg(I), MRI, 1859 Depth + 1, Kind)) 1860 return false; 1861 return true; 1862 } 1863 default: { 1864 auto MOCheck = [&](const MachineOperand &MO) { 1865 if (!MO.isReg()) 1866 return true; 1867 return ::isGuaranteedNotToBeUndefOrPoison(MO.getReg(), MRI, Depth + 1, 1868 Kind); 1869 }; 1870 return !::canCreateUndefOrPoison(Reg, MRI, 1871 /*ConsiderFlagsAndMetadata=*/true, Kind) && 1872 all_of(RegDef->uses(), MOCheck); 1873 } 1874 } 1875 } 1876 1877 bool llvm::canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI, 1878 bool ConsiderFlagsAndMetadata) { 1879 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata, 1880 UndefPoisonKind::UndefOrPoison); 1881 } 1882 1883 bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI, 1884 bool ConsiderFlagsAndMetadata = true) { 1885 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata, 1886 UndefPoisonKind::PoisonOnly); 1887 } 1888 1889 bool llvm::isGuaranteedNotToBeUndefOrPoison(Register Reg, 1890 const MachineRegisterInfo &MRI, 1891 unsigned Depth) { 1892 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth, 1893 UndefPoisonKind::UndefOrPoison); 1894 } 1895 1896 bool llvm::isGuaranteedNotToBePoison(Register Reg, 1897 const MachineRegisterInfo &MRI, 1898 unsigned Depth) { 1899 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth, 1900 UndefPoisonKind::PoisonOnly); 1901 } 1902 1903 bool llvm::isGuaranteedNotToBeUndef(Register Reg, 1904 const MachineRegisterInfo &MRI, 1905 unsigned Depth) { 1906 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth, 1907 UndefPoisonKind::UndefOnly); 1908 } 1909 1910 Type *llvm::getTypeForLLT(LLT Ty, LLVMContext &C) { 1911 if (Ty.isVector()) 1912 return VectorType::get(IntegerType::get(C, Ty.getScalarSizeInBits()), 1913 Ty.getElementCount()); 1914 return IntegerType::get(C, Ty.getSizeInBits()); 1915 } 1916