1 //===-- RISCVInstrInfo.cpp - RISCV Instruction Information ------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the RISCV implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "RISCVInstrInfo.h" 14 #include "MCTargetDesc/RISCVMatInt.h" 15 #include "RISCV.h" 16 #include "RISCVMachineFunctionInfo.h" 17 #include "RISCVSubtarget.h" 18 #include "RISCVTargetMachine.h" 19 #include "llvm/ADT/STLExtras.h" 20 #include "llvm/ADT/SmallVector.h" 21 #include "llvm/Analysis/MemoryLocation.h" 22 #include "llvm/CodeGen/LiveIntervals.h" 23 #include "llvm/CodeGen/LiveVariables.h" 24 #include "llvm/CodeGen/MachineFunctionPass.h" 25 #include "llvm/CodeGen/MachineInstrBuilder.h" 26 #include "llvm/CodeGen/MachineRegisterInfo.h" 27 #include "llvm/CodeGen/RegisterScavenging.h" 28 #include "llvm/MC/MCInstBuilder.h" 29 #include "llvm/MC/TargetRegistry.h" 30 #include "llvm/Support/ErrorHandling.h" 31 32 using namespace llvm; 33 34 #define GEN_CHECK_COMPRESS_INSTR 35 #include "RISCVGenCompressInstEmitter.inc" 36 37 #define GET_INSTRINFO_CTOR_DTOR 38 #include "RISCVGenInstrInfo.inc" 39 40 static cl::opt<bool> PreferWholeRegisterMove( 41 "riscv-prefer-whole-register-move", cl::init(false), cl::Hidden, 42 cl::desc("Prefer whole register move for vector registers.")); 43 44 namespace llvm { 45 namespace RISCVVPseudosTable { 46 47 using namespace RISCV; 48 49 #define GET_RISCVVPseudosTable_IMPL 50 #include "RISCVGenSearchableTables.inc" 51 52 } // namespace RISCVVPseudosTable 53 } // namespace llvm 54 55 RISCVInstrInfo::RISCVInstrInfo(RISCVSubtarget &STI) 56 : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP), 57 STI(STI) {} 58 59 MCInst RISCVInstrInfo::getNop() const { 60 if (STI.getFeatureBits()[RISCV::FeatureStdExtC]) 61 return MCInstBuilder(RISCV::C_NOP); 62 return MCInstBuilder(RISCV::ADDI) 63 .addReg(RISCV::X0) 64 .addReg(RISCV::X0) 65 .addImm(0); 66 } 67 68 unsigned RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 69 int &FrameIndex) const { 70 switch (MI.getOpcode()) { 71 default: 72 return 0; 73 case RISCV::LB: 74 case RISCV::LBU: 75 case RISCV::LH: 76 case RISCV::LHU: 77 case RISCV::FLH: 78 case RISCV::LW: 79 case RISCV::FLW: 80 case RISCV::LWU: 81 case RISCV::LD: 82 case RISCV::FLD: 83 break; 84 } 85 86 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() && 87 MI.getOperand(2).getImm() == 0) { 88 FrameIndex = MI.getOperand(1).getIndex(); 89 return MI.getOperand(0).getReg(); 90 } 91 92 return 0; 93 } 94 95 unsigned RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 96 int &FrameIndex) const { 97 switch (MI.getOpcode()) { 98 default: 99 return 0; 100 case RISCV::SB: 101 case RISCV::SH: 102 case RISCV::SW: 103 case RISCV::FSH: 104 case RISCV::FSW: 105 case RISCV::SD: 106 case RISCV::FSD: 107 break; 108 } 109 110 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() && 111 MI.getOperand(2).getImm() == 0) { 112 FrameIndex = MI.getOperand(1).getIndex(); 113 return MI.getOperand(0).getReg(); 114 } 115 116 return 0; 117 } 118 119 static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg, 120 unsigned NumRegs) { 121 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs; 122 } 123 124 static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI, 125 const MachineBasicBlock &MBB, 126 MachineBasicBlock::const_iterator MBBI, 127 MachineBasicBlock::const_iterator &DefMBBI, 128 RISCVII::VLMUL LMul) { 129 if (PreferWholeRegisterMove) 130 return false; 131 132 assert(MBBI->getOpcode() == TargetOpcode::COPY && 133 "Unexpected COPY instruction."); 134 Register SrcReg = MBBI->getOperand(1).getReg(); 135 const TargetRegisterInfo *TRI = STI.getRegisterInfo(); 136 137 bool FoundDef = false; 138 bool FirstVSetVLI = false; 139 unsigned FirstSEW = 0; 140 while (MBBI != MBB.begin()) { 141 --MBBI; 142 if (MBBI->isMetaInstruction()) 143 continue; 144 145 if (MBBI->getOpcode() == RISCV::PseudoVSETVLI || 146 MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 || 147 MBBI->getOpcode() == RISCV::PseudoVSETIVLI) { 148 // There is a vsetvli between COPY and source define instruction. 149 // vy = def_vop ... (producing instruction) 150 // ... 151 // vsetvli 152 // ... 153 // vx = COPY vy 154 if (!FoundDef) { 155 if (!FirstVSetVLI) { 156 FirstVSetVLI = true; 157 unsigned FirstVType = MBBI->getOperand(2).getImm(); 158 RISCVII::VLMUL FirstLMul = RISCVVType::getVLMUL(FirstVType); 159 FirstSEW = RISCVVType::getSEW(FirstVType); 160 // The first encountered vsetvli must have the same lmul as the 161 // register class of COPY. 162 if (FirstLMul != LMul) 163 return false; 164 } 165 // Only permit `vsetvli x0, x0, vtype` between COPY and the source 166 // define instruction. 167 if (MBBI->getOperand(0).getReg() != RISCV::X0) 168 return false; 169 if (MBBI->getOperand(1).isImm()) 170 return false; 171 if (MBBI->getOperand(1).getReg() != RISCV::X0) 172 return false; 173 continue; 174 } 175 176 // MBBI is the first vsetvli before the producing instruction. 177 unsigned VType = MBBI->getOperand(2).getImm(); 178 // If there is a vsetvli between COPY and the producing instruction. 179 if (FirstVSetVLI) { 180 // If SEW is different, return false. 181 if (RISCVVType::getSEW(VType) != FirstSEW) 182 return false; 183 } 184 185 // If the vsetvli is tail undisturbed, keep the whole register move. 186 if (!RISCVVType::isTailAgnostic(VType)) 187 return false; 188 189 // The checking is conservative. We only have register classes for 190 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v 191 // for fractional LMUL operations. However, we could not use the vsetvli 192 // lmul for widening operations. The result of widening operation is 193 // 2 x LMUL. 194 return LMul == RISCVVType::getVLMUL(VType); 195 } else if (MBBI->isInlineAsm() || MBBI->isCall()) { 196 return false; 197 } else if (MBBI->getNumDefs()) { 198 // Check all the instructions which will change VL. 199 // For example, vleff has implicit def VL. 200 if (MBBI->modifiesRegister(RISCV::VL)) 201 return false; 202 203 // Go through all defined operands, including implicit defines. 204 for (const MachineOperand &MO : MBBI->operands()) { 205 if (!MO.isReg() || !MO.isDef()) 206 continue; 207 if (!FoundDef && TRI->isSubRegisterEq(MO.getReg(), SrcReg)) { 208 // We only permit the source of COPY has the same LMUL as the defined 209 // operand. 210 // There are cases we need to keep the whole register copy if the LMUL 211 // is different. 212 // For example, 213 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m 214 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2 215 // # The COPY may be created by vlmul_trunc intrinsic. 216 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4 217 // 218 // After widening, the valid value will be 4 x e32 elements. If we 219 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements. 220 // FIXME: The COPY of subregister of Zvlsseg register will not be able 221 // to convert to vmv.v.[v|i] under the constraint. 222 if (MO.getReg() != SrcReg) 223 return false; 224 225 // In widening reduction instructions with LMUL_1 input vector case, 226 // only checking the LMUL is insufficient due to reduction result is 227 // always LMUL_1. 228 // For example, 229 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu 230 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27 231 // $v26 = COPY killed renamable $v8 232 // After widening, The valid value will be 1 x e16 elements. If we 233 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements. 234 uint64_t TSFlags = MBBI->getDesc().TSFlags; 235 if (RISCVII::isRVVWideningReduction(TSFlags)) 236 return false; 237 238 // Found the definition. 239 FoundDef = true; 240 DefMBBI = MBBI; 241 // If the producing instruction does not depend on vsetvli, do not 242 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD. 243 if (!RISCVII::hasSEWOp(TSFlags)) 244 return false; 245 break; 246 } 247 } 248 } 249 } 250 251 return false; 252 } 253 254 void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 255 MachineBasicBlock::iterator MBBI, 256 const DebugLoc &DL, MCRegister DstReg, 257 MCRegister SrcReg, bool KillSrc) const { 258 if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) { 259 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg) 260 .addReg(SrcReg, getKillRegState(KillSrc)) 261 .addImm(0); 262 return; 263 } 264 265 // FPR->FPR copies and VR->VR copies. 266 unsigned Opc; 267 bool IsScalableVector = true; 268 unsigned NF = 1; 269 RISCVII::VLMUL LMul = RISCVII::LMUL_1; 270 unsigned SubRegIdx = RISCV::sub_vrm1_0; 271 if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) { 272 Opc = RISCV::FSGNJ_H; 273 IsScalableVector = false; 274 } else if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) { 275 Opc = RISCV::FSGNJ_S; 276 IsScalableVector = false; 277 } else if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) { 278 Opc = RISCV::FSGNJ_D; 279 IsScalableVector = false; 280 } else if (RISCV::VRRegClass.contains(DstReg, SrcReg)) { 281 Opc = RISCV::PseudoVMV1R_V; 282 LMul = RISCVII::LMUL_1; 283 } else if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) { 284 Opc = RISCV::PseudoVMV2R_V; 285 LMul = RISCVII::LMUL_2; 286 } else if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) { 287 Opc = RISCV::PseudoVMV4R_V; 288 LMul = RISCVII::LMUL_4; 289 } else if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) { 290 Opc = RISCV::PseudoVMV8R_V; 291 LMul = RISCVII::LMUL_8; 292 } else if (RISCV::VRN2M1RegClass.contains(DstReg, SrcReg)) { 293 Opc = RISCV::PseudoVMV1R_V; 294 SubRegIdx = RISCV::sub_vrm1_0; 295 NF = 2; 296 LMul = RISCVII::LMUL_1; 297 } else if (RISCV::VRN2M2RegClass.contains(DstReg, SrcReg)) { 298 Opc = RISCV::PseudoVMV2R_V; 299 SubRegIdx = RISCV::sub_vrm2_0; 300 NF = 2; 301 LMul = RISCVII::LMUL_2; 302 } else if (RISCV::VRN2M4RegClass.contains(DstReg, SrcReg)) { 303 Opc = RISCV::PseudoVMV4R_V; 304 SubRegIdx = RISCV::sub_vrm4_0; 305 NF = 2; 306 LMul = RISCVII::LMUL_4; 307 } else if (RISCV::VRN3M1RegClass.contains(DstReg, SrcReg)) { 308 Opc = RISCV::PseudoVMV1R_V; 309 SubRegIdx = RISCV::sub_vrm1_0; 310 NF = 3; 311 LMul = RISCVII::LMUL_1; 312 } else if (RISCV::VRN3M2RegClass.contains(DstReg, SrcReg)) { 313 Opc = RISCV::PseudoVMV2R_V; 314 SubRegIdx = RISCV::sub_vrm2_0; 315 NF = 3; 316 LMul = RISCVII::LMUL_2; 317 } else if (RISCV::VRN4M1RegClass.contains(DstReg, SrcReg)) { 318 Opc = RISCV::PseudoVMV1R_V; 319 SubRegIdx = RISCV::sub_vrm1_0; 320 NF = 4; 321 LMul = RISCVII::LMUL_1; 322 } else if (RISCV::VRN4M2RegClass.contains(DstReg, SrcReg)) { 323 Opc = RISCV::PseudoVMV2R_V; 324 SubRegIdx = RISCV::sub_vrm2_0; 325 NF = 4; 326 LMul = RISCVII::LMUL_2; 327 } else if (RISCV::VRN5M1RegClass.contains(DstReg, SrcReg)) { 328 Opc = RISCV::PseudoVMV1R_V; 329 SubRegIdx = RISCV::sub_vrm1_0; 330 NF = 5; 331 LMul = RISCVII::LMUL_1; 332 } else if (RISCV::VRN6M1RegClass.contains(DstReg, SrcReg)) { 333 Opc = RISCV::PseudoVMV1R_V; 334 SubRegIdx = RISCV::sub_vrm1_0; 335 NF = 6; 336 LMul = RISCVII::LMUL_1; 337 } else if (RISCV::VRN7M1RegClass.contains(DstReg, SrcReg)) { 338 Opc = RISCV::PseudoVMV1R_V; 339 SubRegIdx = RISCV::sub_vrm1_0; 340 NF = 7; 341 LMul = RISCVII::LMUL_1; 342 } else if (RISCV::VRN8M1RegClass.contains(DstReg, SrcReg)) { 343 Opc = RISCV::PseudoVMV1R_V; 344 SubRegIdx = RISCV::sub_vrm1_0; 345 NF = 8; 346 LMul = RISCVII::LMUL_1; 347 } else { 348 llvm_unreachable("Impossible reg-to-reg copy"); 349 } 350 351 if (IsScalableVector) { 352 bool UseVMV_V_V = false; 353 MachineBasicBlock::const_iterator DefMBBI; 354 unsigned DefExplicitOpNum; 355 unsigned VIOpc; 356 if (isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) { 357 UseVMV_V_V = true; 358 DefExplicitOpNum = DefMBBI->getNumExplicitOperands(); 359 // We only need to handle LMUL = 1/2/4/8 here because we only define 360 // vector register classes for LMUL = 1/2/4/8. 361 switch (LMul) { 362 default: 363 llvm_unreachable("Impossible LMUL for vector register copy."); 364 case RISCVII::LMUL_1: 365 Opc = RISCV::PseudoVMV_V_V_M1; 366 VIOpc = RISCV::PseudoVMV_V_I_M1; 367 break; 368 case RISCVII::LMUL_2: 369 Opc = RISCV::PseudoVMV_V_V_M2; 370 VIOpc = RISCV::PseudoVMV_V_I_M2; 371 break; 372 case RISCVII::LMUL_4: 373 Opc = RISCV::PseudoVMV_V_V_M4; 374 VIOpc = RISCV::PseudoVMV_V_I_M4; 375 break; 376 case RISCVII::LMUL_8: 377 Opc = RISCV::PseudoVMV_V_V_M8; 378 VIOpc = RISCV::PseudoVMV_V_I_M8; 379 break; 380 } 381 } 382 383 bool UseVMV_V_I = false; 384 if (UseVMV_V_V && (DefMBBI->getOpcode() == VIOpc)) { 385 UseVMV_V_I = true; 386 Opc = VIOpc; 387 } 388 389 if (NF == 1) { 390 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), DstReg); 391 if (UseVMV_V_I) 392 MIB = MIB.add(DefMBBI->getOperand(1)); 393 else 394 MIB = MIB.addReg(SrcReg, getKillRegState(KillSrc)); 395 if (UseVMV_V_V) { 396 // The last two arguments of vector instructions are 397 // AVL, SEW. We also need to append the implicit-use vl and vtype. 398 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 2)); // AVL 399 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 1)); // SEW 400 MIB.addReg(RISCV::VL, RegState::Implicit); 401 MIB.addReg(RISCV::VTYPE, RegState::Implicit); 402 } 403 } else { 404 const TargetRegisterInfo *TRI = STI.getRegisterInfo(); 405 406 int I = 0, End = NF, Incr = 1; 407 unsigned SrcEncoding = TRI->getEncodingValue(SrcReg); 408 unsigned DstEncoding = TRI->getEncodingValue(DstReg); 409 unsigned LMulVal; 410 bool Fractional; 411 std::tie(LMulVal, Fractional) = RISCVVType::decodeVLMUL(LMul); 412 assert(!Fractional && "It is impossible be fractional lmul here."); 413 if (forwardCopyWillClobberTuple(DstEncoding, SrcEncoding, NF * LMulVal)) { 414 I = NF - 1; 415 End = -1; 416 Incr = -1; 417 } 418 419 for (; I != End; I += Incr) { 420 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), 421 TRI->getSubReg(DstReg, SubRegIdx + I)); 422 if (UseVMV_V_I) 423 MIB = MIB.add(DefMBBI->getOperand(1)); 424 else 425 MIB = MIB.addReg(TRI->getSubReg(SrcReg, SubRegIdx + I), 426 getKillRegState(KillSrc)); 427 if (UseVMV_V_V) { 428 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 2)); // AVL 429 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 1)); // SEW 430 MIB.addReg(RISCV::VL, RegState::Implicit); 431 MIB.addReg(RISCV::VTYPE, RegState::Implicit); 432 } 433 } 434 } 435 } else { 436 BuildMI(MBB, MBBI, DL, get(Opc), DstReg) 437 .addReg(SrcReg, getKillRegState(KillSrc)) 438 .addReg(SrcReg, getKillRegState(KillSrc)); 439 } 440 } 441 442 void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, 443 MachineBasicBlock::iterator I, 444 Register SrcReg, bool IsKill, int FI, 445 const TargetRegisterClass *RC, 446 const TargetRegisterInfo *TRI) const { 447 DebugLoc DL; 448 if (I != MBB.end()) 449 DL = I->getDebugLoc(); 450 451 MachineFunction *MF = MBB.getParent(); 452 MachineFrameInfo &MFI = MF->getFrameInfo(); 453 454 unsigned Opcode; 455 bool IsScalableVector = true; 456 bool IsZvlsseg = true; 457 if (RISCV::GPRRegClass.hasSubClassEq(RC)) { 458 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ? 459 RISCV::SW : RISCV::SD; 460 IsScalableVector = false; 461 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) { 462 Opcode = RISCV::FSH; 463 IsScalableVector = false; 464 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) { 465 Opcode = RISCV::FSW; 466 IsScalableVector = false; 467 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) { 468 Opcode = RISCV::FSD; 469 IsScalableVector = false; 470 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) { 471 Opcode = RISCV::PseudoVSPILL_M1; 472 IsZvlsseg = false; 473 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) { 474 Opcode = RISCV::PseudoVSPILL_M2; 475 IsZvlsseg = false; 476 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) { 477 Opcode = RISCV::PseudoVSPILL_M4; 478 IsZvlsseg = false; 479 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) { 480 Opcode = RISCV::PseudoVSPILL_M8; 481 IsZvlsseg = false; 482 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC)) 483 Opcode = RISCV::PseudoVSPILL2_M1; 484 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC)) 485 Opcode = RISCV::PseudoVSPILL2_M2; 486 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC)) 487 Opcode = RISCV::PseudoVSPILL2_M4; 488 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC)) 489 Opcode = RISCV::PseudoVSPILL3_M1; 490 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC)) 491 Opcode = RISCV::PseudoVSPILL3_M2; 492 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC)) 493 Opcode = RISCV::PseudoVSPILL4_M1; 494 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC)) 495 Opcode = RISCV::PseudoVSPILL4_M2; 496 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC)) 497 Opcode = RISCV::PseudoVSPILL5_M1; 498 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC)) 499 Opcode = RISCV::PseudoVSPILL6_M1; 500 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC)) 501 Opcode = RISCV::PseudoVSPILL7_M1; 502 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC)) 503 Opcode = RISCV::PseudoVSPILL8_M1; 504 else 505 llvm_unreachable("Can't store this register to stack slot"); 506 507 if (IsScalableVector) { 508 MachineMemOperand *MMO = MF->getMachineMemOperand( 509 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 510 MemoryLocation::UnknownSize, MFI.getObjectAlign(FI)); 511 512 MFI.setStackID(FI, TargetStackID::ScalableVector); 513 auto MIB = BuildMI(MBB, I, DL, get(Opcode)) 514 .addReg(SrcReg, getKillRegState(IsKill)) 515 .addFrameIndex(FI) 516 .addMemOperand(MMO); 517 if (IsZvlsseg) { 518 // For spilling/reloading Zvlsseg registers, append the dummy field for 519 // the scaled vector length. The argument will be used when expanding 520 // these pseudo instructions. 521 MIB.addReg(RISCV::X0); 522 } 523 } else { 524 MachineMemOperand *MMO = MF->getMachineMemOperand( 525 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 526 MFI.getObjectSize(FI), MFI.getObjectAlign(FI)); 527 528 BuildMI(MBB, I, DL, get(Opcode)) 529 .addReg(SrcReg, getKillRegState(IsKill)) 530 .addFrameIndex(FI) 531 .addImm(0) 532 .addMemOperand(MMO); 533 } 534 } 535 536 void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 537 MachineBasicBlock::iterator I, 538 Register DstReg, int FI, 539 const TargetRegisterClass *RC, 540 const TargetRegisterInfo *TRI) const { 541 DebugLoc DL; 542 if (I != MBB.end()) 543 DL = I->getDebugLoc(); 544 545 MachineFunction *MF = MBB.getParent(); 546 MachineFrameInfo &MFI = MF->getFrameInfo(); 547 548 unsigned Opcode; 549 bool IsScalableVector = true; 550 bool IsZvlsseg = true; 551 if (RISCV::GPRRegClass.hasSubClassEq(RC)) { 552 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ? 553 RISCV::LW : RISCV::LD; 554 IsScalableVector = false; 555 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) { 556 Opcode = RISCV::FLH; 557 IsScalableVector = false; 558 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) { 559 Opcode = RISCV::FLW; 560 IsScalableVector = false; 561 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) { 562 Opcode = RISCV::FLD; 563 IsScalableVector = false; 564 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) { 565 Opcode = RISCV::PseudoVRELOAD_M1; 566 IsZvlsseg = false; 567 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) { 568 Opcode = RISCV::PseudoVRELOAD_M2; 569 IsZvlsseg = false; 570 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) { 571 Opcode = RISCV::PseudoVRELOAD_M4; 572 IsZvlsseg = false; 573 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) { 574 Opcode = RISCV::PseudoVRELOAD_M8; 575 IsZvlsseg = false; 576 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC)) 577 Opcode = RISCV::PseudoVRELOAD2_M1; 578 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC)) 579 Opcode = RISCV::PseudoVRELOAD2_M2; 580 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC)) 581 Opcode = RISCV::PseudoVRELOAD2_M4; 582 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC)) 583 Opcode = RISCV::PseudoVRELOAD3_M1; 584 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC)) 585 Opcode = RISCV::PseudoVRELOAD3_M2; 586 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC)) 587 Opcode = RISCV::PseudoVRELOAD4_M1; 588 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC)) 589 Opcode = RISCV::PseudoVRELOAD4_M2; 590 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC)) 591 Opcode = RISCV::PseudoVRELOAD5_M1; 592 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC)) 593 Opcode = RISCV::PseudoVRELOAD6_M1; 594 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC)) 595 Opcode = RISCV::PseudoVRELOAD7_M1; 596 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC)) 597 Opcode = RISCV::PseudoVRELOAD8_M1; 598 else 599 llvm_unreachable("Can't load this register from stack slot"); 600 601 if (IsScalableVector) { 602 MachineMemOperand *MMO = MF->getMachineMemOperand( 603 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad, 604 MemoryLocation::UnknownSize, MFI.getObjectAlign(FI)); 605 606 MFI.setStackID(FI, TargetStackID::ScalableVector); 607 auto MIB = BuildMI(MBB, I, DL, get(Opcode), DstReg) 608 .addFrameIndex(FI) 609 .addMemOperand(MMO); 610 if (IsZvlsseg) { 611 // For spilling/reloading Zvlsseg registers, append the dummy field for 612 // the scaled vector length. The argument will be used when expanding 613 // these pseudo instructions. 614 MIB.addReg(RISCV::X0); 615 } 616 } else { 617 MachineMemOperand *MMO = MF->getMachineMemOperand( 618 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad, 619 MFI.getObjectSize(FI), MFI.getObjectAlign(FI)); 620 621 BuildMI(MBB, I, DL, get(Opcode), DstReg) 622 .addFrameIndex(FI) 623 .addImm(0) 624 .addMemOperand(MMO); 625 } 626 } 627 628 void RISCVInstrInfo::movImm(MachineBasicBlock &MBB, 629 MachineBasicBlock::iterator MBBI, 630 const DebugLoc &DL, Register DstReg, uint64_t Val, 631 MachineInstr::MIFlag Flag) const { 632 MachineFunction *MF = MBB.getParent(); 633 MachineRegisterInfo &MRI = MF->getRegInfo(); 634 Register SrcReg = RISCV::X0; 635 Register Result = MRI.createVirtualRegister(&RISCV::GPRRegClass); 636 unsigned Num = 0; 637 638 if (!STI.is64Bit() && !isInt<32>(Val)) 639 report_fatal_error("Should only materialize 32-bit constants for RV32"); 640 641 RISCVMatInt::InstSeq Seq = 642 RISCVMatInt::generateInstSeq(Val, STI.getFeatureBits()); 643 assert(!Seq.empty()); 644 645 for (RISCVMatInt::Inst &Inst : Seq) { 646 // Write the final result to DstReg if it's the last instruction in the Seq. 647 // Otherwise, write the result to the temp register. 648 if (++Num == Seq.size()) 649 Result = DstReg; 650 651 if (Inst.Opc == RISCV::LUI) { 652 BuildMI(MBB, MBBI, DL, get(RISCV::LUI), Result) 653 .addImm(Inst.Imm) 654 .setMIFlag(Flag); 655 } else if (Inst.Opc == RISCV::ADDUW) { 656 BuildMI(MBB, MBBI, DL, get(RISCV::ADDUW), Result) 657 .addReg(SrcReg, RegState::Kill) 658 .addReg(RISCV::X0) 659 .setMIFlag(Flag); 660 } else if (Inst.Opc == RISCV::SH1ADD || Inst.Opc == RISCV::SH2ADD || 661 Inst.Opc == RISCV::SH3ADD) { 662 BuildMI(MBB, MBBI, DL, get(Inst.Opc), Result) 663 .addReg(SrcReg, RegState::Kill) 664 .addReg(SrcReg, RegState::Kill) 665 .setMIFlag(Flag); 666 } else { 667 BuildMI(MBB, MBBI, DL, get(Inst.Opc), Result) 668 .addReg(SrcReg, RegState::Kill) 669 .addImm(Inst.Imm) 670 .setMIFlag(Flag); 671 } 672 // Only the first instruction has X0 as its source. 673 SrcReg = Result; 674 } 675 } 676 677 static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc) { 678 switch (Opc) { 679 default: 680 return RISCVCC::COND_INVALID; 681 case RISCV::BEQ: 682 return RISCVCC::COND_EQ; 683 case RISCV::BNE: 684 return RISCVCC::COND_NE; 685 case RISCV::BLT: 686 return RISCVCC::COND_LT; 687 case RISCV::BGE: 688 return RISCVCC::COND_GE; 689 case RISCV::BLTU: 690 return RISCVCC::COND_LTU; 691 case RISCV::BGEU: 692 return RISCVCC::COND_GEU; 693 } 694 } 695 696 // The contents of values added to Cond are not examined outside of 697 // RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we 698 // push BranchOpcode, Reg1, Reg2. 699 static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target, 700 SmallVectorImpl<MachineOperand> &Cond) { 701 // Block ends with fall-through condbranch. 702 assert(LastInst.getDesc().isConditionalBranch() && 703 "Unknown conditional branch"); 704 Target = LastInst.getOperand(2).getMBB(); 705 unsigned CC = getCondFromBranchOpc(LastInst.getOpcode()); 706 Cond.push_back(MachineOperand::CreateImm(CC)); 707 Cond.push_back(LastInst.getOperand(0)); 708 Cond.push_back(LastInst.getOperand(1)); 709 } 710 711 const MCInstrDesc &RISCVInstrInfo::getBrCond(RISCVCC::CondCode CC) const { 712 switch (CC) { 713 default: 714 llvm_unreachable("Unknown condition code!"); 715 case RISCVCC::COND_EQ: 716 return get(RISCV::BEQ); 717 case RISCVCC::COND_NE: 718 return get(RISCV::BNE); 719 case RISCVCC::COND_LT: 720 return get(RISCV::BLT); 721 case RISCVCC::COND_GE: 722 return get(RISCV::BGE); 723 case RISCVCC::COND_LTU: 724 return get(RISCV::BLTU); 725 case RISCVCC::COND_GEU: 726 return get(RISCV::BGEU); 727 } 728 } 729 730 RISCVCC::CondCode RISCVCC::getOppositeBranchCondition(RISCVCC::CondCode CC) { 731 switch (CC) { 732 default: 733 llvm_unreachable("Unrecognized conditional branch"); 734 case RISCVCC::COND_EQ: 735 return RISCVCC::COND_NE; 736 case RISCVCC::COND_NE: 737 return RISCVCC::COND_EQ; 738 case RISCVCC::COND_LT: 739 return RISCVCC::COND_GE; 740 case RISCVCC::COND_GE: 741 return RISCVCC::COND_LT; 742 case RISCVCC::COND_LTU: 743 return RISCVCC::COND_GEU; 744 case RISCVCC::COND_GEU: 745 return RISCVCC::COND_LTU; 746 } 747 } 748 749 bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB, 750 MachineBasicBlock *&TBB, 751 MachineBasicBlock *&FBB, 752 SmallVectorImpl<MachineOperand> &Cond, 753 bool AllowModify) const { 754 TBB = FBB = nullptr; 755 Cond.clear(); 756 757 // If the block has no terminators, it just falls into the block after it. 758 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); 759 if (I == MBB.end() || !isUnpredicatedTerminator(*I)) 760 return false; 761 762 // Count the number of terminators and find the first unconditional or 763 // indirect branch. 764 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end(); 765 int NumTerminators = 0; 766 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J); 767 J++) { 768 NumTerminators++; 769 if (J->getDesc().isUnconditionalBranch() || 770 J->getDesc().isIndirectBranch()) { 771 FirstUncondOrIndirectBr = J.getReverse(); 772 } 773 } 774 775 // If AllowModify is true, we can erase any terminators after 776 // FirstUncondOrIndirectBR. 777 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) { 778 while (std::next(FirstUncondOrIndirectBr) != MBB.end()) { 779 std::next(FirstUncondOrIndirectBr)->eraseFromParent(); 780 NumTerminators--; 781 } 782 I = FirstUncondOrIndirectBr; 783 } 784 785 // We can't handle blocks that end in an indirect branch. 786 if (I->getDesc().isIndirectBranch()) 787 return true; 788 789 // We can't handle blocks with more than 2 terminators. 790 if (NumTerminators > 2) 791 return true; 792 793 // Handle a single unconditional branch. 794 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) { 795 TBB = getBranchDestBlock(*I); 796 return false; 797 } 798 799 // Handle a single conditional branch. 800 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) { 801 parseCondBranch(*I, TBB, Cond); 802 return false; 803 } 804 805 // Handle a conditional branch followed by an unconditional branch. 806 if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() && 807 I->getDesc().isUnconditionalBranch()) { 808 parseCondBranch(*std::prev(I), TBB, Cond); 809 FBB = getBranchDestBlock(*I); 810 return false; 811 } 812 813 // Otherwise, we can't handle this. 814 return true; 815 } 816 817 unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB, 818 int *BytesRemoved) const { 819 if (BytesRemoved) 820 *BytesRemoved = 0; 821 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); 822 if (I == MBB.end()) 823 return 0; 824 825 if (!I->getDesc().isUnconditionalBranch() && 826 !I->getDesc().isConditionalBranch()) 827 return 0; 828 829 // Remove the branch. 830 if (BytesRemoved) 831 *BytesRemoved += getInstSizeInBytes(*I); 832 I->eraseFromParent(); 833 834 I = MBB.end(); 835 836 if (I == MBB.begin()) 837 return 1; 838 --I; 839 if (!I->getDesc().isConditionalBranch()) 840 return 1; 841 842 // Remove the branch. 843 if (BytesRemoved) 844 *BytesRemoved += getInstSizeInBytes(*I); 845 I->eraseFromParent(); 846 return 2; 847 } 848 849 // Inserts a branch into the end of the specific MachineBasicBlock, returning 850 // the number of instructions inserted. 851 unsigned RISCVInstrInfo::insertBranch( 852 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, 853 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const { 854 if (BytesAdded) 855 *BytesAdded = 0; 856 857 // Shouldn't be a fall through. 858 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 859 assert((Cond.size() == 3 || Cond.size() == 0) && 860 "RISCV branch conditions have two components!"); 861 862 // Unconditional branch. 863 if (Cond.empty()) { 864 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB); 865 if (BytesAdded) 866 *BytesAdded += getInstSizeInBytes(MI); 867 return 1; 868 } 869 870 // Either a one or two-way conditional branch. 871 auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm()); 872 MachineInstr &CondMI = 873 *BuildMI(&MBB, DL, getBrCond(CC)).add(Cond[1]).add(Cond[2]).addMBB(TBB); 874 if (BytesAdded) 875 *BytesAdded += getInstSizeInBytes(CondMI); 876 877 // One-way conditional branch. 878 if (!FBB) 879 return 1; 880 881 // Two-way conditional branch. 882 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB); 883 if (BytesAdded) 884 *BytesAdded += getInstSizeInBytes(MI); 885 return 2; 886 } 887 888 void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, 889 MachineBasicBlock &DestBB, 890 MachineBasicBlock &RestoreBB, 891 const DebugLoc &DL, int64_t BrOffset, 892 RegScavenger *RS) const { 893 assert(RS && "RegScavenger required for long branching"); 894 assert(MBB.empty() && 895 "new block should be inserted for expanding unconditional branch"); 896 assert(MBB.pred_size() == 1); 897 898 MachineFunction *MF = MBB.getParent(); 899 MachineRegisterInfo &MRI = MF->getRegInfo(); 900 901 if (!isInt<32>(BrOffset)) 902 report_fatal_error( 903 "Branch offsets outside of the signed 32-bit range not supported"); 904 905 // FIXME: A virtual register must be used initially, as the register 906 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch 907 // uses the same workaround). 908 Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRRegClass); 909 auto II = MBB.end(); 910 911 MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump)) 912 .addReg(ScratchReg, RegState::Define | RegState::Dead) 913 .addMBB(&DestBB, RISCVII::MO_CALL); 914 915 RS->enterBasicBlockEnd(MBB); 916 unsigned Scav = RS->scavengeRegisterBackwards(RISCV::GPRRegClass, 917 MI.getIterator(), false, 0); 918 // TODO: The case when there is no scavenged register needs special handling. 919 assert(Scav != RISCV::NoRegister && "No register is scavenged!"); 920 MRI.replaceRegWith(ScratchReg, Scav); 921 MRI.clearVirtRegs(); 922 RS->setRegUsed(Scav); 923 } 924 925 bool RISCVInstrInfo::reverseBranchCondition( 926 SmallVectorImpl<MachineOperand> &Cond) const { 927 assert((Cond.size() == 3) && "Invalid branch condition!"); 928 auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm()); 929 Cond[0].setImm(getOppositeBranchCondition(CC)); 930 return false; 931 } 932 933 MachineBasicBlock * 934 RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const { 935 assert(MI.getDesc().isBranch() && "Unexpected opcode!"); 936 // The branch target is always the last operand. 937 int NumOp = MI.getNumExplicitOperands(); 938 return MI.getOperand(NumOp - 1).getMBB(); 939 } 940 941 bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp, 942 int64_t BrOffset) const { 943 unsigned XLen = STI.getXLen(); 944 // Ideally we could determine the supported branch offset from the 945 // RISCVII::FormMask, but this can't be used for Pseudo instructions like 946 // PseudoBR. 947 switch (BranchOp) { 948 default: 949 llvm_unreachable("Unexpected opcode!"); 950 case RISCV::BEQ: 951 case RISCV::BNE: 952 case RISCV::BLT: 953 case RISCV::BGE: 954 case RISCV::BLTU: 955 case RISCV::BGEU: 956 return isIntN(13, BrOffset); 957 case RISCV::JAL: 958 case RISCV::PseudoBR: 959 return isIntN(21, BrOffset); 960 case RISCV::PseudoJump: 961 return isIntN(32, SignExtend64(BrOffset + 0x800, XLen)); 962 } 963 } 964 965 unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 966 unsigned Opcode = MI.getOpcode(); 967 968 switch (Opcode) { 969 default: { 970 if (MI.getParent() && MI.getParent()->getParent()) { 971 const auto MF = MI.getMF(); 972 const auto &TM = static_cast<const RISCVTargetMachine &>(MF->getTarget()); 973 const MCRegisterInfo &MRI = *TM.getMCRegisterInfo(); 974 const MCSubtargetInfo &STI = *TM.getMCSubtargetInfo(); 975 const RISCVSubtarget &ST = MF->getSubtarget<RISCVSubtarget>(); 976 if (isCompressibleInst(MI, &ST, MRI, STI)) 977 return 2; 978 } 979 return get(Opcode).getSize(); 980 } 981 case TargetOpcode::EH_LABEL: 982 case TargetOpcode::IMPLICIT_DEF: 983 case TargetOpcode::KILL: 984 case TargetOpcode::DBG_VALUE: 985 return 0; 986 // These values are determined based on RISCVExpandAtomicPseudoInsts, 987 // RISCVExpandPseudoInsts and RISCVMCCodeEmitter, depending on where the 988 // pseudos are expanded. 989 case RISCV::PseudoCALLReg: 990 case RISCV::PseudoCALL: 991 case RISCV::PseudoJump: 992 case RISCV::PseudoTAIL: 993 case RISCV::PseudoLLA: 994 case RISCV::PseudoLA: 995 case RISCV::PseudoLA_TLS_IE: 996 case RISCV::PseudoLA_TLS_GD: 997 return 8; 998 case RISCV::PseudoAtomicLoadNand32: 999 case RISCV::PseudoAtomicLoadNand64: 1000 return 20; 1001 case RISCV::PseudoMaskedAtomicSwap32: 1002 case RISCV::PseudoMaskedAtomicLoadAdd32: 1003 case RISCV::PseudoMaskedAtomicLoadSub32: 1004 return 28; 1005 case RISCV::PseudoMaskedAtomicLoadNand32: 1006 return 32; 1007 case RISCV::PseudoMaskedAtomicLoadMax32: 1008 case RISCV::PseudoMaskedAtomicLoadMin32: 1009 return 44; 1010 case RISCV::PseudoMaskedAtomicLoadUMax32: 1011 case RISCV::PseudoMaskedAtomicLoadUMin32: 1012 return 36; 1013 case RISCV::PseudoCmpXchg32: 1014 case RISCV::PseudoCmpXchg64: 1015 return 16; 1016 case RISCV::PseudoMaskedCmpXchg32: 1017 return 32; 1018 case TargetOpcode::INLINEASM: 1019 case TargetOpcode::INLINEASM_BR: { 1020 const MachineFunction &MF = *MI.getParent()->getParent(); 1021 const auto &TM = static_cast<const RISCVTargetMachine &>(MF.getTarget()); 1022 return getInlineAsmLength(MI.getOperand(0).getSymbolName(), 1023 *TM.getMCAsmInfo()); 1024 } 1025 case RISCV::PseudoVSPILL2_M1: 1026 case RISCV::PseudoVSPILL2_M2: 1027 case RISCV::PseudoVSPILL2_M4: 1028 case RISCV::PseudoVSPILL3_M1: 1029 case RISCV::PseudoVSPILL3_M2: 1030 case RISCV::PseudoVSPILL4_M1: 1031 case RISCV::PseudoVSPILL4_M2: 1032 case RISCV::PseudoVSPILL5_M1: 1033 case RISCV::PseudoVSPILL6_M1: 1034 case RISCV::PseudoVSPILL7_M1: 1035 case RISCV::PseudoVSPILL8_M1: 1036 case RISCV::PseudoVRELOAD2_M1: 1037 case RISCV::PseudoVRELOAD2_M2: 1038 case RISCV::PseudoVRELOAD2_M4: 1039 case RISCV::PseudoVRELOAD3_M1: 1040 case RISCV::PseudoVRELOAD3_M2: 1041 case RISCV::PseudoVRELOAD4_M1: 1042 case RISCV::PseudoVRELOAD4_M2: 1043 case RISCV::PseudoVRELOAD5_M1: 1044 case RISCV::PseudoVRELOAD6_M1: 1045 case RISCV::PseudoVRELOAD7_M1: 1046 case RISCV::PseudoVRELOAD8_M1: { 1047 // The values are determined based on expandVSPILL and expandVRELOAD that 1048 // expand the pseudos depending on NF. 1049 unsigned NF = isRVVSpillForZvlsseg(Opcode)->first; 1050 return 4 * (2 * NF - 1); 1051 } 1052 } 1053 } 1054 1055 bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const { 1056 const unsigned Opcode = MI.getOpcode(); 1057 switch (Opcode) { 1058 default: 1059 break; 1060 case RISCV::FSGNJ_D: 1061 case RISCV::FSGNJ_S: 1062 // The canonical floating-point move is fsgnj rd, rs, rs. 1063 return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() && 1064 MI.getOperand(1).getReg() == MI.getOperand(2).getReg(); 1065 case RISCV::ADDI: 1066 case RISCV::ORI: 1067 case RISCV::XORI: 1068 return (MI.getOperand(1).isReg() && 1069 MI.getOperand(1).getReg() == RISCV::X0) || 1070 (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0); 1071 } 1072 return MI.isAsCheapAsAMove(); 1073 } 1074 1075 Optional<DestSourcePair> 1076 RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { 1077 if (MI.isMoveReg()) 1078 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 1079 switch (MI.getOpcode()) { 1080 default: 1081 break; 1082 case RISCV::ADDI: 1083 // Operand 1 can be a frameindex but callers expect registers 1084 if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() && 1085 MI.getOperand(2).getImm() == 0) 1086 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 1087 break; 1088 case RISCV::FSGNJ_D: 1089 case RISCV::FSGNJ_S: 1090 // The canonical floating-point move is fsgnj rd, rs, rs. 1091 if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() && 1092 MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) 1093 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 1094 break; 1095 } 1096 return None; 1097 } 1098 1099 bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI, 1100 StringRef &ErrInfo) const { 1101 const MCInstrInfo *MCII = STI.getInstrInfo(); 1102 MCInstrDesc const &Desc = MCII->get(MI.getOpcode()); 1103 1104 for (auto &OI : enumerate(Desc.operands())) { 1105 unsigned OpType = OI.value().OperandType; 1106 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM && 1107 OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) { 1108 const MachineOperand &MO = MI.getOperand(OI.index()); 1109 if (MO.isImm()) { 1110 int64_t Imm = MO.getImm(); 1111 bool Ok; 1112 switch (OpType) { 1113 default: 1114 llvm_unreachable("Unexpected operand type"); 1115 case RISCVOp::OPERAND_UIMM2: 1116 Ok = isUInt<2>(Imm); 1117 break; 1118 case RISCVOp::OPERAND_UIMM3: 1119 Ok = isUInt<3>(Imm); 1120 break; 1121 case RISCVOp::OPERAND_UIMM4: 1122 Ok = isUInt<4>(Imm); 1123 break; 1124 case RISCVOp::OPERAND_UIMM5: 1125 Ok = isUInt<5>(Imm); 1126 break; 1127 case RISCVOp::OPERAND_UIMM7: 1128 Ok = isUInt<7>(Imm); 1129 break; 1130 case RISCVOp::OPERAND_UIMM12: 1131 Ok = isUInt<12>(Imm); 1132 break; 1133 case RISCVOp::OPERAND_SIMM12: 1134 Ok = isInt<12>(Imm); 1135 break; 1136 case RISCVOp::OPERAND_UIMM20: 1137 Ok = isUInt<20>(Imm); 1138 break; 1139 case RISCVOp::OPERAND_UIMMLOG2XLEN: 1140 if (STI.getTargetTriple().isArch64Bit()) 1141 Ok = isUInt<6>(Imm); 1142 else 1143 Ok = isUInt<5>(Imm); 1144 break; 1145 } 1146 if (!Ok) { 1147 ErrInfo = "Invalid immediate"; 1148 return false; 1149 } 1150 } 1151 } 1152 } 1153 1154 return true; 1155 } 1156 1157 // Return true if get the base operand, byte offset of an instruction and the 1158 // memory width. Width is the size of memory that is being loaded/stored. 1159 bool RISCVInstrInfo::getMemOperandWithOffsetWidth( 1160 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset, 1161 unsigned &Width, const TargetRegisterInfo *TRI) const { 1162 if (!LdSt.mayLoadOrStore()) 1163 return false; 1164 1165 // Here we assume the standard RISC-V ISA, which uses a base+offset 1166 // addressing mode. You'll need to relax these conditions to support custom 1167 // load/stores instructions. 1168 if (LdSt.getNumExplicitOperands() != 3) 1169 return false; 1170 if (!LdSt.getOperand(1).isReg() || !LdSt.getOperand(2).isImm()) 1171 return false; 1172 1173 if (!LdSt.hasOneMemOperand()) 1174 return false; 1175 1176 Width = (*LdSt.memoperands_begin())->getSize(); 1177 BaseReg = &LdSt.getOperand(1); 1178 Offset = LdSt.getOperand(2).getImm(); 1179 return true; 1180 } 1181 1182 bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint( 1183 const MachineInstr &MIa, const MachineInstr &MIb) const { 1184 assert(MIa.mayLoadOrStore() && "MIa must be a load or store."); 1185 assert(MIb.mayLoadOrStore() && "MIb must be a load or store."); 1186 1187 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() || 1188 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) 1189 return false; 1190 1191 // Retrieve the base register, offset from the base register and width. Width 1192 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If 1193 // base registers are identical, and the offset of a lower memory access + 1194 // the width doesn't overlap the offset of a higher memory access, 1195 // then the memory accesses are different. 1196 const TargetRegisterInfo *TRI = STI.getRegisterInfo(); 1197 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; 1198 int64_t OffsetA = 0, OffsetB = 0; 1199 unsigned int WidthA = 0, WidthB = 0; 1200 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) && 1201 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) { 1202 if (BaseOpA->isIdenticalTo(*BaseOpB)) { 1203 int LowOffset = std::min(OffsetA, OffsetB); 1204 int HighOffset = std::max(OffsetA, OffsetB); 1205 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 1206 if (LowOffset + LowWidth <= HighOffset) 1207 return true; 1208 } 1209 } 1210 return false; 1211 } 1212 1213 std::pair<unsigned, unsigned> 1214 RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 1215 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK; 1216 return std::make_pair(TF & Mask, TF & ~Mask); 1217 } 1218 1219 ArrayRef<std::pair<unsigned, const char *>> 1220 RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 1221 using namespace RISCVII; 1222 static const std::pair<unsigned, const char *> TargetFlags[] = { 1223 {MO_CALL, "riscv-call"}, 1224 {MO_PLT, "riscv-plt"}, 1225 {MO_LO, "riscv-lo"}, 1226 {MO_HI, "riscv-hi"}, 1227 {MO_PCREL_LO, "riscv-pcrel-lo"}, 1228 {MO_PCREL_HI, "riscv-pcrel-hi"}, 1229 {MO_GOT_HI, "riscv-got-hi"}, 1230 {MO_TPREL_LO, "riscv-tprel-lo"}, 1231 {MO_TPREL_HI, "riscv-tprel-hi"}, 1232 {MO_TPREL_ADD, "riscv-tprel-add"}, 1233 {MO_TLS_GOT_HI, "riscv-tls-got-hi"}, 1234 {MO_TLS_GD_HI, "riscv-tls-gd-hi"}}; 1235 return makeArrayRef(TargetFlags); 1236 } 1237 bool RISCVInstrInfo::isFunctionSafeToOutlineFrom( 1238 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const { 1239 const Function &F = MF.getFunction(); 1240 1241 // Can F be deduplicated by the linker? If it can, don't outline from it. 1242 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage()) 1243 return false; 1244 1245 // Don't outline from functions with section markings; the program could 1246 // expect that all the code is in the named section. 1247 if (F.hasSection()) 1248 return false; 1249 1250 // It's safe to outline from MF. 1251 return true; 1252 } 1253 1254 bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, 1255 unsigned &Flags) const { 1256 // More accurate safety checking is done in getOutliningCandidateInfo. 1257 return true; 1258 } 1259 1260 // Enum values indicating how an outlined call should be constructed. 1261 enum MachineOutlinerConstructionID { 1262 MachineOutlinerDefault 1263 }; 1264 1265 outliner::OutlinedFunction RISCVInstrInfo::getOutliningCandidateInfo( 1266 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { 1267 1268 // First we need to filter out candidates where the X5 register (IE t0) can't 1269 // be used to setup the function call. 1270 auto CannotInsertCall = [](outliner::Candidate &C) { 1271 const TargetRegisterInfo *TRI = C.getMF()->getSubtarget().getRegisterInfo(); 1272 1273 C.initLRU(*TRI); 1274 LiveRegUnits LRU = C.LRU; 1275 return !LRU.available(RISCV::X5); 1276 }; 1277 1278 llvm::erase_if(RepeatedSequenceLocs, CannotInsertCall); 1279 1280 // If the sequence doesn't have enough candidates left, then we're done. 1281 if (RepeatedSequenceLocs.size() < 2) 1282 return outliner::OutlinedFunction(); 1283 1284 unsigned SequenceSize = 0; 1285 1286 auto I = RepeatedSequenceLocs[0].front(); 1287 auto E = std::next(RepeatedSequenceLocs[0].back()); 1288 for (; I != E; ++I) 1289 SequenceSize += getInstSizeInBytes(*I); 1290 1291 // call t0, function = 8 bytes. 1292 unsigned CallOverhead = 8; 1293 for (auto &C : RepeatedSequenceLocs) 1294 C.setCallInfo(MachineOutlinerDefault, CallOverhead); 1295 1296 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled. 1297 unsigned FrameOverhead = 4; 1298 if (RepeatedSequenceLocs[0].getMF()->getSubtarget() 1299 .getFeatureBits()[RISCV::FeatureStdExtC]) 1300 FrameOverhead = 2; 1301 1302 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 1303 FrameOverhead, MachineOutlinerDefault); 1304 } 1305 1306 outliner::InstrType 1307 RISCVInstrInfo::getOutliningType(MachineBasicBlock::iterator &MBBI, 1308 unsigned Flags) const { 1309 MachineInstr &MI = *MBBI; 1310 MachineBasicBlock *MBB = MI.getParent(); 1311 const TargetRegisterInfo *TRI = 1312 MBB->getParent()->getSubtarget().getRegisterInfo(); 1313 1314 // Positions generally can't safely be outlined. 1315 if (MI.isPosition()) { 1316 // We can manually strip out CFI instructions later. 1317 if (MI.isCFIInstruction()) 1318 return outliner::InstrType::Invisible; 1319 1320 return outliner::InstrType::Illegal; 1321 } 1322 1323 // Don't trust the user to write safe inline assembly. 1324 if (MI.isInlineAsm()) 1325 return outliner::InstrType::Illegal; 1326 1327 // We can't outline branches to other basic blocks. 1328 if (MI.isTerminator() && !MBB->succ_empty()) 1329 return outliner::InstrType::Illegal; 1330 1331 // We need support for tail calls to outlined functions before return 1332 // statements can be allowed. 1333 if (MI.isReturn()) 1334 return outliner::InstrType::Illegal; 1335 1336 // Don't allow modifying the X5 register which we use for return addresses for 1337 // these outlined functions. 1338 if (MI.modifiesRegister(RISCV::X5, TRI) || 1339 MI.getDesc().hasImplicitDefOfPhysReg(RISCV::X5)) 1340 return outliner::InstrType::Illegal; 1341 1342 // Make sure the operands don't reference something unsafe. 1343 for (const auto &MO : MI.operands()) 1344 if (MO.isMBB() || MO.isBlockAddress() || MO.isCPI() || MO.isJTI()) 1345 return outliner::InstrType::Illegal; 1346 1347 // Don't allow instructions which won't be materialized to impact outlining 1348 // analysis. 1349 if (MI.isMetaInstruction()) 1350 return outliner::InstrType::Invisible; 1351 1352 return outliner::InstrType::Legal; 1353 } 1354 1355 void RISCVInstrInfo::buildOutlinedFrame( 1356 MachineBasicBlock &MBB, MachineFunction &MF, 1357 const outliner::OutlinedFunction &OF) const { 1358 1359 // Strip out any CFI instructions 1360 bool Changed = true; 1361 while (Changed) { 1362 Changed = false; 1363 auto I = MBB.begin(); 1364 auto E = MBB.end(); 1365 for (; I != E; ++I) { 1366 if (I->isCFIInstruction()) { 1367 I->removeFromParent(); 1368 Changed = true; 1369 break; 1370 } 1371 } 1372 } 1373 1374 MBB.addLiveIn(RISCV::X5); 1375 1376 // Add in a return instruction to the end of the outlined frame. 1377 MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR)) 1378 .addReg(RISCV::X0, RegState::Define) 1379 .addReg(RISCV::X5) 1380 .addImm(0)); 1381 } 1382 1383 MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall( 1384 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, 1385 MachineFunction &MF, const outliner::Candidate &C) const { 1386 1387 // Add in a call instruction to the outlined function at the given location. 1388 It = MBB.insert(It, 1389 BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5) 1390 .addGlobalAddress(M.getNamedValue(MF.getName()), 0, 1391 RISCVII::MO_CALL)); 1392 return It; 1393 } 1394 1395 // clang-format off 1396 #define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL) \ 1397 RISCV::PseudoV##OP##_##TYPE##_##LMUL 1398 1399 #define CASE_VFMA_OPCODE_LMULS(OP, TYPE) \ 1400 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF8): \ 1401 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4): \ 1402 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2): \ 1403 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1): \ 1404 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2): \ 1405 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4): \ 1406 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8) 1407 1408 #define CASE_VFMA_SPLATS(OP) \ 1409 CASE_VFMA_OPCODE_LMULS(OP, VF16): \ 1410 case CASE_VFMA_OPCODE_LMULS(OP, VF32): \ 1411 case CASE_VFMA_OPCODE_LMULS(OP, VF64) 1412 // clang-format on 1413 1414 bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI, 1415 unsigned &SrcOpIdx1, 1416 unsigned &SrcOpIdx2) const { 1417 const MCInstrDesc &Desc = MI.getDesc(); 1418 if (!Desc.isCommutable()) 1419 return false; 1420 1421 switch (MI.getOpcode()) { 1422 case CASE_VFMA_SPLATS(FMADD): 1423 case CASE_VFMA_SPLATS(FMSUB): 1424 case CASE_VFMA_SPLATS(FMACC): 1425 case CASE_VFMA_SPLATS(FMSAC): 1426 case CASE_VFMA_SPLATS(FNMADD): 1427 case CASE_VFMA_SPLATS(FNMSUB): 1428 case CASE_VFMA_SPLATS(FNMACC): 1429 case CASE_VFMA_SPLATS(FNMSAC): 1430 case CASE_VFMA_OPCODE_LMULS(FMACC, VV): 1431 case CASE_VFMA_OPCODE_LMULS(FMSAC, VV): 1432 case CASE_VFMA_OPCODE_LMULS(FNMACC, VV): 1433 case CASE_VFMA_OPCODE_LMULS(FNMSAC, VV): 1434 case CASE_VFMA_OPCODE_LMULS(MADD, VX): 1435 case CASE_VFMA_OPCODE_LMULS(NMSUB, VX): 1436 case CASE_VFMA_OPCODE_LMULS(MACC, VX): 1437 case CASE_VFMA_OPCODE_LMULS(NMSAC, VX): 1438 case CASE_VFMA_OPCODE_LMULS(MACC, VV): 1439 case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): { 1440 // If the tail policy is undisturbed we can't commute. 1441 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags)); 1442 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0) 1443 return false; 1444 1445 // For these instructions we can only swap operand 1 and operand 3 by 1446 // changing the opcode. 1447 unsigned CommutableOpIdx1 = 1; 1448 unsigned CommutableOpIdx2 = 3; 1449 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1, 1450 CommutableOpIdx2)) 1451 return false; 1452 return true; 1453 } 1454 case CASE_VFMA_OPCODE_LMULS(FMADD, VV): 1455 case CASE_VFMA_OPCODE_LMULS(FMSUB, VV): 1456 case CASE_VFMA_OPCODE_LMULS(FNMADD, VV): 1457 case CASE_VFMA_OPCODE_LMULS(FNMSUB, VV): 1458 case CASE_VFMA_OPCODE_LMULS(MADD, VV): 1459 case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): { 1460 // If the tail policy is undisturbed we can't commute. 1461 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags)); 1462 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0) 1463 return false; 1464 1465 // For these instructions we have more freedom. We can commute with the 1466 // other multiplicand or with the addend/subtrahend/minuend. 1467 1468 // Any fixed operand must be from source 1, 2 or 3. 1469 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3) 1470 return false; 1471 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3) 1472 return false; 1473 1474 // It both ops are fixed one must be the tied source. 1475 if (SrcOpIdx1 != CommuteAnyOperandIndex && 1476 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1) 1477 return false; 1478 1479 // Look for two different register operands assumed to be commutable 1480 // regardless of the FMA opcode. The FMA opcode is adjusted later if 1481 // needed. 1482 if (SrcOpIdx1 == CommuteAnyOperandIndex || 1483 SrcOpIdx2 == CommuteAnyOperandIndex) { 1484 // At least one of operands to be commuted is not specified and 1485 // this method is free to choose appropriate commutable operands. 1486 unsigned CommutableOpIdx1 = SrcOpIdx1; 1487 if (SrcOpIdx1 == SrcOpIdx2) { 1488 // Both of operands are not fixed. Set one of commutable 1489 // operands to the tied source. 1490 CommutableOpIdx1 = 1; 1491 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) { 1492 // Only one of the operands is not fixed. 1493 CommutableOpIdx1 = SrcOpIdx2; 1494 } 1495 1496 // CommutableOpIdx1 is well defined now. Let's choose another commutable 1497 // operand and assign its index to CommutableOpIdx2. 1498 unsigned CommutableOpIdx2; 1499 if (CommutableOpIdx1 != 1) { 1500 // If we haven't already used the tied source, we must use it now. 1501 CommutableOpIdx2 = 1; 1502 } else { 1503 Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg(); 1504 1505 // The commuted operands should have different registers. 1506 // Otherwise, the commute transformation does not change anything and 1507 // is useless. We use this as a hint to make our decision. 1508 if (Op1Reg != MI.getOperand(2).getReg()) 1509 CommutableOpIdx2 = 2; 1510 else 1511 CommutableOpIdx2 = 3; 1512 } 1513 1514 // Assign the found pair of commutable indices to SrcOpIdx1 and 1515 // SrcOpIdx2 to return those values. 1516 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1, 1517 CommutableOpIdx2)) 1518 return false; 1519 } 1520 1521 return true; 1522 } 1523 } 1524 1525 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 1526 } 1527 1528 #define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \ 1529 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \ 1530 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \ 1531 break; 1532 1533 #define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \ 1534 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \ 1535 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \ 1536 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \ 1537 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \ 1538 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \ 1539 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \ 1540 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8) 1541 1542 #define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \ 1543 CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, VF16) \ 1544 CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, VF32) \ 1545 CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, VF64) 1546 1547 MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI, 1548 bool NewMI, 1549 unsigned OpIdx1, 1550 unsigned OpIdx2) const { 1551 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 1552 if (NewMI) 1553 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 1554 return MI; 1555 }; 1556 1557 switch (MI.getOpcode()) { 1558 case CASE_VFMA_SPLATS(FMACC): 1559 case CASE_VFMA_SPLATS(FMADD): 1560 case CASE_VFMA_SPLATS(FMSAC): 1561 case CASE_VFMA_SPLATS(FMSUB): 1562 case CASE_VFMA_SPLATS(FNMACC): 1563 case CASE_VFMA_SPLATS(FNMADD): 1564 case CASE_VFMA_SPLATS(FNMSAC): 1565 case CASE_VFMA_SPLATS(FNMSUB): 1566 case CASE_VFMA_OPCODE_LMULS(FMACC, VV): 1567 case CASE_VFMA_OPCODE_LMULS(FMSAC, VV): 1568 case CASE_VFMA_OPCODE_LMULS(FNMACC, VV): 1569 case CASE_VFMA_OPCODE_LMULS(FNMSAC, VV): 1570 case CASE_VFMA_OPCODE_LMULS(MADD, VX): 1571 case CASE_VFMA_OPCODE_LMULS(NMSUB, VX): 1572 case CASE_VFMA_OPCODE_LMULS(MACC, VX): 1573 case CASE_VFMA_OPCODE_LMULS(NMSAC, VX): 1574 case CASE_VFMA_OPCODE_LMULS(MACC, VV): 1575 case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): { 1576 // It only make sense to toggle these between clobbering the 1577 // addend/subtrahend/minuend one of the multiplicands. 1578 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index"); 1579 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index"); 1580 unsigned Opc; 1581 switch (MI.getOpcode()) { 1582 default: 1583 llvm_unreachable("Unexpected opcode"); 1584 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD) 1585 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC) 1586 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB) 1587 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC) 1588 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD) 1589 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC) 1590 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB) 1591 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC) 1592 CASE_VFMA_CHANGE_OPCODE_LMULS(FMACC, FMADD, VV) 1593 CASE_VFMA_CHANGE_OPCODE_LMULS(FMSAC, FMSUB, VV) 1594 CASE_VFMA_CHANGE_OPCODE_LMULS(FNMACC, FNMADD, VV) 1595 CASE_VFMA_CHANGE_OPCODE_LMULS(FNMSAC, FNMSUB, VV) 1596 CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX) 1597 CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX) 1598 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX) 1599 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX) 1600 CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV) 1601 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV) 1602 } 1603 1604 auto &WorkingMI = cloneIfNew(MI); 1605 WorkingMI.setDesc(get(Opc)); 1606 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1607 OpIdx1, OpIdx2); 1608 } 1609 case CASE_VFMA_OPCODE_LMULS(FMADD, VV): 1610 case CASE_VFMA_OPCODE_LMULS(FMSUB, VV): 1611 case CASE_VFMA_OPCODE_LMULS(FNMADD, VV): 1612 case CASE_VFMA_OPCODE_LMULS(FNMSUB, VV): 1613 case CASE_VFMA_OPCODE_LMULS(MADD, VV): 1614 case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): { 1615 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index"); 1616 // If one of the operands, is the addend we need to change opcode. 1617 // Otherwise we're just swapping 2 of the multiplicands. 1618 if (OpIdx1 == 3 || OpIdx2 == 3) { 1619 unsigned Opc; 1620 switch (MI.getOpcode()) { 1621 default: 1622 llvm_unreachable("Unexpected opcode"); 1623 CASE_VFMA_CHANGE_OPCODE_LMULS(FMADD, FMACC, VV) 1624 CASE_VFMA_CHANGE_OPCODE_LMULS(FMSUB, FMSAC, VV) 1625 CASE_VFMA_CHANGE_OPCODE_LMULS(FNMADD, FNMACC, VV) 1626 CASE_VFMA_CHANGE_OPCODE_LMULS(FNMSUB, FNMSAC, VV) 1627 CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV) 1628 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV) 1629 } 1630 1631 auto &WorkingMI = cloneIfNew(MI); 1632 WorkingMI.setDesc(get(Opc)); 1633 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1634 OpIdx1, OpIdx2); 1635 } 1636 // Let the default code handle it. 1637 break; 1638 } 1639 } 1640 1641 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 1642 } 1643 1644 #undef CASE_VFMA_CHANGE_OPCODE_SPLATS 1645 #undef CASE_VFMA_CHANGE_OPCODE_LMULS 1646 #undef CASE_VFMA_CHANGE_OPCODE_COMMON 1647 #undef CASE_VFMA_SPLATS 1648 #undef CASE_VFMA_OPCODE_LMULS 1649 #undef CASE_VFMA_OPCODE_COMMON 1650 1651 // clang-format off 1652 #define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \ 1653 RISCV::PseudoV##OP##_##LMUL##_TIED 1654 1655 #define CASE_WIDEOP_OPCODE_LMULS(OP) \ 1656 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \ 1657 case CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \ 1658 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \ 1659 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \ 1660 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \ 1661 case CASE_WIDEOP_OPCODE_COMMON(OP, M4) 1662 // clang-format on 1663 1664 #define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \ 1665 case RISCV::PseudoV##OP##_##LMUL##_TIED: \ 1666 NewOpc = RISCV::PseudoV##OP##_##LMUL; \ 1667 break; 1668 1669 #define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \ 1670 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \ 1671 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \ 1672 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \ 1673 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \ 1674 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \ 1675 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4) 1676 1677 MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI, 1678 LiveVariables *LV, 1679 LiveIntervals *LIS) const { 1680 switch (MI.getOpcode()) { 1681 default: 1682 break; 1683 case CASE_WIDEOP_OPCODE_LMULS(FWADD_WV): 1684 case CASE_WIDEOP_OPCODE_LMULS(FWSUB_WV): 1685 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV): 1686 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV): 1687 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV): 1688 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): { 1689 // clang-format off 1690 unsigned NewOpc; 1691 switch (MI.getOpcode()) { 1692 default: 1693 llvm_unreachable("Unexpected opcode"); 1694 CASE_WIDEOP_CHANGE_OPCODE_LMULS(FWADD_WV) 1695 CASE_WIDEOP_CHANGE_OPCODE_LMULS(FWSUB_WV) 1696 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV) 1697 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV) 1698 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV) 1699 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV) 1700 } 1701 //clang-format on 1702 1703 MachineBasicBlock &MBB = *MI.getParent(); 1704 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 1705 .add(MI.getOperand(0)) 1706 .add(MI.getOperand(1)) 1707 .add(MI.getOperand(2)) 1708 .add(MI.getOperand(3)) 1709 .add(MI.getOperand(4)); 1710 MIB.copyImplicitOps(MI); 1711 1712 if (LV) { 1713 unsigned NumOps = MI.getNumOperands(); 1714 for (unsigned I = 1; I < NumOps; ++I) { 1715 MachineOperand &Op = MI.getOperand(I); 1716 if (Op.isReg() && Op.isKill()) 1717 LV->replaceKillInstruction(Op.getReg(), MI, *MIB); 1718 } 1719 } 1720 1721 if (LIS) { 1722 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, *MIB); 1723 1724 if (MI.getOperand(0).isEarlyClobber()) { 1725 // Use operand 1 was tied to early-clobber def operand 0, so its live 1726 // interval could have ended at an early-clobber slot. Now they are not 1727 // tied we need to update it to the normal register slot. 1728 LiveInterval &LI = LIS->getInterval(MI.getOperand(1).getReg()); 1729 LiveRange::Segment *S = LI.getSegmentContaining(Idx); 1730 if (S->end == Idx.getRegSlot(true)) 1731 S->end = Idx.getRegSlot(); 1732 } 1733 } 1734 1735 return MIB; 1736 } 1737 } 1738 1739 return nullptr; 1740 } 1741 1742 #undef CASE_WIDEOP_CHANGE_OPCODE_LMULS 1743 #undef CASE_WIDEOP_CHANGE_OPCODE_COMMON 1744 #undef CASE_WIDEOP_OPCODE_LMULS 1745 #undef CASE_WIDEOP_OPCODE_COMMON 1746 1747 Register RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF, 1748 MachineBasicBlock &MBB, 1749 MachineBasicBlock::iterator II, 1750 const DebugLoc &DL, 1751 int64_t Amount, 1752 MachineInstr::MIFlag Flag) const { 1753 assert(Amount > 0 && "There is no need to get VLEN scaled value."); 1754 assert(Amount % 8 == 0 && 1755 "Reserve the stack by the multiple of one vector size."); 1756 1757 MachineRegisterInfo &MRI = MF.getRegInfo(); 1758 const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo(); 1759 int64_t NumOfVReg = Amount / 8; 1760 1761 Register VL = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1762 BuildMI(MBB, II, DL, TII->get(RISCV::PseudoReadVLENB), VL) 1763 .setMIFlag(Flag); 1764 assert(isInt<32>(NumOfVReg) && 1765 "Expect the number of vector registers within 32-bits."); 1766 if (isPowerOf2_32(NumOfVReg)) { 1767 uint32_t ShiftAmount = Log2_32(NumOfVReg); 1768 if (ShiftAmount == 0) 1769 return VL; 1770 BuildMI(MBB, II, DL, TII->get(RISCV::SLLI), VL) 1771 .addReg(VL, RegState::Kill) 1772 .addImm(ShiftAmount) 1773 .setMIFlag(Flag); 1774 } else if (isPowerOf2_32(NumOfVReg - 1)) { 1775 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1776 uint32_t ShiftAmount = Log2_32(NumOfVReg - 1); 1777 BuildMI(MBB, II, DL, TII->get(RISCV::SLLI), ScaledRegister) 1778 .addReg(VL) 1779 .addImm(ShiftAmount) 1780 .setMIFlag(Flag); 1781 BuildMI(MBB, II, DL, TII->get(RISCV::ADD), VL) 1782 .addReg(ScaledRegister, RegState::Kill) 1783 .addReg(VL, RegState::Kill) 1784 .setMIFlag(Flag); 1785 } else if (isPowerOf2_32(NumOfVReg + 1)) { 1786 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1787 uint32_t ShiftAmount = Log2_32(NumOfVReg + 1); 1788 BuildMI(MBB, II, DL, TII->get(RISCV::SLLI), ScaledRegister) 1789 .addReg(VL) 1790 .addImm(ShiftAmount) 1791 .setMIFlag(Flag); 1792 BuildMI(MBB, II, DL, TII->get(RISCV::SUB), VL) 1793 .addReg(ScaledRegister, RegState::Kill) 1794 .addReg(VL, RegState::Kill) 1795 .setMIFlag(Flag); 1796 } else { 1797 Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1798 if (!isInt<12>(NumOfVReg)) 1799 movImm(MBB, II, DL, N, NumOfVReg); 1800 else { 1801 BuildMI(MBB, II, DL, TII->get(RISCV::ADDI), N) 1802 .addReg(RISCV::X0) 1803 .addImm(NumOfVReg) 1804 .setMIFlag(Flag); 1805 } 1806 if (!MF.getSubtarget<RISCVSubtarget>().hasStdExtM()) 1807 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{ 1808 MF.getFunction(), 1809 "M-extension must be enabled to calculate the vscaled size/offset."}); 1810 BuildMI(MBB, II, DL, TII->get(RISCV::MUL), VL) 1811 .addReg(VL, RegState::Kill) 1812 .addReg(N, RegState::Kill) 1813 .setMIFlag(Flag); 1814 } 1815 1816 return VL; 1817 } 1818 1819 static bool isRVVWholeLoadStore(unsigned Opcode) { 1820 switch (Opcode) { 1821 default: 1822 return false; 1823 case RISCV::VS1R_V: 1824 case RISCV::VS2R_V: 1825 case RISCV::VS4R_V: 1826 case RISCV::VS8R_V: 1827 case RISCV::VL1RE8_V: 1828 case RISCV::VL2RE8_V: 1829 case RISCV::VL4RE8_V: 1830 case RISCV::VL8RE8_V: 1831 case RISCV::VL1RE16_V: 1832 case RISCV::VL2RE16_V: 1833 case RISCV::VL4RE16_V: 1834 case RISCV::VL8RE16_V: 1835 case RISCV::VL1RE32_V: 1836 case RISCV::VL2RE32_V: 1837 case RISCV::VL4RE32_V: 1838 case RISCV::VL8RE32_V: 1839 case RISCV::VL1RE64_V: 1840 case RISCV::VL2RE64_V: 1841 case RISCV::VL4RE64_V: 1842 case RISCV::VL8RE64_V: 1843 return true; 1844 } 1845 } 1846 1847 bool RISCVInstrInfo::isRVVSpill(const MachineInstr &MI, bool CheckFIs) const { 1848 // RVV lacks any support for immediate addressing for stack addresses, so be 1849 // conservative. 1850 unsigned Opcode = MI.getOpcode(); 1851 if (!RISCVVPseudosTable::getPseudoInfo(Opcode) && 1852 !isRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode)) 1853 return false; 1854 return !CheckFIs || any_of(MI.operands(), [](const MachineOperand &MO) { 1855 return MO.isFI(); 1856 }); 1857 } 1858 1859 Optional<std::pair<unsigned, unsigned>> 1860 RISCVInstrInfo::isRVVSpillForZvlsseg(unsigned Opcode) const { 1861 switch (Opcode) { 1862 default: 1863 return None; 1864 case RISCV::PseudoVSPILL2_M1: 1865 case RISCV::PseudoVRELOAD2_M1: 1866 return std::make_pair(2u, 1u); 1867 case RISCV::PseudoVSPILL2_M2: 1868 case RISCV::PseudoVRELOAD2_M2: 1869 return std::make_pair(2u, 2u); 1870 case RISCV::PseudoVSPILL2_M4: 1871 case RISCV::PseudoVRELOAD2_M4: 1872 return std::make_pair(2u, 4u); 1873 case RISCV::PseudoVSPILL3_M1: 1874 case RISCV::PseudoVRELOAD3_M1: 1875 return std::make_pair(3u, 1u); 1876 case RISCV::PseudoVSPILL3_M2: 1877 case RISCV::PseudoVRELOAD3_M2: 1878 return std::make_pair(3u, 2u); 1879 case RISCV::PseudoVSPILL4_M1: 1880 case RISCV::PseudoVRELOAD4_M1: 1881 return std::make_pair(4u, 1u); 1882 case RISCV::PseudoVSPILL4_M2: 1883 case RISCV::PseudoVRELOAD4_M2: 1884 return std::make_pair(4u, 2u); 1885 case RISCV::PseudoVSPILL5_M1: 1886 case RISCV::PseudoVRELOAD5_M1: 1887 return std::make_pair(5u, 1u); 1888 case RISCV::PseudoVSPILL6_M1: 1889 case RISCV::PseudoVRELOAD6_M1: 1890 return std::make_pair(6u, 1u); 1891 case RISCV::PseudoVSPILL7_M1: 1892 case RISCV::PseudoVRELOAD7_M1: 1893 return std::make_pair(7u, 1u); 1894 case RISCV::PseudoVSPILL8_M1: 1895 case RISCV::PseudoVRELOAD8_M1: 1896 return std::make_pair(8u, 1u); 1897 } 1898 } 1899