1 //===- MachineCopyPropagation.cpp - Machine Copy Propagation Pass ---------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This is an extremely simple MachineInstr-level copy propagation pass. 11 // 12 // This pass forwards the source of COPYs to the users of their destinations 13 // when doing so is legal. For example: 14 // 15 // %reg1 = COPY %reg0 16 // ... 17 // ... = OP %reg1 18 // 19 // If 20 // - %reg0 has not been clobbered by the time of the use of %reg1 21 // - the register class constraints are satisfied 22 // - the COPY def is the only value that reaches OP 23 // then this pass replaces the above with: 24 // 25 // %reg1 = COPY %reg0 26 // ... 27 // ... = OP %reg0 28 // 29 // This pass also removes some redundant COPYs. For example: 30 // 31 // %R1 = COPY %R0 32 // ... // No clobber of %R1 33 // %R0 = COPY %R1 <<< Removed 34 // 35 // or 36 // 37 // %R1 = COPY %R0 38 // ... // No clobber of %R0 39 // %R1 = COPY %R0 <<< Removed 40 // 41 //===----------------------------------------------------------------------===// 42 43 #include "llvm/ADT/DenseMap.h" 44 #include "llvm/ADT/STLExtras.h" 45 #include "llvm/ADT/SetVector.h" 46 #include "llvm/ADT/SmallVector.h" 47 #include "llvm/ADT/Statistic.h" 48 #include "llvm/ADT/iterator_range.h" 49 #include "llvm/CodeGen/MachineBasicBlock.h" 50 #include "llvm/CodeGen/MachineFunction.h" 51 #include "llvm/CodeGen/MachineFunctionPass.h" 52 #include "llvm/CodeGen/MachineInstr.h" 53 #include "llvm/CodeGen/MachineOperand.h" 54 #include "llvm/CodeGen/MachineRegisterInfo.h" 55 #include "llvm/CodeGen/TargetInstrInfo.h" 56 #include "llvm/CodeGen/TargetRegisterInfo.h" 57 #include "llvm/CodeGen/TargetSubtargetInfo.h" 58 #include "llvm/MC/MCRegisterInfo.h" 59 #include "llvm/Pass.h" 60 #include "llvm/Support/Debug.h" 61 #include "llvm/Support/DebugCounter.h" 62 #include "llvm/Support/raw_ostream.h" 63 #include <cassert> 64 #include <iterator> 65 66 using namespace llvm; 67 68 #define DEBUG_TYPE "machine-cp" 69 70 STATISTIC(NumDeletes, "Number of dead copies deleted"); 71 STATISTIC(NumCopyForwards, "Number of copy uses forwarded"); 72 DEBUG_COUNTER(FwdCounter, "machine-cp-fwd", 73 "Controls which register COPYs are forwarded"); 74 75 namespace { 76 77 using RegList = SmallVector<unsigned, 4>; 78 using SourceMap = DenseMap<unsigned, RegList>; 79 using Reg2MIMap = DenseMap<unsigned, MachineInstr *>; 80 81 class MachineCopyPropagation : public MachineFunctionPass { 82 const TargetRegisterInfo *TRI; 83 const TargetInstrInfo *TII; 84 const MachineRegisterInfo *MRI; 85 86 public: 87 static char ID; // Pass identification, replacement for typeid 88 89 MachineCopyPropagation() : MachineFunctionPass(ID) { 90 initializeMachineCopyPropagationPass(*PassRegistry::getPassRegistry()); 91 } 92 93 void getAnalysisUsage(AnalysisUsage &AU) const override { 94 AU.setPreservesCFG(); 95 MachineFunctionPass::getAnalysisUsage(AU); 96 } 97 98 bool runOnMachineFunction(MachineFunction &MF) override; 99 100 MachineFunctionProperties getRequiredProperties() const override { 101 return MachineFunctionProperties().set( 102 MachineFunctionProperties::Property::NoVRegs); 103 } 104 105 private: 106 void ClobberRegister(unsigned Reg); 107 void ReadRegister(unsigned Reg); 108 void CopyPropagateBlock(MachineBasicBlock &MBB); 109 bool eraseIfRedundant(MachineInstr &Copy, unsigned Src, unsigned Def); 110 void forwardUses(MachineInstr &MI); 111 bool isForwardableRegClassCopy(const MachineInstr &Copy, 112 const MachineInstr &UseI, unsigned UseIdx); 113 bool hasImplicitOverlap(const MachineInstr &MI, const MachineOperand &Use); 114 115 /// Candidates for deletion. 116 SmallSetVector<MachineInstr*, 8> MaybeDeadCopies; 117 118 /// Def -> available copies map. 119 Reg2MIMap AvailCopyMap; 120 121 /// Def -> copies map. 122 Reg2MIMap CopyMap; 123 124 /// Src -> Def map 125 SourceMap SrcMap; 126 127 bool Changed; 128 }; 129 130 } // end anonymous namespace 131 132 char MachineCopyPropagation::ID = 0; 133 134 char &llvm::MachineCopyPropagationID = MachineCopyPropagation::ID; 135 136 INITIALIZE_PASS(MachineCopyPropagation, DEBUG_TYPE, 137 "Machine Copy Propagation Pass", false, false) 138 139 /// Remove any entry in \p Map where the register is a subregister or equal to 140 /// a register contained in \p Regs. 141 static void removeRegsFromMap(Reg2MIMap &Map, const RegList &Regs, 142 const TargetRegisterInfo &TRI) { 143 for (unsigned Reg : Regs) { 144 // Source of copy is no longer available for propagation. 145 for (MCSubRegIterator SR(Reg, &TRI, true); SR.isValid(); ++SR) 146 Map.erase(*SR); 147 } 148 } 149 150 /// Remove any entry in \p Map that is marked clobbered in \p RegMask. 151 /// The map will typically have a lot fewer entries than the regmask clobbers, 152 /// so this is more efficient than iterating the clobbered registers and calling 153 /// ClobberRegister() on them. 154 static void removeClobberedRegsFromMap(Reg2MIMap &Map, 155 const MachineOperand &RegMask) { 156 for (Reg2MIMap::iterator I = Map.begin(), E = Map.end(), Next; I != E; 157 I = Next) { 158 Next = std::next(I); 159 unsigned Reg = I->first; 160 if (RegMask.clobbersPhysReg(Reg)) 161 Map.erase(I); 162 } 163 } 164 165 void MachineCopyPropagation::ClobberRegister(unsigned Reg) { 166 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) { 167 CopyMap.erase(*AI); 168 AvailCopyMap.erase(*AI); 169 170 SourceMap::iterator SI = SrcMap.find(*AI); 171 if (SI != SrcMap.end()) { 172 removeRegsFromMap(AvailCopyMap, SI->second, *TRI); 173 SrcMap.erase(SI); 174 } 175 } 176 } 177 178 void MachineCopyPropagation::ReadRegister(unsigned Reg) { 179 // If 'Reg' is defined by a copy, the copy is no longer a candidate 180 // for elimination. 181 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) { 182 Reg2MIMap::iterator CI = CopyMap.find(*AI); 183 if (CI != CopyMap.end()) { 184 DEBUG(dbgs() << "MCP: Copy is used - not dead: "; CI->second->dump()); 185 MaybeDeadCopies.remove(CI->second); 186 } 187 } 188 } 189 190 /// Return true if \p PreviousCopy did copy register \p Src to register \p Def. 191 /// This fact may have been obscured by sub register usage or may not be true at 192 /// all even though Src and Def are subregisters of the registers used in 193 /// PreviousCopy. e.g. 194 /// isNopCopy("ecx = COPY eax", AX, CX) == true 195 /// isNopCopy("ecx = COPY eax", AH, CL) == false 196 static bool isNopCopy(const MachineInstr &PreviousCopy, unsigned Src, 197 unsigned Def, const TargetRegisterInfo *TRI) { 198 unsigned PreviousSrc = PreviousCopy.getOperand(1).getReg(); 199 unsigned PreviousDef = PreviousCopy.getOperand(0).getReg(); 200 if (Src == PreviousSrc) { 201 assert(Def == PreviousDef); 202 return true; 203 } 204 if (!TRI->isSubRegister(PreviousSrc, Src)) 205 return false; 206 unsigned SubIdx = TRI->getSubRegIndex(PreviousSrc, Src); 207 return SubIdx == TRI->getSubRegIndex(PreviousDef, Def); 208 } 209 210 /// Remove instruction \p Copy if there exists a previous copy that copies the 211 /// register \p Src to the register \p Def; This may happen indirectly by 212 /// copying the super registers. 213 bool MachineCopyPropagation::eraseIfRedundant(MachineInstr &Copy, unsigned Src, 214 unsigned Def) { 215 // Avoid eliminating a copy from/to a reserved registers as we cannot predict 216 // the value (Example: The sparc zero register is writable but stays zero). 217 if (MRI->isReserved(Src) || MRI->isReserved(Def)) 218 return false; 219 220 // Search for an existing copy. 221 Reg2MIMap::iterator CI = AvailCopyMap.find(Def); 222 if (CI == AvailCopyMap.end()) 223 return false; 224 225 // Check that the existing copy uses the correct sub registers. 226 MachineInstr &PrevCopy = *CI->second; 227 if (PrevCopy.getOperand(0).isDead()) 228 return false; 229 if (!isNopCopy(PrevCopy, Src, Def, TRI)) 230 return false; 231 232 DEBUG(dbgs() << "MCP: copy is a NOP, removing: "; Copy.dump()); 233 234 // Copy was redundantly redefining either Src or Def. Remove earlier kill 235 // flags between Copy and PrevCopy because the value will be reused now. 236 assert(Copy.isCopy()); 237 unsigned CopyDef = Copy.getOperand(0).getReg(); 238 assert(CopyDef == Src || CopyDef == Def); 239 for (MachineInstr &MI : 240 make_range(PrevCopy.getIterator(), Copy.getIterator())) 241 MI.clearRegisterKills(CopyDef, TRI); 242 243 Copy.eraseFromParent(); 244 Changed = true; 245 ++NumDeletes; 246 return true; 247 } 248 249 /// Decide whether we should forward the source of \param Copy to its use in 250 /// \param UseI based on the physical register class constraints of the opcode 251 /// and avoiding introducing more cross-class COPYs. 252 bool MachineCopyPropagation::isForwardableRegClassCopy(const MachineInstr &Copy, 253 const MachineInstr &UseI, 254 unsigned UseIdx) { 255 256 unsigned CopySrcReg = Copy.getOperand(1).getReg(); 257 258 // If the new register meets the opcode register constraints, then allow 259 // forwarding. 260 if (const TargetRegisterClass *URC = 261 UseI.getRegClassConstraint(UseIdx, TII, TRI)) 262 return URC->contains(CopySrcReg); 263 264 if (!UseI.isCopy()) 265 return false; 266 267 /// COPYs don't have register class constraints, so if the user instruction 268 /// is a COPY, we just try to avoid introducing additional cross-class 269 /// COPYs. For example: 270 /// 271 /// RegClassA = COPY RegClassB // Copy parameter 272 /// ... 273 /// RegClassB = COPY RegClassA // UseI parameter 274 /// 275 /// which after forwarding becomes 276 /// 277 /// RegClassA = COPY RegClassB 278 /// ... 279 /// RegClassB = COPY RegClassB 280 /// 281 /// so we have reduced the number of cross-class COPYs and potentially 282 /// introduced a nop COPY that can be removed. 283 const TargetRegisterClass *UseDstRC = 284 TRI->getMinimalPhysRegClass(UseI.getOperand(0).getReg()); 285 286 const TargetRegisterClass *SuperRC = UseDstRC; 287 for (TargetRegisterClass::sc_iterator SuperRCI = UseDstRC->getSuperClasses(); 288 SuperRC; SuperRC = *SuperRCI++) 289 if (SuperRC->contains(CopySrcReg)) 290 return true; 291 292 return false; 293 } 294 295 /// Check that \p MI does not have implicit uses that overlap with it's \p Use 296 /// operand (the register being replaced), since these can sometimes be 297 /// implicitly tied to other operands. For example, on AMDGPU: 298 /// 299 /// V_MOVRELS_B32_e32 %VGPR2, %M0<imp-use>, %EXEC<imp-use>, %VGPR2_VGPR3_VGPR4_VGPR5<imp-use> 300 /// 301 /// the %VGPR2 is implicitly tied to the larger reg operand, but we have no 302 /// way of knowing we need to update the latter when updating the former. 303 bool MachineCopyPropagation::hasImplicitOverlap(const MachineInstr &MI, 304 const MachineOperand &Use) { 305 for (const MachineOperand &MIUse : MI.uses()) 306 if (&MIUse != &Use && MIUse.isReg() && MIUse.isImplicit() && 307 MIUse.isUse() && TRI->regsOverlap(Use.getReg(), MIUse.getReg())) 308 return true; 309 310 return false; 311 } 312 313 /// Look for available copies whose destination register is used by \p MI and 314 /// replace the use in \p MI with the copy's source register. 315 void MachineCopyPropagation::forwardUses(MachineInstr &MI) { 316 if (AvailCopyMap.empty()) 317 return; 318 319 // Look for non-tied explicit vreg uses that have an active COPY 320 // instruction that defines the physical register allocated to them. 321 // Replace the vreg with the source of the active COPY. 322 for (unsigned OpIdx = 0, OpEnd = MI.getNumOperands(); OpIdx < OpEnd; 323 ++OpIdx) { 324 MachineOperand &MOUse = MI.getOperand(OpIdx); 325 // Don't forward into undef use operands since doing so can cause problems 326 // with the machine verifier, since it doesn't treat undef reads as reads, 327 // so we can end up with a live range that ends on an undef read, leading to 328 // an error that the live range doesn't end on a read of the live range 329 // register. 330 if (!MOUse.isReg() || MOUse.isTied() || MOUse.isUndef() || MOUse.isDef() || 331 MOUse.isImplicit()) 332 continue; 333 334 if (!MOUse.getReg()) 335 continue; 336 337 // Check that the register is marked 'renamable' so we know it is safe to 338 // rename it without violating any constraints that aren't expressed in the 339 // IR (e.g. ABI or opcode requirements). 340 if (!MOUse.isRenamable()) 341 continue; 342 343 auto CI = AvailCopyMap.find(MOUse.getReg()); 344 if (CI == AvailCopyMap.end()) 345 continue; 346 347 MachineInstr &Copy = *CI->second; 348 unsigned CopyDstReg = Copy.getOperand(0).getReg(); 349 const MachineOperand &CopySrc = Copy.getOperand(1); 350 unsigned CopySrcReg = CopySrc.getReg(); 351 352 // FIXME: Don't handle partial uses of wider COPYs yet. 353 if (MOUse.getReg() != CopyDstReg) { 354 DEBUG(dbgs() << "MCP: FIXME! Not forwarding COPY to sub-register use:\n " 355 << MI); 356 continue; 357 } 358 359 // Don't forward COPYs of reserved regs unless they are constant. 360 if (MRI->isReserved(CopySrcReg) && !MRI->isConstantPhysReg(CopySrcReg)) 361 continue; 362 363 if (!isForwardableRegClassCopy(Copy, MI, OpIdx)) 364 continue; 365 366 if (hasImplicitOverlap(MI, MOUse)) 367 continue; 368 369 if (!DebugCounter::shouldExecute(FwdCounter)) { 370 DEBUG(dbgs() << "MCP: Skipping forwarding due to debug counter:\n " 371 << MI); 372 continue; 373 } 374 375 DEBUG(dbgs() << "MCP: Replacing " << printReg(MOUse.getReg(), TRI) 376 << "\n with " << printReg(CopySrcReg, TRI) << "\n in " 377 << MI << " from " << Copy); 378 379 MOUse.setReg(CopySrcReg); 380 if (!CopySrc.isRenamable()) 381 MOUse.setIsRenamable(false); 382 383 DEBUG(dbgs() << "MCP: After replacement: " << MI << "\n"); 384 385 // Clear kill markers that may have been invalidated. 386 for (MachineInstr &KMI : 387 make_range(Copy.getIterator(), std::next(MI.getIterator()))) 388 KMI.clearRegisterKills(CopySrcReg, TRI); 389 390 ++NumCopyForwards; 391 Changed = true; 392 } 393 } 394 395 void MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) { 396 DEBUG(dbgs() << "MCP: CopyPropagateBlock " << MBB.getName() << "\n"); 397 398 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ) { 399 MachineInstr *MI = &*I; 400 ++I; 401 402 // Analyze copies (which don't overlap themselves). 403 if (MI->isCopy() && !TRI->regsOverlap(MI->getOperand(0).getReg(), 404 MI->getOperand(1).getReg())) { 405 unsigned Def = MI->getOperand(0).getReg(); 406 unsigned Src = MI->getOperand(1).getReg(); 407 408 assert(!TargetRegisterInfo::isVirtualRegister(Def) && 409 !TargetRegisterInfo::isVirtualRegister(Src) && 410 "MachineCopyPropagation should be run after register allocation!"); 411 412 // The two copies cancel out and the source of the first copy 413 // hasn't been overridden, eliminate the second one. e.g. 414 // %ecx = COPY %eax 415 // ... nothing clobbered eax. 416 // %eax = COPY %ecx 417 // => 418 // %ecx = COPY %eax 419 // 420 // or 421 // 422 // %ecx = COPY %eax 423 // ... nothing clobbered eax. 424 // %ecx = COPY %eax 425 // => 426 // %ecx = COPY %eax 427 if (eraseIfRedundant(*MI, Def, Src) || eraseIfRedundant(*MI, Src, Def)) 428 continue; 429 430 forwardUses(*MI); 431 432 // Src may have been changed by forwardUses() 433 Src = MI->getOperand(1).getReg(); 434 435 // If Src is defined by a previous copy, the previous copy cannot be 436 // eliminated. 437 ReadRegister(Src); 438 for (const MachineOperand &MO : MI->implicit_operands()) { 439 if (!MO.isReg() || !MO.readsReg()) 440 continue; 441 unsigned Reg = MO.getReg(); 442 if (!Reg) 443 continue; 444 ReadRegister(Reg); 445 } 446 447 DEBUG(dbgs() << "MCP: Copy is a deletion candidate: "; MI->dump()); 448 449 // Copy is now a candidate for deletion. 450 if (!MRI->isReserved(Def)) 451 MaybeDeadCopies.insert(MI); 452 453 // If 'Def' is previously source of another copy, then this earlier copy's 454 // source is no longer available. e.g. 455 // %xmm9 = copy %xmm2 456 // ... 457 // %xmm2 = copy %xmm0 458 // ... 459 // %xmm2 = copy %xmm9 460 ClobberRegister(Def); 461 for (const MachineOperand &MO : MI->implicit_operands()) { 462 if (!MO.isReg() || !MO.isDef()) 463 continue; 464 unsigned Reg = MO.getReg(); 465 if (!Reg) 466 continue; 467 ClobberRegister(Reg); 468 } 469 470 // Remember Def is defined by the copy. 471 for (MCSubRegIterator SR(Def, TRI, /*IncludeSelf=*/true); SR.isValid(); 472 ++SR) { 473 CopyMap[*SR] = MI; 474 AvailCopyMap[*SR] = MI; 475 } 476 477 // Remember source that's copied to Def. Once it's clobbered, then 478 // it's no longer available for copy propagation. 479 RegList &DestList = SrcMap[Src]; 480 if (!is_contained(DestList, Def)) 481 DestList.push_back(Def); 482 483 continue; 484 } 485 486 // Clobber any earlyclobber regs first. 487 for (const MachineOperand &MO : MI->operands()) 488 if (MO.isReg() && MO.isEarlyClobber()) { 489 unsigned Reg = MO.getReg(); 490 // If we have a tied earlyclobber, that means it is also read by this 491 // instruction, so we need to make sure we don't remove it as dead 492 // later. 493 if (MO.isTied()) 494 ReadRegister(Reg); 495 ClobberRegister(Reg); 496 } 497 498 forwardUses(*MI); 499 500 // Not a copy. 501 SmallVector<unsigned, 2> Defs; 502 const MachineOperand *RegMask = nullptr; 503 for (const MachineOperand &MO : MI->operands()) { 504 if (MO.isRegMask()) 505 RegMask = &MO; 506 if (!MO.isReg()) 507 continue; 508 unsigned Reg = MO.getReg(); 509 if (!Reg) 510 continue; 511 512 assert(!TargetRegisterInfo::isVirtualRegister(Reg) && 513 "MachineCopyPropagation should be run after register allocation!"); 514 515 if (MO.isDef() && !MO.isEarlyClobber()) { 516 Defs.push_back(Reg); 517 continue; 518 } else if (MO.readsReg()) 519 ReadRegister(Reg); 520 } 521 522 // The instruction has a register mask operand which means that it clobbers 523 // a large set of registers. Treat clobbered registers the same way as 524 // defined registers. 525 if (RegMask) { 526 // Erase any MaybeDeadCopies whose destination register is clobbered. 527 for (SmallSetVector<MachineInstr *, 8>::iterator DI = 528 MaybeDeadCopies.begin(); 529 DI != MaybeDeadCopies.end();) { 530 MachineInstr *MaybeDead = *DI; 531 unsigned Reg = MaybeDead->getOperand(0).getReg(); 532 assert(!MRI->isReserved(Reg)); 533 534 if (!RegMask->clobbersPhysReg(Reg)) { 535 ++DI; 536 continue; 537 } 538 539 DEBUG(dbgs() << "MCP: Removing copy due to regmask clobbering: "; 540 MaybeDead->dump()); 541 542 // erase() will return the next valid iterator pointing to the next 543 // element after the erased one. 544 DI = MaybeDeadCopies.erase(DI); 545 MaybeDead->eraseFromParent(); 546 Changed = true; 547 ++NumDeletes; 548 } 549 550 removeClobberedRegsFromMap(AvailCopyMap, *RegMask); 551 removeClobberedRegsFromMap(CopyMap, *RegMask); 552 for (SourceMap::iterator I = SrcMap.begin(), E = SrcMap.end(), Next; 553 I != E; I = Next) { 554 Next = std::next(I); 555 if (RegMask->clobbersPhysReg(I->first)) { 556 removeRegsFromMap(AvailCopyMap, I->second, *TRI); 557 SrcMap.erase(I); 558 } 559 } 560 } 561 562 // Any previous copy definition or reading the Defs is no longer available. 563 for (unsigned Reg : Defs) 564 ClobberRegister(Reg); 565 } 566 567 // If MBB doesn't have successors, delete the copies whose defs are not used. 568 // If MBB does have successors, then conservative assume the defs are live-out 569 // since we don't want to trust live-in lists. 570 if (MBB.succ_empty()) { 571 for (MachineInstr *MaybeDead : MaybeDeadCopies) { 572 DEBUG(dbgs() << "MCP: Removing copy due to no live-out succ: "; 573 MaybeDead->dump()); 574 assert(!MRI->isReserved(MaybeDead->getOperand(0).getReg())); 575 MaybeDead->eraseFromParent(); 576 Changed = true; 577 ++NumDeletes; 578 } 579 } 580 581 MaybeDeadCopies.clear(); 582 AvailCopyMap.clear(); 583 CopyMap.clear(); 584 SrcMap.clear(); 585 } 586 587 bool MachineCopyPropagation::runOnMachineFunction(MachineFunction &MF) { 588 if (skipFunction(MF.getFunction())) 589 return false; 590 591 Changed = false; 592 593 TRI = MF.getSubtarget().getRegisterInfo(); 594 TII = MF.getSubtarget().getInstrInfo(); 595 MRI = &MF.getRegInfo(); 596 597 for (MachineBasicBlock &MBB : MF) 598 CopyPropagateBlock(MBB); 599 600 return Changed; 601 } 602