Lines Matching +full:block +full:- +full:copy
1 //====- X86FlagsCopyLowering.cpp - Lowers COPY nodes of EFLAGS ------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// Lowers COPY nodes of EFLAGS by directly extracting and preserving individual
17 /// and other non-status flags being overwritten. Using sequences involving
21 //===----------------------------------------------------------------------===//
62 #define PASS_KEY "x86-flags-copy-lowering"
80 StringRef getPassName() const override { return "X86 EFLAGS copy lowering"; }
120 "X86 EFLAGS copy lowering", false, false)
122 "X86 EFLAGS copy lowering", false, false)
146 "Split instruction must be in the split block!");
157 assert(!std::prev(PrevI.getIterator())->isTerminator() &&
163 // Analyze the original block to see if we are actually splitting an edge
180 // Insert the new block immediately after the current one. Any existing
181 // fallthrough will be sunk into this new block anyways.
184 // Splice the tail of instructions into the new block.
187 // Copy the necessary succesors (and their probability info) into the new
188 // block.
196 // Now replace all of the moved successors in the original block with the new
197 // block. This will merge their probabilities.
204 "Failed to make the new block a successor!");
216 assert(OpMBB.isMBB() && "Block operand to a PHI is not a block!");
247 if (FlagDef->isDead() && X86::getNFVariant(MI.getOpcode()))
259 TII = Subtarget->getInstrInfo();
260 TRI = Subtarget->getRegisterInfo();
267 if (none_of(MRI->def_instructions(X86::EFLAGS), [](const MachineInstr &MI) {
268 return MI.getOpcode() == TargetOpcode::COPY;
280 MDT = &MDTWrapper->getDomTree();
283 OwnedMDT->getBase().recalculate(MF);
287 // Collect the copies in RPO so that when there are chains where a copy is in
295 if (MI.getOpcode() == TargetOpcode::COPY &&
299 // Try to elminate the copys by transform the instructions between copy and
302 // %1:gr64 = COPY $eflags
303 // OP1 implicit-def dead $eflags
304 // $eflags = COPY %1
307 // ->
311 if (Subtarget->hasNF()) {
319 MachineOperand &VOp = CopyI->getOperand(1);
320 MachineInstr *CopyDefI = MRI->getVRegDef(VOp.getReg());
321 MachineBasicBlock *CopyIMBB = CopyI->getParent();
322 MachineBasicBlock *CopyDefIMBB = CopyDefI->getParent();
323 // Walk all basic blocks reachable in depth-first iteration on the inverse
327 // has to be NF-convertible if it clobbers flags.
332 ? MBB->begin()
334 E = (MBB != CopyIMBB) ? MBB->end()
351 CopyI->eraseFromParent();
352 if (MRI->use_nodbg_empty(CopyDefI->getOperand(0).getReg())) {
354 CopyDefI->eraseFromParent();
358 unsigned NewOpc = X86::getNFVariant(Clobber->getOpcode());
360 Clobber->setDesc(TII->get(NewOpc));
361 Clobber->removeOperand(
362 Clobber->findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr)
363 ->getOperandNo());
370 BI->addLiveIn(X86::EFLAGS);
381 // %1:gr64 = COPY $eflags
382 // OP1 implicit-def dead $eflags
383 // $eflags = COPY %1
386 // ->
389 // OP1 implicit-def dead $eflags
390 // TEST8rr %1, %1, implicit-def $eflags
393 MachineBasicBlock &MBB = *CopyI->getParent();
395 MachineOperand &VOp = CopyI->getOperand(1);
397 "The input to the copy for EFLAGS should always be a register!");
398 MachineInstr &CopyDefI = *MRI->getVRegDef(VOp.getReg());
399 if (CopyDefI.getOpcode() != TargetOpcode::COPY) {
406 // we *might* want, computing them in every place where we copy *out* of
415 dbgs() << "ERROR: Encountered unexpected def of an eflags copy: ";
418 "Cannot lower EFLAGS copy unless it is defined in turn by a copy!");
422 // All uses of the EFLAGS copy are now rewritten, kill the copy into
423 // eflags and if dead the copy from.
424 CopyI->eraseFromParent();
425 if (MRI->use_empty(CopyDefI.getOperand(0).getReg()))
430 MachineOperand &DOp = CopyI->getOperand(0);
432 assert(DOp.getReg() == X86::EFLAGS && "Unexpected copy def register!");
440 LLVM_DEBUG(dbgs() << "Rewriting copy: "; CopyI->dump());
442 // Walk up across live-in EFLAGS to find where they were actually def'ed.
444 // This copy's def may just be part of a region of blocks covered by
458 // Flag any instruction (other than the copy we are
466 assert(MDT->dominates(BeginMBB, EndMBB) &&
475 for (auto *PredMBB : MBB->predecessors()) {
478 if (HasEFLAGSClobber(PredMBB->begin(), PredMBB->end()))
480 // Enqueue this block to walk its predecessors.
487 while (TestMBB->isLiveIn(X86::EFLAGS) && !TestMBB->pred_empty() &&
488 !HasEFLAGSClobber(TestMBB->begin(), TestPos)) {
492 std::accumulate(std::next(TestMBB->pred_begin()), TestMBB->pred_end(),
493 *TestMBB->pred_begin(),
495 return MDT->findNearestCommonDominator(LHS, RHS);
499 // the hoist block. A clobber anywhere in any of these blocks the hoist.
505 if (HasEFLAGSClobber(HoistMBB->getFirstTerminator()->getIterator(),
506 HoistMBB->instr_end()))
511 TestPos = TestMBB->getFirstTerminator()->getIterator();
517 llvm::reverse(llvm::make_range(TestMBB->instr_begin(), TestPos)),
521 if (DefIt.base() != TestMBB->instr_begin()) {
523 DefIt->dump();
525 dbgs() << " Using live-in flags for BB:\n";
526 TestMBB->dump();
537 // very few of them and we expect to not revisit the same copy definition
543 // a single basic block but we may have to scan multiple blocks if the
544 // EFLAGS copy lives into successors.
552 // Track when if/when we find a kill of the flags in this block.
555 // In most cases, we walk from the beginning to the end of the block. But
556 // when the block is the same block as the copy is from, we will visit it
557 // twice. The first time we start from the copy and go to the end. The
558 // second time we start from the beginning and go to the copy. This lets
565 ? std::next(CopyI->getIterator())
570 // If we are in the original copy block and encounter either the copy
571 // def or the copy itself, break so that we don't re-process any part of
572 // the block or process the instructions in the range that was copied
577 "original block.");
593 if (FlagUse->isKill())
600 // tail calls, as those are not introduced into the X86 MI until post-RA
614 // Otherwise we can just rewrite in-place.
616 if (Opc == TargetOpcode::COPY) {
617 // Just replace this copy with the original copy def.
618 MRI->replaceRegWith(MI.getOperand(0).getReg(),
634 // If the flags were killed, we're done with this block.
638 // Otherwise we need to scan successors for ones where the flags live-in
641 if (SuccMBB->isLiveIn(X86::EFLAGS) &&
644 // test basic block dominates all of the use basic blocks. Further, we
645 // can't have a cycle from the test block back to itself as that would
650 // this EFLAGS copy along. This isn't as bad as fully general PHI
655 // a hard check even in non-debug builds here.
656 if (SuccMBB == TestMBB || !MDT->dominates(TestMBB, SuccMBB)) {
660 "basic block! Rewriting this would require inserting PHI "
662 "block:\n";
663 TestMBB->dump();
664 dbgs() << "Use block:\n";
665 SuccMBB->dump();
668 "Cannot lower EFLAGS copy when original copy def "
675 SuccMBB->removeLiveIn(X86::EFLAGS);
680 // because if there are multiple jumps in a single basic block we'll have
684 // Past the first jump within a basic block we need to split the blocks
686 if (JmpI->getParent() == LastJmpMBB)
687 splitBlock(*JmpI->getParent(), *JmpI, *TII);
689 LastJmpMBB = JmpI->getParent();
694 // FIXME: Mark the last use of EFLAGS before the copy's def as a kill if
695 // the copy's def operand is itself a kill.
701 if (MI.getOpcode() == TargetOpcode::COPY &&
704 LLVM_DEBUG(dbgs() << "ERROR: Found a COPY involving EFLAGS: ";
706 llvm_unreachable("Unlowered EFLAGS copy!");
714 /// can re-use them rather than adding duplicates.
726 "A non-storing SETcc should always define a register!");
741 Register Reg = MRI->createVirtualRegister(PromoteRC);
742 auto SetI = BuildMI(TestMBB, TestPos, TestLoc, TII->get(X86::SETCCr), Reg)
745 LLVM_DEBUG(dbgs() << " save cond: "; SetI->dump());
768 BuildMI(MBB, Pos, Loc, TII->get(X86::TEST8rr)).addReg(Reg).addReg(Reg);
770 LLVM_DEBUG(dbgs() << " test cond: "; TestI->dump());
791 "Cannot have a non-register defined operand to SETcc!");
795 MRI->clearKillFlags(OldReg);
796 MRI->replaceRegWith(OldReg, CondReg);
803 TII->get(X86::MOV8mr));
804 // Copy the address operands.
831 Register TmpReg = MRI->createVirtualRegister(PromoteRC);
834 TII->get(Subtarget->hasNDD() ? X86::ADD8ri_ND : X86::ADD8ri))
839 LLVM_DEBUG(dbgs() << " add cond: "; AddI->dump());
841 MI.findRegisterUseOperand(X86::EFLAGS, /*TRI=*/nullptr)->setIsKill(true);
912 MI.setDesc(TII->get(getOpcodeWithCC(MI.getOpcode(), NewCC)));
914 MI.getOperand(MI.getDesc().getNumOperands() - 1).setImm(NewCC);
916 MI.findRegisterUseOperand(X86::EFLAGS, /*TRI=*/nullptr)->setIsKill(true);