1 //===-- PPCExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains a pass that expands atomic pseudo instructions into 10 // target instructions post RA. With such method, LL/SC loop is considered as 11 // a whole blob and make spilling unlikely happens in the LL/SC loop. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/PPCPredicates.h" 16 #include "PPC.h" 17 #include "PPCInstrInfo.h" 18 #include "PPCTargetMachine.h" 19 20 #include "llvm/CodeGen/LivePhysRegs.h" 21 #include "llvm/CodeGen/MachineFunctionPass.h" 22 #include "llvm/CodeGen/MachineInstrBuilder.h" 23 24 using namespace llvm; 25 26 #define DEBUG_TYPE "ppc-atomic-expand" 27 28 namespace { 29 30 class PPCExpandAtomicPseudo : public MachineFunctionPass { 31 public: 32 const PPCInstrInfo *TII; 33 const PPCRegisterInfo *TRI; 34 static char ID; 35 36 PPCExpandAtomicPseudo() : MachineFunctionPass(ID) { 37 initializePPCExpandAtomicPseudoPass(*PassRegistry::getPassRegistry()); 38 } 39 40 bool runOnMachineFunction(MachineFunction &MF) override; 41 42 private: 43 bool expandMI(MachineBasicBlock &MBB, MachineInstr &MI, 44 MachineBasicBlock::iterator &NMBBI); 45 bool expandAtomicRMW128(MachineBasicBlock &MBB, MachineInstr &MI, 46 MachineBasicBlock::iterator &NMBBI); 47 bool expandAtomicCmpSwap128(MachineBasicBlock &MBB, MachineInstr &MI, 48 MachineBasicBlock::iterator &NMBBI); 49 }; 50 51 static void PairedCopy(const PPCInstrInfo *TII, MachineBasicBlock &MBB, 52 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, 53 Register Dest0, Register Dest1, Register Src0, 54 Register Src1) { 55 const MCInstrDesc &OR = TII->get(PPC::OR8); 56 const MCInstrDesc &XOR = TII->get(PPC::XOR8); 57 if (Dest0 == Src1 && Dest1 == Src0) { 58 // The most tricky case, swapping values. 59 BuildMI(MBB, MBBI, DL, XOR, Dest0).addReg(Dest0).addReg(Dest1); 60 BuildMI(MBB, MBBI, DL, XOR, Dest1).addReg(Dest0).addReg(Dest1); 61 BuildMI(MBB, MBBI, DL, XOR, Dest0).addReg(Dest0).addReg(Dest1); 62 } else if (Dest0 != Src0 || Dest1 != Src1) { 63 if (Dest0 == Src1 || Dest1 != Src0) { 64 BuildMI(MBB, MBBI, DL, OR, Dest1).addReg(Src1).addReg(Src1); 65 BuildMI(MBB, MBBI, DL, OR, Dest0).addReg(Src0).addReg(Src0); 66 } else { 67 BuildMI(MBB, MBBI, DL, OR, Dest0).addReg(Src0).addReg(Src0); 68 BuildMI(MBB, MBBI, DL, OR, Dest1).addReg(Src1).addReg(Src1); 69 } 70 } 71 } 72 73 bool PPCExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) { 74 bool Changed = false; 75 TII = static_cast<const PPCInstrInfo *>(MF.getSubtarget().getInstrInfo()); 76 TRI = &TII->getRegisterInfo(); 77 for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { 78 MachineBasicBlock &MBB = *I; 79 for (MachineBasicBlock::iterator MBBI = MBB.begin(), MBBE = MBB.end(); 80 MBBI != MBBE;) { 81 MachineInstr &MI = *MBBI; 82 MachineBasicBlock::iterator NMBBI = std::next(MBBI); 83 Changed |= expandMI(MBB, MI, NMBBI); 84 MBBI = NMBBI; 85 } 86 } 87 if (Changed) 88 MF.RenumberBlocks(); 89 return Changed; 90 } 91 92 bool PPCExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB, MachineInstr &MI, 93 MachineBasicBlock::iterator &NMBBI) { 94 switch (MI.getOpcode()) { 95 case PPC::ATOMIC_SWAP_I128: 96 case PPC::ATOMIC_LOAD_ADD_I128: 97 case PPC::ATOMIC_LOAD_SUB_I128: 98 case PPC::ATOMIC_LOAD_XOR_I128: 99 case PPC::ATOMIC_LOAD_NAND_I128: 100 case PPC::ATOMIC_LOAD_AND_I128: 101 case PPC::ATOMIC_LOAD_OR_I128: 102 return expandAtomicRMW128(MBB, MI, NMBBI); 103 case PPC::ATOMIC_CMP_SWAP_I128: 104 return expandAtomicCmpSwap128(MBB, MI, NMBBI); 105 case PPC::BUILD_QUADWORD: { 106 Register Dst = MI.getOperand(0).getReg(); 107 Register DstHi = TRI->getSubReg(Dst, PPC::sub_gp8_x0); 108 Register DstLo = TRI->getSubReg(Dst, PPC::sub_gp8_x1); 109 Register Lo = MI.getOperand(1).getReg(); 110 Register Hi = MI.getOperand(2).getReg(); 111 PairedCopy(TII, MBB, MI, MI.getDebugLoc(), DstHi, DstLo, Hi, Lo); 112 MI.eraseFromParent(); 113 return true; 114 } 115 default: 116 return false; 117 } 118 } 119 120 bool PPCExpandAtomicPseudo::expandAtomicRMW128( 121 MachineBasicBlock &MBB, MachineInstr &MI, 122 MachineBasicBlock::iterator &NMBBI) { 123 const MCInstrDesc &LL = TII->get(PPC::LQARX); 124 const MCInstrDesc &SC = TII->get(PPC::STQCX); 125 DebugLoc DL = MI.getDebugLoc(); 126 MachineFunction *MF = MBB.getParent(); 127 const BasicBlock *BB = MBB.getBasicBlock(); 128 // Create layout of control flow. 129 MachineFunction::iterator MFI = ++MBB.getIterator(); 130 MachineBasicBlock *LoopMBB = MF->CreateMachineBasicBlock(BB); 131 MachineBasicBlock *ExitMBB = MF->CreateMachineBasicBlock(BB); 132 MF->insert(MFI, LoopMBB); 133 MF->insert(MFI, ExitMBB); 134 ExitMBB->splice(ExitMBB->begin(), &MBB, std::next(MI.getIterator()), 135 MBB.end()); 136 ExitMBB->transferSuccessorsAndUpdatePHIs(&MBB); 137 MBB.addSuccessor(LoopMBB); 138 139 // For non-min/max operations, control flow is kinda like: 140 // MBB: 141 // ... 142 // LoopMBB: 143 // lqarx in, ptr 144 // addc out.sub_x1, in.sub_x1, op.sub_x1 145 // adde out.sub_x0, in.sub_x0, op.sub_x0 146 // stqcx out, ptr 147 // bne- LoopMBB 148 // ExitMBB: 149 // ... 150 Register Old = MI.getOperand(0).getReg(); 151 Register OldHi = TRI->getSubReg(Old, PPC::sub_gp8_x0); 152 Register OldLo = TRI->getSubReg(Old, PPC::sub_gp8_x1); 153 Register Scratch = MI.getOperand(1).getReg(); 154 Register ScratchHi = TRI->getSubReg(Scratch, PPC::sub_gp8_x0); 155 Register ScratchLo = TRI->getSubReg(Scratch, PPC::sub_gp8_x1); 156 Register RA = MI.getOperand(2).getReg(); 157 Register RB = MI.getOperand(3).getReg(); 158 Register IncrLo = MI.getOperand(4).getReg(); 159 Register IncrHi = MI.getOperand(5).getReg(); 160 unsigned RMWOpcode = MI.getOpcode(); 161 162 MachineBasicBlock *CurrentMBB = LoopMBB; 163 BuildMI(CurrentMBB, DL, LL, Old).addReg(RA).addReg(RB); 164 165 switch (RMWOpcode) { 166 case PPC::ATOMIC_SWAP_I128: 167 PairedCopy(TII, *CurrentMBB, CurrentMBB->end(), DL, ScratchHi, ScratchLo, 168 IncrHi, IncrLo); 169 break; 170 case PPC::ATOMIC_LOAD_ADD_I128: 171 BuildMI(CurrentMBB, DL, TII->get(PPC::ADDC8), ScratchLo) 172 .addReg(IncrLo) 173 .addReg(OldLo); 174 BuildMI(CurrentMBB, DL, TII->get(PPC::ADDE8), ScratchHi) 175 .addReg(IncrHi) 176 .addReg(OldHi); 177 break; 178 case PPC::ATOMIC_LOAD_SUB_I128: 179 BuildMI(CurrentMBB, DL, TII->get(PPC::SUBFC8), ScratchLo) 180 .addReg(IncrLo) 181 .addReg(OldLo); 182 BuildMI(CurrentMBB, DL, TII->get(PPC::SUBFE8), ScratchHi) 183 .addReg(IncrHi) 184 .addReg(OldHi); 185 break; 186 187 #define TRIVIAL_ATOMICRMW(Opcode, Instr) \ 188 case Opcode: \ 189 BuildMI(CurrentMBB, DL, TII->get((Instr)), ScratchLo) \ 190 .addReg(IncrLo) \ 191 .addReg(OldLo); \ 192 BuildMI(CurrentMBB, DL, TII->get((Instr)), ScratchHi) \ 193 .addReg(IncrHi) \ 194 .addReg(OldHi); \ 195 break 196 197 TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_OR_I128, PPC::OR8); 198 TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_XOR_I128, PPC::XOR8); 199 TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_AND_I128, PPC::AND8); 200 TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_NAND_I128, PPC::NAND8); 201 #undef TRIVIAL_ATOMICRMW 202 default: 203 llvm_unreachable("Unhandled atomic RMW operation"); 204 } 205 BuildMI(CurrentMBB, DL, SC).addReg(Scratch).addReg(RA).addReg(RB); 206 BuildMI(CurrentMBB, DL, TII->get(PPC::BCC)) 207 .addImm(PPC::PRED_NE) 208 .addReg(PPC::CR0) 209 .addMBB(LoopMBB); 210 CurrentMBB->addSuccessor(LoopMBB); 211 CurrentMBB->addSuccessor(ExitMBB); 212 recomputeLiveIns(*LoopMBB); 213 recomputeLiveIns(*ExitMBB); 214 NMBBI = MBB.end(); 215 MI.eraseFromParent(); 216 return true; 217 } 218 219 bool PPCExpandAtomicPseudo::expandAtomicCmpSwap128( 220 MachineBasicBlock &MBB, MachineInstr &MI, 221 MachineBasicBlock::iterator &NMBBI) { 222 const MCInstrDesc &LL = TII->get(PPC::LQARX); 223 const MCInstrDesc &SC = TII->get(PPC::STQCX); 224 DebugLoc DL = MI.getDebugLoc(); 225 MachineFunction *MF = MBB.getParent(); 226 const BasicBlock *BB = MBB.getBasicBlock(); 227 Register Old = MI.getOperand(0).getReg(); 228 Register OldHi = TRI->getSubReg(Old, PPC::sub_gp8_x0); 229 Register OldLo = TRI->getSubReg(Old, PPC::sub_gp8_x1); 230 Register Scratch = MI.getOperand(1).getReg(); 231 Register ScratchHi = TRI->getSubReg(Scratch, PPC::sub_gp8_x0); 232 Register ScratchLo = TRI->getSubReg(Scratch, PPC::sub_gp8_x1); 233 Register RA = MI.getOperand(2).getReg(); 234 Register RB = MI.getOperand(3).getReg(); 235 Register CmpLo = MI.getOperand(4).getReg(); 236 Register CmpHi = MI.getOperand(5).getReg(); 237 Register NewLo = MI.getOperand(6).getReg(); 238 Register NewHi = MI.getOperand(7).getReg(); 239 // Create layout of control flow. 240 // loop: 241 // old = lqarx ptr 242 // <compare old, cmp> 243 // bne 0, fail 244 // succ: 245 // stqcx new ptr 246 // bne 0, loop 247 // b exit 248 // fail: 249 // stqcx old ptr 250 // exit: 251 // .... 252 MachineFunction::iterator MFI = ++MBB.getIterator(); 253 MachineBasicBlock *LoopCmpMBB = MF->CreateMachineBasicBlock(BB); 254 MachineBasicBlock *CmpSuccMBB = MF->CreateMachineBasicBlock(BB); 255 MachineBasicBlock *CmpFailMBB = MF->CreateMachineBasicBlock(BB); 256 MachineBasicBlock *ExitMBB = MF->CreateMachineBasicBlock(BB); 257 MF->insert(MFI, LoopCmpMBB); 258 MF->insert(MFI, CmpSuccMBB); 259 MF->insert(MFI, CmpFailMBB); 260 MF->insert(MFI, ExitMBB); 261 ExitMBB->splice(ExitMBB->begin(), &MBB, std::next(MI.getIterator()), 262 MBB.end()); 263 ExitMBB->transferSuccessorsAndUpdatePHIs(&MBB); 264 MBB.addSuccessor(LoopCmpMBB); 265 // Build loop. 266 MachineBasicBlock *CurrentMBB = LoopCmpMBB; 267 BuildMI(CurrentMBB, DL, LL, Old).addReg(RA).addReg(RB); 268 BuildMI(CurrentMBB, DL, TII->get(PPC::XOR8), ScratchLo) 269 .addReg(OldLo) 270 .addReg(CmpLo); 271 BuildMI(CurrentMBB, DL, TII->get(PPC::XOR8), ScratchHi) 272 .addReg(OldHi) 273 .addReg(CmpHi); 274 BuildMI(CurrentMBB, DL, TII->get(PPC::OR8_rec), ScratchLo) 275 .addReg(ScratchLo) 276 .addReg(ScratchHi); 277 BuildMI(CurrentMBB, DL, TII->get(PPC::BCC)) 278 .addImm(PPC::PRED_NE) 279 .addReg(PPC::CR0) 280 .addMBB(CmpFailMBB); 281 CurrentMBB->addSuccessor(CmpSuccMBB); 282 CurrentMBB->addSuccessor(CmpFailMBB); 283 // Build succ. 284 CurrentMBB = CmpSuccMBB; 285 PairedCopy(TII, *CurrentMBB, CurrentMBB->end(), DL, ScratchHi, ScratchLo, 286 NewHi, NewLo); 287 BuildMI(CurrentMBB, DL, SC).addReg(Scratch).addReg(RA).addReg(RB); 288 BuildMI(CurrentMBB, DL, TII->get(PPC::BCC)) 289 .addImm(PPC::PRED_NE) 290 .addReg(PPC::CR0) 291 .addMBB(LoopCmpMBB); 292 BuildMI(CurrentMBB, DL, TII->get(PPC::B)).addMBB(ExitMBB); 293 CurrentMBB->addSuccessor(LoopCmpMBB); 294 CurrentMBB->addSuccessor(ExitMBB); 295 CurrentMBB = CmpFailMBB; 296 BuildMI(CurrentMBB, DL, SC).addReg(Old).addReg(RA).addReg(RB); 297 CurrentMBB->addSuccessor(ExitMBB); 298 299 recomputeLiveIns(*LoopCmpMBB); 300 recomputeLiveIns(*CmpSuccMBB); 301 recomputeLiveIns(*CmpFailMBB); 302 recomputeLiveIns(*ExitMBB); 303 NMBBI = MBB.end(); 304 MI.eraseFromParent(); 305 return true; 306 } 307 308 } // namespace 309 310 INITIALIZE_PASS(PPCExpandAtomicPseudo, DEBUG_TYPE, "PowerPC Expand Atomic", 311 false, false) 312 313 char PPCExpandAtomicPseudo::ID = 0; 314 FunctionPass *llvm::createPPCExpandAtomicPseudoPass() { 315 return new PPCExpandAtomicPseudo(); 316 } 317