1 //===---------- PPCTLSDynamicCall.cpp - TLS Dynamic Call Fixup ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass expands ADDItls{ld,gd}LADDR[32] machine instructions into 10 // separate ADDItls[gd]L[32] and GETtlsADDR[32] instructions, both of 11 // which define GPR3. A copy is added from GPR3 to the target virtual 12 // register of the original instruction. The GETtlsADDR[32] is really 13 // a call instruction, so its target register is constrained to be GPR3. 14 // This is not true of ADDItls[gd]L[32], but there is a legacy linker 15 // optimization bug that requires the target register of the addi of 16 // a local- or general-dynamic TLS access sequence to be GPR3. 17 // 18 // This is done in a late pass so that TLS variable accesses can be 19 // fully commoned by MachineCSE. 20 // 21 //===----------------------------------------------------------------------===// 22 23 #include "PPC.h" 24 #include "PPCInstrInfo.h" 25 #include "PPCTargetMachine.h" 26 #include "llvm/CodeGen/LiveIntervals.h" 27 #include "llvm/CodeGen/MachineFrameInfo.h" 28 #include "llvm/CodeGen/MachineFunctionPass.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/InitializePasses.h" 31 #include "llvm/Support/Debug.h" 32 #include "llvm/Support/raw_ostream.h" 33 34 using namespace llvm; 35 36 #define DEBUG_TYPE "ppc-tls-dynamic-call" 37 38 namespace { 39 struct PPCTLSDynamicCall : public MachineFunctionPass { 40 static char ID; 41 PPCTLSDynamicCall() : MachineFunctionPass(ID) { 42 initializePPCTLSDynamicCallPass(*PassRegistry::getPassRegistry()); 43 } 44 45 const PPCInstrInfo *TII; 46 47 protected: 48 bool processBlock(MachineBasicBlock &MBB) { 49 bool Changed = false; 50 bool NeedFence = true; 51 const PPCSubtarget &Subtarget = 52 MBB.getParent()->getSubtarget<PPCSubtarget>(); 53 bool Is64Bit = Subtarget.isPPC64(); 54 bool IsAIX = Subtarget.isAIXABI(); 55 bool IsLargeModel = 56 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Large; 57 bool IsPCREL = false; 58 MachineFunction *MF = MBB.getParent(); 59 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 60 61 for (MachineBasicBlock::iterator I = MBB.begin(), IE = MBB.end(); 62 I != IE;) { 63 MachineInstr &MI = *I; 64 IsPCREL = isPCREL(MI); 65 // There are a number of slight differences in code generation 66 // when we call .__get_tpointer (32-bit AIX TLS). 67 bool IsTLSTPRelMI = MI.getOpcode() == PPC::GETtlsTpointer32AIX; 68 bool IsTLSLDAIXMI = (MI.getOpcode() == PPC::TLSLDAIX8 || 69 MI.getOpcode() == PPC::TLSLDAIX); 70 71 if (MI.getOpcode() != PPC::ADDItlsgdLADDR && 72 MI.getOpcode() != PPC::ADDItlsldLADDR && 73 MI.getOpcode() != PPC::ADDItlsgdLADDR32 && 74 MI.getOpcode() != PPC::ADDItlsldLADDR32 && 75 MI.getOpcode() != PPC::TLSGDAIX && 76 MI.getOpcode() != PPC::TLSGDAIX8 && !IsTLSTPRelMI && !IsPCREL && 77 !IsTLSLDAIXMI) { 78 // Although we create ADJCALLSTACKDOWN and ADJCALLSTACKUP 79 // as scheduling fences, we skip creating fences if we already 80 // have existing ADJCALLSTACKDOWN/UP to avoid nesting, 81 // which causes verification error with -verify-machineinstrs. 82 if (MI.getOpcode() == PPC::ADJCALLSTACKDOWN) 83 NeedFence = false; 84 else if (MI.getOpcode() == PPC::ADJCALLSTACKUP) 85 NeedFence = true; 86 87 ++I; 88 continue; 89 } 90 91 LLVM_DEBUG(dbgs() << "TLS Dynamic Call Fixup:\n " << MI); 92 93 Register OutReg = MI.getOperand(0).getReg(); 94 Register InReg = PPC::NoRegister; 95 Register GPR3 = Is64Bit ? PPC::X3 : PPC::R3; 96 Register GPR4 = Is64Bit ? PPC::X4 : PPC::R4; 97 if (!IsPCREL && !IsTLSTPRelMI) 98 InReg = MI.getOperand(1).getReg(); 99 DebugLoc DL = MI.getDebugLoc(); 100 101 unsigned Opc1, Opc2; 102 switch (MI.getOpcode()) { 103 default: 104 llvm_unreachable("Opcode inconsistency error"); 105 case PPC::ADDItlsgdLADDR: 106 Opc1 = PPC::ADDItlsgdL; 107 Opc2 = PPC::GETtlsADDR; 108 break; 109 case PPC::ADDItlsldLADDR: 110 Opc1 = PPC::ADDItlsldL; 111 Opc2 = PPC::GETtlsldADDR; 112 break; 113 case PPC::ADDItlsgdLADDR32: 114 Opc1 = PPC::ADDItlsgdL32; 115 Opc2 = PPC::GETtlsADDR32; 116 break; 117 case PPC::ADDItlsldLADDR32: 118 Opc1 = PPC::ADDItlsldL32; 119 Opc2 = PPC::GETtlsldADDR32; 120 break; 121 case PPC::TLSLDAIX: 122 // TLSLDAIX is expanded to one copy and GET_TLS_MOD, so we only set 123 // Opc2 here. 124 Opc2 = PPC::GETtlsMOD32AIX; 125 break; 126 case PPC::TLSLDAIX8: 127 // TLSLDAIX8 is expanded to one copy and GET_TLS_MOD, so we only set 128 // Opc2 here. 129 Opc2 = PPC::GETtlsMOD64AIX; 130 break; 131 case PPC::TLSGDAIX8: 132 // TLSGDAIX8 is expanded to two copies and GET_TLS_ADDR, so we only 133 // set Opc2 here. 134 Opc2 = PPC::GETtlsADDR64AIX; 135 break; 136 case PPC::TLSGDAIX: 137 // TLSGDAIX is expanded to two copies and GET_TLS_ADDR, so we only 138 // set Opc2 here. 139 Opc2 = PPC::GETtlsADDR32AIX; 140 break; 141 case PPC::GETtlsTpointer32AIX: 142 // GETtlsTpointer32AIX is expanded to a call to GET_TPOINTER on AIX 143 // 32-bit mode within PPCAsmPrinter. This instruction does not need 144 // to change, so Opc2 is set to the same instruction opcode. 145 Opc2 = PPC::GETtlsTpointer32AIX; 146 break; 147 case PPC::PADDI8pc: 148 assert(IsPCREL && "Expecting General/Local Dynamic PCRel"); 149 Opc1 = PPC::PADDI8pc; 150 Opc2 = MI.getOperand(2).getTargetFlags() == 151 PPCII::MO_GOT_TLSGD_PCREL_FLAG 152 ? PPC::GETtlsADDRPCREL 153 : PPC::GETtlsldADDRPCREL; 154 } 155 156 // We create ADJCALLSTACKUP and ADJCALLSTACKDOWN around _tls_get_addr 157 // as scheduling fence to avoid it is scheduled before 158 // mflr in the prologue and the address in LR is clobbered (PR25839). 159 // We don't really need to save data to the stack - the clobbered 160 // registers are already saved when the SDNode (e.g. PPCaddiTlsgdLAddr) 161 // gets translated to the pseudo instruction (e.g. ADDItlsgdLADDR). 162 if (NeedFence) { 163 MBB.getParent()->getFrameInfo().setAdjustsStack(true); 164 BuildMI(MBB, I, DL, TII->get(PPC::ADJCALLSTACKDOWN)).addImm(0) 165 .addImm(0); 166 } 167 168 if (IsAIX) { 169 if (IsTLSLDAIXMI) { 170 // The relative order between the node that loads the variable 171 // offset from the TOC, and the .__tls_get_mod node is being tuned 172 // here. It is better to put the variable offset TOC load after the 173 // call, since this node can use clobbers r4/r5. 174 // Search for the pattern of the two nodes that load from the TOC 175 // (either for the variable offset or for the module handle), and 176 // then move the variable offset TOC load right before the node that 177 // uses the OutReg of the .__tls_get_mod node. 178 unsigned LDTocOp = 179 Is64Bit ? (IsLargeModel ? PPC::LDtocL : PPC::LDtoc) 180 : (IsLargeModel ? PPC::LWZtocL : PPC::LWZtoc); 181 if (!RegInfo.use_empty(OutReg)) { 182 std::set<MachineInstr *> Uses; 183 // Collect all instructions that use the OutReg. 184 for (MachineOperand &MO : RegInfo.use_operands(OutReg)) 185 Uses.insert(MO.getParent()); 186 // Find the first user (e.g.: lwax/stfdx) of the OutReg within the 187 // current BB. 188 MachineBasicBlock::iterator UseIter = MBB.begin(); 189 for (MachineBasicBlock::iterator IE = MBB.end(); UseIter != IE; 190 ++UseIter) 191 if (Uses.count(&*UseIter)) 192 break; 193 194 // Additional handling is required when UserIter (the first user 195 // of OutReg) is pointing to a valid node that loads from the TOC. 196 // Check the pattern and do the movement if the pattern matches. 197 if (UseIter != MBB.end()) { 198 // Collect all associated nodes that load from the TOC. Use 199 // hasOneDef() to guard against unexpected scenarios. 200 std::set<MachineInstr *> LoadFromTocs; 201 for (MachineOperand &MO : UseIter->operands()) 202 if (MO.isReg() && MO.isUse()) { 203 Register MOReg = MO.getReg(); 204 if (RegInfo.hasOneDef(MOReg)) { 205 MachineInstr *Temp = 206 RegInfo.getOneDef(MOReg)->getParent(); 207 // For the current TLSLDAIX node, get the corresponding 208 // node that loads from the TOC for the InReg. Otherwise, 209 // Temp probably pointed to the variable offset TOC load 210 // we would like to move. 211 if (Temp == &MI && RegInfo.hasOneDef(InReg)) 212 Temp = RegInfo.getOneDef(InReg)->getParent(); 213 if (Temp->getOpcode() == LDTocOp) 214 LoadFromTocs.insert(Temp); 215 } else { 216 // FIXME: analyze this scenario if there is one. 217 LoadFromTocs.clear(); 218 break; 219 } 220 } 221 222 // Check the two nodes that loaded from the TOC: one should be 223 // "_$TLSML", and the other will be moved before the node that 224 // uses the OutReg of the .__tls_get_mod node. 225 if (LoadFromTocs.size() == 2) { 226 MachineBasicBlock::iterator TLSMLIter = MBB.end(); 227 MachineBasicBlock::iterator OffsetIter = MBB.end(); 228 // Make sure the two nodes that loaded from the TOC are within 229 // the current BB, and that one of them is from the "_$TLSML" 230 // pseudo symbol, while the other is from the variable. 231 for (MachineBasicBlock::iterator I = MBB.begin(), 232 IE = MBB.end(); 233 I != IE; ++I) 234 if (LoadFromTocs.count(&*I)) { 235 MachineOperand MO = I->getOperand(1); 236 if (MO.isGlobal() && MO.getGlobal()->hasName() && 237 MO.getGlobal()->getName() == "_$TLSML") 238 TLSMLIter = I; 239 else 240 OffsetIter = I; 241 } 242 // Perform the movement when the desired scenario has been 243 // identified, which should be when both of the iterators are 244 // valid. 245 if (TLSMLIter != MBB.end() && OffsetIter != MBB.end()) 246 OffsetIter->moveBefore(&*UseIter); 247 } 248 } 249 } 250 // The module-handle is copied into r3. The copy is followed by 251 // GETtlsMOD32AIX/GETtlsMOD64AIX. 252 BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), GPR3) 253 .addReg(InReg); 254 // The call to .__tls_get_mod. 255 BuildMI(MBB, I, DL, TII->get(Opc2), GPR3).addReg(GPR3); 256 } else if (!IsTLSTPRelMI) { 257 // The variable offset and region handle (for TLSGD) are copied in 258 // r4 and r3. The copies are followed by 259 // GETtlsADDR32AIX/GETtlsADDR64AIX. 260 BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), GPR4) 261 .addReg(MI.getOperand(1).getReg()); 262 BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), GPR3) 263 .addReg(MI.getOperand(2).getReg()); 264 BuildMI(MBB, I, DL, TII->get(Opc2), GPR3).addReg(GPR3).addReg(GPR4); 265 } else 266 // The opcode of GETtlsTpointer32AIX does not change, because later 267 // this instruction will be expanded into a call to .__get_tpointer, 268 // which will return the thread pointer into r3. 269 BuildMI(MBB, I, DL, TII->get(Opc2), GPR3); 270 } else { 271 MachineInstr *Addi; 272 if (IsPCREL) { 273 Addi = BuildMI(MBB, I, DL, TII->get(Opc1), GPR3).addImm(0); 274 } else { 275 // Expand into two ops built prior to the existing instruction. 276 assert(InReg != PPC::NoRegister && "Operand must be a register"); 277 Addi = BuildMI(MBB, I, DL, TII->get(Opc1), GPR3).addReg(InReg); 278 } 279 280 Addi->addOperand(MI.getOperand(2)); 281 282 MachineInstr *Call = 283 (BuildMI(MBB, I, DL, TII->get(Opc2), GPR3).addReg(GPR3)); 284 if (IsPCREL) 285 Call->addOperand(MI.getOperand(2)); 286 else 287 Call->addOperand(MI.getOperand(3)); 288 } 289 if (NeedFence) 290 BuildMI(MBB, I, DL, TII->get(PPC::ADJCALLSTACKUP)).addImm(0).addImm(0); 291 292 BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), OutReg) 293 .addReg(GPR3); 294 295 // Move past the original instruction and remove it. 296 ++I; 297 MI.removeFromParent(); 298 299 Changed = true; 300 } 301 302 return Changed; 303 } 304 305 public: 306 bool isPCREL(const MachineInstr &MI) { 307 return (MI.getOpcode() == PPC::PADDI8pc) && 308 (MI.getOperand(2).getTargetFlags() == 309 PPCII::MO_GOT_TLSGD_PCREL_FLAG || 310 MI.getOperand(2).getTargetFlags() == 311 PPCII::MO_GOT_TLSLD_PCREL_FLAG); 312 } 313 314 bool runOnMachineFunction(MachineFunction &MF) override { 315 TII = MF.getSubtarget<PPCSubtarget>().getInstrInfo(); 316 317 bool Changed = false; 318 319 for (MachineBasicBlock &B : llvm::make_early_inc_range(MF)) 320 if (processBlock(B)) 321 Changed = true; 322 323 return Changed; 324 } 325 326 void getAnalysisUsage(AnalysisUsage &AU) const override { 327 AU.addRequired<LiveIntervalsWrapperPass>(); 328 AU.addRequired<SlotIndexesWrapperPass>(); 329 MachineFunctionPass::getAnalysisUsage(AU); 330 } 331 }; 332 } 333 334 INITIALIZE_PASS_BEGIN(PPCTLSDynamicCall, DEBUG_TYPE, 335 "PowerPC TLS Dynamic Call Fixup", false, false) 336 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass) 337 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass) 338 INITIALIZE_PASS_END(PPCTLSDynamicCall, DEBUG_TYPE, 339 "PowerPC TLS Dynamic Call Fixup", false, false) 340 341 char PPCTLSDynamicCall::ID = 0; 342 FunctionPass* 343 llvm::createPPCTLSDynamicCallPass() { return new PPCTLSDynamicCall(); } 344