1 //===---------- PPCTLSDynamicCall.cpp - TLS Dynamic Call Fixup ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass expands ADDItls{ld,gd}LADDR[32] machine instructions into 10 // separate ADDItls[gd]L[32] and GETtlsADDR[32] instructions, both of 11 // which define GPR3. A copy is added from GPR3 to the target virtual 12 // register of the original instruction. The GETtlsADDR[32] is really 13 // a call instruction, so its target register is constrained to be GPR3. 14 // This is not true of ADDItls[gd]L[32], but there is a legacy linker 15 // optimization bug that requires the target register of the addi of 16 // a local- or general-dynamic TLS access sequence to be GPR3. 17 // 18 // This is done in a late pass so that TLS variable accesses can be 19 // fully commoned by MachineCSE. 20 // 21 //===----------------------------------------------------------------------===// 22 23 #include "PPC.h" 24 #include "PPCInstrBuilder.h" 25 #include "PPCInstrInfo.h" 26 #include "PPCTargetMachine.h" 27 #include "llvm/CodeGen/LiveIntervals.h" 28 #include "llvm/CodeGen/MachineFunctionPass.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/InitializePasses.h" 31 #include "llvm/Support/Debug.h" 32 #include "llvm/Support/raw_ostream.h" 33 34 using namespace llvm; 35 36 #define DEBUG_TYPE "ppc-tls-dynamic-call" 37 38 namespace { 39 struct PPCTLSDynamicCall : public MachineFunctionPass { 40 static char ID; 41 PPCTLSDynamicCall() : MachineFunctionPass(ID) { 42 initializePPCTLSDynamicCallPass(*PassRegistry::getPassRegistry()); 43 } 44 45 const PPCInstrInfo *TII; 46 47 protected: 48 bool processBlock(MachineBasicBlock &MBB) { 49 bool Changed = false; 50 bool NeedFence = true; 51 const PPCSubtarget &Subtarget = 52 MBB.getParent()->getSubtarget<PPCSubtarget>(); 53 bool Is64Bit = Subtarget.isPPC64(); 54 bool IsAIX = Subtarget.isAIXABI(); 55 bool IsLargeModel = 56 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Large; 57 bool IsPCREL = false; 58 MachineFunction *MF = MBB.getParent(); 59 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 60 61 for (MachineBasicBlock::iterator I = MBB.begin(), IE = MBB.end(); 62 I != IE;) { 63 MachineInstr &MI = *I; 64 IsPCREL = isPCREL(MI); 65 // There are a number of slight differences in code generation 66 // when we call .__get_tpointer (32-bit AIX TLS). 67 bool IsTLSTPRelMI = MI.getOpcode() == PPC::GETtlsTpointer32AIX; 68 bool IsTLSLDAIXMI = (MI.getOpcode() == PPC::TLSLDAIX8 || 69 MI.getOpcode() == PPC::TLSLDAIX); 70 71 if (MI.getOpcode() != PPC::ADDItlsgdLADDR && 72 MI.getOpcode() != PPC::ADDItlsldLADDR && 73 MI.getOpcode() != PPC::ADDItlsgdLADDR32 && 74 MI.getOpcode() != PPC::ADDItlsldLADDR32 && 75 MI.getOpcode() != PPC::TLSGDAIX && 76 MI.getOpcode() != PPC::TLSGDAIX8 && !IsTLSTPRelMI && !IsPCREL && 77 !IsTLSLDAIXMI) { 78 // Although we create ADJCALLSTACKDOWN and ADJCALLSTACKUP 79 // as scheduling fences, we skip creating fences if we already 80 // have existing ADJCALLSTACKDOWN/UP to avoid nesting, 81 // which causes verification error with -verify-machineinstrs. 82 if (MI.getOpcode() == PPC::ADJCALLSTACKDOWN) 83 NeedFence = false; 84 else if (MI.getOpcode() == PPC::ADJCALLSTACKUP) 85 NeedFence = true; 86 87 ++I; 88 continue; 89 } 90 91 LLVM_DEBUG(dbgs() << "TLS Dynamic Call Fixup:\n " << MI); 92 93 Register OutReg = MI.getOperand(0).getReg(); 94 Register InReg = PPC::NoRegister; 95 Register GPR3 = Is64Bit ? PPC::X3 : PPC::R3; 96 Register GPR4 = Is64Bit ? PPC::X4 : PPC::R4; 97 if (!IsPCREL && !IsTLSTPRelMI) 98 InReg = MI.getOperand(1).getReg(); 99 DebugLoc DL = MI.getDebugLoc(); 100 101 unsigned Opc1, Opc2; 102 switch (MI.getOpcode()) { 103 default: 104 llvm_unreachable("Opcode inconsistency error"); 105 case PPC::ADDItlsgdLADDR: 106 Opc1 = PPC::ADDItlsgdL; 107 Opc2 = PPC::GETtlsADDR; 108 break; 109 case PPC::ADDItlsldLADDR: 110 Opc1 = PPC::ADDItlsldL; 111 Opc2 = PPC::GETtlsldADDR; 112 break; 113 case PPC::ADDItlsgdLADDR32: 114 Opc1 = PPC::ADDItlsgdL32; 115 Opc2 = PPC::GETtlsADDR32; 116 break; 117 case PPC::ADDItlsldLADDR32: 118 Opc1 = PPC::ADDItlsldL32; 119 Opc2 = PPC::GETtlsldADDR32; 120 break; 121 case PPC::TLSLDAIX: 122 // TLSLDAIX is expanded to one copy and GET_TLS_MOD, so we only set 123 // Opc2 here. 124 Opc2 = PPC::GETtlsMOD32AIX; 125 break; 126 case PPC::TLSLDAIX8: 127 // TLSLDAIX8 is expanded to one copy and GET_TLS_MOD, so we only set 128 // Opc2 here. 129 Opc2 = PPC::GETtlsMOD64AIX; 130 break; 131 case PPC::TLSGDAIX8: 132 // TLSGDAIX8 is expanded to two copies and GET_TLS_ADDR, so we only 133 // set Opc2 here. 134 Opc2 = PPC::GETtlsADDR64AIX; 135 break; 136 case PPC::TLSGDAIX: 137 // TLSGDAIX is expanded to two copies and GET_TLS_ADDR, so we only 138 // set Opc2 here. 139 Opc2 = PPC::GETtlsADDR32AIX; 140 break; 141 case PPC::GETtlsTpointer32AIX: 142 // GETtlsTpointer32AIX is expanded to a call to GET_TPOINTER on AIX 143 // 32-bit mode within PPCAsmPrinter. This instruction does not need 144 // to change, so Opc2 is set to the same instruction opcode. 145 Opc2 = PPC::GETtlsTpointer32AIX; 146 break; 147 case PPC::PADDI8pc: 148 assert(IsPCREL && "Expecting General/Local Dynamic PCRel"); 149 Opc1 = PPC::PADDI8pc; 150 Opc2 = MI.getOperand(2).getTargetFlags() == 151 PPCII::MO_GOT_TLSGD_PCREL_FLAG 152 ? PPC::GETtlsADDRPCREL 153 : PPC::GETtlsldADDRPCREL; 154 } 155 156 // We create ADJCALLSTACKUP and ADJCALLSTACKDOWN around _tls_get_addr 157 // as scheduling fence to avoid it is scheduled before 158 // mflr in the prologue and the address in LR is clobbered (PR25839). 159 // We don't really need to save data to the stack - the clobbered 160 // registers are already saved when the SDNode (e.g. PPCaddiTlsgdLAddr) 161 // gets translated to the pseudo instruction (e.g. ADDItlsgdLADDR). 162 if (NeedFence) 163 BuildMI(MBB, I, DL, TII->get(PPC::ADJCALLSTACKDOWN)).addImm(0) 164 .addImm(0); 165 166 if (IsAIX) { 167 if (IsTLSLDAIXMI) { 168 // The relative order between the node that loads the variable 169 // offset from the TOC, and the .__tls_get_mod node is being tuned 170 // here. It is better to put the variable offset TOC load after the 171 // call, since this node can use clobbers r4/r5. 172 // Search for the pattern of the two nodes that load from the TOC 173 // (either for the variable offset or for the module handle), and 174 // then move the variable offset TOC load right before the node that 175 // uses the OutReg of the .__tls_get_mod node. 176 unsigned LDTocOp = 177 Is64Bit ? (IsLargeModel ? PPC::LDtocL : PPC::LDtoc) 178 : (IsLargeModel ? PPC::LWZtocL : PPC::LWZtoc); 179 if (!RegInfo.use_empty(OutReg)) { 180 std::set<MachineInstr *> Uses; 181 // Collect all instructions that use the OutReg. 182 for (MachineOperand &MO : RegInfo.use_operands(OutReg)) 183 Uses.insert(MO.getParent()); 184 // Find the first user (e.g.: lwax/stfdx) of the OutReg within the 185 // current BB. 186 MachineBasicBlock::iterator UseIter = MBB.begin(); 187 for (MachineBasicBlock::iterator IE = MBB.end(); UseIter != IE; 188 ++UseIter) 189 if (Uses.count(&*UseIter)) 190 break; 191 192 // Additional handling is required when UserIter (the first user 193 // of OutReg) is pointing to a valid node that loads from the TOC. 194 // Check the pattern and do the movement if the pattern matches. 195 if (UseIter != MBB.end()) { 196 // Collect all associated nodes that load from the TOC. Use 197 // hasOneDef() to guard against unexpected scenarios. 198 std::set<MachineInstr *> LoadFromTocs; 199 for (MachineOperand &MO : UseIter->operands()) 200 if (MO.isReg() && MO.isUse()) { 201 Register MOReg = MO.getReg(); 202 if (RegInfo.hasOneDef(MOReg)) { 203 MachineInstr *Temp = 204 RegInfo.getOneDef(MOReg)->getParent(); 205 // For the current TLSLDAIX node, get the corresponding 206 // node that loads from the TOC for the InReg. Otherwise, 207 // Temp probably pointed to the variable offset TOC load 208 // we would like to move. 209 if (Temp == &MI && RegInfo.hasOneDef(InReg)) 210 Temp = RegInfo.getOneDef(InReg)->getParent(); 211 if (Temp->getOpcode() == LDTocOp) 212 LoadFromTocs.insert(Temp); 213 } else { 214 // FIXME: analyze this scenario if there is one. 215 LoadFromTocs.clear(); 216 break; 217 } 218 } 219 220 // Check the two nodes that loaded from the TOC: one should be 221 // "_$TLSML", and the other will be moved before the node that 222 // uses the OutReg of the .__tls_get_mod node. 223 if (LoadFromTocs.size() == 2) { 224 MachineBasicBlock::iterator TLSMLIter = MBB.end(); 225 MachineBasicBlock::iterator OffsetIter = MBB.end(); 226 // Make sure the two nodes that loaded from the TOC are within 227 // the current BB, and that one of them is from the "_$TLSML" 228 // pseudo symbol, while the other is from the variable. 229 for (MachineBasicBlock::iterator I = MBB.begin(), 230 IE = MBB.end(); 231 I != IE; ++I) 232 if (LoadFromTocs.count(&*I)) { 233 MachineOperand MO = I->getOperand(1); 234 if (MO.isGlobal() && MO.getGlobal()->hasName() && 235 MO.getGlobal()->getName() == "_$TLSML") 236 TLSMLIter = I; 237 else 238 OffsetIter = I; 239 } 240 // Perform the movement when the desired scenario has been 241 // identified, which should be when both of the iterators are 242 // valid. 243 if (TLSMLIter != MBB.end() && OffsetIter != MBB.end()) 244 OffsetIter->moveBefore(&*UseIter); 245 } 246 } 247 } 248 // The module-handle is copied into r3. The copy is followed by 249 // GETtlsMOD32AIX/GETtlsMOD64AIX. 250 BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), GPR3) 251 .addReg(InReg); 252 // The call to .__tls_get_mod. 253 BuildMI(MBB, I, DL, TII->get(Opc2), GPR3).addReg(GPR3); 254 } else if (!IsTLSTPRelMI) { 255 // The variable offset and region handle (for TLSGD) are copied in 256 // r4 and r3. The copies are followed by 257 // GETtlsADDR32AIX/GETtlsADDR64AIX. 258 BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), GPR4) 259 .addReg(MI.getOperand(1).getReg()); 260 BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), GPR3) 261 .addReg(MI.getOperand(2).getReg()); 262 BuildMI(MBB, I, DL, TII->get(Opc2), GPR3).addReg(GPR3).addReg(GPR4); 263 } else 264 // The opcode of GETtlsTpointer32AIX does not change, because later 265 // this instruction will be expanded into a call to .__get_tpointer, 266 // which will return the thread pointer into r3. 267 BuildMI(MBB, I, DL, TII->get(Opc2), GPR3); 268 } else { 269 MachineInstr *Addi; 270 if (IsPCREL) { 271 Addi = BuildMI(MBB, I, DL, TII->get(Opc1), GPR3).addImm(0); 272 } else { 273 // Expand into two ops built prior to the existing instruction. 274 assert(InReg != PPC::NoRegister && "Operand must be a register"); 275 Addi = BuildMI(MBB, I, DL, TII->get(Opc1), GPR3).addReg(InReg); 276 } 277 278 Addi->addOperand(MI.getOperand(2)); 279 280 MachineInstr *Call = 281 (BuildMI(MBB, I, DL, TII->get(Opc2), GPR3).addReg(GPR3)); 282 if (IsPCREL) 283 Call->addOperand(MI.getOperand(2)); 284 else 285 Call->addOperand(MI.getOperand(3)); 286 } 287 if (NeedFence) 288 BuildMI(MBB, I, DL, TII->get(PPC::ADJCALLSTACKUP)).addImm(0).addImm(0); 289 290 BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), OutReg) 291 .addReg(GPR3); 292 293 // Move past the original instruction and remove it. 294 ++I; 295 MI.removeFromParent(); 296 297 Changed = true; 298 } 299 300 return Changed; 301 } 302 303 public: 304 bool isPCREL(const MachineInstr &MI) { 305 return (MI.getOpcode() == PPC::PADDI8pc) && 306 (MI.getOperand(2).getTargetFlags() == 307 PPCII::MO_GOT_TLSGD_PCREL_FLAG || 308 MI.getOperand(2).getTargetFlags() == 309 PPCII::MO_GOT_TLSLD_PCREL_FLAG); 310 } 311 312 bool runOnMachineFunction(MachineFunction &MF) override { 313 TII = MF.getSubtarget<PPCSubtarget>().getInstrInfo(); 314 315 bool Changed = false; 316 317 for (MachineBasicBlock &B : llvm::make_early_inc_range(MF)) 318 if (processBlock(B)) 319 Changed = true; 320 321 return Changed; 322 } 323 324 void getAnalysisUsage(AnalysisUsage &AU) const override { 325 AU.addRequired<LiveIntervals>(); 326 AU.addRequired<SlotIndexes>(); 327 MachineFunctionPass::getAnalysisUsage(AU); 328 } 329 }; 330 } 331 332 INITIALIZE_PASS_BEGIN(PPCTLSDynamicCall, DEBUG_TYPE, 333 "PowerPC TLS Dynamic Call Fixup", false, false) 334 INITIALIZE_PASS_DEPENDENCY(LiveIntervals) 335 INITIALIZE_PASS_DEPENDENCY(SlotIndexes) 336 INITIALIZE_PASS_END(PPCTLSDynamicCall, DEBUG_TYPE, 337 "PowerPC TLS Dynamic Call Fixup", false, false) 338 339 char PPCTLSDynamicCall::ID = 0; 340 FunctionPass* 341 llvm::createPPCTLSDynamicCallPass() { return new PPCTLSDynamicCall(); } 342