1 //===----- BPFMISimplifyPatchable.cpp - MI Simplify Patchable Insts -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass targets a subset of instructions like below 10 // ld_imm64 r1, @global 11 // ldd r2, r1, 0 12 // add r3, struct_base_reg, r2 13 // 14 // Here @global should represent an AMA (abstruct member access). 15 // Such an access is subject to bpf load time patching. After this pass, the 16 // code becomes 17 // ld_imm64 r1, @global 18 // add r3, struct_base_reg, r1 19 // 20 // Eventually, at BTF output stage, a relocation record will be generated 21 // for ld_imm64 which should be replaced later by bpf loader: 22 // r1 = <calculated field_info> 23 // add r3, struct_base_reg, r1 24 // 25 //===----------------------------------------------------------------------===// 26 27 #include "BPF.h" 28 #include "BPFCORE.h" 29 #include "BPFInstrInfo.h" 30 #include "BPFTargetMachine.h" 31 #include "llvm/CodeGen/MachineInstrBuilder.h" 32 #include "llvm/CodeGen/MachineRegisterInfo.h" 33 #include "llvm/Support/Debug.h" 34 35 using namespace llvm; 36 37 #define DEBUG_TYPE "bpf-mi-simplify-patchable" 38 39 namespace { 40 41 struct BPFMISimplifyPatchable : public MachineFunctionPass { 42 43 static char ID; 44 const BPFInstrInfo *TII; 45 MachineFunction *MF; 46 47 BPFMISimplifyPatchable() : MachineFunctionPass(ID) { 48 initializeBPFMISimplifyPatchablePass(*PassRegistry::getPassRegistry()); 49 } 50 51 private: 52 // Initialize class variables. 53 void initialize(MachineFunction &MFParm); 54 55 bool removeLD(void); 56 void processCandidate(MachineRegisterInfo *MRI, MachineBasicBlock &MBB, 57 MachineInstr &MI, Register &SrcReg, Register &DstReg, 58 const GlobalValue *GVal); 59 void processDstReg(MachineRegisterInfo *MRI, Register &DstReg, 60 Register &SrcReg, const GlobalValue *GVal, 61 bool doSrcRegProp); 62 void processInst(MachineRegisterInfo *MRI, MachineInstr *Inst, 63 MachineOperand *RelocOp, const GlobalValue *GVal); 64 void checkADDrr(MachineRegisterInfo *MRI, MachineOperand *RelocOp, 65 const GlobalValue *GVal); 66 void checkShift(MachineRegisterInfo *MRI, MachineBasicBlock &MBB, 67 MachineOperand *RelocOp, const GlobalValue *GVal, 68 unsigned Opcode); 69 70 public: 71 // Main entry point for this pass. 72 bool runOnMachineFunction(MachineFunction &MF) override { 73 if (!skipFunction(MF.getFunction())) { 74 initialize(MF); 75 } 76 return removeLD(); 77 } 78 }; 79 80 // Initialize class variables. 81 void BPFMISimplifyPatchable::initialize(MachineFunction &MFParm) { 82 MF = &MFParm; 83 TII = MF->getSubtarget<BPFSubtarget>().getInstrInfo(); 84 LLVM_DEBUG(dbgs() << "*** BPF simplify patchable insts pass ***\n\n"); 85 } 86 87 void BPFMISimplifyPatchable::checkADDrr(MachineRegisterInfo *MRI, 88 MachineOperand *RelocOp, const GlobalValue *GVal) { 89 const MachineInstr *Inst = RelocOp->getParent(); 90 const MachineOperand *Op1 = &Inst->getOperand(1); 91 const MachineOperand *Op2 = &Inst->getOperand(2); 92 const MachineOperand *BaseOp = (RelocOp == Op1) ? Op2 : Op1; 93 94 // Go through all uses of %1 as in %1 = ADD_rr %2, %3 95 const MachineOperand Op0 = Inst->getOperand(0); 96 auto Begin = MRI->use_begin(Op0.getReg()), End = MRI->use_end(); 97 decltype(End) NextI; 98 for (auto I = Begin; I != End; I = NextI) { 99 NextI = std::next(I); 100 // The candidate needs to have a unique definition. 101 if (!MRI->getUniqueVRegDef(I->getReg())) 102 continue; 103 104 MachineInstr *DefInst = I->getParent(); 105 unsigned Opcode = DefInst->getOpcode(); 106 unsigned COREOp; 107 if (Opcode == BPF::LDB || Opcode == BPF::LDH || Opcode == BPF::LDW || 108 Opcode == BPF::LDD || Opcode == BPF::STB || Opcode == BPF::STH || 109 Opcode == BPF::STW || Opcode == BPF::STD) 110 COREOp = BPF::CORE_MEM; 111 else if (Opcode == BPF::LDB32 || Opcode == BPF::LDH32 || 112 Opcode == BPF::LDW32 || Opcode == BPF::STB32 || 113 Opcode == BPF::STH32 || Opcode == BPF::STW32) 114 COREOp = BPF::CORE_ALU32_MEM; 115 else 116 continue; 117 118 // It must be a form of %1 = *(type *)(%2 + 0) or *(type *)(%2 + 0) = %1. 119 const MachineOperand &ImmOp = DefInst->getOperand(2); 120 if (!ImmOp.isImm() || ImmOp.getImm() != 0) 121 continue; 122 123 BuildMI(*DefInst->getParent(), *DefInst, DefInst->getDebugLoc(), TII->get(COREOp)) 124 .add(DefInst->getOperand(0)).addImm(Opcode).add(*BaseOp) 125 .addGlobalAddress(GVal); 126 DefInst->eraseFromParent(); 127 } 128 } 129 130 void BPFMISimplifyPatchable::checkShift(MachineRegisterInfo *MRI, 131 MachineBasicBlock &MBB, MachineOperand *RelocOp, const GlobalValue *GVal, 132 unsigned Opcode) { 133 // Relocation operand should be the operand #2. 134 MachineInstr *Inst = RelocOp->getParent(); 135 if (RelocOp != &Inst->getOperand(2)) 136 return; 137 138 BuildMI(MBB, *Inst, Inst->getDebugLoc(), TII->get(BPF::CORE_SHIFT)) 139 .add(Inst->getOperand(0)).addImm(Opcode) 140 .add(Inst->getOperand(1)).addGlobalAddress(GVal); 141 Inst->eraseFromParent(); 142 } 143 144 void BPFMISimplifyPatchable::processCandidate(MachineRegisterInfo *MRI, 145 MachineBasicBlock &MBB, MachineInstr &MI, Register &SrcReg, 146 Register &DstReg, const GlobalValue *GVal) { 147 if (MRI->getRegClass(DstReg) == &BPF::GPR32RegClass) { 148 // We can optimize such a pattern: 149 // %1:gpr = LD_imm64 @"llvm.s:0:4$0:2" 150 // %2:gpr32 = LDW32 %1:gpr, 0 151 // %3:gpr = SUBREG_TO_REG 0, %2:gpr32, %subreg.sub_32 152 // %4:gpr = ADD_rr %0:gpr, %3:gpr 153 // or similar patterns below for non-alu32 case. 154 auto Begin = MRI->use_begin(DstReg), End = MRI->use_end(); 155 decltype(End) NextI; 156 for (auto I = Begin; I != End; I = NextI) { 157 NextI = std::next(I); 158 if (!MRI->getUniqueVRegDef(I->getReg())) 159 continue; 160 161 unsigned Opcode = I->getParent()->getOpcode(); 162 if (Opcode == BPF::SUBREG_TO_REG) { 163 Register TmpReg = I->getParent()->getOperand(0).getReg(); 164 processDstReg(MRI, TmpReg, DstReg, GVal, false); 165 } 166 } 167 168 BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(BPF::COPY), DstReg) 169 .addReg(SrcReg, 0, BPF::sub_32); 170 return; 171 } 172 173 // All uses of DstReg replaced by SrcReg 174 processDstReg(MRI, DstReg, SrcReg, GVal, true); 175 } 176 177 void BPFMISimplifyPatchable::processDstReg(MachineRegisterInfo *MRI, 178 Register &DstReg, Register &SrcReg, const GlobalValue *GVal, 179 bool doSrcRegProp) { 180 auto Begin = MRI->use_begin(DstReg), End = MRI->use_end(); 181 decltype(End) NextI; 182 for (auto I = Begin; I != End; I = NextI) { 183 NextI = std::next(I); 184 if (doSrcRegProp) 185 I->setReg(SrcReg); 186 187 // The candidate needs to have a unique definition. 188 if (MRI->getUniqueVRegDef(I->getReg())) 189 processInst(MRI, I->getParent(), &*I, GVal); 190 } 191 } 192 193 // Check to see whether we could do some optimization 194 // to attach relocation to downstream dependent instructions. 195 // Two kinds of patterns are recognized below: 196 // Pattern 1: 197 // %1 = LD_imm64 @"llvm.b:0:4$0:1" <== patch_imm = 4 198 // %2 = LDD %1, 0 <== this insn will be removed 199 // %3 = ADD_rr %0, %2 200 // %4 = LDW[32] %3, 0 OR STW[32] %4, %3, 0 201 // The `%4 = ...` will be transformed to 202 // CORE_[ALU32_]MEM(%4, mem_opcode, %0, @"llvm.b:0:4$0:1") 203 // and later on, BTF emit phase will translate to 204 // %4 = LDW[32] %0, 4 STW[32] %4, %0, 4 205 // and attach a relocation to it. 206 // Pattern 2: 207 // %15 = LD_imm64 @"llvm.t:5:63$0:2" <== relocation type 5 208 // %16 = LDD %15, 0 <== this insn will be removed 209 // %17 = SRA_rr %14, %16 210 // The `%17 = ...` will be transformed to 211 // %17 = CORE_SHIFT(SRA_ri, %14, @"llvm.t:5:63$0:2") 212 // and later on, BTF emit phase will translate to 213 // %r4 = SRA_ri %r4, 63 214 void BPFMISimplifyPatchable::processInst(MachineRegisterInfo *MRI, 215 MachineInstr *Inst, MachineOperand *RelocOp, const GlobalValue *GVal) { 216 unsigned Opcode = Inst->getOpcode(); 217 if (Opcode == BPF::ADD_rr) 218 checkADDrr(MRI, RelocOp, GVal); 219 else if (Opcode == BPF::SLL_rr) 220 checkShift(MRI, *Inst->getParent(), RelocOp, GVal, BPF::SLL_ri); 221 else if (Opcode == BPF::SRA_rr) 222 checkShift(MRI, *Inst->getParent(), RelocOp, GVal, BPF::SRA_ri); 223 else if (Opcode == BPF::SRL_rr) 224 checkShift(MRI, *Inst->getParent(), RelocOp, GVal, BPF::SRL_ri); 225 } 226 227 /// Remove unneeded Load instructions. 228 bool BPFMISimplifyPatchable::removeLD() { 229 MachineRegisterInfo *MRI = &MF->getRegInfo(); 230 MachineInstr *ToErase = nullptr; 231 bool Changed = false; 232 233 for (MachineBasicBlock &MBB : *MF) { 234 for (MachineInstr &MI : MBB) { 235 if (ToErase) { 236 ToErase->eraseFromParent(); 237 ToErase = nullptr; 238 } 239 240 // Ensure the register format is LOAD <reg>, <reg>, 0 241 if (MI.getOpcode() != BPF::LDD && MI.getOpcode() != BPF::LDW && 242 MI.getOpcode() != BPF::LDH && MI.getOpcode() != BPF::LDB && 243 MI.getOpcode() != BPF::LDW32 && MI.getOpcode() != BPF::LDH32 && 244 MI.getOpcode() != BPF::LDB32) 245 continue; 246 247 if (!MI.getOperand(0).isReg() || !MI.getOperand(1).isReg()) 248 continue; 249 250 if (!MI.getOperand(2).isImm() || MI.getOperand(2).getImm()) 251 continue; 252 253 Register DstReg = MI.getOperand(0).getReg(); 254 Register SrcReg = MI.getOperand(1).getReg(); 255 256 MachineInstr *DefInst = MRI->getUniqueVRegDef(SrcReg); 257 if (!DefInst) 258 continue; 259 260 bool IsCandidate = false; 261 const GlobalValue *GVal = nullptr; 262 if (DefInst->getOpcode() == BPF::LD_imm64) { 263 const MachineOperand &MO = DefInst->getOperand(1); 264 if (MO.isGlobal()) { 265 GVal = MO.getGlobal(); 266 auto *GVar = dyn_cast<GlobalVariable>(GVal); 267 if (GVar) { 268 // Global variables representing structure offset or 269 // patchable extern globals. 270 if (GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) { 271 assert(MI.getOperand(2).getImm() == 0); 272 IsCandidate = true; 273 } 274 } 275 } 276 } 277 278 if (!IsCandidate) 279 continue; 280 281 processCandidate(MRI, MBB, MI, SrcReg, DstReg, GVal); 282 283 ToErase = &MI; 284 Changed = true; 285 } 286 } 287 288 return Changed; 289 } 290 291 } // namespace 292 293 INITIALIZE_PASS(BPFMISimplifyPatchable, DEBUG_TYPE, 294 "BPF PreEmit SimplifyPatchable", false, false) 295 296 char BPFMISimplifyPatchable::ID = 0; 297 FunctionPass *llvm::createBPFMISimplifyPatchablePass() { 298 return new BPFMISimplifyPatchable(); 299 } 300