1 //===------ BPFAbstractMemberAccess.cpp - Abstracting Member Accesses -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass abstracted struct/union member accesses in order to support 10 // compile-once run-everywhere (CO-RE). The CO-RE intends to compile the program 11 // which can run on different kernels. In particular, if bpf program tries to 12 // access a particular kernel data structure member, the details of the 13 // intermediate member access will be remembered so bpf loader can do 14 // necessary adjustment right before program loading. 15 // 16 // For example, 17 // 18 // struct s { 19 // int a; 20 // int b; 21 // }; 22 // struct t { 23 // struct s c; 24 // int d; 25 // }; 26 // struct t e; 27 // 28 // For the member access e.c.b, the compiler will generate code 29 // &e + 4 30 // 31 // The compile-once run-everywhere instead generates the following code 32 // r = 4 33 // &e + r 34 // The "4" in "r = 4" can be changed based on a particular kernel version. 35 // For example, on a particular kernel version, if struct s is changed to 36 // 37 // struct s { 38 // int new_field; 39 // int a; 40 // int b; 41 // } 42 // 43 // By repeating the member access on the host, the bpf loader can 44 // adjust "r = 4" as "r = 8". 45 // 46 // This feature relies on the following three intrinsic calls: 47 // addr = preserve_array_access_index(base, dimension, index) 48 // addr = preserve_union_access_index(base, di_index) 49 // !llvm.preserve.access.index <union_ditype> 50 // addr = preserve_struct_access_index(base, gep_index, di_index) 51 // !llvm.preserve.access.index <struct_ditype> 52 // 53 // Bitfield member access needs special attention. User cannot take the 54 // address of a bitfield acceess. To facilitate kernel verifier 55 // for easy bitfield code optimization, a new clang intrinsic is introduced: 56 // uint32_t __builtin_preserve_field_info(member_access, info_kind) 57 // In IR, a chain with two (or more) intrinsic calls will be generated: 58 // ... 59 // addr = preserve_struct_access_index(base, 1, 1) !struct s 60 // uint32_t result = bpf_preserve_field_info(addr, info_kind) 61 // 62 // Suppose the info_kind is FIELD_SIGNEDNESS, 63 // The above two IR intrinsics will be replaced with 64 // a relocatable insn: 65 // signness = /* signness of member_access */ 66 // and signness can be changed by bpf loader based on the 67 // types on the host. 68 // 69 // User can also test whether a field exists or not with 70 // uint32_t result = bpf_preserve_field_info(member_access, FIELD_EXISTENCE) 71 // The field will be always available (result = 1) during initial 72 // compilation, but bpf loader can patch with the correct value 73 // on the target host where the member_access may or may not be available 74 // 75 //===----------------------------------------------------------------------===// 76 77 #include "BPF.h" 78 #include "BPFCORE.h" 79 #include "BPFTargetMachine.h" 80 #include "llvm/BinaryFormat/Dwarf.h" 81 #include "llvm/DebugInfo/BTF/BTF.h" 82 #include "llvm/IR/DebugInfoMetadata.h" 83 #include "llvm/IR/GlobalVariable.h" 84 #include "llvm/IR/Instruction.h" 85 #include "llvm/IR/Instructions.h" 86 #include "llvm/IR/IntrinsicsBPF.h" 87 #include "llvm/IR/Module.h" 88 #include "llvm/IR/PassManager.h" 89 #include "llvm/IR/Type.h" 90 #include "llvm/IR/User.h" 91 #include "llvm/IR/Value.h" 92 #include "llvm/IR/ValueHandle.h" 93 #include "llvm/Pass.h" 94 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 95 #include <stack> 96 97 #define DEBUG_TYPE "bpf-abstract-member-access" 98 99 namespace llvm { 100 constexpr StringRef BPFCoreSharedInfo::AmaAttr; 101 uint32_t BPFCoreSharedInfo::SeqNum; 102 103 Instruction *BPFCoreSharedInfo::insertPassThrough(Module *M, BasicBlock *BB, 104 Instruction *Input, 105 Instruction *Before) { 106 Function *Fn = Intrinsic::getOrInsertDeclaration( 107 M, Intrinsic::bpf_passthrough, {Input->getType(), Input->getType()}); 108 Constant *SeqNumVal = ConstantInt::get(Type::getInt32Ty(BB->getContext()), 109 BPFCoreSharedInfo::SeqNum++); 110 111 auto *NewInst = CallInst::Create(Fn, {SeqNumVal, Input}); 112 NewInst->insertBefore(Before->getIterator()); 113 return NewInst; 114 } 115 } // namespace llvm 116 117 using namespace llvm; 118 119 namespace { 120 class BPFAbstractMemberAccess final { 121 public: 122 BPFAbstractMemberAccess(BPFTargetMachine *TM) : TM(TM) {} 123 124 bool run(Function &F); 125 126 struct CallInfo { 127 uint32_t Kind; 128 uint32_t AccessIndex; 129 MaybeAlign RecordAlignment; 130 MDNode *Metadata; 131 WeakTrackingVH Base; 132 }; 133 typedef std::stack<std::pair<CallInst *, CallInfo>> CallInfoStack; 134 135 private: 136 enum : uint32_t { 137 BPFPreserveArrayAI = 1, 138 BPFPreserveUnionAI = 2, 139 BPFPreserveStructAI = 3, 140 BPFPreserveFieldInfoAI = 4, 141 }; 142 143 TargetMachine *TM; 144 const DataLayout *DL = nullptr; 145 Module *M = nullptr; 146 147 static std::map<std::string, GlobalVariable *> GEPGlobals; 148 // A map to link preserve_*_access_index intrinsic calls. 149 std::map<CallInst *, std::pair<CallInst *, CallInfo>> AIChain; 150 // A map to hold all the base preserve_*_access_index intrinsic calls. 151 // The base call is not an input of any other preserve_* 152 // intrinsics. 153 std::map<CallInst *, CallInfo> BaseAICalls; 154 // A map to hold <AnonRecord, TypeDef> relationships 155 std::map<DICompositeType *, DIDerivedType *> AnonRecords; 156 157 void CheckAnonRecordType(DIDerivedType *ParentTy, DIType *Ty); 158 void CheckCompositeType(DIDerivedType *ParentTy, DICompositeType *CTy); 159 void CheckDerivedType(DIDerivedType *ParentTy, DIDerivedType *DTy); 160 void ResetMetadata(struct CallInfo &CInfo); 161 162 bool doTransformation(Function &F); 163 164 void traceAICall(CallInst *Call, CallInfo &ParentInfo); 165 void traceBitCast(BitCastInst *BitCast, CallInst *Parent, 166 CallInfo &ParentInfo); 167 void traceGEP(GetElementPtrInst *GEP, CallInst *Parent, 168 CallInfo &ParentInfo); 169 void collectAICallChains(Function &F); 170 171 bool IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &Cinfo); 172 bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI, 173 const MDNode *ChildMeta); 174 bool removePreserveAccessIndexIntrinsic(Function &F); 175 bool HasPreserveFieldInfoCall(CallInfoStack &CallStack); 176 void GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment, 177 uint32_t &StartBitOffset, uint32_t &EndBitOffset); 178 uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy, 179 uint32_t AccessIndex, uint32_t PatchImm, 180 MaybeAlign RecordAlignment); 181 182 Value *computeBaseAndAccessKey(CallInst *Call, CallInfo &CInfo, 183 std::string &AccessKey, MDNode *&BaseMeta); 184 MDNode *computeAccessKey(CallInst *Call, CallInfo &CInfo, 185 std::string &AccessKey, bool &IsInt32Ret); 186 bool transformGEPChain(CallInst *Call, CallInfo &CInfo); 187 }; 188 189 std::map<std::string, GlobalVariable *> BPFAbstractMemberAccess::GEPGlobals; 190 } // End anonymous namespace 191 192 bool BPFAbstractMemberAccess::run(Function &F) { 193 LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n"); 194 195 M = F.getParent(); 196 if (!M) 197 return false; 198 199 // Bail out if no debug info. 200 if (M->debug_compile_units().empty()) 201 return false; 202 203 // For each argument/return/local_variable type, trace the type 204 // pattern like '[derived_type]* [composite_type]' to check 205 // and remember (anon record -> typedef) relations where the 206 // anon record is defined as 207 // typedef [const/volatile/restrict]* [anon record] 208 DISubprogram *SP = F.getSubprogram(); 209 if (SP && SP->isDefinition()) { 210 for (DIType *Ty: SP->getType()->getTypeArray()) 211 CheckAnonRecordType(nullptr, Ty); 212 for (const DINode *DN : SP->getRetainedNodes()) { 213 if (const auto *DV = dyn_cast<DILocalVariable>(DN)) 214 CheckAnonRecordType(nullptr, DV->getType()); 215 } 216 } 217 218 DL = &M->getDataLayout(); 219 return doTransformation(F); 220 } 221 222 void BPFAbstractMemberAccess::ResetMetadata(struct CallInfo &CInfo) { 223 if (auto Ty = dyn_cast<DICompositeType>(CInfo.Metadata)) { 224 auto It = AnonRecords.find(Ty); 225 if (It != AnonRecords.end() && It->second != nullptr) 226 CInfo.Metadata = It->second; 227 } 228 } 229 230 void BPFAbstractMemberAccess::CheckCompositeType(DIDerivedType *ParentTy, 231 DICompositeType *CTy) { 232 if (!CTy->getName().empty() || !ParentTy || 233 ParentTy->getTag() != dwarf::DW_TAG_typedef) 234 return; 235 236 auto [It, Inserted] = AnonRecords.try_emplace(CTy, ParentTy); 237 // Two or more typedef's may point to the same anon record. 238 // If this is the case, set the typedef DIType to be nullptr 239 // to indicate the duplication case. 240 if (!Inserted && It->second != ParentTy) 241 It->second = nullptr; 242 } 243 244 void BPFAbstractMemberAccess::CheckDerivedType(DIDerivedType *ParentTy, 245 DIDerivedType *DTy) { 246 DIType *BaseType = DTy->getBaseType(); 247 if (!BaseType) 248 return; 249 250 unsigned Tag = DTy->getTag(); 251 if (Tag == dwarf::DW_TAG_pointer_type) 252 CheckAnonRecordType(nullptr, BaseType); 253 else if (Tag == dwarf::DW_TAG_typedef) 254 CheckAnonRecordType(DTy, BaseType); 255 else 256 CheckAnonRecordType(ParentTy, BaseType); 257 } 258 259 void BPFAbstractMemberAccess::CheckAnonRecordType(DIDerivedType *ParentTy, 260 DIType *Ty) { 261 if (!Ty) 262 return; 263 264 if (auto *CTy = dyn_cast<DICompositeType>(Ty)) 265 return CheckCompositeType(ParentTy, CTy); 266 else if (auto *DTy = dyn_cast<DIDerivedType>(Ty)) 267 return CheckDerivedType(ParentTy, DTy); 268 } 269 270 static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef) { 271 if (Tag != dwarf::DW_TAG_typedef && Tag != dwarf::DW_TAG_const_type && 272 Tag != dwarf::DW_TAG_volatile_type && 273 Tag != dwarf::DW_TAG_restrict_type && 274 Tag != dwarf::DW_TAG_member) 275 return false; 276 if (Tag == dwarf::DW_TAG_typedef && !skipTypedef) 277 return false; 278 return true; 279 } 280 281 static DIType * stripQualifiers(DIType *Ty, bool skipTypedef = true) { 282 while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) { 283 if (!SkipDIDerivedTag(DTy->getTag(), skipTypedef)) 284 break; 285 Ty = DTy->getBaseType(); 286 } 287 return Ty; 288 } 289 290 static const DIType * stripQualifiers(const DIType *Ty) { 291 while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) { 292 if (!SkipDIDerivedTag(DTy->getTag(), true)) 293 break; 294 Ty = DTy->getBaseType(); 295 } 296 return Ty; 297 } 298 299 static uint32_t calcArraySize(const DICompositeType *CTy, uint32_t StartDim) { 300 DINodeArray Elements = CTy->getElements(); 301 uint32_t DimSize = 1; 302 for (uint32_t I = StartDim; I < Elements.size(); ++I) { 303 if (auto *Element = dyn_cast_or_null<DINode>(Elements[I])) 304 if (Element->getTag() == dwarf::DW_TAG_subrange_type) { 305 const DISubrange *SR = cast<DISubrange>(Element); 306 auto *CI = dyn_cast<ConstantInt *>(SR->getCount()); 307 DimSize *= CI->getSExtValue(); 308 } 309 } 310 311 return DimSize; 312 } 313 314 static Type *getBaseElementType(const CallInst *Call) { 315 // Element type is stored in an elementtype() attribute on the first param. 316 return Call->getParamElementType(0); 317 } 318 319 static uint64_t getConstant(const Value *IndexValue) { 320 const ConstantInt *CV = dyn_cast<ConstantInt>(IndexValue); 321 assert(CV); 322 return CV->getValue().getZExtValue(); 323 } 324 325 /// Check whether a call is a preserve_*_access_index intrinsic call or not. 326 bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call, 327 CallInfo &CInfo) { 328 if (!Call) 329 return false; 330 331 const auto *GV = dyn_cast<GlobalValue>(Call->getCalledOperand()); 332 if (!GV) 333 return false; 334 if (GV->getName().starts_with("llvm.preserve.array.access.index")) { 335 CInfo.Kind = BPFPreserveArrayAI; 336 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); 337 if (!CInfo.Metadata) 338 report_fatal_error("Missing metadata for llvm.preserve.array.access.index intrinsic"); 339 CInfo.AccessIndex = getConstant(Call->getArgOperand(2)); 340 CInfo.Base = Call->getArgOperand(0); 341 CInfo.RecordAlignment = DL->getABITypeAlign(getBaseElementType(Call)); 342 return true; 343 } 344 if (GV->getName().starts_with("llvm.preserve.union.access.index")) { 345 CInfo.Kind = BPFPreserveUnionAI; 346 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); 347 if (!CInfo.Metadata) 348 report_fatal_error("Missing metadata for llvm.preserve.union.access.index intrinsic"); 349 ResetMetadata(CInfo); 350 CInfo.AccessIndex = getConstant(Call->getArgOperand(1)); 351 CInfo.Base = Call->getArgOperand(0); 352 return true; 353 } 354 if (GV->getName().starts_with("llvm.preserve.struct.access.index")) { 355 CInfo.Kind = BPFPreserveStructAI; 356 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); 357 if (!CInfo.Metadata) 358 report_fatal_error("Missing metadata for llvm.preserve.struct.access.index intrinsic"); 359 ResetMetadata(CInfo); 360 CInfo.AccessIndex = getConstant(Call->getArgOperand(2)); 361 CInfo.Base = Call->getArgOperand(0); 362 CInfo.RecordAlignment = DL->getABITypeAlign(getBaseElementType(Call)); 363 return true; 364 } 365 if (GV->getName().starts_with("llvm.bpf.preserve.field.info")) { 366 CInfo.Kind = BPFPreserveFieldInfoAI; 367 CInfo.Metadata = nullptr; 368 // Check validity of info_kind as clang did not check this. 369 uint64_t InfoKind = getConstant(Call->getArgOperand(1)); 370 if (InfoKind >= BTF::MAX_FIELD_RELOC_KIND) 371 report_fatal_error("Incorrect info_kind for llvm.bpf.preserve.field.info intrinsic"); 372 CInfo.AccessIndex = InfoKind; 373 return true; 374 } 375 if (GV->getName().starts_with("llvm.bpf.preserve.type.info")) { 376 CInfo.Kind = BPFPreserveFieldInfoAI; 377 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); 378 if (!CInfo.Metadata) 379 report_fatal_error("Missing metadata for llvm.preserve.type.info intrinsic"); 380 uint64_t Flag = getConstant(Call->getArgOperand(1)); 381 if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_TYPE_INFO_FLAG) 382 report_fatal_error("Incorrect flag for llvm.bpf.preserve.type.info intrinsic"); 383 if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_EXISTENCE) 384 CInfo.AccessIndex = BTF::TYPE_EXISTENCE; 385 else if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_MATCH) 386 CInfo.AccessIndex = BTF::TYPE_MATCH; 387 else 388 CInfo.AccessIndex = BTF::TYPE_SIZE; 389 return true; 390 } 391 if (GV->getName().starts_with("llvm.bpf.preserve.enum.value")) { 392 CInfo.Kind = BPFPreserveFieldInfoAI; 393 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); 394 if (!CInfo.Metadata) 395 report_fatal_error("Missing metadata for llvm.preserve.enum.value intrinsic"); 396 uint64_t Flag = getConstant(Call->getArgOperand(2)); 397 if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_ENUM_VALUE_FLAG) 398 report_fatal_error("Incorrect flag for llvm.bpf.preserve.enum.value intrinsic"); 399 if (Flag == BPFCoreSharedInfo::PRESERVE_ENUM_VALUE_EXISTENCE) 400 CInfo.AccessIndex = BTF::ENUM_VALUE_EXISTENCE; 401 else 402 CInfo.AccessIndex = BTF::ENUM_VALUE; 403 return true; 404 } 405 406 return false; 407 } 408 409 static void replaceWithGEP(CallInst *Call, uint32_t DimensionIndex, 410 uint32_t GEPIndex) { 411 uint32_t Dimension = 1; 412 if (DimensionIndex > 0) 413 Dimension = getConstant(Call->getArgOperand(DimensionIndex)); 414 415 Constant *Zero = 416 ConstantInt::get(Type::getInt32Ty(Call->getParent()->getContext()), 0); 417 SmallVector<Value *, 4> IdxList; 418 for (unsigned I = 0; I < Dimension; ++I) 419 IdxList.push_back(Zero); 420 IdxList.push_back(Call->getArgOperand(GEPIndex)); 421 422 auto *GEP = GetElementPtrInst::CreateInBounds(getBaseElementType(Call), 423 Call->getArgOperand(0), IdxList, 424 "", Call->getIterator()); 425 Call->replaceAllUsesWith(GEP); 426 Call->eraseFromParent(); 427 } 428 429 void BPFCoreSharedInfo::removeArrayAccessCall(CallInst *Call) { 430 replaceWithGEP(Call, 1, 2); 431 } 432 433 void BPFCoreSharedInfo::removeStructAccessCall(CallInst *Call) { 434 replaceWithGEP(Call, 0, 1); 435 } 436 437 void BPFCoreSharedInfo::removeUnionAccessCall(CallInst *Call) { 438 Call->replaceAllUsesWith(Call->getArgOperand(0)); 439 Call->eraseFromParent(); 440 } 441 442 bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) { 443 std::vector<CallInst *> PreserveArrayIndexCalls; 444 std::vector<CallInst *> PreserveUnionIndexCalls; 445 std::vector<CallInst *> PreserveStructIndexCalls; 446 bool Found = false; 447 448 for (auto &BB : F) 449 for (auto &I : BB) { 450 auto *Call = dyn_cast<CallInst>(&I); 451 CallInfo CInfo; 452 if (!IsPreserveDIAccessIndexCall(Call, CInfo)) 453 continue; 454 455 Found = true; 456 if (CInfo.Kind == BPFPreserveArrayAI) 457 PreserveArrayIndexCalls.push_back(Call); 458 else if (CInfo.Kind == BPFPreserveUnionAI) 459 PreserveUnionIndexCalls.push_back(Call); 460 else 461 PreserveStructIndexCalls.push_back(Call); 462 } 463 464 // do the following transformation: 465 // . addr = preserve_array_access_index(base, dimension, index) 466 // is transformed to 467 // addr = GEP(base, dimenion's zero's, index) 468 // . addr = preserve_union_access_index(base, di_index) 469 // is transformed to 470 // addr = base, i.e., all usages of "addr" are replaced by "base". 471 // . addr = preserve_struct_access_index(base, gep_index, di_index) 472 // is transformed to 473 // addr = GEP(base, 0, gep_index) 474 for (CallInst *Call : PreserveArrayIndexCalls) 475 BPFCoreSharedInfo::removeArrayAccessCall(Call); 476 for (CallInst *Call : PreserveStructIndexCalls) 477 BPFCoreSharedInfo::removeStructAccessCall(Call); 478 for (CallInst *Call : PreserveUnionIndexCalls) 479 BPFCoreSharedInfo::removeUnionAccessCall(Call); 480 481 return Found; 482 } 483 484 /// Check whether the access index chain is valid. We check 485 /// here because there may be type casts between two 486 /// access indexes. We want to ensure memory access still valid. 487 bool BPFAbstractMemberAccess::IsValidAIChain(const MDNode *ParentType, 488 uint32_t ParentAI, 489 const MDNode *ChildType) { 490 if (!ChildType) 491 return true; // preserve_field_info, no type comparison needed. 492 493 const DIType *PType = stripQualifiers(cast<DIType>(ParentType)); 494 const DIType *CType = stripQualifiers(cast<DIType>(ChildType)); 495 496 // Child is a derived/pointer type, which is due to type casting. 497 // Pointer type cannot be in the middle of chain. 498 if (isa<DIDerivedType>(CType)) 499 return false; 500 501 // Parent is a pointer type. 502 if (const auto *PtrTy = dyn_cast<DIDerivedType>(PType)) { 503 if (PtrTy->getTag() != dwarf::DW_TAG_pointer_type) 504 return false; 505 return stripQualifiers(PtrTy->getBaseType()) == CType; 506 } 507 508 // Otherwise, struct/union/array types 509 const auto *PTy = dyn_cast<DICompositeType>(PType); 510 const auto *CTy = dyn_cast<DICompositeType>(CType); 511 assert(PTy && CTy && "ParentType or ChildType is null or not composite"); 512 513 uint32_t PTyTag = PTy->getTag(); 514 assert(PTyTag == dwarf::DW_TAG_array_type || 515 PTyTag == dwarf::DW_TAG_structure_type || 516 PTyTag == dwarf::DW_TAG_union_type); 517 518 uint32_t CTyTag = CTy->getTag(); 519 assert(CTyTag == dwarf::DW_TAG_array_type || 520 CTyTag == dwarf::DW_TAG_structure_type || 521 CTyTag == dwarf::DW_TAG_union_type); 522 523 // Multi dimensional arrays, base element should be the same 524 if (PTyTag == dwarf::DW_TAG_array_type && PTyTag == CTyTag) 525 return PTy->getBaseType() == CTy->getBaseType(); 526 527 DIType *Ty; 528 if (PTyTag == dwarf::DW_TAG_array_type) 529 Ty = PTy->getBaseType(); 530 else 531 Ty = dyn_cast<DIType>(PTy->getElements()[ParentAI]); 532 533 return dyn_cast<DICompositeType>(stripQualifiers(Ty)) == CTy; 534 } 535 536 void BPFAbstractMemberAccess::traceAICall(CallInst *Call, 537 CallInfo &ParentInfo) { 538 for (User *U : Call->users()) { 539 Instruction *Inst = dyn_cast<Instruction>(U); 540 if (!Inst) 541 continue; 542 543 if (auto *BI = dyn_cast<BitCastInst>(Inst)) { 544 traceBitCast(BI, Call, ParentInfo); 545 } else if (auto *CI = dyn_cast<CallInst>(Inst)) { 546 CallInfo ChildInfo; 547 548 if (IsPreserveDIAccessIndexCall(CI, ChildInfo) && 549 IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex, 550 ChildInfo.Metadata)) { 551 AIChain[CI] = std::make_pair(Call, ParentInfo); 552 traceAICall(CI, ChildInfo); 553 } else { 554 BaseAICalls[Call] = ParentInfo; 555 } 556 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { 557 if (GI->hasAllZeroIndices()) 558 traceGEP(GI, Call, ParentInfo); 559 else 560 BaseAICalls[Call] = ParentInfo; 561 } else { 562 BaseAICalls[Call] = ParentInfo; 563 } 564 } 565 } 566 567 void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast, 568 CallInst *Parent, 569 CallInfo &ParentInfo) { 570 for (User *U : BitCast->users()) { 571 Instruction *Inst = dyn_cast<Instruction>(U); 572 if (!Inst) 573 continue; 574 575 if (auto *BI = dyn_cast<BitCastInst>(Inst)) { 576 traceBitCast(BI, Parent, ParentInfo); 577 } else if (auto *CI = dyn_cast<CallInst>(Inst)) { 578 CallInfo ChildInfo; 579 if (IsPreserveDIAccessIndexCall(CI, ChildInfo) && 580 IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex, 581 ChildInfo.Metadata)) { 582 AIChain[CI] = std::make_pair(Parent, ParentInfo); 583 traceAICall(CI, ChildInfo); 584 } else { 585 BaseAICalls[Parent] = ParentInfo; 586 } 587 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { 588 if (GI->hasAllZeroIndices()) 589 traceGEP(GI, Parent, ParentInfo); 590 else 591 BaseAICalls[Parent] = ParentInfo; 592 } else { 593 BaseAICalls[Parent] = ParentInfo; 594 } 595 } 596 } 597 598 void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent, 599 CallInfo &ParentInfo) { 600 for (User *U : GEP->users()) { 601 Instruction *Inst = dyn_cast<Instruction>(U); 602 if (!Inst) 603 continue; 604 605 if (auto *BI = dyn_cast<BitCastInst>(Inst)) { 606 traceBitCast(BI, Parent, ParentInfo); 607 } else if (auto *CI = dyn_cast<CallInst>(Inst)) { 608 CallInfo ChildInfo; 609 if (IsPreserveDIAccessIndexCall(CI, ChildInfo) && 610 IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex, 611 ChildInfo.Metadata)) { 612 AIChain[CI] = std::make_pair(Parent, ParentInfo); 613 traceAICall(CI, ChildInfo); 614 } else { 615 BaseAICalls[Parent] = ParentInfo; 616 } 617 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { 618 if (GI->hasAllZeroIndices()) 619 traceGEP(GI, Parent, ParentInfo); 620 else 621 BaseAICalls[Parent] = ParentInfo; 622 } else { 623 BaseAICalls[Parent] = ParentInfo; 624 } 625 } 626 } 627 628 void BPFAbstractMemberAccess::collectAICallChains(Function &F) { 629 AIChain.clear(); 630 BaseAICalls.clear(); 631 632 for (auto &BB : F) 633 for (auto &I : BB) { 634 CallInfo CInfo; 635 auto *Call = dyn_cast<CallInst>(&I); 636 if (!IsPreserveDIAccessIndexCall(Call, CInfo) || 637 AIChain.find(Call) != AIChain.end()) 638 continue; 639 640 traceAICall(Call, CInfo); 641 } 642 } 643 644 /// Get the start and the end of storage offset for \p MemberTy. 645 void BPFAbstractMemberAccess::GetStorageBitRange(DIDerivedType *MemberTy, 646 Align RecordAlignment, 647 uint32_t &StartBitOffset, 648 uint32_t &EndBitOffset) { 649 uint32_t MemberBitSize = MemberTy->getSizeInBits(); 650 uint32_t MemberBitOffset = MemberTy->getOffsetInBits(); 651 652 if (RecordAlignment > 8) { 653 // If the Bits are within an aligned 8-byte, set the RecordAlignment 654 // to 8, other report the fatal error. 655 if (MemberBitOffset / 64 != (MemberBitOffset + MemberBitSize) / 64) 656 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, " 657 "requiring too big alignment"); 658 RecordAlignment = Align(8); 659 } 660 661 uint32_t AlignBits = RecordAlignment.value() * 8; 662 if (MemberBitSize > AlignBits) 663 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, " 664 "bitfield size greater than record alignment"); 665 666 StartBitOffset = MemberBitOffset & ~(AlignBits - 1); 667 if ((StartBitOffset + AlignBits) < (MemberBitOffset + MemberBitSize)) 668 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, " 669 "cross alignment boundary"); 670 EndBitOffset = StartBitOffset + AlignBits; 671 } 672 673 uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind, 674 DICompositeType *CTy, 675 uint32_t AccessIndex, 676 uint32_t PatchImm, 677 MaybeAlign RecordAlignment) { 678 if (InfoKind == BTF::FIELD_EXISTENCE) 679 return 1; 680 681 uint32_t Tag = CTy->getTag(); 682 if (InfoKind == BTF::FIELD_BYTE_OFFSET) { 683 if (Tag == dwarf::DW_TAG_array_type) { 684 auto *EltTy = stripQualifiers(CTy->getBaseType()); 685 PatchImm += AccessIndex * calcArraySize(CTy, 1) * 686 (EltTy->getSizeInBits() >> 3); 687 } else if (Tag == dwarf::DW_TAG_structure_type) { 688 auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); 689 if (!MemberTy->isBitField()) { 690 PatchImm += MemberTy->getOffsetInBits() >> 3; 691 } else { 692 unsigned SBitOffset, NextSBitOffset; 693 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, 694 NextSBitOffset); 695 PatchImm += SBitOffset >> 3; 696 } 697 } 698 return PatchImm; 699 } 700 701 if (InfoKind == BTF::FIELD_BYTE_SIZE) { 702 if (Tag == dwarf::DW_TAG_array_type) { 703 auto *EltTy = stripQualifiers(CTy->getBaseType()); 704 return calcArraySize(CTy, 1) * (EltTy->getSizeInBits() >> 3); 705 } else { 706 auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); 707 uint32_t SizeInBits = MemberTy->getSizeInBits(); 708 if (!MemberTy->isBitField()) 709 return SizeInBits >> 3; 710 711 unsigned SBitOffset, NextSBitOffset; 712 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, 713 NextSBitOffset); 714 SizeInBits = NextSBitOffset - SBitOffset; 715 if (SizeInBits & (SizeInBits - 1)) 716 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info"); 717 return SizeInBits >> 3; 718 } 719 } 720 721 if (InfoKind == BTF::FIELD_SIGNEDNESS) { 722 const DIType *BaseTy; 723 if (Tag == dwarf::DW_TAG_array_type) { 724 // Signedness only checked when final array elements are accessed. 725 if (CTy->getElements().size() != 1) 726 report_fatal_error("Invalid array expression for llvm.bpf.preserve.field.info"); 727 BaseTy = stripQualifiers(CTy->getBaseType()); 728 } else { 729 auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); 730 BaseTy = stripQualifiers(MemberTy->getBaseType()); 731 } 732 733 // Only basic types and enum types have signedness. 734 const auto *BTy = dyn_cast<DIBasicType>(BaseTy); 735 while (!BTy) { 736 const auto *CompTy = dyn_cast<DICompositeType>(BaseTy); 737 // Report an error if the field expression does not have signedness. 738 if (!CompTy || CompTy->getTag() != dwarf::DW_TAG_enumeration_type) 739 report_fatal_error("Invalid field expression for llvm.bpf.preserve.field.info"); 740 BaseTy = stripQualifiers(CompTy->getBaseType()); 741 BTy = dyn_cast<DIBasicType>(BaseTy); 742 } 743 uint32_t Encoding = BTy->getEncoding(); 744 return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char); 745 } 746 747 if (InfoKind == BTF::FIELD_LSHIFT_U64) { 748 // The value is loaded into a value with FIELD_BYTE_SIZE size, 749 // and then zero or sign extended to U64. 750 // FIELD_LSHIFT_U64 and FIELD_RSHIFT_U64 are operations 751 // to extract the original value. 752 const Triple &Triple = TM->getTargetTriple(); 753 DIDerivedType *MemberTy = nullptr; 754 bool IsBitField = false; 755 uint32_t SizeInBits; 756 757 if (Tag == dwarf::DW_TAG_array_type) { 758 auto *EltTy = stripQualifiers(CTy->getBaseType()); 759 SizeInBits = calcArraySize(CTy, 1) * EltTy->getSizeInBits(); 760 } else { 761 MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); 762 SizeInBits = MemberTy->getSizeInBits(); 763 IsBitField = MemberTy->isBitField(); 764 } 765 766 if (!IsBitField) { 767 if (SizeInBits > 64) 768 report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); 769 return 64 - SizeInBits; 770 } 771 772 unsigned SBitOffset, NextSBitOffset; 773 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, NextSBitOffset); 774 if (NextSBitOffset - SBitOffset > 64) 775 report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); 776 777 unsigned OffsetInBits = MemberTy->getOffsetInBits(); 778 if (Triple.getArch() == Triple::bpfel) 779 return SBitOffset + 64 - OffsetInBits - SizeInBits; 780 else 781 return OffsetInBits + 64 - NextSBitOffset; 782 } 783 784 if (InfoKind == BTF::FIELD_RSHIFT_U64) { 785 DIDerivedType *MemberTy = nullptr; 786 bool IsBitField = false; 787 uint32_t SizeInBits; 788 if (Tag == dwarf::DW_TAG_array_type) { 789 auto *EltTy = stripQualifiers(CTy->getBaseType()); 790 SizeInBits = calcArraySize(CTy, 1) * EltTy->getSizeInBits(); 791 } else { 792 MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); 793 SizeInBits = MemberTy->getSizeInBits(); 794 IsBitField = MemberTy->isBitField(); 795 } 796 797 if (!IsBitField) { 798 if (SizeInBits > 64) 799 report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); 800 return 64 - SizeInBits; 801 } 802 803 unsigned SBitOffset, NextSBitOffset; 804 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, NextSBitOffset); 805 if (NextSBitOffset - SBitOffset > 64) 806 report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); 807 808 return 64 - SizeInBits; 809 } 810 811 llvm_unreachable("Unknown llvm.bpf.preserve.field.info info kind"); 812 } 813 814 bool BPFAbstractMemberAccess::HasPreserveFieldInfoCall(CallInfoStack &CallStack) { 815 // This is called in error return path, no need to maintain CallStack. 816 while (CallStack.size()) { 817 auto StackElem = CallStack.top(); 818 if (StackElem.second.Kind == BPFPreserveFieldInfoAI) 819 return true; 820 CallStack.pop(); 821 } 822 return false; 823 } 824 825 /// Compute the base of the whole preserve_* intrinsics chains, i.e., the base 826 /// pointer of the first preserve_*_access_index call, and construct the access 827 /// string, which will be the name of a global variable. 828 Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call, 829 CallInfo &CInfo, 830 std::string &AccessKey, 831 MDNode *&TypeMeta) { 832 Value *Base = nullptr; 833 std::string TypeName; 834 CallInfoStack CallStack; 835 836 // Put the access chain into a stack with the top as the head of the chain. 837 while (Call) { 838 CallStack.push(std::make_pair(Call, CInfo)); 839 CInfo = AIChain[Call].second; 840 Call = AIChain[Call].first; 841 } 842 843 // The access offset from the base of the head of chain is also 844 // calculated here as all debuginfo types are available. 845 846 // Get type name and calculate the first index. 847 // We only want to get type name from typedef, structure or union. 848 // If user wants a relocation like 849 // int *p; ... __builtin_preserve_access_index(&p[4]) ... 850 // or 851 // int a[10][20]; ... __builtin_preserve_access_index(&a[2][3]) ... 852 // we will skip them. 853 uint32_t FirstIndex = 0; 854 uint32_t PatchImm = 0; // AccessOffset or the requested field info 855 uint32_t InfoKind = BTF::FIELD_BYTE_OFFSET; 856 while (CallStack.size()) { 857 auto StackElem = CallStack.top(); 858 Call = StackElem.first; 859 CInfo = StackElem.second; 860 861 if (!Base) 862 Base = CInfo.Base; 863 864 DIType *PossibleTypeDef = stripQualifiers(cast<DIType>(CInfo.Metadata), 865 false); 866 DIType *Ty = stripQualifiers(PossibleTypeDef); 867 if (CInfo.Kind == BPFPreserveUnionAI || 868 CInfo.Kind == BPFPreserveStructAI) { 869 // struct or union type. If the typedef is in the metadata, always 870 // use the typedef. 871 TypeName = std::string(PossibleTypeDef->getName()); 872 TypeMeta = PossibleTypeDef; 873 PatchImm += FirstIndex * (Ty->getSizeInBits() >> 3); 874 break; 875 } 876 877 assert(CInfo.Kind == BPFPreserveArrayAI); 878 879 // Array entries will always be consumed for accumulative initial index. 880 CallStack.pop(); 881 882 // BPFPreserveArrayAI 883 uint64_t AccessIndex = CInfo.AccessIndex; 884 885 DIType *BaseTy = nullptr; 886 bool CheckElemType = false; 887 if (const auto *CTy = dyn_cast<DICompositeType>(Ty)) { 888 // array type 889 assert(CTy->getTag() == dwarf::DW_TAG_array_type); 890 891 892 FirstIndex += AccessIndex * calcArraySize(CTy, 1); 893 BaseTy = stripQualifiers(CTy->getBaseType()); 894 CheckElemType = CTy->getElements().size() == 1; 895 } else { 896 // pointer type 897 auto *DTy = cast<DIDerivedType>(Ty); 898 assert(DTy->getTag() == dwarf::DW_TAG_pointer_type); 899 900 BaseTy = stripQualifiers(DTy->getBaseType()); 901 CTy = dyn_cast<DICompositeType>(BaseTy); 902 if (!CTy) { 903 CheckElemType = true; 904 } else if (CTy->getTag() != dwarf::DW_TAG_array_type) { 905 FirstIndex += AccessIndex; 906 CheckElemType = true; 907 } else { 908 FirstIndex += AccessIndex * calcArraySize(CTy, 0); 909 } 910 } 911 912 if (CheckElemType) { 913 auto *CTy = dyn_cast<DICompositeType>(BaseTy); 914 if (!CTy) { 915 if (HasPreserveFieldInfoCall(CallStack)) 916 report_fatal_error("Invalid field access for llvm.preserve.field.info intrinsic"); 917 return nullptr; 918 } 919 920 unsigned CTag = CTy->getTag(); 921 if (CTag == dwarf::DW_TAG_structure_type || CTag == dwarf::DW_TAG_union_type) { 922 TypeName = std::string(CTy->getName()); 923 } else { 924 if (HasPreserveFieldInfoCall(CallStack)) 925 report_fatal_error("Invalid field access for llvm.preserve.field.info intrinsic"); 926 return nullptr; 927 } 928 TypeMeta = CTy; 929 PatchImm += FirstIndex * (CTy->getSizeInBits() >> 3); 930 break; 931 } 932 } 933 assert(TypeName.size()); 934 AccessKey += std::to_string(FirstIndex); 935 936 // Traverse the rest of access chain to complete offset calculation 937 // and access key construction. 938 while (CallStack.size()) { 939 auto StackElem = CallStack.top(); 940 CInfo = StackElem.second; 941 CallStack.pop(); 942 943 if (CInfo.Kind == BPFPreserveFieldInfoAI) { 944 InfoKind = CInfo.AccessIndex; 945 if (InfoKind == BTF::FIELD_EXISTENCE) 946 PatchImm = 1; 947 break; 948 } 949 950 // If the next Call (the top of the stack) is a BPFPreserveFieldInfoAI, 951 // the action will be extracting field info. 952 if (CallStack.size()) { 953 auto StackElem2 = CallStack.top(); 954 CallInfo CInfo2 = StackElem2.second; 955 if (CInfo2.Kind == BPFPreserveFieldInfoAI) { 956 InfoKind = CInfo2.AccessIndex; 957 assert(CallStack.size() == 1); 958 } 959 } 960 961 // Access Index 962 uint64_t AccessIndex = CInfo.AccessIndex; 963 AccessKey += ":" + std::to_string(AccessIndex); 964 965 MDNode *MDN = CInfo.Metadata; 966 // At this stage, it cannot be pointer type. 967 auto *CTy = cast<DICompositeType>(stripQualifiers(cast<DIType>(MDN))); 968 PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex, PatchImm, 969 CInfo.RecordAlignment); 970 } 971 972 // Access key is the 973 // "llvm." + type name + ":" + reloc type + ":" + patched imm + "$" + 974 // access string, 975 // uniquely identifying one relocation. 976 // The prefix "llvm." indicates this is a temporary global, which should 977 // not be emitted to ELF file. 978 AccessKey = "llvm." + TypeName + ":" + std::to_string(InfoKind) + ":" + 979 std::to_string(PatchImm) + "$" + AccessKey; 980 981 return Base; 982 } 983 984 MDNode *BPFAbstractMemberAccess::computeAccessKey(CallInst *Call, 985 CallInfo &CInfo, 986 std::string &AccessKey, 987 bool &IsInt32Ret) { 988 DIType *Ty = stripQualifiers(cast<DIType>(CInfo.Metadata), false); 989 assert(!Ty->getName().empty()); 990 991 int64_t PatchImm; 992 std::string AccessStr("0"); 993 if (CInfo.AccessIndex == BTF::TYPE_EXISTENCE || 994 CInfo.AccessIndex == BTF::TYPE_MATCH) { 995 PatchImm = 1; 996 } else if (CInfo.AccessIndex == BTF::TYPE_SIZE) { 997 // typedef debuginfo type has size 0, get the eventual base type. 998 DIType *BaseTy = stripQualifiers(Ty, true); 999 PatchImm = BaseTy->getSizeInBits() / 8; 1000 } else { 1001 // ENUM_VALUE_EXISTENCE and ENUM_VALUE 1002 IsInt32Ret = false; 1003 1004 // The argument could be a global variable or a getelementptr with base to 1005 // a global variable depending on whether the clang option `opaque-options` 1006 // is set or not. 1007 const GlobalVariable *GV = 1008 cast<GlobalVariable>(Call->getArgOperand(1)->stripPointerCasts()); 1009 assert(GV->hasInitializer()); 1010 const ConstantDataArray *DA = cast<ConstantDataArray>(GV->getInitializer()); 1011 assert(DA->isString()); 1012 StringRef ValueStr = DA->getAsString(); 1013 1014 // ValueStr format: <EnumeratorStr>:<Value> 1015 size_t Separator = ValueStr.find_first_of(':'); 1016 StringRef EnumeratorStr = ValueStr.substr(0, Separator); 1017 1018 // Find enumerator index in the debuginfo 1019 DIType *BaseTy = stripQualifiers(Ty, true); 1020 const auto *CTy = cast<DICompositeType>(BaseTy); 1021 assert(CTy->getTag() == dwarf::DW_TAG_enumeration_type); 1022 int EnumIndex = 0; 1023 for (const auto Element : CTy->getElements()) { 1024 const auto *Enum = cast<DIEnumerator>(Element); 1025 if (Enum->getName() == EnumeratorStr) { 1026 AccessStr = std::to_string(EnumIndex); 1027 break; 1028 } 1029 EnumIndex++; 1030 } 1031 1032 if (CInfo.AccessIndex == BTF::ENUM_VALUE) { 1033 StringRef EValueStr = ValueStr.substr(Separator + 1); 1034 PatchImm = std::stoll(std::string(EValueStr)); 1035 } else { 1036 PatchImm = 1; 1037 } 1038 } 1039 1040 AccessKey = "llvm." + Ty->getName().str() + ":" + 1041 std::to_string(CInfo.AccessIndex) + std::string(":") + 1042 std::to_string(PatchImm) + std::string("$") + AccessStr; 1043 1044 return Ty; 1045 } 1046 1047 /// Call/Kind is the base preserve_*_access_index() call. Attempts to do 1048 /// transformation to a chain of relocable GEPs. 1049 bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call, 1050 CallInfo &CInfo) { 1051 std::string AccessKey; 1052 MDNode *TypeMeta; 1053 Value *Base = nullptr; 1054 bool IsInt32Ret; 1055 1056 IsInt32Ret = CInfo.Kind == BPFPreserveFieldInfoAI; 1057 if (CInfo.Kind == BPFPreserveFieldInfoAI && CInfo.Metadata) { 1058 TypeMeta = computeAccessKey(Call, CInfo, AccessKey, IsInt32Ret); 1059 } else { 1060 Base = computeBaseAndAccessKey(Call, CInfo, AccessKey, TypeMeta); 1061 if (!Base) 1062 return false; 1063 } 1064 1065 BasicBlock *BB = Call->getParent(); 1066 GlobalVariable *GV; 1067 1068 if (GEPGlobals.find(AccessKey) == GEPGlobals.end()) { 1069 IntegerType *VarType; 1070 if (IsInt32Ret) 1071 VarType = Type::getInt32Ty(BB->getContext()); // 32bit return value 1072 else 1073 VarType = Type::getInt64Ty(BB->getContext()); // 64bit ptr or enum value 1074 1075 GV = new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage, 1076 nullptr, AccessKey); 1077 GV->addAttribute(BPFCoreSharedInfo::AmaAttr); 1078 GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta); 1079 GEPGlobals[AccessKey] = GV; 1080 } else { 1081 GV = GEPGlobals[AccessKey]; 1082 } 1083 1084 if (CInfo.Kind == BPFPreserveFieldInfoAI) { 1085 // Load the global variable which represents the returned field info. 1086 LoadInst *LDInst; 1087 if (IsInt32Ret) 1088 LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV, "", 1089 Call->getIterator()); 1090 else 1091 LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "", 1092 Call->getIterator()); 1093 1094 Instruction *PassThroughInst = 1095 BPFCoreSharedInfo::insertPassThrough(M, BB, LDInst, Call); 1096 Call->replaceAllUsesWith(PassThroughInst); 1097 Call->eraseFromParent(); 1098 return true; 1099 } 1100 1101 // For any original GEP Call and Base %2 like 1102 // %4 = bitcast %struct.net_device** %dev1 to i64* 1103 // it is transformed to: 1104 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0 1105 // %7 = bitcast %struct.sk_buff* %2 to i8* 1106 // %8 = getelementptr i8, i8* %7, %6 1107 // %9 = bitcast i8* %8 to i64* 1108 // using %9 instead of %4 1109 // The original Call inst is removed. 1110 1111 // Load the global variable. 1112 auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "", 1113 Call->getIterator()); 1114 1115 // Generate a BitCast 1116 auto *BCInst = 1117 new BitCastInst(Base, PointerType::getUnqual(BB->getContext())); 1118 BCInst->insertBefore(Call->getIterator()); 1119 1120 // Generate a GetElementPtr 1121 auto *GEP = GetElementPtrInst::Create(Type::getInt8Ty(BB->getContext()), 1122 BCInst, LDInst); 1123 GEP->insertBefore(Call->getIterator()); 1124 1125 // Generate a BitCast 1126 auto *BCInst2 = new BitCastInst(GEP, Call->getType()); 1127 BCInst2->insertBefore(Call->getIterator()); 1128 1129 // For the following code, 1130 // Block0: 1131 // ... 1132 // if (...) goto Block1 else ... 1133 // Block1: 1134 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0 1135 // %7 = bitcast %struct.sk_buff* %2 to i8* 1136 // %8 = getelementptr i8, i8* %7, %6 1137 // ... 1138 // goto CommonExit 1139 // Block2: 1140 // ... 1141 // if (...) goto Block3 else ... 1142 // Block3: 1143 // %6 = load llvm.bpf_map:0:40$0:0:0:2:0 1144 // %7 = bitcast %struct.sk_buff* %2 to i8* 1145 // %8 = getelementptr i8, i8* %7, %6 1146 // ... 1147 // goto CommonExit 1148 // CommonExit 1149 // SimplifyCFG may generate: 1150 // Block0: 1151 // ... 1152 // if (...) goto Block_Common else ... 1153 // Block2: 1154 // ... 1155 // if (...) goto Block_Common else ... 1156 // Block_Common: 1157 // PHI = [llvm.sk_buff:0:50$0:0:0:2:0, llvm.bpf_map:0:40$0:0:0:2:0] 1158 // %6 = load PHI 1159 // %7 = bitcast %struct.sk_buff* %2 to i8* 1160 // %8 = getelementptr i8, i8* %7, %6 1161 // ... 1162 // goto CommonExit 1163 // For the above code, we cannot perform proper relocation since 1164 // "load PHI" has two possible relocations. 1165 // 1166 // To prevent above tail merging, we use __builtin_bpf_passthrough() 1167 // where one of its parameters is a seq_num. Since two 1168 // __builtin_bpf_passthrough() funcs will always have different seq_num, 1169 // tail merging cannot happen. The __builtin_bpf_passthrough() will be 1170 // removed in the beginning of Target IR passes. 1171 // 1172 // This approach is also used in other places when global var 1173 // representing a relocation is used. 1174 Instruction *PassThroughInst = 1175 BPFCoreSharedInfo::insertPassThrough(M, BB, BCInst2, Call); 1176 Call->replaceAllUsesWith(PassThroughInst); 1177 Call->eraseFromParent(); 1178 1179 return true; 1180 } 1181 1182 bool BPFAbstractMemberAccess::doTransformation(Function &F) { 1183 bool Transformed = false; 1184 1185 // Collect PreserveDIAccessIndex Intrinsic call chains. 1186 // The call chains will be used to generate the access 1187 // patterns similar to GEP. 1188 collectAICallChains(F); 1189 1190 for (auto &C : BaseAICalls) 1191 Transformed = transformGEPChain(C.first, C.second) || Transformed; 1192 1193 return removePreserveAccessIndexIntrinsic(F) || Transformed; 1194 } 1195 1196 PreservedAnalyses 1197 BPFAbstractMemberAccessPass::run(Function &F, FunctionAnalysisManager &AM) { 1198 return BPFAbstractMemberAccess(TM).run(F) ? PreservedAnalyses::none() 1199 : PreservedAnalyses::all(); 1200 } 1201