Lines Matching defs:IRB
806 IRBuilder<> IRB(*C);
819 IRB.getVoidTy(), IRB.getInt32Ty());
824 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
825 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
826 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
827 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
828 IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
847 M, "__msan_metadata_ptr_for_load_n", PtrTy, IRB.getInt64Ty());
849 M, "__msan_metadata_ptr_for_store_n", PtrTy, IRB.getInt64Ty());
853 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
855 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy);
869 IRBuilder<> IRB(*C);
879 IRB.getVoidTy(), IRB.getInt32Ty());
883 WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
889 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
895 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
903 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
910 getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
918 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
923 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy,
924 IRB.getInt32Ty());
929 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy);
932 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
934 IRB.getVoidTy(), PtrTy, IntptrTy);
944 IRBuilder<> IRB(*C);
949 TLI.getAttrList(C, {0}, /*Signed=*/false, /*Ret=*/true), IRB.getInt32Ty(),
950 IRB.getInt32Ty());
953 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty());
960 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
963 "__msan_instrument_asm_store", IRB.getVoidTy(), PtrTy, IntptrTy);
1070 IRBuilder<> IRB(*C);
1071 IntptrTy = IRB.getIntPtrTy(DL);
1072 OriginTy = IRB.getInt32Ty();
1073 PtrTy = IRB.getPtrTy();
1080 M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
1082 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1083 IRB.getInt32(TrackOrigins), "__msan_track_origins");
1087 M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
1088 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1090 IRB.getInt32(Recover), "__msan_keep_going");
1109 virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
1213 IRBuilder<> IRB(FnPrologueEnd);
1214 insertKmsanPrologue(IRB);
1241 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1244 return IRB.CreateCall(MS.MsanChainOriginFn, V);
1247 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1253 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
1254 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
1258 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1269 Value *Size = IRB.CreateTypeSize(MS.IntptrTy, TS);
1271 IRB.CreateAdd(Size, ConstantInt::get(MS.IntptrTy, kOriginSize - 1));
1273 IRB.CreateUDiv(RoundUp, ConstantInt::get(MS.IntptrTy, kOriginSize));
1275 SplitBlockAndInsertSimpleForLoop(End, IRB.GetInsertPoint());
1276 IRB.SetInsertPoint(InsertPt);
1278 Value *GEP = IRB.CreateGEP(MS.OriginTy, OriginPtr, Index);
1279 IRB.CreateAlignedStore(Origin, GEP, kMinOriginAlignment);
1288 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1289 Value *IntptrOriginPtr = IRB.CreatePointerCast(OriginPtr, MS.PtrTy);
1291 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
1293 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
1301 i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
1302 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
1307 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1313 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1322 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1335 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1336 CallBase *CB = IRB.CreateCall(Fn, {ConvertedShadow2, Addr, Origin});
1340 Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1342 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
1351 IRBuilder<> IRB(SI);
1360 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1362 StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
1370 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
1387 /// Helper function to insert a warning at IRB's current insert point.
1388 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1390 Origin = (Value *)IRB.getInt32(0);
1393 if (shouldDisambiguateWarningLocation(IRB.getCurrentDebugLocation())) {
1401 if (NewDebugLoc && NewDebugLoc != IRB.getCurrentDebugLocation()) {
1404 IRBuilder<> IRBOrigin(&*IRB.GetInsertPoint());
1412 IRB.CreateCall(MS.WarningFn, Origin)->setCannotMerge();
1414 IRB.CreateCall(MS.WarningFn)->setCannotMerge();
1420 void materializeOneCheck(IRBuilder<> &IRB, Value *ConvertedShadow,
1429 ConvertedShadow = convertShadowToScalar(ConvertedShadow, IRB);
1431 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1432 CallBase *CB = IRB.CreateCall(
1434 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)});
1438 Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1440 Cmp, &*IRB.GetInsertPoint(),
1443 IRB.SetInsertPoint(CheckTerm);
1444 insertWarningFn(IRB, Origin);
1459 IRBuilder<> IRB(Instruction);
1470 insertWarningFn(IRB, ShadowData.Origin);
1480 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1489 Shadow = convertToBool(Shadow, IRB, "_mscmp");
1490 ConvertedShadow = convertToBool(ConvertedShadow, IRB, "_mscmp");
1491 Shadow = IRB.CreateOr(Shadow, ConvertedShadow, "_msor");
1496 IRBuilder<> IRB(Instruction);
1497 materializeOneCheck(IRB, Shadow, nullptr);
1526 void insertKmsanPrologue(IRBuilder<> &IRB) {
1527 Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
1528 Constant *Zero = IRB.getInt32(0);
1529 MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1530 {Zero, IRB.getInt32(0)}, "param_shadow");
1531 MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1532 {Zero, IRB.getInt32(1)}, "retval_shadow");
1533 MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1534 {Zero, IRB.getInt32(2)}, "va_arg_shadow");
1535 MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1536 {Zero, IRB.getInt32(3)}, "va_arg_origin");
1538 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1539 {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
1540 MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1541 {Zero, IRB.getInt32(5)}, "param_origin");
1543 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1544 {Zero, IRB.getInt32(6)}, "retval_origin");
1546 MS.MsanMetadataAlloca = IRB.CreateAlloca(MS.MsanMetadata, 0u);
1635 IRBuilder<> &IRB) {
1636 Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0);
1641 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1642 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1645 Aggregator = IRB.CreateOr(Aggregator, ShadowBool);
1655 IRBuilder<> &IRB) {
1657 return IRB.getIntN(/* width */ 1, /* value */ 0);
1659 Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
1660 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1663 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1664 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1665 Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
1673 Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1675 return collapseStructShadow(Struct, V, IRB);
1677 return collapseArrayShadow(Array, V, IRB);
1680 return convertShadowToScalar(IRB.CreateOrReduce(V), IRB);
1683 return IRB.CreateBitCast(V, IntegerType::get(*MS.C, BitWidth));
1689 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1692 return convertToBool(convertShadowToScalar(V, IRB), IRB, name);
1696 return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name);
1737 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1739 Value *OffsetLong = IRB.CreatePointerCast(Addr, IntptrTy);
1742 OffsetLong = IRB.CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1745 OffsetLong = IRB.CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1758 getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1767 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1771 IRB.CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1773 Value *ShadowPtr = IRB.CreateIntToPtr(
1782 IRB.CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1785 OriginLong = IRB.CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1787 OriginPtr = IRB.CreateIntToPtr(
1794 Value *createMetadataCall(IRBuilder<> &IRB, FunctionCallee Callee,
1797 IRB.CreateCall(Callee,
1799 return IRB.CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
1802 return IRB.CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1806 IRBuilder<> &IRB,
1814 Value *AddrCast = IRB.CreatePointerCast(Addr, MS.PtrTy);
1816 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
1820 IRB,
1824 Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
1825 ShadowPtr = IRB.CreatePointerCast(ShadowPtr, MS.PtrTy);
1826 Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
1835 IRBuilder<> &IRB,
1841 return getShadowOriginPtrKernelNoVec(Addr, IRB, ShadowTy, isStore);
1847 FixedVectorType::get(IRB.getPtrTy(), NumElements));
1851 FixedVectorType::get(IRB.getPtrTy(), NumElements));
1854 IRB.CreateExtractElement(Addr, ConstantInt::get(IRB.getInt32Ty(), i));
1856 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy, isStore);
1858 ShadowPtrs = IRB.CreateInsertElement(
1859 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.getInt32Ty(), i));
1861 OriginPtrs = IRB.CreateInsertElement(
1862 OriginPtrs, OriginPtr, ConstantInt::get(IRB.getInt32Ty(), i));
1867 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1872 return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1873 return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1879 Value *getShadowPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1880 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1882 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1883 return IRB.CreateIntToPtr(Base, IRB.getPtrTy(0), "_msarg");
1887 Value *getOriginPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1890 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1892 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1893 return IRB.CreateIntToPtr(Base, IRB.getPtrTy(0), "_msarg_o");
1897 Value *getShadowPtrForRetval(IRBuilder<> &IRB) {
1898 return IRB.CreatePointerCast(MS.RetvalTLS, IRB.getPtrTy(0), "_msret");
2172 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
2186 return ConstantDataVector::get(IRB.getContext(), OrderingTable);
2206 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
2220 return ConstantDataVector::get(IRB.getContext(), OrderingTable);
2249 NextNodeIRBuilder IRB(&I);
2256 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2258 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
2273 &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment));
2293 IRBuilder<> IRB(&I);
2296 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, getShadowTy(Val), Align(1),
2309 IRB.CreateStore(getCleanShadow(Val), ShadowPtr);
2328 IRBuilder<> IRB(&I);
2329 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
2336 IRBuilder<> IRB(&I);
2339 setShadow(&I, IRB.CreateInsertElement(Shadow0, Shadow1, I.getOperand(2),
2345 IRBuilder<> IRB(&I);
2348 setShadow(&I, IRB.CreateShuffleVector(Shadow0, Shadow1, I.getShuffleMask(),
2355 IRBuilder<> IRB(&I);
2356 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
2361 IRBuilder<> IRB(&I);
2362 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
2367 IRBuilder<> IRB(&I);
2368 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
2379 IRBuilder<> IRB(&I);
2380 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
2385 IRBuilder<> IRB(&I);
2386 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2392 IRBuilder<> IRB(&I);
2393 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2411 IRBuilder<> IRB(&I);
2422 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2423 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2425 Value *S1S2 = IRB.CreateAnd(S1, S2);
2426 Value *V1S2 = IRB.CreateAnd(V1, S2);
2427 Value *S1V2 = IRB.CreateAnd(S1, V2);
2428 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2433 IRBuilder<> IRB(&I);
2441 Value *V1 = IRB.CreateNot(I.getOperand(0));
2442 Value *V2 = IRB.CreateNot(I.getOperand(1));
2444 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2445 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2447 Value *S1S2 = IRB.CreateAnd(S1, S2);
2448 Value *V1S2 = IRB.CreateAnd(V1, S2);
2449 Value *S1V2 = IRB.CreateAnd(S1, V2);
2450 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2472 IRBuilder<> &IRB;
2476 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2477 : IRB(IRB), MSV(MSV) {}
2486 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2487 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
2499 Value *Cond = MSV->convertToBool(OpShadow, IRB);
2500 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2519 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
2533 MSV->paintOrigin(IRB, Origin, OriginPtr, TS, kMinOriginAlignment);
2545 IRBuilder<> IRB(&I);
2546 OriginCombiner OC(this, IRB);
2562 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2570 return IRB.CreateICmpNE(V, getCleanShadow(V));
2573 return IRB.CreateIntCast(V, dstTy, Signed);
2577 return IRB.CreateIntCast(V, dstTy, Signed);
2578 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
2580 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
2581 return IRB.CreateBitCast(V2, dstTy);
2586 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2591 return IRB.CreatePtrToInt(V, ShadowTy);
2593 return IRB.CreateBitCast(V, ShadowTy);
2598 IRBuilder<> IRB(&I);
2599 ShadowAndOriginCombiner SC(this, IRB);
2645 IRBuilder<> IRB(&I);
2647 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
2670 IRBuilder<> IRB(&I);
2692 IRBuilder<> IRB(&I);
2701 A = IRB.CreatePointerCast(A, Sa->getType());
2702 B = IRB.CreatePointerCast(B, Sb->getType());
2707 Value *C = IRB.CreateXor(A, B);
2708 Value *Sc = IRB.CreateOr(Sa, Sb);
2716 Value *LHS = IRB.CreateICmpNE(Sc, Zero);
2718 IRB.CreateICmpEQ(IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero);
2719 Value *Si = IRB.CreateAnd(LHS, RHS);
2731 IRBuilder<> IRB(&I);
2740 A = IRB.CreatePointerCast(A, Sa->getType());
2741 B = IRB.CreatePointerCast(B, Sb->getType());
2758 V = IRB.CreateXor(V, ConstantInt::get(V->getType(), MinVal));
2761 Value *Min = IRB.CreateAnd(V, IRB.CreateNot(S));
2762 Value *Max = IRB.CreateOr(V, S);
2768 Value *S1 = IRB.CreateICmp(I.getUnsignedPredicate(), Amin, Bmax);
2769 Value *S2 = IRB.CreateICmp(I.getUnsignedPredicate(), Amax, Bmin);
2771 Value *Si = IRB.CreateXor(S1, S2);
2799 IRBuilder<> IRB(&I);
2800 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2841 IRBuilder<> IRB(&I);
2847 IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
2849 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2850 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2859 IRBuilder<> IRB(&I);
2866 IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
2868 Value *Shift = IRB.CreateIntrinsic(I.getIntrinsicID(), S2Conv->getType(),
2870 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2886 IRBuilder<> IRB(&I);
2887 IRB.CreateCall(MS.MemmoveFn,
2889 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2909 IRBuilder<> IRB(&I);
2910 IRB.CreateCall(MS.MemcpyFn,
2912 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2918 IRBuilder<> IRB(&I);
2919 IRB.CreateCall(
2922 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2923 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2936 IRBuilder<> IRB(&I);
2944 Addr, IRB, Shadow->getType(), Align(1), /*isStore*/ true);
2945 IRB.CreateAlignedStore(Shadow, ShadowPtr, Align(1));
2952 IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2961 IRBuilder<> IRB(&I);
2971 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2973 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
2983 setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
3015 IRBuilder<> IRB(&I);
3016 ShadowAndOriginCombiner SC(this, IRB);
3088 IRBuilder<> IRB(&I);
3091 setShadow(&I, IRB.CreateIntrinsic(Intrinsic::bswap, ArrayRef(&OpType, 1),
3097 IRBuilder<> IRB(&I);
3101 Value *BoolShadow = IRB.CreateIsNotNull(getShadow(Src), "_mscz_bs");
3106 Value *BoolZeroPoison = IRB.CreateIsNull(Src, "_mscz_bzp");
3107 BoolShadow = IRB.CreateOr(BoolShadow, BoolZeroPoison, "_mscz_bs");
3111 IRB.CreateSExt(BoolShadow, getShadowTy(Src), "_mscz_os");
3134 IRBuilder<> IRB(&I);
3164 AggShadow = IRB.CreateExtractElement(
3165 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
3167 Value *MoreShadow = IRB.CreateExtractElement(
3168 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
3169 AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
3185 ResultShadow = IRB.CreateInsertElement(
3187 ConstantInt::get(IRB.getInt32Ty(), i));
3199 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3201 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
3203 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
3204 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
3209 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3210 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
3211 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
3212 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
3215 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
3218 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
3219 return IRB.CreateSExt(S2, T);
3231 IRBuilder<> IRB(&I);
3236 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3237 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
3240 Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(),
3241 {IRB.CreateBitCast(S1, V1->getType()), V2});
3242 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
3243 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
3297 IRBuilder<> IRB(&I);
3308 S1 = IRB.CreateBitCast(S1, T);
3309 S2 = IRB.CreateBitCast(S2, T);
3312 IRB.CreateSExt(IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
3314 IRB.CreateSExt(IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
3316 S1_ext = IRB.CreateBitCast(S1_ext, getMMXVectorTy(64));
3317 S2_ext = IRB.CreateBitCast(S2_ext, getMMXVectorTy(64));
3320 Value *S = IRB.CreateIntrinsic(getSignedPackIntrinsic(I.getIntrinsicID()),
3324 S = IRB.CreateBitCast(S, getShadowTy(&I));
3341 Value *findDppPoisonedOutput(IRBuilder<> &IRB, Value *S, unsigned SrcMask,
3346 S = IRB.CreateSelect(createDppMask(Width, SrcMask), S,
3348 Value *SElem = IRB.CreateOrReduce(S);
3349 Value *IsClean = IRB.CreateIsNull(SElem, "_msdpp");
3352 return IRB.CreateSelect(
3368 IRBuilder<> IRB(&I);
3372 Value *S = IRB.CreateOr(S0, S1);
3383 Value *SI1 = findDppPoisonedOutput(IRB, S, SrcMask, DstMask);
3387 SI1 = IRB.CreateOr(
3388 SI1, findDppPoisonedOutput(IRB, S, SrcMask << 4, DstMask << 4));
3392 S = IRB.CreateSExt(SI1, S->getType(), "_msdpp");
3398 Value *convertBlendvToSelectMask(IRBuilder<> &IRB, Value *C) {
3399 C = CreateAppToShadowCast(IRB, C);
3402 C = IRB.CreateAShr(C, ElSize - 1);
3403 FVT = FixedVectorType::get(IRB.getInt1Ty(), FVT->getNumElements());
3404 return IRB.CreateTrunc(C, FVT);
3417 IRBuilder<> IRB(&I);
3419 C = convertBlendvToSelectMask(IRB, C);
3420 Sc = convertBlendvToSelectMask(IRB, Sc);
3436 IRBuilder<> IRB(&I);
3439 Value *S = IRB.CreateOr(Shadow0, Shadow1);
3440 S = IRB.CreateBitCast(S, ResTy);
3441 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
3443 S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
3444 S = IRB.CreateBitCast(S, getShadowTy(&I));
3454 IRBuilder<> IRB(&I);
3457 Value *S = IRB.CreateOr(Shadow0, Shadow1);
3458 S = IRB.CreateBitCast(S, ResTy);
3459 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
3461 S = IRB.CreateBitCast(S, getShadowTy(&I));
3470 IRBuilder<> IRB(&I);
3474 Value *S0 = IRB.CreateOr(Shadow0, Shadow1);
3475 Value *S = IRB.CreateSExt(
3476 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
3485 IRBuilder<> IRB(&I);
3488 Value *S0 = IRB.CreateOr(Shadow0, Shadow1);
3489 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
3497 IRBuilder<> IRB(&I);
3498 Value *S = IRB.CreateOrReduce(getShadow(&I, 0));
3507 IRBuilder<> IRB(&I);
3509 Value *OperandUnsetBits = IRB.CreateNot(I.getOperand(0));
3510 Value *OperandUnsetOrPoison = IRB.CreateOr(OperandUnsetBits, OperandShadow);
3512 Value *OutShadowMask = IRB.CreateAndReduce(OperandUnsetOrPoison);
3514 Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3515 Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3525 IRBuilder<> IRB(&I);
3527 Value *OperandSetOrPoison = IRB.CreateOr(I.getOperand(0), OperandShadow);
3529 Value *OutShadowMask = IRB.CreateAndReduce(OperandSetOrPoison);
3531 Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3532 Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3539 IRBuilder<> IRB(&I);
3541 Type *Ty = IRB.getInt32Ty();
3543 getShadowOriginPtr(Addr, IRB, Ty, Align(1), /*isStore*/ true).first;
3545 IRB.CreateStore(getCleanShadow(Ty), ShadowPtr);
3555 IRBuilder<> IRB(&I);
3557 Type *Ty = IRB.getInt32Ty();
3561 getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
3566 Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
3567 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
3573 IRBuilder<> IRB(&I);
3593 getShadowOriginPtr(Ptr, IRB, ElementShadowTy, Align, /*isStore*/ false);
3596 IRB.CreateMaskedExpandLoad(ShadowTy, ShadowPtr, Align, Mask,
3606 IRBuilder<> IRB(&I);
3621 getShadowOriginPtr(Ptr, IRB, ElementShadowTy, Align, /*isStore*/ true);
3623 IRB.CreateMaskedCompressStore(Shadow, ShadowPtr, Align, Mask);
3629 IRBuilder<> IRB(&I);
3639 Value *MaskedPtrShadow = IRB.CreateSelect(
3654 Ptrs, IRB, ElementShadowTy, Alignment, /*isStore*/ false);
3657 IRB.CreateMaskedGather(ShadowTy, ShadowPtrs, Alignment, Mask,
3667 IRBuilder<> IRB(&I);
3677 Value *MaskedPtrShadow = IRB.CreateSelect(
3687 Ptrs, IRB, ElementShadowTy, Alignment, /*isStore*/ true);
3689 IRB.CreateMaskedScatter(Shadow, ShadowPtrs, Alignment, Mask);
3699 IRBuilder<> IRB(&I);
3715 Ptr, IRB, Shadow->getType(), Alignment, /*isStore*/ true);
3717 IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment, Mask);
3723 paintOrigin(IRB, getOrigin(V), OriginPtr,
3733 IRBuilder<> IRB(&I);
3754 getShadowOriginPtr(Ptr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3755 setShadow(&I, IRB.CreateMaskedLoad(ShadowTy, ShadowPtr, Alignment, Mask,
3762 Value *MaskedPassThruShadow = IRB.CreateAnd(
3763 getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
3765 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB, "_mscmp");
3767 Value *PtrOrigin = IRB.CreateLoad(MS.OriginTy, OriginPtr);
3768 Value *Origin = IRB.CreateSelect(NotNull, getOrigin(PassThru), PtrOrigin);
3787 IRBuilder<> IRB(&I);
3810 Dst, IRB, SrcShadow->getType(), Alignment, /*isStore*/ true);
3819 ShadowArgs.append(1, IRB.CreateBitCast(SrcShadow, Src->getType()));
3822 IRB.CreateIntrinsic(IRB.getVoidTy(), I.getIntrinsicID(), ShadowArgs);
3830 paintOrigin(IRB, getOrigin(Src), DstOriginPtr,
3851 IRBuilder<> IRB(&I);
3868 getShadowOriginPtr(Src, IRB, SrcShadowTy, Alignment, /*isStore*/ false);
3875 IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(), ShadowArgs);
3880 setShadow(&I, IRB.CreateBitCast(CI, getShadowTy(&I)));
3888 Value *PtrSrcOrigin = IRB.CreateLoad(MS.OriginTy, SrcOriginPtr);
3898 IRBuilder<> IRB(&I);
3903 SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)),
3906 Value *S = IRB.CreateCall(I.getCalledFunction(),
3908 S = IRB.CreateOr(SMask, S);
3930 IRBuilder<> IRB(&I);
3936 Value *Shuf0 = IRB.CreateShuffleVector(getShadow(&I, 0),
3938 Value *Shuf1 = IRB.CreateShuffleVector(getShadow(&I, 1),
3940 ShadowAndOriginCombiner SOC(this, IRB);
3948 IRBuilder<> IRB(&I);
3958 Value *Shadow = IRB.CreateShuffleVector(First, Second, Mask);
3965 IRBuilder<> IRB(&I);
3968 Value *Or = IRB.CreateOr(Shadow0, Shadow1);
3969 Value *NZ = IRB.CreateICmpNE(Or, Constant::getNullValue(Or->getType()));
3970 Value *Scalar = convertShadowToScalar(NZ, IRB);
3971 Value *Shadow = IRB.CreateZExt(Scalar, getShadowTy(&I));
3978 IRBuilder<> IRB(&I);
3983 Value *OrShadow = IRB.CreateOr(First, Second);
3989 Value *Shadow = IRB.CreateShuffleVector(First, OrShadow, Mask);
4003 IRBuilder<> IRB(&I);
4004 ShadowAndOriginCombiner SC(this, IRB);
4017 IRBuilder<> IRB(&I);
4023 IRBuilder<> IRB(&I);
4025 setShadow(&I, IRB.CreateICmpNE(Shadow, getCleanShadow(Shadow)));
4030 IRBuilder<> IRB(&I);
4033 Value *ShadowElt0 = IRB.CreateOr(Shadow0, Shadow1);
4035 IRB.CreateICmpNE(ShadowElt0, getCleanShadow(ShadowElt0));
4038 Shadow = IRB.CreateInsertValue(Shadow, ShadowElt0, 0);
4039 Shadow = IRB.CreateInsertValue(Shadow, ShadowElt1, 1);
4141 IRBuilder<> IRB(&I);
4195 Addr, IRB, OutputShadowTy, Align(1), /*isStore*/ true);
4199 IRB.CreateIntrinsic(IRB.getVoidTy(), I.getIntrinsicID(), ShadowArgs);
4210 OriginCombiner OC(this, IRB);
4245 IRBuilder<> IRB(&I);
4257 IRB.CreateBitCast(Shadow, I.getArgOperand(i)->getType()));
4267 IRB.CreateIntrinsic(I.getType(), shadowIntrinsicID, ShadowArgs);
4274 CreateShadowCast(IRB, getShadow(&I, i), CombinedShadow->getType());
4275 CombinedShadow = IRB.CreateOr(Shadow, CombinedShadow, "_msprop");
4278 setShadow(&I, IRB.CreateBitCast(CombinedShadow, getShadowTy(&I)));
4784 IRBuilder<> IRB(&CB);
4792 IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
4815 IRBuilder<> IRB(&CB);
4822 IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
4826 getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1),
4831 IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size,
4888 IRBuilder<> IRB(&CB);
4927 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4944 getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment,
4947 Store = IRB.CreateMemSet(ArgShadowBase,
4948 Constant::getNullValue(IRB.getInt8Ty()),
4951 Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
4954 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4958 IRB.CreateMemCpy(
4971 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
4975 IRB.CreateStore(getOrigin(A),
4976 getOriginPtrForArgument(IRB, ArgOffset));
4990 VAHelper->visitCallBase(CB, IRB);
5051 IRBuilder<> IRB(&I);
5058 Value *ShadowPtr = getShadowPtrForRetval(IRB);
5076 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
5078 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval());
5083 IRBuilder<> IRB(&I);
5091 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
5095 &I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(), "_msphi_o"));
5110 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
5112 IRB.CreateCall(MS.MsanPoisonStackFn, {&I, Len});
5116 &I, IRB, IRB.getInt8Ty(), Align(1), /*isStore*/ true);
5118 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
5119 IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlign());
5126 IRB.CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
5129 IRB.CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr});
5134 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
5137 IRB.CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr});
5139 IRB.CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len});
5146 NextNodeIRBuilder IRB(InsPoint);
5149 Value *Len = IRB.CreateTypeSize(MS.IntptrTy, TS);
5151 Len = IRB.CreateMul(Len,
5152 IRB.CreateZExtOrTrunc(I.getArraySize(), MS.IntptrTy));
5155 poisonAllocaKmsan(I, IRB, Len);
5157 poisonAllocaUserspace(I, IRB, Len);
5178 IRBuilder<> IRB(&I);
5189 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
5203 C = CreateAppToShadowCast(IRB, C);
5204 D = CreateAppToShadowCast(IRB, D);
5207 Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd});
5209 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
5215 B = convertToBool(B, IRB);
5216 Sb = convertToBool(Sb, IRB);
5220 setOrigin(&I, IRB.CreateSelect(Sb, Ob, IRB.CreateSelect(B, Oc, Od)));
5244 IRBuilder<> IRB(&I);
5249 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
5256 IRBuilder<> IRB(&I);
5262 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
5293 IRBuilder<> &IRB, const DataLayout &DL,
5309 Value *SizeVal = IRB.CreateTypeSize(MS.IntptrTy, Size);
5311 IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Operand, SizeVal});
5318 getShadowOriginPtrUserspace(Operand, IRB, IRB.getInt8Ty(), Align(1));
5320 IRB.CreateAlignedStore(getCleanShadow(ElemTy), ShadowPtr, Align(1));
5322 IRB.CreateMemSet(ShadowPtr, ConstantInt::getNullValue(IRB.getInt8Ty()),
5371 IRBuilder<> IRB(&I);
5381 instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL,
5389 instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL,
5429 Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
5430 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
5431 return IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
5435 Value *getShadowPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
5436 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
5437 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
5438 return IRB.CreateIntToPtr(Base, MS.PtrTy, "_msarg_va_s");
5442 Value *getShadowPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset,
5447 return getShadowPtrForVAArgument(IRB, ArgOffset);
5451 Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, int ArgOffset) {
5452 Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
5456 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
5457 return IRB.CreateIntToPtr(Base, MS.PtrTy, "_msarg_va_o");
5460 void CleanUnusedTLS(IRBuilder<> &IRB, Value *ShadowBase,
5468 ConstantInt::getSigned(IRB.getInt32Ty(), kParamTLSSize - BaseOffset);
5469 IRB.CreateMemSet(ShadowBase, ConstantInt::getNullValue(IRB.getInt8Ty()),
5474 IRBuilder<> IRB(&I);
5478 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
5480 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
5550 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5570 Value *ShadowBase = getShadowPtrForVAArgument(IRB, OverflowOffset);
5573 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
5577 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
5583 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
5585 IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
5588 IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr,
5599 ShadowBase = getShadowPtrForVAArgument(IRB, GpOffset);
5601 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset);
5606 ShadowBase = getShadowPtrForVAArgument(IRB, FpOffset);
5608 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5618 ShadowBase = getShadowPtrForVAArgument(IRB, OverflowOffset);
5620 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
5625 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
5635 IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment);
5639 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5645 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
5646 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5655 IRBuilder<> IRB(MSV.FnPrologueEnd);
5657 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5658 Value *CopySize = IRB.CreateAdd(
5660 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5662 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
5665 Value *SrcSize = IRB.CreateBinaryIntrinsic(
5668 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
5671 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5673 IRB.CreateMemCpy(VAArgTLSOriginCopy, kShadowTLSAlignment,
5681 NextNodeIRBuilder IRB(OrigInst);
5684 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
5685 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5688 Value *RegSaveAreaPtr = IRB.CreateLoad(MS.PtrTy, RegSaveAreaPtrPtr);
5692 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
5694 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5697 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5699 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
5700 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5704 IRB.CreateLoad(MS.PtrTy, OverflowArgAreaPtrPtr);
5707 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
5709 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
5711 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5714 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
5716 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5777 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5795 Base = getShadowPtrForVAArgument(IRB, GrOffset);
5799 Base = getShadowPtrForVAArgument(IRB, VrOffset);
5810 Base = getShadowPtrForVAArgument(IRB, BaseOffset);
5814 CleanUnusedTLS(IRB, Base, BaseOffset);
5823 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
5826 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
5827 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5831 Value *getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
5832 Value *SaveAreaPtrPtr = IRB.CreateIntToPtr(
5833 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5836 return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
5840 Value *getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
5841 Value *SaveAreaPtr = IRB.CreateIntToPtr(
5842 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5845 Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
5846 return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
5855 IRBuilder<> IRB(MSV.FnPrologueEnd);
5857 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5858 Value *CopySize = IRB.CreateAdd(
5860 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5862 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
5865 Value *SrcSize = IRB.CreateBinaryIntrinsic(
5868 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
5878 NextNodeIRBuilder IRB(OrigInst);
5894 Type *RegSaveAreaPtrTy = IRB.getPtrTy();
5898 IRB.CreateIntToPtr(getVAField64(IRB, VAListTag, 0), RegSaveAreaPtrTy);
5901 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
5902 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
5904 Value *GrRegSaveAreaPtr = IRB.CreateIntToPtr(
5905 IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea), RegSaveAreaPtrTy);
5908 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
5909 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
5911 Value *VrRegSaveAreaPtr = IRB.CreateIntToPtr(
5912 IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea), RegSaveAreaPtrTy);
5919 IRB.CreateAdd(GrArgSize, GrOffSaveArea);
5922 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
5927 IRB.CreateInBoundsPtrAdd(VAArgTLSCopy, GrRegSaveAreaShadowPtrOff);
5928 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
5930 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8),
5935 IRB.CreateAdd(VrArgSize, VrOffSaveArea);
5938 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
5942 Value *VrSrcPtr = IRB.CreateInBoundsPtrAdd(
5943 IRB.CreateInBoundsPtrAdd(VAArgTLSCopy,
5944 IRB.getInt32(AArch64VrBegOffset)),
5946 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
5948 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8),
5953 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
5957 Value *StackSrcPtr = IRB.CreateInBoundsPtrAdd(
5958 VAArgTLSCopy, IRB.getInt32(AArch64VAEndOffset));
5960 IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr,
5975 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
6011 getShadowPtrForVAArgument(IRB, VAArgOffset - VAArgBase, ArgSize);
6015 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
6018 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
6048 getShadowPtrForVAArgument(IRB, VAArgOffset - VAArgBase, ArgSize);
6050 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
6063 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
6069 IRBuilder<> IRB(MSV.FnPrologueEnd);
6070 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
6077 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
6079 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
6082 Value *SrcSize = IRB.CreateBinaryIntrinsic(
6084 ConstantInt::get(IRB.getInt64Ty(), kParamTLSSize));
6085 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
6093 NextNodeIRBuilder IRB(OrigInst);
6095 Value *RegSaveAreaPtrPtr = IRB.CreatePtrToInt(VAListTag, MS.IntptrTy);
6100 IRB.CreateAdd(RegSaveAreaPtrPtr, ConstantInt::get(MS.IntptrTy, 8));
6102 RegSaveAreaPtrPtr = IRB.CreateIntToPtr(RegSaveAreaPtrPtr, MS.PtrTy);
6104 Value *RegSaveAreaPtr = IRB.CreateLoad(MS.PtrTy, RegSaveAreaPtrPtr);
6110 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
6112 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
6188 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
6226 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
6228 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
6245 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
6247 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
6274 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
6277 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
6292 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.getInt64Ty(),
6294 ShadowBase = IRB.CreateIntToPtr(ShadowBase, MS.PtrTy, "_msarg_va_s");
6295 IRB.CreateStore(Shadow, ShadowBase);
6299 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
6304 IRB.getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
6305 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
6308 void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) {
6309 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
6310 IRB.CreateAdd(
6311 IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
6314 Value *RegSaveAreaPtr = IRB.CreateLoad(MS.PtrTy, RegSaveAreaPtrPtr);
6318 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment,
6325 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
6328 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
6334 void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
6335 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
6336 IRB.CreateAdd(
6337 IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
6340 Value *OverflowArgAreaPtr = IRB.CreateLoad(MS.PtrTy, OverflowArgAreaPtrPtr);
6344 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
6346 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
6348 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
6351 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
6353 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
6364 IRBuilder<> IRB(MSV.FnPrologueEnd);
6366 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
6368 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
6370 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
6372 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
6375 Value *SrcSize = IRB.CreateBinaryIntrinsic(
6378 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
6381 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
6383 IRB.CreateMemCpy(VAArgTLSOriginCopy, kShadowTLSAlignment,
6391 NextNodeIRBuilder IRB(OrigInst);
6393 copyRegSaveArea(IRB, VAListTag);
6394 copyOverflowArea(IRB, VAListTag);
6408 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
6424 Value *Base = getShadowPtrForVAArgument(IRB, VAArgOffset, ArgSize);
6428 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
6431 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
6448 Base = getShadowPtrForVAArgument(IRB, VAArgOffset, ArgSize);
6450 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
6460 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
6466 IRBuilder<> IRB(MSV.FnPrologueEnd);
6467 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
6473 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
6475 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
6478 Value *SrcSize = IRB.CreateBinaryIntrinsic(
6480 ConstantInt::get(IRB.getInt64Ty(), kParamTLSSize));
6481 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
6488 NextNodeIRBuilder IRB(OrigInst);
6492 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
6495 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
6501 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
6503 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
6519 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
6534 Value *Base = getShadowPtrForVAArgument(IRB, VAArgOffset, ArgSize);
6539 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
6545 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
6551 IRBuilder<> IRB(MSV.FnPrologueEnd);
6552 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
6558 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
6560 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
6563 Value *SrcSize = IRB.CreateBinaryIntrinsic(
6565 ConstantInt::get(IRB.getInt64Ty(), kParamTLSSize));
6566 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
6573 NextNodeIRBuilder IRB(OrigInst);
6577 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
6580 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
6586 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
6588 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
6606 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {}