Lines Matching defs:IRB
779 IRBuilder<> IRB(*C);
792 IRB.getVoidTy(), IRB.getInt32Ty());
797 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
798 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
799 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
800 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
801 IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
806 MsanMetadata = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
807 PointerType::get(IRB.getInt32Ty(), 0));
815 M, name_load, PointerType::get(IRB.getInt8Ty(), 0));
817 M, name_store, PointerType::get(IRB.getInt8Ty(), 0));
821 M, "__msan_metadata_ptr_for_load_n", PointerType::get(IRB.getInt8Ty(), 0),
822 IRB.getInt64Ty());
825 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
829 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
831 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy);
844 IRBuilder<> IRB(*C);
854 IRB.getVoidTy(), IRB.getInt32Ty());
858 WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
864 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
870 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
878 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
885 getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
893 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
898 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy,
899 IRB.getInt32Ty());
904 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy);
907 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
909 IRB.getVoidTy(), PtrTy, IntptrTy);
918 IRBuilder<> IRB(*C);
923 TLI.getAttrList(C, {0}, /*Signed=*/false, /*Ret=*/true), IRB.getInt32Ty(),
924 IRB.getInt32Ty());
927 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty());
934 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
937 M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
938 PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
1045 IRBuilder<> IRB(*C);
1046 IntptrTy = IRB.getIntPtrTy(DL);
1047 OriginTy = IRB.getInt32Ty();
1048 PtrTy = IRB.getPtrTy();
1055 M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
1057 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1058 IRB.getInt32(TrackOrigins), "__msan_track_origins");
1062 M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
1063 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1065 IRB.getInt32(Recover), "__msan_keep_going");
1084 virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
1187 IRBuilder<> IRB(FnPrologueEnd);
1188 insertKmsanPrologue(IRB);
1215 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1218 return IRB.CreateCall(MS.MsanChainOriginFn, V);
1221 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1227 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
1228 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
1232 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1243 Value *Size = IRB.CreateTypeSize(MS.IntptrTy, TS);
1245 IRB.CreateAdd(Size, ConstantInt::get(MS.IntptrTy, kOriginSize - 1));
1247 IRB.CreateUDiv(RoundUp, ConstantInt::get(MS.IntptrTy, kOriginSize));
1249 SplitBlockAndInsertSimpleForLoop(End, &*IRB.GetInsertPoint());
1250 IRB.SetInsertPoint(InsertPt);
1252 Value *GEP = IRB.CreateGEP(MS.OriginTy, OriginPtr, Index);
1253 IRB.CreateAlignedStore(Origin, GEP, kMinOriginAlignment);
1262 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1264 IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
1266 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
1268 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
1276 i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
1277 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
1282 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1288 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1297 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1310 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1311 CallBase *CB = IRB.CreateCall(Fn, {ConvertedShadow2, Addr, Origin});
1315 Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1317 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
1326 IRBuilder<> IRB(SI);
1335 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1337 StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
1345 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
1362 /// Helper function to insert a warning at IRB's current insert point.
1363 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1365 Origin = (Value *)IRB.getInt32(0);
1368 if (shouldDisambiguateWarningLocation(IRB.getCurrentDebugLocation())) {
1376 if (NewDebugLoc && NewDebugLoc != IRB.getCurrentDebugLocation()) {
1379 IRBuilder<> IRBOrigin(&*IRB.GetInsertPoint());
1387 IRB.CreateCall(MS.WarningFn, Origin)->setCannotMerge();
1389 IRB.CreateCall(MS.WarningFn)->setCannotMerge();
1395 void materializeOneCheck(IRBuilder<> &IRB, Value *ConvertedShadow,
1404 ConvertedShadow = convertShadowToScalar(ConvertedShadow, IRB);
1406 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1407 CallBase *CB = IRB.CreateCall(
1409 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)});
1413 Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1415 Cmp, &*IRB.GetInsertPoint(),
1418 IRB.SetInsertPoint(CheckTerm);
1419 insertWarningFn(IRB, Origin);
1434 IRBuilder<> IRB(Instruction);
1445 insertWarningFn(IRB, ShadowData.Origin);
1455 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1464 Shadow = convertToBool(Shadow, IRB, "_mscmp");
1465 ConvertedShadow = convertToBool(ConvertedShadow, IRB, "_mscmp");
1466 Shadow = IRB.CreateOr(Shadow, ConvertedShadow, "_msor");
1471 IRBuilder<> IRB(Instruction);
1472 materializeOneCheck(IRB, Shadow, nullptr);
1501 void insertKmsanPrologue(IRBuilder<> &IRB) {
1502 Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
1503 Constant *Zero = IRB.getInt32(0);
1504 MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1505 {Zero, IRB.getInt32(0)}, "param_shadow");
1506 MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1507 {Zero, IRB.getInt32(1)}, "retval_shadow");
1508 MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1509 {Zero, IRB.getInt32(2)}, "va_arg_shadow");
1510 MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1511 {Zero, IRB.getInt32(3)}, "va_arg_origin");
1513 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1514 {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
1515 MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1516 {Zero, IRB.getInt32(5)}, "param_origin");
1518 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1519 {Zero, IRB.getInt32(6)}, "retval_origin");
1521 MS.MsanMetadataAlloca = IRB.CreateAlloca(MS.MsanMetadata, 0u);
1610 IRBuilder<> &IRB) {
1611 Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0);
1616 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1617 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1620 Aggregator = IRB.CreateOr(Aggregator, ShadowBool);
1630 IRBuilder<> &IRB) {
1632 return IRB.getIntN(/* width */ 1, /* value */ 0);
1634 Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
1635 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1638 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1639 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1640 Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
1648 Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1650 return collapseStructShadow(Struct, V, IRB);
1652 return collapseArrayShadow(Array, V, IRB);
1655 return convertShadowToScalar(IRB.CreateOrReduce(V), IRB);
1658 return IRB.CreateBitCast(V, IntegerType::get(*MS.C, BitWidth));
1664 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1667 return convertToBool(convertShadowToScalar(V, IRB), IRB, name);
1671 return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name);
1709 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1711 Value *OffsetLong = IRB.CreatePointerCast(Addr, IntptrTy);
1714 OffsetLong = IRB.CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1717 OffsetLong = IRB.CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1730 getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1739 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1743 IRB.CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1745 Value *ShadowPtr = IRB.CreateIntToPtr(
1754 IRB.CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1757 OriginLong = IRB.CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1759 OriginPtr = IRB.CreateIntToPtr(
1766 Value *createMetadataCall(IRBuilder<> &IRB, FunctionCallee Callee,
1769 IRB.CreateCall(Callee,
1771 return IRB.CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
1774 return IRB.CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1778 IRBuilder<> &IRB,
1787 IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
1789 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
1793 IRB,
1797 Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
1798 ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
1799 Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
1808 IRBuilder<> &IRB,
1814 return getShadowOriginPtrKernelNoVec(Addr, IRB, ShadowTy, isStore);
1820 FixedVectorType::get(IRB.getPtrTy(), NumElements));
1824 FixedVectorType::get(IRB.getPtrTy(), NumElements));
1827 IRB.CreateExtractElement(Addr, ConstantInt::get(IRB.getInt32Ty(), i));
1829 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy, isStore);
1831 ShadowPtrs = IRB.CreateInsertElement(
1832 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.getInt32Ty(), i));
1834 OriginPtrs = IRB.CreateInsertElement(
1835 OriginPtrs, OriginPtr, ConstantInt::get(IRB.getInt32Ty(), i));
1840 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1845 return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1846 return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1852 Value *getShadowPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1853 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1855 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1856 return IRB.CreateIntToPtr(Base, IRB.getPtrTy(0), "_msarg");
1860 Value *getOriginPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1863 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1865 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1866 return IRB.CreateIntToPtr(Base, IRB.getPtrTy(0), "_msarg_o");
1870 Value *getShadowPtrForRetval(IRBuilder<> &IRB) {
1871 return IRB.CreatePointerCast(MS.RetvalTLS, IRB.getPtrTy(0), "_msret");
2147 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
2161 return ConstantDataVector::get(IRB.getContext(), OrderingTable);
2181 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
2195 return ConstantDataVector::get(IRB.getContext(), OrderingTable);
2224 NextNodeIRBuilder IRB(&I);
2231 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2233 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
2248 &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment));
2268 IRBuilder<> IRB(&I);
2271 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, getShadowTy(Val), Align(1),
2284 IRB.CreateStore(getCleanShadow(Val), ShadowPtr);
2303 IRBuilder<> IRB(&I);
2304 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
2311 IRBuilder<> IRB(&I);
2314 setShadow(&I, IRB.CreateInsertElement(Shadow0, Shadow1, I.getOperand(2),
2320 IRBuilder<> IRB(&I);
2323 setShadow(&I, IRB.CreateShuffleVector(Shadow0, Shadow1, I.getShuffleMask(),
2330 IRBuilder<> IRB(&I);
2331 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
2336 IRBuilder<> IRB(&I);
2337 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
2342 IRBuilder<> IRB(&I);
2343 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
2354 IRBuilder<> IRB(&I);
2355 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
2360 IRBuilder<> IRB(&I);
2361 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2367 IRBuilder<> IRB(&I);
2368 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2386 IRBuilder<> IRB(&I);
2397 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2398 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2400 Value *S1S2 = IRB.CreateAnd(S1, S2);
2401 Value *V1S2 = IRB.CreateAnd(V1, S2);
2402 Value *S1V2 = IRB.CreateAnd(S1, V2);
2403 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2408 IRBuilder<> IRB(&I);
2416 Value *V1 = IRB.CreateNot(I.getOperand(0));
2417 Value *V2 = IRB.CreateNot(I.getOperand(1));
2419 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2420 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2422 Value *S1S2 = IRB.CreateAnd(S1, S2);
2423 Value *V1S2 = IRB.CreateAnd(V1, S2);
2424 Value *S1V2 = IRB.CreateAnd(S1, V2);
2425 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2447 IRBuilder<> &IRB;
2451 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2452 : IRB(IRB), MSV(MSV) {}
2461 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2462 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
2474 Value *Cond = MSV->convertToBool(OpShadow, IRB);
2475 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2494 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
2508 MSV->paintOrigin(IRB, Origin, OriginPtr, TS, kMinOriginAlignment);
2520 IRBuilder<> IRB(&I);
2521 OriginCombiner OC(this, IRB);
2537 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2545 return IRB.CreateICmpNE(V, getCleanShadow(V));
2548 return IRB.CreateIntCast(V, dstTy, Signed);
2552 return IRB.CreateIntCast(V, dstTy, Signed);
2553 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
2555 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
2556 return IRB.CreateBitCast(V2, dstTy);
2561 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2566 return IRB.CreatePtrToInt(V, ShadowTy);
2568 return IRB.CreateBitCast(V, ShadowTy);
2573 IRBuilder<> IRB(&I);
2574 ShadowAndOriginCombiner SC(this, IRB);
2620 IRBuilder<> IRB(&I);
2622 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
2645 IRBuilder<> IRB(&I);
2667 IRBuilder<> IRB(&I);
2676 A = IRB.CreatePointerCast(A, Sa->getType());
2677 B = IRB.CreatePointerCast(B, Sb->getType());
2682 Value *C = IRB.CreateXor(A, B);
2683 Value *Sc = IRB.CreateOr(Sa, Sb);
2691 Value *LHS = IRB.CreateICmpNE(Sc, Zero);
2693 IRB.CreateICmpEQ(IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero);
2694 Value *Si = IRB.CreateAnd(LHS, RHS);
2702 Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2706 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2707 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2709 return IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)),
2713 return IRB.CreateAnd(A, IRB.CreateNot(Sa));
2719 Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2723 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2724 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2726 return IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)),
2730 return IRB.CreateOr(A, Sa);
2740 IRBuilder<> IRB(&I);
2749 A = IRB.CreatePointerCast(A, Sa->getType());
2750 B = IRB.CreatePointerCast(B, Sb->getType());
2756 Value *S1 = IRB.CreateICmp(I.getPredicate(),
2757 getLowestPossibleValue(IRB, A, Sa, IsSigned),
2758 getHighestPossibleValue(IRB, B, Sb, IsSigned));
2759 Value *S2 = IRB.CreateICmp(I.getPredicate(),
2760 getHighestPossibleValue(IRB, A, Sa, IsSigned),
2761 getLowestPossibleValue(IRB, B, Sb, IsSigned));
2762 Value *Si = IRB.CreateXor(S1, S2);
2790 IRBuilder<> IRB(&I);
2791 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2832 IRBuilder<> IRB(&I);
2838 IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
2840 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2841 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2850 IRBuilder<> IRB(&I);
2857 IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
2861 Value *Shift = IRB.CreateCall(Intrin, {S0, S1, V2});
2862 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2878 IRBuilder<> IRB(&I);
2879 IRB.CreateCall(MS.MemmoveFn,
2881 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2901 IRBuilder<> IRB(&I);
2902 IRB.CreateCall(MS.MemcpyFn,
2904 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2910 IRBuilder<> IRB(&I);
2911 IRB.CreateCall(
2914 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2915 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2928 IRBuilder<> IRB(&I);
2936 Addr, IRB, Shadow->getType(), Align(1), /*isStore*/ true);
2937 IRB.CreateAlignedStore(Shadow, ShadowPtr, Align(1));
2944 IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2953 IRBuilder<> IRB(&I);
2963 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2965 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
2975 setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
3001 IRBuilder<> IRB(&I);
3002 ShadowAndOriginCombiner SC(this, IRB);
3061 IRBuilder<> IRB(&I);
3066 setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
3071 IRBuilder<> IRB(&I);
3075 Value *BoolShadow = IRB.CreateIsNotNull(getShadow(Src), "_mscz_bs");
3080 Value *BoolZeroPoison = IRB.CreateIsNull(Src, "_mscz_bzp");
3081 BoolShadow = IRB.CreateOr(BoolShadow, BoolZeroPoison, "_mscz_bs");
3085 IRB.CreateSExt(BoolShadow, getShadowTy(Src), "_mscz_os");
3108 IRBuilder<> IRB(&I);
3138 AggShadow = IRB.CreateExtractElement(
3139 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
3141 Value *MoreShadow = IRB.CreateExtractElement(
3142 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
3143 AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
3159 ResultShadow = IRB.CreateInsertElement(
3161 ConstantInt::get(IRB.getInt32Ty(), i));
3173 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3175 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
3177 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
3178 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
3183 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3184 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
3185 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
3186 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
3189 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
3192 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
3193 return IRB.CreateSExt(S2, T);
3205 IRBuilder<> IRB(&I);
3210 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3211 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
3214 Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(),
3215 {IRB.CreateBitCast(S1, V1->getType()), V2});
3216 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
3217 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
3271 IRBuilder<> IRB(&I);
3281 S1 = IRB.CreateBitCast(S1, T);
3282 S2 = IRB.CreateBitCast(S2, T);
3285 IRB.CreateSExt(IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
3287 IRB.CreateSExt(IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
3290 S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
3291 S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
3298 IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
3300 S = IRB.CreateBitCast(S, getShadowTy(&I));
3317 Value *findDppPoisonedOutput(IRBuilder<> &IRB, Value *S, unsigned SrcMask,
3322 S = IRB.CreateSelect(createDppMask(Width, SrcMask), S,
3324 Value *SElem = IRB.CreateOrReduce(S);
3325 Value *IsClean = IRB.CreateIsNull(SElem, "_msdpp");
3328 return IRB.CreateSelect(
3344 IRBuilder<> IRB(&I);
3348 Value *S = IRB.CreateOr(S0, S1);
3359 Value *SI1 = findDppPoisonedOutput(IRB, S, SrcMask, DstMask);
3363 SI1 = IRB.CreateOr(
3364 SI1, findDppPoisonedOutput(IRB, S, SrcMask << 4, DstMask << 4));
3368 S = IRB.CreateSExt(SI1, S->getType(), "_msdpp");
3374 Value *convertBlendvToSelectMask(IRBuilder<> &IRB, Value *C) {
3375 C = CreateAppToShadowCast(IRB, C);
3378 C = IRB.CreateAShr(C, ElSize - 1);
3379 FVT = FixedVectorType::get(IRB.getInt1Ty(), FVT->getNumElements());
3380 return IRB.CreateTrunc(C, FVT);
3393 IRBuilder<> IRB(&I);
3395 C = convertBlendvToSelectMask(IRB, C);
3396 Sc = convertBlendvToSelectMask(IRB, Sc);
3413 IRBuilder<> IRB(&I);
3416 Value *S = IRB.CreateOr(Shadow0, Shadow1);
3417 S = IRB.CreateBitCast(S, ResTy);
3418 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
3420 S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
3421 S = IRB.CreateBitCast(S, getShadowTy(&I));
3431 IRBuilder<> IRB(&I);
3434 Value *S = IRB.CreateOr(Shadow0, Shadow1);
3435 S = IRB.CreateBitCast(S, ResTy);
3436 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
3438 S = IRB.CreateBitCast(S, getShadowTy(&I));
3447 IRBuilder<> IRB(&I);
3451 Value *S0 = IRB.CreateOr(Shadow0, Shadow1);
3452 Value *S = IRB.CreateSExt(
3453 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
3462 IRBuilder<> IRB(&I);
3465 Value *S0 = IRB.CreateOr(Shadow0, Shadow1);
3466 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
3474 IRBuilder<> IRB(&I);
3475 Value *S = IRB.CreateOrReduce(getShadow(&I, 0));
3484 IRBuilder<> IRB(&I);
3486 Value *OperandUnsetBits = IRB.CreateNot(I.getOperand(0));
3487 Value *OperandUnsetOrPoison = IRB.CreateOr(OperandUnsetBits, OperandShadow);
3489 Value *OutShadowMask = IRB.CreateAndReduce(OperandUnsetOrPoison);
3491 Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3492 Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3502 IRBuilder<> IRB(&I);
3504 Value *OperandSetOrPoison = IRB.CreateOr(I.getOperand(0), OperandShadow);
3506 Value *OutShadowMask = IRB.CreateAndReduce(OperandSetOrPoison);
3508 Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3509 Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3516 IRBuilder<> IRB(&I);
3518 Type *Ty = IRB.getInt32Ty();
3520 getShadowOriginPtr(Addr, IRB, Ty, Align(1), /*isStore*/ true).first;
3522 IRB.CreateStore(getCleanShadow(Ty), ShadowPtr);
3532 IRBuilder<> IRB(&I);
3534 Type *Ty = IRB.getInt32Ty();
3538 getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
3543 Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
3544 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
3550 IRBuilder<> IRB(&I);
3569 getShadowOriginPtr(Ptr, IRB, ElementShadowTy, {}, /*isStore*/ false);
3571 Value *Shadow = IRB.CreateMaskedExpandLoad(
3581 IRBuilder<> IRB(&I);
3595 getShadowOriginPtr(Ptr, IRB, ElementShadowTy, {}, /*isStore*/ true);
3597 IRB.CreateMaskedCompressStore(Shadow, ShadowPtr, Mask);
3603 IRBuilder<> IRB(&I);
3613 Value *MaskedPtrShadow = IRB.CreateSelect(
3628 Ptrs, IRB, ElementShadowTy, Alignment, /*isStore*/ false);
3631 IRB.CreateMaskedGather(ShadowTy, ShadowPtrs, Alignment, Mask,
3641 IRBuilder<> IRB(&I);
3651 Value *MaskedPtrShadow = IRB.CreateSelect(
3661 Ptrs, IRB, ElementShadowTy, Alignment, /*isStore*/ true);
3663 IRB.CreateMaskedScatter(Shadow, ShadowPtrs, Alignment, Mask);
3669 IRBuilder<> IRB(&I);
3685 Ptr, IRB, Shadow->getType(), Alignment, /*isStore*/ true);
3687 IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment, Mask);
3693 paintOrigin(IRB, getOrigin(V), OriginPtr,
3699 IRBuilder<> IRB(&I);
3720 getShadowOriginPtr(Ptr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3721 setShadow(&I, IRB.CreateMaskedLoad(ShadowTy, ShadowPtr, Alignment, Mask,
3728 Value *MaskedPassThruShadow = IRB.CreateAnd(
3729 getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
3731 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB, "_mscmp");
3733 Value *PtrOrigin = IRB.CreateLoad(MS.OriginTy, OriginPtr);
3734 Value *Origin = IRB.CreateSelect(NotNull, getOrigin(PassThru), PtrOrigin);
3745 IRBuilder<> IRB(&I);
3750 SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)),
3753 Value *S = IRB.CreateCall(I.getCalledFunction(),
3755 S = IRB.CreateOr(SMask, S);
3777 IRBuilder<> IRB(&I);
3783 Value *Shuf0 = IRB.CreateShuffleVector(getShadow(&I, 0),
3785 Value *Shuf1 = IRB.CreateShuffleVector(getShadow(&I, 1),
3787 ShadowAndOriginCombiner SOC(this, IRB);
3795 IRBuilder<> IRB(&I);
3805 Value *Shadow = IRB.CreateShuffleVector(First, Second, Mask);
3812 IRBuilder<> IRB(&I);
3815 Value *Or = IRB.CreateOr(Shadow0, Shadow1);
3816 Value *NZ = IRB.CreateICmpNE(Or, Constant::getNullValue(Or->getType()));
3817 Value *Scalar = convertShadowToScalar(NZ, IRB);
3818 Value *Shadow = IRB.CreateZExt(Scalar, getShadowTy(&I));
3825 IRBuilder<> IRB(&I);
3830 Value *OrShadow = IRB.CreateOr(First, Second);
3836 Value *Shadow = IRB.CreateShuffleVector(First, OrShadow, Mask);
3850 IRBuilder<> IRB(&I);
3856 IRBuilder<> IRB(&I);
3858 setShadow(&I, IRB.CreateICmpNE(Shadow, getCleanShadow(Shadow)));
3863 IRBuilder<> IRB(&I);
3866 Value *ShadowElt0 = IRB.CreateOr(Shadow0, Shadow1);
3868 IRB.CreateICmpNE(ShadowElt0, getCleanShadow(ShadowElt0));
3871 Shadow = IRB.CreateInsertValue(Shadow, ShadowElt0, 0);
3872 Shadow = IRB.CreateInsertValue(Shadow, ShadowElt1, 1);
3884 IRBuilder<> IRB(&I);
3923 getShadowOriginPtr(Addr, IRB, ShadowTy, Align(1), /*isStore*/ true);
3932 OriginCombiner OC(this, IRB);
4298 IRBuilder<> IRB(&CB);
4306 IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
4329 IRBuilder<> IRB(&CB);
4336 IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
4340 getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1),
4345 IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size,
4402 IRBuilder<> IRB(&CB);
4441 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4458 getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment,
4461 Store = IRB.CreateMemSet(ArgShadowBase,
4462 Constant::getNullValue(IRB.getInt8Ty()),
4465 Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
4468 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4472 IRB.CreateMemCpy(
4485 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
4489 IRB.CreateStore(getOrigin(A),
4490 getOriginPtrForArgument(IRB, ArgOffset));
4504 VAHelper->visitCallBase(CB, IRB);
4566 IRBuilder<> IRB(&I);
4573 Value *ShadowPtr = getShadowPtrForRetval(IRB);
4591 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
4593 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval());
4598 IRBuilder<> IRB(&I);
4606 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
4610 &I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(), "_msphi_o"));
4625 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
4627 IRB.CreateCall(MS.MsanPoisonStackFn, {&I, Len});
4631 &I, IRB, IRB.getInt8Ty(), Align(1), /*isStore*/ true);
4633 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
4634 IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlign());
4641 IRB.CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
4644 IRB.CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr});
4649 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
4652 IRB.CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr});
4654 IRB.CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len});
4661 NextNodeIRBuilder IRB(InsPoint);
4664 Value *Len = IRB.CreateTypeSize(MS.IntptrTy, TS);
4666 Len = IRB.CreateMul(Len,
4667 IRB.CreateZExtOrTrunc(I.getArraySize(), MS.IntptrTy));
4670 poisonAllocaKmsan(I, IRB, Len);
4672 poisonAllocaUserspace(I, IRB, Len);
4693 IRBuilder<> IRB(&I);
4704 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
4718 C = CreateAppToShadowCast(IRB, C);
4719 D = CreateAppToShadowCast(IRB, D);
4722 Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd});
4724 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
4730 B = convertToBool(B, IRB);
4731 Sb = convertToBool(Sb, IRB);
4735 setOrigin(&I, IRB.CreateSelect(Sb, Ob, IRB.CreateSelect(B, Oc, Od)));
4759 IRBuilder<> IRB(&I);
4764 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
4771 IRBuilder<> IRB(&I);
4777 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
4808 IRBuilder<> &IRB, const DataLayout &DL,
4824 Value *SizeVal = IRB.CreateTypeSize(MS.IntptrTy, Size);
4826 IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Operand, SizeVal});
4833 getShadowOriginPtrUserspace(Operand, IRB, IRB.getInt8Ty(), Align(1));
4835 IRB.CreateAlignedStore(getCleanShadow(ElemTy), ShadowPtr, Align(1));
4837 IRB.CreateMemSet(ShadowPtr, ConstantInt::getNullValue(IRB.getInt8Ty()),
4886 IRBuilder<> IRB(&I);
4896 instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL,
4904 instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL,
4944 Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
4945 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4946 return IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4950 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4952 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4953 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4954 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4959 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4964 return getShadowPtrForVAArgument(Ty, IRB, ArgOffset);
4968 Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, int ArgOffset) {
4969 Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
4973 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4974 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
4978 void CleanUnusedTLS(IRBuilder<> &IRB, Value *ShadowBase,
4986 ConstantInt::getSigned(IRB.getInt32Ty(), kParamTLSSize - BaseOffset);
4987 IRB.CreateMemSet(ShadowBase, ConstantInt::getNullValue(IRB.getInt8Ty()),
4992 IRBuilder<> IRB(&I);
4996 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4998 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
5068 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5089 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
5092 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
5096 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
5102 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
5104 IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
5107 IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr,
5118 ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, GpOffset);
5120 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset);
5125 ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, FpOffset);
5127 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5138 getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset);
5140 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
5145 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
5155 IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment);
5159 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5165 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
5166 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5175 IRBuilder<> IRB(MSV.FnPrologueEnd);
5177 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5178 Value *CopySize = IRB.CreateAdd(
5180 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5182 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
5185 Value *SrcSize = IRB.CreateBinaryIntrinsic(
5188 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
5191 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5193 IRB.CreateMemCpy(VAArgTLSOriginCopy, kShadowTLSAlignment,
5201 NextNodeIRBuilder IRB(OrigInst);
5205 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
5206 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5210 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5214 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
5216 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5219 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5222 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
5223 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5227 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5230 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
5232 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
5234 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5237 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
5239 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5256 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5270 Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize);
5275 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
5278 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
5281 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5287 IRBuilder<> IRB(MSV.FnPrologueEnd);
5288 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5290 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5295 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5297 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
5300 Value *SrcSize = IRB.CreateBinaryIntrinsic(
5303 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
5310 NextNodeIRBuilder IRB(OrigInst);
5314 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5317 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5321 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
5323 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5383 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5401 Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset);
5405 Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset);
5416 Base = getShadowPtrForVAArgument(A->getType(), IRB, BaseOffset);
5420 CleanUnusedTLS(IRB, Base, BaseOffset);
5429 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
5432 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
5433 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5437 Value *getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
5438 Value *SaveAreaPtrPtr = IRB.CreateIntToPtr(
5439 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5442 return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
5446 Value *getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
5447 Value *SaveAreaPtr = IRB.CreateIntToPtr(
5448 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5451 Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
5452 return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
5461 IRBuilder<> IRB(MSV.FnPrologueEnd);
5463 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5464 Value *CopySize = IRB.CreateAdd(
5466 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5468 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
5471 Value *SrcSize = IRB.CreateBinaryIntrinsic(
5474 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
5484 NextNodeIRBuilder IRB(OrigInst);
5500 Type *RegSaveAreaPtrTy = IRB.getPtrTy();
5504 IRB.CreateIntToPtr(getVAField64(IRB, VAListTag, 0), RegSaveAreaPtrTy);
5507 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
5508 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
5510 Value *GrRegSaveAreaPtr = IRB.CreateIntToPtr(
5511 IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea), RegSaveAreaPtrTy);
5514 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
5515 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
5517 Value *VrRegSaveAreaPtr = IRB.CreateIntToPtr(
5518 IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea), RegSaveAreaPtrTy);
5525 IRB.CreateAdd(GrArgSize, GrOffSaveArea);
5528 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
5533 IRB.CreateInBoundsPtrAdd(VAArgTLSCopy, GrRegSaveAreaShadowPtrOff);
5534 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
5536 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8),
5541 IRB.CreateAdd(VrArgSize, VrOffSaveArea);
5544 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
5548 Value *VrSrcPtr = IRB.CreateInBoundsPtrAdd(
5549 IRB.CreateInBoundsPtrAdd(VAArgTLSCopy,
5550 IRB.getInt32(AArch64VrBegOffset)),
5552 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
5554 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8),
5559 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
5563 Value *StackSrcPtr = IRB.CreateInBoundsPtrAdd(
5564 VAArgTLSCopy, IRB.getInt32(AArch64VAEndOffset));
5566 IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr,
5581 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5612 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
5616 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
5619 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
5648 Base = getShadowPtrForVAArgument(A->getType(), IRB,
5651 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
5661 ConstantInt::get(IRB.getInt64Ty(), VAArgOffset - VAArgBase);
5664 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5670 IRBuilder<> IRB(MSV.FnPrologueEnd);
5671 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5673 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5679 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5681 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
5684 Value *SrcSize = IRB.CreateBinaryIntrinsic(
5687 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
5694 NextNodeIRBuilder IRB(OrigInst);
5698 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5701 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5705 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
5707 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5783 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5821 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5823 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5840 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5842 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5869 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5872 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5887 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.getInt64Ty(),
5889 ShadowBase = IRB.CreateIntToPtr(
5891 IRB.CreateStore(Shadow, ShadowBase);
5895 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5900 IRB.getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5901 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5904 void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) {
5906 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
5907 IRB.CreateAdd(
5908 IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5911 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5915 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment,
5922 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5925 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5931 void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
5933 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
5934 IRB.CreateAdd(
5935 IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5939 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5943 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
5945 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
5947 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5950 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
5952 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5963 IRBuilder<> IRB(MSV.FnPrologueEnd);
5965 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5967 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5969 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5971 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()),
5974 Value *SrcSize = IRB.CreateBinaryIntrinsic(
5977 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS,
5980 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5982 IRB.CreateMemCpy(VAArgTLSOriginCopy, kShadowTLSAlignment,
5990 NextNodeIRBuilder IRB(OrigInst);
5992 copyRegSaveArea(IRB, VAListTag);
5993 copyOverflowArea(IRB, VAListTag);
6007 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {}