Lines Matching defs:PMV
693 raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
703 PrintObj(PMV.WordType);
705 PrintObj(PMV.ValueType);
707 PrintObj(PMV.AlignedAddr);
708 O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n';
710 PrintObj(PMV.ShiftAmt);
712 PrintObj(PMV.Mask);
714 PrintObj(PMV.Inv_Mask);
741 PartwordMaskValues PMV;
748 PMV.ValueType = PMV.IntValueType = ValueType;
749 if (PMV.ValueType->isFloatingPointTy() || PMV.ValueType->isVectorTy())
750 PMV.IntValueType =
753 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(Ctx, MinWordSize * 8)
755 if (PMV.ValueType == PMV.WordType) {
756 PMV.AlignedAddr = Addr;
757 PMV.AlignedAddrAlignment = AddrAlign;
758 PMV.ShiftAmt = ConstantInt::get(PMV.ValueType, 0);
759 PMV.Mask = ConstantInt::get(PMV.ValueType, ~0, /*isSigned*/ true);
760 return PMV;
763 PMV.AlignedAddrAlignment = Align(MinWordSize);
772 PMV.AlignedAddr = Builder.CreateIntrinsic(
781 PMV.AlignedAddr = Addr;
787 PMV.ShiftAmt = Builder.CreateShl(PtrLSB, 3);
790 PMV.ShiftAmt = Builder.CreateShl(
794 PMV.ShiftAmt = Builder.CreateTrunc(PMV.ShiftAmt, PMV.WordType, "ShiftAmt");
795 PMV.Mask = Builder.CreateShl(
796 ConstantInt::get(PMV.WordType, (1 << (ValueSize * 8)) - 1), PMV.ShiftAmt,
799 PMV.Inv_Mask = Builder.CreateNot(PMV.Mask, "Inv_Mask");
801 return PMV;
805 const PartwordMaskValues &PMV) {
806 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
807 if (PMV.WordType == PMV.ValueType)
810 Value *Shift = Builder.CreateLShr(WideWord, PMV.ShiftAmt, "shifted");
811 Value *Trunc = Builder.CreateTrunc(Shift, PMV.IntValueType, "extracted");
812 return Builder.CreateBitCast(Trunc, PMV.ValueType);
816 Value *Updated, const PartwordMaskValues &PMV) {
817 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
818 assert(Updated->getType() == PMV.ValueType && "Value type mismatch");
819 if (PMV.WordType == PMV.ValueType)
822 Updated = Builder.CreateBitCast(Updated, PMV.IntValueType);
824 Value *ZExt = Builder.CreateZExt(Updated, PMV.WordType, "extended");
826 Builder.CreateShl(ZExt, PMV.ShiftAmt, "shifted", /*HasNUW*/ true);
827 Value *And = Builder.CreateAnd(WideWord, PMV.Inv_Mask, "unmasked");
838 const PartwordMaskValues &PMV) {
841 // to merge bits from two values without requiring PMV.Inv_Mask.
844 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
857 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask);
858 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
875 Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV);
877 Value *FinalVal = insertMaskedValue(Builder, Loaded, NewVal, PMV);
906 PartwordMaskValues PMV =
913 Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType);
915 Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
921 AI->getValOperand(), PMV);
926 OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr,
927 PMV.AlignedAddrAlignment, MemOpOrder, SSID,
931 OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr,
932 PMV.AlignedAddrAlignment, MemOpOrder,
936 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
980 PartwordMaskValues PMV =
985 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
986 PMV.ShiftAmt, "ValOperand_Shifted");
992 Builder.CreateOr(ValOperand_Shifted, PMV.Inv_Mask, "AndOperand");
997 Op, PMV.AlignedAddr, NewOperand, PMV.AlignedAddrAlignment,
1002 Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV);
1016 // [[Setup mask values PMV.*]]
1017 // %NewVal_Shifted = shl i32 %NewVal, %PMV.ShiftAmt
1018 // %Cmp_Shifted = shl i32 %Cmp, %PMV.ShiftAmt
1020 // %InitLoaded_MaskOut = and i32 %InitLoaded, %PMV.Inv_Mask
1027 // %NewCI = cmpxchg i32* %PMV.AlignedAddr, i32 %FullWord_Cmp,
1034 // %OldVal_MaskOut = and i32 %OldVal, %PMV.Inv_Mask
1039 // %tmp1 = lshr i32 %OldVal, %PMV.ShiftAmt
1064 PartwordMaskValues PMV =
1070 Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt);
1072 Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt);
1076 LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr);
1078 Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask);
1083 PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2);
1090 PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, PMV.AlignedAddrAlignment,
1113 Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask);
1123 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV);
1148 PartwordMaskValues PMV =
1161 Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType),
1162 PMV.ShiftAmt, "ValOperand_Shifted");
1164 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt,
1166 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
1175 PartwordMaskValues PMV = createMaskInstrs(
1180 Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt,
1183 Builder.CreateZExt(CI->getNewValOperand(), PMV.WordType), PMV.ShiftAmt,
1186 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask,
1188 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV);
1192 CmpVal_Shifted, Builder.CreateAnd(OldVal, PMV.Mask), "Success");
1392 PartwordMaskValues PMV =
1400 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder);
1402 extractMaskedValue(Builder, UnreleasedLoad, PMV);
1417 Builder.CreatePHI(PMV.WordType, 2, "loaded.trystore");
1420 insertMaskedValue(Builder, LoadedTryStore, CI->getNewValOperand(), PMV);
1422 PMV.AlignedAddr, MemOpOrder);
1433 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder);
1434 Value *SecondLoadExtract = extractMaskedValue(Builder, SecondLoad, PMV);
1495 Value *Loaded = extractMaskedValue(Builder, LoadedFull, PMV);