Lines Matching full:offsets
86 // Decompose a ptr into Base and Offsets, potentially using a GEP to return a
87 // scalar base and vector offsets, or else fallback to using a base of 0 and
89 Value *decomposePtr(Value *Ptr, Value *&Offsets, int &Scale,
92 // Check for a getelementptr and deduce base and offsets from it, on success
93 // returning the base directly and the offsets indirectly using the Offsets
95 Value *decomposeGEP(Value *&Offsets, FixedVectorType *Ty,
108 // Create a gather from a base + vector of offsets
122 // Create a scatter to a base + vector of offsets
123 Instruction *tryCreateMaskedScatterOffset(IntrinsicInst *I, Value *Offsets,
134 // QI gathers and scatters can increment their offsets on their own if
138 // QI gathers/scatters can increment their offsets on their own if the
145 // Optimise the base and offsets of the given address
148 Value *foldGEP(GetElementPtrInst *GEP, Value *&Offsets, unsigned &Scale,
150 // Check whether these offsets could be moved out of the loop they're in
151 bool optimiseOffsets(Value *Offsets, BasicBlock *BB, LoopInfo *LI);
185 static bool checkOffsetSize(Value *Offsets, unsigned TargetElemCount) {
186 // Offsets that are not of type <N x i32> are sign extended by the
189 // positive offsets - i.e., the offsets are not allowed to be variables we
191 // Additionally, <N x i32> offsets have to either originate from a zext of a
197 unsigned OffsetElemSize = cast<FixedVectorType>(Offsets->getType())
201 Constant *ConstOff = dyn_cast<Constant>(Offsets);
227 Value *MVEGatherScatterLowering::decomposePtr(Value *Ptr, Value *&Offsets,
232 if (Value *V = decomposeGEP(Offsets, Ty, GEP, Builder)) {
241 // BasePtr of 0 with Ptr as the Offsets, so long as there are only 4
248 Offsets = Builder.CreatePtrToInt(
254 Value *MVEGatherScatterLowering::decomposeGEP(Value *&Offsets,
264 << " Looking at intrinsic for base + vector of offsets\n");
266 Offsets = GEP->getOperand(1);
268 !isa<FixedVectorType>(Offsets->getType()))
276 Offsets = GEP->getOperand(1);
278 cast<FixedVectorType>(Offsets->getType())->getNumElements();
282 ZExtInst *ZextOffs = dyn_cast<ZExtInst>(Offsets);
284 Offsets = ZextOffs->getOperand(0);
285 FixedVectorType *OffsetType = cast<FixedVectorType>(Offsets->getType());
287 // If the offsets are already being zext-ed to <N x i32>, that relieves us of
292 if (!checkOffsetSize(Offsets, OffsetsElemCount))
297 if (Ty != Offsets->getType()) {
300 Offsets = Builder.CreateTrunc(Offsets, Ty);
302 Offsets = Builder.CreateZExt(Offsets, VectorType::getInteger(Ty));
306 LLVM_DEBUG(dbgs() << "masked gathers/scatters: found correct offsets\n");
548 Value *Offsets;
551 Ptr, Offsets, Scale, cast<FixedVectorType>(ResultTy), MemoryTy, Builder);
561 {ResultTy, BasePtr->getType(), Offsets->getType(), Mask->getType()},
562 {BasePtr, Offsets, Builder.getInt32(MemoryTy->getScalarSizeInBits()),
567 {ResultTy, BasePtr->getType(), Offsets->getType()},
568 {BasePtr, Offsets, Builder.getInt32(MemoryTy->getScalarSizeInBits()),
671 << " to base + vector of offsets\n");
701 Value *Offsets;
704 Ptr, Offsets, Scale, cast<FixedVectorType>(InputTy), MemoryTy, Builder);
713 {BasePtr->getType(), Offsets->getType(), Input->getType(),
715 {BasePtr, Offsets, Input,
721 {BasePtr->getType(), Offsets->getType(), Input->getType()},
722 {BasePtr, Offsets, Input,
743 // Decompose the GEP into Base and Offsets
745 Value *Offsets;
746 Value *BasePtr = decomposeGEP(Offsets, Ty, GEP, Builder);
753 // The gep was in charge of making sure the offsets are scaled correctly
766 if (auto *Load = tryCreateIncrementingWBGatScat(I, BasePtr, Offsets,
774 std::pair<Value *, int64_t> Add = getVarAndConst(Offsets, TypeScale);
780 // Make sure the offsets are scaled correctly
786 // Add the base to the offsets
803 IntrinsicInst *I, Value *BasePtr, Value *Offsets, unsigned TypeScale,
808 // Offsets that are worth merging into this instruction will be incremented
810 PHINode *Phi = dyn_cast<PHINode>(Offsets);
822 Offsets = Phi->getIncomingValue(IncrementIndex);
824 std::pair<Value *, int64_t> Add = getVarAndConst(Offsets, TypeScale);
838 // Make sure the offsets are scaled correctly
844 // Add the base to the offsets
880 Instruction *AddInst = cast<Instruction>(Offsets);
945 // Check whether all usages of this instruction are as offsets of
972 bool MVEGatherScatterLowering::optimiseOffsets(Value *Offsets, BasicBlock *BB,
975 << *Offsets << "\n");
978 if (!isa<Instruction>(Offsets))
980 Instruction *Offs = cast<Instruction>(Offsets);
1148 LLVM_DEBUG(dbgs() << "masked gathers/scatters: incompatible gep offsets\n");
1189 Value *&Offsets, unsigned &Scale,
1192 Offsets = GEP->getOperand(1);
1194 // We only merge geps with constant offsets, because only for those
1196 if (GEP->getNumIndices() != 1 || !isa<Constant>(Offsets))
1200 Value *BaseBasePtr = foldGEP(BaseGEP, Offsets, Scale, Builder);
1203 Offsets = CheckAndCreateOffsetAdd(
1204 Offsets, Scale, GEP->getOperand(1),
1206 if (Offsets == nullptr)
1224 Value *Offsets;
1226 Value *Base = foldGEP(GEP, Offsets, Scale, Builder);
1231 if (Offsets && Base && Base != GEP) {
1237 Builder.getInt8Ty(), Builder.CreateBitCast(Base, BaseTy), Offsets,