Lines Matching defs:GEP
34 // each GEP, wasting tons of registers. It emits the following PTX for the
50 // It works by splitting each GEP into a variadic base and a constant offset.
81 // Another improvement enabled by the LowerGEP flag is to lower a GEP with
102 // lower a GEP with multiple indices into arithmetic operations:
124 // If the target uses alias analysis in codegen, this pass will lower a GEP
148 // LICM (Loop Invariant Code Motion) can not hoist/sink a GEP of multiple
149 // indices if one of the index is variant. If we lower such GEP into invariant
152 // target's addressing modes. A GEP with multiple indices may not match and will
153 // not be sunk. If we lower such GEP into smaller parts, CGP may sink some of
203 cl::desc("Do not separate the constant offset from a GEP instruction"),
216 /// A helper class for separating a constant offset from a GEP index.
218 /// In real programs, a GEP index may be more complicated than a simple addition
223 /// Therefore, this class looks into the expression that computes a given GEP
231 /// Extracts a constant offset from the given GEP index. It returns the
234 /// \p Idx The given GEP index
235 /// \p GEP The given GEP
238 static Value *Extract(Value *Idx, GetElementPtrInst *GEP,
241 /// Looks for a constant offset from the given GEP index without extracting
244 static int64_t Find(Value *Idx, GetElementPtrInst *GEP);
257 /// GEP index
259 /// GEP index
261 /// an index of an inbounds GEP is guaranteed to be
270 /// After finding the constant offset C from the GEP index I, we build a new
287 /// After the first step of rebuilding the GEP index without the constant
304 /// Reassociates the GEP index to the form I' + C and returns I'.
322 /// The path from the constant offset to the old GEP index. e.g., if the GEP
327 /// This path helps to rebuild the new GEP index.
340 /// A pass that tries to split every GEP in the function into a variadic
367 /// A pass that tries to split every GEP in the function into a variadic
390 /// Tries to split the given GEP into a variadic base and a constant offset,
392 bool splitGEP(GetElementPtrInst *GEP);
394 /// Tries to reorder the given GEP with the GEP that produces the base if
397 bool reorderGEP(GetElementPtrInst *GEP, TargetTransformInfo &TTI);
399 /// Lower a GEP with multiple indices into multiple GEPs with a single index.
400 /// Function splitGEP already split the original GEP into a variadic part and
404 /// \p Variadic The variadic part of the original GEP.
409 /// Lower a GEP with multiple indices into ptrtoint+arithmetics+inttoptr form.
410 /// Function splitGEP already split the original GEP into a variadic part and
414 /// \p Variadic The variadic part of the original GEP.
424 int64_t accumulateByteOffset(GetElementPtrInst *GEP, bool &NeedsExtraction);
427 /// simplify the logic of splitting a GEP. For example, if a + b is a
431 /// the pointer size, because LLVM conceptually sign-extends GEP indices to
441 bool canonicalizeArrayIndicesToIndexSize(GetElementPtrInst *GEP);
467 // Swap the index operand of two GEP.
470 // Check if it is safe to swap operand of two GEP.
481 /// Whether to lower a GEP with multiple indices into arithmetic operations or
553 // Leveraging this invariant, we can trace into an sext'ed inbound GEP
781 Value *ConstantOffsetExtractor::Extract(Value *Idx, GetElementPtrInst *GEP,
783 ConstantOffsetExtractor Extractor(GEP->getIterator());
787 GEP->isInBounds());
792 // Separates the constant offset from the GEP index.
798 int64_t ConstantOffsetExtractor::Find(Value *Idx, GetElementPtrInst *GEP) {
799 // If Idx is an index of an inbound GEP, Idx is guaranteed to be non-negative.
800 return ConstantOffsetExtractor(GEP->getIterator())
802 GEP->isInBounds())
807 GetElementPtrInst *GEP) {
809 Type *PtrIdxTy = DL->getIndexType(GEP->getType());
810 gep_type_iterator GTI = gep_type_begin(*GEP);
811 for (User::op_iterator I = GEP->op_begin() + 1, E = GEP->op_end();
817 GEP->getIterator());
826 SeparateConstOffsetFromGEP::accumulateByteOffset(GetElementPtrInst *GEP,
830 gep_type_iterator GTI = gep_type_begin(*GEP);
831 for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I, ++GTI) {
837 // Tries to extract a constant offset from this GEP index.
839 ConstantOffsetExtractor::Find(GEP->getOperand(I), GEP);
842 // A GEP may have multiple indices. We accumulate the extracted
844 // the original GEP with this byte offset.
850 uint64_t Field = cast<ConstantInt>(GEP->getOperand(I))->getZExtValue();
876 // Create an ugly GEP for each sequential index. We don't create GEPs for
898 // Create an ugly GEP with a single index for each index.
905 // Create a GEP with the constant offset index.
912 // If we created a GEP with constant index, and the base is loop invariant,
913 // then we swap the first one with it, so LICM can move constant GEP out
973 bool SeparateConstOffsetFromGEP::reorderGEP(GetElementPtrInst *GEP,
975 auto PtrGEP = dyn_cast<GetElementPtrInst>(GEP->getPointerOperand());
986 if (!TTI.isLegalAddressingMode(GEP->getResultElementType(),
991 bool GEPInBounds = GEP->isInBounds();
998 IsChainInBounds &= all_of(GEP->indices(), IsKnownNonNegative);
1003 IRBuilder<> Builder(GEP);
1004 // For trivial GEP chains, we can swap the indices.
1006 GEP->getSourceElementType(), PtrGEP->getPointerOperand(),
1007 SmallVector<Value *, 4>(GEP->indices()), "", IsChainInBounds);
1011 GEP->replaceAllUsesWith(NewGEP);
1012 RecursivelyDeleteTriviallyDeadInstructions(GEP);
1016 bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
1018 if (GEP->getType()->isVectorTy())
1023 if (GEP->hasAllConstantIndices())
1026 bool Changed = canonicalizeArrayIndicesToIndexSize(GEP);
1029 int64_t AccumulativeByteOffset = accumulateByteOffset(GEP, NeedsExtraction);
1031 TargetTransformInfo &TTI = GetTTI(*GEP->getFunction());
1034 Changed |= reorderGEP(GEP, TTI);
1038 // If LowerGEP is disabled, before really splitting the GEP, check whether the
1046 unsigned AddrSpace = GEP->getPointerAddressSpace();
1047 if (!TTI.isLegalAddressingMode(GEP->getResultElementType(),
1055 // Remove the constant offset in each sequential index. The resultant GEP
1062 gep_type_iterator GTI = gep_type_begin(*GEP);
1063 for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I, ++GTI) {
1069 // Splits this GEP index into a variadic part and a constant offset, and
1071 Value *OldIdx = GEP->getOperand(I);
1074 ConstantOffsetExtractor::Extract(OldIdx, GEP, UserChainTail);
1077 GEP->setOperand(I, NewIdx);
1106 bool GEPWasInBounds = GEP->isInBounds();
1107 GEP->setNoWrapFlags(GEPNoWrapFlags::none());
1109 // Lowers a GEP to either GEPs with a single index or arithmetic operations.
1117 unsigned AddrSpace = GEP->getPointerAddressSpace();
1122 lowerToSingleIndexGEPs(GEP, AccumulativeByteOffset);
1124 lowerToArithmetics(GEP, AccumulativeByteOffset);
1128 // No need to create another GEP if the accumulative byte offset is 0.
1149 Instruction *NewGEP = GEP->clone();
1150 NewGEP->insertBefore(GEP->getIterator());
1152 Type *PtrIdxTy = DL->getIndexType(GEP->getType());
1153 IRBuilder<> Builder(GEP);
1156 GEP->getName(), GEPWasInBounds));
1157 NewGEP->copyMetadata(*GEP);
1159 GEP->replaceAllUsesWith(NewGEP);
1160 GEP->eraseFromParent();
1189 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I))
1190 Changed |= splitGEP(GEP);
1191 // No need to split GEP ConstantExprs because all its indices are constant
1319 // Give up if the index of the first GEP is loop invariant.
1329 // Check if the second operand of first GEP has constant coefficient.
1331 // hoisting the second GEP out because the second GEP can be folded away.