Lines Matching full:load
1 //===- LoopLoadElimination.cpp - Loop Load Elimination Pass ---------------===//
9 // This file implement a loop-aware load elimination pass.
14 // of the corresponding load. This makes the load dead.
18 // load.
66 #define LLE_OPTION "loop-load-elim"
70 "runtime-check-per-loop-load-elim", cl::Hidden,
71 cl::desc("Max number of memchecks allowed per eliminated load on average"),
75 "loop-load-elimination-scev-check-threshold", cl::init(8), cl::Hidden,
77 "Load Elimination"));
85 LoadInst *Load;
88 StoreToLoadForwardingCandidate(LoadInst *Load, StoreInst *Store)
89 : Load(Load), Store(Store) {}
91 /// Return true if the dependence from the store to the load has an
96 Value *LoadPtr = Load->getPointerOperand();
98 Type *LoadType = getLoadStoreType(Load);
99 auto &DL = Load->getDataLayout();
115 // benefits of load elimination. To safely implement support for non-unit
137 Value *getLoadPtr() const { return Load->getPointerOperand(); }
143 OS.indent(2) << *Cand.Load << "\n";
162 /// Return true if the load is not executed on all paths in the loop.
163 static bool isLoadConditional(LoadInst *Load, Loop *L) {
164 return Load->getParent() != L->getHeader();
178 /// this loop and find store->load dependences.
191 // Find store->load dependences (consequently true dep). Both lexically
221 auto *Load = dyn_cast<LoadInst>(Destination);
222 if (!Load)
227 getLoadStoreType(Store), getLoadStoreType(Load),
231 Candidates.emplace_front(Load, Store);
236 return LoadsWithUnknownDepedence.count(C.Load);
249 /// If a load has multiple candidates associated (i.e. different
281 LoadToSingleCand.insert(std::make_pair(Cand.Load, &Cand));
284 // Already multiple stores forward to this load.
290 // long as they both have a dependence distance of one to the load.
294 // They are in the same block, the later one will forward to the load.
303 if (LoadToSingleCand[Cand.Load] != &Cand) {
307 << " The load may have multiple stores forwarding to "
318 /// We need a check if one is a pointer for a candidate load and the other is
332 /// forwarding store to a load.
358 return getInstrIndex(A.Load) <
359 getInstrIndex(B.Load);
361 ->Load;
373 // forwarded-to load. Collect the pointer for the stores.
427 // %x = load %gep_i
434 // %x.initial = load %gep_0
437 // %x = load %gep_i <---- now dead
441 Value *Ptr = Cand.Load->getPointerOperand();
448 new LoadInst(Cand.Load->getType(), InitialPtr, "load_initial",
449 /* isVolatile */ false, Cand.Load->getAlign(),
462 auto &DL = Cand.Load->getDataLayout();
473 // Because it casts the old `load` value and is used by the new `phi`
474 // which replaces the old `load`, we give the `load`'s debug location
476 cast<Instruction>(StoreValue)->setDebugLoc(Cand.Load->getDebugLoc());
481 Cand.Load->replaceAllUsesWith(PHI);
482 PHI->setDebugLoc(Cand.Load->getDebugLoc());
485 /// Top-level driver for each loop: find store->load forwarding
491 // Look for store-to-load forwarding cases across the
495 // %x = load %gep_i
502 // %x.initial = load %gep_0
505 // %x = load %gep_i <---- now dead
509 // First start with store->load dependences.
514 // Generate an index for each load and store according to the original
518 // To keep things simple for now, remove those where the load is potentially
534 // If the load is conditional we can't hoist its 0-iteration instance to
537 if (isLoadConditional(Cand.Load, L))
541 // thus we load the value in the next iteration.
545 assert(isa<SCEVAddRecExpr>(PSE.getSCEV(Cand.Load->getPointerOperand())) &&
555 << ". Valid store-to-load forwarding across the loop backedge\n");
611 PSE.getSCEV(Cand.Load->getPointerOperand())) ||
618 // Next, propagate the value stored by the store to the users of the load.
619 // Also for the first iteration, generate the initial value of the load.
632 /// Maps the load/store instructions to their index according to