1 //===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the implementation of the scalar evolution analysis 11 // engine, which is used primarily to analyze expressions involving induction 12 // variables in loops. 13 // 14 // There are several aspects to this library. First is the representation of 15 // scalar expressions, which are represented as subclasses of the SCEV class. 16 // These classes are used to represent certain types of subexpressions that we 17 // can handle. We only create one SCEV of a particular shape, so 18 // pointer-comparisons for equality are legal. 19 // 20 // One important aspect of the SCEV objects is that they are never cyclic, even 21 // if there is a cycle in the dataflow for an expression (ie, a PHI node). If 22 // the PHI node is one of the idioms that we can represent (e.g., a polynomial 23 // recurrence) then we represent it directly as a recurrence node, otherwise we 24 // represent it as a SCEVUnknown node. 25 // 26 // In addition to being able to represent expressions of various types, we also 27 // have folders that are used to build the *canonical* representation for a 28 // particular expression. These folders are capable of using a variety of 29 // rewrite rules to simplify the expressions. 30 // 31 // Once the folders are defined, we can implement the more interesting 32 // higher-level code, such as the code that recognizes PHI nodes of various 33 // types, computes the execution count of a loop, etc. 34 // 35 // TODO: We should use these routines and value representations to implement 36 // dependence analysis! 37 // 38 //===----------------------------------------------------------------------===// 39 // 40 // There are several good references for the techniques used in this analysis. 41 // 42 // Chains of recurrences -- a method to expedite the evaluation 43 // of closed-form functions 44 // Olaf Bachmann, Paul S. Wang, Eugene V. Zima 45 // 46 // On computational properties of chains of recurrences 47 // Eugene V. Zima 48 // 49 // Symbolic Evaluation of Chains of Recurrences for Loop Optimization 50 // Robert A. van Engelen 51 // 52 // Efficient Symbolic Analysis for Optimizing Compilers 53 // Robert A. van Engelen 54 // 55 // Using the chains of recurrences algebra for data dependence testing and 56 // induction variable substitution 57 // MS Thesis, Johnie Birch 58 // 59 //===----------------------------------------------------------------------===// 60 61 #include "llvm/Analysis/ScalarEvolution.h" 62 #include "llvm/ADT/Optional.h" 63 #include "llvm/ADT/STLExtras.h" 64 #include "llvm/ADT/SmallPtrSet.h" 65 #include "llvm/ADT/Statistic.h" 66 #include "llvm/Analysis/AssumptionTracker.h" 67 #include "llvm/Analysis/ConstantFolding.h" 68 #include "llvm/Analysis/InstructionSimplify.h" 69 #include "llvm/Analysis/LoopInfo.h" 70 #include "llvm/Analysis/ScalarEvolutionExpressions.h" 71 #include "llvm/Analysis/ValueTracking.h" 72 #include "llvm/IR/ConstantRange.h" 73 #include "llvm/IR/Constants.h" 74 #include "llvm/IR/DataLayout.h" 75 #include "llvm/IR/DerivedTypes.h" 76 #include "llvm/IR/Dominators.h" 77 #include "llvm/IR/GetElementPtrTypeIterator.h" 78 #include "llvm/IR/GlobalAlias.h" 79 #include "llvm/IR/GlobalVariable.h" 80 #include "llvm/IR/InstIterator.h" 81 #include "llvm/IR/Instructions.h" 82 #include "llvm/IR/LLVMContext.h" 83 #include "llvm/IR/Metadata.h" 84 #include "llvm/IR/Operator.h" 85 #include "llvm/Support/CommandLine.h" 86 #include "llvm/Support/Debug.h" 87 #include "llvm/Support/ErrorHandling.h" 88 #include "llvm/Support/MathExtras.h" 89 #include "llvm/Support/raw_ostream.h" 90 #include "llvm/Target/TargetLibraryInfo.h" 91 #include <algorithm> 92 using namespace llvm; 93 94 #define DEBUG_TYPE "scalar-evolution" 95 96 STATISTIC(NumArrayLenItCounts, 97 "Number of trip counts computed with array length"); 98 STATISTIC(NumTripCountsComputed, 99 "Number of loops with predictable loop counts"); 100 STATISTIC(NumTripCountsNotComputed, 101 "Number of loops without predictable loop counts"); 102 STATISTIC(NumBruteForceTripCountsComputed, 103 "Number of loops with trip counts computed by force"); 104 105 static cl::opt<unsigned> 106 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden, 107 cl::desc("Maximum number of iterations SCEV will " 108 "symbolically execute a constant " 109 "derived loop"), 110 cl::init(100)); 111 112 // FIXME: Enable this with XDEBUG when the test suite is clean. 113 static cl::opt<bool> 114 VerifySCEV("verify-scev", 115 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)")); 116 117 INITIALIZE_PASS_BEGIN(ScalarEvolution, "scalar-evolution", 118 "Scalar Evolution Analysis", false, true) 119 INITIALIZE_PASS_DEPENDENCY(AssumptionTracker) 120 INITIALIZE_PASS_DEPENDENCY(LoopInfo) 121 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 122 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfo) 123 INITIALIZE_PASS_END(ScalarEvolution, "scalar-evolution", 124 "Scalar Evolution Analysis", false, true) 125 char ScalarEvolution::ID = 0; 126 127 //===----------------------------------------------------------------------===// 128 // SCEV class definitions 129 //===----------------------------------------------------------------------===// 130 131 //===----------------------------------------------------------------------===// 132 // Implementation of the SCEV class. 133 // 134 135 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 136 void SCEV::dump() const { 137 print(dbgs()); 138 dbgs() << '\n'; 139 } 140 #endif 141 142 void SCEV::print(raw_ostream &OS) const { 143 switch (static_cast<SCEVTypes>(getSCEVType())) { 144 case scConstant: 145 cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false); 146 return; 147 case scTruncate: { 148 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this); 149 const SCEV *Op = Trunc->getOperand(); 150 OS << "(trunc " << *Op->getType() << " " << *Op << " to " 151 << *Trunc->getType() << ")"; 152 return; 153 } 154 case scZeroExtend: { 155 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(this); 156 const SCEV *Op = ZExt->getOperand(); 157 OS << "(zext " << *Op->getType() << " " << *Op << " to " 158 << *ZExt->getType() << ")"; 159 return; 160 } 161 case scSignExtend: { 162 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(this); 163 const SCEV *Op = SExt->getOperand(); 164 OS << "(sext " << *Op->getType() << " " << *Op << " to " 165 << *SExt->getType() << ")"; 166 return; 167 } 168 case scAddRecExpr: { 169 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this); 170 OS << "{" << *AR->getOperand(0); 171 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i) 172 OS << ",+," << *AR->getOperand(i); 173 OS << "}<"; 174 if (AR->getNoWrapFlags(FlagNUW)) 175 OS << "nuw><"; 176 if (AR->getNoWrapFlags(FlagNSW)) 177 OS << "nsw><"; 178 if (AR->getNoWrapFlags(FlagNW) && 179 !AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW))) 180 OS << "nw><"; 181 AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false); 182 OS << ">"; 183 return; 184 } 185 case scAddExpr: 186 case scMulExpr: 187 case scUMaxExpr: 188 case scSMaxExpr: { 189 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this); 190 const char *OpStr = nullptr; 191 switch (NAry->getSCEVType()) { 192 case scAddExpr: OpStr = " + "; break; 193 case scMulExpr: OpStr = " * "; break; 194 case scUMaxExpr: OpStr = " umax "; break; 195 case scSMaxExpr: OpStr = " smax "; break; 196 } 197 OS << "("; 198 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 199 I != E; ++I) { 200 OS << **I; 201 if (std::next(I) != E) 202 OS << OpStr; 203 } 204 OS << ")"; 205 switch (NAry->getSCEVType()) { 206 case scAddExpr: 207 case scMulExpr: 208 if (NAry->getNoWrapFlags(FlagNUW)) 209 OS << "<nuw>"; 210 if (NAry->getNoWrapFlags(FlagNSW)) 211 OS << "<nsw>"; 212 } 213 return; 214 } 215 case scUDivExpr: { 216 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this); 217 OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")"; 218 return; 219 } 220 case scUnknown: { 221 const SCEVUnknown *U = cast<SCEVUnknown>(this); 222 Type *AllocTy; 223 if (U->isSizeOf(AllocTy)) { 224 OS << "sizeof(" << *AllocTy << ")"; 225 return; 226 } 227 if (U->isAlignOf(AllocTy)) { 228 OS << "alignof(" << *AllocTy << ")"; 229 return; 230 } 231 232 Type *CTy; 233 Constant *FieldNo; 234 if (U->isOffsetOf(CTy, FieldNo)) { 235 OS << "offsetof(" << *CTy << ", "; 236 FieldNo->printAsOperand(OS, false); 237 OS << ")"; 238 return; 239 } 240 241 // Otherwise just print it normally. 242 U->getValue()->printAsOperand(OS, false); 243 return; 244 } 245 case scCouldNotCompute: 246 OS << "***COULDNOTCOMPUTE***"; 247 return; 248 } 249 llvm_unreachable("Unknown SCEV kind!"); 250 } 251 252 Type *SCEV::getType() const { 253 switch (static_cast<SCEVTypes>(getSCEVType())) { 254 case scConstant: 255 return cast<SCEVConstant>(this)->getType(); 256 case scTruncate: 257 case scZeroExtend: 258 case scSignExtend: 259 return cast<SCEVCastExpr>(this)->getType(); 260 case scAddRecExpr: 261 case scMulExpr: 262 case scUMaxExpr: 263 case scSMaxExpr: 264 return cast<SCEVNAryExpr>(this)->getType(); 265 case scAddExpr: 266 return cast<SCEVAddExpr>(this)->getType(); 267 case scUDivExpr: 268 return cast<SCEVUDivExpr>(this)->getType(); 269 case scUnknown: 270 return cast<SCEVUnknown>(this)->getType(); 271 case scCouldNotCompute: 272 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 273 } 274 llvm_unreachable("Unknown SCEV kind!"); 275 } 276 277 bool SCEV::isZero() const { 278 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 279 return SC->getValue()->isZero(); 280 return false; 281 } 282 283 bool SCEV::isOne() const { 284 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 285 return SC->getValue()->isOne(); 286 return false; 287 } 288 289 bool SCEV::isAllOnesValue() const { 290 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 291 return SC->getValue()->isAllOnesValue(); 292 return false; 293 } 294 295 /// isNonConstantNegative - Return true if the specified scev is negated, but 296 /// not a constant. 297 bool SCEV::isNonConstantNegative() const { 298 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(this); 299 if (!Mul) return false; 300 301 // If there is a constant factor, it will be first. 302 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0)); 303 if (!SC) return false; 304 305 // Return true if the value is negative, this matches things like (-42 * V). 306 return SC->getValue()->getValue().isNegative(); 307 } 308 309 SCEVCouldNotCompute::SCEVCouldNotCompute() : 310 SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {} 311 312 bool SCEVCouldNotCompute::classof(const SCEV *S) { 313 return S->getSCEVType() == scCouldNotCompute; 314 } 315 316 const SCEV *ScalarEvolution::getConstant(ConstantInt *V) { 317 FoldingSetNodeID ID; 318 ID.AddInteger(scConstant); 319 ID.AddPointer(V); 320 void *IP = nullptr; 321 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 322 SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V); 323 UniqueSCEVs.InsertNode(S, IP); 324 return S; 325 } 326 327 const SCEV *ScalarEvolution::getConstant(const APInt &Val) { 328 return getConstant(ConstantInt::get(getContext(), Val)); 329 } 330 331 const SCEV * 332 ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) { 333 IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty)); 334 return getConstant(ConstantInt::get(ITy, V, isSigned)); 335 } 336 337 SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, 338 unsigned SCEVTy, const SCEV *op, Type *ty) 339 : SCEV(ID, SCEVTy), Op(op), Ty(ty) {} 340 341 SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, 342 const SCEV *op, Type *ty) 343 : SCEVCastExpr(ID, scTruncate, op, ty) { 344 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 345 (Ty->isIntegerTy() || Ty->isPointerTy()) && 346 "Cannot truncate non-integer value!"); 347 } 348 349 SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, 350 const SCEV *op, Type *ty) 351 : SCEVCastExpr(ID, scZeroExtend, op, ty) { 352 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 353 (Ty->isIntegerTy() || Ty->isPointerTy()) && 354 "Cannot zero extend non-integer value!"); 355 } 356 357 SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, 358 const SCEV *op, Type *ty) 359 : SCEVCastExpr(ID, scSignExtend, op, ty) { 360 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 361 (Ty->isIntegerTy() || Ty->isPointerTy()) && 362 "Cannot sign extend non-integer value!"); 363 } 364 365 void SCEVUnknown::deleted() { 366 // Clear this SCEVUnknown from various maps. 367 SE->forgetMemoizedResults(this); 368 369 // Remove this SCEVUnknown from the uniquing map. 370 SE->UniqueSCEVs.RemoveNode(this); 371 372 // Release the value. 373 setValPtr(nullptr); 374 } 375 376 void SCEVUnknown::allUsesReplacedWith(Value *New) { 377 // Clear this SCEVUnknown from various maps. 378 SE->forgetMemoizedResults(this); 379 380 // Remove this SCEVUnknown from the uniquing map. 381 SE->UniqueSCEVs.RemoveNode(this); 382 383 // Update this SCEVUnknown to point to the new value. This is needed 384 // because there may still be outstanding SCEVs which still point to 385 // this SCEVUnknown. 386 setValPtr(New); 387 } 388 389 bool SCEVUnknown::isSizeOf(Type *&AllocTy) const { 390 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 391 if (VCE->getOpcode() == Instruction::PtrToInt) 392 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 393 if (CE->getOpcode() == Instruction::GetElementPtr && 394 CE->getOperand(0)->isNullValue() && 395 CE->getNumOperands() == 2) 396 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1))) 397 if (CI->isOne()) { 398 AllocTy = cast<PointerType>(CE->getOperand(0)->getType()) 399 ->getElementType(); 400 return true; 401 } 402 403 return false; 404 } 405 406 bool SCEVUnknown::isAlignOf(Type *&AllocTy) const { 407 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 408 if (VCE->getOpcode() == Instruction::PtrToInt) 409 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 410 if (CE->getOpcode() == Instruction::GetElementPtr && 411 CE->getOperand(0)->isNullValue()) { 412 Type *Ty = 413 cast<PointerType>(CE->getOperand(0)->getType())->getElementType(); 414 if (StructType *STy = dyn_cast<StructType>(Ty)) 415 if (!STy->isPacked() && 416 CE->getNumOperands() == 3 && 417 CE->getOperand(1)->isNullValue()) { 418 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2))) 419 if (CI->isOne() && 420 STy->getNumElements() == 2 && 421 STy->getElementType(0)->isIntegerTy(1)) { 422 AllocTy = STy->getElementType(1); 423 return true; 424 } 425 } 426 } 427 428 return false; 429 } 430 431 bool SCEVUnknown::isOffsetOf(Type *&CTy, Constant *&FieldNo) const { 432 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 433 if (VCE->getOpcode() == Instruction::PtrToInt) 434 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 435 if (CE->getOpcode() == Instruction::GetElementPtr && 436 CE->getNumOperands() == 3 && 437 CE->getOperand(0)->isNullValue() && 438 CE->getOperand(1)->isNullValue()) { 439 Type *Ty = 440 cast<PointerType>(CE->getOperand(0)->getType())->getElementType(); 441 // Ignore vector types here so that ScalarEvolutionExpander doesn't 442 // emit getelementptrs that index into vectors. 443 if (Ty->isStructTy() || Ty->isArrayTy()) { 444 CTy = Ty; 445 FieldNo = CE->getOperand(2); 446 return true; 447 } 448 } 449 450 return false; 451 } 452 453 //===----------------------------------------------------------------------===// 454 // SCEV Utilities 455 //===----------------------------------------------------------------------===// 456 457 namespace { 458 /// SCEVComplexityCompare - Return true if the complexity of the LHS is less 459 /// than the complexity of the RHS. This comparator is used to canonicalize 460 /// expressions. 461 class SCEVComplexityCompare { 462 const LoopInfo *const LI; 463 public: 464 explicit SCEVComplexityCompare(const LoopInfo *li) : LI(li) {} 465 466 // Return true or false if LHS is less than, or at least RHS, respectively. 467 bool operator()(const SCEV *LHS, const SCEV *RHS) const { 468 return compare(LHS, RHS) < 0; 469 } 470 471 // Return negative, zero, or positive, if LHS is less than, equal to, or 472 // greater than RHS, respectively. A three-way result allows recursive 473 // comparisons to be more efficient. 474 int compare(const SCEV *LHS, const SCEV *RHS) const { 475 // Fast-path: SCEVs are uniqued so we can do a quick equality check. 476 if (LHS == RHS) 477 return 0; 478 479 // Primarily, sort the SCEVs by their getSCEVType(). 480 unsigned LType = LHS->getSCEVType(), RType = RHS->getSCEVType(); 481 if (LType != RType) 482 return (int)LType - (int)RType; 483 484 // Aside from the getSCEVType() ordering, the particular ordering 485 // isn't very important except that it's beneficial to be consistent, 486 // so that (a + b) and (b + a) don't end up as different expressions. 487 switch (static_cast<SCEVTypes>(LType)) { 488 case scUnknown: { 489 const SCEVUnknown *LU = cast<SCEVUnknown>(LHS); 490 const SCEVUnknown *RU = cast<SCEVUnknown>(RHS); 491 492 // Sort SCEVUnknown values with some loose heuristics. TODO: This is 493 // not as complete as it could be. 494 const Value *LV = LU->getValue(), *RV = RU->getValue(); 495 496 // Order pointer values after integer values. This helps SCEVExpander 497 // form GEPs. 498 bool LIsPointer = LV->getType()->isPointerTy(), 499 RIsPointer = RV->getType()->isPointerTy(); 500 if (LIsPointer != RIsPointer) 501 return (int)LIsPointer - (int)RIsPointer; 502 503 // Compare getValueID values. 504 unsigned LID = LV->getValueID(), 505 RID = RV->getValueID(); 506 if (LID != RID) 507 return (int)LID - (int)RID; 508 509 // Sort arguments by their position. 510 if (const Argument *LA = dyn_cast<Argument>(LV)) { 511 const Argument *RA = cast<Argument>(RV); 512 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo(); 513 return (int)LArgNo - (int)RArgNo; 514 } 515 516 // For instructions, compare their loop depth, and their operand 517 // count. This is pretty loose. 518 if (const Instruction *LInst = dyn_cast<Instruction>(LV)) { 519 const Instruction *RInst = cast<Instruction>(RV); 520 521 // Compare loop depths. 522 const BasicBlock *LParent = LInst->getParent(), 523 *RParent = RInst->getParent(); 524 if (LParent != RParent) { 525 unsigned LDepth = LI->getLoopDepth(LParent), 526 RDepth = LI->getLoopDepth(RParent); 527 if (LDepth != RDepth) 528 return (int)LDepth - (int)RDepth; 529 } 530 531 // Compare the number of operands. 532 unsigned LNumOps = LInst->getNumOperands(), 533 RNumOps = RInst->getNumOperands(); 534 return (int)LNumOps - (int)RNumOps; 535 } 536 537 return 0; 538 } 539 540 case scConstant: { 541 const SCEVConstant *LC = cast<SCEVConstant>(LHS); 542 const SCEVConstant *RC = cast<SCEVConstant>(RHS); 543 544 // Compare constant values. 545 const APInt &LA = LC->getValue()->getValue(); 546 const APInt &RA = RC->getValue()->getValue(); 547 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth(); 548 if (LBitWidth != RBitWidth) 549 return (int)LBitWidth - (int)RBitWidth; 550 return LA.ult(RA) ? -1 : 1; 551 } 552 553 case scAddRecExpr: { 554 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(LHS); 555 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS); 556 557 // Compare addrec loop depths. 558 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop(); 559 if (LLoop != RLoop) { 560 unsigned LDepth = LLoop->getLoopDepth(), 561 RDepth = RLoop->getLoopDepth(); 562 if (LDepth != RDepth) 563 return (int)LDepth - (int)RDepth; 564 } 565 566 // Addrec complexity grows with operand count. 567 unsigned LNumOps = LA->getNumOperands(), RNumOps = RA->getNumOperands(); 568 if (LNumOps != RNumOps) 569 return (int)LNumOps - (int)RNumOps; 570 571 // Lexicographically compare. 572 for (unsigned i = 0; i != LNumOps; ++i) { 573 long X = compare(LA->getOperand(i), RA->getOperand(i)); 574 if (X != 0) 575 return X; 576 } 577 578 return 0; 579 } 580 581 case scAddExpr: 582 case scMulExpr: 583 case scSMaxExpr: 584 case scUMaxExpr: { 585 const SCEVNAryExpr *LC = cast<SCEVNAryExpr>(LHS); 586 const SCEVNAryExpr *RC = cast<SCEVNAryExpr>(RHS); 587 588 // Lexicographically compare n-ary expressions. 589 unsigned LNumOps = LC->getNumOperands(), RNumOps = RC->getNumOperands(); 590 if (LNumOps != RNumOps) 591 return (int)LNumOps - (int)RNumOps; 592 593 for (unsigned i = 0; i != LNumOps; ++i) { 594 if (i >= RNumOps) 595 return 1; 596 long X = compare(LC->getOperand(i), RC->getOperand(i)); 597 if (X != 0) 598 return X; 599 } 600 return (int)LNumOps - (int)RNumOps; 601 } 602 603 case scUDivExpr: { 604 const SCEVUDivExpr *LC = cast<SCEVUDivExpr>(LHS); 605 const SCEVUDivExpr *RC = cast<SCEVUDivExpr>(RHS); 606 607 // Lexicographically compare udiv expressions. 608 long X = compare(LC->getLHS(), RC->getLHS()); 609 if (X != 0) 610 return X; 611 return compare(LC->getRHS(), RC->getRHS()); 612 } 613 614 case scTruncate: 615 case scZeroExtend: 616 case scSignExtend: { 617 const SCEVCastExpr *LC = cast<SCEVCastExpr>(LHS); 618 const SCEVCastExpr *RC = cast<SCEVCastExpr>(RHS); 619 620 // Compare cast expressions by operand. 621 return compare(LC->getOperand(), RC->getOperand()); 622 } 623 624 case scCouldNotCompute: 625 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 626 } 627 llvm_unreachable("Unknown SCEV kind!"); 628 } 629 }; 630 } 631 632 /// GroupByComplexity - Given a list of SCEV objects, order them by their 633 /// complexity, and group objects of the same complexity together by value. 634 /// When this routine is finished, we know that any duplicates in the vector are 635 /// consecutive and that complexity is monotonically increasing. 636 /// 637 /// Note that we go take special precautions to ensure that we get deterministic 638 /// results from this routine. In other words, we don't want the results of 639 /// this to depend on where the addresses of various SCEV objects happened to 640 /// land in memory. 641 /// 642 static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops, 643 LoopInfo *LI) { 644 if (Ops.size() < 2) return; // Noop 645 if (Ops.size() == 2) { 646 // This is the common case, which also happens to be trivially simple. 647 // Special case it. 648 const SCEV *&LHS = Ops[0], *&RHS = Ops[1]; 649 if (SCEVComplexityCompare(LI)(RHS, LHS)) 650 std::swap(LHS, RHS); 651 return; 652 } 653 654 // Do the rough sort by complexity. 655 std::stable_sort(Ops.begin(), Ops.end(), SCEVComplexityCompare(LI)); 656 657 // Now that we are sorted by complexity, group elements of the same 658 // complexity. Note that this is, at worst, N^2, but the vector is likely to 659 // be extremely short in practice. Note that we take this approach because we 660 // do not want to depend on the addresses of the objects we are grouping. 661 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) { 662 const SCEV *S = Ops[i]; 663 unsigned Complexity = S->getSCEVType(); 664 665 // If there are any objects of the same complexity and same value as this 666 // one, group them. 667 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) { 668 if (Ops[j] == S) { // Found a duplicate. 669 // Move it to immediately after i'th element. 670 std::swap(Ops[i+1], Ops[j]); 671 ++i; // no need to rescan it. 672 if (i == e-2) return; // Done! 673 } 674 } 675 } 676 } 677 678 namespace { 679 struct FindSCEVSize { 680 int Size; 681 FindSCEVSize() : Size(0) {} 682 683 bool follow(const SCEV *S) { 684 ++Size; 685 // Keep looking at all operands of S. 686 return true; 687 } 688 bool isDone() const { 689 return false; 690 } 691 }; 692 } 693 694 // Returns the size of the SCEV S. 695 static inline int sizeOfSCEV(const SCEV *S) { 696 FindSCEVSize F; 697 SCEVTraversal<FindSCEVSize> ST(F); 698 ST.visitAll(S); 699 return F.Size; 700 } 701 702 namespace { 703 704 struct SCEVDivision : public SCEVVisitor<SCEVDivision, void> { 705 public: 706 // Computes the Quotient and Remainder of the division of Numerator by 707 // Denominator. 708 static void divide(ScalarEvolution &SE, const SCEV *Numerator, 709 const SCEV *Denominator, const SCEV **Quotient, 710 const SCEV **Remainder) { 711 assert(Numerator && Denominator && "Uninitialized SCEV"); 712 713 SCEVDivision D(SE, Numerator, Denominator); 714 715 // Check for the trivial case here to avoid having to check for it in the 716 // rest of the code. 717 if (Numerator == Denominator) { 718 *Quotient = D.One; 719 *Remainder = D.Zero; 720 return; 721 } 722 723 if (Numerator->isZero()) { 724 *Quotient = D.Zero; 725 *Remainder = D.Zero; 726 return; 727 } 728 729 // Split the Denominator when it is a product. 730 if (const SCEVMulExpr *T = dyn_cast<const SCEVMulExpr>(Denominator)) { 731 const SCEV *Q, *R; 732 *Quotient = Numerator; 733 for (const SCEV *Op : T->operands()) { 734 divide(SE, *Quotient, Op, &Q, &R); 735 *Quotient = Q; 736 737 // Bail out when the Numerator is not divisible by one of the terms of 738 // the Denominator. 739 if (!R->isZero()) { 740 *Quotient = D.Zero; 741 *Remainder = Numerator; 742 return; 743 } 744 } 745 *Remainder = D.Zero; 746 return; 747 } 748 749 D.visit(Numerator); 750 *Quotient = D.Quotient; 751 *Remainder = D.Remainder; 752 } 753 754 // Except in the trivial case described above, we do not know how to divide 755 // Expr by Denominator for the following functions with empty implementation. 756 void visitTruncateExpr(const SCEVTruncateExpr *Numerator) {} 757 void visitZeroExtendExpr(const SCEVZeroExtendExpr *Numerator) {} 758 void visitSignExtendExpr(const SCEVSignExtendExpr *Numerator) {} 759 void visitUDivExpr(const SCEVUDivExpr *Numerator) {} 760 void visitSMaxExpr(const SCEVSMaxExpr *Numerator) {} 761 void visitUMaxExpr(const SCEVUMaxExpr *Numerator) {} 762 void visitUnknown(const SCEVUnknown *Numerator) {} 763 void visitCouldNotCompute(const SCEVCouldNotCompute *Numerator) {} 764 765 void visitConstant(const SCEVConstant *Numerator) { 766 if (const SCEVConstant *D = dyn_cast<SCEVConstant>(Denominator)) { 767 APInt NumeratorVal = Numerator->getValue()->getValue(); 768 APInt DenominatorVal = D->getValue()->getValue(); 769 uint32_t NumeratorBW = NumeratorVal.getBitWidth(); 770 uint32_t DenominatorBW = DenominatorVal.getBitWidth(); 771 772 if (NumeratorBW > DenominatorBW) 773 DenominatorVal = DenominatorVal.sext(NumeratorBW); 774 else if (NumeratorBW < DenominatorBW) 775 NumeratorVal = NumeratorVal.sext(DenominatorBW); 776 777 APInt QuotientVal(NumeratorVal.getBitWidth(), 0); 778 APInt RemainderVal(NumeratorVal.getBitWidth(), 0); 779 APInt::sdivrem(NumeratorVal, DenominatorVal, QuotientVal, RemainderVal); 780 Quotient = SE.getConstant(QuotientVal); 781 Remainder = SE.getConstant(RemainderVal); 782 return; 783 } 784 } 785 786 void visitAddRecExpr(const SCEVAddRecExpr *Numerator) { 787 const SCEV *StartQ, *StartR, *StepQ, *StepR; 788 assert(Numerator->isAffine() && "Numerator should be affine"); 789 divide(SE, Numerator->getStart(), Denominator, &StartQ, &StartR); 790 divide(SE, Numerator->getStepRecurrence(SE), Denominator, &StepQ, &StepR); 791 Quotient = SE.getAddRecExpr(StartQ, StepQ, Numerator->getLoop(), 792 Numerator->getNoWrapFlags()); 793 Remainder = SE.getAddRecExpr(StartR, StepR, Numerator->getLoop(), 794 Numerator->getNoWrapFlags()); 795 } 796 797 void visitAddExpr(const SCEVAddExpr *Numerator) { 798 SmallVector<const SCEV *, 2> Qs, Rs; 799 Type *Ty = Denominator->getType(); 800 801 for (const SCEV *Op : Numerator->operands()) { 802 const SCEV *Q, *R; 803 divide(SE, Op, Denominator, &Q, &R); 804 805 // Bail out if types do not match. 806 if (Ty != Q->getType() || Ty != R->getType()) { 807 Quotient = Zero; 808 Remainder = Numerator; 809 return; 810 } 811 812 Qs.push_back(Q); 813 Rs.push_back(R); 814 } 815 816 if (Qs.size() == 1) { 817 Quotient = Qs[0]; 818 Remainder = Rs[0]; 819 return; 820 } 821 822 Quotient = SE.getAddExpr(Qs); 823 Remainder = SE.getAddExpr(Rs); 824 } 825 826 void visitMulExpr(const SCEVMulExpr *Numerator) { 827 SmallVector<const SCEV *, 2> Qs; 828 Type *Ty = Denominator->getType(); 829 830 bool FoundDenominatorTerm = false; 831 for (const SCEV *Op : Numerator->operands()) { 832 // Bail out if types do not match. 833 if (Ty != Op->getType()) { 834 Quotient = Zero; 835 Remainder = Numerator; 836 return; 837 } 838 839 if (FoundDenominatorTerm) { 840 Qs.push_back(Op); 841 continue; 842 } 843 844 // Check whether Denominator divides one of the product operands. 845 const SCEV *Q, *R; 846 divide(SE, Op, Denominator, &Q, &R); 847 if (!R->isZero()) { 848 Qs.push_back(Op); 849 continue; 850 } 851 852 // Bail out if types do not match. 853 if (Ty != Q->getType()) { 854 Quotient = Zero; 855 Remainder = Numerator; 856 return; 857 } 858 859 FoundDenominatorTerm = true; 860 Qs.push_back(Q); 861 } 862 863 if (FoundDenominatorTerm) { 864 Remainder = Zero; 865 if (Qs.size() == 1) 866 Quotient = Qs[0]; 867 else 868 Quotient = SE.getMulExpr(Qs); 869 return; 870 } 871 872 if (!isa<SCEVUnknown>(Denominator)) { 873 Quotient = Zero; 874 Remainder = Numerator; 875 return; 876 } 877 878 // The Remainder is obtained by replacing Denominator by 0 in Numerator. 879 ValueToValueMap RewriteMap; 880 RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] = 881 cast<SCEVConstant>(Zero)->getValue(); 882 Remainder = SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true); 883 884 if (Remainder->isZero()) { 885 // The Quotient is obtained by replacing Denominator by 1 in Numerator. 886 RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] = 887 cast<SCEVConstant>(One)->getValue(); 888 Quotient = 889 SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true); 890 return; 891 } 892 893 // Quotient is (Numerator - Remainder) divided by Denominator. 894 const SCEV *Q, *R; 895 const SCEV *Diff = SE.getMinusSCEV(Numerator, Remainder); 896 if (sizeOfSCEV(Diff) > sizeOfSCEV(Numerator)) { 897 // This SCEV does not seem to simplify: fail the division here. 898 Quotient = Zero; 899 Remainder = Numerator; 900 return; 901 } 902 divide(SE, Diff, Denominator, &Q, &R); 903 assert(R == Zero && 904 "(Numerator - Remainder) should evenly divide Denominator"); 905 Quotient = Q; 906 } 907 908 private: 909 SCEVDivision(ScalarEvolution &S, const SCEV *Numerator, 910 const SCEV *Denominator) 911 : SE(S), Denominator(Denominator) { 912 Zero = SE.getConstant(Denominator->getType(), 0); 913 One = SE.getConstant(Denominator->getType(), 1); 914 915 // By default, we don't know how to divide Expr by Denominator. 916 // Providing the default here simplifies the rest of the code. 917 Quotient = Zero; 918 Remainder = Numerator; 919 } 920 921 ScalarEvolution &SE; 922 const SCEV *Denominator, *Quotient, *Remainder, *Zero, *One; 923 }; 924 925 } 926 927 //===----------------------------------------------------------------------===// 928 // Simple SCEV method implementations 929 //===----------------------------------------------------------------------===// 930 931 /// BinomialCoefficient - Compute BC(It, K). The result has width W. 932 /// Assume, K > 0. 933 static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K, 934 ScalarEvolution &SE, 935 Type *ResultTy) { 936 // Handle the simplest case efficiently. 937 if (K == 1) 938 return SE.getTruncateOrZeroExtend(It, ResultTy); 939 940 // We are using the following formula for BC(It, K): 941 // 942 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K! 943 // 944 // Suppose, W is the bitwidth of the return value. We must be prepared for 945 // overflow. Hence, we must assure that the result of our computation is 946 // equal to the accurate one modulo 2^W. Unfortunately, division isn't 947 // safe in modular arithmetic. 948 // 949 // However, this code doesn't use exactly that formula; the formula it uses 950 // is something like the following, where T is the number of factors of 2 in 951 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is 952 // exponentiation: 953 // 954 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T) 955 // 956 // This formula is trivially equivalent to the previous formula. However, 957 // this formula can be implemented much more efficiently. The trick is that 958 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular 959 // arithmetic. To do exact division in modular arithmetic, all we have 960 // to do is multiply by the inverse. Therefore, this step can be done at 961 // width W. 962 // 963 // The next issue is how to safely do the division by 2^T. The way this 964 // is done is by doing the multiplication step at a width of at least W + T 965 // bits. This way, the bottom W+T bits of the product are accurate. Then, 966 // when we perform the division by 2^T (which is equivalent to a right shift 967 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get 968 // truncated out after the division by 2^T. 969 // 970 // In comparison to just directly using the first formula, this technique 971 // is much more efficient; using the first formula requires W * K bits, 972 // but this formula less than W + K bits. Also, the first formula requires 973 // a division step, whereas this formula only requires multiplies and shifts. 974 // 975 // It doesn't matter whether the subtraction step is done in the calculation 976 // width or the input iteration count's width; if the subtraction overflows, 977 // the result must be zero anyway. We prefer here to do it in the width of 978 // the induction variable because it helps a lot for certain cases; CodeGen 979 // isn't smart enough to ignore the overflow, which leads to much less 980 // efficient code if the width of the subtraction is wider than the native 981 // register width. 982 // 983 // (It's possible to not widen at all by pulling out factors of 2 before 984 // the multiplication; for example, K=2 can be calculated as 985 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires 986 // extra arithmetic, so it's not an obvious win, and it gets 987 // much more complicated for K > 3.) 988 989 // Protection from insane SCEVs; this bound is conservative, 990 // but it probably doesn't matter. 991 if (K > 1000) 992 return SE.getCouldNotCompute(); 993 994 unsigned W = SE.getTypeSizeInBits(ResultTy); 995 996 // Calculate K! / 2^T and T; we divide out the factors of two before 997 // multiplying for calculating K! / 2^T to avoid overflow. 998 // Other overflow doesn't matter because we only care about the bottom 999 // W bits of the result. 1000 APInt OddFactorial(W, 1); 1001 unsigned T = 1; 1002 for (unsigned i = 3; i <= K; ++i) { 1003 APInt Mult(W, i); 1004 unsigned TwoFactors = Mult.countTrailingZeros(); 1005 T += TwoFactors; 1006 Mult = Mult.lshr(TwoFactors); 1007 OddFactorial *= Mult; 1008 } 1009 1010 // We need at least W + T bits for the multiplication step 1011 unsigned CalculationBits = W + T; 1012 1013 // Calculate 2^T, at width T+W. 1014 APInt DivFactor = APInt::getOneBitSet(CalculationBits, T); 1015 1016 // Calculate the multiplicative inverse of K! / 2^T; 1017 // this multiplication factor will perform the exact division by 1018 // K! / 2^T. 1019 APInt Mod = APInt::getSignedMinValue(W+1); 1020 APInt MultiplyFactor = OddFactorial.zext(W+1); 1021 MultiplyFactor = MultiplyFactor.multiplicativeInverse(Mod); 1022 MultiplyFactor = MultiplyFactor.trunc(W); 1023 1024 // Calculate the product, at width T+W 1025 IntegerType *CalculationTy = IntegerType::get(SE.getContext(), 1026 CalculationBits); 1027 const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy); 1028 for (unsigned i = 1; i != K; ++i) { 1029 const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i)); 1030 Dividend = SE.getMulExpr(Dividend, 1031 SE.getTruncateOrZeroExtend(S, CalculationTy)); 1032 } 1033 1034 // Divide by 2^T 1035 const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor)); 1036 1037 // Truncate the result, and divide by K! / 2^T. 1038 1039 return SE.getMulExpr(SE.getConstant(MultiplyFactor), 1040 SE.getTruncateOrZeroExtend(DivResult, ResultTy)); 1041 } 1042 1043 /// evaluateAtIteration - Return the value of this chain of recurrences at 1044 /// the specified iteration number. We can evaluate this recurrence by 1045 /// multiplying each element in the chain by the binomial coefficient 1046 /// corresponding to it. In other words, we can evaluate {A,+,B,+,C,+,D} as: 1047 /// 1048 /// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3) 1049 /// 1050 /// where BC(It, k) stands for binomial coefficient. 1051 /// 1052 const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It, 1053 ScalarEvolution &SE) const { 1054 const SCEV *Result = getStart(); 1055 for (unsigned i = 1, e = getNumOperands(); i != e; ++i) { 1056 // The computation is correct in the face of overflow provided that the 1057 // multiplication is performed _after_ the evaluation of the binomial 1058 // coefficient. 1059 const SCEV *Coeff = BinomialCoefficient(It, i, SE, getType()); 1060 if (isa<SCEVCouldNotCompute>(Coeff)) 1061 return Coeff; 1062 1063 Result = SE.getAddExpr(Result, SE.getMulExpr(getOperand(i), Coeff)); 1064 } 1065 return Result; 1066 } 1067 1068 //===----------------------------------------------------------------------===// 1069 // SCEV Expression folder implementations 1070 //===----------------------------------------------------------------------===// 1071 1072 const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, 1073 Type *Ty) { 1074 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) && 1075 "This is not a truncating conversion!"); 1076 assert(isSCEVable(Ty) && 1077 "This is not a conversion to a SCEVable type!"); 1078 Ty = getEffectiveSCEVType(Ty); 1079 1080 FoldingSetNodeID ID; 1081 ID.AddInteger(scTruncate); 1082 ID.AddPointer(Op); 1083 ID.AddPointer(Ty); 1084 void *IP = nullptr; 1085 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1086 1087 // Fold if the operand is constant. 1088 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1089 return getConstant( 1090 cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty))); 1091 1092 // trunc(trunc(x)) --> trunc(x) 1093 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) 1094 return getTruncateExpr(ST->getOperand(), Ty); 1095 1096 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing 1097 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op)) 1098 return getTruncateOrSignExtend(SS->getOperand(), Ty); 1099 1100 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing 1101 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1102 return getTruncateOrZeroExtend(SZ->getOperand(), Ty); 1103 1104 // trunc(x1+x2+...+xN) --> trunc(x1)+trunc(x2)+...+trunc(xN) if we can 1105 // eliminate all the truncates. 1106 if (const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Op)) { 1107 SmallVector<const SCEV *, 4> Operands; 1108 bool hasTrunc = false; 1109 for (unsigned i = 0, e = SA->getNumOperands(); i != e && !hasTrunc; ++i) { 1110 const SCEV *S = getTruncateExpr(SA->getOperand(i), Ty); 1111 hasTrunc = isa<SCEVTruncateExpr>(S); 1112 Operands.push_back(S); 1113 } 1114 if (!hasTrunc) 1115 return getAddExpr(Operands); 1116 UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. 1117 } 1118 1119 // trunc(x1*x2*...*xN) --> trunc(x1)*trunc(x2)*...*trunc(xN) if we can 1120 // eliminate all the truncates. 1121 if (const SCEVMulExpr *SM = dyn_cast<SCEVMulExpr>(Op)) { 1122 SmallVector<const SCEV *, 4> Operands; 1123 bool hasTrunc = false; 1124 for (unsigned i = 0, e = SM->getNumOperands(); i != e && !hasTrunc; ++i) { 1125 const SCEV *S = getTruncateExpr(SM->getOperand(i), Ty); 1126 hasTrunc = isa<SCEVTruncateExpr>(S); 1127 Operands.push_back(S); 1128 } 1129 if (!hasTrunc) 1130 return getMulExpr(Operands); 1131 UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. 1132 } 1133 1134 // If the input value is a chrec scev, truncate the chrec's operands. 1135 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) { 1136 SmallVector<const SCEV *, 4> Operands; 1137 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) 1138 Operands.push_back(getTruncateExpr(AddRec->getOperand(i), Ty)); 1139 return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap); 1140 } 1141 1142 // The cast wasn't folded; create an explicit cast node. We can reuse 1143 // the existing insert position since if we get here, we won't have 1144 // made any changes which would invalidate it. 1145 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), 1146 Op, Ty); 1147 UniqueSCEVs.InsertNode(S, IP); 1148 return S; 1149 } 1150 1151 const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, 1152 Type *Ty) { 1153 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1154 "This is not an extending conversion!"); 1155 assert(isSCEVable(Ty) && 1156 "This is not a conversion to a SCEVable type!"); 1157 Ty = getEffectiveSCEVType(Ty); 1158 1159 // Fold if the operand is constant. 1160 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1161 return getConstant( 1162 cast<ConstantInt>(ConstantExpr::getZExt(SC->getValue(), Ty))); 1163 1164 // zext(zext(x)) --> zext(x) 1165 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1166 return getZeroExtendExpr(SZ->getOperand(), Ty); 1167 1168 // Before doing any expensive analysis, check to see if we've already 1169 // computed a SCEV for this Op and Ty. 1170 FoldingSetNodeID ID; 1171 ID.AddInteger(scZeroExtend); 1172 ID.AddPointer(Op); 1173 ID.AddPointer(Ty); 1174 void *IP = nullptr; 1175 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1176 1177 // zext(trunc(x)) --> zext(x) or x or trunc(x) 1178 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) { 1179 // It's possible the bits taken off by the truncate were all zero bits. If 1180 // so, we should be able to simplify this further. 1181 const SCEV *X = ST->getOperand(); 1182 ConstantRange CR = getUnsignedRange(X); 1183 unsigned TruncBits = getTypeSizeInBits(ST->getType()); 1184 unsigned NewBits = getTypeSizeInBits(Ty); 1185 if (CR.truncate(TruncBits).zeroExtend(NewBits).contains( 1186 CR.zextOrTrunc(NewBits))) 1187 return getTruncateOrZeroExtend(X, Ty); 1188 } 1189 1190 // If the input value is a chrec scev, and we can prove that the value 1191 // did not overflow the old, smaller, value, we can zero extend all of the 1192 // operands (often constants). This allows analysis of something like 1193 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; } 1194 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) 1195 if (AR->isAffine()) { 1196 const SCEV *Start = AR->getStart(); 1197 const SCEV *Step = AR->getStepRecurrence(*this); 1198 unsigned BitWidth = getTypeSizeInBits(AR->getType()); 1199 const Loop *L = AR->getLoop(); 1200 1201 // If we have special knowledge that this addrec won't overflow, 1202 // we don't need to do any further analysis. 1203 if (AR->getNoWrapFlags(SCEV::FlagNUW)) 1204 return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1205 getZeroExtendExpr(Step, Ty), 1206 L, AR->getNoWrapFlags()); 1207 1208 // Check whether the backedge-taken count is SCEVCouldNotCompute. 1209 // Note that this serves two purposes: It filters out loops that are 1210 // simply not analyzable, and it covers the case where this code is 1211 // being called from within backedge-taken count analysis, such that 1212 // attempting to ask for the backedge-taken count would likely result 1213 // in infinite recursion. In the later case, the analysis code will 1214 // cope with a conservative value, and it will take care to purge 1215 // that value once it has finished. 1216 const SCEV *MaxBECount = getMaxBackedgeTakenCount(L); 1217 if (!isa<SCEVCouldNotCompute>(MaxBECount)) { 1218 // Manually compute the final value for AR, checking for 1219 // overflow. 1220 1221 // Check whether the backedge-taken count can be losslessly casted to 1222 // the addrec's type. The count is always unsigned. 1223 const SCEV *CastedMaxBECount = 1224 getTruncateOrZeroExtend(MaxBECount, Start->getType()); 1225 const SCEV *RecastedMaxBECount = 1226 getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); 1227 if (MaxBECount == RecastedMaxBECount) { 1228 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); 1229 // Check whether Start+Step*MaxBECount has no unsigned overflow. 1230 const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step); 1231 const SCEV *ZAdd = getZeroExtendExpr(getAddExpr(Start, ZMul), WideTy); 1232 const SCEV *WideStart = getZeroExtendExpr(Start, WideTy); 1233 const SCEV *WideMaxBECount = 1234 getZeroExtendExpr(CastedMaxBECount, WideTy); 1235 const SCEV *OperandExtendedAdd = 1236 getAddExpr(WideStart, 1237 getMulExpr(WideMaxBECount, 1238 getZeroExtendExpr(Step, WideTy))); 1239 if (ZAdd == OperandExtendedAdd) { 1240 // Cache knowledge of AR NUW, which is propagated to this AddRec. 1241 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 1242 // Return the expression with the addrec on the outside. 1243 return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1244 getZeroExtendExpr(Step, Ty), 1245 L, AR->getNoWrapFlags()); 1246 } 1247 // Similar to above, only this time treat the step value as signed. 1248 // This covers loops that count down. 1249 OperandExtendedAdd = 1250 getAddExpr(WideStart, 1251 getMulExpr(WideMaxBECount, 1252 getSignExtendExpr(Step, WideTy))); 1253 if (ZAdd == OperandExtendedAdd) { 1254 // Cache knowledge of AR NW, which is propagated to this AddRec. 1255 // Negative step causes unsigned wrap, but it still can't self-wrap. 1256 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1257 // Return the expression with the addrec on the outside. 1258 return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1259 getSignExtendExpr(Step, Ty), 1260 L, AR->getNoWrapFlags()); 1261 } 1262 } 1263 1264 // If the backedge is guarded by a comparison with the pre-inc value 1265 // the addrec is safe. Also, if the entry is guarded by a comparison 1266 // with the start value and the backedge is guarded by a comparison 1267 // with the post-inc value, the addrec is safe. 1268 if (isKnownPositive(Step)) { 1269 const SCEV *N = getConstant(APInt::getMinValue(BitWidth) - 1270 getUnsignedRange(Step).getUnsignedMax()); 1271 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, AR, N) || 1272 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_ULT, Start, N) && 1273 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, 1274 AR->getPostIncExpr(*this), N))) { 1275 // Cache knowledge of AR NUW, which is propagated to this AddRec. 1276 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 1277 // Return the expression with the addrec on the outside. 1278 return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1279 getZeroExtendExpr(Step, Ty), 1280 L, AR->getNoWrapFlags()); 1281 } 1282 } else if (isKnownNegative(Step)) { 1283 const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) - 1284 getSignedRange(Step).getSignedMin()); 1285 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) || 1286 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_UGT, Start, N) && 1287 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, 1288 AR->getPostIncExpr(*this), N))) { 1289 // Cache knowledge of AR NW, which is propagated to this AddRec. 1290 // Negative step causes unsigned wrap, but it still can't self-wrap. 1291 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1292 // Return the expression with the addrec on the outside. 1293 return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1294 getSignExtendExpr(Step, Ty), 1295 L, AR->getNoWrapFlags()); 1296 } 1297 } 1298 } 1299 } 1300 1301 // The cast wasn't folded; create an explicit cast node. 1302 // Recompute the insert position, as it may have been invalidated. 1303 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1304 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator), 1305 Op, Ty); 1306 UniqueSCEVs.InsertNode(S, IP); 1307 return S; 1308 } 1309 1310 // Get the limit of a recurrence such that incrementing by Step cannot cause 1311 // signed overflow as long as the value of the recurrence within the loop does 1312 // not exceed this limit before incrementing. 1313 static const SCEV *getOverflowLimitForStep(const SCEV *Step, 1314 ICmpInst::Predicate *Pred, 1315 ScalarEvolution *SE) { 1316 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType()); 1317 if (SE->isKnownPositive(Step)) { 1318 *Pred = ICmpInst::ICMP_SLT; 1319 return SE->getConstant(APInt::getSignedMinValue(BitWidth) - 1320 SE->getSignedRange(Step).getSignedMax()); 1321 } 1322 if (SE->isKnownNegative(Step)) { 1323 *Pred = ICmpInst::ICMP_SGT; 1324 return SE->getConstant(APInt::getSignedMaxValue(BitWidth) - 1325 SE->getSignedRange(Step).getSignedMin()); 1326 } 1327 return nullptr; 1328 } 1329 1330 // The recurrence AR has been shown to have no signed wrap. Typically, if we can 1331 // prove NSW for AR, then we can just as easily prove NSW for its preincrement 1332 // or postincrement sibling. This allows normalizing a sign extended AddRec as 1333 // such: {sext(Step + Start),+,Step} => {(Step + sext(Start),+,Step} As a 1334 // result, the expression "Step + sext(PreIncAR)" is congruent with 1335 // "sext(PostIncAR)" 1336 static const SCEV *getPreStartForSignExtend(const SCEVAddRecExpr *AR, 1337 Type *Ty, 1338 ScalarEvolution *SE) { 1339 const Loop *L = AR->getLoop(); 1340 const SCEV *Start = AR->getStart(); 1341 const SCEV *Step = AR->getStepRecurrence(*SE); 1342 1343 // Check for a simple looking step prior to loop entry. 1344 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start); 1345 if (!SA) 1346 return nullptr; 1347 1348 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV 1349 // subtraction is expensive. For this purpose, perform a quick and dirty 1350 // difference, by checking for Step in the operand list. 1351 SmallVector<const SCEV *, 4> DiffOps; 1352 for (const SCEV *Op : SA->operands()) 1353 if (Op != Step) 1354 DiffOps.push_back(Op); 1355 1356 if (DiffOps.size() == SA->getNumOperands()) 1357 return nullptr; 1358 1359 // This is a postinc AR. Check for overflow on the preinc recurrence using the 1360 // same three conditions that getSignExtendedExpr checks. 1361 1362 // 1. NSW flags on the step increment. 1363 const SCEV *PreStart = SE->getAddExpr(DiffOps, SA->getNoWrapFlags()); 1364 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>( 1365 SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap)); 1366 1367 if (PreAR && PreAR->getNoWrapFlags(SCEV::FlagNSW)) 1368 return PreStart; 1369 1370 // 2. Direct overflow check on the step operation's expression. 1371 unsigned BitWidth = SE->getTypeSizeInBits(AR->getType()); 1372 Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2); 1373 const SCEV *OperandExtendedStart = 1374 SE->getAddExpr(SE->getSignExtendExpr(PreStart, WideTy), 1375 SE->getSignExtendExpr(Step, WideTy)); 1376 if (SE->getSignExtendExpr(Start, WideTy) == OperandExtendedStart) { 1377 // Cache knowledge of PreAR NSW. 1378 if (PreAR) 1379 const_cast<SCEVAddRecExpr *>(PreAR)->setNoWrapFlags(SCEV::FlagNSW); 1380 // FIXME: this optimization needs a unit test 1381 DEBUG(dbgs() << "SCEV: untested prestart overflow check\n"); 1382 return PreStart; 1383 } 1384 1385 // 3. Loop precondition. 1386 ICmpInst::Predicate Pred; 1387 const SCEV *OverflowLimit = getOverflowLimitForStep(Step, &Pred, SE); 1388 1389 if (OverflowLimit && 1390 SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit)) { 1391 return PreStart; 1392 } 1393 return nullptr; 1394 } 1395 1396 // Get the normalized sign-extended expression for this AddRec's Start. 1397 static const SCEV *getSignExtendAddRecStart(const SCEVAddRecExpr *AR, 1398 Type *Ty, 1399 ScalarEvolution *SE) { 1400 const SCEV *PreStart = getPreStartForSignExtend(AR, Ty, SE); 1401 if (!PreStart) 1402 return SE->getSignExtendExpr(AR->getStart(), Ty); 1403 1404 return SE->getAddExpr(SE->getSignExtendExpr(AR->getStepRecurrence(*SE), Ty), 1405 SE->getSignExtendExpr(PreStart, Ty)); 1406 } 1407 1408 const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, 1409 Type *Ty) { 1410 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1411 "This is not an extending conversion!"); 1412 assert(isSCEVable(Ty) && 1413 "This is not a conversion to a SCEVable type!"); 1414 Ty = getEffectiveSCEVType(Ty); 1415 1416 // Fold if the operand is constant. 1417 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1418 return getConstant( 1419 cast<ConstantInt>(ConstantExpr::getSExt(SC->getValue(), Ty))); 1420 1421 // sext(sext(x)) --> sext(x) 1422 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op)) 1423 return getSignExtendExpr(SS->getOperand(), Ty); 1424 1425 // sext(zext(x)) --> zext(x) 1426 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1427 return getZeroExtendExpr(SZ->getOperand(), Ty); 1428 1429 // Before doing any expensive analysis, check to see if we've already 1430 // computed a SCEV for this Op and Ty. 1431 FoldingSetNodeID ID; 1432 ID.AddInteger(scSignExtend); 1433 ID.AddPointer(Op); 1434 ID.AddPointer(Ty); 1435 void *IP = nullptr; 1436 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1437 1438 // If the input value is provably positive, build a zext instead. 1439 if (isKnownNonNegative(Op)) 1440 return getZeroExtendExpr(Op, Ty); 1441 1442 // sext(trunc(x)) --> sext(x) or x or trunc(x) 1443 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) { 1444 // It's possible the bits taken off by the truncate were all sign bits. If 1445 // so, we should be able to simplify this further. 1446 const SCEV *X = ST->getOperand(); 1447 ConstantRange CR = getSignedRange(X); 1448 unsigned TruncBits = getTypeSizeInBits(ST->getType()); 1449 unsigned NewBits = getTypeSizeInBits(Ty); 1450 if (CR.truncate(TruncBits).signExtend(NewBits).contains( 1451 CR.sextOrTrunc(NewBits))) 1452 return getTruncateOrSignExtend(X, Ty); 1453 } 1454 1455 // sext(C1 + (C2 * x)) --> C1 + sext(C2 * x) if C1 < C2 1456 if (auto SA = dyn_cast<SCEVAddExpr>(Op)) { 1457 if (SA->getNumOperands() == 2) { 1458 auto SC1 = dyn_cast<SCEVConstant>(SA->getOperand(0)); 1459 auto SMul = dyn_cast<SCEVMulExpr>(SA->getOperand(1)); 1460 if (SMul && SC1) { 1461 if (auto SC2 = dyn_cast<SCEVConstant>(SMul->getOperand(0))) { 1462 const APInt &C1 = SC1->getValue()->getValue(); 1463 const APInt &C2 = SC2->getValue()->getValue(); 1464 if (C1.isStrictlyPositive() && C2.isStrictlyPositive() && 1465 C2.ugt(C1) && C2.isPowerOf2()) 1466 return getAddExpr(getSignExtendExpr(SC1, Ty), 1467 getSignExtendExpr(SMul, Ty)); 1468 } 1469 } 1470 } 1471 } 1472 // If the input value is a chrec scev, and we can prove that the value 1473 // did not overflow the old, smaller, value, we can sign extend all of the 1474 // operands (often constants). This allows analysis of something like 1475 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; } 1476 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) 1477 if (AR->isAffine()) { 1478 const SCEV *Start = AR->getStart(); 1479 const SCEV *Step = AR->getStepRecurrence(*this); 1480 unsigned BitWidth = getTypeSizeInBits(AR->getType()); 1481 const Loop *L = AR->getLoop(); 1482 1483 // If we have special knowledge that this addrec won't overflow, 1484 // we don't need to do any further analysis. 1485 if (AR->getNoWrapFlags(SCEV::FlagNSW)) 1486 return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1487 getSignExtendExpr(Step, Ty), 1488 L, SCEV::FlagNSW); 1489 1490 // Check whether the backedge-taken count is SCEVCouldNotCompute. 1491 // Note that this serves two purposes: It filters out loops that are 1492 // simply not analyzable, and it covers the case where this code is 1493 // being called from within backedge-taken count analysis, such that 1494 // attempting to ask for the backedge-taken count would likely result 1495 // in infinite recursion. In the later case, the analysis code will 1496 // cope with a conservative value, and it will take care to purge 1497 // that value once it has finished. 1498 const SCEV *MaxBECount = getMaxBackedgeTakenCount(L); 1499 if (!isa<SCEVCouldNotCompute>(MaxBECount)) { 1500 // Manually compute the final value for AR, checking for 1501 // overflow. 1502 1503 // Check whether the backedge-taken count can be losslessly casted to 1504 // the addrec's type. The count is always unsigned. 1505 const SCEV *CastedMaxBECount = 1506 getTruncateOrZeroExtend(MaxBECount, Start->getType()); 1507 const SCEV *RecastedMaxBECount = 1508 getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); 1509 if (MaxBECount == RecastedMaxBECount) { 1510 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); 1511 // Check whether Start+Step*MaxBECount has no signed overflow. 1512 const SCEV *SMul = getMulExpr(CastedMaxBECount, Step); 1513 const SCEV *SAdd = getSignExtendExpr(getAddExpr(Start, SMul), WideTy); 1514 const SCEV *WideStart = getSignExtendExpr(Start, WideTy); 1515 const SCEV *WideMaxBECount = 1516 getZeroExtendExpr(CastedMaxBECount, WideTy); 1517 const SCEV *OperandExtendedAdd = 1518 getAddExpr(WideStart, 1519 getMulExpr(WideMaxBECount, 1520 getSignExtendExpr(Step, WideTy))); 1521 if (SAdd == OperandExtendedAdd) { 1522 // Cache knowledge of AR NSW, which is propagated to this AddRec. 1523 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1524 // Return the expression with the addrec on the outside. 1525 return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1526 getSignExtendExpr(Step, Ty), 1527 L, AR->getNoWrapFlags()); 1528 } 1529 // Similar to above, only this time treat the step value as unsigned. 1530 // This covers loops that count up with an unsigned step. 1531 OperandExtendedAdd = 1532 getAddExpr(WideStart, 1533 getMulExpr(WideMaxBECount, 1534 getZeroExtendExpr(Step, WideTy))); 1535 if (SAdd == OperandExtendedAdd) { 1536 // Cache knowledge of AR NSW, which is propagated to this AddRec. 1537 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1538 // Return the expression with the addrec on the outside. 1539 return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1540 getZeroExtendExpr(Step, Ty), 1541 L, AR->getNoWrapFlags()); 1542 } 1543 } 1544 1545 // If the backedge is guarded by a comparison with the pre-inc value 1546 // the addrec is safe. Also, if the entry is guarded by a comparison 1547 // with the start value and the backedge is guarded by a comparison 1548 // with the post-inc value, the addrec is safe. 1549 ICmpInst::Predicate Pred; 1550 const SCEV *OverflowLimit = getOverflowLimitForStep(Step, &Pred, this); 1551 if (OverflowLimit && 1552 (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) || 1553 (isLoopEntryGuardedByCond(L, Pred, Start, OverflowLimit) && 1554 isLoopBackedgeGuardedByCond(L, Pred, AR->getPostIncExpr(*this), 1555 OverflowLimit)))) { 1556 // Cache knowledge of AR NSW, then propagate NSW to the wide AddRec. 1557 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1558 return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1559 getSignExtendExpr(Step, Ty), 1560 L, AR->getNoWrapFlags()); 1561 } 1562 } 1563 // If Start and Step are constants, check if we can apply this 1564 // transformation: 1565 // sext{C1,+,C2} --> C1 + sext{0,+,C2} if C1 < C2 1566 auto SC1 = dyn_cast<SCEVConstant>(Start); 1567 auto SC2 = dyn_cast<SCEVConstant>(Step); 1568 if (SC1 && SC2) { 1569 const APInt &C1 = SC1->getValue()->getValue(); 1570 const APInt &C2 = SC2->getValue()->getValue(); 1571 if (C1.isStrictlyPositive() && C2.isStrictlyPositive() && C2.ugt(C1) && 1572 C2.isPowerOf2()) { 1573 Start = getSignExtendExpr(Start, Ty); 1574 const SCEV *NewAR = getAddRecExpr(getConstant(AR->getType(), 0), Step, 1575 L, AR->getNoWrapFlags()); 1576 return getAddExpr(Start, getSignExtendExpr(NewAR, Ty)); 1577 } 1578 } 1579 } 1580 1581 // The cast wasn't folded; create an explicit cast node. 1582 // Recompute the insert position, as it may have been invalidated. 1583 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1584 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator), 1585 Op, Ty); 1586 UniqueSCEVs.InsertNode(S, IP); 1587 return S; 1588 } 1589 1590 /// getAnyExtendExpr - Return a SCEV for the given operand extended with 1591 /// unspecified bits out to the given type. 1592 /// 1593 const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op, 1594 Type *Ty) { 1595 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1596 "This is not an extending conversion!"); 1597 assert(isSCEVable(Ty) && 1598 "This is not a conversion to a SCEVable type!"); 1599 Ty = getEffectiveSCEVType(Ty); 1600 1601 // Sign-extend negative constants. 1602 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1603 if (SC->getValue()->getValue().isNegative()) 1604 return getSignExtendExpr(Op, Ty); 1605 1606 // Peel off a truncate cast. 1607 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) { 1608 const SCEV *NewOp = T->getOperand(); 1609 if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty)) 1610 return getAnyExtendExpr(NewOp, Ty); 1611 return getTruncateOrNoop(NewOp, Ty); 1612 } 1613 1614 // Next try a zext cast. If the cast is folded, use it. 1615 const SCEV *ZExt = getZeroExtendExpr(Op, Ty); 1616 if (!isa<SCEVZeroExtendExpr>(ZExt)) 1617 return ZExt; 1618 1619 // Next try a sext cast. If the cast is folded, use it. 1620 const SCEV *SExt = getSignExtendExpr(Op, Ty); 1621 if (!isa<SCEVSignExtendExpr>(SExt)) 1622 return SExt; 1623 1624 // Force the cast to be folded into the operands of an addrec. 1625 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) { 1626 SmallVector<const SCEV *, 4> Ops; 1627 for (const SCEV *Op : AR->operands()) 1628 Ops.push_back(getAnyExtendExpr(Op, Ty)); 1629 return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW); 1630 } 1631 1632 // If the expression is obviously signed, use the sext cast value. 1633 if (isa<SCEVSMaxExpr>(Op)) 1634 return SExt; 1635 1636 // Absent any other information, use the zext cast value. 1637 return ZExt; 1638 } 1639 1640 /// CollectAddOperandsWithScales - Process the given Ops list, which is 1641 /// a list of operands to be added under the given scale, update the given 1642 /// map. This is a helper function for getAddRecExpr. As an example of 1643 /// what it does, given a sequence of operands that would form an add 1644 /// expression like this: 1645 /// 1646 /// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r) 1647 /// 1648 /// where A and B are constants, update the map with these values: 1649 /// 1650 /// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0) 1651 /// 1652 /// and add 13 + A*B*29 to AccumulatedConstant. 1653 /// This will allow getAddRecExpr to produce this: 1654 /// 1655 /// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B) 1656 /// 1657 /// This form often exposes folding opportunities that are hidden in 1658 /// the original operand list. 1659 /// 1660 /// Return true iff it appears that any interesting folding opportunities 1661 /// may be exposed. This helps getAddRecExpr short-circuit extra work in 1662 /// the common case where no interesting opportunities are present, and 1663 /// is also used as a check to avoid infinite recursion. 1664 /// 1665 static bool 1666 CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M, 1667 SmallVectorImpl<const SCEV *> &NewOps, 1668 APInt &AccumulatedConstant, 1669 const SCEV *const *Ops, size_t NumOperands, 1670 const APInt &Scale, 1671 ScalarEvolution &SE) { 1672 bool Interesting = false; 1673 1674 // Iterate over the add operands. They are sorted, with constants first. 1675 unsigned i = 0; 1676 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) { 1677 ++i; 1678 // Pull a buried constant out to the outside. 1679 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero()) 1680 Interesting = true; 1681 AccumulatedConstant += Scale * C->getValue()->getValue(); 1682 } 1683 1684 // Next comes everything else. We're especially interested in multiplies 1685 // here, but they're in the middle, so just visit the rest with one loop. 1686 for (; i != NumOperands; ++i) { 1687 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]); 1688 if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) { 1689 APInt NewScale = 1690 Scale * cast<SCEVConstant>(Mul->getOperand(0))->getValue()->getValue(); 1691 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) { 1692 // A multiplication of a constant with another add; recurse. 1693 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1)); 1694 Interesting |= 1695 CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant, 1696 Add->op_begin(), Add->getNumOperands(), 1697 NewScale, SE); 1698 } else { 1699 // A multiplication of a constant with some other value. Update 1700 // the map. 1701 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin()+1, Mul->op_end()); 1702 const SCEV *Key = SE.getMulExpr(MulOps); 1703 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair = 1704 M.insert(std::make_pair(Key, NewScale)); 1705 if (Pair.second) { 1706 NewOps.push_back(Pair.first->first); 1707 } else { 1708 Pair.first->second += NewScale; 1709 // The map already had an entry for this value, which may indicate 1710 // a folding opportunity. 1711 Interesting = true; 1712 } 1713 } 1714 } else { 1715 // An ordinary operand. Update the map. 1716 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair = 1717 M.insert(std::make_pair(Ops[i], Scale)); 1718 if (Pair.second) { 1719 NewOps.push_back(Pair.first->first); 1720 } else { 1721 Pair.first->second += Scale; 1722 // The map already had an entry for this value, which may indicate 1723 // a folding opportunity. 1724 Interesting = true; 1725 } 1726 } 1727 } 1728 1729 return Interesting; 1730 } 1731 1732 namespace { 1733 struct APIntCompare { 1734 bool operator()(const APInt &LHS, const APInt &RHS) const { 1735 return LHS.ult(RHS); 1736 } 1737 }; 1738 } 1739 1740 /// getAddExpr - Get a canonical add expression, or something simpler if 1741 /// possible. 1742 const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, 1743 SCEV::NoWrapFlags Flags) { 1744 assert(!(Flags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) && 1745 "only nuw or nsw allowed"); 1746 assert(!Ops.empty() && "Cannot get empty add!"); 1747 if (Ops.size() == 1) return Ops[0]; 1748 #ifndef NDEBUG 1749 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 1750 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 1751 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 1752 "SCEVAddExpr operand types don't match!"); 1753 #endif 1754 1755 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 1756 // And vice-versa. 1757 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 1758 SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask); 1759 if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) { 1760 bool All = true; 1761 for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(), 1762 E = Ops.end(); I != E; ++I) 1763 if (!isKnownNonNegative(*I)) { 1764 All = false; 1765 break; 1766 } 1767 if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 1768 } 1769 1770 // Sort by complexity, this groups all similar expression types together. 1771 GroupByComplexity(Ops, LI); 1772 1773 // If there are any constants, fold them together. 1774 unsigned Idx = 0; 1775 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 1776 ++Idx; 1777 assert(Idx < Ops.size()); 1778 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 1779 // We found two constants, fold them together! 1780 Ops[0] = getConstant(LHSC->getValue()->getValue() + 1781 RHSC->getValue()->getValue()); 1782 if (Ops.size() == 2) return Ops[0]; 1783 Ops.erase(Ops.begin()+1); // Erase the folded element 1784 LHSC = cast<SCEVConstant>(Ops[0]); 1785 } 1786 1787 // If we are left with a constant zero being added, strip it off. 1788 if (LHSC->getValue()->isZero()) { 1789 Ops.erase(Ops.begin()); 1790 --Idx; 1791 } 1792 1793 if (Ops.size() == 1) return Ops[0]; 1794 } 1795 1796 // Okay, check to see if the same value occurs in the operand list more than 1797 // once. If so, merge them together into an multiply expression. Since we 1798 // sorted the list, these values are required to be adjacent. 1799 Type *Ty = Ops[0]->getType(); 1800 bool FoundMatch = false; 1801 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i) 1802 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2 1803 // Scan ahead to count how many equal operands there are. 1804 unsigned Count = 2; 1805 while (i+Count != e && Ops[i+Count] == Ops[i]) 1806 ++Count; 1807 // Merge the values into a multiply. 1808 const SCEV *Scale = getConstant(Ty, Count); 1809 const SCEV *Mul = getMulExpr(Scale, Ops[i]); 1810 if (Ops.size() == Count) 1811 return Mul; 1812 Ops[i] = Mul; 1813 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count); 1814 --i; e -= Count - 1; 1815 FoundMatch = true; 1816 } 1817 if (FoundMatch) 1818 return getAddExpr(Ops, Flags); 1819 1820 // Check for truncates. If all the operands are truncated from the same 1821 // type, see if factoring out the truncate would permit the result to be 1822 // folded. eg., trunc(x) + m*trunc(n) --> trunc(x + trunc(m)*n) 1823 // if the contents of the resulting outer trunc fold to something simple. 1824 for (; Idx < Ops.size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) { 1825 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Ops[Idx]); 1826 Type *DstType = Trunc->getType(); 1827 Type *SrcType = Trunc->getOperand()->getType(); 1828 SmallVector<const SCEV *, 8> LargeOps; 1829 bool Ok = true; 1830 // Check all the operands to see if they can be represented in the 1831 // source type of the truncate. 1832 for (unsigned i = 0, e = Ops.size(); i != e; ++i) { 1833 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Ops[i])) { 1834 if (T->getOperand()->getType() != SrcType) { 1835 Ok = false; 1836 break; 1837 } 1838 LargeOps.push_back(T->getOperand()); 1839 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) { 1840 LargeOps.push_back(getAnyExtendExpr(C, SrcType)); 1841 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) { 1842 SmallVector<const SCEV *, 8> LargeMulOps; 1843 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) { 1844 if (const SCEVTruncateExpr *T = 1845 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) { 1846 if (T->getOperand()->getType() != SrcType) { 1847 Ok = false; 1848 break; 1849 } 1850 LargeMulOps.push_back(T->getOperand()); 1851 } else if (const SCEVConstant *C = 1852 dyn_cast<SCEVConstant>(M->getOperand(j))) { 1853 LargeMulOps.push_back(getAnyExtendExpr(C, SrcType)); 1854 } else { 1855 Ok = false; 1856 break; 1857 } 1858 } 1859 if (Ok) 1860 LargeOps.push_back(getMulExpr(LargeMulOps)); 1861 } else { 1862 Ok = false; 1863 break; 1864 } 1865 } 1866 if (Ok) { 1867 // Evaluate the expression in the larger type. 1868 const SCEV *Fold = getAddExpr(LargeOps, Flags); 1869 // If it folds to something simple, use it. Otherwise, don't. 1870 if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold)) 1871 return getTruncateExpr(Fold, DstType); 1872 } 1873 } 1874 1875 // Skip past any other cast SCEVs. 1876 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr) 1877 ++Idx; 1878 1879 // If there are add operands they would be next. 1880 if (Idx < Ops.size()) { 1881 bool DeletedAdd = false; 1882 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) { 1883 // If we have an add, expand the add operands onto the end of the operands 1884 // list. 1885 Ops.erase(Ops.begin()+Idx); 1886 Ops.append(Add->op_begin(), Add->op_end()); 1887 DeletedAdd = true; 1888 } 1889 1890 // If we deleted at least one add, we added operands to the end of the list, 1891 // and they are not necessarily sorted. Recurse to resort and resimplify 1892 // any operands we just acquired. 1893 if (DeletedAdd) 1894 return getAddExpr(Ops); 1895 } 1896 1897 // Skip over the add expression until we get to a multiply. 1898 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr) 1899 ++Idx; 1900 1901 // Check to see if there are any folding opportunities present with 1902 // operands multiplied by constant values. 1903 if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) { 1904 uint64_t BitWidth = getTypeSizeInBits(Ty); 1905 DenseMap<const SCEV *, APInt> M; 1906 SmallVector<const SCEV *, 8> NewOps; 1907 APInt AccumulatedConstant(BitWidth, 0); 1908 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant, 1909 Ops.data(), Ops.size(), 1910 APInt(BitWidth, 1), *this)) { 1911 // Some interesting folding opportunity is present, so its worthwhile to 1912 // re-generate the operands list. Group the operands by constant scale, 1913 // to avoid multiplying by the same constant scale multiple times. 1914 std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists; 1915 for (SmallVectorImpl<const SCEV *>::const_iterator I = NewOps.begin(), 1916 E = NewOps.end(); I != E; ++I) 1917 MulOpLists[M.find(*I)->second].push_back(*I); 1918 // Re-generate the operands list. 1919 Ops.clear(); 1920 if (AccumulatedConstant != 0) 1921 Ops.push_back(getConstant(AccumulatedConstant)); 1922 for (std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare>::iterator 1923 I = MulOpLists.begin(), E = MulOpLists.end(); I != E; ++I) 1924 if (I->first != 0) 1925 Ops.push_back(getMulExpr(getConstant(I->first), 1926 getAddExpr(I->second))); 1927 if (Ops.empty()) 1928 return getConstant(Ty, 0); 1929 if (Ops.size() == 1) 1930 return Ops[0]; 1931 return getAddExpr(Ops); 1932 } 1933 } 1934 1935 // If we are adding something to a multiply expression, make sure the 1936 // something is not already an operand of the multiply. If so, merge it into 1937 // the multiply. 1938 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) { 1939 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]); 1940 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) { 1941 const SCEV *MulOpSCEV = Mul->getOperand(MulOp); 1942 if (isa<SCEVConstant>(MulOpSCEV)) 1943 continue; 1944 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) 1945 if (MulOpSCEV == Ops[AddOp]) { 1946 // Fold W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1)) 1947 const SCEV *InnerMul = Mul->getOperand(MulOp == 0); 1948 if (Mul->getNumOperands() != 2) { 1949 // If the multiply has more than two operands, we must get the 1950 // Y*Z term. 1951 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), 1952 Mul->op_begin()+MulOp); 1953 MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end()); 1954 InnerMul = getMulExpr(MulOps); 1955 } 1956 const SCEV *One = getConstant(Ty, 1); 1957 const SCEV *AddOne = getAddExpr(One, InnerMul); 1958 const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV); 1959 if (Ops.size() == 2) return OuterMul; 1960 if (AddOp < Idx) { 1961 Ops.erase(Ops.begin()+AddOp); 1962 Ops.erase(Ops.begin()+Idx-1); 1963 } else { 1964 Ops.erase(Ops.begin()+Idx); 1965 Ops.erase(Ops.begin()+AddOp-1); 1966 } 1967 Ops.push_back(OuterMul); 1968 return getAddExpr(Ops); 1969 } 1970 1971 // Check this multiply against other multiplies being added together. 1972 for (unsigned OtherMulIdx = Idx+1; 1973 OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]); 1974 ++OtherMulIdx) { 1975 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]); 1976 // If MulOp occurs in OtherMul, we can fold the two multiplies 1977 // together. 1978 for (unsigned OMulOp = 0, e = OtherMul->getNumOperands(); 1979 OMulOp != e; ++OMulOp) 1980 if (OtherMul->getOperand(OMulOp) == MulOpSCEV) { 1981 // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E)) 1982 const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0); 1983 if (Mul->getNumOperands() != 2) { 1984 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), 1985 Mul->op_begin()+MulOp); 1986 MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end()); 1987 InnerMul1 = getMulExpr(MulOps); 1988 } 1989 const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0); 1990 if (OtherMul->getNumOperands() != 2) { 1991 SmallVector<const SCEV *, 4> MulOps(OtherMul->op_begin(), 1992 OtherMul->op_begin()+OMulOp); 1993 MulOps.append(OtherMul->op_begin()+OMulOp+1, OtherMul->op_end()); 1994 InnerMul2 = getMulExpr(MulOps); 1995 } 1996 const SCEV *InnerMulSum = getAddExpr(InnerMul1,InnerMul2); 1997 const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum); 1998 if (Ops.size() == 2) return OuterMul; 1999 Ops.erase(Ops.begin()+Idx); 2000 Ops.erase(Ops.begin()+OtherMulIdx-1); 2001 Ops.push_back(OuterMul); 2002 return getAddExpr(Ops); 2003 } 2004 } 2005 } 2006 } 2007 2008 // If there are any add recurrences in the operands list, see if any other 2009 // added values are loop invariant. If so, we can fold them into the 2010 // recurrence. 2011 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr) 2012 ++Idx; 2013 2014 // Scan over all recurrences, trying to fold loop invariants into them. 2015 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) { 2016 // Scan all of the other operands to this add and add them to the vector if 2017 // they are loop invariant w.r.t. the recurrence. 2018 SmallVector<const SCEV *, 8> LIOps; 2019 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]); 2020 const Loop *AddRecLoop = AddRec->getLoop(); 2021 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2022 if (isLoopInvariant(Ops[i], AddRecLoop)) { 2023 LIOps.push_back(Ops[i]); 2024 Ops.erase(Ops.begin()+i); 2025 --i; --e; 2026 } 2027 2028 // If we found some loop invariants, fold them into the recurrence. 2029 if (!LIOps.empty()) { 2030 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step} 2031 LIOps.push_back(AddRec->getStart()); 2032 2033 SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(), 2034 AddRec->op_end()); 2035 AddRecOps[0] = getAddExpr(LIOps); 2036 2037 // Build the new addrec. Propagate the NUW and NSW flags if both the 2038 // outer add and the inner addrec are guaranteed to have no overflow. 2039 // Always propagate NW. 2040 Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW)); 2041 const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags); 2042 2043 // If all of the other operands were loop invariant, we are done. 2044 if (Ops.size() == 1) return NewRec; 2045 2046 // Otherwise, add the folded AddRec by the non-invariant parts. 2047 for (unsigned i = 0;; ++i) 2048 if (Ops[i] == AddRec) { 2049 Ops[i] = NewRec; 2050 break; 2051 } 2052 return getAddExpr(Ops); 2053 } 2054 2055 // Okay, if there weren't any loop invariants to be folded, check to see if 2056 // there are multiple AddRec's with the same loop induction variable being 2057 // added together. If so, we can fold them. 2058 for (unsigned OtherIdx = Idx+1; 2059 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2060 ++OtherIdx) 2061 if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) { 2062 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L> 2063 SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(), 2064 AddRec->op_end()); 2065 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2066 ++OtherIdx) 2067 if (const SCEVAddRecExpr *OtherAddRec = 2068 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx])) 2069 if (OtherAddRec->getLoop() == AddRecLoop) { 2070 for (unsigned i = 0, e = OtherAddRec->getNumOperands(); 2071 i != e; ++i) { 2072 if (i >= AddRecOps.size()) { 2073 AddRecOps.append(OtherAddRec->op_begin()+i, 2074 OtherAddRec->op_end()); 2075 break; 2076 } 2077 AddRecOps[i] = getAddExpr(AddRecOps[i], 2078 OtherAddRec->getOperand(i)); 2079 } 2080 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; 2081 } 2082 // Step size has changed, so we cannot guarantee no self-wraparound. 2083 Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap); 2084 return getAddExpr(Ops); 2085 } 2086 2087 // Otherwise couldn't fold anything into this recurrence. Move onto the 2088 // next one. 2089 } 2090 2091 // Okay, it looks like we really DO need an add expr. Check to see if we 2092 // already have one, otherwise create a new one. 2093 FoldingSetNodeID ID; 2094 ID.AddInteger(scAddExpr); 2095 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2096 ID.AddPointer(Ops[i]); 2097 void *IP = nullptr; 2098 SCEVAddExpr *S = 2099 static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2100 if (!S) { 2101 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2102 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2103 S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator), 2104 O, Ops.size()); 2105 UniqueSCEVs.InsertNode(S, IP); 2106 } 2107 S->setNoWrapFlags(Flags); 2108 return S; 2109 } 2110 2111 static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) { 2112 uint64_t k = i*j; 2113 if (j > 1 && k / j != i) Overflow = true; 2114 return k; 2115 } 2116 2117 /// Compute the result of "n choose k", the binomial coefficient. If an 2118 /// intermediate computation overflows, Overflow will be set and the return will 2119 /// be garbage. Overflow is not cleared on absence of overflow. 2120 static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) { 2121 // We use the multiplicative formula: 2122 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 . 2123 // At each iteration, we take the n-th term of the numeral and divide by the 2124 // (k-n)th term of the denominator. This division will always produce an 2125 // integral result, and helps reduce the chance of overflow in the 2126 // intermediate computations. However, we can still overflow even when the 2127 // final result would fit. 2128 2129 if (n == 0 || n == k) return 1; 2130 if (k > n) return 0; 2131 2132 if (k > n/2) 2133 k = n-k; 2134 2135 uint64_t r = 1; 2136 for (uint64_t i = 1; i <= k; ++i) { 2137 r = umul_ov(r, n-(i-1), Overflow); 2138 r /= i; 2139 } 2140 return r; 2141 } 2142 2143 /// Determine if any of the operands in this SCEV are a constant or if 2144 /// any of the add or multiply expressions in this SCEV contain a constant. 2145 static bool containsConstantSomewhere(const SCEV *StartExpr) { 2146 SmallVector<const SCEV *, 4> Ops; 2147 Ops.push_back(StartExpr); 2148 while (!Ops.empty()) { 2149 const SCEV *CurrentExpr = Ops.pop_back_val(); 2150 if (isa<SCEVConstant>(*CurrentExpr)) 2151 return true; 2152 2153 if (isa<SCEVAddExpr>(*CurrentExpr) || isa<SCEVMulExpr>(*CurrentExpr)) { 2154 const auto *CurrentNAry = cast<SCEVNAryExpr>(CurrentExpr); 2155 for (const SCEV *Operand : CurrentNAry->operands()) 2156 Ops.push_back(Operand); 2157 } 2158 } 2159 return false; 2160 } 2161 2162 /// getMulExpr - Get a canonical multiply expression, or something simpler if 2163 /// possible. 2164 const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops, 2165 SCEV::NoWrapFlags Flags) { 2166 assert(Flags == maskFlags(Flags, SCEV::FlagNUW | SCEV::FlagNSW) && 2167 "only nuw or nsw allowed"); 2168 assert(!Ops.empty() && "Cannot get empty mul!"); 2169 if (Ops.size() == 1) return Ops[0]; 2170 #ifndef NDEBUG 2171 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2172 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2173 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2174 "SCEVMulExpr operand types don't match!"); 2175 #endif 2176 2177 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 2178 // And vice-versa. 2179 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 2180 SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask); 2181 if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) { 2182 bool All = true; 2183 for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(), 2184 E = Ops.end(); I != E; ++I) 2185 if (!isKnownNonNegative(*I)) { 2186 All = false; 2187 break; 2188 } 2189 if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 2190 } 2191 2192 // Sort by complexity, this groups all similar expression types together. 2193 GroupByComplexity(Ops, LI); 2194 2195 // If there are any constants, fold them together. 2196 unsigned Idx = 0; 2197 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2198 2199 // C1*(C2+V) -> C1*C2 + C1*V 2200 if (Ops.size() == 2) 2201 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) 2202 // If any of Add's ops are Adds or Muls with a constant, 2203 // apply this transformation as well. 2204 if (Add->getNumOperands() == 2) 2205 if (containsConstantSomewhere(Add)) 2206 return getAddExpr(getMulExpr(LHSC, Add->getOperand(0)), 2207 getMulExpr(LHSC, Add->getOperand(1))); 2208 2209 ++Idx; 2210 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2211 // We found two constants, fold them together! 2212 ConstantInt *Fold = ConstantInt::get(getContext(), 2213 LHSC->getValue()->getValue() * 2214 RHSC->getValue()->getValue()); 2215 Ops[0] = getConstant(Fold); 2216 Ops.erase(Ops.begin()+1); // Erase the folded element 2217 if (Ops.size() == 1) return Ops[0]; 2218 LHSC = cast<SCEVConstant>(Ops[0]); 2219 } 2220 2221 // If we are left with a constant one being multiplied, strip it off. 2222 if (cast<SCEVConstant>(Ops[0])->getValue()->equalsInt(1)) { 2223 Ops.erase(Ops.begin()); 2224 --Idx; 2225 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isZero()) { 2226 // If we have a multiply of zero, it will always be zero. 2227 return Ops[0]; 2228 } else if (Ops[0]->isAllOnesValue()) { 2229 // If we have a mul by -1 of an add, try distributing the -1 among the 2230 // add operands. 2231 if (Ops.size() == 2) { 2232 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) { 2233 SmallVector<const SCEV *, 4> NewOps; 2234 bool AnyFolded = false; 2235 for (SCEVAddRecExpr::op_iterator I = Add->op_begin(), 2236 E = Add->op_end(); I != E; ++I) { 2237 const SCEV *Mul = getMulExpr(Ops[0], *I); 2238 if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true; 2239 NewOps.push_back(Mul); 2240 } 2241 if (AnyFolded) 2242 return getAddExpr(NewOps); 2243 } 2244 else if (const SCEVAddRecExpr * 2245 AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) { 2246 // Negation preserves a recurrence's no self-wrap property. 2247 SmallVector<const SCEV *, 4> Operands; 2248 for (SCEVAddRecExpr::op_iterator I = AddRec->op_begin(), 2249 E = AddRec->op_end(); I != E; ++I) { 2250 Operands.push_back(getMulExpr(Ops[0], *I)); 2251 } 2252 return getAddRecExpr(Operands, AddRec->getLoop(), 2253 AddRec->getNoWrapFlags(SCEV::FlagNW)); 2254 } 2255 } 2256 } 2257 2258 if (Ops.size() == 1) 2259 return Ops[0]; 2260 } 2261 2262 // Skip over the add expression until we get to a multiply. 2263 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr) 2264 ++Idx; 2265 2266 // If there are mul operands inline them all into this expression. 2267 if (Idx < Ops.size()) { 2268 bool DeletedMul = false; 2269 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) { 2270 // If we have an mul, expand the mul operands onto the end of the operands 2271 // list. 2272 Ops.erase(Ops.begin()+Idx); 2273 Ops.append(Mul->op_begin(), Mul->op_end()); 2274 DeletedMul = true; 2275 } 2276 2277 // If we deleted at least one mul, we added operands to the end of the list, 2278 // and they are not necessarily sorted. Recurse to resort and resimplify 2279 // any operands we just acquired. 2280 if (DeletedMul) 2281 return getMulExpr(Ops); 2282 } 2283 2284 // If there are any add recurrences in the operands list, see if any other 2285 // added values are loop invariant. If so, we can fold them into the 2286 // recurrence. 2287 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr) 2288 ++Idx; 2289 2290 // Scan over all recurrences, trying to fold loop invariants into them. 2291 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) { 2292 // Scan all of the other operands to this mul and add them to the vector if 2293 // they are loop invariant w.r.t. the recurrence. 2294 SmallVector<const SCEV *, 8> LIOps; 2295 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]); 2296 const Loop *AddRecLoop = AddRec->getLoop(); 2297 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2298 if (isLoopInvariant(Ops[i], AddRecLoop)) { 2299 LIOps.push_back(Ops[i]); 2300 Ops.erase(Ops.begin()+i); 2301 --i; --e; 2302 } 2303 2304 // If we found some loop invariants, fold them into the recurrence. 2305 if (!LIOps.empty()) { 2306 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step} 2307 SmallVector<const SCEV *, 4> NewOps; 2308 NewOps.reserve(AddRec->getNumOperands()); 2309 const SCEV *Scale = getMulExpr(LIOps); 2310 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) 2311 NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i))); 2312 2313 // Build the new addrec. Propagate the NUW and NSW flags if both the 2314 // outer mul and the inner addrec are guaranteed to have no overflow. 2315 // 2316 // No self-wrap cannot be guaranteed after changing the step size, but 2317 // will be inferred if either NUW or NSW is true. 2318 Flags = AddRec->getNoWrapFlags(clearFlags(Flags, SCEV::FlagNW)); 2319 const SCEV *NewRec = getAddRecExpr(NewOps, AddRecLoop, Flags); 2320 2321 // If all of the other operands were loop invariant, we are done. 2322 if (Ops.size() == 1) return NewRec; 2323 2324 // Otherwise, multiply the folded AddRec by the non-invariant parts. 2325 for (unsigned i = 0;; ++i) 2326 if (Ops[i] == AddRec) { 2327 Ops[i] = NewRec; 2328 break; 2329 } 2330 return getMulExpr(Ops); 2331 } 2332 2333 // Okay, if there weren't any loop invariants to be folded, check to see if 2334 // there are multiple AddRec's with the same loop induction variable being 2335 // multiplied together. If so, we can fold them. 2336 2337 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L> 2338 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [ 2339 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z 2340 // ]]],+,...up to x=2n}. 2341 // Note that the arguments to choose() are always integers with values 2342 // known at compile time, never SCEV objects. 2343 // 2344 // The implementation avoids pointless extra computations when the two 2345 // addrec's are of different length (mathematically, it's equivalent to 2346 // an infinite stream of zeros on the right). 2347 bool OpsModified = false; 2348 for (unsigned OtherIdx = Idx+1; 2349 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2350 ++OtherIdx) { 2351 const SCEVAddRecExpr *OtherAddRec = 2352 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]); 2353 if (!OtherAddRec || OtherAddRec->getLoop() != AddRecLoop) 2354 continue; 2355 2356 bool Overflow = false; 2357 Type *Ty = AddRec->getType(); 2358 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64; 2359 SmallVector<const SCEV*, 7> AddRecOps; 2360 for (int x = 0, xe = AddRec->getNumOperands() + 2361 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) { 2362 const SCEV *Term = getConstant(Ty, 0); 2363 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) { 2364 uint64_t Coeff1 = Choose(x, 2*x - y, Overflow); 2365 for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1), 2366 ze = std::min(x+1, (int)OtherAddRec->getNumOperands()); 2367 z < ze && !Overflow; ++z) { 2368 uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow); 2369 uint64_t Coeff; 2370 if (LargerThan64Bits) 2371 Coeff = umul_ov(Coeff1, Coeff2, Overflow); 2372 else 2373 Coeff = Coeff1*Coeff2; 2374 const SCEV *CoeffTerm = getConstant(Ty, Coeff); 2375 const SCEV *Term1 = AddRec->getOperand(y-z); 2376 const SCEV *Term2 = OtherAddRec->getOperand(z); 2377 Term = getAddExpr(Term, getMulExpr(CoeffTerm, Term1,Term2)); 2378 } 2379 } 2380 AddRecOps.push_back(Term); 2381 } 2382 if (!Overflow) { 2383 const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(), 2384 SCEV::FlagAnyWrap); 2385 if (Ops.size() == 2) return NewAddRec; 2386 Ops[Idx] = NewAddRec; 2387 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; 2388 OpsModified = true; 2389 AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec); 2390 if (!AddRec) 2391 break; 2392 } 2393 } 2394 if (OpsModified) 2395 return getMulExpr(Ops); 2396 2397 // Otherwise couldn't fold anything into this recurrence. Move onto the 2398 // next one. 2399 } 2400 2401 // Okay, it looks like we really DO need an mul expr. Check to see if we 2402 // already have one, otherwise create a new one. 2403 FoldingSetNodeID ID; 2404 ID.AddInteger(scMulExpr); 2405 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2406 ID.AddPointer(Ops[i]); 2407 void *IP = nullptr; 2408 SCEVMulExpr *S = 2409 static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2410 if (!S) { 2411 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2412 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2413 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator), 2414 O, Ops.size()); 2415 UniqueSCEVs.InsertNode(S, IP); 2416 } 2417 S->setNoWrapFlags(Flags); 2418 return S; 2419 } 2420 2421 /// getUDivExpr - Get a canonical unsigned division expression, or something 2422 /// simpler if possible. 2423 const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS, 2424 const SCEV *RHS) { 2425 assert(getEffectiveSCEVType(LHS->getType()) == 2426 getEffectiveSCEVType(RHS->getType()) && 2427 "SCEVUDivExpr operand types don't match!"); 2428 2429 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) { 2430 if (RHSC->getValue()->equalsInt(1)) 2431 return LHS; // X udiv 1 --> x 2432 // If the denominator is zero, the result of the udiv is undefined. Don't 2433 // try to analyze it, because the resolution chosen here may differ from 2434 // the resolution chosen in other parts of the compiler. 2435 if (!RHSC->getValue()->isZero()) { 2436 // Determine if the division can be folded into the operands of 2437 // its operands. 2438 // TODO: Generalize this to non-constants by using known-bits information. 2439 Type *Ty = LHS->getType(); 2440 unsigned LZ = RHSC->getValue()->getValue().countLeadingZeros(); 2441 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1; 2442 // For non-power-of-two values, effectively round the value up to the 2443 // nearest power of two. 2444 if (!RHSC->getValue()->getValue().isPowerOf2()) 2445 ++MaxShiftAmt; 2446 IntegerType *ExtTy = 2447 IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt); 2448 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) 2449 if (const SCEVConstant *Step = 2450 dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) { 2451 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded. 2452 const APInt &StepInt = Step->getValue()->getValue(); 2453 const APInt &DivInt = RHSC->getValue()->getValue(); 2454 if (!StepInt.urem(DivInt) && 2455 getZeroExtendExpr(AR, ExtTy) == 2456 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), 2457 getZeroExtendExpr(Step, ExtTy), 2458 AR->getLoop(), SCEV::FlagAnyWrap)) { 2459 SmallVector<const SCEV *, 4> Operands; 2460 for (unsigned i = 0, e = AR->getNumOperands(); i != e; ++i) 2461 Operands.push_back(getUDivExpr(AR->getOperand(i), RHS)); 2462 return getAddRecExpr(Operands, AR->getLoop(), 2463 SCEV::FlagNW); 2464 } 2465 /// Get a canonical UDivExpr for a recurrence. 2466 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0. 2467 // We can currently only fold X%N if X is constant. 2468 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart()); 2469 if (StartC && !DivInt.urem(StepInt) && 2470 getZeroExtendExpr(AR, ExtTy) == 2471 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), 2472 getZeroExtendExpr(Step, ExtTy), 2473 AR->getLoop(), SCEV::FlagAnyWrap)) { 2474 const APInt &StartInt = StartC->getValue()->getValue(); 2475 const APInt &StartRem = StartInt.urem(StepInt); 2476 if (StartRem != 0) 2477 LHS = getAddRecExpr(getConstant(StartInt - StartRem), Step, 2478 AR->getLoop(), SCEV::FlagNW); 2479 } 2480 } 2481 // (A*B)/C --> A*(B/C) if safe and B/C can be folded. 2482 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) { 2483 SmallVector<const SCEV *, 4> Operands; 2484 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) 2485 Operands.push_back(getZeroExtendExpr(M->getOperand(i), ExtTy)); 2486 if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands)) 2487 // Find an operand that's safely divisible. 2488 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 2489 const SCEV *Op = M->getOperand(i); 2490 const SCEV *Div = getUDivExpr(Op, RHSC); 2491 if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) { 2492 Operands = SmallVector<const SCEV *, 4>(M->op_begin(), 2493 M->op_end()); 2494 Operands[i] = Div; 2495 return getMulExpr(Operands); 2496 } 2497 } 2498 } 2499 // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded. 2500 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) { 2501 SmallVector<const SCEV *, 4> Operands; 2502 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) 2503 Operands.push_back(getZeroExtendExpr(A->getOperand(i), ExtTy)); 2504 if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) { 2505 Operands.clear(); 2506 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) { 2507 const SCEV *Op = getUDivExpr(A->getOperand(i), RHS); 2508 if (isa<SCEVUDivExpr>(Op) || 2509 getMulExpr(Op, RHS) != A->getOperand(i)) 2510 break; 2511 Operands.push_back(Op); 2512 } 2513 if (Operands.size() == A->getNumOperands()) 2514 return getAddExpr(Operands); 2515 } 2516 } 2517 2518 // Fold if both operands are constant. 2519 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { 2520 Constant *LHSCV = LHSC->getValue(); 2521 Constant *RHSCV = RHSC->getValue(); 2522 return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV, 2523 RHSCV))); 2524 } 2525 } 2526 } 2527 2528 FoldingSetNodeID ID; 2529 ID.AddInteger(scUDivExpr); 2530 ID.AddPointer(LHS); 2531 ID.AddPointer(RHS); 2532 void *IP = nullptr; 2533 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2534 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator), 2535 LHS, RHS); 2536 UniqueSCEVs.InsertNode(S, IP); 2537 return S; 2538 } 2539 2540 static const APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) { 2541 APInt A = C1->getValue()->getValue().abs(); 2542 APInt B = C2->getValue()->getValue().abs(); 2543 uint32_t ABW = A.getBitWidth(); 2544 uint32_t BBW = B.getBitWidth(); 2545 2546 if (ABW > BBW) 2547 B = B.zext(ABW); 2548 else if (ABW < BBW) 2549 A = A.zext(BBW); 2550 2551 return APIntOps::GreatestCommonDivisor(A, B); 2552 } 2553 2554 /// getUDivExactExpr - Get a canonical unsigned division expression, or 2555 /// something simpler if possible. There is no representation for an exact udiv 2556 /// in SCEV IR, but we can attempt to remove factors from the LHS and RHS. 2557 /// We can't do this when it's not exact because the udiv may be clearing bits. 2558 const SCEV *ScalarEvolution::getUDivExactExpr(const SCEV *LHS, 2559 const SCEV *RHS) { 2560 // TODO: we could try to find factors in all sorts of things, but for now we 2561 // just deal with u/exact (multiply, constant). See SCEVDivision towards the 2562 // end of this file for inspiration. 2563 2564 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS); 2565 if (!Mul) 2566 return getUDivExpr(LHS, RHS); 2567 2568 if (const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(RHS)) { 2569 // If the mulexpr multiplies by a constant, then that constant must be the 2570 // first element of the mulexpr. 2571 if (const SCEVConstant *LHSCst = 2572 dyn_cast<SCEVConstant>(Mul->getOperand(0))) { 2573 if (LHSCst == RHSCst) { 2574 SmallVector<const SCEV *, 2> Operands; 2575 Operands.append(Mul->op_begin() + 1, Mul->op_end()); 2576 return getMulExpr(Operands); 2577 } 2578 2579 // We can't just assume that LHSCst divides RHSCst cleanly, it could be 2580 // that there's a factor provided by one of the other terms. We need to 2581 // check. 2582 APInt Factor = gcd(LHSCst, RHSCst); 2583 if (!Factor.isIntN(1)) { 2584 LHSCst = cast<SCEVConstant>( 2585 getConstant(LHSCst->getValue()->getValue().udiv(Factor))); 2586 RHSCst = cast<SCEVConstant>( 2587 getConstant(RHSCst->getValue()->getValue().udiv(Factor))); 2588 SmallVector<const SCEV *, 2> Operands; 2589 Operands.push_back(LHSCst); 2590 Operands.append(Mul->op_begin() + 1, Mul->op_end()); 2591 LHS = getMulExpr(Operands); 2592 RHS = RHSCst; 2593 Mul = dyn_cast<SCEVMulExpr>(LHS); 2594 if (!Mul) 2595 return getUDivExactExpr(LHS, RHS); 2596 } 2597 } 2598 } 2599 2600 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) { 2601 if (Mul->getOperand(i) == RHS) { 2602 SmallVector<const SCEV *, 2> Operands; 2603 Operands.append(Mul->op_begin(), Mul->op_begin() + i); 2604 Operands.append(Mul->op_begin() + i + 1, Mul->op_end()); 2605 return getMulExpr(Operands); 2606 } 2607 } 2608 2609 return getUDivExpr(LHS, RHS); 2610 } 2611 2612 /// getAddRecExpr - Get an add recurrence expression for the specified loop. 2613 /// Simplify the expression as much as possible. 2614 const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step, 2615 const Loop *L, 2616 SCEV::NoWrapFlags Flags) { 2617 SmallVector<const SCEV *, 4> Operands; 2618 Operands.push_back(Start); 2619 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step)) 2620 if (StepChrec->getLoop() == L) { 2621 Operands.append(StepChrec->op_begin(), StepChrec->op_end()); 2622 return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW)); 2623 } 2624 2625 Operands.push_back(Step); 2626 return getAddRecExpr(Operands, L, Flags); 2627 } 2628 2629 /// getAddRecExpr - Get an add recurrence expression for the specified loop. 2630 /// Simplify the expression as much as possible. 2631 const SCEV * 2632 ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands, 2633 const Loop *L, SCEV::NoWrapFlags Flags) { 2634 if (Operands.size() == 1) return Operands[0]; 2635 #ifndef NDEBUG 2636 Type *ETy = getEffectiveSCEVType(Operands[0]->getType()); 2637 for (unsigned i = 1, e = Operands.size(); i != e; ++i) 2638 assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy && 2639 "SCEVAddRecExpr operand types don't match!"); 2640 for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2641 assert(isLoopInvariant(Operands[i], L) && 2642 "SCEVAddRecExpr operand is not loop-invariant!"); 2643 #endif 2644 2645 if (Operands.back()->isZero()) { 2646 Operands.pop_back(); 2647 return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0} --> X 2648 } 2649 2650 // It's tempting to want to call getMaxBackedgeTakenCount count here and 2651 // use that information to infer NUW and NSW flags. However, computing a 2652 // BE count requires calling getAddRecExpr, so we may not yet have a 2653 // meaningful BE count at this point (and if we don't, we'd be stuck 2654 // with a SCEVCouldNotCompute as the cached BE count). 2655 2656 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 2657 // And vice-versa. 2658 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 2659 SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask); 2660 if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) { 2661 bool All = true; 2662 for (SmallVectorImpl<const SCEV *>::const_iterator I = Operands.begin(), 2663 E = Operands.end(); I != E; ++I) 2664 if (!isKnownNonNegative(*I)) { 2665 All = false; 2666 break; 2667 } 2668 if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 2669 } 2670 2671 // Canonicalize nested AddRecs in by nesting them in order of loop depth. 2672 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) { 2673 const Loop *NestedLoop = NestedAR->getLoop(); 2674 if (L->contains(NestedLoop) ? 2675 (L->getLoopDepth() < NestedLoop->getLoopDepth()) : 2676 (!NestedLoop->contains(L) && 2677 DT->dominates(L->getHeader(), NestedLoop->getHeader()))) { 2678 SmallVector<const SCEV *, 4> NestedOperands(NestedAR->op_begin(), 2679 NestedAR->op_end()); 2680 Operands[0] = NestedAR->getStart(); 2681 // AddRecs require their operands be loop-invariant with respect to their 2682 // loops. Don't perform this transformation if it would break this 2683 // requirement. 2684 bool AllInvariant = true; 2685 for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2686 if (!isLoopInvariant(Operands[i], L)) { 2687 AllInvariant = false; 2688 break; 2689 } 2690 if (AllInvariant) { 2691 // Create a recurrence for the outer loop with the same step size. 2692 // 2693 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the 2694 // inner recurrence has the same property. 2695 SCEV::NoWrapFlags OuterFlags = 2696 maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags()); 2697 2698 NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags); 2699 AllInvariant = true; 2700 for (unsigned i = 0, e = NestedOperands.size(); i != e; ++i) 2701 if (!isLoopInvariant(NestedOperands[i], NestedLoop)) { 2702 AllInvariant = false; 2703 break; 2704 } 2705 if (AllInvariant) { 2706 // Ok, both add recurrences are valid after the transformation. 2707 // 2708 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if 2709 // the outer recurrence has the same property. 2710 SCEV::NoWrapFlags InnerFlags = 2711 maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags); 2712 return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags); 2713 } 2714 } 2715 // Reset Operands to its original state. 2716 Operands[0] = NestedAR; 2717 } 2718 } 2719 2720 // Okay, it looks like we really DO need an addrec expr. Check to see if we 2721 // already have one, otherwise create a new one. 2722 FoldingSetNodeID ID; 2723 ID.AddInteger(scAddRecExpr); 2724 for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2725 ID.AddPointer(Operands[i]); 2726 ID.AddPointer(L); 2727 void *IP = nullptr; 2728 SCEVAddRecExpr *S = 2729 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2730 if (!S) { 2731 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Operands.size()); 2732 std::uninitialized_copy(Operands.begin(), Operands.end(), O); 2733 S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator), 2734 O, Operands.size(), L); 2735 UniqueSCEVs.InsertNode(S, IP); 2736 } 2737 S->setNoWrapFlags(Flags); 2738 return S; 2739 } 2740 2741 const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS, 2742 const SCEV *RHS) { 2743 SmallVector<const SCEV *, 2> Ops; 2744 Ops.push_back(LHS); 2745 Ops.push_back(RHS); 2746 return getSMaxExpr(Ops); 2747 } 2748 2749 const SCEV * 2750 ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) { 2751 assert(!Ops.empty() && "Cannot get empty smax!"); 2752 if (Ops.size() == 1) return Ops[0]; 2753 #ifndef NDEBUG 2754 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2755 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2756 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2757 "SCEVSMaxExpr operand types don't match!"); 2758 #endif 2759 2760 // Sort by complexity, this groups all similar expression types together. 2761 GroupByComplexity(Ops, LI); 2762 2763 // If there are any constants, fold them together. 2764 unsigned Idx = 0; 2765 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2766 ++Idx; 2767 assert(Idx < Ops.size()); 2768 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2769 // We found two constants, fold them together! 2770 ConstantInt *Fold = ConstantInt::get(getContext(), 2771 APIntOps::smax(LHSC->getValue()->getValue(), 2772 RHSC->getValue()->getValue())); 2773 Ops[0] = getConstant(Fold); 2774 Ops.erase(Ops.begin()+1); // Erase the folded element 2775 if (Ops.size() == 1) return Ops[0]; 2776 LHSC = cast<SCEVConstant>(Ops[0]); 2777 } 2778 2779 // If we are left with a constant minimum-int, strip it off. 2780 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(true)) { 2781 Ops.erase(Ops.begin()); 2782 --Idx; 2783 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(true)) { 2784 // If we have an smax with a constant maximum-int, it will always be 2785 // maximum-int. 2786 return Ops[0]; 2787 } 2788 2789 if (Ops.size() == 1) return Ops[0]; 2790 } 2791 2792 // Find the first SMax 2793 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scSMaxExpr) 2794 ++Idx; 2795 2796 // Check to see if one of the operands is an SMax. If so, expand its operands 2797 // onto our operand list, and recurse to simplify. 2798 if (Idx < Ops.size()) { 2799 bool DeletedSMax = false; 2800 while (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(Ops[Idx])) { 2801 Ops.erase(Ops.begin()+Idx); 2802 Ops.append(SMax->op_begin(), SMax->op_end()); 2803 DeletedSMax = true; 2804 } 2805 2806 if (DeletedSMax) 2807 return getSMaxExpr(Ops); 2808 } 2809 2810 // Okay, check to see if the same value occurs in the operand list twice. If 2811 // so, delete one. Since we sorted the list, these values are required to 2812 // be adjacent. 2813 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) 2814 // X smax Y smax Y --> X smax Y 2815 // X smax Y --> X, if X is always greater than Y 2816 if (Ops[i] == Ops[i+1] || 2817 isKnownPredicate(ICmpInst::ICMP_SGE, Ops[i], Ops[i+1])) { 2818 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); 2819 --i; --e; 2820 } else if (isKnownPredicate(ICmpInst::ICMP_SLE, Ops[i], Ops[i+1])) { 2821 Ops.erase(Ops.begin()+i, Ops.begin()+i+1); 2822 --i; --e; 2823 } 2824 2825 if (Ops.size() == 1) return Ops[0]; 2826 2827 assert(!Ops.empty() && "Reduced smax down to nothing!"); 2828 2829 // Okay, it looks like we really DO need an smax expr. Check to see if we 2830 // already have one, otherwise create a new one. 2831 FoldingSetNodeID ID; 2832 ID.AddInteger(scSMaxExpr); 2833 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2834 ID.AddPointer(Ops[i]); 2835 void *IP = nullptr; 2836 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2837 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2838 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2839 SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator), 2840 O, Ops.size()); 2841 UniqueSCEVs.InsertNode(S, IP); 2842 return S; 2843 } 2844 2845 const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS, 2846 const SCEV *RHS) { 2847 SmallVector<const SCEV *, 2> Ops; 2848 Ops.push_back(LHS); 2849 Ops.push_back(RHS); 2850 return getUMaxExpr(Ops); 2851 } 2852 2853 const SCEV * 2854 ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) { 2855 assert(!Ops.empty() && "Cannot get empty umax!"); 2856 if (Ops.size() == 1) return Ops[0]; 2857 #ifndef NDEBUG 2858 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2859 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2860 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2861 "SCEVUMaxExpr operand types don't match!"); 2862 #endif 2863 2864 // Sort by complexity, this groups all similar expression types together. 2865 GroupByComplexity(Ops, LI); 2866 2867 // If there are any constants, fold them together. 2868 unsigned Idx = 0; 2869 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2870 ++Idx; 2871 assert(Idx < Ops.size()); 2872 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2873 // We found two constants, fold them together! 2874 ConstantInt *Fold = ConstantInt::get(getContext(), 2875 APIntOps::umax(LHSC->getValue()->getValue(), 2876 RHSC->getValue()->getValue())); 2877 Ops[0] = getConstant(Fold); 2878 Ops.erase(Ops.begin()+1); // Erase the folded element 2879 if (Ops.size() == 1) return Ops[0]; 2880 LHSC = cast<SCEVConstant>(Ops[0]); 2881 } 2882 2883 // If we are left with a constant minimum-int, strip it off. 2884 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(false)) { 2885 Ops.erase(Ops.begin()); 2886 --Idx; 2887 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(false)) { 2888 // If we have an umax with a constant maximum-int, it will always be 2889 // maximum-int. 2890 return Ops[0]; 2891 } 2892 2893 if (Ops.size() == 1) return Ops[0]; 2894 } 2895 2896 // Find the first UMax 2897 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scUMaxExpr) 2898 ++Idx; 2899 2900 // Check to see if one of the operands is a UMax. If so, expand its operands 2901 // onto our operand list, and recurse to simplify. 2902 if (Idx < Ops.size()) { 2903 bool DeletedUMax = false; 2904 while (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(Ops[Idx])) { 2905 Ops.erase(Ops.begin()+Idx); 2906 Ops.append(UMax->op_begin(), UMax->op_end()); 2907 DeletedUMax = true; 2908 } 2909 2910 if (DeletedUMax) 2911 return getUMaxExpr(Ops); 2912 } 2913 2914 // Okay, check to see if the same value occurs in the operand list twice. If 2915 // so, delete one. Since we sorted the list, these values are required to 2916 // be adjacent. 2917 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) 2918 // X umax Y umax Y --> X umax Y 2919 // X umax Y --> X, if X is always greater than Y 2920 if (Ops[i] == Ops[i+1] || 2921 isKnownPredicate(ICmpInst::ICMP_UGE, Ops[i], Ops[i+1])) { 2922 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); 2923 --i; --e; 2924 } else if (isKnownPredicate(ICmpInst::ICMP_ULE, Ops[i], Ops[i+1])) { 2925 Ops.erase(Ops.begin()+i, Ops.begin()+i+1); 2926 --i; --e; 2927 } 2928 2929 if (Ops.size() == 1) return Ops[0]; 2930 2931 assert(!Ops.empty() && "Reduced umax down to nothing!"); 2932 2933 // Okay, it looks like we really DO need a umax expr. Check to see if we 2934 // already have one, otherwise create a new one. 2935 FoldingSetNodeID ID; 2936 ID.AddInteger(scUMaxExpr); 2937 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2938 ID.AddPointer(Ops[i]); 2939 void *IP = nullptr; 2940 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2941 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2942 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2943 SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator), 2944 O, Ops.size()); 2945 UniqueSCEVs.InsertNode(S, IP); 2946 return S; 2947 } 2948 2949 const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS, 2950 const SCEV *RHS) { 2951 // ~smax(~x, ~y) == smin(x, y). 2952 return getNotSCEV(getSMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS))); 2953 } 2954 2955 const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, 2956 const SCEV *RHS) { 2957 // ~umax(~x, ~y) == umin(x, y) 2958 return getNotSCEV(getUMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS))); 2959 } 2960 2961 const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) { 2962 // If we have DataLayout, we can bypass creating a target-independent 2963 // constant expression and then folding it back into a ConstantInt. 2964 // This is just a compile-time optimization. 2965 if (DL) 2966 return getConstant(IntTy, DL->getTypeAllocSize(AllocTy)); 2967 2968 Constant *C = ConstantExpr::getSizeOf(AllocTy); 2969 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) 2970 if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI)) 2971 C = Folded; 2972 Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy)); 2973 assert(Ty == IntTy && "Effective SCEV type doesn't match"); 2974 return getTruncateOrZeroExtend(getSCEV(C), Ty); 2975 } 2976 2977 const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy, 2978 StructType *STy, 2979 unsigned FieldNo) { 2980 // If we have DataLayout, we can bypass creating a target-independent 2981 // constant expression and then folding it back into a ConstantInt. 2982 // This is just a compile-time optimization. 2983 if (DL) { 2984 return getConstant(IntTy, 2985 DL->getStructLayout(STy)->getElementOffset(FieldNo)); 2986 } 2987 2988 Constant *C = ConstantExpr::getOffsetOf(STy, FieldNo); 2989 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) 2990 if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI)) 2991 C = Folded; 2992 2993 Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(STy)); 2994 return getTruncateOrZeroExtend(getSCEV(C), Ty); 2995 } 2996 2997 const SCEV *ScalarEvolution::getUnknown(Value *V) { 2998 // Don't attempt to do anything other than create a SCEVUnknown object 2999 // here. createSCEV only calls getUnknown after checking for all other 3000 // interesting possibilities, and any other code that calls getUnknown 3001 // is doing so in order to hide a value from SCEV canonicalization. 3002 3003 FoldingSetNodeID ID; 3004 ID.AddInteger(scUnknown); 3005 ID.AddPointer(V); 3006 void *IP = nullptr; 3007 if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) { 3008 assert(cast<SCEVUnknown>(S)->getValue() == V && 3009 "Stale SCEVUnknown in uniquing map!"); 3010 return S; 3011 } 3012 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this, 3013 FirstUnknown); 3014 FirstUnknown = cast<SCEVUnknown>(S); 3015 UniqueSCEVs.InsertNode(S, IP); 3016 return S; 3017 } 3018 3019 //===----------------------------------------------------------------------===// 3020 // Basic SCEV Analysis and PHI Idiom Recognition Code 3021 // 3022 3023 /// isSCEVable - Test if values of the given type are analyzable within 3024 /// the SCEV framework. This primarily includes integer types, and it 3025 /// can optionally include pointer types if the ScalarEvolution class 3026 /// has access to target-specific information. 3027 bool ScalarEvolution::isSCEVable(Type *Ty) const { 3028 // Integers and pointers are always SCEVable. 3029 return Ty->isIntegerTy() || Ty->isPointerTy(); 3030 } 3031 3032 /// getTypeSizeInBits - Return the size in bits of the specified type, 3033 /// for which isSCEVable must return true. 3034 uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const { 3035 assert(isSCEVable(Ty) && "Type is not SCEVable!"); 3036 3037 // If we have a DataLayout, use it! 3038 if (DL) 3039 return DL->getTypeSizeInBits(Ty); 3040 3041 // Integer types have fixed sizes. 3042 if (Ty->isIntegerTy()) 3043 return Ty->getPrimitiveSizeInBits(); 3044 3045 // The only other support type is pointer. Without DataLayout, conservatively 3046 // assume pointers are 64-bit. 3047 assert(Ty->isPointerTy() && "isSCEVable permitted a non-SCEVable type!"); 3048 return 64; 3049 } 3050 3051 /// getEffectiveSCEVType - Return a type with the same bitwidth as 3052 /// the given type and which represents how SCEV will treat the given 3053 /// type, for which isSCEVable must return true. For pointer types, 3054 /// this is the pointer-sized integer type. 3055 Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const { 3056 assert(isSCEVable(Ty) && "Type is not SCEVable!"); 3057 3058 if (Ty->isIntegerTy()) { 3059 return Ty; 3060 } 3061 3062 // The only other support type is pointer. 3063 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!"); 3064 3065 if (DL) 3066 return DL->getIntPtrType(Ty); 3067 3068 // Without DataLayout, conservatively assume pointers are 64-bit. 3069 return Type::getInt64Ty(getContext()); 3070 } 3071 3072 const SCEV *ScalarEvolution::getCouldNotCompute() { 3073 return &CouldNotCompute; 3074 } 3075 3076 namespace { 3077 // Helper class working with SCEVTraversal to figure out if a SCEV contains 3078 // a SCEVUnknown with null value-pointer. FindInvalidSCEVUnknown::FindOne 3079 // is set iff if find such SCEVUnknown. 3080 // 3081 struct FindInvalidSCEVUnknown { 3082 bool FindOne; 3083 FindInvalidSCEVUnknown() { FindOne = false; } 3084 bool follow(const SCEV *S) { 3085 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 3086 case scConstant: 3087 return false; 3088 case scUnknown: 3089 if (!cast<SCEVUnknown>(S)->getValue()) 3090 FindOne = true; 3091 return false; 3092 default: 3093 return true; 3094 } 3095 } 3096 bool isDone() const { return FindOne; } 3097 }; 3098 } 3099 3100 bool ScalarEvolution::checkValidity(const SCEV *S) const { 3101 FindInvalidSCEVUnknown F; 3102 SCEVTraversal<FindInvalidSCEVUnknown> ST(F); 3103 ST.visitAll(S); 3104 3105 return !F.FindOne; 3106 } 3107 3108 /// getSCEV - Return an existing SCEV if it exists, otherwise analyze the 3109 /// expression and create a new one. 3110 const SCEV *ScalarEvolution::getSCEV(Value *V) { 3111 assert(isSCEVable(V->getType()) && "Value is not SCEVable!"); 3112 3113 ValueExprMapType::iterator I = ValueExprMap.find_as(V); 3114 if (I != ValueExprMap.end()) { 3115 const SCEV *S = I->second; 3116 if (checkValidity(S)) 3117 return S; 3118 else 3119 ValueExprMap.erase(I); 3120 } 3121 const SCEV *S = createSCEV(V); 3122 3123 // The process of creating a SCEV for V may have caused other SCEVs 3124 // to have been created, so it's necessary to insert the new entry 3125 // from scratch, rather than trying to remember the insert position 3126 // above. 3127 ValueExprMap.insert(std::make_pair(SCEVCallbackVH(V, this), S)); 3128 return S; 3129 } 3130 3131 /// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V 3132 /// 3133 const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V) { 3134 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) 3135 return getConstant( 3136 cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue()))); 3137 3138 Type *Ty = V->getType(); 3139 Ty = getEffectiveSCEVType(Ty); 3140 return getMulExpr(V, 3141 getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty)))); 3142 } 3143 3144 /// getNotSCEV - Return a SCEV corresponding to ~V = -1-V 3145 const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) { 3146 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) 3147 return getConstant( 3148 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue()))); 3149 3150 Type *Ty = V->getType(); 3151 Ty = getEffectiveSCEVType(Ty); 3152 const SCEV *AllOnes = 3153 getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))); 3154 return getMinusSCEV(AllOnes, V); 3155 } 3156 3157 /// getMinusSCEV - Return LHS-RHS. Minus is represented in SCEV as A+B*-1. 3158 const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS, 3159 SCEV::NoWrapFlags Flags) { 3160 assert(!maskFlags(Flags, SCEV::FlagNUW) && "subtraction does not have NUW"); 3161 3162 // Fast path: X - X --> 0. 3163 if (LHS == RHS) 3164 return getConstant(LHS->getType(), 0); 3165 3166 // X - Y --> X + -Y 3167 return getAddExpr(LHS, getNegativeSCEV(RHS), Flags); 3168 } 3169 3170 /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the 3171 /// input value to the specified type. If the type must be extended, it is zero 3172 /// extended. 3173 const SCEV * 3174 ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty) { 3175 Type *SrcTy = V->getType(); 3176 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3177 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3178 "Cannot truncate or zero extend with non-integer arguments!"); 3179 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3180 return V; // No conversion 3181 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) 3182 return getTruncateExpr(V, Ty); 3183 return getZeroExtendExpr(V, Ty); 3184 } 3185 3186 /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion of the 3187 /// input value to the specified type. If the type must be extended, it is sign 3188 /// extended. 3189 const SCEV * 3190 ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, 3191 Type *Ty) { 3192 Type *SrcTy = V->getType(); 3193 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3194 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3195 "Cannot truncate or zero extend with non-integer arguments!"); 3196 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3197 return V; // No conversion 3198 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) 3199 return getTruncateExpr(V, Ty); 3200 return getSignExtendExpr(V, Ty); 3201 } 3202 3203 /// getNoopOrZeroExtend - Return a SCEV corresponding to a conversion of the 3204 /// input value to the specified type. If the type must be extended, it is zero 3205 /// extended. The conversion must not be narrowing. 3206 const SCEV * 3207 ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) { 3208 Type *SrcTy = V->getType(); 3209 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3210 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3211 "Cannot noop or zero extend with non-integer arguments!"); 3212 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 3213 "getNoopOrZeroExtend cannot truncate!"); 3214 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3215 return V; // No conversion 3216 return getZeroExtendExpr(V, Ty); 3217 } 3218 3219 /// getNoopOrSignExtend - Return a SCEV corresponding to a conversion of the 3220 /// input value to the specified type. If the type must be extended, it is sign 3221 /// extended. The conversion must not be narrowing. 3222 const SCEV * 3223 ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) { 3224 Type *SrcTy = V->getType(); 3225 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3226 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3227 "Cannot noop or sign extend with non-integer arguments!"); 3228 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 3229 "getNoopOrSignExtend cannot truncate!"); 3230 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3231 return V; // No conversion 3232 return getSignExtendExpr(V, Ty); 3233 } 3234 3235 /// getNoopOrAnyExtend - Return a SCEV corresponding to a conversion of 3236 /// the input value to the specified type. If the type must be extended, 3237 /// it is extended with unspecified bits. The conversion must not be 3238 /// narrowing. 3239 const SCEV * 3240 ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) { 3241 Type *SrcTy = V->getType(); 3242 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3243 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3244 "Cannot noop or any extend with non-integer arguments!"); 3245 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 3246 "getNoopOrAnyExtend cannot truncate!"); 3247 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3248 return V; // No conversion 3249 return getAnyExtendExpr(V, Ty); 3250 } 3251 3252 /// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the 3253 /// input value to the specified type. The conversion must not be widening. 3254 const SCEV * 3255 ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) { 3256 Type *SrcTy = V->getType(); 3257 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3258 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3259 "Cannot truncate or noop with non-integer arguments!"); 3260 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) && 3261 "getTruncateOrNoop cannot extend!"); 3262 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3263 return V; // No conversion 3264 return getTruncateExpr(V, Ty); 3265 } 3266 3267 /// getUMaxFromMismatchedTypes - Promote the operands to the wider of 3268 /// the types using zero-extension, and then perform a umax operation 3269 /// with them. 3270 const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS, 3271 const SCEV *RHS) { 3272 const SCEV *PromotedLHS = LHS; 3273 const SCEV *PromotedRHS = RHS; 3274 3275 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType())) 3276 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType()); 3277 else 3278 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType()); 3279 3280 return getUMaxExpr(PromotedLHS, PromotedRHS); 3281 } 3282 3283 /// getUMinFromMismatchedTypes - Promote the operands to the wider of 3284 /// the types using zero-extension, and then perform a umin operation 3285 /// with them. 3286 const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS, 3287 const SCEV *RHS) { 3288 const SCEV *PromotedLHS = LHS; 3289 const SCEV *PromotedRHS = RHS; 3290 3291 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType())) 3292 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType()); 3293 else 3294 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType()); 3295 3296 return getUMinExpr(PromotedLHS, PromotedRHS); 3297 } 3298 3299 /// getPointerBase - Transitively follow the chain of pointer-type operands 3300 /// until reaching a SCEV that does not have a single pointer operand. This 3301 /// returns a SCEVUnknown pointer for well-formed pointer-type expressions, 3302 /// but corner cases do exist. 3303 const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) { 3304 // A pointer operand may evaluate to a nonpointer expression, such as null. 3305 if (!V->getType()->isPointerTy()) 3306 return V; 3307 3308 if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) { 3309 return getPointerBase(Cast->getOperand()); 3310 } 3311 else if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) { 3312 const SCEV *PtrOp = nullptr; 3313 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 3314 I != E; ++I) { 3315 if ((*I)->getType()->isPointerTy()) { 3316 // Cannot find the base of an expression with multiple pointer operands. 3317 if (PtrOp) 3318 return V; 3319 PtrOp = *I; 3320 } 3321 } 3322 if (!PtrOp) 3323 return V; 3324 return getPointerBase(PtrOp); 3325 } 3326 return V; 3327 } 3328 3329 /// PushDefUseChildren - Push users of the given Instruction 3330 /// onto the given Worklist. 3331 static void 3332 PushDefUseChildren(Instruction *I, 3333 SmallVectorImpl<Instruction *> &Worklist) { 3334 // Push the def-use children onto the Worklist stack. 3335 for (User *U : I->users()) 3336 Worklist.push_back(cast<Instruction>(U)); 3337 } 3338 3339 /// ForgetSymbolicValue - This looks up computed SCEV values for all 3340 /// instructions that depend on the given instruction and removes them from 3341 /// the ValueExprMapType map if they reference SymName. This is used during PHI 3342 /// resolution. 3343 void 3344 ScalarEvolution::ForgetSymbolicName(Instruction *PN, const SCEV *SymName) { 3345 SmallVector<Instruction *, 16> Worklist; 3346 PushDefUseChildren(PN, Worklist); 3347 3348 SmallPtrSet<Instruction *, 8> Visited; 3349 Visited.insert(PN); 3350 while (!Worklist.empty()) { 3351 Instruction *I = Worklist.pop_back_val(); 3352 if (!Visited.insert(I).second) 3353 continue; 3354 3355 ValueExprMapType::iterator It = 3356 ValueExprMap.find_as(static_cast<Value *>(I)); 3357 if (It != ValueExprMap.end()) { 3358 const SCEV *Old = It->second; 3359 3360 // Short-circuit the def-use traversal if the symbolic name 3361 // ceases to appear in expressions. 3362 if (Old != SymName && !hasOperand(Old, SymName)) 3363 continue; 3364 3365 // SCEVUnknown for a PHI either means that it has an unrecognized 3366 // structure, it's a PHI that's in the progress of being computed 3367 // by createNodeForPHI, or it's a single-value PHI. In the first case, 3368 // additional loop trip count information isn't going to change anything. 3369 // In the second case, createNodeForPHI will perform the necessary 3370 // updates on its own when it gets to that point. In the third, we do 3371 // want to forget the SCEVUnknown. 3372 if (!isa<PHINode>(I) || 3373 !isa<SCEVUnknown>(Old) || 3374 (I != PN && Old == SymName)) { 3375 forgetMemoizedResults(Old); 3376 ValueExprMap.erase(It); 3377 } 3378 } 3379 3380 PushDefUseChildren(I, Worklist); 3381 } 3382 } 3383 3384 /// createNodeForPHI - PHI nodes have two cases. Either the PHI node exists in 3385 /// a loop header, making it a potential recurrence, or it doesn't. 3386 /// 3387 const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { 3388 if (const Loop *L = LI->getLoopFor(PN->getParent())) 3389 if (L->getHeader() == PN->getParent()) { 3390 // The loop may have multiple entrances or multiple exits; we can analyze 3391 // this phi as an addrec if it has a unique entry value and a unique 3392 // backedge value. 3393 Value *BEValueV = nullptr, *StartValueV = nullptr; 3394 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 3395 Value *V = PN->getIncomingValue(i); 3396 if (L->contains(PN->getIncomingBlock(i))) { 3397 if (!BEValueV) { 3398 BEValueV = V; 3399 } else if (BEValueV != V) { 3400 BEValueV = nullptr; 3401 break; 3402 } 3403 } else if (!StartValueV) { 3404 StartValueV = V; 3405 } else if (StartValueV != V) { 3406 StartValueV = nullptr; 3407 break; 3408 } 3409 } 3410 if (BEValueV && StartValueV) { 3411 // While we are analyzing this PHI node, handle its value symbolically. 3412 const SCEV *SymbolicName = getUnknown(PN); 3413 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() && 3414 "PHI node already processed?"); 3415 ValueExprMap.insert(std::make_pair(SCEVCallbackVH(PN, this), SymbolicName)); 3416 3417 // Using this symbolic name for the PHI, analyze the value coming around 3418 // the back-edge. 3419 const SCEV *BEValue = getSCEV(BEValueV); 3420 3421 // NOTE: If BEValue is loop invariant, we know that the PHI node just 3422 // has a special value for the first iteration of the loop. 3423 3424 // If the value coming around the backedge is an add with the symbolic 3425 // value we just inserted, then we found a simple induction variable! 3426 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) { 3427 // If there is a single occurrence of the symbolic value, replace it 3428 // with a recurrence. 3429 unsigned FoundIndex = Add->getNumOperands(); 3430 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i) 3431 if (Add->getOperand(i) == SymbolicName) 3432 if (FoundIndex == e) { 3433 FoundIndex = i; 3434 break; 3435 } 3436 3437 if (FoundIndex != Add->getNumOperands()) { 3438 // Create an add with everything but the specified operand. 3439 SmallVector<const SCEV *, 8> Ops; 3440 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i) 3441 if (i != FoundIndex) 3442 Ops.push_back(Add->getOperand(i)); 3443 const SCEV *Accum = getAddExpr(Ops); 3444 3445 // This is not a valid addrec if the step amount is varying each 3446 // loop iteration, but is not itself an addrec in this loop. 3447 if (isLoopInvariant(Accum, L) || 3448 (isa<SCEVAddRecExpr>(Accum) && 3449 cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) { 3450 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; 3451 3452 // If the increment doesn't overflow, then neither the addrec nor 3453 // the post-increment will overflow. 3454 if (const AddOperator *OBO = dyn_cast<AddOperator>(BEValueV)) { 3455 if (OBO->hasNoUnsignedWrap()) 3456 Flags = setFlags(Flags, SCEV::FlagNUW); 3457 if (OBO->hasNoSignedWrap()) 3458 Flags = setFlags(Flags, SCEV::FlagNSW); 3459 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(BEValueV)) { 3460 // If the increment is an inbounds GEP, then we know the address 3461 // space cannot be wrapped around. We cannot make any guarantee 3462 // about signed or unsigned overflow because pointers are 3463 // unsigned but we may have a negative index from the base 3464 // pointer. We can guarantee that no unsigned wrap occurs if the 3465 // indices form a positive value. 3466 if (GEP->isInBounds()) { 3467 Flags = setFlags(Flags, SCEV::FlagNW); 3468 3469 const SCEV *Ptr = getSCEV(GEP->getPointerOperand()); 3470 if (isKnownPositive(getMinusSCEV(getSCEV(GEP), Ptr))) 3471 Flags = setFlags(Flags, SCEV::FlagNUW); 3472 } 3473 } else if (const SubOperator *OBO = 3474 dyn_cast<SubOperator>(BEValueV)) { 3475 if (OBO->hasNoUnsignedWrap()) 3476 Flags = setFlags(Flags, SCEV::FlagNUW); 3477 if (OBO->hasNoSignedWrap()) 3478 Flags = setFlags(Flags, SCEV::FlagNSW); 3479 } 3480 3481 const SCEV *StartVal = getSCEV(StartValueV); 3482 const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags); 3483 3484 // Since the no-wrap flags are on the increment, they apply to the 3485 // post-incremented value as well. 3486 if (isLoopInvariant(Accum, L)) 3487 (void)getAddRecExpr(getAddExpr(StartVal, Accum), 3488 Accum, L, Flags); 3489 3490 // Okay, for the entire analysis of this edge we assumed the PHI 3491 // to be symbolic. We now need to go back and purge all of the 3492 // entries for the scalars that use the symbolic expression. 3493 ForgetSymbolicName(PN, SymbolicName); 3494 ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV; 3495 return PHISCEV; 3496 } 3497 } 3498 } else if (const SCEVAddRecExpr *AddRec = 3499 dyn_cast<SCEVAddRecExpr>(BEValue)) { 3500 // Otherwise, this could be a loop like this: 3501 // i = 0; for (j = 1; ..; ++j) { .... i = j; } 3502 // In this case, j = {1,+,1} and BEValue is j. 3503 // Because the other in-value of i (0) fits the evolution of BEValue 3504 // i really is an addrec evolution. 3505 if (AddRec->getLoop() == L && AddRec->isAffine()) { 3506 const SCEV *StartVal = getSCEV(StartValueV); 3507 3508 // If StartVal = j.start - j.stride, we can use StartVal as the 3509 // initial step of the addrec evolution. 3510 if (StartVal == getMinusSCEV(AddRec->getOperand(0), 3511 AddRec->getOperand(1))) { 3512 // FIXME: For constant StartVal, we should be able to infer 3513 // no-wrap flags. 3514 const SCEV *PHISCEV = 3515 getAddRecExpr(StartVal, AddRec->getOperand(1), L, 3516 SCEV::FlagAnyWrap); 3517 3518 // Okay, for the entire analysis of this edge we assumed the PHI 3519 // to be symbolic. We now need to go back and purge all of the 3520 // entries for the scalars that use the symbolic expression. 3521 ForgetSymbolicName(PN, SymbolicName); 3522 ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV; 3523 return PHISCEV; 3524 } 3525 } 3526 } 3527 } 3528 } 3529 3530 // If the PHI has a single incoming value, follow that value, unless the 3531 // PHI's incoming blocks are in a different loop, in which case doing so 3532 // risks breaking LCSSA form. Instcombine would normally zap these, but 3533 // it doesn't have DominatorTree information, so it may miss cases. 3534 if (Value *V = SimplifyInstruction(PN, DL, TLI, DT, AT)) 3535 if (LI->replacementPreservesLCSSAForm(PN, V)) 3536 return getSCEV(V); 3537 3538 // If it's not a loop phi, we can't handle it yet. 3539 return getUnknown(PN); 3540 } 3541 3542 /// createNodeForGEP - Expand GEP instructions into add and multiply 3543 /// operations. This allows them to be analyzed by regular SCEV code. 3544 /// 3545 const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) { 3546 Type *IntPtrTy = getEffectiveSCEVType(GEP->getType()); 3547 Value *Base = GEP->getOperand(0); 3548 // Don't attempt to analyze GEPs over unsized objects. 3549 if (!Base->getType()->getPointerElementType()->isSized()) 3550 return getUnknown(GEP); 3551 3552 // Don't blindly transfer the inbounds flag from the GEP instruction to the 3553 // Add expression, because the Instruction may be guarded by control flow 3554 // and the no-overflow bits may not be valid for the expression in any 3555 // context. 3556 SCEV::NoWrapFlags Wrap = GEP->isInBounds() ? SCEV::FlagNSW : SCEV::FlagAnyWrap; 3557 3558 const SCEV *TotalOffset = getConstant(IntPtrTy, 0); 3559 gep_type_iterator GTI = gep_type_begin(GEP); 3560 for (GetElementPtrInst::op_iterator I = std::next(GEP->op_begin()), 3561 E = GEP->op_end(); 3562 I != E; ++I) { 3563 Value *Index = *I; 3564 // Compute the (potentially symbolic) offset in bytes for this index. 3565 if (StructType *STy = dyn_cast<StructType>(*GTI++)) { 3566 // For a struct, add the member offset. 3567 unsigned FieldNo = cast<ConstantInt>(Index)->getZExtValue(); 3568 const SCEV *FieldOffset = getOffsetOfExpr(IntPtrTy, STy, FieldNo); 3569 3570 // Add the field offset to the running total offset. 3571 TotalOffset = getAddExpr(TotalOffset, FieldOffset); 3572 } else { 3573 // For an array, add the element offset, explicitly scaled. 3574 const SCEV *ElementSize = getSizeOfExpr(IntPtrTy, *GTI); 3575 const SCEV *IndexS = getSCEV(Index); 3576 // Getelementptr indices are signed. 3577 IndexS = getTruncateOrSignExtend(IndexS, IntPtrTy); 3578 3579 // Multiply the index by the element size to compute the element offset. 3580 const SCEV *LocalOffset = getMulExpr(IndexS, ElementSize, Wrap); 3581 3582 // Add the element offset to the running total offset. 3583 TotalOffset = getAddExpr(TotalOffset, LocalOffset); 3584 } 3585 } 3586 3587 // Get the SCEV for the GEP base. 3588 const SCEV *BaseS = getSCEV(Base); 3589 3590 // Add the total offset from all the GEP indices to the base. 3591 return getAddExpr(BaseS, TotalOffset, Wrap); 3592 } 3593 3594 /// GetMinTrailingZeros - Determine the minimum number of zero bits that S is 3595 /// guaranteed to end in (at every loop iteration). It is, at the same time, 3596 /// the minimum number of times S is divisible by 2. For example, given {4,+,8} 3597 /// it returns 2. If S is guaranteed to be 0, it returns the bitwidth of S. 3598 uint32_t 3599 ScalarEvolution::GetMinTrailingZeros(const SCEV *S) { 3600 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 3601 return C->getValue()->getValue().countTrailingZeros(); 3602 3603 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(S)) 3604 return std::min(GetMinTrailingZeros(T->getOperand()), 3605 (uint32_t)getTypeSizeInBits(T->getType())); 3606 3607 if (const SCEVZeroExtendExpr *E = dyn_cast<SCEVZeroExtendExpr>(S)) { 3608 uint32_t OpRes = GetMinTrailingZeros(E->getOperand()); 3609 return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ? 3610 getTypeSizeInBits(E->getType()) : OpRes; 3611 } 3612 3613 if (const SCEVSignExtendExpr *E = dyn_cast<SCEVSignExtendExpr>(S)) { 3614 uint32_t OpRes = GetMinTrailingZeros(E->getOperand()); 3615 return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ? 3616 getTypeSizeInBits(E->getType()) : OpRes; 3617 } 3618 3619 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(S)) { 3620 // The result is the min of all operands results. 3621 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0)); 3622 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i) 3623 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i))); 3624 return MinOpRes; 3625 } 3626 3627 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) { 3628 // The result is the sum of all operands results. 3629 uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0)); 3630 uint32_t BitWidth = getTypeSizeInBits(M->getType()); 3631 for (unsigned i = 1, e = M->getNumOperands(); 3632 SumOpRes != BitWidth && i != e; ++i) 3633 SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)), 3634 BitWidth); 3635 return SumOpRes; 3636 } 3637 3638 if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) { 3639 // The result is the min of all operands results. 3640 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0)); 3641 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i) 3642 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i))); 3643 return MinOpRes; 3644 } 3645 3646 if (const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) { 3647 // The result is the min of all operands results. 3648 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0)); 3649 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i) 3650 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i))); 3651 return MinOpRes; 3652 } 3653 3654 if (const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) { 3655 // The result is the min of all operands results. 3656 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0)); 3657 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i) 3658 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i))); 3659 return MinOpRes; 3660 } 3661 3662 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 3663 // For a SCEVUnknown, ask ValueTracking. 3664 unsigned BitWidth = getTypeSizeInBits(U->getType()); 3665 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); 3666 computeKnownBits(U->getValue(), Zeros, Ones, DL, 0, AT, nullptr, DT); 3667 return Zeros.countTrailingOnes(); 3668 } 3669 3670 // SCEVUDivExpr 3671 return 0; 3672 } 3673 3674 /// GetRangeFromMetadata - Helper method to assign a range to V from 3675 /// metadata present in the IR. 3676 static Optional<ConstantRange> GetRangeFromMetadata(Value *V) { 3677 if (Instruction *I = dyn_cast<Instruction>(V)) { 3678 if (MDNode *MD = I->getMetadata(LLVMContext::MD_range)) { 3679 ConstantRange TotalRange( 3680 cast<IntegerType>(I->getType())->getBitWidth(), false); 3681 3682 unsigned NumRanges = MD->getNumOperands() / 2; 3683 assert(NumRanges >= 1); 3684 3685 for (unsigned i = 0; i < NumRanges; ++i) { 3686 ConstantInt *Lower = 3687 mdconst::extract<ConstantInt>(MD->getOperand(2 * i + 0)); 3688 ConstantInt *Upper = 3689 mdconst::extract<ConstantInt>(MD->getOperand(2 * i + 1)); 3690 ConstantRange Range(Lower->getValue(), Upper->getValue()); 3691 TotalRange = TotalRange.unionWith(Range); 3692 } 3693 3694 return TotalRange; 3695 } 3696 } 3697 3698 return None; 3699 } 3700 3701 /// getUnsignedRange - Determine the unsigned range for a particular SCEV. 3702 /// 3703 ConstantRange 3704 ScalarEvolution::getUnsignedRange(const SCEV *S) { 3705 // See if we've computed this range already. 3706 DenseMap<const SCEV *, ConstantRange>::iterator I = UnsignedRanges.find(S); 3707 if (I != UnsignedRanges.end()) 3708 return I->second; 3709 3710 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 3711 return setUnsignedRange(C, ConstantRange(C->getValue()->getValue())); 3712 3713 unsigned BitWidth = getTypeSizeInBits(S->getType()); 3714 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true); 3715 3716 // If the value has known zeros, the maximum unsigned value will have those 3717 // known zeros as well. 3718 uint32_t TZ = GetMinTrailingZeros(S); 3719 if (TZ != 0) 3720 ConservativeResult = 3721 ConstantRange(APInt::getMinValue(BitWidth), 3722 APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1); 3723 3724 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 3725 ConstantRange X = getUnsignedRange(Add->getOperand(0)); 3726 for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) 3727 X = X.add(getUnsignedRange(Add->getOperand(i))); 3728 return setUnsignedRange(Add, ConservativeResult.intersectWith(X)); 3729 } 3730 3731 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 3732 ConstantRange X = getUnsignedRange(Mul->getOperand(0)); 3733 for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) 3734 X = X.multiply(getUnsignedRange(Mul->getOperand(i))); 3735 return setUnsignedRange(Mul, ConservativeResult.intersectWith(X)); 3736 } 3737 3738 if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) { 3739 ConstantRange X = getUnsignedRange(SMax->getOperand(0)); 3740 for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) 3741 X = X.smax(getUnsignedRange(SMax->getOperand(i))); 3742 return setUnsignedRange(SMax, ConservativeResult.intersectWith(X)); 3743 } 3744 3745 if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) { 3746 ConstantRange X = getUnsignedRange(UMax->getOperand(0)); 3747 for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) 3748 X = X.umax(getUnsignedRange(UMax->getOperand(i))); 3749 return setUnsignedRange(UMax, ConservativeResult.intersectWith(X)); 3750 } 3751 3752 if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) { 3753 ConstantRange X = getUnsignedRange(UDiv->getLHS()); 3754 ConstantRange Y = getUnsignedRange(UDiv->getRHS()); 3755 return setUnsignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y))); 3756 } 3757 3758 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) { 3759 ConstantRange X = getUnsignedRange(ZExt->getOperand()); 3760 return setUnsignedRange(ZExt, 3761 ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); 3762 } 3763 3764 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) { 3765 ConstantRange X = getUnsignedRange(SExt->getOperand()); 3766 return setUnsignedRange(SExt, 3767 ConservativeResult.intersectWith(X.signExtend(BitWidth))); 3768 } 3769 3770 if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) { 3771 ConstantRange X = getUnsignedRange(Trunc->getOperand()); 3772 return setUnsignedRange(Trunc, 3773 ConservativeResult.intersectWith(X.truncate(BitWidth))); 3774 } 3775 3776 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) { 3777 // If there's no unsigned wrap, the value will never be less than its 3778 // initial value. 3779 if (AddRec->getNoWrapFlags(SCEV::FlagNUW)) 3780 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(AddRec->getStart())) 3781 if (!C->getValue()->isZero()) 3782 ConservativeResult = 3783 ConservativeResult.intersectWith( 3784 ConstantRange(C->getValue()->getValue(), APInt(BitWidth, 0))); 3785 3786 // TODO: non-affine addrec 3787 if (AddRec->isAffine()) { 3788 Type *Ty = AddRec->getType(); 3789 const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop()); 3790 if (!isa<SCEVCouldNotCompute>(MaxBECount) && 3791 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) { 3792 MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty); 3793 3794 const SCEV *Start = AddRec->getStart(); 3795 const SCEV *Step = AddRec->getStepRecurrence(*this); 3796 3797 ConstantRange StartRange = getUnsignedRange(Start); 3798 ConstantRange StepRange = getSignedRange(Step); 3799 ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); 3800 ConstantRange EndRange = 3801 StartRange.add(MaxBECountRange.multiply(StepRange)); 3802 3803 // Check for overflow. This must be done with ConstantRange arithmetic 3804 // because we could be called from within the ScalarEvolution overflow 3805 // checking code. 3806 ConstantRange ExtStartRange = StartRange.zextOrTrunc(BitWidth*2+1); 3807 ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1); 3808 ConstantRange ExtMaxBECountRange = 3809 MaxBECountRange.zextOrTrunc(BitWidth*2+1); 3810 ConstantRange ExtEndRange = EndRange.zextOrTrunc(BitWidth*2+1); 3811 if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) != 3812 ExtEndRange) 3813 return setUnsignedRange(AddRec, ConservativeResult); 3814 3815 APInt Min = APIntOps::umin(StartRange.getUnsignedMin(), 3816 EndRange.getUnsignedMin()); 3817 APInt Max = APIntOps::umax(StartRange.getUnsignedMax(), 3818 EndRange.getUnsignedMax()); 3819 if (Min.isMinValue() && Max.isMaxValue()) 3820 return setUnsignedRange(AddRec, ConservativeResult); 3821 return setUnsignedRange(AddRec, 3822 ConservativeResult.intersectWith(ConstantRange(Min, Max+1))); 3823 } 3824 } 3825 3826 return setUnsignedRange(AddRec, ConservativeResult); 3827 } 3828 3829 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 3830 // Check if the IR explicitly contains !range metadata. 3831 Optional<ConstantRange> MDRange = GetRangeFromMetadata(U->getValue()); 3832 if (MDRange.hasValue()) 3833 ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue()); 3834 3835 // For a SCEVUnknown, ask ValueTracking. 3836 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); 3837 computeKnownBits(U->getValue(), Zeros, Ones, DL, 0, AT, nullptr, DT); 3838 if (Ones == ~Zeros + 1) 3839 return setUnsignedRange(U, ConservativeResult); 3840 return setUnsignedRange(U, 3841 ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1))); 3842 } 3843 3844 return setUnsignedRange(S, ConservativeResult); 3845 } 3846 3847 /// getSignedRange - Determine the signed range for a particular SCEV. 3848 /// 3849 ConstantRange 3850 ScalarEvolution::getSignedRange(const SCEV *S) { 3851 // See if we've computed this range already. 3852 DenseMap<const SCEV *, ConstantRange>::iterator I = SignedRanges.find(S); 3853 if (I != SignedRanges.end()) 3854 return I->second; 3855 3856 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 3857 return setSignedRange(C, ConstantRange(C->getValue()->getValue())); 3858 3859 unsigned BitWidth = getTypeSizeInBits(S->getType()); 3860 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true); 3861 3862 // If the value has known zeros, the maximum signed value will have those 3863 // known zeros as well. 3864 uint32_t TZ = GetMinTrailingZeros(S); 3865 if (TZ != 0) 3866 ConservativeResult = 3867 ConstantRange(APInt::getSignedMinValue(BitWidth), 3868 APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1); 3869 3870 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 3871 ConstantRange X = getSignedRange(Add->getOperand(0)); 3872 for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) 3873 X = X.add(getSignedRange(Add->getOperand(i))); 3874 return setSignedRange(Add, ConservativeResult.intersectWith(X)); 3875 } 3876 3877 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 3878 ConstantRange X = getSignedRange(Mul->getOperand(0)); 3879 for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) 3880 X = X.multiply(getSignedRange(Mul->getOperand(i))); 3881 return setSignedRange(Mul, ConservativeResult.intersectWith(X)); 3882 } 3883 3884 if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) { 3885 ConstantRange X = getSignedRange(SMax->getOperand(0)); 3886 for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) 3887 X = X.smax(getSignedRange(SMax->getOperand(i))); 3888 return setSignedRange(SMax, ConservativeResult.intersectWith(X)); 3889 } 3890 3891 if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) { 3892 ConstantRange X = getSignedRange(UMax->getOperand(0)); 3893 for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) 3894 X = X.umax(getSignedRange(UMax->getOperand(i))); 3895 return setSignedRange(UMax, ConservativeResult.intersectWith(X)); 3896 } 3897 3898 if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) { 3899 ConstantRange X = getSignedRange(UDiv->getLHS()); 3900 ConstantRange Y = getSignedRange(UDiv->getRHS()); 3901 return setSignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y))); 3902 } 3903 3904 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) { 3905 ConstantRange X = getSignedRange(ZExt->getOperand()); 3906 return setSignedRange(ZExt, 3907 ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); 3908 } 3909 3910 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) { 3911 ConstantRange X = getSignedRange(SExt->getOperand()); 3912 return setSignedRange(SExt, 3913 ConservativeResult.intersectWith(X.signExtend(BitWidth))); 3914 } 3915 3916 if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) { 3917 ConstantRange X = getSignedRange(Trunc->getOperand()); 3918 return setSignedRange(Trunc, 3919 ConservativeResult.intersectWith(X.truncate(BitWidth))); 3920 } 3921 3922 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) { 3923 // If there's no signed wrap, and all the operands have the same sign or 3924 // zero, the value won't ever change sign. 3925 if (AddRec->getNoWrapFlags(SCEV::FlagNSW)) { 3926 bool AllNonNeg = true; 3927 bool AllNonPos = true; 3928 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { 3929 if (!isKnownNonNegative(AddRec->getOperand(i))) AllNonNeg = false; 3930 if (!isKnownNonPositive(AddRec->getOperand(i))) AllNonPos = false; 3931 } 3932 if (AllNonNeg) 3933 ConservativeResult = ConservativeResult.intersectWith( 3934 ConstantRange(APInt(BitWidth, 0), 3935 APInt::getSignedMinValue(BitWidth))); 3936 else if (AllNonPos) 3937 ConservativeResult = ConservativeResult.intersectWith( 3938 ConstantRange(APInt::getSignedMinValue(BitWidth), 3939 APInt(BitWidth, 1))); 3940 } 3941 3942 // TODO: non-affine addrec 3943 if (AddRec->isAffine()) { 3944 Type *Ty = AddRec->getType(); 3945 const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop()); 3946 if (!isa<SCEVCouldNotCompute>(MaxBECount) && 3947 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) { 3948 MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty); 3949 3950 const SCEV *Start = AddRec->getStart(); 3951 const SCEV *Step = AddRec->getStepRecurrence(*this); 3952 3953 ConstantRange StartRange = getSignedRange(Start); 3954 ConstantRange StepRange = getSignedRange(Step); 3955 ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); 3956 ConstantRange EndRange = 3957 StartRange.add(MaxBECountRange.multiply(StepRange)); 3958 3959 // Check for overflow. This must be done with ConstantRange arithmetic 3960 // because we could be called from within the ScalarEvolution overflow 3961 // checking code. 3962 ConstantRange ExtStartRange = StartRange.sextOrTrunc(BitWidth*2+1); 3963 ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1); 3964 ConstantRange ExtMaxBECountRange = 3965 MaxBECountRange.zextOrTrunc(BitWidth*2+1); 3966 ConstantRange ExtEndRange = EndRange.sextOrTrunc(BitWidth*2+1); 3967 if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) != 3968 ExtEndRange) 3969 return setSignedRange(AddRec, ConservativeResult); 3970 3971 APInt Min = APIntOps::smin(StartRange.getSignedMin(), 3972 EndRange.getSignedMin()); 3973 APInt Max = APIntOps::smax(StartRange.getSignedMax(), 3974 EndRange.getSignedMax()); 3975 if (Min.isMinSignedValue() && Max.isMaxSignedValue()) 3976 return setSignedRange(AddRec, ConservativeResult); 3977 return setSignedRange(AddRec, 3978 ConservativeResult.intersectWith(ConstantRange(Min, Max+1))); 3979 } 3980 } 3981 3982 return setSignedRange(AddRec, ConservativeResult); 3983 } 3984 3985 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 3986 // Check if the IR explicitly contains !range metadata. 3987 Optional<ConstantRange> MDRange = GetRangeFromMetadata(U->getValue()); 3988 if (MDRange.hasValue()) 3989 ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue()); 3990 3991 // For a SCEVUnknown, ask ValueTracking. 3992 if (!U->getValue()->getType()->isIntegerTy() && !DL) 3993 return setSignedRange(U, ConservativeResult); 3994 unsigned NS = ComputeNumSignBits(U->getValue(), DL, 0, AT, nullptr, DT); 3995 if (NS <= 1) 3996 return setSignedRange(U, ConservativeResult); 3997 return setSignedRange(U, ConservativeResult.intersectWith( 3998 ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1), 3999 APInt::getSignedMaxValue(BitWidth).ashr(NS - 1)+1))); 4000 } 4001 4002 return setSignedRange(S, ConservativeResult); 4003 } 4004 4005 /// createSCEV - We know that there is no SCEV for the specified value. 4006 /// Analyze the expression. 4007 /// 4008 const SCEV *ScalarEvolution::createSCEV(Value *V) { 4009 if (!isSCEVable(V->getType())) 4010 return getUnknown(V); 4011 4012 unsigned Opcode = Instruction::UserOp1; 4013 if (Instruction *I = dyn_cast<Instruction>(V)) { 4014 Opcode = I->getOpcode(); 4015 4016 // Don't attempt to analyze instructions in blocks that aren't 4017 // reachable. Such instructions don't matter, and they aren't required 4018 // to obey basic rules for definitions dominating uses which this 4019 // analysis depends on. 4020 if (!DT->isReachableFromEntry(I->getParent())) 4021 return getUnknown(V); 4022 } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) 4023 Opcode = CE->getOpcode(); 4024 else if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) 4025 return getConstant(CI); 4026 else if (isa<ConstantPointerNull>(V)) 4027 return getConstant(V->getType(), 0); 4028 else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 4029 return GA->mayBeOverridden() ? getUnknown(V) : getSCEV(GA->getAliasee()); 4030 else 4031 return getUnknown(V); 4032 4033 Operator *U = cast<Operator>(V); 4034 switch (Opcode) { 4035 case Instruction::Add: { 4036 // The simple thing to do would be to just call getSCEV on both operands 4037 // and call getAddExpr with the result. However if we're looking at a 4038 // bunch of things all added together, this can be quite inefficient, 4039 // because it leads to N-1 getAddExpr calls for N ultimate operands. 4040 // Instead, gather up all the operands and make a single getAddExpr call. 4041 // LLVM IR canonical form means we need only traverse the left operands. 4042 // 4043 // Don't apply this instruction's NSW or NUW flags to the new 4044 // expression. The instruction may be guarded by control flow that the 4045 // no-wrap behavior depends on. Non-control-equivalent instructions can be 4046 // mapped to the same SCEV expression, and it would be incorrect to transfer 4047 // NSW/NUW semantics to those operations. 4048 SmallVector<const SCEV *, 4> AddOps; 4049 AddOps.push_back(getSCEV(U->getOperand(1))); 4050 for (Value *Op = U->getOperand(0); ; Op = U->getOperand(0)) { 4051 unsigned Opcode = Op->getValueID() - Value::InstructionVal; 4052 if (Opcode != Instruction::Add && Opcode != Instruction::Sub) 4053 break; 4054 U = cast<Operator>(Op); 4055 const SCEV *Op1 = getSCEV(U->getOperand(1)); 4056 if (Opcode == Instruction::Sub) 4057 AddOps.push_back(getNegativeSCEV(Op1)); 4058 else 4059 AddOps.push_back(Op1); 4060 } 4061 AddOps.push_back(getSCEV(U->getOperand(0))); 4062 return getAddExpr(AddOps); 4063 } 4064 case Instruction::Mul: { 4065 // Don't transfer NSW/NUW for the same reason as AddExpr. 4066 SmallVector<const SCEV *, 4> MulOps; 4067 MulOps.push_back(getSCEV(U->getOperand(1))); 4068 for (Value *Op = U->getOperand(0); 4069 Op->getValueID() == Instruction::Mul + Value::InstructionVal; 4070 Op = U->getOperand(0)) { 4071 U = cast<Operator>(Op); 4072 MulOps.push_back(getSCEV(U->getOperand(1))); 4073 } 4074 MulOps.push_back(getSCEV(U->getOperand(0))); 4075 return getMulExpr(MulOps); 4076 } 4077 case Instruction::UDiv: 4078 return getUDivExpr(getSCEV(U->getOperand(0)), 4079 getSCEV(U->getOperand(1))); 4080 case Instruction::Sub: 4081 return getMinusSCEV(getSCEV(U->getOperand(0)), 4082 getSCEV(U->getOperand(1))); 4083 case Instruction::And: 4084 // For an expression like x&255 that merely masks off the high bits, 4085 // use zext(trunc(x)) as the SCEV expression. 4086 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) { 4087 if (CI->isNullValue()) 4088 return getSCEV(U->getOperand(1)); 4089 if (CI->isAllOnesValue()) 4090 return getSCEV(U->getOperand(0)); 4091 const APInt &A = CI->getValue(); 4092 4093 // Instcombine's ShrinkDemandedConstant may strip bits out of 4094 // constants, obscuring what would otherwise be a low-bits mask. 4095 // Use computeKnownBits to compute what ShrinkDemandedConstant 4096 // knew about to reconstruct a low-bits mask value. 4097 unsigned LZ = A.countLeadingZeros(); 4098 unsigned TZ = A.countTrailingZeros(); 4099 unsigned BitWidth = A.getBitWidth(); 4100 APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0); 4101 computeKnownBits(U->getOperand(0), KnownZero, KnownOne, DL, 4102 0, AT, nullptr, DT); 4103 4104 APInt EffectiveMask = 4105 APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ); 4106 if ((LZ != 0 || TZ != 0) && !((~A & ~KnownZero) & EffectiveMask)) { 4107 const SCEV *MulCount = getConstant( 4108 ConstantInt::get(getContext(), APInt::getOneBitSet(BitWidth, TZ))); 4109 return getMulExpr( 4110 getZeroExtendExpr( 4111 getTruncateExpr( 4112 getUDivExactExpr(getSCEV(U->getOperand(0)), MulCount), 4113 IntegerType::get(getContext(), BitWidth - LZ - TZ)), 4114 U->getType()), 4115 MulCount); 4116 } 4117 } 4118 break; 4119 4120 case Instruction::Or: 4121 // If the RHS of the Or is a constant, we may have something like: 4122 // X*4+1 which got turned into X*4|1. Handle this as an Add so loop 4123 // optimizations will transparently handle this case. 4124 // 4125 // In order for this transformation to be safe, the LHS must be of the 4126 // form X*(2^n) and the Or constant must be less than 2^n. 4127 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) { 4128 const SCEV *LHS = getSCEV(U->getOperand(0)); 4129 const APInt &CIVal = CI->getValue(); 4130 if (GetMinTrailingZeros(LHS) >= 4131 (CIVal.getBitWidth() - CIVal.countLeadingZeros())) { 4132 // Build a plain add SCEV. 4133 const SCEV *S = getAddExpr(LHS, getSCEV(CI)); 4134 // If the LHS of the add was an addrec and it has no-wrap flags, 4135 // transfer the no-wrap flags, since an or won't introduce a wrap. 4136 if (const SCEVAddRecExpr *NewAR = dyn_cast<SCEVAddRecExpr>(S)) { 4137 const SCEVAddRecExpr *OldAR = cast<SCEVAddRecExpr>(LHS); 4138 const_cast<SCEVAddRecExpr *>(NewAR)->setNoWrapFlags( 4139 OldAR->getNoWrapFlags()); 4140 } 4141 return S; 4142 } 4143 } 4144 break; 4145 case Instruction::Xor: 4146 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) { 4147 // If the RHS of the xor is a signbit, then this is just an add. 4148 // Instcombine turns add of signbit into xor as a strength reduction step. 4149 if (CI->getValue().isSignBit()) 4150 return getAddExpr(getSCEV(U->getOperand(0)), 4151 getSCEV(U->getOperand(1))); 4152 4153 // If the RHS of xor is -1, then this is a not operation. 4154 if (CI->isAllOnesValue()) 4155 return getNotSCEV(getSCEV(U->getOperand(0))); 4156 4157 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask. 4158 // This is a variant of the check for xor with -1, and it handles 4159 // the case where instcombine has trimmed non-demanded bits out 4160 // of an xor with -1. 4161 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U->getOperand(0))) 4162 if (ConstantInt *LCI = dyn_cast<ConstantInt>(BO->getOperand(1))) 4163 if (BO->getOpcode() == Instruction::And && 4164 LCI->getValue() == CI->getValue()) 4165 if (const SCEVZeroExtendExpr *Z = 4166 dyn_cast<SCEVZeroExtendExpr>(getSCEV(U->getOperand(0)))) { 4167 Type *UTy = U->getType(); 4168 const SCEV *Z0 = Z->getOperand(); 4169 Type *Z0Ty = Z0->getType(); 4170 unsigned Z0TySize = getTypeSizeInBits(Z0Ty); 4171 4172 // If C is a low-bits mask, the zero extend is serving to 4173 // mask off the high bits. Complement the operand and 4174 // re-apply the zext. 4175 if (APIntOps::isMask(Z0TySize, CI->getValue())) 4176 return getZeroExtendExpr(getNotSCEV(Z0), UTy); 4177 4178 // If C is a single bit, it may be in the sign-bit position 4179 // before the zero-extend. In this case, represent the xor 4180 // using an add, which is equivalent, and re-apply the zext. 4181 APInt Trunc = CI->getValue().trunc(Z0TySize); 4182 if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() && 4183 Trunc.isSignBit()) 4184 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)), 4185 UTy); 4186 } 4187 } 4188 break; 4189 4190 case Instruction::Shl: 4191 // Turn shift left of a constant amount into a multiply. 4192 if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) { 4193 uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth(); 4194 4195 // If the shift count is not less than the bitwidth, the result of 4196 // the shift is undefined. Don't try to analyze it, because the 4197 // resolution chosen here may differ from the resolution chosen in 4198 // other parts of the compiler. 4199 if (SA->getValue().uge(BitWidth)) 4200 break; 4201 4202 Constant *X = ConstantInt::get(getContext(), 4203 APInt::getOneBitSet(BitWidth, SA->getZExtValue())); 4204 return getMulExpr(getSCEV(U->getOperand(0)), getSCEV(X)); 4205 } 4206 break; 4207 4208 case Instruction::LShr: 4209 // Turn logical shift right of a constant into a unsigned divide. 4210 if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) { 4211 uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth(); 4212 4213 // If the shift count is not less than the bitwidth, the result of 4214 // the shift is undefined. Don't try to analyze it, because the 4215 // resolution chosen here may differ from the resolution chosen in 4216 // other parts of the compiler. 4217 if (SA->getValue().uge(BitWidth)) 4218 break; 4219 4220 Constant *X = ConstantInt::get(getContext(), 4221 APInt::getOneBitSet(BitWidth, SA->getZExtValue())); 4222 return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(X)); 4223 } 4224 break; 4225 4226 case Instruction::AShr: 4227 // For a two-shift sext-inreg, use sext(trunc(x)) as the SCEV expression. 4228 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) 4229 if (Operator *L = dyn_cast<Operator>(U->getOperand(0))) 4230 if (L->getOpcode() == Instruction::Shl && 4231 L->getOperand(1) == U->getOperand(1)) { 4232 uint64_t BitWidth = getTypeSizeInBits(U->getType()); 4233 4234 // If the shift count is not less than the bitwidth, the result of 4235 // the shift is undefined. Don't try to analyze it, because the 4236 // resolution chosen here may differ from the resolution chosen in 4237 // other parts of the compiler. 4238 if (CI->getValue().uge(BitWidth)) 4239 break; 4240 4241 uint64_t Amt = BitWidth - CI->getZExtValue(); 4242 if (Amt == BitWidth) 4243 return getSCEV(L->getOperand(0)); // shift by zero --> noop 4244 return 4245 getSignExtendExpr(getTruncateExpr(getSCEV(L->getOperand(0)), 4246 IntegerType::get(getContext(), 4247 Amt)), 4248 U->getType()); 4249 } 4250 break; 4251 4252 case Instruction::Trunc: 4253 return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType()); 4254 4255 case Instruction::ZExt: 4256 return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType()); 4257 4258 case Instruction::SExt: 4259 return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType()); 4260 4261 case Instruction::BitCast: 4262 // BitCasts are no-op casts so we just eliminate the cast. 4263 if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType())) 4264 return getSCEV(U->getOperand(0)); 4265 break; 4266 4267 // It's tempting to handle inttoptr and ptrtoint as no-ops, however this can 4268 // lead to pointer expressions which cannot safely be expanded to GEPs, 4269 // because ScalarEvolution doesn't respect the GEP aliasing rules when 4270 // simplifying integer expressions. 4271 4272 case Instruction::GetElementPtr: 4273 return createNodeForGEP(cast<GEPOperator>(U)); 4274 4275 case Instruction::PHI: 4276 return createNodeForPHI(cast<PHINode>(U)); 4277 4278 case Instruction::Select: 4279 // This could be a smax or umax that was lowered earlier. 4280 // Try to recover it. 4281 if (ICmpInst *ICI = dyn_cast<ICmpInst>(U->getOperand(0))) { 4282 Value *LHS = ICI->getOperand(0); 4283 Value *RHS = ICI->getOperand(1); 4284 switch (ICI->getPredicate()) { 4285 case ICmpInst::ICMP_SLT: 4286 case ICmpInst::ICMP_SLE: 4287 std::swap(LHS, RHS); 4288 // fall through 4289 case ICmpInst::ICMP_SGT: 4290 case ICmpInst::ICMP_SGE: 4291 // a >s b ? a+x : b+x -> smax(a, b)+x 4292 // a >s b ? b+x : a+x -> smin(a, b)+x 4293 if (LHS->getType() == U->getType()) { 4294 const SCEV *LS = getSCEV(LHS); 4295 const SCEV *RS = getSCEV(RHS); 4296 const SCEV *LA = getSCEV(U->getOperand(1)); 4297 const SCEV *RA = getSCEV(U->getOperand(2)); 4298 const SCEV *LDiff = getMinusSCEV(LA, LS); 4299 const SCEV *RDiff = getMinusSCEV(RA, RS); 4300 if (LDiff == RDiff) 4301 return getAddExpr(getSMaxExpr(LS, RS), LDiff); 4302 LDiff = getMinusSCEV(LA, RS); 4303 RDiff = getMinusSCEV(RA, LS); 4304 if (LDiff == RDiff) 4305 return getAddExpr(getSMinExpr(LS, RS), LDiff); 4306 } 4307 break; 4308 case ICmpInst::ICMP_ULT: 4309 case ICmpInst::ICMP_ULE: 4310 std::swap(LHS, RHS); 4311 // fall through 4312 case ICmpInst::ICMP_UGT: 4313 case ICmpInst::ICMP_UGE: 4314 // a >u b ? a+x : b+x -> umax(a, b)+x 4315 // a >u b ? b+x : a+x -> umin(a, b)+x 4316 if (LHS->getType() == U->getType()) { 4317 const SCEV *LS = getSCEV(LHS); 4318 const SCEV *RS = getSCEV(RHS); 4319 const SCEV *LA = getSCEV(U->getOperand(1)); 4320 const SCEV *RA = getSCEV(U->getOperand(2)); 4321 const SCEV *LDiff = getMinusSCEV(LA, LS); 4322 const SCEV *RDiff = getMinusSCEV(RA, RS); 4323 if (LDiff == RDiff) 4324 return getAddExpr(getUMaxExpr(LS, RS), LDiff); 4325 LDiff = getMinusSCEV(LA, RS); 4326 RDiff = getMinusSCEV(RA, LS); 4327 if (LDiff == RDiff) 4328 return getAddExpr(getUMinExpr(LS, RS), LDiff); 4329 } 4330 break; 4331 case ICmpInst::ICMP_NE: 4332 // n != 0 ? n+x : 1+x -> umax(n, 1)+x 4333 if (LHS->getType() == U->getType() && 4334 isa<ConstantInt>(RHS) && 4335 cast<ConstantInt>(RHS)->isZero()) { 4336 const SCEV *One = getConstant(LHS->getType(), 1); 4337 const SCEV *LS = getSCEV(LHS); 4338 const SCEV *LA = getSCEV(U->getOperand(1)); 4339 const SCEV *RA = getSCEV(U->getOperand(2)); 4340 const SCEV *LDiff = getMinusSCEV(LA, LS); 4341 const SCEV *RDiff = getMinusSCEV(RA, One); 4342 if (LDiff == RDiff) 4343 return getAddExpr(getUMaxExpr(One, LS), LDiff); 4344 } 4345 break; 4346 case ICmpInst::ICMP_EQ: 4347 // n == 0 ? 1+x : n+x -> umax(n, 1)+x 4348 if (LHS->getType() == U->getType() && 4349 isa<ConstantInt>(RHS) && 4350 cast<ConstantInt>(RHS)->isZero()) { 4351 const SCEV *One = getConstant(LHS->getType(), 1); 4352 const SCEV *LS = getSCEV(LHS); 4353 const SCEV *LA = getSCEV(U->getOperand(1)); 4354 const SCEV *RA = getSCEV(U->getOperand(2)); 4355 const SCEV *LDiff = getMinusSCEV(LA, One); 4356 const SCEV *RDiff = getMinusSCEV(RA, LS); 4357 if (LDiff == RDiff) 4358 return getAddExpr(getUMaxExpr(One, LS), LDiff); 4359 } 4360 break; 4361 default: 4362 break; 4363 } 4364 } 4365 4366 default: // We cannot analyze this expression. 4367 break; 4368 } 4369 4370 return getUnknown(V); 4371 } 4372 4373 4374 4375 //===----------------------------------------------------------------------===// 4376 // Iteration Count Computation Code 4377 // 4378 4379 unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L) { 4380 if (BasicBlock *ExitingBB = L->getExitingBlock()) 4381 return getSmallConstantTripCount(L, ExitingBB); 4382 4383 // No trip count information for multiple exits. 4384 return 0; 4385 } 4386 4387 /// getSmallConstantTripCount - Returns the maximum trip count of this loop as a 4388 /// normal unsigned value. Returns 0 if the trip count is unknown or not 4389 /// constant. Will also return 0 if the maximum trip count is very large (>= 4390 /// 2^32). 4391 /// 4392 /// This "trip count" assumes that control exits via ExitingBlock. More 4393 /// precisely, it is the number of times that control may reach ExitingBlock 4394 /// before taking the branch. For loops with multiple exits, it may not be the 4395 /// number times that the loop header executes because the loop may exit 4396 /// prematurely via another branch. 4397 unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L, 4398 BasicBlock *ExitingBlock) { 4399 assert(ExitingBlock && "Must pass a non-null exiting block!"); 4400 assert(L->isLoopExiting(ExitingBlock) && 4401 "Exiting block must actually branch out of the loop!"); 4402 const SCEVConstant *ExitCount = 4403 dyn_cast<SCEVConstant>(getExitCount(L, ExitingBlock)); 4404 if (!ExitCount) 4405 return 0; 4406 4407 ConstantInt *ExitConst = ExitCount->getValue(); 4408 4409 // Guard against huge trip counts. 4410 if (ExitConst->getValue().getActiveBits() > 32) 4411 return 0; 4412 4413 // In case of integer overflow, this returns 0, which is correct. 4414 return ((unsigned)ExitConst->getZExtValue()) + 1; 4415 } 4416 4417 unsigned ScalarEvolution::getSmallConstantTripMultiple(Loop *L) { 4418 if (BasicBlock *ExitingBB = L->getExitingBlock()) 4419 return getSmallConstantTripMultiple(L, ExitingBB); 4420 4421 // No trip multiple information for multiple exits. 4422 return 0; 4423 } 4424 4425 /// getSmallConstantTripMultiple - Returns the largest constant divisor of the 4426 /// trip count of this loop as a normal unsigned value, if possible. This 4427 /// means that the actual trip count is always a multiple of the returned 4428 /// value (don't forget the trip count could very well be zero as well!). 4429 /// 4430 /// Returns 1 if the trip count is unknown or not guaranteed to be the 4431 /// multiple of a constant (which is also the case if the trip count is simply 4432 /// constant, use getSmallConstantTripCount for that case), Will also return 1 4433 /// if the trip count is very large (>= 2^32). 4434 /// 4435 /// As explained in the comments for getSmallConstantTripCount, this assumes 4436 /// that control exits the loop via ExitingBlock. 4437 unsigned 4438 ScalarEvolution::getSmallConstantTripMultiple(Loop *L, 4439 BasicBlock *ExitingBlock) { 4440 assert(ExitingBlock && "Must pass a non-null exiting block!"); 4441 assert(L->isLoopExiting(ExitingBlock) && 4442 "Exiting block must actually branch out of the loop!"); 4443 const SCEV *ExitCount = getExitCount(L, ExitingBlock); 4444 if (ExitCount == getCouldNotCompute()) 4445 return 1; 4446 4447 // Get the trip count from the BE count by adding 1. 4448 const SCEV *TCMul = getAddExpr(ExitCount, 4449 getConstant(ExitCount->getType(), 1)); 4450 // FIXME: SCEV distributes multiplication as V1*C1 + V2*C1. We could attempt 4451 // to factor simple cases. 4452 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(TCMul)) 4453 TCMul = Mul->getOperand(0); 4454 4455 const SCEVConstant *MulC = dyn_cast<SCEVConstant>(TCMul); 4456 if (!MulC) 4457 return 1; 4458 4459 ConstantInt *Result = MulC->getValue(); 4460 4461 // Guard against huge trip counts (this requires checking 4462 // for zero to handle the case where the trip count == -1 and the 4463 // addition wraps). 4464 if (!Result || Result->getValue().getActiveBits() > 32 || 4465 Result->getValue().getActiveBits() == 0) 4466 return 1; 4467 4468 return (unsigned)Result->getZExtValue(); 4469 } 4470 4471 // getExitCount - Get the expression for the number of loop iterations for which 4472 // this loop is guaranteed not to exit via ExitingBlock. Otherwise return 4473 // SCEVCouldNotCompute. 4474 const SCEV *ScalarEvolution::getExitCount(Loop *L, BasicBlock *ExitingBlock) { 4475 return getBackedgeTakenInfo(L).getExact(ExitingBlock, this); 4476 } 4477 4478 /// getBackedgeTakenCount - If the specified loop has a predictable 4479 /// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute 4480 /// object. The backedge-taken count is the number of times the loop header 4481 /// will be branched to from within the loop. This is one less than the 4482 /// trip count of the loop, since it doesn't count the first iteration, 4483 /// when the header is branched to from outside the loop. 4484 /// 4485 /// Note that it is not valid to call this method on a loop without a 4486 /// loop-invariant backedge-taken count (see 4487 /// hasLoopInvariantBackedgeTakenCount). 4488 /// 4489 const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L) { 4490 return getBackedgeTakenInfo(L).getExact(this); 4491 } 4492 4493 /// getMaxBackedgeTakenCount - Similar to getBackedgeTakenCount, except 4494 /// return the least SCEV value that is known never to be less than the 4495 /// actual backedge taken count. 4496 const SCEV *ScalarEvolution::getMaxBackedgeTakenCount(const Loop *L) { 4497 return getBackedgeTakenInfo(L).getMax(this); 4498 } 4499 4500 /// PushLoopPHIs - Push PHI nodes in the header of the given loop 4501 /// onto the given Worklist. 4502 static void 4503 PushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) { 4504 BasicBlock *Header = L->getHeader(); 4505 4506 // Push all Loop-header PHIs onto the Worklist stack. 4507 for (BasicBlock::iterator I = Header->begin(); 4508 PHINode *PN = dyn_cast<PHINode>(I); ++I) 4509 Worklist.push_back(PN); 4510 } 4511 4512 const ScalarEvolution::BackedgeTakenInfo & 4513 ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { 4514 // Initially insert an invalid entry for this loop. If the insertion 4515 // succeeds, proceed to actually compute a backedge-taken count and 4516 // update the value. The temporary CouldNotCompute value tells SCEV 4517 // code elsewhere that it shouldn't attempt to request a new 4518 // backedge-taken count, which could result in infinite recursion. 4519 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair = 4520 BackedgeTakenCounts.insert(std::make_pair(L, BackedgeTakenInfo())); 4521 if (!Pair.second) 4522 return Pair.first->second; 4523 4524 // ComputeBackedgeTakenCount may allocate memory for its result. Inserting it 4525 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result 4526 // must be cleared in this scope. 4527 BackedgeTakenInfo Result = ComputeBackedgeTakenCount(L); 4528 4529 if (Result.getExact(this) != getCouldNotCompute()) { 4530 assert(isLoopInvariant(Result.getExact(this), L) && 4531 isLoopInvariant(Result.getMax(this), L) && 4532 "Computed backedge-taken count isn't loop invariant for loop!"); 4533 ++NumTripCountsComputed; 4534 } 4535 else if (Result.getMax(this) == getCouldNotCompute() && 4536 isa<PHINode>(L->getHeader()->begin())) { 4537 // Only count loops that have phi nodes as not being computable. 4538 ++NumTripCountsNotComputed; 4539 } 4540 4541 // Now that we know more about the trip count for this loop, forget any 4542 // existing SCEV values for PHI nodes in this loop since they are only 4543 // conservative estimates made without the benefit of trip count 4544 // information. This is similar to the code in forgetLoop, except that 4545 // it handles SCEVUnknown PHI nodes specially. 4546 if (Result.hasAnyInfo()) { 4547 SmallVector<Instruction *, 16> Worklist; 4548 PushLoopPHIs(L, Worklist); 4549 4550 SmallPtrSet<Instruction *, 8> Visited; 4551 while (!Worklist.empty()) { 4552 Instruction *I = Worklist.pop_back_val(); 4553 if (!Visited.insert(I).second) 4554 continue; 4555 4556 ValueExprMapType::iterator It = 4557 ValueExprMap.find_as(static_cast<Value *>(I)); 4558 if (It != ValueExprMap.end()) { 4559 const SCEV *Old = It->second; 4560 4561 // SCEVUnknown for a PHI either means that it has an unrecognized 4562 // structure, or it's a PHI that's in the progress of being computed 4563 // by createNodeForPHI. In the former case, additional loop trip 4564 // count information isn't going to change anything. In the later 4565 // case, createNodeForPHI will perform the necessary updates on its 4566 // own when it gets to that point. 4567 if (!isa<PHINode>(I) || !isa<SCEVUnknown>(Old)) { 4568 forgetMemoizedResults(Old); 4569 ValueExprMap.erase(It); 4570 } 4571 if (PHINode *PN = dyn_cast<PHINode>(I)) 4572 ConstantEvolutionLoopExitValue.erase(PN); 4573 } 4574 4575 PushDefUseChildren(I, Worklist); 4576 } 4577 } 4578 4579 // Re-lookup the insert position, since the call to 4580 // ComputeBackedgeTakenCount above could result in a 4581 // recusive call to getBackedgeTakenInfo (on a different 4582 // loop), which would invalidate the iterator computed 4583 // earlier. 4584 return BackedgeTakenCounts.find(L)->second = Result; 4585 } 4586 4587 /// forgetLoop - This method should be called by the client when it has 4588 /// changed a loop in a way that may effect ScalarEvolution's ability to 4589 /// compute a trip count, or if the loop is deleted. 4590 void ScalarEvolution::forgetLoop(const Loop *L) { 4591 // Drop any stored trip count value. 4592 DenseMap<const Loop*, BackedgeTakenInfo>::iterator BTCPos = 4593 BackedgeTakenCounts.find(L); 4594 if (BTCPos != BackedgeTakenCounts.end()) { 4595 BTCPos->second.clear(); 4596 BackedgeTakenCounts.erase(BTCPos); 4597 } 4598 4599 // Drop information about expressions based on loop-header PHIs. 4600 SmallVector<Instruction *, 16> Worklist; 4601 PushLoopPHIs(L, Worklist); 4602 4603 SmallPtrSet<Instruction *, 8> Visited; 4604 while (!Worklist.empty()) { 4605 Instruction *I = Worklist.pop_back_val(); 4606 if (!Visited.insert(I).second) 4607 continue; 4608 4609 ValueExprMapType::iterator It = 4610 ValueExprMap.find_as(static_cast<Value *>(I)); 4611 if (It != ValueExprMap.end()) { 4612 forgetMemoizedResults(It->second); 4613 ValueExprMap.erase(It); 4614 if (PHINode *PN = dyn_cast<PHINode>(I)) 4615 ConstantEvolutionLoopExitValue.erase(PN); 4616 } 4617 4618 PushDefUseChildren(I, Worklist); 4619 } 4620 4621 // Forget all contained loops too, to avoid dangling entries in the 4622 // ValuesAtScopes map. 4623 for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) 4624 forgetLoop(*I); 4625 } 4626 4627 /// forgetValue - This method should be called by the client when it has 4628 /// changed a value in a way that may effect its value, or which may 4629 /// disconnect it from a def-use chain linking it to a loop. 4630 void ScalarEvolution::forgetValue(Value *V) { 4631 Instruction *I = dyn_cast<Instruction>(V); 4632 if (!I) return; 4633 4634 // Drop information about expressions based on loop-header PHIs. 4635 SmallVector<Instruction *, 16> Worklist; 4636 Worklist.push_back(I); 4637 4638 SmallPtrSet<Instruction *, 8> Visited; 4639 while (!Worklist.empty()) { 4640 I = Worklist.pop_back_val(); 4641 if (!Visited.insert(I).second) 4642 continue; 4643 4644 ValueExprMapType::iterator It = 4645 ValueExprMap.find_as(static_cast<Value *>(I)); 4646 if (It != ValueExprMap.end()) { 4647 forgetMemoizedResults(It->second); 4648 ValueExprMap.erase(It); 4649 if (PHINode *PN = dyn_cast<PHINode>(I)) 4650 ConstantEvolutionLoopExitValue.erase(PN); 4651 } 4652 4653 PushDefUseChildren(I, Worklist); 4654 } 4655 } 4656 4657 /// getExact - Get the exact loop backedge taken count considering all loop 4658 /// exits. A computable result can only be return for loops with a single exit. 4659 /// Returning the minimum taken count among all exits is incorrect because one 4660 /// of the loop's exit limit's may have been skipped. HowFarToZero assumes that 4661 /// the limit of each loop test is never skipped. This is a valid assumption as 4662 /// long as the loop exits via that test. For precise results, it is the 4663 /// caller's responsibility to specify the relevant loop exit using 4664 /// getExact(ExitingBlock, SE). 4665 const SCEV * 4666 ScalarEvolution::BackedgeTakenInfo::getExact(ScalarEvolution *SE) const { 4667 // If any exits were not computable, the loop is not computable. 4668 if (!ExitNotTaken.isCompleteList()) return SE->getCouldNotCompute(); 4669 4670 // We need exactly one computable exit. 4671 if (!ExitNotTaken.ExitingBlock) return SE->getCouldNotCompute(); 4672 assert(ExitNotTaken.ExactNotTaken && "uninitialized not-taken info"); 4673 4674 const SCEV *BECount = nullptr; 4675 for (const ExitNotTakenInfo *ENT = &ExitNotTaken; 4676 ENT != nullptr; ENT = ENT->getNextExit()) { 4677 4678 assert(ENT->ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV"); 4679 4680 if (!BECount) 4681 BECount = ENT->ExactNotTaken; 4682 else if (BECount != ENT->ExactNotTaken) 4683 return SE->getCouldNotCompute(); 4684 } 4685 assert(BECount && "Invalid not taken count for loop exit"); 4686 return BECount; 4687 } 4688 4689 /// getExact - Get the exact not taken count for this loop exit. 4690 const SCEV * 4691 ScalarEvolution::BackedgeTakenInfo::getExact(BasicBlock *ExitingBlock, 4692 ScalarEvolution *SE) const { 4693 for (const ExitNotTakenInfo *ENT = &ExitNotTaken; 4694 ENT != nullptr; ENT = ENT->getNextExit()) { 4695 4696 if (ENT->ExitingBlock == ExitingBlock) 4697 return ENT->ExactNotTaken; 4698 } 4699 return SE->getCouldNotCompute(); 4700 } 4701 4702 /// getMax - Get the max backedge taken count for the loop. 4703 const SCEV * 4704 ScalarEvolution::BackedgeTakenInfo::getMax(ScalarEvolution *SE) const { 4705 return Max ? Max : SE->getCouldNotCompute(); 4706 } 4707 4708 bool ScalarEvolution::BackedgeTakenInfo::hasOperand(const SCEV *S, 4709 ScalarEvolution *SE) const { 4710 if (Max && Max != SE->getCouldNotCompute() && SE->hasOperand(Max, S)) 4711 return true; 4712 4713 if (!ExitNotTaken.ExitingBlock) 4714 return false; 4715 4716 for (const ExitNotTakenInfo *ENT = &ExitNotTaken; 4717 ENT != nullptr; ENT = ENT->getNextExit()) { 4718 4719 if (ENT->ExactNotTaken != SE->getCouldNotCompute() 4720 && SE->hasOperand(ENT->ExactNotTaken, S)) { 4721 return true; 4722 } 4723 } 4724 return false; 4725 } 4726 4727 /// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each 4728 /// computable exit into a persistent ExitNotTakenInfo array. 4729 ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo( 4730 SmallVectorImpl< std::pair<BasicBlock *, const SCEV *> > &ExitCounts, 4731 bool Complete, const SCEV *MaxCount) : Max(MaxCount) { 4732 4733 if (!Complete) 4734 ExitNotTaken.setIncomplete(); 4735 4736 unsigned NumExits = ExitCounts.size(); 4737 if (NumExits == 0) return; 4738 4739 ExitNotTaken.ExitingBlock = ExitCounts[0].first; 4740 ExitNotTaken.ExactNotTaken = ExitCounts[0].second; 4741 if (NumExits == 1) return; 4742 4743 // Handle the rare case of multiple computable exits. 4744 ExitNotTakenInfo *ENT = new ExitNotTakenInfo[NumExits-1]; 4745 4746 ExitNotTakenInfo *PrevENT = &ExitNotTaken; 4747 for (unsigned i = 1; i < NumExits; ++i, PrevENT = ENT, ++ENT) { 4748 PrevENT->setNextExit(ENT); 4749 ENT->ExitingBlock = ExitCounts[i].first; 4750 ENT->ExactNotTaken = ExitCounts[i].second; 4751 } 4752 } 4753 4754 /// clear - Invalidate this result and free the ExitNotTakenInfo array. 4755 void ScalarEvolution::BackedgeTakenInfo::clear() { 4756 ExitNotTaken.ExitingBlock = nullptr; 4757 ExitNotTaken.ExactNotTaken = nullptr; 4758 delete[] ExitNotTaken.getNextExit(); 4759 } 4760 4761 /// ComputeBackedgeTakenCount - Compute the number of times the backedge 4762 /// of the specified loop will execute. 4763 ScalarEvolution::BackedgeTakenInfo 4764 ScalarEvolution::ComputeBackedgeTakenCount(const Loop *L) { 4765 SmallVector<BasicBlock *, 8> ExitingBlocks; 4766 L->getExitingBlocks(ExitingBlocks); 4767 4768 SmallVector<std::pair<BasicBlock *, const SCEV *>, 4> ExitCounts; 4769 bool CouldComputeBECount = true; 4770 BasicBlock *Latch = L->getLoopLatch(); // may be NULL. 4771 const SCEV *MustExitMaxBECount = nullptr; 4772 const SCEV *MayExitMaxBECount = nullptr; 4773 4774 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts 4775 // and compute maxBECount. 4776 for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { 4777 BasicBlock *ExitBB = ExitingBlocks[i]; 4778 ExitLimit EL = ComputeExitLimit(L, ExitBB); 4779 4780 // 1. For each exit that can be computed, add an entry to ExitCounts. 4781 // CouldComputeBECount is true only if all exits can be computed. 4782 if (EL.Exact == getCouldNotCompute()) 4783 // We couldn't compute an exact value for this exit, so 4784 // we won't be able to compute an exact value for the loop. 4785 CouldComputeBECount = false; 4786 else 4787 ExitCounts.push_back(std::make_pair(ExitBB, EL.Exact)); 4788 4789 // 2. Derive the loop's MaxBECount from each exit's max number of 4790 // non-exiting iterations. Partition the loop exits into two kinds: 4791 // LoopMustExits and LoopMayExits. 4792 // 4793 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it 4794 // is a LoopMayExit. If any computable LoopMustExit is found, then 4795 // MaxBECount is the minimum EL.Max of computable LoopMustExits. Otherwise, 4796 // MaxBECount is conservatively the maximum EL.Max, where CouldNotCompute is 4797 // considered greater than any computable EL.Max. 4798 if (EL.Max != getCouldNotCompute() && Latch && 4799 DT->dominates(ExitBB, Latch)) { 4800 if (!MustExitMaxBECount) 4801 MustExitMaxBECount = EL.Max; 4802 else { 4803 MustExitMaxBECount = 4804 getUMinFromMismatchedTypes(MustExitMaxBECount, EL.Max); 4805 } 4806 } else if (MayExitMaxBECount != getCouldNotCompute()) { 4807 if (!MayExitMaxBECount || EL.Max == getCouldNotCompute()) 4808 MayExitMaxBECount = EL.Max; 4809 else { 4810 MayExitMaxBECount = 4811 getUMaxFromMismatchedTypes(MayExitMaxBECount, EL.Max); 4812 } 4813 } 4814 } 4815 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount : 4816 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute()); 4817 return BackedgeTakenInfo(ExitCounts, CouldComputeBECount, MaxBECount); 4818 } 4819 4820 /// ComputeExitLimit - Compute the number of times the backedge of the specified 4821 /// loop will execute if it exits via the specified block. 4822 ScalarEvolution::ExitLimit 4823 ScalarEvolution::ComputeExitLimit(const Loop *L, BasicBlock *ExitingBlock) { 4824 4825 // Okay, we've chosen an exiting block. See what condition causes us to 4826 // exit at this block and remember the exit block and whether all other targets 4827 // lead to the loop header. 4828 bool MustExecuteLoopHeader = true; 4829 BasicBlock *Exit = nullptr; 4830 for (succ_iterator SI = succ_begin(ExitingBlock), SE = succ_end(ExitingBlock); 4831 SI != SE; ++SI) 4832 if (!L->contains(*SI)) { 4833 if (Exit) // Multiple exit successors. 4834 return getCouldNotCompute(); 4835 Exit = *SI; 4836 } else if (*SI != L->getHeader()) { 4837 MustExecuteLoopHeader = false; 4838 } 4839 4840 // At this point, we know we have a conditional branch that determines whether 4841 // the loop is exited. However, we don't know if the branch is executed each 4842 // time through the loop. If not, then the execution count of the branch will 4843 // not be equal to the trip count of the loop. 4844 // 4845 // Currently we check for this by checking to see if the Exit branch goes to 4846 // the loop header. If so, we know it will always execute the same number of 4847 // times as the loop. We also handle the case where the exit block *is* the 4848 // loop header. This is common for un-rotated loops. 4849 // 4850 // If both of those tests fail, walk up the unique predecessor chain to the 4851 // header, stopping if there is an edge that doesn't exit the loop. If the 4852 // header is reached, the execution count of the branch will be equal to the 4853 // trip count of the loop. 4854 // 4855 // More extensive analysis could be done to handle more cases here. 4856 // 4857 if (!MustExecuteLoopHeader && ExitingBlock != L->getHeader()) { 4858 // The simple checks failed, try climbing the unique predecessor chain 4859 // up to the header. 4860 bool Ok = false; 4861 for (BasicBlock *BB = ExitingBlock; BB; ) { 4862 BasicBlock *Pred = BB->getUniquePredecessor(); 4863 if (!Pred) 4864 return getCouldNotCompute(); 4865 TerminatorInst *PredTerm = Pred->getTerminator(); 4866 for (unsigned i = 0, e = PredTerm->getNumSuccessors(); i != e; ++i) { 4867 BasicBlock *PredSucc = PredTerm->getSuccessor(i); 4868 if (PredSucc == BB) 4869 continue; 4870 // If the predecessor has a successor that isn't BB and isn't 4871 // outside the loop, assume the worst. 4872 if (L->contains(PredSucc)) 4873 return getCouldNotCompute(); 4874 } 4875 if (Pred == L->getHeader()) { 4876 Ok = true; 4877 break; 4878 } 4879 BB = Pred; 4880 } 4881 if (!Ok) 4882 return getCouldNotCompute(); 4883 } 4884 4885 bool IsOnlyExit = (L->getExitingBlock() != nullptr); 4886 TerminatorInst *Term = ExitingBlock->getTerminator(); 4887 if (BranchInst *BI = dyn_cast<BranchInst>(Term)) { 4888 assert(BI->isConditional() && "If unconditional, it can't be in loop!"); 4889 // Proceed to the next level to examine the exit condition expression. 4890 return ComputeExitLimitFromCond(L, BI->getCondition(), BI->getSuccessor(0), 4891 BI->getSuccessor(1), 4892 /*ControlsExit=*/IsOnlyExit); 4893 } 4894 4895 if (SwitchInst *SI = dyn_cast<SwitchInst>(Term)) 4896 return ComputeExitLimitFromSingleExitSwitch(L, SI, Exit, 4897 /*ControlsExit=*/IsOnlyExit); 4898 4899 return getCouldNotCompute(); 4900 } 4901 4902 /// ComputeExitLimitFromCond - Compute the number of times the 4903 /// backedge of the specified loop will execute if its exit condition 4904 /// were a conditional branch of ExitCond, TBB, and FBB. 4905 /// 4906 /// @param ControlsExit is true if ExitCond directly controls the exit 4907 /// branch. In this case, we can assume that the loop exits only if the 4908 /// condition is true and can infer that failing to meet the condition prior to 4909 /// integer wraparound results in undefined behavior. 4910 ScalarEvolution::ExitLimit 4911 ScalarEvolution::ComputeExitLimitFromCond(const Loop *L, 4912 Value *ExitCond, 4913 BasicBlock *TBB, 4914 BasicBlock *FBB, 4915 bool ControlsExit) { 4916 // Check if the controlling expression for this loop is an And or Or. 4917 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(ExitCond)) { 4918 if (BO->getOpcode() == Instruction::And) { 4919 // Recurse on the operands of the and. 4920 bool EitherMayExit = L->contains(TBB); 4921 ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB, 4922 ControlsExit && !EitherMayExit); 4923 ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB, 4924 ControlsExit && !EitherMayExit); 4925 const SCEV *BECount = getCouldNotCompute(); 4926 const SCEV *MaxBECount = getCouldNotCompute(); 4927 if (EitherMayExit) { 4928 // Both conditions must be true for the loop to continue executing. 4929 // Choose the less conservative count. 4930 if (EL0.Exact == getCouldNotCompute() || 4931 EL1.Exact == getCouldNotCompute()) 4932 BECount = getCouldNotCompute(); 4933 else 4934 BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact); 4935 if (EL0.Max == getCouldNotCompute()) 4936 MaxBECount = EL1.Max; 4937 else if (EL1.Max == getCouldNotCompute()) 4938 MaxBECount = EL0.Max; 4939 else 4940 MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max); 4941 } else { 4942 // Both conditions must be true at the same time for the loop to exit. 4943 // For now, be conservative. 4944 assert(L->contains(FBB) && "Loop block has no successor in loop!"); 4945 if (EL0.Max == EL1.Max) 4946 MaxBECount = EL0.Max; 4947 if (EL0.Exact == EL1.Exact) 4948 BECount = EL0.Exact; 4949 } 4950 4951 return ExitLimit(BECount, MaxBECount); 4952 } 4953 if (BO->getOpcode() == Instruction::Or) { 4954 // Recurse on the operands of the or. 4955 bool EitherMayExit = L->contains(FBB); 4956 ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB, 4957 ControlsExit && !EitherMayExit); 4958 ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB, 4959 ControlsExit && !EitherMayExit); 4960 const SCEV *BECount = getCouldNotCompute(); 4961 const SCEV *MaxBECount = getCouldNotCompute(); 4962 if (EitherMayExit) { 4963 // Both conditions must be false for the loop to continue executing. 4964 // Choose the less conservative count. 4965 if (EL0.Exact == getCouldNotCompute() || 4966 EL1.Exact == getCouldNotCompute()) 4967 BECount = getCouldNotCompute(); 4968 else 4969 BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact); 4970 if (EL0.Max == getCouldNotCompute()) 4971 MaxBECount = EL1.Max; 4972 else if (EL1.Max == getCouldNotCompute()) 4973 MaxBECount = EL0.Max; 4974 else 4975 MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max); 4976 } else { 4977 // Both conditions must be false at the same time for the loop to exit. 4978 // For now, be conservative. 4979 assert(L->contains(TBB) && "Loop block has no successor in loop!"); 4980 if (EL0.Max == EL1.Max) 4981 MaxBECount = EL0.Max; 4982 if (EL0.Exact == EL1.Exact) 4983 BECount = EL0.Exact; 4984 } 4985 4986 return ExitLimit(BECount, MaxBECount); 4987 } 4988 } 4989 4990 // With an icmp, it may be feasible to compute an exact backedge-taken count. 4991 // Proceed to the next level to examine the icmp. 4992 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond)) 4993 return ComputeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit); 4994 4995 // Check for a constant condition. These are normally stripped out by 4996 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to 4997 // preserve the CFG and is temporarily leaving constant conditions 4998 // in place. 4999 if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) { 5000 if (L->contains(FBB) == !CI->getZExtValue()) 5001 // The backedge is always taken. 5002 return getCouldNotCompute(); 5003 else 5004 // The backedge is never taken. 5005 return getConstant(CI->getType(), 0); 5006 } 5007 5008 // If it's not an integer or pointer comparison then compute it the hard way. 5009 return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB)); 5010 } 5011 5012 /// ComputeExitLimitFromICmp - Compute the number of times the 5013 /// backedge of the specified loop will execute if its exit condition 5014 /// were a conditional branch of the ICmpInst ExitCond, TBB, and FBB. 5015 ScalarEvolution::ExitLimit 5016 ScalarEvolution::ComputeExitLimitFromICmp(const Loop *L, 5017 ICmpInst *ExitCond, 5018 BasicBlock *TBB, 5019 BasicBlock *FBB, 5020 bool ControlsExit) { 5021 5022 // If the condition was exit on true, convert the condition to exit on false 5023 ICmpInst::Predicate Cond; 5024 if (!L->contains(FBB)) 5025 Cond = ExitCond->getPredicate(); 5026 else 5027 Cond = ExitCond->getInversePredicate(); 5028 5029 // Handle common loops like: for (X = "string"; *X; ++X) 5030 if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0))) 5031 if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) { 5032 ExitLimit ItCnt = 5033 ComputeLoadConstantCompareExitLimit(LI, RHS, L, Cond); 5034 if (ItCnt.hasAnyInfo()) 5035 return ItCnt; 5036 } 5037 5038 const SCEV *LHS = getSCEV(ExitCond->getOperand(0)); 5039 const SCEV *RHS = getSCEV(ExitCond->getOperand(1)); 5040 5041 // Try to evaluate any dependencies out of the loop. 5042 LHS = getSCEVAtScope(LHS, L); 5043 RHS = getSCEVAtScope(RHS, L); 5044 5045 // At this point, we would like to compute how many iterations of the 5046 // loop the predicate will return true for these inputs. 5047 if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) { 5048 // If there is a loop-invariant, force it into the RHS. 5049 std::swap(LHS, RHS); 5050 Cond = ICmpInst::getSwappedPredicate(Cond); 5051 } 5052 5053 // Simplify the operands before analyzing them. 5054 (void)SimplifyICmpOperands(Cond, LHS, RHS); 5055 5056 // If we have a comparison of a chrec against a constant, try to use value 5057 // ranges to answer this query. 5058 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) 5059 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS)) 5060 if (AddRec->getLoop() == L) { 5061 // Form the constant range. 5062 ConstantRange CompRange( 5063 ICmpInst::makeConstantRange(Cond, RHSC->getValue()->getValue())); 5064 5065 const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this); 5066 if (!isa<SCEVCouldNotCompute>(Ret)) return Ret; 5067 } 5068 5069 switch (Cond) { 5070 case ICmpInst::ICMP_NE: { // while (X != Y) 5071 // Convert to: while (X-Y != 0) 5072 ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit); 5073 if (EL.hasAnyInfo()) return EL; 5074 break; 5075 } 5076 case ICmpInst::ICMP_EQ: { // while (X == Y) 5077 // Convert to: while (X-Y == 0) 5078 ExitLimit EL = HowFarToNonZero(getMinusSCEV(LHS, RHS), L); 5079 if (EL.hasAnyInfo()) return EL; 5080 break; 5081 } 5082 case ICmpInst::ICMP_SLT: 5083 case ICmpInst::ICMP_ULT: { // while (X < Y) 5084 bool IsSigned = Cond == ICmpInst::ICMP_SLT; 5085 ExitLimit EL = HowManyLessThans(LHS, RHS, L, IsSigned, ControlsExit); 5086 if (EL.hasAnyInfo()) return EL; 5087 break; 5088 } 5089 case ICmpInst::ICMP_SGT: 5090 case ICmpInst::ICMP_UGT: { // while (X > Y) 5091 bool IsSigned = Cond == ICmpInst::ICMP_SGT; 5092 ExitLimit EL = HowManyGreaterThans(LHS, RHS, L, IsSigned, ControlsExit); 5093 if (EL.hasAnyInfo()) return EL; 5094 break; 5095 } 5096 default: 5097 #if 0 5098 dbgs() << "ComputeBackedgeTakenCount "; 5099 if (ExitCond->getOperand(0)->getType()->isUnsigned()) 5100 dbgs() << "[unsigned] "; 5101 dbgs() << *LHS << " " 5102 << Instruction::getOpcodeName(Instruction::ICmp) 5103 << " " << *RHS << "\n"; 5104 #endif 5105 break; 5106 } 5107 return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB)); 5108 } 5109 5110 ScalarEvolution::ExitLimit 5111 ScalarEvolution::ComputeExitLimitFromSingleExitSwitch(const Loop *L, 5112 SwitchInst *Switch, 5113 BasicBlock *ExitingBlock, 5114 bool ControlsExit) { 5115 assert(!L->contains(ExitingBlock) && "Not an exiting block!"); 5116 5117 // Give up if the exit is the default dest of a switch. 5118 if (Switch->getDefaultDest() == ExitingBlock) 5119 return getCouldNotCompute(); 5120 5121 assert(L->contains(Switch->getDefaultDest()) && 5122 "Default case must not exit the loop!"); 5123 const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L); 5124 const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock)); 5125 5126 // while (X != Y) --> while (X-Y != 0) 5127 ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit); 5128 if (EL.hasAnyInfo()) 5129 return EL; 5130 5131 return getCouldNotCompute(); 5132 } 5133 5134 static ConstantInt * 5135 EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C, 5136 ScalarEvolution &SE) { 5137 const SCEV *InVal = SE.getConstant(C); 5138 const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE); 5139 assert(isa<SCEVConstant>(Val) && 5140 "Evaluation of SCEV at constant didn't fold correctly?"); 5141 return cast<SCEVConstant>(Val)->getValue(); 5142 } 5143 5144 /// ComputeLoadConstantCompareExitLimit - Given an exit condition of 5145 /// 'icmp op load X, cst', try to see if we can compute the backedge 5146 /// execution count. 5147 ScalarEvolution::ExitLimit 5148 ScalarEvolution::ComputeLoadConstantCompareExitLimit( 5149 LoadInst *LI, 5150 Constant *RHS, 5151 const Loop *L, 5152 ICmpInst::Predicate predicate) { 5153 5154 if (LI->isVolatile()) return getCouldNotCompute(); 5155 5156 // Check to see if the loaded pointer is a getelementptr of a global. 5157 // TODO: Use SCEV instead of manually grubbing with GEPs. 5158 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0)); 5159 if (!GEP) return getCouldNotCompute(); 5160 5161 // Make sure that it is really a constant global we are gepping, with an 5162 // initializer, and make sure the first IDX is really 0. 5163 GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)); 5164 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() || 5165 GEP->getNumOperands() < 3 || !isa<Constant>(GEP->getOperand(1)) || 5166 !cast<Constant>(GEP->getOperand(1))->isNullValue()) 5167 return getCouldNotCompute(); 5168 5169 // Okay, we allow one non-constant index into the GEP instruction. 5170 Value *VarIdx = nullptr; 5171 std::vector<Constant*> Indexes; 5172 unsigned VarIdxNum = 0; 5173 for (unsigned i = 2, e = GEP->getNumOperands(); i != e; ++i) 5174 if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { 5175 Indexes.push_back(CI); 5176 } else if (!isa<ConstantInt>(GEP->getOperand(i))) { 5177 if (VarIdx) return getCouldNotCompute(); // Multiple non-constant idx's. 5178 VarIdx = GEP->getOperand(i); 5179 VarIdxNum = i-2; 5180 Indexes.push_back(nullptr); 5181 } 5182 5183 // Loop-invariant loads may be a byproduct of loop optimization. Skip them. 5184 if (!VarIdx) 5185 return getCouldNotCompute(); 5186 5187 // Okay, we know we have a (load (gep GV, 0, X)) comparison with a constant. 5188 // Check to see if X is a loop variant variable value now. 5189 const SCEV *Idx = getSCEV(VarIdx); 5190 Idx = getSCEVAtScope(Idx, L); 5191 5192 // We can only recognize very limited forms of loop index expressions, in 5193 // particular, only affine AddRec's like {C1,+,C2}. 5194 const SCEVAddRecExpr *IdxExpr = dyn_cast<SCEVAddRecExpr>(Idx); 5195 if (!IdxExpr || !IdxExpr->isAffine() || isLoopInvariant(IdxExpr, L) || 5196 !isa<SCEVConstant>(IdxExpr->getOperand(0)) || 5197 !isa<SCEVConstant>(IdxExpr->getOperand(1))) 5198 return getCouldNotCompute(); 5199 5200 unsigned MaxSteps = MaxBruteForceIterations; 5201 for (unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) { 5202 ConstantInt *ItCst = ConstantInt::get( 5203 cast<IntegerType>(IdxExpr->getType()), IterationNum); 5204 ConstantInt *Val = EvaluateConstantChrecAtConstant(IdxExpr, ItCst, *this); 5205 5206 // Form the GEP offset. 5207 Indexes[VarIdxNum] = Val; 5208 5209 Constant *Result = ConstantFoldLoadThroughGEPIndices(GV->getInitializer(), 5210 Indexes); 5211 if (!Result) break; // Cannot compute! 5212 5213 // Evaluate the condition for this iteration. 5214 Result = ConstantExpr::getICmp(predicate, Result, RHS); 5215 if (!isa<ConstantInt>(Result)) break; // Couldn't decide for sure 5216 if (cast<ConstantInt>(Result)->getValue().isMinValue()) { 5217 #if 0 5218 dbgs() << "\n***\n*** Computed loop count " << *ItCst 5219 << "\n*** From global " << *GV << "*** BB: " << *L->getHeader() 5220 << "***\n"; 5221 #endif 5222 ++NumArrayLenItCounts; 5223 return getConstant(ItCst); // Found terminating iteration! 5224 } 5225 } 5226 return getCouldNotCompute(); 5227 } 5228 5229 5230 /// CanConstantFold - Return true if we can constant fold an instruction of the 5231 /// specified type, assuming that all operands were constants. 5232 static bool CanConstantFold(const Instruction *I) { 5233 if (isa<BinaryOperator>(I) || isa<CmpInst>(I) || 5234 isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) || 5235 isa<LoadInst>(I)) 5236 return true; 5237 5238 if (const CallInst *CI = dyn_cast<CallInst>(I)) 5239 if (const Function *F = CI->getCalledFunction()) 5240 return canConstantFoldCallTo(F); 5241 return false; 5242 } 5243 5244 /// Determine whether this instruction can constant evolve within this loop 5245 /// assuming its operands can all constant evolve. 5246 static bool canConstantEvolve(Instruction *I, const Loop *L) { 5247 // An instruction outside of the loop can't be derived from a loop PHI. 5248 if (!L->contains(I)) return false; 5249 5250 if (isa<PHINode>(I)) { 5251 if (L->getHeader() == I->getParent()) 5252 return true; 5253 else 5254 // We don't currently keep track of the control flow needed to evaluate 5255 // PHIs, so we cannot handle PHIs inside of loops. 5256 return false; 5257 } 5258 5259 // If we won't be able to constant fold this expression even if the operands 5260 // are constants, bail early. 5261 return CanConstantFold(I); 5262 } 5263 5264 /// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by 5265 /// recursing through each instruction operand until reaching a loop header phi. 5266 static PHINode * 5267 getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L, 5268 DenseMap<Instruction *, PHINode *> &PHIMap) { 5269 5270 // Otherwise, we can evaluate this instruction if all of its operands are 5271 // constant or derived from a PHI node themselves. 5272 PHINode *PHI = nullptr; 5273 for (Instruction::op_iterator OpI = UseInst->op_begin(), 5274 OpE = UseInst->op_end(); OpI != OpE; ++OpI) { 5275 5276 if (isa<Constant>(*OpI)) continue; 5277 5278 Instruction *OpInst = dyn_cast<Instruction>(*OpI); 5279 if (!OpInst || !canConstantEvolve(OpInst, L)) return nullptr; 5280 5281 PHINode *P = dyn_cast<PHINode>(OpInst); 5282 if (!P) 5283 // If this operand is already visited, reuse the prior result. 5284 // We may have P != PHI if this is the deepest point at which the 5285 // inconsistent paths meet. 5286 P = PHIMap.lookup(OpInst); 5287 if (!P) { 5288 // Recurse and memoize the results, whether a phi is found or not. 5289 // This recursive call invalidates pointers into PHIMap. 5290 P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap); 5291 PHIMap[OpInst] = P; 5292 } 5293 if (!P) 5294 return nullptr; // Not evolving from PHI 5295 if (PHI && PHI != P) 5296 return nullptr; // Evolving from multiple different PHIs. 5297 PHI = P; 5298 } 5299 // This is a expression evolving from a constant PHI! 5300 return PHI; 5301 } 5302 5303 /// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node 5304 /// in the loop that V is derived from. We allow arbitrary operations along the 5305 /// way, but the operands of an operation must either be constants or a value 5306 /// derived from a constant PHI. If this expression does not fit with these 5307 /// constraints, return null. 5308 static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) { 5309 Instruction *I = dyn_cast<Instruction>(V); 5310 if (!I || !canConstantEvolve(I, L)) return nullptr; 5311 5312 if (PHINode *PN = dyn_cast<PHINode>(I)) { 5313 return PN; 5314 } 5315 5316 // Record non-constant instructions contained by the loop. 5317 DenseMap<Instruction *, PHINode *> PHIMap; 5318 return getConstantEvolvingPHIOperands(I, L, PHIMap); 5319 } 5320 5321 /// EvaluateExpression - Given an expression that passes the 5322 /// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node 5323 /// in the loop has the value PHIVal. If we can't fold this expression for some 5324 /// reason, return null. 5325 static Constant *EvaluateExpression(Value *V, const Loop *L, 5326 DenseMap<Instruction *, Constant *> &Vals, 5327 const DataLayout *DL, 5328 const TargetLibraryInfo *TLI) { 5329 // Convenient constant check, but redundant for recursive calls. 5330 if (Constant *C = dyn_cast<Constant>(V)) return C; 5331 Instruction *I = dyn_cast<Instruction>(V); 5332 if (!I) return nullptr; 5333 5334 if (Constant *C = Vals.lookup(I)) return C; 5335 5336 // An instruction inside the loop depends on a value outside the loop that we 5337 // weren't given a mapping for, or a value such as a call inside the loop. 5338 if (!canConstantEvolve(I, L)) return nullptr; 5339 5340 // An unmapped PHI can be due to a branch or another loop inside this loop, 5341 // or due to this not being the initial iteration through a loop where we 5342 // couldn't compute the evolution of this particular PHI last time. 5343 if (isa<PHINode>(I)) return nullptr; 5344 5345 std::vector<Constant*> Operands(I->getNumOperands()); 5346 5347 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) { 5348 Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i)); 5349 if (!Operand) { 5350 Operands[i] = dyn_cast<Constant>(I->getOperand(i)); 5351 if (!Operands[i]) return nullptr; 5352 continue; 5353 } 5354 Constant *C = EvaluateExpression(Operand, L, Vals, DL, TLI); 5355 Vals[Operand] = C; 5356 if (!C) return nullptr; 5357 Operands[i] = C; 5358 } 5359 5360 if (CmpInst *CI = dyn_cast<CmpInst>(I)) 5361 return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0], 5362 Operands[1], DL, TLI); 5363 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 5364 if (!LI->isVolatile()) 5365 return ConstantFoldLoadFromConstPtr(Operands[0], DL); 5366 } 5367 return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands, DL, 5368 TLI); 5369 } 5370 5371 /// getConstantEvolutionLoopExitValue - If we know that the specified Phi is 5372 /// in the header of its containing loop, we know the loop executes a 5373 /// constant number of times, and the PHI node is just a recurrence 5374 /// involving constants, fold it. 5375 Constant * 5376 ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN, 5377 const APInt &BEs, 5378 const Loop *L) { 5379 DenseMap<PHINode*, Constant*>::const_iterator I = 5380 ConstantEvolutionLoopExitValue.find(PN); 5381 if (I != ConstantEvolutionLoopExitValue.end()) 5382 return I->second; 5383 5384 if (BEs.ugt(MaxBruteForceIterations)) 5385 return ConstantEvolutionLoopExitValue[PN] = nullptr; // Not going to evaluate it. 5386 5387 Constant *&RetVal = ConstantEvolutionLoopExitValue[PN]; 5388 5389 DenseMap<Instruction *, Constant *> CurrentIterVals; 5390 BasicBlock *Header = L->getHeader(); 5391 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!"); 5392 5393 // Since the loop is canonicalized, the PHI node must have two entries. One 5394 // entry must be a constant (coming in from outside of the loop), and the 5395 // second must be derived from the same PHI. 5396 bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1)); 5397 PHINode *PHI = nullptr; 5398 for (BasicBlock::iterator I = Header->begin(); 5399 (PHI = dyn_cast<PHINode>(I)); ++I) { 5400 Constant *StartCST = 5401 dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge)); 5402 if (!StartCST) continue; 5403 CurrentIterVals[PHI] = StartCST; 5404 } 5405 if (!CurrentIterVals.count(PN)) 5406 return RetVal = nullptr; 5407 5408 Value *BEValue = PN->getIncomingValue(SecondIsBackedge); 5409 5410 // Execute the loop symbolically to determine the exit value. 5411 if (BEs.getActiveBits() >= 32) 5412 return RetVal = nullptr; // More than 2^32-1 iterations?? Not doing it! 5413 5414 unsigned NumIterations = BEs.getZExtValue(); // must be in range 5415 unsigned IterationNum = 0; 5416 for (; ; ++IterationNum) { 5417 if (IterationNum == NumIterations) 5418 return RetVal = CurrentIterVals[PN]; // Got exit value! 5419 5420 // Compute the value of the PHIs for the next iteration. 5421 // EvaluateExpression adds non-phi values to the CurrentIterVals map. 5422 DenseMap<Instruction *, Constant *> NextIterVals; 5423 Constant *NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, 5424 TLI); 5425 if (!NextPHI) 5426 return nullptr; // Couldn't evaluate! 5427 NextIterVals[PN] = NextPHI; 5428 5429 bool StoppedEvolving = NextPHI == CurrentIterVals[PN]; 5430 5431 // Also evaluate the other PHI nodes. However, we don't get to stop if we 5432 // cease to be able to evaluate one of them or if they stop evolving, 5433 // because that doesn't necessarily prevent us from computing PN. 5434 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute; 5435 for (DenseMap<Instruction *, Constant *>::const_iterator 5436 I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){ 5437 PHINode *PHI = dyn_cast<PHINode>(I->first); 5438 if (!PHI || PHI == PN || PHI->getParent() != Header) continue; 5439 PHIsToCompute.push_back(std::make_pair(PHI, I->second)); 5440 } 5441 // We use two distinct loops because EvaluateExpression may invalidate any 5442 // iterators into CurrentIterVals. 5443 for (SmallVectorImpl<std::pair<PHINode *, Constant*> >::const_iterator 5444 I = PHIsToCompute.begin(), E = PHIsToCompute.end(); I != E; ++I) { 5445 PHINode *PHI = I->first; 5446 Constant *&NextPHI = NextIterVals[PHI]; 5447 if (!NextPHI) { // Not already computed. 5448 Value *BEValue = PHI->getIncomingValue(SecondIsBackedge); 5449 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI); 5450 } 5451 if (NextPHI != I->second) 5452 StoppedEvolving = false; 5453 } 5454 5455 // If all entries in CurrentIterVals == NextIterVals then we can stop 5456 // iterating, the loop can't continue to change. 5457 if (StoppedEvolving) 5458 return RetVal = CurrentIterVals[PN]; 5459 5460 CurrentIterVals.swap(NextIterVals); 5461 } 5462 } 5463 5464 /// ComputeExitCountExhaustively - If the loop is known to execute a 5465 /// constant number of times (the condition evolves only from constants), 5466 /// try to evaluate a few iterations of the loop until we get the exit 5467 /// condition gets a value of ExitWhen (true or false). If we cannot 5468 /// evaluate the trip count of the loop, return getCouldNotCompute(). 5469 const SCEV *ScalarEvolution::ComputeExitCountExhaustively(const Loop *L, 5470 Value *Cond, 5471 bool ExitWhen) { 5472 PHINode *PN = getConstantEvolvingPHI(Cond, L); 5473 if (!PN) return getCouldNotCompute(); 5474 5475 // If the loop is canonicalized, the PHI will have exactly two entries. 5476 // That's the only form we support here. 5477 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute(); 5478 5479 DenseMap<Instruction *, Constant *> CurrentIterVals; 5480 BasicBlock *Header = L->getHeader(); 5481 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!"); 5482 5483 // One entry must be a constant (coming in from outside of the loop), and the 5484 // second must be derived from the same PHI. 5485 bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1)); 5486 PHINode *PHI = nullptr; 5487 for (BasicBlock::iterator I = Header->begin(); 5488 (PHI = dyn_cast<PHINode>(I)); ++I) { 5489 Constant *StartCST = 5490 dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge)); 5491 if (!StartCST) continue; 5492 CurrentIterVals[PHI] = StartCST; 5493 } 5494 if (!CurrentIterVals.count(PN)) 5495 return getCouldNotCompute(); 5496 5497 // Okay, we find a PHI node that defines the trip count of this loop. Execute 5498 // the loop symbolically to determine when the condition gets a value of 5499 // "ExitWhen". 5500 5501 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis. 5502 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){ 5503 ConstantInt *CondVal = 5504 dyn_cast_or_null<ConstantInt>(EvaluateExpression(Cond, L, CurrentIterVals, 5505 DL, TLI)); 5506 5507 // Couldn't symbolically evaluate. 5508 if (!CondVal) return getCouldNotCompute(); 5509 5510 if (CondVal->getValue() == uint64_t(ExitWhen)) { 5511 ++NumBruteForceTripCountsComputed; 5512 return getConstant(Type::getInt32Ty(getContext()), IterationNum); 5513 } 5514 5515 // Update all the PHI nodes for the next iteration. 5516 DenseMap<Instruction *, Constant *> NextIterVals; 5517 5518 // Create a list of which PHIs we need to compute. We want to do this before 5519 // calling EvaluateExpression on them because that may invalidate iterators 5520 // into CurrentIterVals. 5521 SmallVector<PHINode *, 8> PHIsToCompute; 5522 for (DenseMap<Instruction *, Constant *>::const_iterator 5523 I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){ 5524 PHINode *PHI = dyn_cast<PHINode>(I->first); 5525 if (!PHI || PHI->getParent() != Header) continue; 5526 PHIsToCompute.push_back(PHI); 5527 } 5528 for (SmallVectorImpl<PHINode *>::const_iterator I = PHIsToCompute.begin(), 5529 E = PHIsToCompute.end(); I != E; ++I) { 5530 PHINode *PHI = *I; 5531 Constant *&NextPHI = NextIterVals[PHI]; 5532 if (NextPHI) continue; // Already computed! 5533 5534 Value *BEValue = PHI->getIncomingValue(SecondIsBackedge); 5535 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI); 5536 } 5537 CurrentIterVals.swap(NextIterVals); 5538 } 5539 5540 // Too many iterations were needed to evaluate. 5541 return getCouldNotCompute(); 5542 } 5543 5544 /// getSCEVAtScope - Return a SCEV expression for the specified value 5545 /// at the specified scope in the program. The L value specifies a loop 5546 /// nest to evaluate the expression at, where null is the top-level or a 5547 /// specified loop is immediately inside of the loop. 5548 /// 5549 /// This method can be used to compute the exit value for a variable defined 5550 /// in a loop by querying what the value will hold in the parent loop. 5551 /// 5552 /// In the case that a relevant loop exit value cannot be computed, the 5553 /// original value V is returned. 5554 const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) { 5555 // Check to see if we've folded this expression at this loop before. 5556 SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values = ValuesAtScopes[V]; 5557 for (unsigned u = 0; u < Values.size(); u++) { 5558 if (Values[u].first == L) 5559 return Values[u].second ? Values[u].second : V; 5560 } 5561 Values.push_back(std::make_pair(L, static_cast<const SCEV *>(nullptr))); 5562 // Otherwise compute it. 5563 const SCEV *C = computeSCEVAtScope(V, L); 5564 SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values2 = ValuesAtScopes[V]; 5565 for (unsigned u = Values2.size(); u > 0; u--) { 5566 if (Values2[u - 1].first == L) { 5567 Values2[u - 1].second = C; 5568 break; 5569 } 5570 } 5571 return C; 5572 } 5573 5574 /// This builds up a Constant using the ConstantExpr interface. That way, we 5575 /// will return Constants for objects which aren't represented by a 5576 /// SCEVConstant, because SCEVConstant is restricted to ConstantInt. 5577 /// Returns NULL if the SCEV isn't representable as a Constant. 5578 static Constant *BuildConstantFromSCEV(const SCEV *V) { 5579 switch (static_cast<SCEVTypes>(V->getSCEVType())) { 5580 case scCouldNotCompute: 5581 case scAddRecExpr: 5582 break; 5583 case scConstant: 5584 return cast<SCEVConstant>(V)->getValue(); 5585 case scUnknown: 5586 return dyn_cast<Constant>(cast<SCEVUnknown>(V)->getValue()); 5587 case scSignExtend: { 5588 const SCEVSignExtendExpr *SS = cast<SCEVSignExtendExpr>(V); 5589 if (Constant *CastOp = BuildConstantFromSCEV(SS->getOperand())) 5590 return ConstantExpr::getSExt(CastOp, SS->getType()); 5591 break; 5592 } 5593 case scZeroExtend: { 5594 const SCEVZeroExtendExpr *SZ = cast<SCEVZeroExtendExpr>(V); 5595 if (Constant *CastOp = BuildConstantFromSCEV(SZ->getOperand())) 5596 return ConstantExpr::getZExt(CastOp, SZ->getType()); 5597 break; 5598 } 5599 case scTruncate: { 5600 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(V); 5601 if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand())) 5602 return ConstantExpr::getTrunc(CastOp, ST->getType()); 5603 break; 5604 } 5605 case scAddExpr: { 5606 const SCEVAddExpr *SA = cast<SCEVAddExpr>(V); 5607 if (Constant *C = BuildConstantFromSCEV(SA->getOperand(0))) { 5608 if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) { 5609 unsigned AS = PTy->getAddressSpace(); 5610 Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS); 5611 C = ConstantExpr::getBitCast(C, DestPtrTy); 5612 } 5613 for (unsigned i = 1, e = SA->getNumOperands(); i != e; ++i) { 5614 Constant *C2 = BuildConstantFromSCEV(SA->getOperand(i)); 5615 if (!C2) return nullptr; 5616 5617 // First pointer! 5618 if (!C->getType()->isPointerTy() && C2->getType()->isPointerTy()) { 5619 unsigned AS = C2->getType()->getPointerAddressSpace(); 5620 std::swap(C, C2); 5621 Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS); 5622 // The offsets have been converted to bytes. We can add bytes to an 5623 // i8* by GEP with the byte count in the first index. 5624 C = ConstantExpr::getBitCast(C, DestPtrTy); 5625 } 5626 5627 // Don't bother trying to sum two pointers. We probably can't 5628 // statically compute a load that results from it anyway. 5629 if (C2->getType()->isPointerTy()) 5630 return nullptr; 5631 5632 if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) { 5633 if (PTy->getElementType()->isStructTy()) 5634 C2 = ConstantExpr::getIntegerCast( 5635 C2, Type::getInt32Ty(C->getContext()), true); 5636 C = ConstantExpr::getGetElementPtr(C, C2); 5637 } else 5638 C = ConstantExpr::getAdd(C, C2); 5639 } 5640 return C; 5641 } 5642 break; 5643 } 5644 case scMulExpr: { 5645 const SCEVMulExpr *SM = cast<SCEVMulExpr>(V); 5646 if (Constant *C = BuildConstantFromSCEV(SM->getOperand(0))) { 5647 // Don't bother with pointers at all. 5648 if (C->getType()->isPointerTy()) return nullptr; 5649 for (unsigned i = 1, e = SM->getNumOperands(); i != e; ++i) { 5650 Constant *C2 = BuildConstantFromSCEV(SM->getOperand(i)); 5651 if (!C2 || C2->getType()->isPointerTy()) return nullptr; 5652 C = ConstantExpr::getMul(C, C2); 5653 } 5654 return C; 5655 } 5656 break; 5657 } 5658 case scUDivExpr: { 5659 const SCEVUDivExpr *SU = cast<SCEVUDivExpr>(V); 5660 if (Constant *LHS = BuildConstantFromSCEV(SU->getLHS())) 5661 if (Constant *RHS = BuildConstantFromSCEV(SU->getRHS())) 5662 if (LHS->getType() == RHS->getType()) 5663 return ConstantExpr::getUDiv(LHS, RHS); 5664 break; 5665 } 5666 case scSMaxExpr: 5667 case scUMaxExpr: 5668 break; // TODO: smax, umax. 5669 } 5670 return nullptr; 5671 } 5672 5673 const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) { 5674 if (isa<SCEVConstant>(V)) return V; 5675 5676 // If this instruction is evolved from a constant-evolving PHI, compute the 5677 // exit value from the loop without using SCEVs. 5678 if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) { 5679 if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) { 5680 const Loop *LI = (*this->LI)[I->getParent()]; 5681 if (LI && LI->getParentLoop() == L) // Looking for loop exit value. 5682 if (PHINode *PN = dyn_cast<PHINode>(I)) 5683 if (PN->getParent() == LI->getHeader()) { 5684 // Okay, there is no closed form solution for the PHI node. Check 5685 // to see if the loop that contains it has a known backedge-taken 5686 // count. If so, we may be able to force computation of the exit 5687 // value. 5688 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI); 5689 if (const SCEVConstant *BTCC = 5690 dyn_cast<SCEVConstant>(BackedgeTakenCount)) { 5691 // Okay, we know how many times the containing loop executes. If 5692 // this is a constant evolving PHI node, get the final value at 5693 // the specified iteration number. 5694 Constant *RV = getConstantEvolutionLoopExitValue(PN, 5695 BTCC->getValue()->getValue(), 5696 LI); 5697 if (RV) return getSCEV(RV); 5698 } 5699 } 5700 5701 // Okay, this is an expression that we cannot symbolically evaluate 5702 // into a SCEV. Check to see if it's possible to symbolically evaluate 5703 // the arguments into constants, and if so, try to constant propagate the 5704 // result. This is particularly useful for computing loop exit values. 5705 if (CanConstantFold(I)) { 5706 SmallVector<Constant *, 4> Operands; 5707 bool MadeImprovement = false; 5708 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) { 5709 Value *Op = I->getOperand(i); 5710 if (Constant *C = dyn_cast<Constant>(Op)) { 5711 Operands.push_back(C); 5712 continue; 5713 } 5714 5715 // If any of the operands is non-constant and if they are 5716 // non-integer and non-pointer, don't even try to analyze them 5717 // with scev techniques. 5718 if (!isSCEVable(Op->getType())) 5719 return V; 5720 5721 const SCEV *OrigV = getSCEV(Op); 5722 const SCEV *OpV = getSCEVAtScope(OrigV, L); 5723 MadeImprovement |= OrigV != OpV; 5724 5725 Constant *C = BuildConstantFromSCEV(OpV); 5726 if (!C) return V; 5727 if (C->getType() != Op->getType()) 5728 C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false, 5729 Op->getType(), 5730 false), 5731 C, Op->getType()); 5732 Operands.push_back(C); 5733 } 5734 5735 // Check to see if getSCEVAtScope actually made an improvement. 5736 if (MadeImprovement) { 5737 Constant *C = nullptr; 5738 if (const CmpInst *CI = dyn_cast<CmpInst>(I)) 5739 C = ConstantFoldCompareInstOperands(CI->getPredicate(), 5740 Operands[0], Operands[1], DL, 5741 TLI); 5742 else if (const LoadInst *LI = dyn_cast<LoadInst>(I)) { 5743 if (!LI->isVolatile()) 5744 C = ConstantFoldLoadFromConstPtr(Operands[0], DL); 5745 } else 5746 C = ConstantFoldInstOperands(I->getOpcode(), I->getType(), 5747 Operands, DL, TLI); 5748 if (!C) return V; 5749 return getSCEV(C); 5750 } 5751 } 5752 } 5753 5754 // This is some other type of SCEVUnknown, just return it. 5755 return V; 5756 } 5757 5758 if (const SCEVCommutativeExpr *Comm = dyn_cast<SCEVCommutativeExpr>(V)) { 5759 // Avoid performing the look-up in the common case where the specified 5760 // expression has no loop-variant portions. 5761 for (unsigned i = 0, e = Comm->getNumOperands(); i != e; ++i) { 5762 const SCEV *OpAtScope = getSCEVAtScope(Comm->getOperand(i), L); 5763 if (OpAtScope != Comm->getOperand(i)) { 5764 // Okay, at least one of these operands is loop variant but might be 5765 // foldable. Build a new instance of the folded commutative expression. 5766 SmallVector<const SCEV *, 8> NewOps(Comm->op_begin(), 5767 Comm->op_begin()+i); 5768 NewOps.push_back(OpAtScope); 5769 5770 for (++i; i != e; ++i) { 5771 OpAtScope = getSCEVAtScope(Comm->getOperand(i), L); 5772 NewOps.push_back(OpAtScope); 5773 } 5774 if (isa<SCEVAddExpr>(Comm)) 5775 return getAddExpr(NewOps); 5776 if (isa<SCEVMulExpr>(Comm)) 5777 return getMulExpr(NewOps); 5778 if (isa<SCEVSMaxExpr>(Comm)) 5779 return getSMaxExpr(NewOps); 5780 if (isa<SCEVUMaxExpr>(Comm)) 5781 return getUMaxExpr(NewOps); 5782 llvm_unreachable("Unknown commutative SCEV type!"); 5783 } 5784 } 5785 // If we got here, all operands are loop invariant. 5786 return Comm; 5787 } 5788 5789 if (const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) { 5790 const SCEV *LHS = getSCEVAtScope(Div->getLHS(), L); 5791 const SCEV *RHS = getSCEVAtScope(Div->getRHS(), L); 5792 if (LHS == Div->getLHS() && RHS == Div->getRHS()) 5793 return Div; // must be loop invariant 5794 return getUDivExpr(LHS, RHS); 5795 } 5796 5797 // If this is a loop recurrence for a loop that does not contain L, then we 5798 // are dealing with the final value computed by the loop. 5799 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) { 5800 // First, attempt to evaluate each operand. 5801 // Avoid performing the look-up in the common case where the specified 5802 // expression has no loop-variant portions. 5803 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { 5804 const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L); 5805 if (OpAtScope == AddRec->getOperand(i)) 5806 continue; 5807 5808 // Okay, at least one of these operands is loop variant but might be 5809 // foldable. Build a new instance of the folded commutative expression. 5810 SmallVector<const SCEV *, 8> NewOps(AddRec->op_begin(), 5811 AddRec->op_begin()+i); 5812 NewOps.push_back(OpAtScope); 5813 for (++i; i != e; ++i) 5814 NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L)); 5815 5816 const SCEV *FoldedRec = 5817 getAddRecExpr(NewOps, AddRec->getLoop(), 5818 AddRec->getNoWrapFlags(SCEV::FlagNW)); 5819 AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec); 5820 // The addrec may be folded to a nonrecurrence, for example, if the 5821 // induction variable is multiplied by zero after constant folding. Go 5822 // ahead and return the folded value. 5823 if (!AddRec) 5824 return FoldedRec; 5825 break; 5826 } 5827 5828 // If the scope is outside the addrec's loop, evaluate it by using the 5829 // loop exit value of the addrec. 5830 if (!AddRec->getLoop()->contains(L)) { 5831 // To evaluate this recurrence, we need to know how many times the AddRec 5832 // loop iterates. Compute this now. 5833 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop()); 5834 if (BackedgeTakenCount == getCouldNotCompute()) return AddRec; 5835 5836 // Then, evaluate the AddRec. 5837 return AddRec->evaluateAtIteration(BackedgeTakenCount, *this); 5838 } 5839 5840 return AddRec; 5841 } 5842 5843 if (const SCEVZeroExtendExpr *Cast = dyn_cast<SCEVZeroExtendExpr>(V)) { 5844 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 5845 if (Op == Cast->getOperand()) 5846 return Cast; // must be loop invariant 5847 return getZeroExtendExpr(Op, Cast->getType()); 5848 } 5849 5850 if (const SCEVSignExtendExpr *Cast = dyn_cast<SCEVSignExtendExpr>(V)) { 5851 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 5852 if (Op == Cast->getOperand()) 5853 return Cast; // must be loop invariant 5854 return getSignExtendExpr(Op, Cast->getType()); 5855 } 5856 5857 if (const SCEVTruncateExpr *Cast = dyn_cast<SCEVTruncateExpr>(V)) { 5858 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 5859 if (Op == Cast->getOperand()) 5860 return Cast; // must be loop invariant 5861 return getTruncateExpr(Op, Cast->getType()); 5862 } 5863 5864 llvm_unreachable("Unknown SCEV type!"); 5865 } 5866 5867 /// getSCEVAtScope - This is a convenience function which does 5868 /// getSCEVAtScope(getSCEV(V), L). 5869 const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) { 5870 return getSCEVAtScope(getSCEV(V), L); 5871 } 5872 5873 /// SolveLinEquationWithOverflow - Finds the minimum unsigned root of the 5874 /// following equation: 5875 /// 5876 /// A * X = B (mod N) 5877 /// 5878 /// where N = 2^BW and BW is the common bit width of A and B. The signedness of 5879 /// A and B isn't important. 5880 /// 5881 /// If the equation does not have a solution, SCEVCouldNotCompute is returned. 5882 static const SCEV *SolveLinEquationWithOverflow(const APInt &A, const APInt &B, 5883 ScalarEvolution &SE) { 5884 uint32_t BW = A.getBitWidth(); 5885 assert(BW == B.getBitWidth() && "Bit widths must be the same."); 5886 assert(A != 0 && "A must be non-zero."); 5887 5888 // 1. D = gcd(A, N) 5889 // 5890 // The gcd of A and N may have only one prime factor: 2. The number of 5891 // trailing zeros in A is its multiplicity 5892 uint32_t Mult2 = A.countTrailingZeros(); 5893 // D = 2^Mult2 5894 5895 // 2. Check if B is divisible by D. 5896 // 5897 // B is divisible by D if and only if the multiplicity of prime factor 2 for B 5898 // is not less than multiplicity of this prime factor for D. 5899 if (B.countTrailingZeros() < Mult2) 5900 return SE.getCouldNotCompute(); 5901 5902 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic 5903 // modulo (N / D). 5904 // 5905 // (N / D) may need BW+1 bits in its representation. Hence, we'll use this 5906 // bit width during computations. 5907 APInt AD = A.lshr(Mult2).zext(BW + 1); // AD = A / D 5908 APInt Mod(BW + 1, 0); 5909 Mod.setBit(BW - Mult2); // Mod = N / D 5910 APInt I = AD.multiplicativeInverse(Mod); 5911 5912 // 4. Compute the minimum unsigned root of the equation: 5913 // I * (B / D) mod (N / D) 5914 APInt Result = (I * B.lshr(Mult2).zext(BW + 1)).urem(Mod); 5915 5916 // The result is guaranteed to be less than 2^BW so we may truncate it to BW 5917 // bits. 5918 return SE.getConstant(Result.trunc(BW)); 5919 } 5920 5921 /// SolveQuadraticEquation - Find the roots of the quadratic equation for the 5922 /// given quadratic chrec {L,+,M,+,N}. This returns either the two roots (which 5923 /// might be the same) or two SCEVCouldNotCompute objects. 5924 /// 5925 static std::pair<const SCEV *,const SCEV *> 5926 SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) { 5927 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!"); 5928 const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0)); 5929 const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1)); 5930 const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2)); 5931 5932 // We currently can only solve this if the coefficients are constants. 5933 if (!LC || !MC || !NC) { 5934 const SCEV *CNC = SE.getCouldNotCompute(); 5935 return std::make_pair(CNC, CNC); 5936 } 5937 5938 uint32_t BitWidth = LC->getValue()->getValue().getBitWidth(); 5939 const APInt &L = LC->getValue()->getValue(); 5940 const APInt &M = MC->getValue()->getValue(); 5941 const APInt &N = NC->getValue()->getValue(); 5942 APInt Two(BitWidth, 2); 5943 APInt Four(BitWidth, 4); 5944 5945 { 5946 using namespace APIntOps; 5947 const APInt& C = L; 5948 // Convert from chrec coefficients to polynomial coefficients AX^2+BX+C 5949 // The B coefficient is M-N/2 5950 APInt B(M); 5951 B -= sdiv(N,Two); 5952 5953 // The A coefficient is N/2 5954 APInt A(N.sdiv(Two)); 5955 5956 // Compute the B^2-4ac term. 5957 APInt SqrtTerm(B); 5958 SqrtTerm *= B; 5959 SqrtTerm -= Four * (A * C); 5960 5961 if (SqrtTerm.isNegative()) { 5962 // The loop is provably infinite. 5963 const SCEV *CNC = SE.getCouldNotCompute(); 5964 return std::make_pair(CNC, CNC); 5965 } 5966 5967 // Compute sqrt(B^2-4ac). This is guaranteed to be the nearest 5968 // integer value or else APInt::sqrt() will assert. 5969 APInt SqrtVal(SqrtTerm.sqrt()); 5970 5971 // Compute the two solutions for the quadratic formula. 5972 // The divisions must be performed as signed divisions. 5973 APInt NegB(-B); 5974 APInt TwoA(A << 1); 5975 if (TwoA.isMinValue()) { 5976 const SCEV *CNC = SE.getCouldNotCompute(); 5977 return std::make_pair(CNC, CNC); 5978 } 5979 5980 LLVMContext &Context = SE.getContext(); 5981 5982 ConstantInt *Solution1 = 5983 ConstantInt::get(Context, (NegB + SqrtVal).sdiv(TwoA)); 5984 ConstantInt *Solution2 = 5985 ConstantInt::get(Context, (NegB - SqrtVal).sdiv(TwoA)); 5986 5987 return std::make_pair(SE.getConstant(Solution1), 5988 SE.getConstant(Solution2)); 5989 } // end APIntOps namespace 5990 } 5991 5992 /// HowFarToZero - Return the number of times a backedge comparing the specified 5993 /// value to zero will execute. If not computable, return CouldNotCompute. 5994 /// 5995 /// This is only used for loops with a "x != y" exit test. The exit condition is 5996 /// now expressed as a single expression, V = x-y. So the exit test is 5997 /// effectively V != 0. We know and take advantage of the fact that this 5998 /// expression only being used in a comparison by zero context. 5999 ScalarEvolution::ExitLimit 6000 ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit) { 6001 // If the value is a constant 6002 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) { 6003 // If the value is already zero, the branch will execute zero times. 6004 if (C->getValue()->isZero()) return C; 6005 return getCouldNotCompute(); // Otherwise it will loop infinitely. 6006 } 6007 6008 const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V); 6009 if (!AddRec || AddRec->getLoop() != L) 6010 return getCouldNotCompute(); 6011 6012 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of 6013 // the quadratic equation to solve it. 6014 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) { 6015 std::pair<const SCEV *,const SCEV *> Roots = 6016 SolveQuadraticEquation(AddRec, *this); 6017 const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first); 6018 const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second); 6019 if (R1 && R2) { 6020 #if 0 6021 dbgs() << "HFTZ: " << *V << " - sol#1: " << *R1 6022 << " sol#2: " << *R2 << "\n"; 6023 #endif 6024 // Pick the smallest positive root value. 6025 if (ConstantInt *CB = 6026 dyn_cast<ConstantInt>(ConstantExpr::getICmp(CmpInst::ICMP_ULT, 6027 R1->getValue(), 6028 R2->getValue()))) { 6029 if (CB->getZExtValue() == false) 6030 std::swap(R1, R2); // R1 is the minimum root now. 6031 6032 // We can only use this value if the chrec ends up with an exact zero 6033 // value at this index. When solving for "X*X != 5", for example, we 6034 // should not accept a root of 2. 6035 const SCEV *Val = AddRec->evaluateAtIteration(R1, *this); 6036 if (Val->isZero()) 6037 return R1; // We found a quadratic root! 6038 } 6039 } 6040 return getCouldNotCompute(); 6041 } 6042 6043 // Otherwise we can only handle this if it is affine. 6044 if (!AddRec->isAffine()) 6045 return getCouldNotCompute(); 6046 6047 // If this is an affine expression, the execution count of this branch is 6048 // the minimum unsigned root of the following equation: 6049 // 6050 // Start + Step*N = 0 (mod 2^BW) 6051 // 6052 // equivalent to: 6053 // 6054 // Step*N = -Start (mod 2^BW) 6055 // 6056 // where BW is the common bit width of Start and Step. 6057 6058 // Get the initial value for the loop. 6059 const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop()); 6060 const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop()); 6061 6062 // For now we handle only constant steps. 6063 // 6064 // TODO: Handle a nonconstant Step given AddRec<NUW>. If the 6065 // AddRec is NUW, then (in an unsigned sense) it cannot be counting up to wrap 6066 // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step. 6067 // We have not yet seen any such cases. 6068 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step); 6069 if (!StepC || StepC->getValue()->equalsInt(0)) 6070 return getCouldNotCompute(); 6071 6072 // For positive steps (counting up until unsigned overflow): 6073 // N = -Start/Step (as unsigned) 6074 // For negative steps (counting down to zero): 6075 // N = Start/-Step 6076 // First compute the unsigned distance from zero in the direction of Step. 6077 bool CountDown = StepC->getValue()->getValue().isNegative(); 6078 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start); 6079 6080 // Handle unitary steps, which cannot wraparound. 6081 // 1*N = -Start; -1*N = Start (mod 2^BW), so: 6082 // N = Distance (as unsigned) 6083 if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) { 6084 ConstantRange CR = getUnsignedRange(Start); 6085 const SCEV *MaxBECount; 6086 if (!CountDown && CR.getUnsignedMin().isMinValue()) 6087 // When counting up, the worst starting value is 1, not 0. 6088 MaxBECount = CR.getUnsignedMax().isMinValue() 6089 ? getConstant(APInt::getMinValue(CR.getBitWidth())) 6090 : getConstant(APInt::getMaxValue(CR.getBitWidth())); 6091 else 6092 MaxBECount = getConstant(CountDown ? CR.getUnsignedMax() 6093 : -CR.getUnsignedMin()); 6094 return ExitLimit(Distance, MaxBECount); 6095 } 6096 6097 // As a special case, handle the instance where Step is a positive power of 6098 // two. In this case, determining whether Step divides Distance evenly can be 6099 // done by counting and comparing the number of trailing zeros of Step and 6100 // Distance. 6101 if (!CountDown) { 6102 const APInt &StepV = StepC->getValue()->getValue(); 6103 // StepV.isPowerOf2() returns true if StepV is an positive power of two. It 6104 // also returns true if StepV is maximally negative (eg, INT_MIN), but that 6105 // case is not handled as this code is guarded by !CountDown. 6106 if (StepV.isPowerOf2() && 6107 GetMinTrailingZeros(Distance) >= StepV.countTrailingZeros()) 6108 return getUDivExactExpr(Distance, Step); 6109 } 6110 6111 // If the condition controls loop exit (the loop exits only if the expression 6112 // is true) and the addition is no-wrap we can use unsigned divide to 6113 // compute the backedge count. In this case, the step may not divide the 6114 // distance, but we don't care because if the condition is "missed" the loop 6115 // will have undefined behavior due to wrapping. 6116 if (ControlsExit && AddRec->getNoWrapFlags(SCEV::FlagNW)) { 6117 const SCEV *Exact = 6118 getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step); 6119 return ExitLimit(Exact, Exact); 6120 } 6121 6122 // Then, try to solve the above equation provided that Start is constant. 6123 if (const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start)) 6124 return SolveLinEquationWithOverflow(StepC->getValue()->getValue(), 6125 -StartC->getValue()->getValue(), 6126 *this); 6127 return getCouldNotCompute(); 6128 } 6129 6130 /// HowFarToNonZero - Return the number of times a backedge checking the 6131 /// specified value for nonzero will execute. If not computable, return 6132 /// CouldNotCompute 6133 ScalarEvolution::ExitLimit 6134 ScalarEvolution::HowFarToNonZero(const SCEV *V, const Loop *L) { 6135 // Loops that look like: while (X == 0) are very strange indeed. We don't 6136 // handle them yet except for the trivial case. This could be expanded in the 6137 // future as needed. 6138 6139 // If the value is a constant, check to see if it is known to be non-zero 6140 // already. If so, the backedge will execute zero times. 6141 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) { 6142 if (!C->getValue()->isNullValue()) 6143 return getConstant(C->getType(), 0); 6144 return getCouldNotCompute(); // Otherwise it will loop infinitely. 6145 } 6146 6147 // We could implement others, but I really doubt anyone writes loops like 6148 // this, and if they did, they would already be constant folded. 6149 return getCouldNotCompute(); 6150 } 6151 6152 /// getPredecessorWithUniqueSuccessorForBB - Return a predecessor of BB 6153 /// (which may not be an immediate predecessor) which has exactly one 6154 /// successor from which BB is reachable, or null if no such block is 6155 /// found. 6156 /// 6157 std::pair<BasicBlock *, BasicBlock *> 6158 ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB) { 6159 // If the block has a unique predecessor, then there is no path from the 6160 // predecessor to the block that does not go through the direct edge 6161 // from the predecessor to the block. 6162 if (BasicBlock *Pred = BB->getSinglePredecessor()) 6163 return std::make_pair(Pred, BB); 6164 6165 // A loop's header is defined to be a block that dominates the loop. 6166 // If the header has a unique predecessor outside the loop, it must be 6167 // a block that has exactly one successor that can reach the loop. 6168 if (Loop *L = LI->getLoopFor(BB)) 6169 return std::make_pair(L->getLoopPredecessor(), L->getHeader()); 6170 6171 return std::pair<BasicBlock *, BasicBlock *>(); 6172 } 6173 6174 /// HasSameValue - SCEV structural equivalence is usually sufficient for 6175 /// testing whether two expressions are equal, however for the purposes of 6176 /// looking for a condition guarding a loop, it can be useful to be a little 6177 /// more general, since a front-end may have replicated the controlling 6178 /// expression. 6179 /// 6180 static bool HasSameValue(const SCEV *A, const SCEV *B) { 6181 // Quick check to see if they are the same SCEV. 6182 if (A == B) return true; 6183 6184 // Otherwise, if they're both SCEVUnknown, it's possible that they hold 6185 // two different instructions with the same value. Check for this case. 6186 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A)) 6187 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B)) 6188 if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue())) 6189 if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue())) 6190 if (AI->isIdenticalTo(BI) && !AI->mayReadFromMemory()) 6191 return true; 6192 6193 // Otherwise assume they may have a different value. 6194 return false; 6195 } 6196 6197 /// SimplifyICmpOperands - Simplify LHS and RHS in a comparison with 6198 /// predicate Pred. Return true iff any changes were made. 6199 /// 6200 bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, 6201 const SCEV *&LHS, const SCEV *&RHS, 6202 unsigned Depth) { 6203 bool Changed = false; 6204 6205 // If we hit the max recursion limit bail out. 6206 if (Depth >= 3) 6207 return false; 6208 6209 // Canonicalize a constant to the right side. 6210 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { 6211 // Check for both operands constant. 6212 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) { 6213 if (ConstantExpr::getICmp(Pred, 6214 LHSC->getValue(), 6215 RHSC->getValue())->isNullValue()) 6216 goto trivially_false; 6217 else 6218 goto trivially_true; 6219 } 6220 // Otherwise swap the operands to put the constant on the right. 6221 std::swap(LHS, RHS); 6222 Pred = ICmpInst::getSwappedPredicate(Pred); 6223 Changed = true; 6224 } 6225 6226 // If we're comparing an addrec with a value which is loop-invariant in the 6227 // addrec's loop, put the addrec on the left. Also make a dominance check, 6228 // as both operands could be addrecs loop-invariant in each other's loop. 6229 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) { 6230 const Loop *L = AR->getLoop(); 6231 if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) { 6232 std::swap(LHS, RHS); 6233 Pred = ICmpInst::getSwappedPredicate(Pred); 6234 Changed = true; 6235 } 6236 } 6237 6238 // If there's a constant operand, canonicalize comparisons with boundary 6239 // cases, and canonicalize *-or-equal comparisons to regular comparisons. 6240 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) { 6241 const APInt &RA = RC->getValue()->getValue(); 6242 switch (Pred) { 6243 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 6244 case ICmpInst::ICMP_EQ: 6245 case ICmpInst::ICMP_NE: 6246 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b. 6247 if (!RA) 6248 if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(LHS)) 6249 if (const SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(AE->getOperand(0))) 6250 if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 && 6251 ME->getOperand(0)->isAllOnesValue()) { 6252 RHS = AE->getOperand(1); 6253 LHS = ME->getOperand(1); 6254 Changed = true; 6255 } 6256 break; 6257 case ICmpInst::ICMP_UGE: 6258 if ((RA - 1).isMinValue()) { 6259 Pred = ICmpInst::ICMP_NE; 6260 RHS = getConstant(RA - 1); 6261 Changed = true; 6262 break; 6263 } 6264 if (RA.isMaxValue()) { 6265 Pred = ICmpInst::ICMP_EQ; 6266 Changed = true; 6267 break; 6268 } 6269 if (RA.isMinValue()) goto trivially_true; 6270 6271 Pred = ICmpInst::ICMP_UGT; 6272 RHS = getConstant(RA - 1); 6273 Changed = true; 6274 break; 6275 case ICmpInst::ICMP_ULE: 6276 if ((RA + 1).isMaxValue()) { 6277 Pred = ICmpInst::ICMP_NE; 6278 RHS = getConstant(RA + 1); 6279 Changed = true; 6280 break; 6281 } 6282 if (RA.isMinValue()) { 6283 Pred = ICmpInst::ICMP_EQ; 6284 Changed = true; 6285 break; 6286 } 6287 if (RA.isMaxValue()) goto trivially_true; 6288 6289 Pred = ICmpInst::ICMP_ULT; 6290 RHS = getConstant(RA + 1); 6291 Changed = true; 6292 break; 6293 case ICmpInst::ICMP_SGE: 6294 if ((RA - 1).isMinSignedValue()) { 6295 Pred = ICmpInst::ICMP_NE; 6296 RHS = getConstant(RA - 1); 6297 Changed = true; 6298 break; 6299 } 6300 if (RA.isMaxSignedValue()) { 6301 Pred = ICmpInst::ICMP_EQ; 6302 Changed = true; 6303 break; 6304 } 6305 if (RA.isMinSignedValue()) goto trivially_true; 6306 6307 Pred = ICmpInst::ICMP_SGT; 6308 RHS = getConstant(RA - 1); 6309 Changed = true; 6310 break; 6311 case ICmpInst::ICMP_SLE: 6312 if ((RA + 1).isMaxSignedValue()) { 6313 Pred = ICmpInst::ICMP_NE; 6314 RHS = getConstant(RA + 1); 6315 Changed = true; 6316 break; 6317 } 6318 if (RA.isMinSignedValue()) { 6319 Pred = ICmpInst::ICMP_EQ; 6320 Changed = true; 6321 break; 6322 } 6323 if (RA.isMaxSignedValue()) goto trivially_true; 6324 6325 Pred = ICmpInst::ICMP_SLT; 6326 RHS = getConstant(RA + 1); 6327 Changed = true; 6328 break; 6329 case ICmpInst::ICMP_UGT: 6330 if (RA.isMinValue()) { 6331 Pred = ICmpInst::ICMP_NE; 6332 Changed = true; 6333 break; 6334 } 6335 if ((RA + 1).isMaxValue()) { 6336 Pred = ICmpInst::ICMP_EQ; 6337 RHS = getConstant(RA + 1); 6338 Changed = true; 6339 break; 6340 } 6341 if (RA.isMaxValue()) goto trivially_false; 6342 break; 6343 case ICmpInst::ICMP_ULT: 6344 if (RA.isMaxValue()) { 6345 Pred = ICmpInst::ICMP_NE; 6346 Changed = true; 6347 break; 6348 } 6349 if ((RA - 1).isMinValue()) { 6350 Pred = ICmpInst::ICMP_EQ; 6351 RHS = getConstant(RA - 1); 6352 Changed = true; 6353 break; 6354 } 6355 if (RA.isMinValue()) goto trivially_false; 6356 break; 6357 case ICmpInst::ICMP_SGT: 6358 if (RA.isMinSignedValue()) { 6359 Pred = ICmpInst::ICMP_NE; 6360 Changed = true; 6361 break; 6362 } 6363 if ((RA + 1).isMaxSignedValue()) { 6364 Pred = ICmpInst::ICMP_EQ; 6365 RHS = getConstant(RA + 1); 6366 Changed = true; 6367 break; 6368 } 6369 if (RA.isMaxSignedValue()) goto trivially_false; 6370 break; 6371 case ICmpInst::ICMP_SLT: 6372 if (RA.isMaxSignedValue()) { 6373 Pred = ICmpInst::ICMP_NE; 6374 Changed = true; 6375 break; 6376 } 6377 if ((RA - 1).isMinSignedValue()) { 6378 Pred = ICmpInst::ICMP_EQ; 6379 RHS = getConstant(RA - 1); 6380 Changed = true; 6381 break; 6382 } 6383 if (RA.isMinSignedValue()) goto trivially_false; 6384 break; 6385 } 6386 } 6387 6388 // Check for obvious equality. 6389 if (HasSameValue(LHS, RHS)) { 6390 if (ICmpInst::isTrueWhenEqual(Pred)) 6391 goto trivially_true; 6392 if (ICmpInst::isFalseWhenEqual(Pred)) 6393 goto trivially_false; 6394 } 6395 6396 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by 6397 // adding or subtracting 1 from one of the operands. 6398 switch (Pred) { 6399 case ICmpInst::ICMP_SLE: 6400 if (!getSignedRange(RHS).getSignedMax().isMaxSignedValue()) { 6401 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, 6402 SCEV::FlagNSW); 6403 Pred = ICmpInst::ICMP_SLT; 6404 Changed = true; 6405 } else if (!getSignedRange(LHS).getSignedMin().isMinSignedValue()) { 6406 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS, 6407 SCEV::FlagNSW); 6408 Pred = ICmpInst::ICMP_SLT; 6409 Changed = true; 6410 } 6411 break; 6412 case ICmpInst::ICMP_SGE: 6413 if (!getSignedRange(RHS).getSignedMin().isMinSignedValue()) { 6414 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS, 6415 SCEV::FlagNSW); 6416 Pred = ICmpInst::ICMP_SGT; 6417 Changed = true; 6418 } else if (!getSignedRange(LHS).getSignedMax().isMaxSignedValue()) { 6419 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, 6420 SCEV::FlagNSW); 6421 Pred = ICmpInst::ICMP_SGT; 6422 Changed = true; 6423 } 6424 break; 6425 case ICmpInst::ICMP_ULE: 6426 if (!getUnsignedRange(RHS).getUnsignedMax().isMaxValue()) { 6427 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, 6428 SCEV::FlagNUW); 6429 Pred = ICmpInst::ICMP_ULT; 6430 Changed = true; 6431 } else if (!getUnsignedRange(LHS).getUnsignedMin().isMinValue()) { 6432 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS, 6433 SCEV::FlagNUW); 6434 Pred = ICmpInst::ICMP_ULT; 6435 Changed = true; 6436 } 6437 break; 6438 case ICmpInst::ICMP_UGE: 6439 if (!getUnsignedRange(RHS).getUnsignedMin().isMinValue()) { 6440 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS, 6441 SCEV::FlagNUW); 6442 Pred = ICmpInst::ICMP_UGT; 6443 Changed = true; 6444 } else if (!getUnsignedRange(LHS).getUnsignedMax().isMaxValue()) { 6445 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, 6446 SCEV::FlagNUW); 6447 Pred = ICmpInst::ICMP_UGT; 6448 Changed = true; 6449 } 6450 break; 6451 default: 6452 break; 6453 } 6454 6455 // TODO: More simplifications are possible here. 6456 6457 // Recursively simplify until we either hit a recursion limit or nothing 6458 // changes. 6459 if (Changed) 6460 return SimplifyICmpOperands(Pred, LHS, RHS, Depth+1); 6461 6462 return Changed; 6463 6464 trivially_true: 6465 // Return 0 == 0. 6466 LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); 6467 Pred = ICmpInst::ICMP_EQ; 6468 return true; 6469 6470 trivially_false: 6471 // Return 0 != 0. 6472 LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); 6473 Pred = ICmpInst::ICMP_NE; 6474 return true; 6475 } 6476 6477 bool ScalarEvolution::isKnownNegative(const SCEV *S) { 6478 return getSignedRange(S).getSignedMax().isNegative(); 6479 } 6480 6481 bool ScalarEvolution::isKnownPositive(const SCEV *S) { 6482 return getSignedRange(S).getSignedMin().isStrictlyPositive(); 6483 } 6484 6485 bool ScalarEvolution::isKnownNonNegative(const SCEV *S) { 6486 return !getSignedRange(S).getSignedMin().isNegative(); 6487 } 6488 6489 bool ScalarEvolution::isKnownNonPositive(const SCEV *S) { 6490 return !getSignedRange(S).getSignedMax().isStrictlyPositive(); 6491 } 6492 6493 bool ScalarEvolution::isKnownNonZero(const SCEV *S) { 6494 return isKnownNegative(S) || isKnownPositive(S); 6495 } 6496 6497 bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred, 6498 const SCEV *LHS, const SCEV *RHS) { 6499 // Canonicalize the inputs first. 6500 (void)SimplifyICmpOperands(Pred, LHS, RHS); 6501 6502 // If LHS or RHS is an addrec, check to see if the condition is true in 6503 // every iteration of the loop. 6504 // If LHS and RHS are both addrec, both conditions must be true in 6505 // every iteration of the loop. 6506 const SCEVAddRecExpr *LAR = dyn_cast<SCEVAddRecExpr>(LHS); 6507 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS); 6508 bool LeftGuarded = false; 6509 bool RightGuarded = false; 6510 if (LAR) { 6511 const Loop *L = LAR->getLoop(); 6512 if (isLoopEntryGuardedByCond(L, Pred, LAR->getStart(), RHS) && 6513 isLoopBackedgeGuardedByCond(L, Pred, LAR->getPostIncExpr(*this), RHS)) { 6514 if (!RAR) return true; 6515 LeftGuarded = true; 6516 } 6517 } 6518 if (RAR) { 6519 const Loop *L = RAR->getLoop(); 6520 if (isLoopEntryGuardedByCond(L, Pred, LHS, RAR->getStart()) && 6521 isLoopBackedgeGuardedByCond(L, Pred, LHS, RAR->getPostIncExpr(*this))) { 6522 if (!LAR) return true; 6523 RightGuarded = true; 6524 } 6525 } 6526 if (LeftGuarded && RightGuarded) 6527 return true; 6528 6529 // Otherwise see what can be done with known constant ranges. 6530 return isKnownPredicateWithRanges(Pred, LHS, RHS); 6531 } 6532 6533 bool 6534 ScalarEvolution::isKnownPredicateWithRanges(ICmpInst::Predicate Pred, 6535 const SCEV *LHS, const SCEV *RHS) { 6536 if (HasSameValue(LHS, RHS)) 6537 return ICmpInst::isTrueWhenEqual(Pred); 6538 6539 // This code is split out from isKnownPredicate because it is called from 6540 // within isLoopEntryGuardedByCond. 6541 switch (Pred) { 6542 default: 6543 llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 6544 case ICmpInst::ICMP_SGT: 6545 std::swap(LHS, RHS); 6546 case ICmpInst::ICMP_SLT: { 6547 ConstantRange LHSRange = getSignedRange(LHS); 6548 ConstantRange RHSRange = getSignedRange(RHS); 6549 if (LHSRange.getSignedMax().slt(RHSRange.getSignedMin())) 6550 return true; 6551 if (LHSRange.getSignedMin().sge(RHSRange.getSignedMax())) 6552 return false; 6553 break; 6554 } 6555 case ICmpInst::ICMP_SGE: 6556 std::swap(LHS, RHS); 6557 case ICmpInst::ICMP_SLE: { 6558 ConstantRange LHSRange = getSignedRange(LHS); 6559 ConstantRange RHSRange = getSignedRange(RHS); 6560 if (LHSRange.getSignedMax().sle(RHSRange.getSignedMin())) 6561 return true; 6562 if (LHSRange.getSignedMin().sgt(RHSRange.getSignedMax())) 6563 return false; 6564 break; 6565 } 6566 case ICmpInst::ICMP_UGT: 6567 std::swap(LHS, RHS); 6568 case ICmpInst::ICMP_ULT: { 6569 ConstantRange LHSRange = getUnsignedRange(LHS); 6570 ConstantRange RHSRange = getUnsignedRange(RHS); 6571 if (LHSRange.getUnsignedMax().ult(RHSRange.getUnsignedMin())) 6572 return true; 6573 if (LHSRange.getUnsignedMin().uge(RHSRange.getUnsignedMax())) 6574 return false; 6575 break; 6576 } 6577 case ICmpInst::ICMP_UGE: 6578 std::swap(LHS, RHS); 6579 case ICmpInst::ICMP_ULE: { 6580 ConstantRange LHSRange = getUnsignedRange(LHS); 6581 ConstantRange RHSRange = getUnsignedRange(RHS); 6582 if (LHSRange.getUnsignedMax().ule(RHSRange.getUnsignedMin())) 6583 return true; 6584 if (LHSRange.getUnsignedMin().ugt(RHSRange.getUnsignedMax())) 6585 return false; 6586 break; 6587 } 6588 case ICmpInst::ICMP_NE: { 6589 if (getUnsignedRange(LHS).intersectWith(getUnsignedRange(RHS)).isEmptySet()) 6590 return true; 6591 if (getSignedRange(LHS).intersectWith(getSignedRange(RHS)).isEmptySet()) 6592 return true; 6593 6594 const SCEV *Diff = getMinusSCEV(LHS, RHS); 6595 if (isKnownNonZero(Diff)) 6596 return true; 6597 break; 6598 } 6599 case ICmpInst::ICMP_EQ: 6600 // The check at the top of the function catches the case where 6601 // the values are known to be equal. 6602 break; 6603 } 6604 return false; 6605 } 6606 6607 /// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is 6608 /// protected by a conditional between LHS and RHS. This is used to 6609 /// to eliminate casts. 6610 bool 6611 ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L, 6612 ICmpInst::Predicate Pred, 6613 const SCEV *LHS, const SCEV *RHS) { 6614 // Interpret a null as meaning no loop, where there is obviously no guard 6615 // (interprocedural conditions notwithstanding). 6616 if (!L) return true; 6617 6618 if (isKnownPredicateWithRanges(Pred, LHS, RHS)) return true; 6619 6620 BasicBlock *Latch = L->getLoopLatch(); 6621 if (!Latch) 6622 return false; 6623 6624 BranchInst *LoopContinuePredicate = 6625 dyn_cast<BranchInst>(Latch->getTerminator()); 6626 if (LoopContinuePredicate && LoopContinuePredicate->isConditional() && 6627 isImpliedCond(Pred, LHS, RHS, 6628 LoopContinuePredicate->getCondition(), 6629 LoopContinuePredicate->getSuccessor(0) != L->getHeader())) 6630 return true; 6631 6632 // Check conditions due to any @llvm.assume intrinsics. 6633 for (auto &CI : AT->assumptions(F)) { 6634 if (!DT->dominates(CI, Latch->getTerminator())) 6635 continue; 6636 6637 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false)) 6638 return true; 6639 } 6640 6641 return false; 6642 } 6643 6644 /// isLoopEntryGuardedByCond - Test whether entry to the loop is protected 6645 /// by a conditional between LHS and RHS. This is used to help avoid max 6646 /// expressions in loop trip counts, and to eliminate casts. 6647 bool 6648 ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, 6649 ICmpInst::Predicate Pred, 6650 const SCEV *LHS, const SCEV *RHS) { 6651 // Interpret a null as meaning no loop, where there is obviously no guard 6652 // (interprocedural conditions notwithstanding). 6653 if (!L) return false; 6654 6655 if (isKnownPredicateWithRanges(Pred, LHS, RHS)) return true; 6656 6657 // Starting at the loop predecessor, climb up the predecessor chain, as long 6658 // as there are predecessors that can be found that have unique successors 6659 // leading to the original header. 6660 for (std::pair<BasicBlock *, BasicBlock *> 6661 Pair(L->getLoopPredecessor(), L->getHeader()); 6662 Pair.first; 6663 Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) { 6664 6665 BranchInst *LoopEntryPredicate = 6666 dyn_cast<BranchInst>(Pair.first->getTerminator()); 6667 if (!LoopEntryPredicate || 6668 LoopEntryPredicate->isUnconditional()) 6669 continue; 6670 6671 if (isImpliedCond(Pred, LHS, RHS, 6672 LoopEntryPredicate->getCondition(), 6673 LoopEntryPredicate->getSuccessor(0) != Pair.second)) 6674 return true; 6675 } 6676 6677 // Check conditions due to any @llvm.assume intrinsics. 6678 for (auto &CI : AT->assumptions(F)) { 6679 if (!DT->dominates(CI, L->getHeader())) 6680 continue; 6681 6682 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false)) 6683 return true; 6684 } 6685 6686 return false; 6687 } 6688 6689 /// RAII wrapper to prevent recursive application of isImpliedCond. 6690 /// ScalarEvolution's PendingLoopPredicates set must be empty unless we are 6691 /// currently evaluating isImpliedCond. 6692 struct MarkPendingLoopPredicate { 6693 Value *Cond; 6694 DenseSet<Value*> &LoopPreds; 6695 bool Pending; 6696 6697 MarkPendingLoopPredicate(Value *C, DenseSet<Value*> &LP) 6698 : Cond(C), LoopPreds(LP) { 6699 Pending = !LoopPreds.insert(Cond).second; 6700 } 6701 ~MarkPendingLoopPredicate() { 6702 if (!Pending) 6703 LoopPreds.erase(Cond); 6704 } 6705 }; 6706 6707 /// isImpliedCond - Test whether the condition described by Pred, LHS, 6708 /// and RHS is true whenever the given Cond value evaluates to true. 6709 bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, 6710 const SCEV *LHS, const SCEV *RHS, 6711 Value *FoundCondValue, 6712 bool Inverse) { 6713 MarkPendingLoopPredicate Mark(FoundCondValue, PendingLoopPredicates); 6714 if (Mark.Pending) 6715 return false; 6716 6717 // Recursively handle And and Or conditions. 6718 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) { 6719 if (BO->getOpcode() == Instruction::And) { 6720 if (!Inverse) 6721 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) || 6722 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse); 6723 } else if (BO->getOpcode() == Instruction::Or) { 6724 if (Inverse) 6725 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) || 6726 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse); 6727 } 6728 } 6729 6730 ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue); 6731 if (!ICI) return false; 6732 6733 // Bail if the ICmp's operands' types are wider than the needed type 6734 // before attempting to call getSCEV on them. This avoids infinite 6735 // recursion, since the analysis of widening casts can require loop 6736 // exit condition information for overflow checking, which would 6737 // lead back here. 6738 if (getTypeSizeInBits(LHS->getType()) < 6739 getTypeSizeInBits(ICI->getOperand(0)->getType())) 6740 return false; 6741 6742 // Now that we found a conditional branch that dominates the loop or controls 6743 // the loop latch. Check to see if it is the comparison we are looking for. 6744 ICmpInst::Predicate FoundPred; 6745 if (Inverse) 6746 FoundPred = ICI->getInversePredicate(); 6747 else 6748 FoundPred = ICI->getPredicate(); 6749 6750 const SCEV *FoundLHS = getSCEV(ICI->getOperand(0)); 6751 const SCEV *FoundRHS = getSCEV(ICI->getOperand(1)); 6752 6753 // Balance the types. The case where FoundLHS' type is wider than 6754 // LHS' type is checked for above. 6755 if (getTypeSizeInBits(LHS->getType()) > 6756 getTypeSizeInBits(FoundLHS->getType())) { 6757 if (CmpInst::isSigned(FoundPred)) { 6758 FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType()); 6759 FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType()); 6760 } else { 6761 FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType()); 6762 FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType()); 6763 } 6764 } 6765 6766 // Canonicalize the query to match the way instcombine will have 6767 // canonicalized the comparison. 6768 if (SimplifyICmpOperands(Pred, LHS, RHS)) 6769 if (LHS == RHS) 6770 return CmpInst::isTrueWhenEqual(Pred); 6771 if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS)) 6772 if (FoundLHS == FoundRHS) 6773 return CmpInst::isFalseWhenEqual(FoundPred); 6774 6775 // Check to see if we can make the LHS or RHS match. 6776 if (LHS == FoundRHS || RHS == FoundLHS) { 6777 if (isa<SCEVConstant>(RHS)) { 6778 std::swap(FoundLHS, FoundRHS); 6779 FoundPred = ICmpInst::getSwappedPredicate(FoundPred); 6780 } else { 6781 std::swap(LHS, RHS); 6782 Pred = ICmpInst::getSwappedPredicate(Pred); 6783 } 6784 } 6785 6786 // Check whether the found predicate is the same as the desired predicate. 6787 if (FoundPred == Pred) 6788 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS); 6789 6790 // Check whether swapping the found predicate makes it the same as the 6791 // desired predicate. 6792 if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) { 6793 if (isa<SCEVConstant>(RHS)) 6794 return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS); 6795 else 6796 return isImpliedCondOperands(ICmpInst::getSwappedPredicate(Pred), 6797 RHS, LHS, FoundLHS, FoundRHS); 6798 } 6799 6800 // Check if we can make progress by sharpening ranges. 6801 if (FoundPred == ICmpInst::ICMP_NE && 6802 (isa<SCEVConstant>(FoundLHS) || isa<SCEVConstant>(FoundRHS))) { 6803 6804 const SCEVConstant *C = nullptr; 6805 const SCEV *V = nullptr; 6806 6807 if (isa<SCEVConstant>(FoundLHS)) { 6808 C = cast<SCEVConstant>(FoundLHS); 6809 V = FoundRHS; 6810 } else { 6811 C = cast<SCEVConstant>(FoundRHS); 6812 V = FoundLHS; 6813 } 6814 6815 // The guarding predicate tells us that C != V. If the known range 6816 // of V is [C, t), we can sharpen the range to [C + 1, t). The 6817 // range we consider has to correspond to same signedness as the 6818 // predicate we're interested in folding. 6819 6820 APInt Min = ICmpInst::isSigned(Pred) ? 6821 getSignedRange(V).getSignedMin() : getUnsignedRange(V).getUnsignedMin(); 6822 6823 if (Min == C->getValue()->getValue()) { 6824 // Given (V >= Min && V != Min) we conclude V >= (Min + 1). 6825 // This is true even if (Min + 1) wraps around -- in case of 6826 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)). 6827 6828 APInt SharperMin = Min + 1; 6829 6830 switch (Pred) { 6831 case ICmpInst::ICMP_SGE: 6832 case ICmpInst::ICMP_UGE: 6833 // We know V `Pred` SharperMin. If this implies LHS `Pred` 6834 // RHS, we're done. 6835 if (isImpliedCondOperands(Pred, LHS, RHS, V, 6836 getConstant(SharperMin))) 6837 return true; 6838 6839 case ICmpInst::ICMP_SGT: 6840 case ICmpInst::ICMP_UGT: 6841 // We know from the range information that (V `Pred` Min || 6842 // V == Min). We know from the guarding condition that !(V 6843 // == Min). This gives us 6844 // 6845 // V `Pred` Min || V == Min && !(V == Min) 6846 // => V `Pred` Min 6847 // 6848 // If V `Pred` Min implies LHS `Pred` RHS, we're done. 6849 6850 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min))) 6851 return true; 6852 6853 default: 6854 // No change 6855 break; 6856 } 6857 } 6858 } 6859 6860 // Check whether the actual condition is beyond sufficient. 6861 if (FoundPred == ICmpInst::ICMP_EQ) 6862 if (ICmpInst::isTrueWhenEqual(Pred)) 6863 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS)) 6864 return true; 6865 if (Pred == ICmpInst::ICMP_NE) 6866 if (!ICmpInst::isTrueWhenEqual(FoundPred)) 6867 if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS)) 6868 return true; 6869 6870 // Otherwise assume the worst. 6871 return false; 6872 } 6873 6874 /// isImpliedCondOperands - Test whether the condition described by Pred, 6875 /// LHS, and RHS is true whenever the condition described by Pred, FoundLHS, 6876 /// and FoundRHS is true. 6877 bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred, 6878 const SCEV *LHS, const SCEV *RHS, 6879 const SCEV *FoundLHS, 6880 const SCEV *FoundRHS) { 6881 return isImpliedCondOperandsHelper(Pred, LHS, RHS, 6882 FoundLHS, FoundRHS) || 6883 // ~x < ~y --> x > y 6884 isImpliedCondOperandsHelper(Pred, LHS, RHS, 6885 getNotSCEV(FoundRHS), 6886 getNotSCEV(FoundLHS)); 6887 } 6888 6889 6890 /// If Expr computes ~A, return A else return nullptr 6891 static const SCEV *MatchNotExpr(const SCEV *Expr) { 6892 const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Expr); 6893 if (!Add || Add->getNumOperands() != 2) return nullptr; 6894 6895 const SCEVConstant *AddLHS = dyn_cast<SCEVConstant>(Add->getOperand(0)); 6896 if (!(AddLHS && AddLHS->getValue()->getValue().isAllOnesValue())) 6897 return nullptr; 6898 6899 const SCEVMulExpr *AddRHS = dyn_cast<SCEVMulExpr>(Add->getOperand(1)); 6900 if (!AddRHS || AddRHS->getNumOperands() != 2) return nullptr; 6901 6902 const SCEVConstant *MulLHS = dyn_cast<SCEVConstant>(AddRHS->getOperand(0)); 6903 if (!(MulLHS && MulLHS->getValue()->getValue().isAllOnesValue())) 6904 return nullptr; 6905 6906 return AddRHS->getOperand(1); 6907 } 6908 6909 6910 /// Is MaybeMaxExpr an SMax or UMax of Candidate and some other values? 6911 template<typename MaxExprType> 6912 static bool IsMaxConsistingOf(const SCEV *MaybeMaxExpr, 6913 const SCEV *Candidate) { 6914 const MaxExprType *MaxExpr = dyn_cast<MaxExprType>(MaybeMaxExpr); 6915 if (!MaxExpr) return false; 6916 6917 auto It = std::find(MaxExpr->op_begin(), MaxExpr->op_end(), Candidate); 6918 return It != MaxExpr->op_end(); 6919 } 6920 6921 6922 /// Is MaybeMinExpr an SMin or UMin of Candidate and some other values? 6923 template<typename MaxExprType> 6924 static bool IsMinConsistingOf(ScalarEvolution &SE, 6925 const SCEV *MaybeMinExpr, 6926 const SCEV *Candidate) { 6927 const SCEV *MaybeMaxExpr = MatchNotExpr(MaybeMinExpr); 6928 if (!MaybeMaxExpr) 6929 return false; 6930 6931 return IsMaxConsistingOf<MaxExprType>(MaybeMaxExpr, SE.getNotSCEV(Candidate)); 6932 } 6933 6934 6935 /// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max 6936 /// expression? 6937 static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, 6938 ICmpInst::Predicate Pred, 6939 const SCEV *LHS, const SCEV *RHS) { 6940 switch (Pred) { 6941 default: 6942 return false; 6943 6944 case ICmpInst::ICMP_SGE: 6945 std::swap(LHS, RHS); 6946 // fall through 6947 case ICmpInst::ICMP_SLE: 6948 return 6949 // min(A, ...) <= A 6950 IsMinConsistingOf<SCEVSMaxExpr>(SE, LHS, RHS) || 6951 // A <= max(A, ...) 6952 IsMaxConsistingOf<SCEVSMaxExpr>(RHS, LHS); 6953 6954 case ICmpInst::ICMP_UGE: 6955 std::swap(LHS, RHS); 6956 // fall through 6957 case ICmpInst::ICMP_ULE: 6958 return 6959 // min(A, ...) <= A 6960 IsMinConsistingOf<SCEVUMaxExpr>(SE, LHS, RHS) || 6961 // A <= max(A, ...) 6962 IsMaxConsistingOf<SCEVUMaxExpr>(RHS, LHS); 6963 } 6964 6965 llvm_unreachable("covered switch fell through?!"); 6966 } 6967 6968 /// isImpliedCondOperandsHelper - Test whether the condition described by 6969 /// Pred, LHS, and RHS is true whenever the condition described by Pred, 6970 /// FoundLHS, and FoundRHS is true. 6971 bool 6972 ScalarEvolution::isImpliedCondOperandsHelper(ICmpInst::Predicate Pred, 6973 const SCEV *LHS, const SCEV *RHS, 6974 const SCEV *FoundLHS, 6975 const SCEV *FoundRHS) { 6976 auto IsKnownPredicateFull = 6977 [this](ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS) { 6978 return isKnownPredicateWithRanges(Pred, LHS, RHS) || 6979 IsKnownPredicateViaMinOrMax(*this, Pred, LHS, RHS); 6980 }; 6981 6982 switch (Pred) { 6983 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 6984 case ICmpInst::ICMP_EQ: 6985 case ICmpInst::ICMP_NE: 6986 if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS)) 6987 return true; 6988 break; 6989 case ICmpInst::ICMP_SLT: 6990 case ICmpInst::ICMP_SLE: 6991 if (IsKnownPredicateFull(ICmpInst::ICMP_SLE, LHS, FoundLHS) && 6992 IsKnownPredicateFull(ICmpInst::ICMP_SGE, RHS, FoundRHS)) 6993 return true; 6994 break; 6995 case ICmpInst::ICMP_SGT: 6996 case ICmpInst::ICMP_SGE: 6997 if (IsKnownPredicateFull(ICmpInst::ICMP_SGE, LHS, FoundLHS) && 6998 IsKnownPredicateFull(ICmpInst::ICMP_SLE, RHS, FoundRHS)) 6999 return true; 7000 break; 7001 case ICmpInst::ICMP_ULT: 7002 case ICmpInst::ICMP_ULE: 7003 if (IsKnownPredicateFull(ICmpInst::ICMP_ULE, LHS, FoundLHS) && 7004 IsKnownPredicateFull(ICmpInst::ICMP_UGE, RHS, FoundRHS)) 7005 return true; 7006 break; 7007 case ICmpInst::ICMP_UGT: 7008 case ICmpInst::ICMP_UGE: 7009 if (IsKnownPredicateFull(ICmpInst::ICMP_UGE, LHS, FoundLHS) && 7010 IsKnownPredicateFull(ICmpInst::ICMP_ULE, RHS, FoundRHS)) 7011 return true; 7012 break; 7013 } 7014 7015 return false; 7016 } 7017 7018 // Verify if an linear IV with positive stride can overflow when in a 7019 // less-than comparison, knowing the invariant term of the comparison, the 7020 // stride and the knowledge of NSW/NUW flags on the recurrence. 7021 bool ScalarEvolution::doesIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride, 7022 bool IsSigned, bool NoWrap) { 7023 if (NoWrap) return false; 7024 7025 unsigned BitWidth = getTypeSizeInBits(RHS->getType()); 7026 const SCEV *One = getConstant(Stride->getType(), 1); 7027 7028 if (IsSigned) { 7029 APInt MaxRHS = getSignedRange(RHS).getSignedMax(); 7030 APInt MaxValue = APInt::getSignedMaxValue(BitWidth); 7031 APInt MaxStrideMinusOne = getSignedRange(getMinusSCEV(Stride, One)) 7032 .getSignedMax(); 7033 7034 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow! 7035 return (MaxValue - MaxStrideMinusOne).slt(MaxRHS); 7036 } 7037 7038 APInt MaxRHS = getUnsignedRange(RHS).getUnsignedMax(); 7039 APInt MaxValue = APInt::getMaxValue(BitWidth); 7040 APInt MaxStrideMinusOne = getUnsignedRange(getMinusSCEV(Stride, One)) 7041 .getUnsignedMax(); 7042 7043 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow! 7044 return (MaxValue - MaxStrideMinusOne).ult(MaxRHS); 7045 } 7046 7047 // Verify if an linear IV with negative stride can overflow when in a 7048 // greater-than comparison, knowing the invariant term of the comparison, 7049 // the stride and the knowledge of NSW/NUW flags on the recurrence. 7050 bool ScalarEvolution::doesIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride, 7051 bool IsSigned, bool NoWrap) { 7052 if (NoWrap) return false; 7053 7054 unsigned BitWidth = getTypeSizeInBits(RHS->getType()); 7055 const SCEV *One = getConstant(Stride->getType(), 1); 7056 7057 if (IsSigned) { 7058 APInt MinRHS = getSignedRange(RHS).getSignedMin(); 7059 APInt MinValue = APInt::getSignedMinValue(BitWidth); 7060 APInt MaxStrideMinusOne = getSignedRange(getMinusSCEV(Stride, One)) 7061 .getSignedMax(); 7062 7063 // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow! 7064 return (MinValue + MaxStrideMinusOne).sgt(MinRHS); 7065 } 7066 7067 APInt MinRHS = getUnsignedRange(RHS).getUnsignedMin(); 7068 APInt MinValue = APInt::getMinValue(BitWidth); 7069 APInt MaxStrideMinusOne = getUnsignedRange(getMinusSCEV(Stride, One)) 7070 .getUnsignedMax(); 7071 7072 // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow! 7073 return (MinValue + MaxStrideMinusOne).ugt(MinRHS); 7074 } 7075 7076 // Compute the backedge taken count knowing the interval difference, the 7077 // stride and presence of the equality in the comparison. 7078 const SCEV *ScalarEvolution::computeBECount(const SCEV *Delta, const SCEV *Step, 7079 bool Equality) { 7080 const SCEV *One = getConstant(Step->getType(), 1); 7081 Delta = Equality ? getAddExpr(Delta, Step) 7082 : getAddExpr(Delta, getMinusSCEV(Step, One)); 7083 return getUDivExpr(Delta, Step); 7084 } 7085 7086 /// HowManyLessThans - Return the number of times a backedge containing the 7087 /// specified less-than comparison will execute. If not computable, return 7088 /// CouldNotCompute. 7089 /// 7090 /// @param ControlsExit is true when the LHS < RHS condition directly controls 7091 /// the branch (loops exits only if condition is true). In this case, we can use 7092 /// NoWrapFlags to skip overflow checks. 7093 ScalarEvolution::ExitLimit 7094 ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS, 7095 const Loop *L, bool IsSigned, 7096 bool ControlsExit) { 7097 // We handle only IV < Invariant 7098 if (!isLoopInvariant(RHS, L)) 7099 return getCouldNotCompute(); 7100 7101 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS); 7102 7103 // Avoid weird loops 7104 if (!IV || IV->getLoop() != L || !IV->isAffine()) 7105 return getCouldNotCompute(); 7106 7107 bool NoWrap = ControlsExit && 7108 IV->getNoWrapFlags(IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW); 7109 7110 const SCEV *Stride = IV->getStepRecurrence(*this); 7111 7112 // Avoid negative or zero stride values 7113 if (!isKnownPositive(Stride)) 7114 return getCouldNotCompute(); 7115 7116 // Avoid proven overflow cases: this will ensure that the backedge taken count 7117 // will not generate any unsigned overflow. Relaxed no-overflow conditions 7118 // exploit NoWrapFlags, allowing to optimize in presence of undefined 7119 // behaviors like the case of C language. 7120 if (!Stride->isOne() && doesIVOverflowOnLT(RHS, Stride, IsSigned, NoWrap)) 7121 return getCouldNotCompute(); 7122 7123 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT 7124 : ICmpInst::ICMP_ULT; 7125 const SCEV *Start = IV->getStart(); 7126 const SCEV *End = RHS; 7127 if (!isLoopEntryGuardedByCond(L, Cond, getMinusSCEV(Start, Stride), RHS)) { 7128 const SCEV *Diff = getMinusSCEV(RHS, Start); 7129 // If we have NoWrap set, then we can assume that the increment won't 7130 // overflow, in which case if RHS - Start is a constant, we don't need to 7131 // do a max operation since we can just figure it out statically 7132 if (NoWrap && isa<SCEVConstant>(Diff)) { 7133 APInt D = dyn_cast<const SCEVConstant>(Diff)->getValue()->getValue(); 7134 if (D.isNegative()) 7135 End = Start; 7136 } else 7137 End = IsSigned ? getSMaxExpr(RHS, Start) 7138 : getUMaxExpr(RHS, Start); 7139 } 7140 7141 const SCEV *BECount = computeBECount(getMinusSCEV(End, Start), Stride, false); 7142 7143 APInt MinStart = IsSigned ? getSignedRange(Start).getSignedMin() 7144 : getUnsignedRange(Start).getUnsignedMin(); 7145 7146 APInt MinStride = IsSigned ? getSignedRange(Stride).getSignedMin() 7147 : getUnsignedRange(Stride).getUnsignedMin(); 7148 7149 unsigned BitWidth = getTypeSizeInBits(LHS->getType()); 7150 APInt Limit = IsSigned ? APInt::getSignedMaxValue(BitWidth) - (MinStride - 1) 7151 : APInt::getMaxValue(BitWidth) - (MinStride - 1); 7152 7153 // Although End can be a MAX expression we estimate MaxEnd considering only 7154 // the case End = RHS. This is safe because in the other case (End - Start) 7155 // is zero, leading to a zero maximum backedge taken count. 7156 APInt MaxEnd = 7157 IsSigned ? APIntOps::smin(getSignedRange(RHS).getSignedMax(), Limit) 7158 : APIntOps::umin(getUnsignedRange(RHS).getUnsignedMax(), Limit); 7159 7160 const SCEV *MaxBECount; 7161 if (isa<SCEVConstant>(BECount)) 7162 MaxBECount = BECount; 7163 else 7164 MaxBECount = computeBECount(getConstant(MaxEnd - MinStart), 7165 getConstant(MinStride), false); 7166 7167 if (isa<SCEVCouldNotCompute>(MaxBECount)) 7168 MaxBECount = BECount; 7169 7170 return ExitLimit(BECount, MaxBECount); 7171 } 7172 7173 ScalarEvolution::ExitLimit 7174 ScalarEvolution::HowManyGreaterThans(const SCEV *LHS, const SCEV *RHS, 7175 const Loop *L, bool IsSigned, 7176 bool ControlsExit) { 7177 // We handle only IV > Invariant 7178 if (!isLoopInvariant(RHS, L)) 7179 return getCouldNotCompute(); 7180 7181 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS); 7182 7183 // Avoid weird loops 7184 if (!IV || IV->getLoop() != L || !IV->isAffine()) 7185 return getCouldNotCompute(); 7186 7187 bool NoWrap = ControlsExit && 7188 IV->getNoWrapFlags(IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW); 7189 7190 const SCEV *Stride = getNegativeSCEV(IV->getStepRecurrence(*this)); 7191 7192 // Avoid negative or zero stride values 7193 if (!isKnownPositive(Stride)) 7194 return getCouldNotCompute(); 7195 7196 // Avoid proven overflow cases: this will ensure that the backedge taken count 7197 // will not generate any unsigned overflow. Relaxed no-overflow conditions 7198 // exploit NoWrapFlags, allowing to optimize in presence of undefined 7199 // behaviors like the case of C language. 7200 if (!Stride->isOne() && doesIVOverflowOnGT(RHS, Stride, IsSigned, NoWrap)) 7201 return getCouldNotCompute(); 7202 7203 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT 7204 : ICmpInst::ICMP_UGT; 7205 7206 const SCEV *Start = IV->getStart(); 7207 const SCEV *End = RHS; 7208 if (!isLoopEntryGuardedByCond(L, Cond, getAddExpr(Start, Stride), RHS)) { 7209 const SCEV *Diff = getMinusSCEV(RHS, Start); 7210 // If we have NoWrap set, then we can assume that the increment won't 7211 // overflow, in which case if RHS - Start is a constant, we don't need to 7212 // do a max operation since we can just figure it out statically 7213 if (NoWrap && isa<SCEVConstant>(Diff)) { 7214 APInt D = dyn_cast<const SCEVConstant>(Diff)->getValue()->getValue(); 7215 if (!D.isNegative()) 7216 End = Start; 7217 } else 7218 End = IsSigned ? getSMinExpr(RHS, Start) 7219 : getUMinExpr(RHS, Start); 7220 } 7221 7222 const SCEV *BECount = computeBECount(getMinusSCEV(Start, End), Stride, false); 7223 7224 APInt MaxStart = IsSigned ? getSignedRange(Start).getSignedMax() 7225 : getUnsignedRange(Start).getUnsignedMax(); 7226 7227 APInt MinStride = IsSigned ? getSignedRange(Stride).getSignedMin() 7228 : getUnsignedRange(Stride).getUnsignedMin(); 7229 7230 unsigned BitWidth = getTypeSizeInBits(LHS->getType()); 7231 APInt Limit = IsSigned ? APInt::getSignedMinValue(BitWidth) + (MinStride - 1) 7232 : APInt::getMinValue(BitWidth) + (MinStride - 1); 7233 7234 // Although End can be a MIN expression we estimate MinEnd considering only 7235 // the case End = RHS. This is safe because in the other case (Start - End) 7236 // is zero, leading to a zero maximum backedge taken count. 7237 APInt MinEnd = 7238 IsSigned ? APIntOps::smax(getSignedRange(RHS).getSignedMin(), Limit) 7239 : APIntOps::umax(getUnsignedRange(RHS).getUnsignedMin(), Limit); 7240 7241 7242 const SCEV *MaxBECount = getCouldNotCompute(); 7243 if (isa<SCEVConstant>(BECount)) 7244 MaxBECount = BECount; 7245 else 7246 MaxBECount = computeBECount(getConstant(MaxStart - MinEnd), 7247 getConstant(MinStride), false); 7248 7249 if (isa<SCEVCouldNotCompute>(MaxBECount)) 7250 MaxBECount = BECount; 7251 7252 return ExitLimit(BECount, MaxBECount); 7253 } 7254 7255 /// getNumIterationsInRange - Return the number of iterations of this loop that 7256 /// produce values in the specified constant range. Another way of looking at 7257 /// this is that it returns the first iteration number where the value is not in 7258 /// the condition, thus computing the exit count. If the iteration count can't 7259 /// be computed, an instance of SCEVCouldNotCompute is returned. 7260 const SCEV *SCEVAddRecExpr::getNumIterationsInRange(ConstantRange Range, 7261 ScalarEvolution &SE) const { 7262 if (Range.isFullSet()) // Infinite loop. 7263 return SE.getCouldNotCompute(); 7264 7265 // If the start is a non-zero constant, shift the range to simplify things. 7266 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(getStart())) 7267 if (!SC->getValue()->isZero()) { 7268 SmallVector<const SCEV *, 4> Operands(op_begin(), op_end()); 7269 Operands[0] = SE.getConstant(SC->getType(), 0); 7270 const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop(), 7271 getNoWrapFlags(FlagNW)); 7272 if (const SCEVAddRecExpr *ShiftedAddRec = 7273 dyn_cast<SCEVAddRecExpr>(Shifted)) 7274 return ShiftedAddRec->getNumIterationsInRange( 7275 Range.subtract(SC->getValue()->getValue()), SE); 7276 // This is strange and shouldn't happen. 7277 return SE.getCouldNotCompute(); 7278 } 7279 7280 // The only time we can solve this is when we have all constant indices. 7281 // Otherwise, we cannot determine the overflow conditions. 7282 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 7283 if (!isa<SCEVConstant>(getOperand(i))) 7284 return SE.getCouldNotCompute(); 7285 7286 7287 // Okay at this point we know that all elements of the chrec are constants and 7288 // that the start element is zero. 7289 7290 // First check to see if the range contains zero. If not, the first 7291 // iteration exits. 7292 unsigned BitWidth = SE.getTypeSizeInBits(getType()); 7293 if (!Range.contains(APInt(BitWidth, 0))) 7294 return SE.getConstant(getType(), 0); 7295 7296 if (isAffine()) { 7297 // If this is an affine expression then we have this situation: 7298 // Solve {0,+,A} in Range === Ax in Range 7299 7300 // We know that zero is in the range. If A is positive then we know that 7301 // the upper value of the range must be the first possible exit value. 7302 // If A is negative then the lower of the range is the last possible loop 7303 // value. Also note that we already checked for a full range. 7304 APInt One(BitWidth,1); 7305 APInt A = cast<SCEVConstant>(getOperand(1))->getValue()->getValue(); 7306 APInt End = A.sge(One) ? (Range.getUpper() - One) : Range.getLower(); 7307 7308 // The exit value should be (End+A)/A. 7309 APInt ExitVal = (End + A).udiv(A); 7310 ConstantInt *ExitValue = ConstantInt::get(SE.getContext(), ExitVal); 7311 7312 // Evaluate at the exit value. If we really did fall out of the valid 7313 // range, then we computed our trip count, otherwise wrap around or other 7314 // things must have happened. 7315 ConstantInt *Val = EvaluateConstantChrecAtConstant(this, ExitValue, SE); 7316 if (Range.contains(Val->getValue())) 7317 return SE.getCouldNotCompute(); // Something strange happened 7318 7319 // Ensure that the previous value is in the range. This is a sanity check. 7320 assert(Range.contains( 7321 EvaluateConstantChrecAtConstant(this, 7322 ConstantInt::get(SE.getContext(), ExitVal - One), SE)->getValue()) && 7323 "Linear scev computation is off in a bad way!"); 7324 return SE.getConstant(ExitValue); 7325 } else if (isQuadratic()) { 7326 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of the 7327 // quadratic equation to solve it. To do this, we must frame our problem in 7328 // terms of figuring out when zero is crossed, instead of when 7329 // Range.getUpper() is crossed. 7330 SmallVector<const SCEV *, 4> NewOps(op_begin(), op_end()); 7331 NewOps[0] = SE.getNegativeSCEV(SE.getConstant(Range.getUpper())); 7332 const SCEV *NewAddRec = SE.getAddRecExpr(NewOps, getLoop(), 7333 // getNoWrapFlags(FlagNW) 7334 FlagAnyWrap); 7335 7336 // Next, solve the constructed addrec 7337 std::pair<const SCEV *,const SCEV *> Roots = 7338 SolveQuadraticEquation(cast<SCEVAddRecExpr>(NewAddRec), SE); 7339 const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first); 7340 const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second); 7341 if (R1) { 7342 // Pick the smallest positive root value. 7343 if (ConstantInt *CB = 7344 dyn_cast<ConstantInt>(ConstantExpr::getICmp(ICmpInst::ICMP_ULT, 7345 R1->getValue(), R2->getValue()))) { 7346 if (CB->getZExtValue() == false) 7347 std::swap(R1, R2); // R1 is the minimum root now. 7348 7349 // Make sure the root is not off by one. The returned iteration should 7350 // not be in the range, but the previous one should be. When solving 7351 // for "X*X < 5", for example, we should not return a root of 2. 7352 ConstantInt *R1Val = EvaluateConstantChrecAtConstant(this, 7353 R1->getValue(), 7354 SE); 7355 if (Range.contains(R1Val->getValue())) { 7356 // The next iteration must be out of the range... 7357 ConstantInt *NextVal = 7358 ConstantInt::get(SE.getContext(), R1->getValue()->getValue()+1); 7359 7360 R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE); 7361 if (!Range.contains(R1Val->getValue())) 7362 return SE.getConstant(NextVal); 7363 return SE.getCouldNotCompute(); // Something strange happened 7364 } 7365 7366 // If R1 was not in the range, then it is a good return value. Make 7367 // sure that R1-1 WAS in the range though, just in case. 7368 ConstantInt *NextVal = 7369 ConstantInt::get(SE.getContext(), R1->getValue()->getValue()-1); 7370 R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE); 7371 if (Range.contains(R1Val->getValue())) 7372 return R1; 7373 return SE.getCouldNotCompute(); // Something strange happened 7374 } 7375 } 7376 } 7377 7378 return SE.getCouldNotCompute(); 7379 } 7380 7381 namespace { 7382 struct FindUndefs { 7383 bool Found; 7384 FindUndefs() : Found(false) {} 7385 7386 bool follow(const SCEV *S) { 7387 if (const SCEVUnknown *C = dyn_cast<SCEVUnknown>(S)) { 7388 if (isa<UndefValue>(C->getValue())) 7389 Found = true; 7390 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) { 7391 if (isa<UndefValue>(C->getValue())) 7392 Found = true; 7393 } 7394 7395 // Keep looking if we haven't found it yet. 7396 return !Found; 7397 } 7398 bool isDone() const { 7399 // Stop recursion if we have found an undef. 7400 return Found; 7401 } 7402 }; 7403 } 7404 7405 // Return true when S contains at least an undef value. 7406 static inline bool 7407 containsUndefs(const SCEV *S) { 7408 FindUndefs F; 7409 SCEVTraversal<FindUndefs> ST(F); 7410 ST.visitAll(S); 7411 7412 return F.Found; 7413 } 7414 7415 namespace { 7416 // Collect all steps of SCEV expressions. 7417 struct SCEVCollectStrides { 7418 ScalarEvolution &SE; 7419 SmallVectorImpl<const SCEV *> &Strides; 7420 7421 SCEVCollectStrides(ScalarEvolution &SE, SmallVectorImpl<const SCEV *> &S) 7422 : SE(SE), Strides(S) {} 7423 7424 bool follow(const SCEV *S) { 7425 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) 7426 Strides.push_back(AR->getStepRecurrence(SE)); 7427 return true; 7428 } 7429 bool isDone() const { return false; } 7430 }; 7431 7432 // Collect all SCEVUnknown and SCEVMulExpr expressions. 7433 struct SCEVCollectTerms { 7434 SmallVectorImpl<const SCEV *> &Terms; 7435 7436 SCEVCollectTerms(SmallVectorImpl<const SCEV *> &T) 7437 : Terms(T) {} 7438 7439 bool follow(const SCEV *S) { 7440 if (isa<SCEVUnknown>(S) || isa<SCEVMulExpr>(S)) { 7441 if (!containsUndefs(S)) 7442 Terms.push_back(S); 7443 7444 // Stop recursion: once we collected a term, do not walk its operands. 7445 return false; 7446 } 7447 7448 // Keep looking. 7449 return true; 7450 } 7451 bool isDone() const { return false; } 7452 }; 7453 } 7454 7455 /// Find parametric terms in this SCEVAddRecExpr. 7456 void SCEVAddRecExpr::collectParametricTerms( 7457 ScalarEvolution &SE, SmallVectorImpl<const SCEV *> &Terms) const { 7458 SmallVector<const SCEV *, 4> Strides; 7459 SCEVCollectStrides StrideCollector(SE, Strides); 7460 visitAll(this, StrideCollector); 7461 7462 DEBUG({ 7463 dbgs() << "Strides:\n"; 7464 for (const SCEV *S : Strides) 7465 dbgs() << *S << "\n"; 7466 }); 7467 7468 for (const SCEV *S : Strides) { 7469 SCEVCollectTerms TermCollector(Terms); 7470 visitAll(S, TermCollector); 7471 } 7472 7473 DEBUG({ 7474 dbgs() << "Terms:\n"; 7475 for (const SCEV *T : Terms) 7476 dbgs() << *T << "\n"; 7477 }); 7478 } 7479 7480 static bool findArrayDimensionsRec(ScalarEvolution &SE, 7481 SmallVectorImpl<const SCEV *> &Terms, 7482 SmallVectorImpl<const SCEV *> &Sizes) { 7483 int Last = Terms.size() - 1; 7484 const SCEV *Step = Terms[Last]; 7485 7486 // End of recursion. 7487 if (Last == 0) { 7488 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Step)) { 7489 SmallVector<const SCEV *, 2> Qs; 7490 for (const SCEV *Op : M->operands()) 7491 if (!isa<SCEVConstant>(Op)) 7492 Qs.push_back(Op); 7493 7494 Step = SE.getMulExpr(Qs); 7495 } 7496 7497 Sizes.push_back(Step); 7498 return true; 7499 } 7500 7501 for (const SCEV *&Term : Terms) { 7502 // Normalize the terms before the next call to findArrayDimensionsRec. 7503 const SCEV *Q, *R; 7504 SCEVDivision::divide(SE, Term, Step, &Q, &R); 7505 7506 // Bail out when GCD does not evenly divide one of the terms. 7507 if (!R->isZero()) 7508 return false; 7509 7510 Term = Q; 7511 } 7512 7513 // Remove all SCEVConstants. 7514 Terms.erase(std::remove_if(Terms.begin(), Terms.end(), [](const SCEV *E) { 7515 return isa<SCEVConstant>(E); 7516 }), 7517 Terms.end()); 7518 7519 if (Terms.size() > 0) 7520 if (!findArrayDimensionsRec(SE, Terms, Sizes)) 7521 return false; 7522 7523 Sizes.push_back(Step); 7524 return true; 7525 } 7526 7527 namespace { 7528 struct FindParameter { 7529 bool FoundParameter; 7530 FindParameter() : FoundParameter(false) {} 7531 7532 bool follow(const SCEV *S) { 7533 if (isa<SCEVUnknown>(S)) { 7534 FoundParameter = true; 7535 // Stop recursion: we found a parameter. 7536 return false; 7537 } 7538 // Keep looking. 7539 return true; 7540 } 7541 bool isDone() const { 7542 // Stop recursion if we have found a parameter. 7543 return FoundParameter; 7544 } 7545 }; 7546 } 7547 7548 // Returns true when S contains at least a SCEVUnknown parameter. 7549 static inline bool 7550 containsParameters(const SCEV *S) { 7551 FindParameter F; 7552 SCEVTraversal<FindParameter> ST(F); 7553 ST.visitAll(S); 7554 7555 return F.FoundParameter; 7556 } 7557 7558 // Returns true when one of the SCEVs of Terms contains a SCEVUnknown parameter. 7559 static inline bool 7560 containsParameters(SmallVectorImpl<const SCEV *> &Terms) { 7561 for (const SCEV *T : Terms) 7562 if (containsParameters(T)) 7563 return true; 7564 return false; 7565 } 7566 7567 // Return the number of product terms in S. 7568 static inline int numberOfTerms(const SCEV *S) { 7569 if (const SCEVMulExpr *Expr = dyn_cast<SCEVMulExpr>(S)) 7570 return Expr->getNumOperands(); 7571 return 1; 7572 } 7573 7574 static const SCEV *removeConstantFactors(ScalarEvolution &SE, const SCEV *T) { 7575 if (isa<SCEVConstant>(T)) 7576 return nullptr; 7577 7578 if (isa<SCEVUnknown>(T)) 7579 return T; 7580 7581 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(T)) { 7582 SmallVector<const SCEV *, 2> Factors; 7583 for (const SCEV *Op : M->operands()) 7584 if (!isa<SCEVConstant>(Op)) 7585 Factors.push_back(Op); 7586 7587 return SE.getMulExpr(Factors); 7588 } 7589 7590 return T; 7591 } 7592 7593 /// Return the size of an element read or written by Inst. 7594 const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) { 7595 Type *Ty; 7596 if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) 7597 Ty = Store->getValueOperand()->getType(); 7598 else if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) 7599 Ty = Load->getType(); 7600 else 7601 return nullptr; 7602 7603 Type *ETy = getEffectiveSCEVType(PointerType::getUnqual(Ty)); 7604 return getSizeOfExpr(ETy, Ty); 7605 } 7606 7607 /// Second step of delinearization: compute the array dimensions Sizes from the 7608 /// set of Terms extracted from the memory access function of this SCEVAddRec. 7609 void ScalarEvolution::findArrayDimensions(SmallVectorImpl<const SCEV *> &Terms, 7610 SmallVectorImpl<const SCEV *> &Sizes, 7611 const SCEV *ElementSize) const { 7612 7613 if (Terms.size() < 1 || !ElementSize) 7614 return; 7615 7616 // Early return when Terms do not contain parameters: we do not delinearize 7617 // non parametric SCEVs. 7618 if (!containsParameters(Terms)) 7619 return; 7620 7621 DEBUG({ 7622 dbgs() << "Terms:\n"; 7623 for (const SCEV *T : Terms) 7624 dbgs() << *T << "\n"; 7625 }); 7626 7627 // Remove duplicates. 7628 std::sort(Terms.begin(), Terms.end()); 7629 Terms.erase(std::unique(Terms.begin(), Terms.end()), Terms.end()); 7630 7631 // Put larger terms first. 7632 std::sort(Terms.begin(), Terms.end(), [](const SCEV *LHS, const SCEV *RHS) { 7633 return numberOfTerms(LHS) > numberOfTerms(RHS); 7634 }); 7635 7636 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 7637 7638 // Divide all terms by the element size. 7639 for (const SCEV *&Term : Terms) { 7640 const SCEV *Q, *R; 7641 SCEVDivision::divide(SE, Term, ElementSize, &Q, &R); 7642 Term = Q; 7643 } 7644 7645 SmallVector<const SCEV *, 4> NewTerms; 7646 7647 // Remove constant factors. 7648 for (const SCEV *T : Terms) 7649 if (const SCEV *NewT = removeConstantFactors(SE, T)) 7650 NewTerms.push_back(NewT); 7651 7652 DEBUG({ 7653 dbgs() << "Terms after sorting:\n"; 7654 for (const SCEV *T : NewTerms) 7655 dbgs() << *T << "\n"; 7656 }); 7657 7658 if (NewTerms.empty() || 7659 !findArrayDimensionsRec(SE, NewTerms, Sizes)) { 7660 Sizes.clear(); 7661 return; 7662 } 7663 7664 // The last element to be pushed into Sizes is the size of an element. 7665 Sizes.push_back(ElementSize); 7666 7667 DEBUG({ 7668 dbgs() << "Sizes:\n"; 7669 for (const SCEV *S : Sizes) 7670 dbgs() << *S << "\n"; 7671 }); 7672 } 7673 7674 /// Third step of delinearization: compute the access functions for the 7675 /// Subscripts based on the dimensions in Sizes. 7676 void SCEVAddRecExpr::computeAccessFunctions( 7677 ScalarEvolution &SE, SmallVectorImpl<const SCEV *> &Subscripts, 7678 SmallVectorImpl<const SCEV *> &Sizes) const { 7679 7680 // Early exit in case this SCEV is not an affine multivariate function. 7681 if (Sizes.empty() || !this->isAffine()) 7682 return; 7683 7684 const SCEV *Res = this; 7685 int Last = Sizes.size() - 1; 7686 for (int i = Last; i >= 0; i--) { 7687 const SCEV *Q, *R; 7688 SCEVDivision::divide(SE, Res, Sizes[i], &Q, &R); 7689 7690 DEBUG({ 7691 dbgs() << "Res: " << *Res << "\n"; 7692 dbgs() << "Sizes[i]: " << *Sizes[i] << "\n"; 7693 dbgs() << "Res divided by Sizes[i]:\n"; 7694 dbgs() << "Quotient: " << *Q << "\n"; 7695 dbgs() << "Remainder: " << *R << "\n"; 7696 }); 7697 7698 Res = Q; 7699 7700 // Do not record the last subscript corresponding to the size of elements in 7701 // the array. 7702 if (i == Last) { 7703 7704 // Bail out if the remainder is too complex. 7705 if (isa<SCEVAddRecExpr>(R)) { 7706 Subscripts.clear(); 7707 Sizes.clear(); 7708 return; 7709 } 7710 7711 continue; 7712 } 7713 7714 // Record the access function for the current subscript. 7715 Subscripts.push_back(R); 7716 } 7717 7718 // Also push in last position the remainder of the last division: it will be 7719 // the access function of the innermost dimension. 7720 Subscripts.push_back(Res); 7721 7722 std::reverse(Subscripts.begin(), Subscripts.end()); 7723 7724 DEBUG({ 7725 dbgs() << "Subscripts:\n"; 7726 for (const SCEV *S : Subscripts) 7727 dbgs() << *S << "\n"; 7728 }); 7729 } 7730 7731 /// Splits the SCEV into two vectors of SCEVs representing the subscripts and 7732 /// sizes of an array access. Returns the remainder of the delinearization that 7733 /// is the offset start of the array. The SCEV->delinearize algorithm computes 7734 /// the multiples of SCEV coefficients: that is a pattern matching of sub 7735 /// expressions in the stride and base of a SCEV corresponding to the 7736 /// computation of a GCD (greatest common divisor) of base and stride. When 7737 /// SCEV->delinearize fails, it returns the SCEV unchanged. 7738 /// 7739 /// For example: when analyzing the memory access A[i][j][k] in this loop nest 7740 /// 7741 /// void foo(long n, long m, long o, double A[n][m][o]) { 7742 /// 7743 /// for (long i = 0; i < n; i++) 7744 /// for (long j = 0; j < m; j++) 7745 /// for (long k = 0; k < o; k++) 7746 /// A[i][j][k] = 1.0; 7747 /// } 7748 /// 7749 /// the delinearization input is the following AddRec SCEV: 7750 /// 7751 /// AddRec: {{{%A,+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k> 7752 /// 7753 /// From this SCEV, we are able to say that the base offset of the access is %A 7754 /// because it appears as an offset that does not divide any of the strides in 7755 /// the loops: 7756 /// 7757 /// CHECK: Base offset: %A 7758 /// 7759 /// and then SCEV->delinearize determines the size of some of the dimensions of 7760 /// the array as these are the multiples by which the strides are happening: 7761 /// 7762 /// CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of sizeof(double) bytes. 7763 /// 7764 /// Note that the outermost dimension remains of UnknownSize because there are 7765 /// no strides that would help identifying the size of the last dimension: when 7766 /// the array has been statically allocated, one could compute the size of that 7767 /// dimension by dividing the overall size of the array by the size of the known 7768 /// dimensions: %m * %o * 8. 7769 /// 7770 /// Finally delinearize provides the access functions for the array reference 7771 /// that does correspond to A[i][j][k] of the above C testcase: 7772 /// 7773 /// CHECK: ArrayRef[{0,+,1}<%for.i>][{0,+,1}<%for.j>][{0,+,1}<%for.k>] 7774 /// 7775 /// The testcases are checking the output of a function pass: 7776 /// DelinearizationPass that walks through all loads and stores of a function 7777 /// asking for the SCEV of the memory access with respect to all enclosing 7778 /// loops, calling SCEV->delinearize on that and printing the results. 7779 7780 void SCEVAddRecExpr::delinearize(ScalarEvolution &SE, 7781 SmallVectorImpl<const SCEV *> &Subscripts, 7782 SmallVectorImpl<const SCEV *> &Sizes, 7783 const SCEV *ElementSize) const { 7784 // First step: collect parametric terms. 7785 SmallVector<const SCEV *, 4> Terms; 7786 collectParametricTerms(SE, Terms); 7787 7788 if (Terms.empty()) 7789 return; 7790 7791 // Second step: find subscript sizes. 7792 SE.findArrayDimensions(Terms, Sizes, ElementSize); 7793 7794 if (Sizes.empty()) 7795 return; 7796 7797 // Third step: compute the access functions for each subscript. 7798 computeAccessFunctions(SE, Subscripts, Sizes); 7799 7800 if (Subscripts.empty()) 7801 return; 7802 7803 DEBUG({ 7804 dbgs() << "succeeded to delinearize " << *this << "\n"; 7805 dbgs() << "ArrayDecl[UnknownSize]"; 7806 for (const SCEV *S : Sizes) 7807 dbgs() << "[" << *S << "]"; 7808 7809 dbgs() << "\nArrayRef"; 7810 for (const SCEV *S : Subscripts) 7811 dbgs() << "[" << *S << "]"; 7812 dbgs() << "\n"; 7813 }); 7814 } 7815 7816 //===----------------------------------------------------------------------===// 7817 // SCEVCallbackVH Class Implementation 7818 //===----------------------------------------------------------------------===// 7819 7820 void ScalarEvolution::SCEVCallbackVH::deleted() { 7821 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!"); 7822 if (PHINode *PN = dyn_cast<PHINode>(getValPtr())) 7823 SE->ConstantEvolutionLoopExitValue.erase(PN); 7824 SE->ValueExprMap.erase(getValPtr()); 7825 // this now dangles! 7826 } 7827 7828 void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) { 7829 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!"); 7830 7831 // Forget all the expressions associated with users of the old value, 7832 // so that future queries will recompute the expressions using the new 7833 // value. 7834 Value *Old = getValPtr(); 7835 SmallVector<User *, 16> Worklist(Old->user_begin(), Old->user_end()); 7836 SmallPtrSet<User *, 8> Visited; 7837 while (!Worklist.empty()) { 7838 User *U = Worklist.pop_back_val(); 7839 // Deleting the Old value will cause this to dangle. Postpone 7840 // that until everything else is done. 7841 if (U == Old) 7842 continue; 7843 if (!Visited.insert(U).second) 7844 continue; 7845 if (PHINode *PN = dyn_cast<PHINode>(U)) 7846 SE->ConstantEvolutionLoopExitValue.erase(PN); 7847 SE->ValueExprMap.erase(U); 7848 Worklist.insert(Worklist.end(), U->user_begin(), U->user_end()); 7849 } 7850 // Delete the Old value. 7851 if (PHINode *PN = dyn_cast<PHINode>(Old)) 7852 SE->ConstantEvolutionLoopExitValue.erase(PN); 7853 SE->ValueExprMap.erase(Old); 7854 // this now dangles! 7855 } 7856 7857 ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se) 7858 : CallbackVH(V), SE(se) {} 7859 7860 //===----------------------------------------------------------------------===// 7861 // ScalarEvolution Class Implementation 7862 //===----------------------------------------------------------------------===// 7863 7864 ScalarEvolution::ScalarEvolution() 7865 : FunctionPass(ID), ValuesAtScopes(64), LoopDispositions(64), 7866 BlockDispositions(64), FirstUnknown(nullptr) { 7867 initializeScalarEvolutionPass(*PassRegistry::getPassRegistry()); 7868 } 7869 7870 bool ScalarEvolution::runOnFunction(Function &F) { 7871 this->F = &F; 7872 AT = &getAnalysis<AssumptionTracker>(); 7873 LI = &getAnalysis<LoopInfo>(); 7874 DataLayoutPass *DLP = getAnalysisIfAvailable<DataLayoutPass>(); 7875 DL = DLP ? &DLP->getDataLayout() : nullptr; 7876 TLI = &getAnalysis<TargetLibraryInfo>(); 7877 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); 7878 return false; 7879 } 7880 7881 void ScalarEvolution::releaseMemory() { 7882 // Iterate through all the SCEVUnknown instances and call their 7883 // destructors, so that they release their references to their values. 7884 for (SCEVUnknown *U = FirstUnknown; U; U = U->Next) 7885 U->~SCEVUnknown(); 7886 FirstUnknown = nullptr; 7887 7888 ValueExprMap.clear(); 7889 7890 // Free any extra memory created for ExitNotTakenInfo in the unlikely event 7891 // that a loop had multiple computable exits. 7892 for (DenseMap<const Loop*, BackedgeTakenInfo>::iterator I = 7893 BackedgeTakenCounts.begin(), E = BackedgeTakenCounts.end(); 7894 I != E; ++I) { 7895 I->second.clear(); 7896 } 7897 7898 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage"); 7899 7900 BackedgeTakenCounts.clear(); 7901 ConstantEvolutionLoopExitValue.clear(); 7902 ValuesAtScopes.clear(); 7903 LoopDispositions.clear(); 7904 BlockDispositions.clear(); 7905 UnsignedRanges.clear(); 7906 SignedRanges.clear(); 7907 UniqueSCEVs.clear(); 7908 SCEVAllocator.Reset(); 7909 } 7910 7911 void ScalarEvolution::getAnalysisUsage(AnalysisUsage &AU) const { 7912 AU.setPreservesAll(); 7913 AU.addRequired<AssumptionTracker>(); 7914 AU.addRequiredTransitive<LoopInfo>(); 7915 AU.addRequiredTransitive<DominatorTreeWrapperPass>(); 7916 AU.addRequired<TargetLibraryInfo>(); 7917 } 7918 7919 bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) { 7920 return !isa<SCEVCouldNotCompute>(getBackedgeTakenCount(L)); 7921 } 7922 7923 static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, 7924 const Loop *L) { 7925 // Print all inner loops first 7926 for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) 7927 PrintLoopInfo(OS, SE, *I); 7928 7929 OS << "Loop "; 7930 L->getHeader()->printAsOperand(OS, /*PrintType=*/false); 7931 OS << ": "; 7932 7933 SmallVector<BasicBlock *, 8> ExitBlocks; 7934 L->getExitBlocks(ExitBlocks); 7935 if (ExitBlocks.size() != 1) 7936 OS << "<multiple exits> "; 7937 7938 if (SE->hasLoopInvariantBackedgeTakenCount(L)) { 7939 OS << "backedge-taken count is " << *SE->getBackedgeTakenCount(L); 7940 } else { 7941 OS << "Unpredictable backedge-taken count. "; 7942 } 7943 7944 OS << "\n" 7945 "Loop "; 7946 L->getHeader()->printAsOperand(OS, /*PrintType=*/false); 7947 OS << ": "; 7948 7949 if (!isa<SCEVCouldNotCompute>(SE->getMaxBackedgeTakenCount(L))) { 7950 OS << "max backedge-taken count is " << *SE->getMaxBackedgeTakenCount(L); 7951 } else { 7952 OS << "Unpredictable max backedge-taken count. "; 7953 } 7954 7955 OS << "\n"; 7956 } 7957 7958 void ScalarEvolution::print(raw_ostream &OS, const Module *) const { 7959 // ScalarEvolution's implementation of the print method is to print 7960 // out SCEV values of all instructions that are interesting. Doing 7961 // this potentially causes it to create new SCEV objects though, 7962 // which technically conflicts with the const qualifier. This isn't 7963 // observable from outside the class though, so casting away the 7964 // const isn't dangerous. 7965 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 7966 7967 OS << "Classifying expressions for: "; 7968 F->printAsOperand(OS, /*PrintType=*/false); 7969 OS << "\n"; 7970 for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I) 7971 if (isSCEVable(I->getType()) && !isa<CmpInst>(*I)) { 7972 OS << *I << '\n'; 7973 OS << " --> "; 7974 const SCEV *SV = SE.getSCEV(&*I); 7975 SV->print(OS); 7976 7977 const Loop *L = LI->getLoopFor((*I).getParent()); 7978 7979 const SCEV *AtUse = SE.getSCEVAtScope(SV, L); 7980 if (AtUse != SV) { 7981 OS << " --> "; 7982 AtUse->print(OS); 7983 } 7984 7985 if (L) { 7986 OS << "\t\t" "Exits: "; 7987 const SCEV *ExitValue = SE.getSCEVAtScope(SV, L->getParentLoop()); 7988 if (!SE.isLoopInvariant(ExitValue, L)) { 7989 OS << "<<Unknown>>"; 7990 } else { 7991 OS << *ExitValue; 7992 } 7993 } 7994 7995 OS << "\n"; 7996 } 7997 7998 OS << "Determining loop execution counts for: "; 7999 F->printAsOperand(OS, /*PrintType=*/false); 8000 OS << "\n"; 8001 for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I) 8002 PrintLoopInfo(OS, &SE, *I); 8003 } 8004 8005 ScalarEvolution::LoopDisposition 8006 ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) { 8007 SmallVector<std::pair<const Loop *, LoopDisposition>, 2> &Values = LoopDispositions[S]; 8008 for (unsigned u = 0; u < Values.size(); u++) { 8009 if (Values[u].first == L) 8010 return Values[u].second; 8011 } 8012 Values.push_back(std::make_pair(L, LoopVariant)); 8013 LoopDisposition D = computeLoopDisposition(S, L); 8014 SmallVector<std::pair<const Loop *, LoopDisposition>, 2> &Values2 = LoopDispositions[S]; 8015 for (unsigned u = Values2.size(); u > 0; u--) { 8016 if (Values2[u - 1].first == L) { 8017 Values2[u - 1].second = D; 8018 break; 8019 } 8020 } 8021 return D; 8022 } 8023 8024 ScalarEvolution::LoopDisposition 8025 ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) { 8026 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 8027 case scConstant: 8028 return LoopInvariant; 8029 case scTruncate: 8030 case scZeroExtend: 8031 case scSignExtend: 8032 return getLoopDisposition(cast<SCEVCastExpr>(S)->getOperand(), L); 8033 case scAddRecExpr: { 8034 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S); 8035 8036 // If L is the addrec's loop, it's computable. 8037 if (AR->getLoop() == L) 8038 return LoopComputable; 8039 8040 // Add recurrences are never invariant in the function-body (null loop). 8041 if (!L) 8042 return LoopVariant; 8043 8044 // This recurrence is variant w.r.t. L if L contains AR's loop. 8045 if (L->contains(AR->getLoop())) 8046 return LoopVariant; 8047 8048 // This recurrence is invariant w.r.t. L if AR's loop contains L. 8049 if (AR->getLoop()->contains(L)) 8050 return LoopInvariant; 8051 8052 // This recurrence is variant w.r.t. L if any of its operands 8053 // are variant. 8054 for (SCEVAddRecExpr::op_iterator I = AR->op_begin(), E = AR->op_end(); 8055 I != E; ++I) 8056 if (!isLoopInvariant(*I, L)) 8057 return LoopVariant; 8058 8059 // Otherwise it's loop-invariant. 8060 return LoopInvariant; 8061 } 8062 case scAddExpr: 8063 case scMulExpr: 8064 case scUMaxExpr: 8065 case scSMaxExpr: { 8066 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S); 8067 bool HasVarying = false; 8068 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 8069 I != E; ++I) { 8070 LoopDisposition D = getLoopDisposition(*I, L); 8071 if (D == LoopVariant) 8072 return LoopVariant; 8073 if (D == LoopComputable) 8074 HasVarying = true; 8075 } 8076 return HasVarying ? LoopComputable : LoopInvariant; 8077 } 8078 case scUDivExpr: { 8079 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 8080 LoopDisposition LD = getLoopDisposition(UDiv->getLHS(), L); 8081 if (LD == LoopVariant) 8082 return LoopVariant; 8083 LoopDisposition RD = getLoopDisposition(UDiv->getRHS(), L); 8084 if (RD == LoopVariant) 8085 return LoopVariant; 8086 return (LD == LoopInvariant && RD == LoopInvariant) ? 8087 LoopInvariant : LoopComputable; 8088 } 8089 case scUnknown: 8090 // All non-instruction values are loop invariant. All instructions are loop 8091 // invariant if they are not contained in the specified loop. 8092 // Instructions are never considered invariant in the function body 8093 // (null loop) because they are defined within the "loop". 8094 if (Instruction *I = dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) 8095 return (L && !L->contains(I)) ? LoopInvariant : LoopVariant; 8096 return LoopInvariant; 8097 case scCouldNotCompute: 8098 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 8099 } 8100 llvm_unreachable("Unknown SCEV kind!"); 8101 } 8102 8103 bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) { 8104 return getLoopDisposition(S, L) == LoopInvariant; 8105 } 8106 8107 bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) { 8108 return getLoopDisposition(S, L) == LoopComputable; 8109 } 8110 8111 ScalarEvolution::BlockDisposition 8112 ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) { 8113 SmallVector<std::pair<const BasicBlock *, BlockDisposition>, 2> &Values = BlockDispositions[S]; 8114 for (unsigned u = 0; u < Values.size(); u++) { 8115 if (Values[u].first == BB) 8116 return Values[u].second; 8117 } 8118 Values.push_back(std::make_pair(BB, DoesNotDominateBlock)); 8119 BlockDisposition D = computeBlockDisposition(S, BB); 8120 SmallVector<std::pair<const BasicBlock *, BlockDisposition>, 2> &Values2 = BlockDispositions[S]; 8121 for (unsigned u = Values2.size(); u > 0; u--) { 8122 if (Values2[u - 1].first == BB) { 8123 Values2[u - 1].second = D; 8124 break; 8125 } 8126 } 8127 return D; 8128 } 8129 8130 ScalarEvolution::BlockDisposition 8131 ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) { 8132 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 8133 case scConstant: 8134 return ProperlyDominatesBlock; 8135 case scTruncate: 8136 case scZeroExtend: 8137 case scSignExtend: 8138 return getBlockDisposition(cast<SCEVCastExpr>(S)->getOperand(), BB); 8139 case scAddRecExpr: { 8140 // This uses a "dominates" query instead of "properly dominates" query 8141 // to test for proper dominance too, because the instruction which 8142 // produces the addrec's value is a PHI, and a PHI effectively properly 8143 // dominates its entire containing block. 8144 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S); 8145 if (!DT->dominates(AR->getLoop()->getHeader(), BB)) 8146 return DoesNotDominateBlock; 8147 } 8148 // FALL THROUGH into SCEVNAryExpr handling. 8149 case scAddExpr: 8150 case scMulExpr: 8151 case scUMaxExpr: 8152 case scSMaxExpr: { 8153 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S); 8154 bool Proper = true; 8155 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 8156 I != E; ++I) { 8157 BlockDisposition D = getBlockDisposition(*I, BB); 8158 if (D == DoesNotDominateBlock) 8159 return DoesNotDominateBlock; 8160 if (D == DominatesBlock) 8161 Proper = false; 8162 } 8163 return Proper ? ProperlyDominatesBlock : DominatesBlock; 8164 } 8165 case scUDivExpr: { 8166 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 8167 const SCEV *LHS = UDiv->getLHS(), *RHS = UDiv->getRHS(); 8168 BlockDisposition LD = getBlockDisposition(LHS, BB); 8169 if (LD == DoesNotDominateBlock) 8170 return DoesNotDominateBlock; 8171 BlockDisposition RD = getBlockDisposition(RHS, BB); 8172 if (RD == DoesNotDominateBlock) 8173 return DoesNotDominateBlock; 8174 return (LD == ProperlyDominatesBlock && RD == ProperlyDominatesBlock) ? 8175 ProperlyDominatesBlock : DominatesBlock; 8176 } 8177 case scUnknown: 8178 if (Instruction *I = 8179 dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) { 8180 if (I->getParent() == BB) 8181 return DominatesBlock; 8182 if (DT->properlyDominates(I->getParent(), BB)) 8183 return ProperlyDominatesBlock; 8184 return DoesNotDominateBlock; 8185 } 8186 return ProperlyDominatesBlock; 8187 case scCouldNotCompute: 8188 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 8189 } 8190 llvm_unreachable("Unknown SCEV kind!"); 8191 } 8192 8193 bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) { 8194 return getBlockDisposition(S, BB) >= DominatesBlock; 8195 } 8196 8197 bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) { 8198 return getBlockDisposition(S, BB) == ProperlyDominatesBlock; 8199 } 8200 8201 namespace { 8202 // Search for a SCEV expression node within an expression tree. 8203 // Implements SCEVTraversal::Visitor. 8204 struct SCEVSearch { 8205 const SCEV *Node; 8206 bool IsFound; 8207 8208 SCEVSearch(const SCEV *N): Node(N), IsFound(false) {} 8209 8210 bool follow(const SCEV *S) { 8211 IsFound |= (S == Node); 8212 return !IsFound; 8213 } 8214 bool isDone() const { return IsFound; } 8215 }; 8216 } 8217 8218 bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const { 8219 SCEVSearch Search(Op); 8220 visitAll(S, Search); 8221 return Search.IsFound; 8222 } 8223 8224 void ScalarEvolution::forgetMemoizedResults(const SCEV *S) { 8225 ValuesAtScopes.erase(S); 8226 LoopDispositions.erase(S); 8227 BlockDispositions.erase(S); 8228 UnsignedRanges.erase(S); 8229 SignedRanges.erase(S); 8230 8231 for (DenseMap<const Loop*, BackedgeTakenInfo>::iterator I = 8232 BackedgeTakenCounts.begin(), E = BackedgeTakenCounts.end(); I != E; ) { 8233 BackedgeTakenInfo &BEInfo = I->second; 8234 if (BEInfo.hasOperand(S, this)) { 8235 BEInfo.clear(); 8236 BackedgeTakenCounts.erase(I++); 8237 } 8238 else 8239 ++I; 8240 } 8241 } 8242 8243 typedef DenseMap<const Loop *, std::string> VerifyMap; 8244 8245 /// replaceSubString - Replaces all occurrences of From in Str with To. 8246 static void replaceSubString(std::string &Str, StringRef From, StringRef To) { 8247 size_t Pos = 0; 8248 while ((Pos = Str.find(From, Pos)) != std::string::npos) { 8249 Str.replace(Pos, From.size(), To.data(), To.size()); 8250 Pos += To.size(); 8251 } 8252 } 8253 8254 /// getLoopBackedgeTakenCounts - Helper method for verifyAnalysis. 8255 static void 8256 getLoopBackedgeTakenCounts(Loop *L, VerifyMap &Map, ScalarEvolution &SE) { 8257 for (Loop::reverse_iterator I = L->rbegin(), E = L->rend(); I != E; ++I) { 8258 getLoopBackedgeTakenCounts(*I, Map, SE); // recurse. 8259 8260 std::string &S = Map[L]; 8261 if (S.empty()) { 8262 raw_string_ostream OS(S); 8263 SE.getBackedgeTakenCount(L)->print(OS); 8264 8265 // false and 0 are semantically equivalent. This can happen in dead loops. 8266 replaceSubString(OS.str(), "false", "0"); 8267 // Remove wrap flags, their use in SCEV is highly fragile. 8268 // FIXME: Remove this when SCEV gets smarter about them. 8269 replaceSubString(OS.str(), "<nw>", ""); 8270 replaceSubString(OS.str(), "<nsw>", ""); 8271 replaceSubString(OS.str(), "<nuw>", ""); 8272 } 8273 } 8274 } 8275 8276 void ScalarEvolution::verifyAnalysis() const { 8277 if (!VerifySCEV) 8278 return; 8279 8280 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 8281 8282 // Gather stringified backedge taken counts for all loops using SCEV's caches. 8283 // FIXME: It would be much better to store actual values instead of strings, 8284 // but SCEV pointers will change if we drop the caches. 8285 VerifyMap BackedgeDumpsOld, BackedgeDumpsNew; 8286 for (LoopInfo::reverse_iterator I = LI->rbegin(), E = LI->rend(); I != E; ++I) 8287 getLoopBackedgeTakenCounts(*I, BackedgeDumpsOld, SE); 8288 8289 // Gather stringified backedge taken counts for all loops without using 8290 // SCEV's caches. 8291 SE.releaseMemory(); 8292 for (LoopInfo::reverse_iterator I = LI->rbegin(), E = LI->rend(); I != E; ++I) 8293 getLoopBackedgeTakenCounts(*I, BackedgeDumpsNew, SE); 8294 8295 // Now compare whether they're the same with and without caches. This allows 8296 // verifying that no pass changed the cache. 8297 assert(BackedgeDumpsOld.size() == BackedgeDumpsNew.size() && 8298 "New loops suddenly appeared!"); 8299 8300 for (VerifyMap::iterator OldI = BackedgeDumpsOld.begin(), 8301 OldE = BackedgeDumpsOld.end(), 8302 NewI = BackedgeDumpsNew.begin(); 8303 OldI != OldE; ++OldI, ++NewI) { 8304 assert(OldI->first == NewI->first && "Loop order changed!"); 8305 8306 // Compare the stringified SCEVs. We don't care if undef backedgetaken count 8307 // changes. 8308 // FIXME: We currently ignore SCEV changes from/to CouldNotCompute. This 8309 // means that a pass is buggy or SCEV has to learn a new pattern but is 8310 // usually not harmful. 8311 if (OldI->second != NewI->second && 8312 OldI->second.find("undef") == std::string::npos && 8313 NewI->second.find("undef") == std::string::npos && 8314 OldI->second != "***COULDNOTCOMPUTE***" && 8315 NewI->second != "***COULDNOTCOMPUTE***") { 8316 dbgs() << "SCEVValidator: SCEV for loop '" 8317 << OldI->first->getHeader()->getName() 8318 << "' changed from '" << OldI->second 8319 << "' to '" << NewI->second << "'!\n"; 8320 std::abort(); 8321 } 8322 } 8323 8324 // TODO: Verify more things. 8325 } 8326